diff --git a/.agents/skills/add-block-preview/SKILL.md b/.agents/skills/add-block-preview/SKILL.md index 158922058e5..c583a54c2cf 100644 --- a/.agents/skills/add-block-preview/SKILL.md +++ b/.agents/skills/add-block-preview/SKILL.md @@ -41,7 +41,7 @@ A revealed block that is not globally GA (`enabled !== true`, or env-revealed) r - GA via config (code cleanup pending): `{ "enabled": true }` — suffix disappears everywhere within ~30s (AppConfig TTL) + client refetch. Same runbook as `feature-flags`: edit the hosted document, `aws appconfig start-deployment` with the `sim--fast` strategy (see the infra README). -5. **GA cleanup:** delete `preview: true` from the block (now visible to self-hosters on their next upgrade), add its `BlockMeta` + regen docs, and drop the AppConfig entry. For a v2 upgrade, this is also when v1 gets `hideFromToolbar: true` (the superseded-version paradigm). +5. **GA cleanup:** delete `preview: true` from the block (now visible to self-hosters on their next upgrade), add its `BlockMeta` + regen docs, and drop the AppConfig entry. For a v2 upgrade, this is also when v1 gets `hideFromToolbar: true` **and** `sunset: { status: 'legacy', replacedBy: '' }` (the superseded-version paradigm). Both edits must land in the **same commit** as the `preview: true` removal — `check-block-registry` fails a sunset block whose `replacedBy` is still `preview`, so splitting them breaks the build in between. Also move the block's `BLOCK_DISPLAY_WORKFLOWS` entry (`apps/docs/components/workflow-preview/block-display-workflows.ts`) to the new type, or `BlockPreview` silently renders nothing on the docs page. ## Kill switch (shipped blocks) diff --git a/.agents/skills/add-block/SKILL.md b/.agents/skills/add-block/SKILL.md index 7e1d2cc8054..7de2fa0e872 100644 --- a/.agents/skills/add-block/SKILL.md +++ b/.agents/skills/add-block/SKILL.md @@ -207,6 +207,7 @@ silently available. id: 'channel', title: 'Channel', type: 'channel-selector', + selectorKey: '{service}.channels', serviceId: '{service}', placeholder: 'Select channel', dependsOn: ['credential'], @@ -217,6 +218,7 @@ silently available. id: 'project', title: 'Project', type: 'project-selector', + selectorKey: '{service}.projects', serviceId: '{service}', dependsOn: ['credential'], } @@ -226,6 +228,7 @@ silently available. id: 'file', title: 'File', type: 'file-selector', + selectorKey: '{service}.files', serviceId: '{service}', mimeType: 'application/pdf', dependsOn: ['credential'], @@ -236,6 +239,7 @@ silently available. id: 'user', title: 'User', type: 'user-selector', + selectorKey: '{service}.users', serviceId: '{service}', dependsOn: ['credential'], } @@ -691,6 +695,10 @@ export const ServiceBlock: BlockConfig = { type: 'service', name: 'Service (Legacy)', hideFromToolbar: true, // Hide from toolbar + // Required: drives the amber legacy badge and its click-to-upgrade action. + // `check-block-registry` fails a legacy block with no `replacedBy`, one whose + // target does not exist, or one whose target is itself sunset or still `preview`. + sunset: { status: 'legacy', replacedBy: 'service_v2' }, // ... rest of config } @@ -1065,7 +1073,11 @@ After creating the block, you MUST validate it against every tool it references: A sub-block gets its choices from exactly one of two places. There is no third. -**`selectorKey` — every remote list.** Register the list in `hooks/selectors/providers//selectors.ts`, add its key to `SelectorKey`, and point the sub-block at it. A selector is parameterized by an explicit `SelectorContext`, so the same definition serves the canvas, the workspace-fork sync modal, and anything added later. +**`selectorKey` — every remote list.** Use the `add-selector` skill to add browser-safe metadata in +`apps/sim/lib/selectors/manifest.ts`. Attach `provider-server` selectors under +`apps/sim/lib/selectors/server/providers/` and `internal-server` selectors in +`apps/sim/lib/selectors/server/internal.ts`. Point the sub-block at that key. All remote selectors +execute through `selectors.execute`; never add a client provider module or selector-only fetch route. ```ts { id: 'triggerCredentials', type: 'oauth-input', canonicalParamId: 'oauthCredential', mode: 'trigger' }, @@ -1074,7 +1086,13 @@ A sub-block gets its choices from exactly one of two places. There is no third. { id: 'manualLabelIds', type: 'short-input', mode: 'trigger-advanced' }, ``` -`canonicalParamId: 'oauthCredential'` on the credential sub-block is the line people forget. `buildSelectorContextFromBlock` keys the context on a sub-block's CANONICAL id, so without it `context.oauthCredential` is never set and the picker looks unfixable without reading the store. (A credential field is also recognised by its `oauth-input` TYPE as a fallback, so a block whose shipped param is already named something else does not have to rename it.) +`canonicalParamId: 'oauthCredential'` on the credential sub-block is the line people forget. The +shared context builder projects only active `dependsOn` values and keys canonical pairs by their +canonical id. Exact environment references such as `{{GMAIL_CREDENTIAL_ID}}` stay unresolved in the +browser and are resolved only by the authorized server executor. The builder does not infer a +nonstandard credential id from `type: 'oauth-input'`; give it +`canonicalParamId: 'oauthCredential'`, or declare an explicit manifest `sourceFields` alias when a +legacy source id must be retained. **`options` — everything else.** A static array, or a pure function of the block's own values for a list that narrows to a sibling's selection. No I/O. @@ -1089,5 +1107,6 @@ options: (params) => { Two rules the checks enforce: -- **A secret never enters a selector's `getQueryKey`.** A query key identifies a resource; a credential authorizes access to it. A credential *id* is fine; a typed password is not (see `imap.mailboxes`). +- **Selector query keys contain no context values.** This includes credential IDs, raw secrets, + unresolved references, and hashes of those values; the shared facade uses an opaque local revision. - **A sub-block that `dependsOn` a credential / knowledge-base / table selector must be reconfigurable at fork-sync time** — a `selectorKey`, a canonical pair whose basic member is a selector, or a `short-input`/`long-input`. `bun run check:fork-dependent-coverage` fails otherwise, because a fork sync clears those fields on every push and an unofferable one can never be set anywhere that sticks. diff --git a/.agents/skills/add-connector/SKILL.md b/.agents/skills/add-connector/SKILL.md index ce2e29066ca..c14c5a11ade 100644 --- a/.agents/skills/add-connector/SKILL.md +++ b/.agents/skills/add-connector/SKILL.md @@ -197,7 +197,14 @@ Three field types are supported: `short-input`, `dropdown`, and `selector`. ## Dynamic Selectors (Canonical Pairs) -Use `type: 'selector'` to fetch options dynamically from the existing selector registry (`hooks/selectors/registry.ts`). Selectors are always paired with a manual fallback input using the **canonical pair** pattern — a `selector` field (basic mode) and a `short-input` field (advanced mode) linked by `canonicalParamId`. +Use `type: 'selector'` for a key declared in the browser-safe selector manifest at +`apps/sim/lib/selectors/manifest.ts`. Remote selectors execute through the authorized +`selectors.execute` server operation and a server attachment; connectors never call providers or +resolve credentials in the browser. Apply the `add-selector` skill when the key does not exist. + +Selectors are paired with a manual fallback input using the **canonical pair** pattern — a +`selector` field (basic mode) and a `short-input` field (advanced mode) linked by +`canonicalParamId`. The user sees a toggle button (ArrowLeftRight) to switch between the selector dropdown and manual text input. On submit, the modal resolves each canonical pair to the active mode's value, keyed by `canonicalParamId`. @@ -217,7 +224,7 @@ configFields: [ id: 'baseSelector', title: 'Base', type: 'selector', - selectorKey: 'airtable.bases', // Must exist in hooks/selectors/registry.ts + selectorKey: 'airtable.bases', // Must exist in lib/selectors/manifest.ts canonicalParamId: 'baseId', mode: 'basic', placeholder: 'Select a base', @@ -260,7 +267,9 @@ configFields: [ ### Selector with domain dependency (Jira/Confluence pattern) -When a selector depends on a plain `short-input` field (no canonical pair), `dependsOn` references that field's `id` directly. The `domain` field's value maps to `SelectorContext.domain` automatically via `SELECTOR_CONTEXT_FIELDS`. +When a selector depends on a plain `short-input` field (no canonical pair), `dependsOn` references +that field's `id` directly. Exact references such as `{{JIRA_DOMAIN}}` remain unresolved in the +browser and are resolved only after workspace authorization on the server. ```typescript configFields: [ @@ -296,16 +305,16 @@ configFields: [ ### How `dependsOn` maps to `SelectorContext` -The connector selector field builds a `SelectorContext` from dependency values. For the mapping to work, each dependency's `canonicalParamId` (or field `id` for non-canonical fields) must exist in `SELECTOR_CONTEXT_FIELDS` (`lib/workflows/subblocks/context.ts`): - -``` -oauthCredential, domain, teamId, projectId, knowledgeBaseId, planId, -siteId, collectionId, spreadsheetId, fileId, baseId, datasetId, serviceDeskId -``` +The shared connector context builder projects only active dependencies. A canonical dependency uses +its active basic or advanced value under `canonicalParamId`; a non-canonical dependency uses its +field `id`. The resulting key must be a `SelectorContextKey` in +`apps/sim/lib/selectors/types.ts` and must be explicitly allowed by that selector's manifest entry. +The browser sends the connector's workspace scope, not the complete connector configuration. ### Available selector keys -Check `hooks/selectors/types.ts` for the full `SelectorKey` union. Common ones for connectors: +Check `apps/sim/lib/selectors/manifest.ts` for the exhaustive selector keys. Common ones for +connectors: | SelectorKey | Context Deps | Returns | |-------------|-------------|---------| @@ -607,9 +616,13 @@ export const CONNECTOR_META_REGISTRY: ConnectorMetaRegistry = { - [ ] **Selector fields configured correctly (if applicable):** - Every `type: 'selector'` field has a canonical pair (`short-input` or `dropdown` with same `canonicalParamId` and `mode: 'advanced'`) - `required` is identical on both fields in each canonical pair - - `selectorKey` exists in `hooks/selectors/registry.ts` + - `selectorKey` exists in `apps/sim/lib/selectors/manifest.ts` - `dependsOn` references selector field IDs (not `canonicalParamId`) - - Dependency `canonicalParamId` values exist in `SELECTOR_CONTEXT_FIELDS` + - Each projected dependency key is a `SelectorContextKey` allowed by the selector manifest + - Every remote key has one server attachment with credential provider binding and a reviewed + `fixed`, `credential-bound`, or `user-controlled` destination policy + - No connector selector adds a client provider module, browser token request, or selector-only + API route - [ ] `listDocuments` handles pagination with metadata-based content hashes - [ ] `syncContext.listingCapped = true` set whenever the listing is truncated (max-items cap or transient per-item error) — required to prevent the engine's deletion reconciliation from removing unseen documents - [ ] `contentDeferred: true` used if content requires per-doc API calls (file download, export, blocks fetch) diff --git a/.agents/skills/add-integration/SKILL.md b/.agents/skills/add-integration/SKILL.md index d3d8e8f64ea..b727fa06bb3 100644 --- a/.agents/skills/add-integration/SKILL.md +++ b/.agents/skills/add-integration/SKILL.md @@ -270,15 +270,24 @@ export const {Service}Block: BlockConfig = { { id: 'project', type: 'project-selector', + selectorKey: '{service}.projects', dependsOn: ['credential'], }, { id: 'issue', type: 'file-selector', + selectorKey: '{service}.issues', dependsOn: ['credential', 'project'], } ``` +Every remote `selectorKey` must use the unified server selector path. Apply the `add-selector` skill: +add browser-safe metadata to `apps/sim/lib/selectors/manifest.ts`, reuse or extract a server-only +provider listing primitive, and add a credential- and destination-bound server attachment. Do not +add code under `hooks/selectors/providers`, a provider-specific query key, browser token acquisition, +or a selector-only API route. The shared context builder sends only active `dependsOn` values and +preserves exact `{{KEY}}` environment references for server-side resolution. + **Basic/Advanced mode for dual UX:** ```typescript // Basic: Visual selector @@ -589,7 +598,15 @@ If creating V2 versions (API-aligned outputs): 1. **V2 Tools** - Add `_v2` suffix, version `2.0.0`, flat outputs 2. **V2 Block** - Add `_v2` type, use `createVersionedToolSelector` -3. **V1 Block** - Add `(Legacy)` to name, set `hideFromToolbar: true` +3. **V1 Block** - Add `(Legacy)` to name, set `hideFromToolbar: true`, and add + `sunset: { status: 'legacy', replacedBy: '{service}_v2' }` — `check-block-registry` + fails a legacy block with no `replacedBy`, and the amber legacy badge plus its + click-to-upgrade action read from that field. + + **Only add `replacedBy` once the target is GA.** The same check also fails when + the target is unregistered, itself sunset, or still `preview: true`. If v2 is + preview-gated, leave v1 alone until GA and drop `preview` in the *same commit* + that adds the sunset — splitting them breaks the build in between. 4. **Registry** - Register both versions ```typescript @@ -630,6 +647,10 @@ If creating V2 versions (API-aligned outputs): - [ ] Added credential field with `requiredScopes: getScopesForService('{service}')` - [ ] Added conditional fields per operation - [ ] Set up dependsOn for cascading selectors +- [ ] Every remote `selectorKey` exists in the shared manifest and has one server attachment with + trusted credential provider binding and a fixed, credential-bound, or explicitly reviewed + user-controlled destination policy +- [ ] No selector provider logic, credential resolution, or provider route call runs in the browser - [ ] Configured tools.access with all tool IDs - [ ] Configured tools.config.tool selector - [ ] Defined outputs matching tool outputs @@ -922,7 +943,8 @@ requiredScopes: getScopesForService('{service}'), 3. **Block type is snake_case** - `type: 'stripe'`, not `type: 'Stripe'` 4. **Alphabetical ordering** - Keep imports and registry entries alphabetically sorted 5. **Required can be conditional** - Use `required: { field: 'op', value: 'create' }` instead of always true -6. **DependsOn clears options** - When a dependency changes, selector options are refetched +6. **DependsOn clears options** - When an active dependency changes, the shared selector facade + refetches with an opaque query revision; dependency values and references never enter query keys 7. **Never pass Buffer directly to fetch** - Convert to `new Uint8Array(buffer)` for TypeScript compatibility 8. **Always handle legacy file params** - Keep hidden `fileContent` params for backwards compatibility 9. **Optional fields use advanced mode** - Set `mode: 'advanced'` on rarely-used optional fields diff --git a/.agents/skills/add-permission-group-item/SKILL.md b/.agents/skills/add-permission-group-item/SKILL.md new file mode 100644 index 00000000000..c218ddf7a38 --- /dev/null +++ b/.agents/skills/add-permission-group-item/SKILL.md @@ -0,0 +1,274 @@ +--- +name: add-permission-group-item +description: Add a new governed item to Sim's enterprise permission groups — a boolean restriction, an allowlist, or a denylist — wired end-to-end from the field registry through the capability rule to the server gate that actually refuses. Use when adding a key to `PERMISSION_GROUP_FIELDS` or a capability to `CAPABILITY_RULES`. +argument-hint: +--- + +# Add Permission Group Item Skill + +You are adding one governed item an organization admin can withhold from a cohort of members. One entry in `apps/sim/lib/permission-groups/fields.ts` produces the write schema, the read schema, the `PermissionGroupConfig` type, the defaults, the tolerant parser, and (for a boolean) the admin editor row. + +**The registry does not produce enforcement.** Twelve keys once shipped with a checkbox, a hint, and no server check — an organization that ticked `hideCopilot` believed it had withheld a capability while every route still answered. Hence the `enforcement` field, the required `capability` field on every operation, and `scripts/check-permission-group-enforcement.ts`. You are done when something *refuses*, not when the key parses. + +## Read the system first + +- `lib/permission-groups/fields.ts` — registry, three field builders, `permissionGroupConfigSchema`, `tolerantArray`, `parsePermissionGroupConfig`. There is **no `types.ts`** (folded in here); the DB constraint maps live in `constraints.ts` +- `lib/permission-groups/capabilities.ts` — `CAPABILITY_IDS`, `CAPABILITY_RULES`, `capabilityRefusal`, `refuseCapability`, the static/parameterized split +- `lib/permission-groups/capability-assertions.ts` — the sanctioned assertion API; re-exports `capabilityRefusal`. `capability-error.ts` holds the thrown error, `capability-response.ts` the raw-route 403 +- `lib/permission-groups/integration-allowlist.ts` — the canonicalizing allowlist algebra, over the generated `block-successors.generated.ts` +- `lib/permission-groups/resolve.server.ts` — `resolveWorkspaceGroup`, `resolveVerifiedUserAccessControlContext`, `getUserPermissionConfig`, `getUserPermissionConfigForOrganization`, `mergeEnvAllowlist`. `ee/access-control/utils/permission-check.ts` re-exports it and keeps the executor gates +- `lib/permission-groups/config-scope.server.ts` (`resolvePermissionGroupConfig`, the per-request memo every assertion resolves through) and `request-scope.server.ts` (`withPermissionGroupScope`, deliberately import-free because `withRouteHandler` imports it) +- `lib/core/application/workspace-operation.ts` and `workspace-authorization.ts` — the required `capability` field, and the funnel +- `scripts/check-permission-group-enforcement.ts`, `check-application-graph.ts`, `check-capability-subject.ts` + +(Paths are under `apps/sim/` unless noted.) + +## Step 0: Decide what kind of thing it is + +| Kind | Builder | Default | Semantics | +|---|---|---|---| +| Boolean restriction | `booleanRestriction(enforcement, feature)` | `false` | `true` withholds. Name it `hideX` / `disableX`, never `allowX` | +| Allowlist | `allowlist(item, enforcement, { limited, empty })` | `null` | `null` allows everything; a list names the only permitted members; `[]` permits **none** | +| Denylist | `denylist(item, enforcement, phrasing)` | `[]` | Empty permits everything; members are refused | + +Allowlist when the safe posture is "only what the admin named" and the member set is enumerable (auth modes, connectors, model providers). Denylist when it is "everything except" and the set is open-ended (tool ids, models — an allowlist over a thousand tools grows a hole every time a tool ships). + +**Which mechanism refuses?** The `enforcement` value is a claim the audit checks. + +| Value | Meaning | +|---|---| +| `'capability'` | An operation declares a capability whose rule reads the key; the funnel refuses before the use case runs. Default answer for anything reachable through an application operation | +| `'executor'` | Read per block/tool/model at run time by `assertPermissionsAllowed` in `ee/access-control/utils/permission-check.ts`. Governs what a *run* may do, which no operation gate can express (one API call executes fifty blocks). Only `allowedIntegrations`, `allowedModelProviders`, `deniedModels`, `deniedTools` live here. The matching primitives these four keys are compared with live in `lib/permission-groups/` — `block-access.ts` (exemptions, superseded-version resolution), `operation-access.ts` (`createToolAccessGate`), `model-access.ts` (`createModelAccessGate`), `integration-allowlist.ts` — shared so the run-time gate and the editor/Copilot projections cannot drift. `allowedIntegrations` alone is also asserted outside a run, by `assertSelectorIntegrationAllowed` (`lib/selectors/server/integration-access.ts`) ahead of the provider call in `selectors.execute`, against the selector's own `resourceServiceId` / `integrationBlockTypes` rather than the credentials it accepts — reaching a provider API is a use of the integration, so a key here can still need a non-run enforcement site | +| `'ui-only'` | Hides a surface without withholding it. **Almost never right** — nothing ships as `ui-only`. Justify in the `enforcement` comment why a determined caller reaching the data is acceptable, and expect review to question it | + +**Is it per-operation at all?** `personal_api_key.use` is the one capability that is not: it withholds a *principal kind* across every operation, checked in the funnel's `personal_api_key` branch (`workspace-authorization.ts`) and again in `app/api/v1/middleware.ts`, so no operation declares it and its absence from every `capability:` field is correct rather than a hole. + +**Is the decision knowable from the config alone?** A rule needing a request value (an auth mode, a connector id) is *parameterized* and cannot be declared on an operation — see Step 3. + +**Is it a gate or a projection?** A key that withholds *fields from a response* rather than the response is a projection. `hideTraceSpans` and `hideCostInfo` work this way: the logs routes declare `capability: 'none'` and strip fields, because refusing the read would withhold the status and error message too. Projections have one owner — `lib/logs/log-projection.ts` (`resolveLogFieldProjection`, `projectExecutionData`, `projectCostTotal`), carrying the `permission-group-enforced:` annotations. Add yours there; two copies of a redaction rule is how one of them stops redacting. Corollary: refuse the query that *selects on* a withheld field — otherwise the projection is a filter oracle; `logQuerySelectsCost` / `assertLogCostQueryAllowed` in that same module are the shape. + +## Step 1: Append the field entry — never insert + +```ts + disableWidgetSharing: booleanRestriction('capability', { + id: 'disable-widget-sharing', + label: 'Widget Sharing', + category: 'Collaboration', + hint: 'Prevent sharing a widget outside the workspace.', + }), +``` + +The second argument is the field's `feature` (`PlatformFeatureMeta`); `PLATFORM_FEATURES` spreads it and appends `configKey`, so those four values are what the editor renders. `PLATFORM_FEATURES` is *derived* from the registry in `features.ts`, so a boolean key cannot reach the config without reaching the editor. + +- **Declaration order is the wire order** of `PermissionGroupConfig`, both zod schemas, and every config JSON crossing the API. `fields.test.ts` pins it with a key-order contract test, and `ee/access-control/components/group-detail.tsx` dirty-checks by comparing stringified configs — a moved key fails the suite *and* makes every open editor read as unsaved. Extend the tail; do not tidy the middle. +- **The default must be the permissive value.** Every stored `permission_group.config` row predates your key; `parsePermissionGroupConfig` fills the gap from the default and the update route merges a partial write over the stored config, so a restrictive default silently applies a new restriction to every existing group in every enterprise org. The builders hardcode `false` / `null` / `[]`, so a new key must be *phrased* so the permissive value is falsy: a `requireWidgetApproval` whose safe default is `true` must be inverted before it can use `booleanRestriction`. +- **The checkbox is inverted.** `group-detail.tsx` renders `checked={!editingConfig[feature.configKey]}` — ticked means *allowed*, so an `allowX` name renders backwards. +- **The hint must describe access withheld, never a surface hidden.** A `'capability'` key refuses at the API; "Hide the Tables module from the sidebar" tells an admin they are tidying a nav bar while they revoke a module. The same string is read again by `getActivePermissionGroupRestrictions` in `features.ts` as the prose for an *active* restriction — reaching users through the Copilot workspace VFS and the enterprise platform context — where "hide" is simply false. Write "Revoke the Tables module. Members cannot read or write any table." `PlatformFeatureMeta.hint` carries the rule in its TSDoc. +- **The category must be in `PLATFORM_CATEGORY_ORDER`** (`features.ts`): `Modules`, `Knowledge Base`, `Tables`, `Files`, `Deployment`, `Tools`, `Logs`, `Collaboration`, `Credentials & Access`. An unlisted category renders last. Categories name what is withheld — no surface-shaped section like "Sidebar". + +## Step 2: Only booleans get an admin UI for free + +`PLATFORM_FEATURES` filters on `field.kind === 'boolean-restriction'`. An allowlist or denylist renders **nothing** — the key exists, the API accepts it, no admin can set it. + +Nested pickers hang off the `featureExtras` map in `group-detail.tsx`, keyed by the **feature id of the boolean it nests under**, not the allowlist's own config key: + +```ts + const featureExtras: Partial> = { + 'hide-knowledge-base': , + } +``` + +Copy `setKnowledgeConnectors`. Two load-bearing behaviors: + +- **Refuse an empty selection** (`if (values.length === 0) return`) — an emptied allowlist denies everyone while the parent checkbox still reads as allowed. Withholding the whole thing is what the parent is for. +- **Collapse "all selected" back to `null`** (`values.length === ALL.length ? null : values`) — storing the full set freezes the allowlist at today's members. + +Choose the parent deliberately: `allowedKnowledgeConnectors` nests under `hide-knowledge-base`, not `disable-knowledge-base-creation`, because a connector attaches to an *existing* KB — nesting under creation would dim the picker for exactly the cohort it serves. + +## Step 3: Add the capability id and rule + +Skip only for `'executor'` / `'ui-only'`. Add the id to `CAPABILITY_IDS` and the rule to `CAPABILITY_RULES` in `capabilities.ts`, which uses `satisfies { readonly [K in PermissionGroupCapability]: CapabilityRule }` so a new id fails to compile until its rule exists. + +**Never replace that `satisfies` with a type annotation.** Annotating widens every entry to `CapabilityRule`, at which point `StaticPermissionGroupCapability` — derived by filtering the object's own entries for `kind: 'static'` — resolves to **`never`**: no operation can declare any capability, the type system goes quiet about capabilities entirely, and nothing at runtime looks wrong. `AssertsStaticCapabilityResolves` at the bottom of the file exists to catch it. Same reasoning for any of these registries. + +Capability ids are **domain-shaped** (`tables.create`); config keys are **surface-shaped** (`disableTableCreation`). `CAPABILITY_RULES` is the only place the two vocabularies meet. + +```ts + 'widgets.share': { + kind: 'static', + configKeys: ['disableWidgetSharing'], + detailCode: 'PERMISSION_GROUP_CAPABILITY_BLOCKED', + describe: 'Sharing widgets', + deniedBy: (config) => config.disableWidgetSharing, + }, +``` + +`configKeys` is what the audit reads to prove your key is enforced — it must list every key `deniedBy` reads. `describe` is the subject of one shared sentence, `" is not available under your organization's permission group"`, so make it a singular noun or gerund that agrees with "is". Exactly two functions build it, both defined in `capabilities.ts`: `refuseCapability(cap)` throws it as a `PermissionGroupCapabilityError`; `capabilityRefusal(cap)` returns it as a string for a raw route rendering its own body (`capability-assertions.ts` re-exports it so an inline gate reaches both through one module). Never write the sentence at a call site. + +Use `'PERMISSION_GROUP_CAPABILITY_BLOCKED'` for `detailCode`. Four rules carry a more specific one — `deploy.chat.auth_mode` (`CHAT_AUTH_MODE_NOT_PERMITTED`), `file_share.publish` / `file_share.auth_mode` (`PUBLIC_SHARING_NOT_ALLOWED`), `personal_api_key.use` (`PERSONAL_API_KEYS_DISABLED`) — which is why a call site reads the code off the rule and never spells one out. The set in `lib/core/application/forbidden.ts` is closed **over remedies, not causes** — a new code is warranted only when the remedy differs from "ask an organization admin", and requires an entry in `FORBIDDEN_DETAIL_CODE_DESCRIPTIONS` (a compile-time gate) plus a new value in the generated OpenAPI 403 description. + +A **parameterized** rule is the same shape with `kind: 'parameterized'` and a `deniedBy` taking the request value second — `'knowledge.connectors'` is `(config, connectorType) => allowlistDenies(config.allowedKnowledgeConnectors, connectorType)`. It **cannot be declared on an operation**: the funnel decides from principal, workspace and operation, never request input, and widening it would touch all ~315 operations for the sake of two keys. `defineWorkspaceOperation` throws at definition time (`Operation declares parameterized capability ; assert it from the use case instead`) rather than letting the operation read as gated while the gate never fires. + +## Step 4: Declare it on the operations it governs, or assert it at the call site + +`capability` is **required on the `ApplicationOperation` base type** (`lib/core/application/operation.ts:31`), typed `StaticPermissionGroupCapability | 'none'` — required there, not only on `defineWorkspaceOperation`, so a bare object literal minted by a domain factory does not compile without it (five OAuth-connection operations once shipped capability-less that way) — *and* guarded at definition time (`Operation declares no capability; name one, or 'none' with a reason`). The guard is not redundant: **`apps/sim/tsconfig.json` excludes `*.test.ts` / `*.test.tsx` from type-checking** and the enforcement audit walks past test files, so a fixture is the one construction site no static check reads. An absent capability does not deny — it throws `Cannot read properties of undefined` inside `capabilityDeniedBy`, and **only for a caller whose organization actually has a permission group**. It passes CI and every personal workspace, then fails in the tenants that bought the feature. + +**Static, and the operation is the whole decision** — set `capability` and write no gate code: + +```ts +export const shareWidget = defineWorkspaceOperation({ + id: 'widgets.share', + minimumRole: 'write', + workspaceApiKey: 'allow', + capability: 'widgets.share', + principalKinds: ['session', 'personal_api_key', 'workspace_api_key'], +}) +``` + +**The factory trap.** An operation minted by a factory that does not call `defineWorkspaceOperation` — a hand-frozen object — bypasses the required type *and*, once bypassed, the audit; twenty-one operations across six domains were invisible that way, and the file still printed a tick because some other operation in it was counted. The audit now matches the whole `defineOperation` family, resolves a same-file `function` factory (capability fixed in the body or taken as a positional second argument — `lib/table/application/operations.ts` shows both, with **no default** on the positional form so nothing inherits `tables.use` unreviewed), and cross-checks the members of every exported `*Operations` registry against what it parsed. Keep new operations inside an exported `*Operations` registry, mint them through a `define*Operation` builder taking an object literal with a string `id`, and use a `function` factory rather than an arrow const. + +**Static, but no operation to hang it on** — a raw route or an organization-level action. + +| Helper | Use when | +|---|---| +| `assertWorkspaceCapability(userId, workspaceId, cap, organizationId?)` | inside a use case — the thrown `PermissionGroupCapabilityError` is projected to a 403 for you | +| `isWorkspaceCapabilityWithheld(userId, workspaceId, cap, organizationId?)` | a raw handler rendering its own body — pair with `capabilityRefusal(cap)` | +| `isOrganizationCapabilityWithheld(organizationId, cap)` | an action naming an organization rather than a workspace | +| `isCapabilityWithheldForUser(userId, cap, workspaceId?)` (`lib/permission-groups/user-scope.server.ts`) | a user-level act that *may or may not* name a workspace — a personal API key, a CLI device-auth handoff. Resolves the workspace's group when given one, else falls back to the organization's default group rather than going ungoverned. Deliberately outside `capability-assertions.ts`: it reads org membership through the billing graph, and that module is a guarded root of `check:application-graph` | +| `capabilityDeniedBy(cap, config)` | you already hold a resolved config | + +Annotate the call site either way: + +```ts + // permission-group-enforced: logs.export — raw streaming route, no workspace operation to declare it on + if (capabilityDeniedBy('logs.export', permissionConfig)) { + return capabilityRefusalResponse('logs.export') + } +``` + +`capabilityRefusalResponse` (`lib/permission-groups/capability-response.ts`) is the one builder for that 403 — it renders `capabilityRefusal(cap)` *and* reads `details.code` off the rule, so a hand-rolled `NextResponse.json({ error: … }, { status: 403 })` reports the four specifically-coded capabilities as the generic block. v1 is deliberately not converged on it (`resolveCapabilityRefusal` in `app/api/v1/middleware.ts` renders v1's own `{ error: { code, message } }` envelope). + +`isOrganizationCapabilityWithheld` resolves through `getUserPermissionConfigForOrganization`, reading the organization's **default** group — a non-default group targets specific workspaces. It sits outside the per-request memo because that memo is keyed by user and workspace, and this decision is keyed by organization alone. + +**Parameterized** — the helpers above are all typed `StaticPermissionGroupCapability`, so write a module-local wrapper that reads the rule and raises through `refuseCapability`, and annotate the call site. `assertConnectorTypeAllowed` in `lib/knowledge/application/connectors.ts` is the shape: + +```ts +const RULE = CAPABILITY_RULES['knowledge.connectors'] +if (!userId) return +const config = await resolvePermissionGroupConfig(userId, workspaceId, undefined) +if (config && RULE.deniedBy(config, connectorType)) refuseCapability('knowledge.connectors') +``` + +Always route through `CAPABILITY_RULES` and raise with `refuseCapability` — a config key spelled out inline silently stops denying when renamed, and a hand-written message drifts from the funnel's. `validatePublicFileSharing` and `validateChatDeployAuth` in `ee/access-control/utils/permission-check.ts` are the other two examples. Return early on a missing `userId`: a permission group is a membership of users, so an actorless caller resolves none, and throwing there turns a scheduled sync into a 500 instead of a refusal anyone can act on. + +**Genuinely ungoverned** — write `capability: 'none'` with a `// permission-group-exempt: ` comment directly above it (`'none'` is spelled out because an absent field cannot be told apart from an unreviewed one). A good reason names why no key applies *and* why a gate would be wrong: *"the executor's own per-run store; no group key names it, and refusing would fail runs the group allows"*. + +### Surfaces that do not go through the funnel + +**Whose group applies — never `userId` off whatever identity is nearest.** Each helper below returns `null` for a caller no group governs (workspace key, internal JWT, executor delegation), and `null` is a *pass*, not a denial. + +| You hold | Helper | +|---|---| +| `Principal` | `capabilityGovernedPrincipalUserId` (`lib/core/application`) — mirrors the funnel exactly, executor exemption included | +| v1 `RateLimitResult` | `capabilityGovernedUserId(rateLimit)` (`app/api/v1/middleware.ts`) — branches on `keyType`, never on the presence of `userId` | +| `TableAccessPrincipal` | `capabilityGovernedUserId(principal)` (`app/api/table/utils.ts`) | +| `AuthResult` from `checkSessionOrInternalAuth` | `capabilityGovernedAuthUserId` (same file) — an internal JWT's `userId` is the run's actor, a bystander | + +When the subject is **persisted and read back later** — the table dispatch pipeline stamps it on `table_run_dispatches` / `table_row_executions` so auto-fired cells run under the person the write was gated for — declare it `capabilityGovernedUserId: string | null`, required with an explicit `null` and never optional. An optional field with a fallback is how every producer that had not been taught the distinction silently inherited `triggeredByUserId`, an *attribution* naming the billed account; making omission a compile error is the whole enforcement. A persisted subject also has a lifecycle: `lib/users/account-deletion.ts` cancels the dispatches stamped with a deleted user. + +- **`/api/v1`** authorizes in `app/api/v1/middleware.ts`. Every route threads a `V1RouteCapability` (`StaticPermissionGroupCapability | 'none'`, required and spelled out) whose value must match what its v2 or internal counterpart declares — v1 gets no mapping of its own. `check-capability-subject.ts` audits v1's subjects only, because the bug has shipped and been fixed twice there. +- **Raw internal table routes** (`/api/table/**`) share one gate in `checkAccess` (`app/api/table/utils.ts`), whose signature takes a `TableAccessPrincipal` union — `{ kind: 'user'; userId }` or `{ kind: 'workspace_api_key'; keyCreatorUserId }` — so a bare id no longer type-checks and only the kind that says so skips the gate. `tableAccessPrincipal(rateLimit)` builds it for v1. +- **The route-wrapper graph.** `withRouteHandler` imports `request-scope.server.ts` and nothing heavier. Import a resolver at the *call site*, never from the wrapper or `lib/core/application` — see Step 6. + +## Step 5: Add it to the golden corpus + +Add the key to **both** the `input` and `expected` objects of the `'a fully populated config'` fixture in `lib/permission-groups/fields.test.ts`, set to a non-default value. That fixture is the pinned coercion corpus: a row that changes in a later diff is a semantic decision someone defends rather than a silent regression. The file's other assertions derive from `DEFAULT_PERMISSION_GROUP_CONFIG` (wire order, idempotence, read-schema acceptance, the 2000-iteration seeded fuzz, write/default/read key-set agreement, boolean-to-`PLATFORM_FEATURES` coverage) and pick your key up for free, as does `features.test.ts`. + +**Give the funnel test a real `workspaceOrganizationId`.** `requireCapability` short-circuits on `context.workspaceOrganizationId === null` (`lib/core/application/workspace-authorization.ts:204`), so a fixture whose workspace context leaves it null passes with the gate present *and* with it removed — a vacuous test that reads as load-bearing. + +Add a case to `capabilities.test.ts` for any rule with logic beyond reading one key. For an allowlist assert all three states — `null` permits every member, a populated list only the named ones, `[]` permits **none** — as `capabilities.test.ts` already does for `knowledge.connectors`. + +## Step 6: Keep the graph light + +`scripts/check-application-graph.ts` walks **runtime** `import` / `export … from` edges (`import type` is erased and allowed) out of five guarded roots: + +| Guarded root | Forbidden | +|---|---| +| `lib/core/application/index.ts`, and `lib/permission-groups/` `capabilities.ts` / `capability-assertions.ts` / `config-scope.server.ts` | `providers/`, `blocks/`, `tools/`, `executor/`, `lib/uploads/`, `lib/workflows/` | +| `lib/core/utils/with-route-handler.ts` | those six **plus** `lib/billing/`, `lib/permission-groups/resolve.server`, `lib/auth`, `lib/copilot/`, `lib/knowledge/` | + +`lib/billing/` stays allowed for the funnel roots because `resolve.server.ts` legitimately reads the subscription to decide whether an organization is on an enterprise plan; the wrapper is a lifecycle shim that opens the memo scope and nothing more. That split is why the scope is two files. + +Breaking this never announces itself — past regressions surfaced only as unrelated tests failing on partial mocks of modules they never meant to load. After adding an import, run this audit first. + +## Step 7: Verify + +```bash +bun run check:permission-group-enforcement +bun run check:application-graph +bun run check:capability-subject +cd apps/sim && bun run type-check +cd apps/sim && bunx vitest run lib/permission-groups +``` + +Also `bun run check:api-validation` if you touched a contract or the group routes. `bun run check:audits` runs all of these; it derives its list from the `check:*` scripts in `package.json`, so a new audit is opted *out* deliberately rather than opted in. + +Read the success lines, not the exit codes — the counts should have grown by your operation and capability: + +``` +✓ permission-group enforcement: 322 operations declare a capability, 35 capabilities all enforced +✅ Application graph clean: 5 roots reach none of 11 forbidden module trees +check:capability-subject — 32 v1 files, 5 capability subjects resolved through capabilityGovernedUserId. +``` + +The enforcement audit is all-or-nothing — one success line or findings, no migration mode that exits 0 with work outstanding. Because it reads source text it also refuses success when its own parsers come up empty or disagree with each other; if a self-check fires, teach the parsers the new form rather than working around it. + +The audits prove *reachability*: your capability is named somewhere, your key is read by some rule. They cannot tell whether the rule's logic is right or whether every operation reaching the behavior declares it. Green is not proof the gate fires. + +## Traps + +**An operation carries exactly ONE capability, and a narrower capability must subsume the broader one it replaced.** `knowledge.create` and `knowledge.upload` list `configKeys: ['disableKnowledgeBaseCreation', 'hideKnowledgeBaseTab']` and OR both in `deniedBy`, because moving KB creation off `knowledge.use` would otherwise let a group that withheld the whole module still create one through the API. Any time you re-point an operation to a more specific capability, the specific rule must read both keys. + +**`.catch()` on an array field is a fail-open security bug.** `z.array(item).catch(fallback)` is whole-value tolerant: one bad member discards every good one. On an allowlist the fallback is `null`, and `null` means **unrestricted** — a partly corrupt allowlist stops restricting anything. `tolerantArray` in `fields.ts` filters element by element, keeping what parses and failing closed. Never swap it for `.catch()`, never hand-roll a parallel coercion path. + +**`parsePermissionGroupConfig` must keep its `Array.isArray` guard.** `typeof [] === 'object'`, so a truthy-object check alone lets an array through and `z.object().parse([])` throws — reachable, because the column is `jsonb` and a row can genuinely hold `[]`. The guard returns the defaults there instead of taking down the request. `tolerantArray` carries the mirror-image guard. + +**An empty allowlist denies everything; `null` allows everything.** They must never collapse — not in the parser, the UI setter, or a `deniedBy`. `allowlistDenies` encodes it as `allowed !== null && !allowed.includes(member)`; a `?? []` anywhere on this path inverts the unrestricted case. + +**Canonicalize both halves of an integration allowlist *before* intersecting, never after.** `allowedIntegrations` and the deployment's `ALLOWED_INTEGRATIONS` are written independently, so one can name `slack` and the other `slack_v2`; fold only case and they intersect to nothing, hiding an integration both policies allow. Compose `intersectAccessControlAllowlists` / `toAccessControlAllowlist` / `resolveAccessControlBlockType` from `integration-allowlist.ts` — never a hand-rolled `Set` intersection — and successor-resolve the type you test against the result the same way. They resolve through `block-successors.generated.ts`, a projection of the block registry because `check:application-graph` forbids the funnel from importing `blocks/`; `check:block-successors` fails the build when it drifts. Read that map only through `Object.hasOwn`: the ids arriving are admin-supplied jsonb, and a group naming `constructor` otherwise gets back an inherited function and 500s every enforcement path that reads it. + +**Not everyone goes through the funnel.** + +| Principal | Rule | +|---|---| +| **Workspace API key** | Authorizes as the workspace — no user, so no group resolves and `operation.capability` does not apply. **Never substitute the key's creator** (not in the funnel, `checkAccess`, v1, or the log projection): it applies a bystander's group to every caller of a shared key and breaks the key when that person leaves. The escape is closed at the door — minting a workspace key is itself capability-gated | +| **Delegated `executor` with a `sim_user` subject** | **Role only** (`requireCurrentHumanRole`). A run carries the trigger-er's role but not their capabilities: a capability names what a *person* may reach, while a run reaches resources because a block does. Applying capabilities would make "hide Tables" a kill-switch breaking every workflow with a Table block | +| **Actorless deployment run** (delegated executor, `mode: 'deployment'`, no subject) | Passes through — a deployed workflow acts with the workspace's authority, not its author's group. Denying would 403 every scheduled run, webhook and public-API call the moment a group withheld anything | +| **Copilot** | **NOT exempt.** A delegated principal with a `sim_user` subject whose `serviceId` is anything other than `executor` takes the full `requireCurrentHumanAccess`, capability check included. Copilot acts *as the person* | + +What a run *does* is still governed by `assertPermissionsAllowed`. An item that must bind a deployed run belongs at `enforcement: 'executor'`. + +**Capability is checked after the role check, on purpose.** `requireCurrentHumanAccess` runs `requirePermission` first. `NoWorkspaceAccessError` is concealed as a 404 by the v2 surface so a non-member cannot learn the resource exists; refusing on capability first hands an outsider an oracle for which capabilities the organization withholds. Do not reorder, and do not add a capability check upstream of the role check in a raw route — the v1 middleware states the same rule in its TSDoc. + +## Checklist Before Finishing + +- [ ] Kind and `enforcement` chosen deliberately; `ui-only` justified in writing if used +- [ ] It is a gate, not a projection — a projection belongs in `lib/logs/log-projection.ts` with `capability: 'none'` on the routes, and still refuses queries that select on the withheld field +- [ ] Entry **appended** to `PERMISSION_GROUP_FIELDS`, permissive default, restriction-phrased name +- [ ] Category present in `PLATFORM_CATEGORY_ORDER`, named after what is withheld +- [ ] `hint` says what access is revoked, never "hide" — it is also the active-restriction prose +- [ ] Non-boolean key has a `featureExtras` picker that refuses empty and collapses "all" to `null` +- [ ] Capability id in `CAPABILITY_IDS`, rule in `CAPABILITY_RULES` under `satisfies`, `configKeys` lists every key `deniedBy` reads +- [ ] A narrower capability replacing a broader one also reads the broader key +- [ ] Declared on every operation it governs, or asserted from the use case with a `// permission-group-enforced:` annotation raising through `refuseCapability` / `capabilityRefusal` +- [ ] New operations minted through a `define*Operation` builder and exported from an `*Operations` registry +- [ ] Any `capability: 'none'` carries a `// permission-group-exempt:` reason +- [ ] Every gate's subject comes from the `capabilityGoverned*` helper for the identity it holds, and a persisted subject is a required `string | null` +- [ ] v1 routes thread the capability through `middleware.ts`; table routes pass a `TableAccessPrincipal` +- [ ] An integration-shaped allowlist canonicalizes through `integration-allowlist.ts` on both sides of every comparison +- [ ] Added to the `'a fully populated config'` fixture in `fields.test.ts`, input and expected +- [ ] Allowlist three-state (`null` / populated / `[]`) covered in `capabilities.test.ts` +- [ ] No new runtime import from a guarded root into a forbidden tree +- [ ] All three audits pass and name your capability; `type-check` clean, `lib/permission-groups` suite green diff --git a/.agents/skills/add-selector/SKILL.md b/.agents/skills/add-selector/SKILL.md new file mode 100644 index 00000000000..ec5fc589737 --- /dev/null +++ b/.agents/skills/add-selector/SKILL.md @@ -0,0 +1,133 @@ +--- +name: add-selector +description: Add or update a Sim dynamic selector using the shared manifest, server attachment, and selectors.execute path. Use for provider-backed, internal, or local option lists referenced by block, trigger, or connector selectorKey fields. +argument-hint: +--- + +# Add Selector + +Dynamic selectors expose option metadata while a workflow or connector is being configured. Every +remote selector executes through the authorized `selectors.execute` application operation; the +browser never resolves credentials or calls a provider directly. + +## Read the shared boundary + +Before editing, read: + +- `apps/sim/lib/selectors/types.ts` +- `apps/sim/lib/selectors/manifest.ts` +- `apps/sim/lib/selectors/context.ts` +- `apps/sim/lib/selectors/server/types.ts` +- `apps/sim/lib/selectors/server/registry.ts` +- `apps/sim/hooks/queries/selectors.ts` + +Then read the nearest existing selector attachment and the block, trigger, or connector declaration +that will consume the key. + +## Classify the selector + +- `provider-server`: contacts an external provider or uses provider credentials. +- `internal-server`: reads protected Sim data through an existing authorized application use case. +- `local`: pure browser-safe data with no protected data, credentials, references, or network I/O. + +Add every key to the browser-safe manifest in `lib/selectors/manifest.ts`. `SelectorKey` derives from +that manifest; do not maintain a second union. Manifest entries contain data only: allowed context, +readiness, scope kinds, list/search/detail capabilities, and stale time. Do not import provider SDKs, +credentials, server helpers, or attachment functions into the manifest. + +## Build context from active values + +Declare `dependsOn` on the consuming sub-block or connector field. The shared context builder sends +only declared, active dependencies: + +- Canonical basic/advanced pairs contribute the active value under their canonical key. +- Action and trigger modes contribute only fields active on that surface. +- Exact environment references such as `{{GMAIL_CREDENTIAL_ID}}` remain unresolved in the browser. +- Runtime block-output references are not selector context. +- Embedded environment interpolation such as `https://{{HOST}}/path` is unsupported. +- `impersonateUserEmail` is the one explicit compatibility hint projected when the manifest allows + it, even when it is absent from `dependsOn`. + +Add a new `SelectorContextKey` only when the value is a real, reusable selector dependency. Allow it +explicitly on each relevant manifest entry. Never send a full block or connector configuration. + +## Add the server attachment + +For `provider-server`, add the service's attachment map under +`apps/sim/lib/selectors/server/providers/` and include it in the exhaustive server registry. For +`internal-server`, add the attachment in `apps/sim/lib/selectors/server/internal.ts`. Local keys use +the exhaustive browser-safe registry in `apps/sim/lib/selectors/client/local.ts` and never enter the +server registry. A provider attachment declares: + +- For stored credentials, a credential policy with the exact context field and trusted + `serviceIds`. Raw-connection selectors instead validate the connection material projected from + their allowed context and bind it to a deliberate destination policy. +- Destination policy: `fixed`, `credential-bound`, or `user-controlled`. +- A list/detail adapter that explicitly projects `id`, `label`, and allowlisted scalar `meta`. + +Stored credentials must pass actor-use, workspace, and provider/service binding checks. Do not trust +a provider, service, operation kind, origin, or module name supplied by the browser. + +Choose the destination policy deliberately: + +- `fixed`: provider origin is code-defined. +- `credential-bound`: origin/account/site comes from, or is verified against, the authorized + credential. +- `user-controlled`: the user selects the destination. Hidden use-only authentication requires an + explicit security policy; do not combine it with an arbitrary destination by default. + +Reuse or extract a server-only provider listing primitive. If an existing provider route has +non-selector callers, keep the route as a thin caller of that primitive. If it is selector-only, +move the logic and remove the obsolete route and contract. Never import a route handler or make an +internal HTTP request from an attachment. + +The attachment must return normalized selector results only. It must never deliberately or +wholesale echo selector context, and hidden/server-only resolved material, credential IDs, tokens, +and authentication secrets must never cross the response boundary. Browser-known literals and +viewable personal/shared values are not automatically server-only secrets, but they may appear in +an option only when the adapter intentionally projects them as provider resource metadata. Let the +shared executor own scope authorization, exact-reference resolution, credential authorization, +error projection, and output sanitization; adapters must pass and preserve the executor's abort +signal during provider work. + +## Wire the UI declaration + +Point the block, trigger, or connector field at `selectorKey` and declare its `dependsOn` fields. +Keep connector selector/manual canonical pairs and fork reconfiguration behavior intact. Static +`options` stay local and need no selector. + +Do not add: + +- A module under `hooks/selectors/providers` or any client provider fetcher. +- A provider-specific React Query key. +- A selector-specific OAuth-token request. +- A selector-only API route when the provider primitive can be called directly. + +All server selectors use the shared POST contract and React Query facade. Query identities must stay +opaque and must not include context values, references, credential IDs, secrets, or their hashes. +Selector code must not add context, token, or result caches. The sole existing cache exception is +authorized client-credential resolution after authorization and provider binding: it may reuse the +credential service's TTL-governed, lazily pruned process-local token cache. This exception requires +explicit security-owner acceptance; do not broaden it or describe it as hard-bounded. + +## Focused validation + +Follow nearby Vitest and route-test style. Do not add an authorization matrix for every ordinary +provider attachment; the shared executor tests own shared security behavior. + +Add a focused adapter test when behavior is special, such as pagination, nontrivial destination +binding, provider-specific projection, or a raw-connection policy. For an ordinary fixed-origin OAuth +list, manifest/registry exhaustiveness plus an existing provider primitive test is usually enough. + +Run the smallest relevant set, then: + +```bash +bunx vitest run +bun run --cwd apps/sim type-check +bun run check:fork-dependent-coverage +bun run check:client-boundary +git diff --check +``` + +Confirm there is no browser-side provider call, every server key has one attachment, and every +returned option is explicitly projected. diff --git a/.agents/skills/add-selector/agents/openai.yaml b/.agents/skills/add-selector/agents/openai.yaml new file mode 100644 index 00000000000..c11b8a3bd73 --- /dev/null +++ b/.agents/skills/add-selector/agents/openai.yaml @@ -0,0 +1,5 @@ +interface: + display_name: "Add Selector" + short_description: "Build a secure dynamic selector" + brand_color: "#2563EB" + default_prompt: "Use $add-selector to add or update a Sim dynamic selector through the unified server execution path." diff --git a/.agents/skills/add-trigger/SKILL.md b/.agents/skills/add-trigger/SKILL.md index 2bdfbc8e29f..c648b9d0d61 100644 --- a/.agents/skills/add-trigger/SKILL.md +++ b/.agents/skills/add-trigger/SKILL.md @@ -476,7 +476,9 @@ Add to `helm/sim/values.yaml` under the existing polling cron jobs: A sub-block gets its choices from exactly one of two places. There is no third. -**`selectorKey` — every remote list.** Register the list in `hooks/selectors/providers//selectors.ts`, add its key to `SelectorKey`, and point the sub-block at it. A selector is parameterized by an explicit `SelectorContext`, so the same definition serves the canvas, the workspace-fork sync modal, and anything added later. +**`selectorKey` — every remote list.** Use the `add-selector` skill to add the key to the +browser-safe manifest and attach its provider behavior on the server. All remote selectors execute +through `selectors.execute`; never add a client provider module or selector-only fetch route. ```ts { id: 'triggerCredentials', type: 'oauth-input', canonicalParamId: 'oauthCredential', mode: 'trigger' }, @@ -485,7 +487,11 @@ A sub-block gets its choices from exactly one of two places. There is no third. { id: 'manualLabelIds', type: 'short-input', mode: 'trigger-advanced' }, ``` -`canonicalParamId: 'oauthCredential'` on the credential sub-block is the line people forget. `buildSelectorContextFromBlock` keys the context on a sub-block's CANONICAL id, so without it `context.oauthCredential` is never set and the picker looks unfixable without reading the store. (A credential field is also recognised by its `oauth-input` TYPE as a fallback, so a block whose shipped param is already named something else does not have to rename it.) +`canonicalParamId: 'oauthCredential'` on the credential sub-block is the line people forget. The +shared context builder uses trigger mode and projects only active `dependsOn` values under their +canonical ids. Exact `{{KEY}}` environment references remain unresolved until the authorized server +executor. A credential field is also recognized by its `oauth-input` type as a compatibility +fallback. **`options` — everything else.** A static array, or a pure function of the block's own values for a list that narrows to a sibling's selection. No I/O. @@ -500,7 +506,8 @@ options: (params) => { Two rules the checks enforce: -- **A secret never enters a selector's `getQueryKey`.** A query key identifies a resource; a credential authorizes access to it. A credential *id* is fine; a typed password is not (see `imap.mailboxes`). +- **Selector query keys contain no dependency values.** Credential IDs, secrets, unresolved + references, and their hashes stay out of browser cache identities. - **A sub-block that `dependsOn` a credential / knowledge-base / table selector must be reconfigurable at fork-sync time** — a `selectorKey`, a canonical pair whose basic member is a selector, or a `short-input`/`long-input`. `bun run check:fork-dependent-coverage` fails otherwise, because a fork sync clears those fields on every push and an unofferable one can never be set anywhere that sticks. ## Checklist diff --git a/.agents/skills/babysit/SKILL.md b/.agents/skills/babysit/SKILL.md index 84110c349f0..16f8cef5b32 100644 --- a/.agents/skills/babysit/SKILL.md +++ b/.agents/skills/babysit/SKILL.md @@ -1,6 +1,6 @@ --- name: babysit -description: Drive a PR to a clean review (Greptile 5/5, zero open threads) — ships if needed, keeps it mergeable against staging, triggers Greptile, fixes real findings, replies to and resolves every thread, and loops until clean +description: Drive a PR to a clean review (Greptile 5/5, zero open threads) — ships if needed, keeps it mergeable against staging, re-triggers both Greptile and cubic, fixes real findings, replies to and resolves every thread, and loops until clean --- # Babysit PRs @@ -8,7 +8,8 @@ description: Drive a PR to a clean review (Greptile 5/5, zero open threads) — Owns a PR end-to-end through review: ship it, wait for the automatic review round, and if it isn't already clean, drive fix → reply → resolve → re-review cycles until Greptile reports 5/5 and there are zero open comment threads, keeping the branch mergeable against staging along the -way. Designed to be run under `/loop` (no fixed interval — let it self-pace on review latency) +way. Two bots review this repo — Greptile and cubic — and they behave differently; see +"Two reviewers" below. Designed to be run under `/loop` (no fixed interval — let it self-pace on review latency) so it survives across multiple wakeups in the same session. ## When to use @@ -23,20 +24,49 @@ Needs a PR number. If none is given and there's no open PR for the current branc first (which includes the `origin/staging` sync check — see `.agents/skills/ship/SKILL.md`) to create one. +## Two reviewers + +Both post inline threads that count toward "clean", and they need re-triggering separately: + +| | Greptile (`greptile-apps`) | cubic (`cubic-dev-ai`) | +|---|---|---| +| Verdict | `Confidence Score: X/5` in a summary comment | no score — only inline threads | +| Summary comment | edited in place across rounds | fresh review per run | +| Re-trigger | `@greptile` | `@cubic-dev-ai review this PR` | +| Latency | 1–3 min | 1–3 min | + +Post **both** after every push, as two separate comments. Triggering only Greptile is the easy +mistake: the PR then shows 5/5 with cubic's threads still open from an earlier commit, and its +findings never get re-checked against the fix. + +`@cubic-dev-ai review this PR` is the documented wording — `@cubic` alone does not trigger it. + +cubic reviews the commit that was HEAD when its run started, so a thread can describe code the +next commit already changed. Before treating a cubic finding as real, check whether the current +HEAD still has the problem — a stale round is a reply-and-resolve, not a fix. + ## Definition of "clean" -Both must hold: +All three must hold: 1. The latest Greptile summary comment reports **Confidence Score: 5/5** -2. `reviewThreads` (GraphQL, see below) has **zero threads with `isResolved: false`** +2. `reviewThreads` (GraphQL, see below) has **zero threads with `isResolved: false`**, from + either bot +3. Every check has **finished and passed** — `gh pr checks ` shows no `fail` *and* no + `pending`. A red run is not clean no matter what the reviewers say, and the lint/audit jobs + routinely catch what a local run misses. A `pending` one is not clean either: it has not + reported yet, and treating "not failing" as "passing" reports the PR clean before CI has + had its say. Wait for it — the step-10 stop condition covers a check that never settles. Do not stop early on "no new comments this round" alone — a thread can be open from an earlier -round. Always check both conditions freshly after every push. +round, and cubic often lands its first threads a round after Greptile's. Always check all three +conditions freshly after every push. ## Loop 1. **Check current state** before doing anything, including whether the PR is still mergeable: ```bash gh pr view --json mergeable + gh pr checks | grep -v skipping gh pr view --json comments -q '[.comments[] | select(.author.login=="greptile-apps")] | last | .body' gh api graphql -f query=' query { repository(owner: "", name: "") { pullRequest(number: ) { @@ -51,16 +81,22 @@ round. Always check both conditions freshly after every push. stop yet: re-run the same query with `after: ""` and keep paging until `hasNextPage` is `false` before evaluating "clean." A PR with more than 50 threads is rare but stopping on a partial page would silently miss unresolved ones past the cutoff. - If `mergeable` is `CONFLICTING`, fix that first (step 2). Otherwise, if Greptile is 5/5 and - every thread across all pages has `isResolved: true`, stop — report the outcome (see - "Reporting" below) and skip the rest of this list. + The query returns both bots' threads. A `ReviewThread` has no author of its own — identity + lives on its comments, so read the opener's at `comments.nodes[0].author.login` and do not + add an `author` field at the thread level, which makes the query fail to compile. + If `mergeable` is `CONFLICTING`, fix that first (step 2). If a check is failing, fix that too + — treat it exactly like a review finding. If a check is still `pending`, do not evaluate + "clean" at all: go to step 9 and wait for it. Otherwise, if Greptile is 5/5, every thread + across all pages has `isResolved: true`, and every check has finished and passed, stop — + report the outcome (see "Reporting" below) and skip the rest of this list. 2. **If the PR has a merge conflict**, merge `origin/staging`, resolve the conflicts, run the usual pre-push checks, push, and go to step 8 to re-trigger review. -3. **If no review has run yet** (fresh PR, no Greptile comments): Greptile usually runs - automatically on PR open — confirm via `gh pr checks ` (look for `Greptile Review`) and - wait for that first round before doing anything else. +3. **If no review has run yet** (fresh PR, no bot comments): both run automatically on PR open — + confirm via `gh pr checks ` (look for `Greptile Review` and `cubic · AI code reviewer`) and + wait for both before doing anything else. They finish at different times, so a PR that looks + clean because only one has reported is not clean yet. 4. **If a review round has landed and it isn't clean**: for every thread where `isResolved: false`, triage the finding on its own merits — this is the part that requires @@ -113,14 +149,19 @@ round. Always check both conditions freshly after every push. rounds; checking sync only before the push (step 6) and never after is how a bad push or a PR whose commit history quietly went stale between rounds goes unnoticed. -8. **Re-trigger review** by posting `@greptile` as its own PR comment: +8. **Re-trigger both reviewers**, each as its own PR comment — a combined comment does not + reliably trigger both: ```bash gh pr comment --body "@greptile" + gh pr comment --body "@cubic-dev-ai review this PR" ``` + Then confirm both actually picked it up before waiting — `gh pr checks ` should show + `Greptile Review` and `cubic · AI code reviewer` as `pending`. If one stayed `pass` from the + previous round, its trigger did not land; re-post that one. 9. **Wait for the new round**, then go back to step 1. Pace the wait with `ScheduleWakeup` using - a fallback delay of ~250–300s (Greptile typically takes 1–3 minutes) — never busy-poll - in a sleep loop. Pass the same `/loop babysit PR ` prompt on each wakeup so the loop + a fallback delay of ~300s — both bots take 1–3 minutes, and CI is usually the slowest of the + three — never busy-poll in a sleep loop. Pass the same `/loop babysit PR ` prompt on each wakeup so the loop resumes correctly. 10. **Stop conditions**: clean state reached (see above), or the same unresolved finding or @@ -130,7 +171,8 @@ round. Always check both conditions freshly after every push. ## Reporting When the loop ends, summarize: how many rounds it took, what was actually fixed (one line each), -what was pushed back on as a false positive and why, and the final Greptile score / thread count. +what was pushed back on as a false positive and why, and the final state — Greptile score, open +thread count across both bots, and whether every check finished and passed. ## Public-repo hygiene @@ -150,4 +192,6 @@ notification email. - Never fix a finding with a hacky workaround — if the clean fix isn't obvious, find the sibling pattern elsewhere in the codebase solving the same class of problem and match it. - Never silently drop a finding — every thread gets either a code fix or a reasoned reply. +- Never re-trigger only one reviewer. Both get a comment after every push, and both get confirmed + `pending` before you start waiting. - Always re-run the `/ship`-style sync check before every push in the loop, not just the first. diff --git a/.agents/skills/migrate-application-operation/SKILL.md b/.agents/skills/migrate-application-operation/SKILL.md index 7b51087bb60..2786721895c 100644 --- a/.agents/skills/migrate-application-operation/SKILL.md +++ b/.agents/skills/migrate-application-operation/SKILL.md @@ -167,6 +167,22 @@ Choose principal kinds from actual behavior. Do not accept every principal merel Route declarations, tool adapters, and use cases must use the same literal operation. Runtime operation selection is permitted only from a trusted, code-defined registry. Never accept an operation ID or permission tag from an HTTP body, model argument, or other untrusted input. +### Unified selector execution is one operation + +Dynamic selector dispatch is the deliberate instance of trusted runtime selection. Define and +authorize `selectors.execute` once: it means "enumerate options while configuring a workflow or +workspace resource." The browser supplies only a selector key from the exhaustive browser-safe +manifest, scope, allowlisted context, and list/detail request. After canonical scope authorization, +the application use case selects the matching attachment from the exhaustive server-only registry. + +Provider and internal attachments are trusted implementation adapters under that semantic operation, +not separate application operations. Do not create one operation per selector, provider, or listing +endpoint. Attachments may choose only code-defined credential/service binding, destination policy, +provider primitive, and projection behavior; they must not accept a module, provider, service, +operation kind, origin, or permission tag from the request. The `selectors.execute` use case owns +reference resolution, credential authorization, provider invocation, sanitization, and safe result +projection end to end. + ## Implement the application use case Use `defineAuthorizedWorkspaceUseCase` directly or a thin domain binding that supplies domain-specific authorization options: diff --git a/.agents/skills/validate-connector/SKILL.md b/.agents/skills/validate-connector/SKILL.md index cea6e14ad73..81070a8e2a3 100644 --- a/.agents/skills/validate-connector/SKILL.md +++ b/.agents/skills/validate-connector/SKILL.md @@ -37,11 +37,16 @@ apps/sim/components/icons.tsx # Icon definition for the service If the connector uses selectors, also read: ``` -apps/sim/hooks/selectors/registry.ts # Selector key definitions -apps/sim/hooks/selectors/types.ts # SelectorKey union type -apps/sim/lib/workflows/subblocks/context.ts # SELECTOR_CONTEXT_FIELDS +apps/sim/lib/selectors/manifest.ts # Browser-safe exhaustive metadata +apps/sim/lib/selectors/types.ts # Selector context and option types +apps/sim/lib/selectors/context.ts # Active canonical context projection +apps/sim/lib/selectors/server/registry.ts # Exhaustive server attachments +apps/sim/lib/selectors/server/providers/* # Matching provider attachment ``` +Apply the `validate-selector` skill to the matching key and provider primitive. There is no client +provider selector registry. + ## Step 2: Pull API Documentation Fetch the official API docs for the service. This is the **source of truth** for: @@ -219,8 +224,17 @@ Connectors where the list API already returns content inline (e.g., Slack messag - A `type: 'selector'` field with `selectorKey`, `canonicalParamId`, `mode: 'basic'` - A `type: 'short-input'` field with the same `canonicalParamId`, `mode: 'advanced'` - `required` is identical on both fields in the pair -- [ ] `selectorKey` values exist in the selector registry +- [ ] `selectorKey` values exist in the browser-safe manifest and remote keys have exactly one + server attachment - [ ] `dependsOn` references selector field `id` values, not `canonicalParamId` +- [ ] The shared builder projects only the active canonical dependency into an allowlisted + `SelectorContextKey`; exact `{{KEY}}` references remain unresolved in the browser +- [ ] The attachment binds stored credentials to the actor, workspace, and trusted provider/service + and declares a reviewed `fixed`, `credential-bound`, or `user-controlled` destination policy +- [ ] The connector sends workspace scope through the shared selector transport and does not send + the full connector configuration +- [ ] No client provider selector module, browser token request, provider-specific selector request, + or selector-only route remains ### validateConfig - [ ] Validates all required fields are present before making API calls @@ -305,6 +319,9 @@ Group findings by severity: - `contentHash` mismatch between `listDocuments` stub and `getDocument` return — causes unnecessary re-processing every sync - Server/runtime import in `meta.ts` (e.g. `@/lib/knowledge/...`, `input-validation.server`, `fetchWithRetry`) — pulls server-only code into the client bundle and breaks the build - Connector missing from `connectors/registry.ts` (the client-safe meta registry) — or its entry there imports the runtime module instead of `meta.ts` — the knowledge UI can't render it +- A connector selector resolves shared secrets in the browser, lacks scope or credential provider + binding, combines hidden authentication with an unsafe user-controlled destination, or forwards + protected/provider payload data to the client **Warning** (incorrect behavior, data quality issues, or convention violations): - HTML content not stripped via `htmlToPlainText` @@ -358,6 +375,8 @@ After fixing, confirm: - [ ] Validated data transformation: plain text extraction, HTML stripping, content hashing - [ ] Validated tag definitions match mapTags output, correct fieldTypes - [ ] Validated config fields: canonical pairs, selector keys, required flags +- [ ] Validated each dynamic selector through the shared manifest, server attachment, and + `selectors.execute` boundary - [ ] Validated validateConfig: lightweight check, error messages, retry options - [ ] Validated getDocument: null on 404, all content types handled, no redundant re-fetches, syncContext forwarding - [ ] Validated fetchWithRetry used for all external calls (no raw fetch), VALIDATE_RETRY_OPTIONS threaded through helpers diff --git a/.agents/skills/validate-integration/SKILL.md b/.agents/skills/validate-integration/SKILL.md index a009e51d1ac..541f9e37846 100644 --- a/.agents/skills/validate-integration/SKILL.md +++ b/.agents/skills/validate-integration/SKILL.md @@ -38,6 +38,10 @@ packages/deployment-config/src/service-account-providers.generated.ts # Generate packages/deployment-config/src/service-account-metadata.ts # Handwritten deployment policy ``` +If the block, its triggers, or connector fields use a `selectorKey`, also apply the `validate-selector` skill and read +the key's entry in `apps/sim/lib/selectors/manifest.ts`, its server attachment and provider listing +primitive, and the shared context builder. There is no client provider selector registry. + ## Step 2: Pull API Documentation Fetch the official API docs for the service. This is the **source of truth** for: @@ -279,6 +283,22 @@ For **each tool** in `tools.access`: - [ ] Input types match the subBlock types - [ ] When using `canonicalParamId`, inputs list the canonical ID (not the raw subBlock IDs) +### Dynamic Selectors + +- [ ] Every remote `selectorKey` is classified in the browser-safe manifest and has exactly one + server attachment +- [ ] The manifest allowlists the minimal active `dependsOn` context and matches list/search/detail, + pagination, scope, and stale-time behavior +- [ ] Canonical basic/advanced and trigger/action modes project only their active values; exact + `{{KEY}}` references remain unresolved in the browser +- [ ] Stored credentials are bound to the actor, workspace, and trusted provider/service +- [ ] Each attachment declares and enforces a `fixed`, `credential-bound`, or explicitly reviewed + `user-controlled` destination policy +- [ ] Provider results are explicitly projected to safe option fields; secrets, tokens, credential + IDs, context values, and raw upstream errors do not enter responses, logs, or query keys +- [ ] No selector provider module, provider fetch, or OAuth-token request runs in the browser, and no + selector-only provider route remains + ## Step 5: Validate OAuth Scopes (if OAuth service) Scopes are centralized — the single source of truth is `OAUTH_PROVIDERS` in `lib/oauth/oauth.ts`. @@ -359,6 +379,9 @@ Group findings by severity: legacy headerless/`NULL` data - A tool substitutes secret plaintext into source, leaks private metadata, or generically sanitizes unrelated third-party results +- A selector resolves shared secret plaintext in the browser, lacks credential provider binding or + destination enforcement, or returns provider payloads or protected values across the selector + boundary **Warning** (follows conventions incorrectly or has usability issues): - Optional field not set to `mode: 'advanced'` @@ -453,6 +476,8 @@ After fixing, confirm: - [ ] Confirmed legacy persisted data keeps working and tracked invalid provenance fails closed - [ ] Confirmed ordinary third-party results remain unchanged absent activated Sim provenance - [ ] Validated `{Service}BlockMeta` exported with at least 7 templates +- [ ] Validated every dynamic selector through the shared manifest, server attachment, and + `selectors.execute` boundary - [ ] Reported all issues grouped by severity - [ ] Fixed all critical and warning issues - [ ] Ran `bun run tool-metadata:generate` if any tool outputs/params changed, and confirmed `bun run tool-metadata:check` passes diff --git a/.agents/skills/validate-permission-group-item/SKILL.md b/.agents/skills/validate-permission-group-item/SKILL.md new file mode 100644 index 00000000000..febb6c278ec --- /dev/null +++ b/.agents/skills/validate-permission-group-item/SKILL.md @@ -0,0 +1,157 @@ +--- +name: validate-permission-group-item +description: Audit an existing enterprise permission-group item end-to-end — registry entry, schemas, type, defaults, tolerant parser, admin UI, capability rule, enforcement site, and tests — proving the gate actually refuses rather than assuming it. Use when checking a key in `PERMISSION_GROUP_FIELDS` or a capability in `CAPABILITY_RULES`. +argument-hint: +--- + +# Validate Permission Group Item Skill + +The question is not "does this key exist in the right places" — the registry makes most of that compiler-enforced. It is: + +> **If an organization admin sets this, what refuses, and can I make that refusal happen?** + +Twelve keys once shipped with a checkbox, a hint, and no server check. Every one would have passed a structural audit. Assume nothing enforces until you have found the throw. + +**`add-permission-group-item` owns the procedure and the rationale for every invariant named below.** Read it for *why*; this skill is the checklist. Its "Read the system first" list is the same one — start there. + +## Step 1: Registry entry (`lib/permission-groups/fields.ts`) + +Record the builder, the `enforcement`, and the position. + +- **Default permissive?** The builders hardcode `false` / `null` / `[]`, so the risk is a *name* that inverts the meaning — an `allowX` boolean. The checkbox renders `checked={!editingConfig[feature.configKey]}` (ticked = allowed), so a positively-named boolean renders backwards. +- **Position stable?** Declaration order is the wire order and `fields.test.ts` pins it with a key-order contract test. If `git log -p` shows the key was ever *moved* rather than appended, that shipped as an editor dirty-check regression. +- **Phrasing accurate?** An allowlist's `{ limited, empty }` and a denylist's string are read by `getActivePermissionGroupRestrictions` in `features.ts` and surface to users through the Copilot workspace VFS and the enterprise platform context. Confirm `empty` says "none allowed", not "unrestricted". +- **Does the `hint` tell the truth?** Highest-value read in this step. A `'capability'` key refuses at the API, so a hint saying it hides a tab, module, or nav item "from the sidebar" is a **lie an admin acts on** — they believe they are tidying chrome while withholding a module. The same string is reused as the prose for an *active* restriction, where "hide" is simply false. Any surviving "Hide the …" hint on a `'capability'` key is a finding, not a nit; check `label` and `category` the same way (a "Sidebar" or "Settings Tabs" section makes the claim structurally). + +## Step 2: Schemas, type, defaults, parser + +All derived by `collectFieldProperty` from the same registry. **Do not hand-verify them.** Verify nothing bypasses the derivation: + +```bash +grep -rn "" apps/sim --include='*.ts' --include='*.tsx' \ + | grep -vE 'lib/permission-groups/(fields|resolve\.server|config-scope\.server)\.ts' +``` + +Only the registry and the resolvers are excluded, so `capabilities.ts` stays in the output — its `CAPABILITY_RULES` entry and `deniedBy` are the authoritative reads this step exists to check. Every hit should be a rule's `deniedBy`, an enforcement site, a UI binding, or a test. A route restating the key, a client re-deriving a default, or a second coercion path is a leak. Specifically: + +- **`z.array(...).catch(...)` anywhere on this key's path** — whole-value tolerant, so one bad member discards every good one, and on an allowlist the `null` fallback means unrestricted. That is fail-**open**. `tolerantArray` filters element-wise. Rank a regression here with the enforcement findings. +- **`?? []` applied to an allowlist** — collapses "allows everything" into "allows nothing". +- **A hand-rolled comparison against an integration allowlist.** `allowedIntegrations` and the deployment's `ALLOWED_INTEGRATIONS` are written independently, so one names `slack` where the other names `slack_v2`; anything folding only case intersects them to nothing and hides an integration both allow. Both halves must canonicalize through `integration-allowlist.ts` (`intersectAccessControlAllowlists` / `toAccessControlAllowlist` / `resolveAccessControlBlockType`) *before* intersecting, with the checked type resolved the same way. That module reads `block-successors.generated.ts` through `Object.hasOwn` — a bare bracket lookup answers an admin-supplied `constructor` with an inherited function and 500s every path reading that group; `check:block-successors` catches the map going stale. +- **Any config read not from `parsePermissionGroupConfig` or a `resolvePermissionGroupConfig` caller.** + +Two structural guards must still be present: + +- **`parsePermissionGroupConfig` still tests `Array.isArray(config)`.** `typeof [] === 'object'`, the column is `jsonb` so a row genuinely can hold `[]`, and `z.object().parse([])` throws — the guard is what returns defaults instead of a 500. `tolerantArray` carries the mirror image. +- **`CAPABILITY_RULES` still uses `satisfies`, not an annotation.** An annotation collapses `StaticPermissionGroupCapability` to `never`, silently disabling the type system around capabilities with nothing wrong at runtime. `AssertsStaticCapabilityResolves` catches it; any weakening is a top-tier finding. + +Confirm the assertions at the bottom of `fields.ts` still name a field of this kind (`AssertsAllowlistStaysPrecise`, `AssertsDenylistStaysPrecise`, `AssertsRestrictionStaysPrecise`, `AssertsAuthTypesStayPrecise`, `AssertsParserReturnsTheConfig`) — a zod generic degrading to `unknown` is invisible at runtime and quietly loses every call site's narrowing. + +## Step 3: Admin UI (`ee/access-control/components/group-detail.tsx`) + +- **Boolean:** appears automatically via `PLATFORM_FEATURES`. Confirm its `category` is in `PLATFORM_CATEGORY_ORDER`; an unlisted one renders after every ordered section. +- **Nested allowlist / denylist** (one that qualifies a platform-feature boolean): renders **nothing** unless it is in the `featureExtras` map — keyed by the *parent boolean's feature id*, not the config key. No picker there means no admin can ever set it. Report it. Top-level lists — `allowedIntegrations`, `allowedModelProviders`, `deniedModels`, `deniedTools` — are not in `featureExtras` and must not be reported for it; they render from the dedicated Providers and Blocks sections, so check them there. +- For an **allowlist** picker, check both behaviors: refuses an empty selection (`if (values.length === 0) return`) and collapses a full one back to `null` (otherwise the allowlist freezes at today's members). A **denylist** picker must do neither: clearing every entry is how an admin denies nothing, and a full selection is a real state that denies everything. +- Check the parent is the right one (`allowedKnowledgeConnectors` under `hide-knowledge-base`, not `disable-knowledge-base-creation`). + +## Step 4: Capability rule + +A `'capability'` key must appear in some rule's `configKeys` — the audit asserts this (D) and the converse (E): a key declared `'executor'` or `'ui-only'` that a rule reads is flagged, so a key cannot gain enforcement while staying documented as weaker. Then check what the audit cannot: + +- **`configKeys` lists every key `deniedBy` reads.** The audit parses it textually and never reads the closure; a key read but unlisted is invisible to D and E. +- **`kind` is right.** A rule needing a request value must be `'parameterized'` — and a parameterized rule named on an operation cannot have run in production (`defineWorkspaceOperation` throws at definition time), so something else is wrong. +- **A narrower capability subsumes the broader one it replaced.** An operation carries exactly one capability. Precedent: `knowledge.create` / `knowledge.upload` both read `hideKnowledgeBaseTab`, without which a group withholding the whole module could still create a KB through the API. Check `git log` for a re-pointed `capability:` and verify the narrower rule grew the broader key in the same commit. +- **`detailCode` matches the remedy** — `FORBIDDEN_DETAIL_CODES` is closed over remedies, not causes; otherwise `PERMISSION_GROUP_CAPABILITY_BLOCKED`. Any code in use needs an entry in `FORBIDDEN_DETAIL_CODE_DESCRIPTIONS`, a compile-time gate that also publishes the OpenAPI 403 text. +- **`describe` reads correctly** as the subject of `" is not available under your organization's permission group"` — a singular noun or gerund agreeing with "is". Exactly two functions build that sentence, both defined in `capabilities.ts` (`refuseCapability` throws it, `capabilityRefusal` returns it); any call site writing it out is a drift finding. + +## Step 5: Prove the enforcement — do not assume it + +The step the skill exists for. Find the **actual refusal**, name file and line, and say what a caller sees. + +```bash +grep -rn "''" apps/sim --include='*.ts' --include='*.tsx' +grep -rn "permission-group-enforced: " apps/sim +``` + +The second grep misses a gate whose annotation sits in a TSDoc block above the enclosing statement — read the surrounding function. + +Classify into exactly one of: + +1. **Declared on operations.** The funnel enforces in `requireCurrentHumanAccess` → `requireCapability`. Verify the set is *complete*: enumerate every route and tool reaching the same behavior. One declaring `capability: 'none'` is the hole. +2. **Asserted at a call site** with a `// permission-group-enforced: — ` annotation. Verify it goes through `capability-assertions.ts` (`assertWorkspaceCapability`, `isWorkspaceCapabilityWithheld`, `isOrganizationCapabilityWithheld`, `capabilityDeniedBy`), through `isCapabilityWithheldForUser` (`lib/permission-groups/user-scope.server.ts` — workspace group first, else the organization's default, for a user-level act that may or may not name a workspace; outside `capability-assertions.ts` on purpose because it reads org membership through the billing graph, a guarded root of `check:application-graph`; `app/api/cli/auth/approve/route.ts` is the shape), or a direct `CAPABILITY_RULES[''].deniedBy(...)` rather than reading `config.disableX` inline, **and** that it *raises* through `refuseCapability` / renders `capabilityRefusal(cap)` rather than building its own `ForbiddenOperationError` with a hand-written message — the easy half to miss, because the decision looks right. Use-case shape: `validatePublicFileSharing`, `validateChatDeployAuth` (`ee/access-control/utils/permission-check.ts`), `assertConnectorTypeAllowed` (`lib/knowledge/application/connectors.ts`). Raw-route shape: `app/api/logs/stats/route.ts`, `app/api/table/[tableId]/export/route.ts`. A raw route should render through `capabilityRefusalResponse` (`lib/permission-groups/capability-response.ts`), which reads `details.code` off the rule — a hand-rolled `NextResponse.json({ error: capabilityRefusal(cap) }, { status: 403 })` drops it, reporting the four specifically-coded capabilities (`deploy.chat.auth_mode`, `file_share.publish`, `file_share.auth_mode`, `personal_api_key.use`) as the generic block. Convergence is partial — the inbox, api-keys, oauth-credentials, cli-approve and `logs/export` routes still hand-roll it, harmlessly today because all of their capabilities carry the generic code, so report one only if its capability gains a specific code. v1 is deliberately not converged on it (`resolveCapabilityRefusal` in `app/api/v1/middleware.ts`). +3. **Executor-gated** by `assertPermissionsAllowed`, per block / tool / model, matching through the shared primitives in `lib/permission-groups/` — `block-access.ts`, `operation-access.ts`, `model-access.ts`, `integration-allowlist.ts` — which the editor and Copilot projections read too, so a second copy of a match rule is a finding. Verify the branch throws a real error and that the id it compares against is the vocabulary the admin UI writes — `deniedTools` holds block `tools.access` ids verbatim, version suffix included. `allowedIntegrations` is *also* enforced off the run, by `assertSelectorIntegrationAllowed` (`lib/selectors/server/integration-access.ts`), so an executor key's coverage is not complete until every non-run path that reaches the third party is checked too. +4. **A field projection, not a gate.** `logs.trace_spans` and `logs.cost` withhold fields, so the logs routes correctly declare `capability: 'none'`. Single owner: `lib/logs/log-projection.ts` (`resolveLogFieldProjection`, `projectExecutionData`, `projectCostTotal`), which carries both annotations. A **second** implementation of the same redaction is the finding — as is a query that lets a caller filter or sort on a withheld field, which turns the projection into an oracle. +5. **Nothing.** Report as a defect: "an organization that sets this believes it applied a restriction that does not exist". + +Ahead of all five: `personal_api_key.use` fits none of them. It withholds a *principal kind* across every operation — the funnel's `personal_api_key` branch (`lib/core/application/workspace-authorization.ts`) and `app/api/v1/middleware.ts` — so no operation declares it and `disablePersonalApiKeys` being absent from every `capability:` field is correct, not a hole. + +Then **make the refusal happen**: write a failing case, or remove the gate (the `capability:` field, the `deniedBy` body, the assertion call) and confirm an existing test goes red. A test that still passes with the gate removed proves nothing. Restore afterward. **Check the fixture's `workspaceOrganizationId` first**: `requireCapability` short-circuits when it is `null` (`lib/core/application/workspace-authorization.ts:204`), so a context that leaves it unset passes either way and the existing test proves nothing even before you touch it. + +For an allowlist the three states must be tested separately — `null` permits every member, a populated list only the named ones, `[]` permits **none**. `capabilities.test.ts` pins all three for `knowledge.connectors`; less than that elsewhere is a gap. + +### Who the gate runs against + +**Read the subject, not the nearest user id.** Every capability sink must take its subject from the `capabilityGoverned*` helper for the identity the surface holds — `capabilityGovernedPrincipalUserId` for a `Principal` (`lib/core/application`), `capabilityGovernedUserId` for a v1 `RateLimitResult` or a `TableAccessPrincipal`, `capabilityGovernedAuthUserId` for a `checkSessionOrInternalAuth` result. Each returns `null` where no group governs, and `null` is a pass. Reading `rateLimit.userId`, `auth.userId`, `subjectUserId` or `triggeredByUserId` into a sink is the finding: for a workspace key the first is the key's *creator*, for an internal JWT the second is the run's actor, and the last is a billing *attribution*. `check-capability-subject.ts` audits **v1 only**, so every other surface is on you. Where the subject is persisted and read back later (`capabilityGovernedUserId` on `table_run_dispatches` / `table_row_executions`), it must be declared required as `string | null` — an optional field with a fallback is exactly how producers re-inherited `triggeredByUserId`, so a proposal to make it optional is a finding. + +- **`/api/v1`** authorizes in `app/api/v1/middleware.ts`, not through `authorizeWorkspaceOperation`; `capabilityGovernedUserId(rateLimit)` branches on `keyType`, never on the presence of a user id. Each route also threads a required, spelled-out `V1RouteCapability`. +- **Raw internal table routes** gate `tables.use` in `checkAccess` (`app/api/table/utils.ts`) via a `TableAccessPrincipal` union — `{ kind: 'user'; userId }` or `{ kind: 'workspace_api_key'; keyCreatorUserId }` — so a bare id no longer type-checks. `tableAccessPrincipal(rateLimit)` is the one place v1 builds it. +- **The definition-time `undefined` guard** on `defineWorkspaceOperation` is not redundant even though `capability` is required on the `ApplicationOperation` **base type** (`lib/core/application/operation.ts:31`, not merely on the builder — which is what stops a bare-literal factory from compiling): `apps/sim/tsconfig.json` excludes `*.test.ts` / `*.test.tsx` and the enforcement audit walks past test files, so a fixture is the one construction site no static check reads. Without it a capability-less operation defines cleanly and then throws `Cannot read properties of undefined` inside `capabilityDeniedBy` **only for tenants that actually have a permission group**, passing CI and every personal workspace. A proposal to drop it is a finding. + +## Step 6: Tests + +- **`fields.test.ts`** — the key must be in both the `input` and `expected` halves of the `'a fully populated config'` fixture; that corpus is pinned so a changed row is defended rather than slipping through. The rest of the file derives from `DEFAULT_PERMISSION_GROUP_CONFIG` and needs no per-key edit. +- **`capabilities.test.ts`** — a case for any rule with logic beyond reading one key: subsumption, allowlist three-state, auth-mode membership. +- **`features.test.ts`** — no edit for a boolean; a non-boolean key's `limited` / `empty` prose should be pinned here. +- **`config-scope.server.test.ts`** — the per-request memo. A gate resolving the config outside `resolvePermissionGroupConfig` is a Step 2 finding, not one here. + +## Step 7: Run the checks + +```bash +bun run check:permission-group-enforcement +bun run check:application-graph +bun run check:capability-subject +cd apps/sim && bun run type-check && bunx vitest run lib/permission-groups +``` + +All three are inside `check:audits`, which derives its list from the `check:*` scripts in `package.json` — a new audit is opted *out* deliberately. Read the output, not the exit codes. Reference success lines (counts grow): + +``` +✓ permission-group enforcement: 322 operations declare a capability, 35 capabilities all enforced +✅ Application graph clean: 5 roots reach none of 11 forbidden module trees +check:capability-subject — 32 v1 files, 5 capability subjects resolved through capabilityGovernedUserId. +``` + +| Audit | What it catches | +|---|---| +| `check:permission-group-enforcement` | Every operation declares a capability and every capability is enforced. All-or-nothing — no migration mode exits 0 with work outstanding, so do not go looking for a `pending enforcement:` list | +| `check:application-graph` | The funnel roots (`lib/core/application/index.ts`, `capabilities.ts`, `capability-assertions.ts`, `config-scope.server.ts`) and `with-route-handler.ts` reach no heavy module tree at *runtime* (`import type` is erased and allowed). A gate that imports a resolver into a guarded root is a finding even if the gate is correct; past regressions surfaced only as unrelated tests failing on partial mocks | +| `check:capability-subject` | Every v1 capability sink takes its subject from `capabilityGovernedUserId`, no v1 file outside the middleware imports the permission-group modules, and at least one governed sink was found at all | + +Two ways the enforcement audit passes without proving what you want: + +- **Vacuous parse.** It reads source text with regexes, so it refuses success when the three registries parse to nothing, cross-checks rule count against capability count, reports per call any unreadable `id`, fails a file that mints an operation but parses to **zero** declarations, and flags any exported `*Operations` registry member it read no operation from. If one fires the audit is broken, not the code — fix the parsers rather than leaving it green. (That last guard exists because an operation minted by a factory that never calls the builder bypasses the required type *and* the audit; twenty-one operations across six domains were invisible that way while the file still printed a tick.) +- **A capability declared on an operation nothing routes to.** Assertion C is satisfied by the declaration alone. + +The audits prove *reachability*, never correctness — that a capability is named, a key is read by some rule, a subject came from the right helper. Step 5 is what covers the rest. + +## Known gaps — recognize these, do not re-report them + +Each is deliberate and documented in the code; `add-permission-group-item` carries the full reasoning. + +- **A workspace API key resolves no permission group** — it authorizes as the workspace, so there is no user and `operation.capability` does not apply; the same reasoning shapes `TableAccessPrincipal`, `capabilityGovernedUserId` and the log projection. Substituting the key's creator would apply a bystander's group to every caller of a shared key and break the key when that person left. Minting a workspace key is itself capability-gated. +- **An executor delegation carries role but not capabilities** — a delegated `executor` principal with a `sim_user` subject goes through `requireCurrentHumanRole` only. A capability names what a *person* may reach; applying it to a run makes "hide Tables" a kill-switch for every workflow with a Table block. +- **An actorless deployment run passes through** — a delegated executor principal in `mode: 'deployment'` with no resolvable subject acts with the workspace's authority; denying would 403 every scheduled run, webhook and public-API call. What such a run *does* is still governed by `assertPermissionsAllowed`, which is why the four run-scoped keys carry `enforcement: 'executor'`. +- **Copilot is NOT exempt** — a delegated principal with a `sim_user` subject whose `serviceId` is anything other than `executor` takes the full `requireCurrentHumanAccess`. Copilot acts as the person. A proposal to exempt it is a finding. +- **Capability is checked after the role check** — `NoWorkspaceAccessError` is concealed as a 404 by the v2 surface, so refusing on capability first would hand a non-member an oracle for what the organization withholds. The v1 middleware states the same ordering in its TSDoc. Not a bug. +- **`allowedEgressHosts` does not exist** — there is no network-egress allowlist. Requests for one are a feature, not missing wiring. +- **Nothing currently ships as `ui-only`** — the union member has no user; an absent `ui-only` key is not a gap. + +## Report Format + +1. **Kind and enforcement** — as declared, and whether the declaration is true. +2. **The refusal** — file, line, error thrown, what the caller sees (status, `detailCode`, message). Or: it is a projection, and here is its single owner. Or: nothing refuses. +3. **The subject** — whose user id the gate reads, and that a workspace key reaches it ungated rather than as its creator. +4. **Proof** — the test that fails when the gate is removed, or that no such test exists. +5. **Coverage gaps** — routes, tools, surfaces reaching the same behavior without the gate. +6. **Findings**, ordered: unenforced key > key-creator substituted for the acting principal > fail-open coercion (`.catch()` on an array, a dropped `Array.isArray` guard, `CAPABILITY_RULES` annotated instead of `satisfies`) > incomplete operation coverage > allowlist three-state confusion > **admin copy that misstates the enforcement** > duplicated projection logic > missing admin UI > missing test > cosmetic. + +A hint saying "hide" for a key that 403s is not cosmetic — it is the one defect an admin acts on directly: they tick it believing they hid a link, and members lose the module. Rank it with the enforcement findings. diff --git a/.agents/skills/validate-selector/SKILL.md b/.agents/skills/validate-selector/SKILL.md new file mode 100644 index 00000000000..bb251dada5c --- /dev/null +++ b/.agents/skills/validate-selector/SKILL.md @@ -0,0 +1,103 @@ +--- +name: validate-selector +description: Audit a Sim dynamic selector across its declaration, browser-safe manifest, server attachment, provider primitive, and selectors.execute security boundary. Use when reviewing selector correctness, secret handling, scope authorization, or migration completeness. +argument-hint: +--- + +# Validate Selector + +Validate the complete path, not only the provider adapter. + +## Gather the path + +Read: + +- Every block, trigger, and connector field using the selector key. +- `apps/sim/lib/selectors/manifest.ts` and `types.ts`. +- `apps/sim/lib/selectors/context.ts`. +- The matching server attachment and any shared provider listing primitive. +- `apps/sim/lib/selectors/server/registry.ts`. +- The shared application executor, route contract, client transport, and focused tests when the + finding concerns shared behavior. + +Search literal API paths as well as TypeScript imports before deciding whether an old provider route +or contract is unused. + +## Validate the declaration and manifest + +- The key has exactly one classification and one attachment (`local` keys use the local registry). +- Allowed context is minimal and readiness matches the provider request. +- List, pagination, search, detail, unknown-detail, scope, and stale-time metadata match actual UX. +- `dependsOn` names the required source fields; canonical pairs contribute only their active member. +- Action/trigger and connector surfaces build the same canonical context. +- Exact `{{KEY}}` references remain unresolved in the browser; runtime references are omitted, + while embedded interpolation is forwarded unresolved and rejected by the server executor. +- Query keys contain no context value, reference, credential ID, secret, or hash of one. + +## Validate the server boundary + +For provider and internal selectors, confirm the shared executor owns this order: + +1. Session authentication before request parsing. +2. Canonical workflow/workspace loading and read authorization. +3. Manifest capability and exact-context allowlisting. +4. Exact environment-reference resolution on the server. +5. Credential use, workspace, and trusted provider/service binding. +6. Destination-policy enforcement before provider network access. +7. Provider/internal execution followed by explicit option projection and sanitization. + +The attachment must not duplicate these shared checks. It must not accept a browser-provided module, +provider, service, operation kind, origin, or scope list. + +Review its destination classification: + +- `fixed` origins are code-defined. +- `credential-bound` origins are derived from or checked against the authorized credential. +- `user-controlled` destinations have an explicit policy for hidden use-only authentication and + network safety. + +Missing and inaccessible references, and missing, unauthorized, or provider-mismatched credentials, +must not become existence oracles. Hidden/server-only resolved plaintext, credentials, authentication +material, and raw upstream errors must not enter responses, query/cache/rate keys, selector result +caches, logs, audit metadata, redirects, or error messages. Browser-known literals and viewable +personal/shared values are not automatically protected plaintext, but the executor must still never +deliberately or wholesale echo selector context. Only intentionally projected, normalized +`{ id, label, meta? }` options and bounded cursors may cross the boundary. + +Selector code must not introduce a context, token, or result cache. The sole existing cache +exception is authorized client-credential resolution after authorization and provider binding, +which may reuse the credential service's TTL-governed, lazily pruned process-local token cache. The +cache is not hard-bounded, the exception requires explicit security-owner acceptance, and selector +work must not expand it. + +## Validate provider reuse and browser boundaries + +- The attachment calls a server-only provider primitive, not a route handler or internal HTTP URL. +- A surviving provider route has a proven non-selector caller and delegates to the same primitive. +- There is no client provider selector module, selector-specific token request, or browser request to + a provider-specific selector route. +- Internal selectors delegate to existing authorized use cases rather than querying protected data + in the route or adapter. + +## Tests and report + +Use existing Vitest/route/React Query patterns. Preserve valuable provider tests, but do not demand a +per-selector authorization matrix. Shared executor tests should cover ordering, references, +credential binding, sanitization, and safe errors; adapter tests should cover only special provider +behavior. + +Report critical, warning, and suggestion findings. Treat browser secret resolution, missing scope or +credential authorization, unsafe destination binding, provider payload passthrough, and plaintext +egress as critical. + +Run the smallest relevant focused suites plus: + +```bash +bun run --cwd apps/sim type-check +bun run check:api-validation:strict +bun run check:fork-dependent-coverage +bun run check:client-boundary +git diff --check +``` + +State which live-provider checks remain pending when disposable credentials are unavailable. diff --git a/.agents/skills/validate-selector/agents/openai.yaml b/.agents/skills/validate-selector/agents/openai.yaml new file mode 100644 index 00000000000..62c3fd9a5ff --- /dev/null +++ b/.agents/skills/validate-selector/agents/openai.yaml @@ -0,0 +1,5 @@ +interface: + display_name: "Validate Selector" + short_description: "Audit a dynamic selector boundary" + brand_color: "#B45309" + default_prompt: "Use $validate-selector to audit a Sim dynamic selector for correctness and secure server-side execution." diff --git a/.agents/skills/validate-trigger/SKILL.md b/.agents/skills/validate-trigger/SKILL.md index 7e4bccb84f8..715a6642d28 100644 --- a/.agents/skills/validate-trigger/SKILL.md +++ b/.agents/skills/validate-trigger/SKILL.md @@ -36,6 +36,10 @@ apps/sim/lib/webhooks/provider-subscription-utils.ts # Subscription helpers apps/sim/lib/webhooks/processor.ts # Central webhook processor ``` +If trigger sub-blocks use a `selectorKey`, also apply the `validate-selector` skill and read the +key's browser-safe manifest entry, shared active-value context builder, server attachment, and +provider listing primitive. + ## Step 2: Pull API Documentation Fetch the service's official webhook documentation. This is the **source of truth** for: @@ -76,6 +80,12 @@ If a payload schema is unknown, validation must explicitly recommend: - [ ] Every trigger's `id` matches the convention `{service}_{event_name}` - [ ] Every trigger's `provider` matches the service name used in the handler registry - [ ] `index.ts` barrel exports all triggers +- [ ] Every remote `selectorKey` is present in `apps/sim/lib/selectors/manifest.ts` and has exactly + one server attachment +- [ ] Trigger-mode `dependsOn` fields project only active canonical values; exact `{{KEY}}` + references stay unresolved in the browser +- [ ] Trigger selectors use the shared `selectors.execute` transport, with no client provider + module, browser token request, or selector-only provider route ### Trigger ↔ Provider Alignment (CRITICAL) - [ ] Every trigger ID referenced in `matchEvent` logic exists in `{service}TriggerOptions` @@ -185,6 +195,8 @@ Group findings by severity: - Trigger IDs mismatch between trigger files, registry, and block - `createSubscription` calling wrong API endpoint - Auth comparison using `===` instead of `safeCompare` +- Trigger selector credential/reference resolution occurring in the browser, or a selector missing + scope authorization, credential provider binding, destination enforcement, or safe projection **Warning** (convention violations or usability issues): - Missing `extractIdempotencyId` when the service provides delivery IDs @@ -218,6 +230,7 @@ After fixing, confirm: - [ ] Read all trigger files, provider handler, types, registries, and block - [ ] Pulled and read official webhook/API documentation - [ ] Validated trigger definitions: options, instructions, extra fields, outputs +- [ ] Validated dynamic selector declarations through the shared manifest and server attachment - [ ] Validated primary/secondary trigger distinction (`includeDropdown`) - [ ] Validated provider handler: auth, matchEvent, formatInput, idempotency - [ ] Validated output alignment: every `outputs` key ↔ every `formatInput` key diff --git a/.claude/rules/sim-architecture.md b/.claude/rules/sim-architecture.md index a8b25498eae..d950851a3f1 100644 --- a/.claude/rules/sim-architecture.md +++ b/.claude/rules/sim-architecture.md @@ -62,21 +62,25 @@ Every export of a `'use client'` module becomes a *client reference* on the serv Server code runs in two runtimes with **different environments**. The app container loads the full env from `SIM_ENV_SECRET_ID` (Secrets Manager). Trigger.dev workers — which execute workflows, so every block handler and every tool call — get their env from the Trigger.dev -dashboard, and `trigger.config.ts` syncs only `DB_APP_NAME`. The repo cannot see what the -dashboard holds. +dashboard; `trigger.config.ts` additionally syncs `DB_APP_NAME`, `TRIGGER_DEV_ENABLED`, and the +`FUNCTION_EXECUTION_ENV` vars. The repo cannot see what the dashboard holds. So before replacing a worker's HTTP call to our own API with an in-process call, ask what env that work reads *on the app side*. Anything gated by a `require*Capability` helper is the sharp case: those **throw** when the variable is absent (`requireOAuthClientCapability` → `EnvCapabilityConfigurationError`), and the throw may be caught and reported as something -unrelated. OAuth token refresh is the known example — moving it into the worker turns every -expired credential into `Failed to refresh access token`, while a still-valid token hides the -bug entirely, so it surfaces hours later and only for whoever's token lapsed first. +unrelated — an in-worker OAuth refresh missing a provider's client pair reports every expired +credential as `Failed to refresh access token`, while a still-valid token hides the bug until it +lapses. The required step before such a conversion is verifying the dashboard env holds every +variable the moved code reads (for OAuth refresh: the `OAUTH_CLIENT_CAPABILITIES` key pairs in +`packages/deployment-config/src/env-capabilities.ts`). An in-process conversion is safe when the same work already runs in that runtime (the agent block has always called `executeProviderRequest` in-process, so router and evaluator joining it -is proven), or when the caller and the callee are both the app (a route calling a lib module, an -RSC prefetch reading the data layer). It is not safe on reasoning alone. +is proven; connector sync refreshing OAuth tokens in-worker is what proved credential-token +resolution could move in-process), or when the caller and the callee are both the app (a route +calling a lib module, an RSC prefetch reading the data layer). It is not safe on reasoning +alone — verify the env, then convert. ## Feature Organization diff --git a/.claude/rules/sim-react-performance.md b/.claude/rules/sim-react-performance.md index 2d77b324f24..a78c2e984e4 100644 --- a/.claude/rules/sim-react-performance.md +++ b/.claude/rules/sim-react-performance.md @@ -90,6 +90,15 @@ const [{ id }, { kbName }] = await Promise.all([params, searchParams]) Only keep awaits sequential when a later call genuinely uses an earlier result, or when the ordering is deliberate (rate-limited batches, retry loops, write-then-read). +## Carry exact lifecycle ownership across async boundaries + +When asynchronous work can outlive an execution, session, or resource instance, capture its +opaque ownership token before the first `await` and pass that exact token through completion and +error cleanup. Never re-adopt the current owner from delayed cleanup: a replacement may now own +the same scope. End the lifecycle by exact-token match, and clear shared state only when that end +succeeds. Current-owner adoption is reserved for synchronous user actions that explicitly stop +the current lifecycle. + ## Prefetch dynamic destination lists on intent For long lists of dynamic destinations, do not viewport-prefetch every row and do not assume @@ -99,6 +108,12 @@ server state with the consumer's shared React Query options. A short, cancelable avoids drive-by downloads. Do not treat `touchstart` as intent because it also begins scrolling; let the actual unmodified click start the data request. +A speculative failure must not poison a later visit when the app default disables +`retryOnMount`: remove only that exact failed query while it is inactive, keep failures visible +to mounted consumers, and set the shared options to `retryOnMount: true` so a quick-click failure +can recover after the user leaves and returns. Never carry placeholder data between protected +resource keys (for example, workspace A to workspace B); an explicit loading state is truthful. + If a continuity-focused surface intentionally omits `loading.tsx` so the current view remains mounted until its peer is ready, the intent path must warm both the full route and its critical data. Otherwise keep the loading boundary so dynamic navigation remains responsive. diff --git a/.claude/rules/sim-settings-pages.md b/.claude/rules/sim-settings-pages.md index b65deabafe3..763803ed7db 100644 --- a/.claude/rules/sim-settings-pages.md +++ b/.claude/rules/sim-settings-pages.md @@ -104,6 +104,11 @@ Adding a new settings page: 2. Render the component inside the shell's `effectiveSection` switch in `settings/[section]/settings.tsx`. 3. Build the component body inside `` — no shell, no title block. +4. When a real second consumer or server boundary needs it, extract client-safe React Query options; + otherwise keep them with the hook. Approved intent warmers reuse those exact options and must keep + `check-tool-registry-boundary` green. Warm only authorized destinations, preserve the current + section during the transition, and follow `sim-react-performance.md` recovery rules; never render + temporary default data that will be replaced after load. ## Text-scale tokens (no literal pixel sizes) diff --git a/.claude/rules/sim-url-state.md b/.claude/rules/sim-url-state.md index 8a859e8a547..6790bc1b56d 100644 --- a/.claude/rules/sim-url-state.md +++ b/.claude/rules/sim-url-state.md @@ -46,6 +46,20 @@ These reads/mutations are **not** anti-patterns and stay as-is: - **Route navigations** — `router.push('/path/[id]?folderId=x')` that changes the route *path*, not just the current query. A nuqs setter only mutates the query on the current path; cross-path navigation stays on `router`. - **Read-once auth / redirect signals** — `token`, `callbackUrl`, `redirect`, `error`, `invite_flow`, `new` (invite signup flow), `upgraded`, `redirect_workflow`, etc. These are navigation signals consumed once (often read-then-strip), not synced view-state. Leave them on `useSearchParams`. Key names are per-surface: files' `new` is a genuine nuqs param (`files/search-params.ts`), while invite's `new` is a one-shot signup signal. +### Remembered list-preference exception + +Files, Tables, and Knowledge may persist their last-used filter/sort snapshot through +`useResourceListPreferences`. This is a fallback preference, not a second live source of truth: + +- nuqs remains authoritative while the module is open. +- Zustand is consulted once on a clean module entry, after persisted state hydrates. +- An explicit URL filter/sort parameter wins even when it resolves to the module default. The + complete resolved URL snapshot becomes the remembered value; omitted fields use URL defaults + rather than merging with storage. +- Explicit filter/sort gestures commit the same complete snapshot to nuqs and Zustand together. +- Never mirror subsequent URL changes with a synchronization effect or `popstate` listener. +- Search and folder navigation remain URL-only and are excluded from the persisted snapshot. + ## Per-feature `search-params.ts` — single source of truth Co-locate a `search-params.ts` next to the feature. Export the parser map (and shared options). Both the client (`useQueryStates`/`useQueryState`) and any server component (`createSearchParamsCache` from `nuqs/server`) import from this one file. Import parsers from `nuqs/server` so the module is safe to import in both client and server contexts. diff --git a/.claude/skills/add-permission-group-item b/.claude/skills/add-permission-group-item new file mode 120000 index 00000000000..37547b94cdb --- /dev/null +++ b/.claude/skills/add-permission-group-item @@ -0,0 +1 @@ +../../.agents/skills/add-permission-group-item \ No newline at end of file diff --git a/.claude/skills/add-selector b/.claude/skills/add-selector new file mode 120000 index 00000000000..00e430ae4ce --- /dev/null +++ b/.claude/skills/add-selector @@ -0,0 +1 @@ +../../.agents/skills/add-selector \ No newline at end of file diff --git a/.claude/skills/validate-permission-group-item b/.claude/skills/validate-permission-group-item new file mode 120000 index 00000000000..d867334b614 --- /dev/null +++ b/.claude/skills/validate-permission-group-item @@ -0,0 +1 @@ +../../.agents/skills/validate-permission-group-item \ No newline at end of file diff --git a/.claude/skills/validate-selector b/.claude/skills/validate-selector new file mode 120000 index 00000000000..b3612d7d2a3 --- /dev/null +++ b/.claude/skills/validate-selector @@ -0,0 +1 @@ +../../.agents/skills/validate-selector \ No newline at end of file diff --git a/.devcontainer/docker-compose.yml b/.devcontainer/docker-compose.yml index c7671658080..2689fd7b689 100644 --- a/.devcontainer/docker-compose.yml +++ b/.devcontainer/docker-compose.yml @@ -25,6 +25,11 @@ services: - OLLAMA_URL=${OLLAMA_URL:-http://localhost:11434} - NEXT_PUBLIC_SOCKET_URL=${NEXT_PUBLIC_SOCKET_URL:-} - BUN_INSTALL_CACHE_DIR=/home/bun/.bun/cache + # Lets a workflow reach a service on the Docker host. Reaching it also + # requires naming it in EGRESS_ALLOWED_HOSTS; this only makes the name + # resolve, which it does not on Linux by default. + extra_hosts: + - 'host.docker.internal:host-gateway' depends_on: db: condition: service_healthy diff --git a/.devcontainer/post-create.sh b/.devcontainer/post-create.sh index c7ceff427e6..841d5c2e362 100755 --- a/.devcontainer/post-create.sh +++ b/.devcontainer/post-create.sh @@ -94,8 +94,8 @@ fi # Generate schema and run database migrations echo "🗃️ Running database schema generation and migrations..." echo "Generating schema..." -cd apps/sim -bunx drizzle-kit generate +cd packages/db +bun run db:generate cd ../.. echo "Waiting for database to be ready..." @@ -105,8 +105,8 @@ echo "Waiting for database to be ready..." while [ $timeout -gt 0 ]; do if PGPASSWORD=postgres psql -h db -U postgres -c '\q' 2>/dev/null; then echo "Database is ready!" - cd apps/sim - DATABASE_URL=postgresql://postgres:postgres@db:5432/simstudio bunx drizzle-kit push + cd packages/db + DATABASE_URL=postgresql://postgres:postgres@db:5432/simstudio bun run db:push cd ../.. break fi diff --git a/.devcontainer/sim-commands.sh b/.devcontainer/sim-commands.sh index 640feb2f95a..c8e3ab1a52f 100755 --- a/.devcontainer/sim-commands.sh +++ b/.devcontainer/sim-commands.sh @@ -7,8 +7,8 @@ alias sim-start="cd /workspace && bun run dev:full" alias sim-app="cd /workspace && bun run dev" alias sim-sockets="cd /workspace && bun run dev:sockets" -alias sim-migrate="cd /workspace/apps/sim && bunx drizzle-kit push" -alias sim-generate="cd /workspace/apps/sim && bunx drizzle-kit generate" +alias sim-migrate="cd /workspace/packages/db && bun run db:push" +alias sim-generate="cd /workspace/packages/db && bun run db:generate" alias sim-rebuild="cd /workspace && bun run build && bun run start" alias docs-dev="cd /workspace/apps/docs && bun run dev" diff --git a/.dockerignore b/.dockerignore index 98af3e0c6e5..3a5d3e50438 100644 --- a/.dockerignore +++ b/.dockerignore @@ -32,7 +32,9 @@ Dockerfile* # Build artifacts and caches .next +**/.next .turbo +**/.turbo .cache dist build diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c81d49b83f6..bfac30a378a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -266,7 +266,7 @@ jobs: echo "ERROR: DEV_TRIGGER_ACCESS_TOKEN and TRIGGER_PROJECT_ID repo secrets must both be set" >&2 exit 1 fi - bunx trigger.dev@4.5.7 deploy --env preview --branch dev-sim + bunx trigger.dev@4.5.12 deploy --env preview --branch dev-sim # Main/staging: build AMD64 images and push sha-tagged images to ECR + GHCR. # Runs in parallel with tests — only immutable sha tags are pushed here, and diff --git a/.github/workflows/desktop-release.yml b/.github/workflows/desktop-release.yml index db1e403eb5d..b3c51213545 100644 --- a/.github/workflows/desktop-release.yml +++ b/.github/workflows/desktop-release.yml @@ -55,6 +55,7 @@ jobs: uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6 with: fetch-depth: 0 + persist-credentials: false ref: ${{ github.event_name == 'workflow_dispatch' && inputs.version || github.sha }} # Prerelease versions carry their environment in the tag: -dev.N is a @@ -340,10 +341,18 @@ jobs: exit 1 fi export GH_TOKEN - if ! gh release view "$VERSION" --repo "$RELEASE_REPOSITORY" >/dev/null; then + if ! RELEASE_ID="$( + gh release view "$VERSION" --repo "$RELEASE_REPOSITORY" \ + --json databaseId --jq '.databaseId' + )"; then echo "::error::Release $VERSION does not exist in $RELEASE_REPOSITORY." exit 1 fi + if ! [[ "$RELEASE_ID" =~ ^[0-9]+$ ]]; then + echo "::error::Release $VERSION returned an invalid database ID." + exit 1 + fi + RELEASE_JSON="$(gh api "repos/${RELEASE_REPOSITORY}/releases/${RELEASE_ID}")" SEMVER="${VERSION#v}" ARTIFACTS=( "apps/desktop/release/Sim-${SEMVER}-universal.dmg" @@ -354,11 +363,10 @@ jobs: ) upload_or_verify() { local ARTIFACT="$1" - local NAME SIZE DIGEST RELEASE_JSON REMOTE REMOTE_SIZE REMOTE_DIGEST + local NAME SIZE DIGEST REMOTE REMOTE_SIZE REMOTE_DIGEST NAME="$(basename "$ARTIFACT")" SIZE="$(stat -f%z "$ARTIFACT")" DIGEST="sha256:$(shasum -a 256 "$ARTIFACT" | awk '{print $1}')" - RELEASE_JSON="$(gh api "repos/${RELEASE_REPOSITORY}/releases/tags/${VERSION}")" REMOTE="$(jq -c --arg name "$NAME" '.assets[] | select(.name == $name)' <<< "$RELEASE_JSON")" if [ -n "$REMOTE" ]; then REMOTE_SIZE="$(jq -r '.size' <<< "$REMOTE")" diff --git a/.github/workflows/test-build.yml b/.github/workflows/test-build.yml index 690905622d2..b4d87d8425e 100644 --- a/.github/workflows/test-build.yml +++ b/.github/workflows/test-build.yml @@ -180,13 +180,19 @@ jobs: # Runs the setup CLI's Bun tests plus each workspace's Vitest suite, # without `--coverage`. See the Codecov note below. + # + # apps/sim runs only its first shard here; `test-shard` below runs the + # others. That suite is bound by the single Vite server thread that feeds + # every worker — wall time is flat from 4 to 13 workers — so a bigger + # runner buys nothing and each extra runner takes a proportional slice. - name: Run tests env: NODE_OPTIONS: '--no-warnings --max-old-space-size=8192' NEXT_PUBLIC_APP_URL: 'https://www.sim.ai' DATABASE_URL: 'postgresql://postgres:postgres@localhost:5432/simstudio' - ENCRYPTION_KEY: '7cf672e460e430c1fba707575c2b0e2ad5a99dddf9b7b7e3b5646e630861db1c' # dummy key for CI only + ENCRYPTION_KEY: '0000000000000000000000000000000000000000000000000000000000000000' # dummy key for CI only TURBO_CACHE_DIR: .turbo + SIM_TEST_SHARD: 1/3 run: bun run test - name: Check schema and migrations are in sync @@ -202,6 +208,72 @@ jobs: fi echo "✅ Schema and migrations are in sync" + # The remaining shards of apps/sim's Vitest suite. Everything else — lint, + # the audits, type-check, the other workspaces' suites — lives in + # `test-build` with shard 1; these jobs exist only because that suite cannot + # go faster on one machine (see the "Run tests" note there). Three shards + # put each runner at roughly the fixed cost of checkout + install. The Turbo + # cache disk gets its own key so the shards' entries do not evict each other. + test-shard: + name: Test (shard ${{ matrix.shard }}) + runs-on: ${{ (vars.CI_PROVIDER == '' || vars.CI_PROVIDER == 'blacksmith') && 'blacksmith-8vcpu-ubuntu-2404' || 'ubuntu-latest' }} + timeout-minutes: 15 + strategy: + fail-fast: false + matrix: + shard: [2, 3] + + steps: + - name: Checkout code + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6 + + - name: Setup Bun + uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2 + with: + bun-version: 1.3.14 + + - name: Setup Node + uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6 + with: + node-version: 24 + + - name: Mount Bun cache + uses: ./.github/actions/cache-mount + with: + provider: ${{ vars.CI_PROVIDER }} + key: ${{ github.repository }}-bun-cache-${{ github.event_name }}${{ github.event.pull_request.head.repo.fork && '-fork' || '' }} + path: ~/.bun/install/cache + + - name: Mount node_modules + uses: ./.github/actions/cache-mount + with: + provider: ${{ vars.CI_PROVIDER }} + key: ${{ github.repository }}-node-modules-${{ github.event_name }}${{ github.event.pull_request.head.repo.fork && '-fork' || '' }}-${{ hashFiles('bun.lock') }} + path: ./node_modules + + - name: Mount Turbo cache + uses: ./.github/actions/cache-mount + with: + provider: ${{ vars.CI_PROVIDER }} + key: ${{ github.repository }}-turbo-cache-shard-${{ matrix.shard }}-${{ github.event_name }}${{ github.event.pull_request.head.repo.fork && '-fork' || '' }} + path: ./.turbo + + - name: Install dependencies + run: bun install --frozen-lockfile --ignore-scripts + + - name: Install ripgrep + run: command -v rg || (sudo apt-get update && sudo apt-get install -y ripgrep) + + - name: Run tests (apps/sim shard ${{ matrix.shard }}/3) + env: + NODE_OPTIONS: '--no-warnings --max-old-space-size=8192' + NEXT_PUBLIC_APP_URL: 'https://www.sim.ai' + DATABASE_URL: 'postgresql://postgres:postgres@localhost:5432/simstudio' + ENCRYPTION_KEY: '0000000000000000000000000000000000000000000000000000000000000000' # dummy key for CI only + TURBO_CACHE_DIR: .turbo + SIM_TEST_SHARD: ${{ matrix.shard }}/3 + run: bunx turbo run test --filter=@sim/app + # Next.js production build, in parallel with lint + tests. Sticky disks are # cloned from the last committed snapshot per job and committed last-writer- # wins, so concurrent mounts are safe. The bun/node_modules disks are shared @@ -293,6 +365,6 @@ jobs: STRIPE_WEBHOOK_SECRET: 'dummy_secret_for_ci_only' RESEND_API_KEY: 'dummy_key_for_ci_only' AWS_REGION: 'us-west-2' - ENCRYPTION_KEY: '7cf672e460e430c1fba707575c2b0e2ad5a99dddf9b7b7e3b5646e630861db1c' # dummy key for CI only + ENCRYPTION_KEY: '0000000000000000000000000000000000000000000000000000000000000000' # dummy key for CI only TURBO_CACHE_DIR: .turbo run: bunx turbo run build --filter=@sim/app diff --git a/.gitignore b/.gitignore index 4371d6d73f7..6bbecf8f295 100644 --- a/.gitignore +++ b/.gitignore @@ -84,9 +84,6 @@ start-collector.sh # IntelliJ .idea -## Helm Chart Tests -helm/sim/test - ## Claude Code .claude/launch.json .claude/worktrees/ diff --git a/NOTICE b/NOTICE index 11d32f13cd3..cc292af7ec2 100644 --- a/NOTICE +++ b/NOTICE @@ -1,4 +1,4 @@ Sim Studio Copyright 2026 Sim Studio -This product includes software developed for the Sim project. \ No newline at end of file +This product includes software developed for the Sim project. diff --git a/apps/desktop/README.md b/apps/desktop/README.md index 0d9b1bdfe98..c8deeb969d1 100644 --- a/apps/desktop/README.md +++ b/apps/desktop/README.md @@ -34,7 +34,9 @@ src/main/ # main process (bundled to dist/main.cjs) browser-credentials/ # saved passwords, OS-auth gated, safeStorage at rest browser-sites/ # imported site directory, safeStorage at rest browser-import/ # one-shot import of profiles, cookies and passwords -src/preload/ # contextBridge IPC bridge (bundled to dist/preload.cjs) +src/preload/ # isolated renderer bridges + index.ts # hosted-app contextBridge IPC bridge (dist/preload.cjs) + browser/ # minimal agent-browser credential helper (dist/browser-preload.cjs) native/ # Node-API/AppKit bridge for native macOS Help docs search static/ # bundled local pages (offline.html) e2e/ # Playwright _electron smoke suite @@ -54,7 +56,7 @@ SIM_DESKTOP_ORIGIN=http://localhost:3000 bun run dev # against local sim - `bun run type-check` / `lint:check` — standard workspace checks; CI picks these up automatically via `turbo run`. - `SIM_DESKTOP_USER_DATA=` isolates settings/partition state (used by e2e). -Everything is bundled by esbuild into `dist/main.cjs` + `dist/preload.cjs` — including `electron-updater` and the `@sim/*` packages — so the packaged app has **no runtime node_modules** and `electron-builder` needs no lockfile/npmRebuild step (this is the deliberate workaround for Bun ↔ electron-builder friction; there is no `package-lock.json`). +The main process and two preloads are bundled by esbuild into `dist/main.cjs`, `dist/preload.cjs`, and `dist/browser-preload.cjs`, including `electron-updater` and the `@sim/*` packages. The native `@lydell/node-pty` packages stay external so Electron can load their architecture-specific prebuilds from the packaged runtime `node_modules`; `npmRebuild` remains disabled because those Node-API prebuilds are already ABI-stable. There is no `package-lock.json`. ## Auth model (read before touching auth) @@ -99,7 +101,7 @@ Overall this is **within normal thin-wrapper coupling** — every item is either Local unsigned build: `bun run package:dir` (app in `release/mac-universal/`). Signed: `bun run package:mac` with `CSC_LINK`/`CSC_KEY_PASSWORD` exported. -Pre-release share (no Developer ID yet): `SIM_DESKTOP_DEFAULT_ORIGIN=https://www.dev.sim.ai bun run package:share` builds a DMG whose fresh installs default to that origin (baked at build time; official builds leave it unset → prod) and skips per-file signature timestamps. Recipients must clear quarantine once: `xattr -cr /Applications/Sim.app`. +Local unsigned pre-release share: `SIM_DESKTOP_DEFAULT_ORIGIN=https://www.dev.sim.ai bun run package:share` builds a DMG whose fresh installs default to that origin (baked at build time; official builds leave it unset → prod) and skips per-file signature timestamps. Recipients must clear quarantine once: `xattr -cr /Applications/Sim.app`. The build also derives the app icon from `SIM_DESKTOP_DEFAULT_ORIGIN`. Every channel uses the exact production icon with its white background and black `sim` mark. Non-production channels add a thin outline using existing platform colors: dev uses orange, staging uses Loop blue, and localhost uses Workflow violet. The macOS menu-bar icon also carries a compact `D`, `S`, or `L` subscript for those environments; production remains unmarked. Native Icon Composer assets live in `build/`; `scripts/build.ts` copies the selected variant to the ignored `build/generated-icon.icon` path consumed by electron-builder. Electron-builder compiles it to `Assets.car` and derives the legacy `.icns` fallback from the same source. Matching 512px PNGs in `static/` provide the Dock icon for unpackaged runs. @@ -185,10 +187,11 @@ Raw local file bytes are never exposed through the preload bridge and cannot be ## Known caveats -- Microphone and camera are denied by design (the permission matrix grants only sanitized clipboard writes to the app origin). +- The hosted Sim renderer may request microphone access for voice input from the configured app origin; camera access remains denied. On macOS the shell also requires the operating-system microphone grant. Separately, a page in the isolated agent browser may request microphone or camera only from its main frame after a recent native user gesture; Sim then requires an explicit document-scoped prompt and the operating-system grant where applicable. +- The built-in agent browser is not a general-purpose download manager. Its dedicated partition applies the same bounded policy to every download, including one started by a direct user click: at most 2 GiB per file, two active downloads per task, six app-wide, and a 1 GiB free-disk reserve. A rejected download appears in the browser's downloads menu; use a normal browser for an intentionally larger transfer. - Default Electron ships H.264/AAC/MP3 — do not swap in the codec-free ffmpeg build. - Third-party web analytics (GTM/GA) are blocked at the network layer by default (`blockThirdPartyAnalytics`); first-party PostHog `/ingest` is untouched. -- `Cmd+F` find-in-page overlay is not implemented (Monaco and tables ship their own finds); revisit if users ask. +- `Cmd+F` opens the native find overlay in built-in browser tabs. The hosted Sim workspace continues to use Monaco- and table-specific find surfaces. - Sign-in uses only the `127.0.0.1` loopback callback, which needs no OS registration — so it completes identically under `bun run dev` (unpackaged) and in a packaged build. There is no custom URL scheme. ## Electron upgrades diff --git a/apps/desktop/package.json b/apps/desktop/package.json index 52161642cb8..49f5aa98909 100644 --- a/apps/desktop/package.json +++ b/apps/desktop/package.json @@ -50,9 +50,10 @@ "@sim/tsconfig": "workspace:*", "@types/micromatch": "4.0.10", "@types/node": "24.2.1", - "electron": "43.1.1", + "electron": "43.5.0", "electron-builder": "26.15.3", "esbuild": "0.28.1", + "jsdom": "^26.0.0", "typescript": "^7.0.2", "vitest": "^4.1.0" } diff --git a/apps/desktop/src/main/browser-agent/driver-profile.test.ts b/apps/desktop/src/main/browser-agent/driver-profile.test.ts index fb77abf35d5..90d5dab667d 100644 --- a/apps/desktop/src/main/browser-agent/driver-profile.test.ts +++ b/apps/desktop/src/main/browser-agent/driver-profile.test.ts @@ -10,6 +10,8 @@ const mocks = vi.hoisted(() => ({ vi.mock('@/main/browser-agent/session', () => ({ clearProfileStorage: mocks.clearProfileStorage, initSession: vi.fn(), + isBrowserScopeSuspended: vi.fn(() => false), + resolveBrowserScopeId: vi.fn((scopeId: string) => scopeId), })) vi.mock('@/main/browser-credentials', () => ({ @@ -18,7 +20,12 @@ vi.mock('@/main/browser-credentials', () => ({ initFillCoordinator: vi.fn(), })) -import { clearBrowserProfile, initDriver } from '@/main/browser-agent/driver' +import { + captureBrowserToolQueueBoundary, + clearBrowserProfile, + executeTool, + initDriver, +} from '@/main/browser-agent/driver' import type { ConfigStore } from '@/main/config' describe('clearBrowserProfile', () => { @@ -52,4 +59,35 @@ describe('clearBrowserProfile', () => { expect(mocks.clearCredentials).toHaveBeenCalledTimes(2) expect(config.flush).toHaveBeenCalledTimes(2) }) + + it('invalidates pre-wipe authorization and retires live work before profile teardown', async () => { + initDriver( + { + onPageState: vi.fn(), + onTabsState: vi.fn(), + onSessionStatus: vi.fn(), + onFillAvailability: vi.fn(), + }, + () => null + ) + const boundary = captureBrowserToolQueueBoundary('chat-before-wipe') + expect(boundary).not.toBeNull() + if (!boundary) throw new Error('Expected browser tool authorization admission') + + await clearBrowserProfile() + const staleExecution = await executeTool( + 'chat-before-wipe', + 'browser_list_sessions', + {}, + 'tool-authorized-before-wipe', + boundary + ) + + expect(mocks.clearProfileStorage).toHaveBeenCalledOnce() + expect(mocks.clearCredentials).toHaveBeenCalledOnce() + expect(staleExecution).toMatchObject({ + ok: false, + error: expect.stringContaining('cancelled before it started'), + }) + }) }) diff --git a/apps/desktop/src/main/browser-agent/driver.test.ts b/apps/desktop/src/main/browser-agent/driver.test.ts index ade81aff505..4610e0999f1 100644 --- a/apps/desktop/src/main/browser-agent/driver.test.ts +++ b/apps/desktop/src/main/browser-agent/driver.test.ts @@ -1,3 +1,4 @@ +import { BROWSER_TOOL_QUEUE_WAIT_TIMEOUT_MS } from '@sim/browser-protocol' import type { MenuItemConstructorOptions } from 'electron' import { beforeEach, describe, expect, it, vi } from 'vitest' @@ -33,6 +34,29 @@ function freshDriver(): DriverModule { return driverModule } +type BrowserToolQueueBoundary = NonNullable< + ReturnType +> + +function capturePendingAuthorizations( + driver: DriverModule, + scopeId: string, + count: number = driver.BROWSER_TOOL_ADMISSION_LIMITS.perScope +): BrowserToolQueueBoundary[] { + const boundaries = Array.from({ length: count }, () => + driver.captureBrowserToolQueueBoundary(scopeId) + ) + expect(boundaries.every((boundary) => boundary !== null)).toBe(true) + return boundaries.filter((boundary): boundary is BrowserToolQueueBoundary => boundary !== null) +} + +function releasePendingAuthorizations( + driver: DriverModule, + boundaries: readonly BrowserToolQueueBoundary[] +): void { + for (const boundary of boundaries) driver.releaseBrowserToolQueueBoundary(boundary) +} + /** Match the serialized function invocation, not comments or helper names in its body. */ function isPageCall(expression: string, fnName: string): boolean { return expression.includes(`function ${fnName}(`) @@ -53,6 +77,7 @@ describe('executeTool', () => { }) it('validates navigation URLs before touching the session', async () => { + const grant = vi.spyOn(session, 'grantSiteOriginForAgentNavigation') const result = await driver.executeTool('chat-test', 'browser_navigate', { url: 'file:///etc/passwd', }) @@ -60,6 +85,7 @@ describe('executeTool', () => { ok: false, error: 'URL must be absolute and start with http:// or https://', }) + expect(grant).not.toHaveBeenCalled() }) it('reports missing required parameters by name', async () => { @@ -68,6 +94,23 @@ describe('executeTool', () => { expect(result.error).toMatch(/Missing required parameter "url"/) }) + it('grants only SSRF-checked agent navigation destinations before loading them', async () => { + const grant = vi.spyOn(session, 'grantSiteOriginForAgentNavigation') + const navigations = [ + ['browser_navigate', 'http://127.0.0.1:4011/navigate'], + ['browser_open_url', 'http://127.0.0.1:4012/open'], + ['browser_open_tab', 'http://127.0.0.1:4013/tab'], + ] as const + + for (const [tool, url] of navigations) { + await expect(driver.executeTool('chat-test', tool, { url })).resolves.toMatchObject({ + ok: true, + }) + expect(grant).toHaveBeenCalledWith(expect.anything(), url) + } + expect(grant).toHaveBeenCalledTimes(navigations.length) + }) + it('reports an aborted navigation when Chromium never leaves the current URL', async () => { vi.useFakeTimers() try { @@ -250,6 +293,410 @@ describe('executeTool', () => { } }) + it('does not let a detached takeover poll touch a disposed scope', async () => { + await driver.executeTool('chat-test', 'browser_open_tab', {}) + vi.useFakeTimers() + const hasSession = vi.spyOn(session, 'hasSession') + try { + const takeover = driver.executeTool( + 'chat-test', + 'browser_request_takeover', + { reason: 'Please finish in the browser' }, + 'tool-disposed-takeover' + ) + await vi.advanceTimersByTimeAsync(0) + + driver.disposeBrowserScope('chat-test') + hasSession.mockClear() + await expect(takeover).resolves.toMatchObject({ + ok: false, + error: expect.stringContaining('cancelled'), + }) + await vi.advanceTimersByTimeAsync(1_500) + + expect(hasSession).not.toHaveBeenCalled() + } finally { + hasSession.mockRestore() + vi.useRealTimers() + } + }) + + it('does not let a detached text wait touch a disposed scope', async () => { + await driver.executeTool('chat-test', 'browser_open_tab', {}) + const contents = session.requireTab().view.webContents + vi.mocked(contents.getURL).mockReturnValue('https://example.com/') + let resolvePageProbe: (value: boolean) => void = () => {} + vi.mocked(contents.executeJavaScript).mockImplementation( + () => + new Promise((resolve) => { + resolvePageProbe = resolve + }) + ) + vi.useFakeTimers() + const automationTab = vi.spyOn(session, 'automationTab') + try { + const waiting = driver.executeTool( + 'chat-test', + 'browser_wait_for', + { text: 'ready', timeoutMs: 120_000 }, + 'tool-disposed-wait' + ) + await vi.advanceTimersByTimeAsync(0) + expect(contents.executeJavaScript).toHaveBeenCalled() + + driver.disposeBrowserScope('chat-test') + automationTab.mockClear() + await expect(waiting).resolves.toMatchObject({ + ok: false, + error: expect.stringContaining('cancelled'), + }) + resolvePageProbe(false) + await vi.advanceTimersByTimeAsync(300) + + expect(automationTab).not.toHaveBeenCalled() + } finally { + automationTab.mockRestore() + vi.useRealTimers() + } + }) + + it('does not let a detached screenshot verification touch a disposed scope', async () => { + await driver.executeTool('chat-test', 'browser_open_tab', {}) + const contents = session.requireTab().view.webContents + vi.mocked(contents.getURL).mockReturnValue('https://example.com/') + let resolveCapture: (capture: cdp.ScreenshotCapture) => void = () => {} + const captureScreenshot = vi.spyOn(cdp, 'captureScreenshot').mockImplementation( + () => + new Promise((resolve) => { + resolveCapture = resolve + }) + ) + const automationTab = vi.spyOn(session, 'automationTab') + try { + const screenshot = driver.executeTool( + 'chat-test', + 'browser_screenshot', + {}, + 'tool-disposed-screenshot' + ) + await Promise.resolve() + expect(captureScreenshot).toHaveBeenCalledOnce() + + driver.disposeBrowserScope('chat-test') + automationTab.mockClear() + await expect(screenshot).resolves.toMatchObject({ + ok: false, + error: expect.stringContaining('cancelled'), + }) + resolveCapture({ + dataUrl: 'data:image/jpeg;base64,c2lt', + scale: 1, + viewport: { width: 800, height: 600 }, + imageSize: { width: 800, height: 600 }, + }) + await Promise.resolve() + await Promise.resolve() + + expect(automationTab).not.toHaveBeenCalled() + } finally { + automationTab.mockRestore() + captureScreenshot.mockRestore() + } + }) + + it('cancels active and queued work before closing the browser session', async () => { + await driver.executeTool('chat-test', 'browser_open_tab', {}) + vi.useFakeTimers() + try { + const waiting = driver.executeTool( + 'chat-test', + 'browser_wait_for', + { timeoutMs: 120_000 }, + 'tool-active-at-close' + ) + await vi.advanceTimersByTimeAsync(0) + const queuedOpen = driver.executeTool( + 'chat-test', + 'browser_open_tab', + {}, + 'tool-queued-at-close' + ) + + driver.closeBrowserSession() + + await expect(waiting).resolves.toMatchObject({ + ok: false, + error: expect.stringContaining('cancelled'), + }) + await expect(queuedOpen).resolves.toMatchObject({ + ok: false, + error: expect.stringContaining('cancelled before it started'), + }) + expect(session.withBrowserScope('chat-test', () => session.peekTabsState()).tabs).toEqual([]) + } finally { + vi.useRealTimers() + } + }) + + it('rejects authorization captured before a browser-session teardown', async () => { + const boundary = driver.captureBrowserToolQueueBoundary('chat-test') + expect(boundary).not.toBeNull() + if (!boundary) throw new Error('Expected browser tool authorization admission') + driver.closeBrowserSession() + driver.activateBrowserScope('chat-test') + + await expect( + driver.executeTool( + 'chat-test', + 'browser_open_tab', + {}, + 'tool-authorized-before-close', + boundary + ) + ).resolves.toMatchObject({ + ok: false, + error: expect.stringContaining('cancelled before it started'), + }) + expect(session.withBrowserScope('chat-test', () => session.peekTabsState()).tabs).toEqual([]) + }) + + it('captures a missing scope without materializing driver state', async () => { + const boundary = driver.captureBrowserToolQueueBoundary('chat-not-yet-active') + expect(boundary).not.toBeNull() + if (!boundary) throw new Error('Expected browser tool authorization admission') + + expect(boundary).toMatchObject({ + scopeId: 'chat-not-yet-active', + generation: null, + cancellationEpoch: null, + }) + + driver.activateBrowserScope('chat-not-yet-active') + await expect( + driver.executeTool( + 'chat-not-yet-active', + 'browser_list_tabs', + {}, + 'tool-authorized-before-activation', + boundary + ) + ).resolves.toMatchObject({ ok: true }) + }) + + it('rejects a missing-scope authorization after process-wide browser teardown', async () => { + const boundary = driver.captureBrowserToolQueueBoundary('chat-not-yet-active') + expect(boundary).not.toBeNull() + if (!boundary) throw new Error('Expected browser tool authorization admission') + driver.closeBrowserSession() + driver.activateBrowserScope('chat-not-yet-active') + + await expect( + driver.executeTool( + 'chat-not-yet-active', + 'browser_list_tabs', + {}, + 'tool-authorized-before-global-close', + boundary + ) + ).resolves.toMatchObject({ + ok: false, + error: expect.stringContaining('cancelled before it started'), + }) + }) + + it('rejects a first-use authorization after its scope is disposed and reopened', async () => { + const boundary = driver.captureBrowserToolQueueBoundary('chat-first-use-disposed') + expect(boundary).not.toBeNull() + if (!boundary) throw new Error('Expected browser tool authorization admission') + + driver.disposeBrowserScope('chat-first-use-disposed') + driver.activateBrowserScope('chat-first-use-disposed') + + await expect( + driver.executeTool( + 'chat-first-use-disposed', + 'browser_open_tab', + {}, + 'tool-authorized-before-first-use-disposal', + boundary + ) + ).resolves.toMatchObject({ + ok: false, + error: expect.stringContaining('cancelled before it started'), + }) + expect( + session.withBrowserScope('chat-first-use-disposed', () => session.peekTabsState()).tabs + ).toEqual([]) + }) + + it('rejects a first-use authorization after its scope is suspended and reopened', async () => { + const boundary = driver.captureBrowserToolQueueBoundary('chat-first-use-suspended') + expect(boundary).not.toBeNull() + if (!boundary) throw new Error('Expected browser tool authorization admission') + + expect(driver.suspendBrowserScope('chat-first-use-suspended')).toBe(true) + driver.activateBrowserScope('chat-first-use-suspended') + + await expect( + driver.executeTool( + 'chat-first-use-suspended', + 'browser_open_tab', + {}, + 'tool-authorized-before-first-use-suspension', + boundary + ) + ).resolves.toMatchObject({ + ok: false, + error: expect.stringContaining('cancelled before it started'), + }) + expect( + session.withBrowserScope('chat-first-use-suspended', () => session.peekTabsState()).tabs + ).toEqual([]) + }) + + it('cancels a provisional first-use authorization when its durable scope is disposed', async () => { + const boundary = driver.captureBrowserToolQueueBoundary('pending:first-use-disposed') + expect(boundary).not.toBeNull() + if (!boundary) throw new Error('Expected browser tool authorization admission') + expect(driver.migrateBrowserScope('pending:first-use-disposed', 'chat-first-use-durable')).toBe( + true + ) + + driver.disposeBrowserScope('chat-first-use-durable') + driver.activateBrowserScope('chat-first-use-durable') + + await expect( + driver.executeTool( + 'chat-first-use-durable', + 'browser_open_tab', + {}, + 'tool-authorized-before-migrated-disposal', + boundary + ) + ).resolves.toMatchObject({ + ok: false, + error: expect.stringContaining('cancelled before it started'), + }) + }) + + it('keeps authorization teardown scoped to its existing driver state', async () => { + driver.activateBrowserScope('chat-other') + const boundary = driver.captureBrowserToolQueueBoundary('chat-other') + expect(boundary).not.toBeNull() + if (!boundary) throw new Error('Expected browser tool authorization admission') + + driver.disposeBrowserScope('chat-test') + + await expect( + driver.executeTool( + 'chat-other', + 'browser_list_tabs', + {}, + 'tool-authorized-in-other-scope', + boundary + ) + ).resolves.toMatchObject({ ok: true }) + }) + + it('rejects an existing-scope authorization after disposal and recreation', async () => { + const boundary = driver.captureBrowserToolQueueBoundary('chat-test') + expect(boundary).not.toBeNull() + if (!boundary) throw new Error('Expected browser tool authorization admission') + + driver.disposeBrowserScope('chat-test') + driver.activateBrowserScope('chat-test') + + await expect( + driver.executeTool( + 'chat-test', + 'browser_list_tabs', + {}, + 'tool-authorized-before-scope-disposal', + boundary + ) + ).resolves.toMatchObject({ + ok: false, + error: expect.stringContaining('cancelled before it started'), + }) + }) + + it('bounds pending authorizations without materializing their scopes', () => { + const boundaries = capturePendingAuthorizations(driver, 'chat-pending-authorization') + + expect(boundaries.every((boundary) => boundary?.generation === null)).toBe(true) + expect(driver.captureBrowserToolQueueBoundary('chat-pending-authorization')).toBeNull() + + releasePendingAuthorizations(driver, boundaries) + const replacement = driver.captureBrowserToolQueueBoundary('chat-pending-authorization') + expect(replacement).not.toBeNull() + if (replacement) driver.releaseBrowserToolQueueBoundary(replacement) + }) + + it('retains cancelled authorization admissions until their fetches settle', () => { + const boundaries = capturePendingAuthorizations(driver, 'chat-test') + + expect(driver.cancelActiveTool('chat-test')).toBe(true) + expect(boundaries.every((boundary) => boundary.cancelled)).toBe(true) + expect(driver.captureBrowserToolQueueBoundary('chat-test')).toBeNull() + + releasePendingAuthorizations(driver, boundaries) + const replacement = driver.captureBrowserToolQueueBoundary('chat-test') + expect(replacement).not.toBeNull() + if (replacement) driver.releaseBrowserToolQueueBoundary(replacement) + }) + + it('retains disposed-scope authorization admissions until their fetches settle', () => { + const boundaries = capturePendingAuthorizations(driver, 'chat-disposed-authorizations') + + driver.disposeBrowserScope('chat-disposed-authorizations') + expect(boundaries.every((boundary) => boundary.cancelled)).toBe(true) + expect(driver.captureBrowserToolQueueBoundary('chat-disposed-authorizations')).toBeNull() + + releasePendingAuthorizations(driver, boundaries) + const replacement = driver.captureBrowserToolQueueBoundary('chat-disposed-authorizations') + expect(replacement).not.toBeNull() + if (replacement) driver.releaseBrowserToolQueueBoundary(replacement) + }) + + it('retains suspended-scope authorization admissions until their fetches settle', () => { + const boundaries = capturePendingAuthorizations(driver, 'chat-suspended-authorizations') + + expect(driver.suspendBrowserScope('chat-suspended-authorizations')).toBe(true) + expect(boundaries.every((boundary) => boundary.cancelled)).toBe(true) + expect(driver.captureBrowserToolQueueBoundary('chat-suspended-authorizations')).toBeNull() + + releasePendingAuthorizations(driver, boundaries) + const replacement = driver.captureBrowserToolQueueBoundary('chat-suspended-authorizations') + expect(replacement).not.toBeNull() + if (replacement) driver.releaseBrowserToolQueueBoundary(replacement) + }) + + it('retains process-wide authorization admissions across driver reinitialization', () => { + const boundaries = ['chat-auth-a', 'chat-auth-b', 'chat-auth-c', 'chat-auth-d'].flatMap( + (scopeId) => capturePendingAuthorizations(driver, scopeId) + ) + expect(boundaries).toHaveLength(driver.BROWSER_TOOL_ADMISSION_LIMITS.process) + + driver.initDriver( + { + onPageState: vi.fn(), + onTabsState: vi.fn(), + onSessionStatus: vi.fn(), + onFillAvailability: vi.fn(), + }, + () => null + ) + + expect(boundaries.every((boundary) => boundary.cancelled)).toBe(true) + expect(driver.captureBrowserToolQueueBoundary('chat-after-reinit')).toBeNull() + + driver.releaseBrowserToolQueueBoundary(boundaries[0]) + const replacement = driver.captureBrowserToolQueueBoundary('chat-after-reinit') + expect(replacement).not.toBeNull() + if (replacement) driver.releaseBrowserToolQueueBoundary(replacement) + releasePendingAuthorizations(driver, boundaries.slice(1)) + }) + it('honors cancellation that arrives before the authorized tool invocation', async () => { expect(driver.cancelTool('chat-test', 'tool-before-authorization')).toBe(true) @@ -588,6 +1035,110 @@ describe('executeTool', () => { expect(respond).toHaveBeenCalledWith('request-1', true) }) + it('routes an exact renderer site decision through the scoped session boundary', async () => { + const respond = vi.spyOn(session, 'respondToSitePermission').mockReturnValue(true) + + await driver.handlePanelAction('chat-test', { + action: 'respond-site-permission', + requestId: 'request-1', + allowed: true, + }) + await driver.handlePanelAction('chat-test', { + action: 'respond-site-permission', + requestId: 'request-2', + }) + + expect(respond).toHaveBeenCalledOnce() + expect(respond).toHaveBeenCalledWith('request-1', true) + }) + + it('grants only the exact origin entered through the user omnibox', async () => { + await driver.executeTool('chat-test', 'browser_open_tab', {}) + const contents = session.requireTab().view.webContents + const grant = vi.spyOn(session, 'grantSiteOriginForUserNavigation') + + await driver.handlePanelAction('chat-test', { + action: 'navigate', + url: 'https://docs.example/private?token=secret', + }) + + expect(grant).toHaveBeenCalledOnce() + expect(grant).toHaveBeenCalledWith(contents, 'https://docs.example/private?token=secret') + expect(contents.loadURL).toHaveBeenCalledWith('https://docs.example/private?token=secret') + }) + + it('waits for a selected restored tab before reporting it ready to the model', async () => { + await driver.executeTool('chat-test', 'browser_open_tab', {}) + const tab = session.requireTab() + let releaseRestore = () => {} + const wait = vi.spyOn(session, 'waitForPendingTabRestore').mockImplementation( + () => + new Promise((resolve) => { + releaseRestore = () => resolve(true) + }) + ) + let settled = false + const switched = driver + .executeTool('chat-test', 'browser_switch_tab', { tabId: tab.id }) + .then((result) => { + settled = true + return result + }) + + await Promise.resolve() + expect(settled).toBe(false) + expect(wait).toHaveBeenCalledWith(tab) + + releaseRestore() + await expect(switched).resolves.toMatchObject({ + ok: true, + result: { tabId: tab.id }, + }) + wait.mockRestore() + }) + + it('does not report a timed-out restored tab as ready to the model', async () => { + await driver.executeTool('chat-test', 'browser_open_tab', {}) + const tab = session.requireTab() + const wait = vi.spyOn(session, 'waitForPendingTabRestore').mockResolvedValue(false) + + await expect( + driver.executeTool('chat-test', 'browser_switch_tab', { tabId: tab.id }) + ).resolves.toMatchObject({ + ok: false, + error: expect.stringContaining('did not finish loading'), + }) + + wait.mockRestore() + }) + + it('allows a fifty-second restored-tab consent and load without duplicating the tab', async () => { + vi.useFakeTimers() + try { + await driver.executeTool('chat-test', 'browser_open_tab', {}) + const tab = session.requireTab() + const wait = vi.spyOn(session, 'waitForPendingTabRestore').mockImplementation( + () => + new Promise((resolve) => { + setTimeout(() => resolve(true), 50_000) + }) + ) + + const switched = driver.executeTool('chat-test', 'browser_switch_tab', { tabId: tab.id }) + await vi.advanceTimersByTimeAsync(50_000) + + await expect(switched).resolves.toMatchObject({ + ok: true, + result: { tabId: tab.id }, + }) + expect(session.listTabs()).toHaveLength(1) + expect(session.requireTab()).toBe(tab) + wait.mockRestore() + } finally { + vi.useRealTimers() + } + }) + it('keeps tool queues and tab state isolated by chat scope', async () => { await driver.executeTool('chat-a', 'browser_open_tab', {}) await driver.executeTool('chat-a', 'browser_open_tab', {}) @@ -621,6 +1172,121 @@ describe('executeTool', () => { expect(driver.migrateBrowserScope('pending:other-chat', 'chat-occupied')).toBe(false) }) + it('cancels only the replaced destination authorizations during migration', async () => { + await driver.executeTool('pending:new-chat', 'browser_open_tab', {}) + driver.activateBrowserScope('chat-real') + const sourceBoundary = driver.captureBrowserToolQueueBoundary('pending:new-chat') + const destinationBoundary = driver.captureBrowserToolQueueBoundary('chat-real') + const otherBoundary = driver.captureBrowserToolQueueBoundary('chat-other') + expect(sourceBoundary).not.toBeNull() + expect(destinationBoundary).not.toBeNull() + expect(otherBoundary).not.toBeNull() + if (!sourceBoundary || !destinationBoundary || !otherBoundary) { + throw new Error('Expected browser tool authorization admissions') + } + + expect(driver.migrateBrowserScope('pending:new-chat', 'chat-real')).toBe(true) + + await expect( + driver.executeTool( + 'chat-real', + 'browser_list_tabs', + {}, + 'tool-destination-before-migration', + destinationBoundary + ) + ).resolves.toMatchObject({ + ok: false, + error: expect.stringContaining('cancelled before it started'), + }) + await expect( + driver.executeTool( + 'chat-real', + 'browser_list_tabs', + {}, + 'tool-source-before-migration', + sourceBoundary + ) + ).resolves.toMatchObject({ ok: true }) + await expect( + driver.executeTool( + 'chat-other', + 'browser_list_tabs', + {}, + 'tool-other-during-migration', + otherBoundary + ) + ).resolves.toMatchObject({ ok: true }) + }) + + it('retains replaced destination admissions until their authorization fetches settle', async () => { + await driver.executeTool('pending:new-chat', 'browser_open_tab', {}) + driver.activateBrowserScope('chat-real') + const sourceBoundary = driver.captureBrowserToolQueueBoundary('pending:new-chat') + const destinationBoundaries = capturePendingAuthorizations( + driver, + 'chat-real', + driver.BROWSER_TOOL_ADMISSION_LIMITS.perScope - 1 + ) + expect(sourceBoundary).not.toBeNull() + if (!sourceBoundary) throw new Error('Expected source authorization admission') + + expect(driver.migrateBrowserScope('pending:new-chat', 'chat-real')).toBe(true) + + expect(destinationBoundaries.every((boundary) => boundary.cancelled)).toBe(true) + expect(sourceBoundary.cancelled).toBe(false) + expect(driver.captureBrowserToolQueueBoundary('chat-real')).toBeNull() + + releasePendingAuthorizations(driver, destinationBoundaries) + await expect( + driver.executeTool( + 'chat-real', + 'browser_list_tabs', + {}, + 'tool-source-after-destination-settlement', + sourceBoundary + ) + ).resolves.toMatchObject({ ok: true }) + const replacement = driver.captureBrowserToolQueueBoundary('chat-real') + expect(replacement).not.toBeNull() + if (replacement) driver.releaseBrowserToolQueueBoundary(replacement) + }) + + it('keeps migrated source admissions charged to the durable scope after disposal', () => { + driver.activateBrowserScope('pending:new-chat') + const sourceBoundaries = capturePendingAuthorizations(driver, 'pending:new-chat') + + expect(driver.migrateBrowserScope('pending:new-chat', 'chat-real')).toBe(true) + expect(sourceBoundaries.every((boundary) => boundary.scopeId === 'chat-real')).toBe(true) + + driver.disposeBrowserScope('chat-real') + driver.activateBrowserScope('chat-real') + expect(sourceBoundaries.every((boundary) => boundary.cancelled)).toBe(true) + expect(driver.captureBrowserToolQueueBoundary('chat-real')).toBeNull() + + releasePendingAuthorizations(driver, sourceBoundaries) + const replacement = driver.captureBrowserToolQueueBoundary('chat-real') + expect(replacement).not.toBeNull() + if (replacement) driver.releaseBrowserToolQueueBoundary(replacement) + }) + + it('retains a migrated provisional alias for callbacks until durable disposal', async () => { + await driver.executeTool('pending:new-chat', 'browser_open_tab', {}) + const tab = session.withBrowserScope('pending:new-chat', () => session.requireTab()) + expect(driver.migrateBrowserScope('pending:new-chat', 'chat-real')).toBe(true) + + driver.disposeBrowserScope('pending:new-chat') + + await expect( + driver.executeTool('pending:new-chat', 'browser_list_tabs', {}) + ).resolves.toMatchObject({ + ok: true, + result: { scopeId: 'chat-real', tabs: [{ tabId: tab.id }] }, + }) + driver.disposeBrowserScope('chat-real') + expect(tab.view.webContents.close).toHaveBeenCalledOnce() + }) + it('keeps activation lazy, then restores and disposes through the driver API', async () => { const snapshot: BrowserSessionSnapshot = { v: 1, @@ -753,6 +1419,141 @@ describe('executeTool', () => { } }) + it('expires a bounded queue wait without running the stale action later', async () => { + await driver.executeTool('chat-test', 'browser_open_tab', {}) + const contents = session.requireTab().view.webContents + vi.mocked(contents.loadURL).mockClear() + vi.useFakeTimers() + try { + const waiting = driver.executeTool( + 'chat-test', + 'browser_wait_for', + { timeoutMs: 120_000 }, + 'tool-queue-head' + ) + await vi.advanceTimersByTimeAsync(0) + const queued = driver.executeTool( + 'chat-test', + 'browser_navigate', + { url: 'http://127.0.0.1/expired' }, + 'tool-queue-expired' + ) + + await vi.advanceTimersByTimeAsync(BROWSER_TOOL_QUEUE_WAIT_TIMEOUT_MS) + await expect(queued).resolves.toMatchObject({ + ok: false, + error: expect.stringContaining('waited too long for earlier browser work'), + }) + + expect(driver.cancelTool('chat-test', 'tool-queue-head')).toBe(true) + await vi.advanceTimersByTimeAsync(0) + await expect(waiting).resolves.toMatchObject({ ok: false }) + expect(contents.loadURL).not.toHaveBeenCalledWith('http://127.0.0.1/expired') + } finally { + vi.useRealTimers() + } + }) + + it('bounds one scope queue and admits new work after the held head is cancelled', async () => { + vi.useFakeTimers() + try { + await driver.executeTool('chat-test', 'browser_open_tab', {}) + const contents = session.requireTab().view.webContents + vi.mocked(contents.getURL).mockReturnValue('https://example.com/') + vi.mocked(contents.executeJavaScript).mockImplementation(() => new Promise(() => {})) + + const held = driver.executeTool('chat-test', 'browser_snapshot', {}, 'held-scope-head') + await vi.advanceTimersByTimeAsync(0) + const queued = Array.from( + { length: driver.BROWSER_TOOL_ADMISSION_LIMITS.perScope - 1 }, + (_, index) => + driver.executeTool('chat-test', 'browser_list_tabs', {}, `queued-scope-${index}`) + ) + + await expect( + driver.executeTool('chat-test', 'browser_list_tabs', {}, 'scope-overflow') + ).resolves.toEqual({ + ok: false, + error: + 'This task browser already has too many actions queued. Wait for earlier actions to finish.', + }) + + expect(driver.cancelTool('chat-test', 'held-scope-head')).toBe(true) + await expect(held).resolves.toMatchObject({ + ok: false, + error: expect.stringContaining('cancelled'), + }) + await expect(Promise.all(queued)).resolves.toEqual( + expect.arrayContaining([ + expect.objectContaining({ + ok: true, + result: expect.objectContaining({ tabs: expect.any(Array) }), + }), + ]) + ) + await expect( + driver.executeTool('chat-test', 'browser_list_tabs', {}, 'scope-recovered') + ).resolves.toMatchObject({ ok: true }) + } finally { + vi.useRealTimers() + } + }) + + it('bounds process-wide queues across scopes and recovers capacity on disposal', async () => { + vi.useFakeTimers() + const scopes = Array.from( + { + length: + driver.BROWSER_TOOL_ADMISSION_LIMITS.process / + driver.BROWSER_TOOL_ADMISSION_LIMITS.perScope, + }, + (_, index) => `chat-admission-${index}` + ) + const executions: Array> = [] + try { + for (const scopeId of scopes) { + await driver.executeTool(scopeId, 'browser_open_tab', {}) + const contents = session.withBrowserScope( + scopeId, + () => session.requireTab().view.webContents + ) + vi.mocked(contents.getURL).mockReturnValue('https://example.com/') + vi.mocked(contents.executeJavaScript).mockImplementation(() => new Promise(() => {})) + executions.push( + driver.executeTool(scopeId, 'browser_snapshot', {}, `held-process-${scopeId}`) + ) + await vi.advanceTimersByTimeAsync(0) + for (let index = 1; index < driver.BROWSER_TOOL_ADMISSION_LIMITS.perScope; index++) { + executions.push( + driver.executeTool( + scopeId, + 'browser_list_tabs', + {}, + `queued-process-${scopeId}-${index}` + ) + ) + } + } + + await expect( + driver.executeTool('chat-process-overflow', 'browser_list_tabs', {}, 'process-overflow') + ).resolves.toEqual({ + ok: false, + error: + 'Sim already has too many browser actions queued. Wait for earlier actions to finish.', + }) + + driver.disposeBrowserScope(scopes[0]) + await expect( + driver.executeTool('chat-process-recovered', 'browser_list_tabs', {}, 'process-recovered') + ).resolves.toMatchObject({ ok: true }) + } finally { + for (const scopeId of scopes) driver.disposeBrowserScope(scopeId) + await Promise.allSettled(executions) + vi.useRealTimers() + } + }) + it('sanitizes hostile tab titles before returning them across the tool boundary', async () => { await driver.executeTool('chat-test', 'browser_open_tab', {}) const contents = session.requireTab().view.webContents @@ -1089,6 +1890,10 @@ describe('executeTool', () => { }) describe('browserToolWatchdogMs', () => { + it('budgets restored-tab switching as navigation work', () => { + expect(driverModule.browserToolWatchdogMs('browser_switch_tab', {})).toBe(60_000) + }) + it.each([ ['number', 30_000, 35_000], ['numeric string', '30000', 35_000], diff --git a/apps/desktop/src/main/browser-agent/driver.ts b/apps/desktop/src/main/browser-agent/driver.ts index 216ecd958de..985bc6ea4f6 100644 --- a/apps/desktop/src/main/browser-agent/driver.ts +++ b/apps/desktop/src/main/browser-agent/driver.ts @@ -18,6 +18,8 @@ */ import { BROWSER_DATA_KINDS, + BROWSER_NAVIGATION_NATIVE_WATCHDOG_MS, + BROWSER_TOOL_QUEUE_WAIT_TIMEOUT_MS, type BrowserDataKind, type BrowserKnownSessionsState, type BrowserPageState, @@ -79,8 +81,12 @@ const TAKEOVER_POLL_MS = 1_500 * legitimate tool (browser_wait_for caps at 120s). */ const DEFAULT_TOOL_WATCHDOG_MS = 20_000 -const NAVIGATION_TOOL_WATCHDOG_MS = 30_000 const WAIT_FOR_TOOL_WATCHDOG_GRACE_MS = 5_000 +/** Retained native tool calls: generous for normal serial use, finite under a wedged caller. */ +export const BROWSER_TOOL_ADMISSION_LIMITS = Object.freeze({ + perScope: 16, + process: 64, +}) const MAX_CROSS_ORIGIN_SNAPSHOT_FRAMES = 8 const MAX_CROSS_ORIGIN_SCAN_FRAMES = 32 const COMBINED_SNAPSHOT_LINE_CAP = 900 @@ -94,6 +100,8 @@ export interface DriverCallbacks { onPageState: (state: BrowserPageState) => void onTabsState: (state: BrowserTabsState) => void onSessionStatus: (alive: boolean, scopeId: string) => void + /** Whether a live renderer for the scope registered support for the consent prompt. */ + sitePermissionPromptSupported?: (scopeId: string) => boolean /** Whether the active tab shows a login form Sim holds a credential for. */ onFillAvailability: (available: boolean, scopeId: string) => void /** Live native download state for one isolated browser scope. */ @@ -111,6 +119,8 @@ let configStore: ConfigStore | null = null * actually happened on the page. */ interface DriverScopeState { + /** Unique state generation so teardown cannot suffer an epoch ABA race. */ + generation: number pendingNotices: string[] takeoverActive: boolean takeoverDone: boolean @@ -127,6 +137,10 @@ interface DriverScopeState { toolQueueCancellationEpoch: number lastTabsStateFingerprint: string | null toolQueue: Promise + /** Admissions held by queued and in-flight tools for this scope. */ + toolAdmissions: Set + /** Prevents detached queue entries from running after their scope is torn down. */ + disposed: boolean /** True while activation is the only operation that has touched this scope. */ activationOnly: boolean /** Tab whose latest monotonic element refs are valid for element actions. */ @@ -144,11 +158,20 @@ interface DriverScopeState { /** Captures the native queue boundary before an async authorization round trip. */ export interface BrowserToolQueueBoundary { scopeId: string - cancellationEpoch: number + /** Invalidates authorization captured before a process-wide browser teardown. */ + lifecycleEpoch: number + /** Present only when the scope already existed at capture time. */ + generation: number | null + cancellationEpoch: number | null + cancelled: boolean } +let nextDriverScopeGeneration = 1 +let browserToolQueueLifecycleEpoch = 0 + function createDriverScopeState(): DriverScopeState { return { + generation: nextDriverScopeGeneration++, pendingNotices: [], takeoverActive: false, takeoverDone: false, @@ -160,6 +183,8 @@ function createDriverScopeState(): DriverScopeState { toolQueueCancellationEpoch: 0, lastTabsStateFingerprint: null, toolQueue: Promise.resolve(), + toolAdmissions: new Set(), + disposed: false, activationOnly: true, snapshotTabId: null, snapshotTargets: new Map(), @@ -200,6 +225,8 @@ function frameNavigationEpoch(contents: WebContents, frame: WebFrameMain): numbe const driverScopeStates = new Map() const driverScopeAliases = new Map() +const activeBrowserToolAdmissions = new Set() +const pendingBrowserToolQueueBoundaries = new Set() const CANCELLED_TOOL_TTL_MS = 5 * 60_000 const MAX_CANCELLED_TOOL_TOMBSTONES = 256 const cancelledToolCallIds = new Map() @@ -237,17 +264,98 @@ function driverScopeState(scopeId = session.getBrowserScopeId()): DriverScopeSta return state } -export function captureBrowserToolQueueBoundary(scopeId: string): BrowserToolQueueBoundary { +function reserveBrowserToolAdmission(state: DriverScopeState): symbol { + const admission = Symbol('browser-tool-admission') + state.toolAdmissions.add(admission) + activeBrowserToolAdmissions.add(admission) + return admission +} + +function releaseBrowserToolAdmission(state: DriverScopeState, admission: symbol): void { + state.toolAdmissions.delete(admission) + activeBrowserToolAdmissions.delete(admission) +} + +function retireDriverScopeState(state: DriverScopeState): void { + state.disposed = true + state.toolQueueCancellationEpoch++ + state.toolInvocationEpoch++ + state.toolExecutionEpoch++ + state.activeToolCancel?.() + state.takeoverActive = false + state.takeoverDone = false + state.takeoverResponse = null + state.takeoverInvocationEpoch = null + for (const admission of state.toolAdmissions) { + activeBrowserToolAdmissions.delete(admission) + } + state.toolAdmissions.clear() +} + +function retireAllDriverScopeStates(): void { + browserToolQueueLifecycleEpoch++ + for (const state of driverScopeStates.values()) retireDriverScopeState(state) + for (const boundary of pendingBrowserToolQueueBoundaries) boundary.cancelled = true + driverScopeStates.clear() + activeBrowserToolAdmissions.clear() + driverScopeAliases.clear() +} + +export function captureBrowserToolQueueBoundary(scopeId: string): BrowserToolQueueBoundary | null { const resolvedScopeId = resolveDriverScopeId(scopeId) - return { + const state = driverScopeStates.get(resolvedScopeId) + if ( + activeBrowserToolAdmissions.size + pendingBrowserToolQueueBoundaries.size >= + BROWSER_TOOL_ADMISSION_LIMITS.process || + (state?.toolAdmissions.size ?? 0) + + [...pendingBrowserToolQueueBoundaries].filter( + (boundary) => resolveDriverScopeId(boundary.scopeId) === resolvedScopeId + ).length >= + BROWSER_TOOL_ADMISSION_LIMITS.perScope + ) { + return null + } + const boundary: BrowserToolQueueBoundary = { scopeId: resolvedScopeId, - cancellationEpoch: driverScopeState(resolvedScopeId).toolQueueCancellationEpoch, + lifecycleEpoch: browserToolQueueLifecycleEpoch, + generation: state?.generation ?? null, + cancellationEpoch: state?.toolQueueCancellationEpoch ?? null, + cancelled: false, + } + pendingBrowserToolQueueBoundaries.add(boundary) + return boundary +} + +export function releaseBrowserToolQueueBoundary(boundary: BrowserToolQueueBoundary): void { + pendingBrowserToolQueueBoundaries.delete(boundary) +} + +function cancelPendingBrowserToolQueueBoundaries(scopeId: string): boolean { + const resolvedScopeId = resolveDriverScopeId(scopeId) + let cancelled = false + for (const boundary of pendingBrowserToolQueueBoundaries) { + if (resolveDriverScopeId(boundary.scopeId) !== resolvedScopeId) continue + boundary.cancelled = true + cancelled = true + } + return cancelled +} + +function cancelBrowserToolQueueBoundaries(boundaries: readonly BrowserToolQueueBoundary[]): void { + for (const boundary of boundaries) { + boundary.cancelled = true } } function isBrowserToolQueueBoundaryCurrent(boundary: BrowserToolQueueBoundary): boolean { + if (boundary.cancelled) return false + if (boundary.lifecycleEpoch !== browserToolQueueLifecycleEpoch) return false + if (boundary.generation === null) return true const state = driverScopeStates.get(resolveDriverScopeId(boundary.scopeId)) - return state?.toolQueueCancellationEpoch === boundary.cancellationEpoch + return ( + state?.generation === boundary.generation && + state.toolQueueCancellationEpoch === boundary.cancellationEpoch + ) } function recordNotice(notice: string): void { @@ -265,6 +373,7 @@ function recordNotice(notice: string): void { function pageStateFor(contents: WebContents, tabId: string): BrowserPageState { const issue = session.pageIssueForContents(contents) const mediaPermissionRequest = session.mediaPermissionRequestForContents(contents) + const sitePermissionRequest = session.sitePermissionRequestForScope() return { scopeId: session.getBrowserScopeId(), tabId, @@ -275,6 +384,7 @@ function pageStateFor(contents: WebContents, tabId: string): BrowserPageState { canGoForward: session.canGoForward(contents), ...(issue ? { issue } : {}), ...(mediaPermissionRequest ? { mediaPermissionRequest } : {}), + ...(sitePermissionRequest ? { sitePermissionRequest } : {}), } } @@ -415,8 +525,7 @@ export function initDriver( // session inherits the previous one's pending notices, a takeover still // waiting on a user who is gone, and a fingerprint that suppresses its very // first tab push as a duplicate. - driverScopeStates.clear() - driverScopeAliases.clear() + retireAllDriverScopeStates() cancelledToolCallIds.clear() // The serialization chain, too. A takeover from the previous session can sit // unresolved indefinitely, and its `takeoverDone` flag is reset above — so @@ -458,6 +567,8 @@ export function initDriver( void fillCoordinator()?.refreshAvailability(true) }, onPageStateChanged: pushPageState, + sitePermissionPromptSupported: (scopeId) => + driverCallbacks?.sitePermissionPromptSupported?.(scopeId) === true, onTabsChanged: pushTabsState, onTabThemeChanged: (contents, theme) => { void cdp.setColorScheme(contents, theme).catch((error) => { @@ -603,9 +714,20 @@ export function migrateBrowserScope(fromScopeId: string, toScopeId: string): boo if (from === to) return true const state = driverScopeStates.get(from) const destinationState = driverScopeStates.get(to) + const sourceBoundaries = [...pendingBrowserToolQueueBoundaries].filter( + (boundary) => resolveDriverScopeId(boundary.scopeId) === from + ) + const destinationBoundaries = [...pendingBrowserToolQueueBoundaries].filter( + (boundary) => resolveDriverScopeId(boundary.scopeId) === to + ) if (destinationState && !destinationState.activationOnly) return false if (!session.migrateBrowserScope(from, to)) return false - if (destinationState) driverScopeStates.delete(to) + for (const boundary of sourceBoundaries) boundary.scopeId = to + cancelBrowserToolQueueBoundaries(destinationBoundaries) + if (destinationState) { + retireDriverScopeState(destinationState) + driverScopeStates.delete(to) + } if (state) { driverScopeStates.delete(from) driverScopeStates.set(to, state) @@ -619,10 +741,12 @@ export function disposeBrowserScope(scopeId: string): void { const resolved = resolveDriverScopeId(scopeId) session.disposeBrowserScope(scopeId) if (wasAlias) { - driverScopeAliases.delete(scopeId) return } + cancelPendingBrowserToolQueueBoundaries(resolved) + const state = driverScopeStates.get(resolved) + if (state) retireDriverScopeState(state) driverScopeStates.delete(resolved) for (const [alias, target] of driverScopeAliases) { if (alias === resolved || resolveDriverScopeId(target) === resolved) { @@ -639,6 +763,9 @@ export function disposeBrowserScope(scopeId: string): void { export function suspendBrowserScope(scopeId: string): boolean { const resolved = resolveDriverScopeId(scopeId) if (!session.suspendBrowserScope(resolved)) return false + cancelPendingBrowserToolQueueBoundaries(resolved) + const state = driverScopeStates.get(resolved) + if (state) retireDriverScopeState(state) driverScopeStates.delete(resolved) return true } @@ -681,6 +808,7 @@ export interface ClearBrowserProfileOptions { export async function clearBrowserProfile( options: ClearBrowserProfileOptions = { settingsPersistence: 'required' } ): Promise { + retireAllDriverScopeStates() const settingsCleared = knownSessions?.clear() !== false const outcomes = await Promise.allSettled([session.clearProfileStorage(), clearCredentials()]) // Last, covering the pinned-tab list `clearProfileStorage` just emptied. @@ -701,6 +829,12 @@ export async function clearBrowserProfile( } } +/** Stops every authorized or queued browser action before closing its live pages. */ +export function closeBrowserSession(): void { + retireAllDriverScopeStates() + session.closeSession() +} + function str(params: Record, key: string): string | undefined { const value = params[key] return typeof value === 'string' && value.length > 0 ? value : undefined @@ -731,9 +865,10 @@ export function browserToolWatchdogMs( tool === 'browser_open_url' || tool === 'browser_go_back' || tool === 'browser_go_forward' || - tool === 'browser_open_tab' + tool === 'browser_open_tab' || + tool === 'browser_switch_tab' ) { - return NAVIGATION_TOOL_WATCHDOG_MS + return BROWSER_NAVIGATION_NATIVE_WATCHDOG_MS } if (tool === 'browser_wait_for') { const requested = normalizeBrowserWaitForTimeoutMs(params.timeoutMs) @@ -1054,10 +1189,14 @@ async function navigationResult( return { url: contents.getURL(), title: contents.getTitle() } } -async function loadUrlAndGetResult( +async function loadAgentCheckedUrlAndGetResult( contents: WebContents, url: string ): Promise> { + session.prepareExplicitNavigation(contents) + if (!session.grantSiteOriginForAgentNavigation(contents, url)) { + throw new ToolError('The tab was closed before navigation could start.') + } const beforeUrl = contents.getURL() try { await contents.loadURL(url) @@ -1908,14 +2047,14 @@ async function runTakeover(purpose: string | undefined, invocationEpoch: number) try { for (;;) { await sleep(TAKEOVER_POLL_MS) + if (state.toolInvocationEpoch !== invocationEpoch) { + throw new ToolError('The browser takeover was superseded by a newer browser action.') + } if (!session.hasSession() || contents.isDestroyed()) { throw new ToolError( 'The browser session was closed during takeover. Ask the user what happened, then reopen with browser_navigate.' ) } - if (state.toolInvocationEpoch !== invocationEpoch) { - throw new ToolError('The browser takeover was superseded by a newer browser action.') - } if (state.takeoverDone) { if (purpose === 'sign_in') { const activeContents = session.automationTab()?.view.webContents @@ -1968,7 +2107,7 @@ async function executeToolInner( const tab = session.ensureAutomationTab() const contents = tab.view.webContents assertCurrentExecution() - return await loadUrlAndGetResult(contents, url) + return await loadAgentCheckedUrlAndGetResult(contents, url) } case 'browser_open_url': { @@ -1985,7 +2124,7 @@ async function executeToolInner( const tab = session.ensureAutomationTab() const contents = tab.view.webContents assertCurrentExecution() - const nav = await loadUrlAndGetResult(contents, url) + const nav = await loadAgentCheckedUrlAndGetResult(contents, url) // A failed snapshot (browser-internal page, injection error) should not // fail the open itself — the page is on screen either way. assertCurrentExecution() @@ -2032,7 +2171,7 @@ async function executeToolInner( const contents = tab.view.webContents if (url) { assertCurrentExecution() - const result = await loadUrlAndGetResult(contents, url) + const result = await loadAgentCheckedUrlAndGetResult(contents, url) return { tabId: tab.id, ...result } } return { tabId: tab.id, url: '', title: '' } @@ -2041,7 +2180,17 @@ async function executeToolInner( case 'browser_switch_tab': { invalidateSnapshot() const tab = session.switchAutomationTab(requireStr(params, 'tabId')) + const restored = await session.waitForPendingTabRestore(tab) + assertCurrentExecution() const contents = tab.view.webContents + if (contents.isDestroyed() || session.automationTab()?.id !== tab.id) { + throw new ToolError('The tab was closed or replaced while it was being restored.') + } + if (!restored) { + throw new ToolError( + 'The saved tab did not finish loading. Retry browser_switch_tab, or navigate it to the saved URL from browser_list_tabs.' + ) + } return { tabId: tab.id, url: contents.getURL(), title: contents.getTitle() } } @@ -2072,6 +2221,7 @@ async function executeToolInner( const waitedTab = session.requireAutomationTab() const contents = waitedTab.view.webContents while (Date.now() - startedAt < timeoutMs) { + assertCurrentExecution() const active = session.automationTab() if (active?.id !== waitedTab.id || active.view.webContents !== contents) { throw new ToolError( @@ -2135,6 +2285,7 @@ async function executeToolInner( const capturedViewportUrl = capturedUrl.slice(0, 4096) const capturedViewportTitle = capturedTitle.slice(0, 500) const captureIsCurrent = (): boolean => { + assertCurrentExecution() const activeTab = session.automationTab() return ( activeTab?.id === capturedTab.id && @@ -3753,111 +3904,164 @@ export async function executeTool( toolCallId?: string, authorizationBoundary?: BrowserToolQueueBoundary ): Promise<{ ok: boolean; result?: unknown; error?: string }> { - const queuedAt = Date.now() const resolvedScopeId = resolveDriverScopeId(scopeId) + if (authorizationBoundary) { + releaseBrowserToolQueueBoundary(authorizationBoundary) + if (!isBrowserToolQueueBoundaryCurrent(authorizationBoundary)) { + return { + ok: false, + error: 'This browser action was cancelled before it started.', + } + } + } if (session.isBrowserScopeSuspended(resolvedScopeId)) { return { ok: false, error: 'This task browser is suspended until the task is reopened.', } } + if (activeBrowserToolAdmissions.size >= BROWSER_TOOL_ADMISSION_LIMITS.process) { + return { + ok: false, + error: 'Sim already has too many browser actions queued. Wait for earlier actions to finish.', + } + } const state = driverScopeState(resolvedScopeId) - state.activationOnly = false - const invocationEpoch = ++state.toolInvocationEpoch - const queueCancellationEpoch = state.toolQueueCancellationEpoch - const run = async () => { - const queueWaitMs = Date.now() - queuedAt - const executionStartedAt = Date.now() - if ( - (authorizationBoundary && !isBrowserToolQueueBoundaryCurrent(authorizationBoundary)) || - queueCancellationEpoch !== state.toolQueueCancellationEpoch || - isToolCallCancelled(toolCallId) - ) { - throw new ToolError('This browser action was cancelled before it started.') + if (state.toolAdmissions.size >= BROWSER_TOOL_ADMISSION_LIMITS.perScope) { + return { + ok: false, + error: + 'This task browser already has too many actions queued. Wait for earlier actions to finish.', } - state.activeToolCallId = toolCallId ?? null - let cancelActiveExecution: () => void = () => {} - const cancellation = new Promise((_resolve, reject) => { - cancelActiveExecution = () => reject(new ToolError('This browser action was cancelled.')) - }) - state.activeToolCancel = cancelActiveExecution - return await session.withBrowserScope(resolvedScopeId, async () => { - logger.info('Executing browser tool', { - tool, - toolCallId, - scopeId: resolvedScopeId, - queueWaitMs, - }) - const keepHiddenPageActive = tool !== 'browser_request_takeover' - if (keepHiddenPageActive) { - session.setAutomationActive(true) - } - try { - const executionEpoch = ++state.toolExecutionEpoch - const watchdogMs = browserToolWatchdogMs(tool, params) - const executionDeadline = watchdogMs === null ? undefined : Date.now() + watchdogMs - const assertCurrentExecution = () => { - if (state.toolExecutionEpoch !== executionEpoch) { - throw new ToolError('This browser action expired before it could dispatch input.') - } - } - const execution = executeToolInner( - tool, - params, - assertCurrentExecution, - executionDeadline, - invocationEpoch + } + const admission = reserveBrowserToolAdmission(state) + let admissionReleased = false + const releaseAdmission = () => { + if (admissionReleased) return + admissionReleased = true + releaseBrowserToolAdmission(state, admission) + } + const queuedAt = Date.now() + let queueWaitExpired = false + let queueWaitTimeoutId: ReturnType | undefined + const queueWaitTimeout = new Promise((_resolve, reject) => { + queueWaitTimeoutId = setTimeout(() => { + queueWaitExpired = true + reject( + new ToolError( + 'This browser action waited too long for earlier browser work and was cancelled before it started.' ) - const guardedExecution = - watchdogMs === null - ? execution - : raceAgainstWatchdog(execution, watchdogMs, () => { - if (state.toolExecutionEpoch === executionEpoch) state.toolExecutionEpoch++ - if (tool === 'browser_snapshot' || tool === 'browser_open_url') { - invalidateSnapshot(state) - } - }) - const result = withNotices(await Promise.race([guardedExecution, cancellation])) - logger.info('Browser tool completed', { + ) + }, BROWSER_TOOL_QUEUE_WAIT_TIMEOUT_MS) + }) + try { + state.activationOnly = false + const invocationEpoch = ++state.toolInvocationEpoch + const queueCancellationEpoch = state.toolQueueCancellationEpoch + const run = async () => { + clearTimeout(queueWaitTimeoutId) + const queueWaitMs = Date.now() - queuedAt + const executionStartedAt = Date.now() + if ( + queueWaitExpired || + state.disposed || + (authorizationBoundary && !isBrowserToolQueueBoundaryCurrent(authorizationBoundary)) || + queueCancellationEpoch !== state.toolQueueCancellationEpoch || + isToolCallCancelled(toolCallId) + ) { + throw new ToolError('This browser action was cancelled before it started.') + } + state.activeToolCallId = toolCallId ?? null + let cancelActiveExecution: () => void = () => {} + const cancellation = new Promise((_resolve, reject) => { + cancelActiveExecution = () => reject(new ToolError('This browser action was cancelled.')) + }) + state.activeToolCancel = cancelActiveExecution + return await session.withBrowserScope(resolvedScopeId, async () => { + logger.info('Executing browser tool', { tool, toolCallId, scopeId: resolvedScopeId, queueWaitMs, - executionMs: Date.now() - executionStartedAt, }) - return result - } finally { + const keepHiddenPageActive = tool !== 'browser_request_takeover' if (keepHiddenPageActive) { - session.setAutomationActive(false) + session.setAutomationActive(true) } - if (state.activeToolCancel === cancelActiveExecution) { - state.activeToolCallId = null - state.activeToolCancel = null + try { + const executionEpoch = ++state.toolExecutionEpoch + const watchdogMs = browserToolWatchdogMs(tool, params) + const executionDeadline = watchdogMs === null ? undefined : Date.now() + watchdogMs + const assertCurrentExecution = () => { + if (state.toolExecutionEpoch !== executionEpoch) { + throw new ToolError('This browser action expired before it could dispatch input.') + } + } + const execution = executeToolInner( + tool, + params, + assertCurrentExecution, + executionDeadline, + invocationEpoch + ) + const guardedExecution = + watchdogMs === null + ? execution + : raceAgainstWatchdog(execution, watchdogMs, () => { + if (state.toolExecutionEpoch === executionEpoch) state.toolExecutionEpoch++ + if (tool === 'browser_snapshot' || tool === 'browser_open_url') { + invalidateSnapshot(state) + } + }) + const result = withNotices(await Promise.race([guardedExecution, cancellation])) + logger.info('Browser tool completed', { + tool, + toolCallId, + scopeId: resolvedScopeId, + queueWaitMs, + executionMs: Date.now() - executionStartedAt, + }) + return result + } finally { + if (keepHiddenPageActive && !state.disposed) { + session.setAutomationActive(false) + } + if (state.activeToolCancel === cancelActiveExecution) { + state.activeToolCallId = null + state.activeToolCancel = null + } } - } - }) - } + }) + } - const settled = state.toolQueue.then(run, run) - state.toolQueue = settled.catch(() => {}) - try { - return { ok: true, result: sanitizeBrowserResult(await settled) } - } catch (error) { - // The watchdog cannot cancel an in-flight renderer promise. Invalidate its - // capture token before releasing the queue so a late snapshot cannot - // overwrite refs belonging to a newer tab or snapshot. - if (tool === 'browser_snapshot' || tool === 'browser_open_url') { - invalidateSnapshot(state) - } - const message = String(sanitizeBrowserResult(getErrorMessage(error), undefined, 0, 'error')) - logger.warn('Browser tool failed', { - tool, - toolCallId, - scopeId: resolvedScopeId, - totalMs: Date.now() - queuedAt, - error: message, - }) - return { ok: false, error: message } + const settled = state.toolQueue.then(run, run) + state.toolQueue = settled.catch(() => {}) + settled.then(releaseAdmission, releaseAdmission) + try { + return { + ok: true, + result: sanitizeBrowserResult(await Promise.race([settled, queueWaitTimeout])), + } + } catch (error) { + // The watchdog cannot cancel an in-flight renderer promise. Invalidate its + // capture token before releasing the queue so a late snapshot cannot + // overwrite refs belonging to a newer tab or snapshot. + if (tool === 'browser_snapshot' || tool === 'browser_open_url') { + invalidateSnapshot(state) + } + const message = String(sanitizeBrowserResult(getErrorMessage(error), undefined, 0, 'error')) + logger.warn('Browser tool failed', { + tool, + toolCallId, + scopeId: resolvedScopeId, + totalMs: Date.now() - queuedAt, + error: message, + }) + return { ok: false, error: message } + } + } finally { + clearTimeout(queueWaitTimeoutId) + if (!queueWaitExpired) releaseAdmission() } } @@ -3888,11 +4092,12 @@ export function cancelTool(scopeId: string, toolCallId: string): boolean { /** Cancels the active tool and every older invocation already queued for this scope. */ export function cancelActiveTool(scopeId: string): boolean { const resolvedScopeId = resolveDriverScopeId(scopeId) + const cancelledPendingAuthorization = cancelPendingBrowserToolQueueBoundaries(resolvedScopeId) const state = driverScopeStates.get(resolvedScopeId) - if (!state) return false + if (!state) return cancelledPendingAuthorization state.toolQueueCancellationEpoch++ const toolCallId = state.activeToolCallId - return toolCallId ? cancelTool(resolvedScopeId, toolCallId) : false + return toolCallId ? cancelTool(resolvedScopeId, toolCallId) : cancelledPendingAuthorization } /** Browser-chrome commands from the panel header; fire-and-forget. */ @@ -3923,6 +4128,12 @@ export async function handlePanelAction( } return } + if (action.action === 'respond-site-permission') { + if (typeof action.requestId === 'string' && typeof action.allowed === 'boolean') { + session.respondToSitePermission(action.requestId, action.allowed) + } + return + } // Navigate bootstraps the session: the user can open the panel manually // (before the agent ever touched the browser) and drive it from the URL // bar. The other chrome actions need an existing page. @@ -3930,6 +4141,8 @@ export async function handlePanelAction( if (typeof action.url === 'string' && /^https?:\/\//i.test(action.url)) { session.claimActiveTabForUser() const contents = session.ensureTab().view.webContents + session.prepareExplicitNavigation(contents) + session.grantSiteOriginForUserNavigation(contents, action.url) void contents.loadURL(action.url).catch(() => {}) } return diff --git a/apps/desktop/src/main/browser-agent/session.test.ts b/apps/desktop/src/main/browser-agent/session.test.ts index 84de584d9d1..2f81cc51f8d 100644 --- a/apps/desktop/src/main/browser-agent/session.test.ts +++ b/apps/desktop/src/main/browser-agent/session.test.ts @@ -4,9 +4,21 @@ import { join } from 'node:path' import type { MenuItemConstructorOptions, WebContents } from 'electron' import { beforeEach, describe, expect, it, vi } from 'vitest' -vi.mock('electron', () => import('@/test/electron-mock')) +const { mockLookup } = vi.hoisted(() => ({ mockLookup: vi.fn() })) -import { BrowserWindow, session as electronSession, Menu, shell, systemPreferences } from 'electron' +vi.mock('electron', () => import('@/test/electron-mock')) +vi.mock('node:dns/promises', () => ({ + default: { lookup: mockLookup }, +})) + +import { + BrowserWindow, + dialog, + session as electronSession, + Menu, + shell, + systemPreferences, +} from 'electron' import { BASE_ZOOM_FACTOR, steppedZoomFactor } from '@/main/browser-agent/context-menu' import * as panel from '@/main/browser-agent/panel' import * as sessionModule from '@/main/browser-agent/session' @@ -25,12 +37,14 @@ interface MockView { session: { setPermissionRequestHandler: ReturnType setPermissionCheckHandler: ReturnType + webRequest: { onBeforeRequest: ReturnType } } on: ReturnType setUserAgent: ReturnType setWindowOpenHandler: ReturnType loadURL: ReturnType reload: ReturnType + stop: ReturnType forcefullyCrashRenderer: ReturnType getURL: ReturnType getTitle: ReturnType @@ -92,6 +106,7 @@ function freshSession( onTabCreated: vi.fn(), onActiveTabChanged: vi.fn(), onPageStateChanged: vi.fn(), + sitePermissionPromptSupported: vi.fn(() => true), onTabsChanged: vi.fn(), onTabThemeChanged: vi.fn(), onTabNavigated: vi.fn(), @@ -142,13 +157,159 @@ function hostResizeHandler(win: BrowserWindow): () => void { function mainFrameNavigationStarted( contents: MockView['webContents'], - isSameDocument = false + isSameDocument = false, + url = (contents.getURL as unknown as () => string)() ): void { const handler = contents.on.mock.calls .filter(([eventName]) => eventName === 'did-start-navigation') .at(-1)?.[1] if (typeof handler !== 'function') throw new Error('no navigation-start listener bound') - handler({ isMainFrame: true, isSameDocument }) + handler({ isMainFrame: true, isSameDocument, url }) +} + +function beginMainFrameRequest( + contents: MockView['webContents'], + url: string, + id = 1 +): Promise<{ cancel: boolean }> { + const handler = contents.session.webRequest.onBeforeRequest.mock.calls[0]?.[0] + if (typeof handler !== 'function') throw new Error('no before-request listener bound') + return new Promise((resolve) => { + handler( + { + id, + url, + method: 'GET', + webContents: contents, + resourceType: 'mainFrame', + referrer: (contents.getURL as unknown as () => string)(), + timestamp: Date.now(), + uploadData: [], + }, + resolve + ) + }) +} + +function beginSubresourceRequest( + contents: MockView['webContents'], + url: string, + resourceType: string, + id = 1 +): Promise<{ cancel: boolean }> { + const handler = contents.session.webRequest.onBeforeRequest.mock.calls[0]?.[0] + if (typeof handler !== 'function') throw new Error('no before-request listener bound') + return new Promise((resolve) => { + handler( + { + id, + url, + method: 'GET', + webContents: contents, + resourceType, + referrer: (contents.getURL as unknown as () => string)(), + timestamp: Date.now(), + uploadData: [], + }, + resolve + ) + }) +} + +type MockDownloadDoneState = 'completed' | 'cancelled' | 'interrupted' + +interface MockDownloadHarness { + item: { + getFilename: ReturnType + getMimeType: ReturnType + getReceivedBytes: ReturnType + getTotalBytes: ReturnType + setSavePath: ReturnType + pause: ReturnType + resume: ReturnType + cancel: ReturnType + on: ReturnType + once: ReturnType + } + setReceivedBytes: (bytes: number) => void + setTotalBytes: (bytes: number) => void + emitUpdated: (state?: 'progressing' | 'interrupted') => void + emitDone: (state: MockDownloadDoneState) => void +} + +function deferred(): { + promise: Promise + resolve: (value: T) => void + reject: (reason?: unknown) => void +} { + let resolve!: (value: T) => void + let reject!: (reason?: unknown) => void + const promise = new Promise((resolvePromise, rejectPromise) => { + resolve = resolvePromise + reject = rejectPromise + }) + return { promise, resolve, reject } +} + +function mockDownloadItem({ + filename = 'report.csv', + mimeType = 'text/csv', + receivedBytes: initialReceivedBytes = 0, + totalBytes: initialTotalBytes = 0, +}: { + filename?: string + mimeType?: string + receivedBytes?: number + totalBytes?: number +} = {}): MockDownloadHarness { + let receivedBytes = initialReceivedBytes + let totalBytes = initialTotalBytes + const item = { + getFilename: vi.fn(() => filename), + getMimeType: vi.fn(() => mimeType), + getReceivedBytes: vi.fn(() => receivedBytes), + getTotalBytes: vi.fn(() => totalBytes), + setSavePath: vi.fn(), + pause: vi.fn(), + resume: vi.fn(), + cancel: vi.fn(), + on: vi.fn(), + once: vi.fn(), + } + return { + item, + setReceivedBytes: (bytes) => { + receivedBytes = bytes + }, + setTotalBytes: (bytes) => { + totalBytes = bytes + }, + emitUpdated: (state = 'progressing') => { + const handler = item.on.mock.calls.find(([eventName]) => eventName === 'updated')?.[1] as + | ((event: unknown, nextState: 'progressing' | 'interrupted') => void) + | undefined + handler?.({}, state) + }, + emitDone: (state) => { + const handler = item.once.mock.calls.find(([eventName]) => eventName === 'done')?.[1] as + | ((event: unknown, nextState: MockDownloadDoneState) => void) + | undefined + handler?.({}, state) + }, + } +} + +function startMockDownload(contents: MockView['webContents'], download: MockDownloadHarness): void { + const webSession = contents.session as typeof contents.session & { + on: ReturnType + } + const willDownload = webSession.on.mock.calls.find( + ([eventName]) => eventName === 'will-download' + )?.[1] as + | ((event: unknown, item: MockDownloadHarness['item'], contents: unknown) => void) + | undefined + if (!willDownload) throw new Error('no will-download listener bound') + willDownload({}, download.item, contents) } describe('browser-agent session', () => { @@ -156,6 +317,8 @@ describe('browser-agent session', () => { let session: SessionModule beforeEach(async () => { + mockLookup.mockReset() + mockLookup.mockResolvedValue([{ address: '93.184.216.34', family: 4 }]) win = mainWindowMock() session = freshSession(win) }) @@ -322,6 +485,23 @@ describe('browser-agent session', () => { expect(session.migrateBrowserScope('chat-real', 'occupied')).toBe(false) }) + it('retains a migrated provisional alias until the durable scope is disposed', () => { + const tab = session.withBrowserScope('pending:workspace', () => session.ensureTab()) + expect(session.migrateBrowserScope('pending:workspace', 'chat-real')).toBe(true) + + session.disposeBrowserScope('pending:workspace') + + expect(session.withBrowserScope('pending:workspace', () => session.activeTab())).toBe(tab) + session.withBrowserScope('pending:workspace', () => session.claimActiveTabForUser()) + expect(session.withBrowserScope('chat-real', () => session.activeTab())).toBe(tab) + + session.disposeBrowserScope('chat-real') + expect((tab.view as unknown as MockView).webContents.close).toHaveBeenCalledOnce() + expect( + session.withBrowserScope('pending:workspace', () => session.peekTabsState().tabs) + ).toEqual([]) + }) + it('preserves a persisted destination behind a lazy activation', () => { const existingSnapshot: BrowserSessionSnapshot = { v: 1, @@ -458,6 +638,622 @@ describe('browser-agent session', () => { ) }) + it('selects and starts the active restore before three bounded background loads', async () => { + const tabs = Array.from({ length: 7 }, (_, index) => ({ + url: `https://restore-${index}.example/`, + pinned: index < 2, + })) + const { persistence } = memoryBrowserPersistence({ + 'chat-restore-order': { + v: 1, + tabs, + activeIndex: 5, + downloads: [], + }, + }) + const createdContents: MockView['webContents'][] = [] + const resolveLoads: Array<(() => void) | undefined> = [] + session = freshSession( + win, + { + onTabCreated: (webContents) => { + const contents = webContents as unknown as MockView['webContents'] + const index = createdContents.push(contents) - 1 + contents.loadURL.mockImplementation( + () => + new Promise((resolve) => { + resolveLoads[index] = resolve + }) + ) + }, + }, + persistence + ) + + session.withBrowserScope('chat-restore-order', () => session.restoreBrowserSession()) + + expect( + session.withBrowserScope('chat-restore-order', () => session.getTabsState()) + ).toMatchObject({ + activeTabId: '6', + tabs: [ + { tabId: '1', pinned: true }, + { tabId: '2', pinned: true }, + { tabId: '3', pinned: false }, + { tabId: '4', pinned: false }, + { tabId: '5', pinned: false }, + { tabId: '6', pinned: false, active: true }, + { tabId: '7', pinned: false }, + ], + }) + expect(createdContents[5].loadURL).toHaveBeenCalledWith(tabs[5].url) + expect(createdContents[5].loadURL.mock.invocationCallOrder[0]).toBeLessThan( + createdContents[0].loadURL.mock.invocationCallOrder[0] + ) + expect( + createdContents.filter((contents) => contents.loadURL.mock.calls.length > 0) + ).toHaveLength(4) + expect(createdContents[3].loadURL).not.toHaveBeenCalled() + + resolveLoads[0]?.() + await vi.waitFor(() => { + expect(createdContents[3].loadURL).toHaveBeenCalledWith(tabs[3].url) + }) + }) + + it('preempts a background restore for a user-selected queued tab', async () => { + const tabs = Array.from({ length: 7 }, (_, index) => ({ + url: `https://priority-${index}.example/`, + pinned: false, + })) + const { persistence } = memoryBrowserPersistence({ + 'chat-restore-priority': { + v: 1, + tabs, + activeIndex: 0, + downloads: [], + }, + }) + const createdContents: MockView['webContents'][] = [] + const resolveLoads: Array<(() => void) | undefined> = [] + session = freshSession( + win, + { + onTabCreated: (webContents) => { + const contents = webContents as unknown as MockView['webContents'] + const index = createdContents.push(contents) - 1 + contents.loadURL.mockImplementation( + () => + new Promise((resolve) => { + resolveLoads[index] = resolve + }) + ) + }, + }, + persistence + ) + + session.withBrowserScope('chat-restore-priority', () => { + session.restoreBrowserSession() + session.switchTab('7') + session.closeTab('5') + }) + expect(createdContents[6].loadURL).toHaveBeenCalledWith(tabs[6].url) + expect( + createdContents.slice(1, 4).some((contents) => contents.stop.mock.calls.length > 0) + ).toBe(true) + resolveLoads[1]?.() + expect(createdContents[4].loadURL).not.toHaveBeenCalled() + + resolveLoads[2]?.() + await vi.waitFor(() => { + expect(createdContents[5].loadURL).toHaveBeenCalledWith(tabs[5].url) + }) + expect(createdContents[4].loadURL).not.toHaveBeenCalled() + + resolveLoads[3]?.() + await vi.waitFor(() => { + expect(createdContents[1].loadURL).toHaveBeenCalledTimes(2) + }) + }) + + it('keeps a deferred restore intact when Back and Forward cannot move', () => { + const tabs = Array.from({ length: 6 }, (_, index) => ({ + url: `https://deferred-history-${index}.example/`, + pinned: false, + })) + const { persistence } = memoryBrowserPersistence({ + 'chat-deferred-history': { v: 1, tabs, activeIndex: 0, downloads: [] }, + }) + const createdContents: MockView['webContents'][] = [] + session = freshSession( + win, + { + onTabCreated: (webContents) => { + const contents = webContents as unknown as MockView['webContents'] + createdContents.push(contents) + contents.loadURL.mockImplementation(() => new Promise(() => {})) + }, + }, + persistence + ) + + session.withBrowserScope('chat-deferred-history', () => { + session.restoreBrowserSession() + const deferred = createdContents[5] as unknown as WebContents + expect(session.goBack(deferred)).toBe(false) + expect(session.goForward(deferred)).toBe(false) + session.switchTab('6') + }) + + expect(createdContents[5].loadURL).toHaveBeenCalledWith(tabs[5].url) + }) + + it('promotes a model-selected queued restore and waits for its exact load', async () => { + vi.useFakeTimers() + try { + const tabs = Array.from({ length: 7 }, (_, index) => ({ + url: `https://model-restore-${index}.example/`, + pinned: false, + })) + const { persistence } = memoryBrowserPersistence({ + 'chat-model-restore': { v: 1, tabs, activeIndex: 0, downloads: [] }, + }) + const createdContents: MockView['webContents'][] = [] + const resolveLoads: Array<(() => void) | undefined> = [] + session = freshSession( + win, + { + onTabCreated: (webContents) => { + const contents = webContents as unknown as MockView['webContents'] + const index = createdContents.push(contents) - 1 + contents.loadURL.mockImplementation( + () => + new Promise((resolve) => { + resolveLoads[index] = resolve + }) + ) + }, + }, + persistence + ) + + const selected = session.withBrowserScope('chat-model-restore', () => { + session.restoreBrowserSession() + return session.switchAutomationTab('7') + }) + let ready = false + const selection = session.withBrowserScope('chat-model-restore', () => + session.waitForPendingTabRestore(selected) + ) + void selection.then(() => { + ready = true + }) + + expect(createdContents[6].loadURL).toHaveBeenCalledWith(tabs[6].url) + expect( + createdContents.slice(1, 4).some((contents) => contents.stop.mock.calls.length > 0) + ).toBe(true) + expect(ready).toBe(false) + + await vi.advanceTimersByTimeAsync(15_000) + expect(createdContents[6].stop).not.toHaveBeenCalled() + expect(ready).toBe(false) + + resolveLoads[6]?.() + await expect(selection).resolves.toBe(true) + expect(ready).toBe(true) + } finally { + vi.useRealTimers() + } + }) + + it('extends an in-flight background restore without restarting its load', async () => { + vi.useFakeTimers() + try { + const tabs = Array.from({ length: 4 }, (_, index) => ({ + url: `https://active-restore-${index}.example/`, + pinned: false, + })) + const { persistence } = memoryBrowserPersistence({ + 'chat-active-restore': { v: 1, tabs, activeIndex: 0, downloads: [] }, + }) + const createdContents: MockView['webContents'][] = [] + const selectedLoads: Array<() => void> = [] + session = freshSession( + win, + { + onTabCreated: (webContents) => { + const contents = webContents as unknown as MockView['webContents'] + const index = createdContents.push(contents) - 1 + contents.loadURL.mockImplementation( + () => + new Promise((resolve) => { + if (index === 1) selectedLoads.push(resolve) + }) + ) + }, + }, + persistence + ) + + const selected = session.withBrowserScope('chat-active-restore', () => { + session.restoreBrowserSession() + return session.switchAutomationTab('2') + }) + const selection = session.withBrowserScope('chat-active-restore', () => + session.waitForPendingTabRestore(selected) + ) + + expect(createdContents[1].loadURL).toHaveBeenCalledOnce() + expect(createdContents[1].stop).not.toHaveBeenCalled() + await vi.advanceTimersByTimeAsync(15_000) + expect(createdContents[1].stop).not.toHaveBeenCalled() + + selectedLoads[0]?.() + await expect(selection).resolves.toBe(true) + } finally { + vi.useRealTimers() + } + }) + + it('queues a fifth foreground restore without preempting another foreground restore', async () => { + const snapshots = Object.fromEntries( + Array.from({ length: 5 }, (_, index) => [ + `chat-foreground-${index}`, + { + v: 1 as const, + tabs: [{ url: `https://foreground-${index}.example/`, pinned: false }], + activeIndex: 0, + downloads: [], + }, + ]) + ) + const { persistence } = memoryBrowserPersistence(snapshots) + const createdContents: MockView['webContents'][] = [] + const resolveLoads: Array<(() => void) | undefined> = [] + session = freshSession( + win, + { + onTabCreated: (webContents) => { + const contents = webContents as unknown as MockView['webContents'] + const index = createdContents.push(contents) - 1 + contents.loadURL.mockImplementation( + () => + new Promise((resolve) => { + resolveLoads[index] = resolve + }) + ) + }, + }, + persistence + ) + + for (let index = 0; index < 5; index += 1) { + session.withBrowserScope(`chat-foreground-${index}`, () => session.restoreBrowserSession()) + } + + expect( + createdContents.slice(0, 4).every((contents) => contents.loadURL.mock.calls.length === 1) + ).toBe(true) + expect(createdContents[4].loadURL).not.toHaveBeenCalled() + expect( + createdContents.slice(0, 4).every((contents) => contents.stop.mock.calls.length === 0) + ).toBe(true) + + resolveLoads[0]?.() + await vi.waitFor(() => { + expect(createdContents[4].loadURL).toHaveBeenCalledWith('https://foreground-4.example/') + }) + }) + + it('releases hung global restore slots so another task can make progress', async () => { + vi.useFakeTimers() + try { + const firstTabs = Array.from({ length: 6 }, (_, index) => ({ + url: `https://hung-a-${index}.example/`, + pinned: false, + })) + const secondTabs = Array.from({ length: 2 }, (_, index) => ({ + url: `https://waiting-b-${index}.example/`, + pinned: false, + })) + const { persistence } = memoryBrowserPersistence({ + 'chat-hung-a': { v: 1, tabs: firstTabs, activeIndex: 0, downloads: [] }, + 'chat-waiting-b': { v: 1, tabs: secondTabs, activeIndex: 0, downloads: [] }, + }) + const createdContents: MockView['webContents'][] = [] + session = freshSession( + win, + { + onTabCreated: (webContents) => { + const contents = webContents as unknown as MockView['webContents'] + createdContents.push(contents) + contents.loadURL.mockImplementation(() => new Promise(() => {})) + }, + }, + persistence + ) + + session.withBrowserScope('chat-hung-a', () => session.restoreBrowserSession()) + session.withBrowserScope('chat-waiting-b', () => session.restoreBrowserSession()) + const waitingBackground = createdContents[7] + expect(waitingBackground.loadURL).not.toHaveBeenCalled() + + await vi.advanceTimersByTimeAsync(15_000) + + expect( + createdContents.slice(1, 4).every((contents) => contents.stop.mock.calls.length > 0) + ).toBe(true) + expect(waitingBackground.loadURL).not.toHaveBeenCalled() + + await vi.advanceTimersByTimeAsync(15_000) + + expect(waitingBackground.loadURL).toHaveBeenCalledWith(secondTabs[1].url) + } finally { + vi.useRealTimers() + } + }) + + it('finishes a timed-out restore even when Electron throws while stopping it', async () => { + vi.useFakeTimers() + try { + const restoredUrl = 'https://throwing-stop.example/' + const { persistence } = memoryBrowserPersistence({ + 'chat-throwing-stop': { + v: 1, + tabs: [{ url: restoredUrl, pinned: false }], + activeIndex: 0, + downloads: [], + }, + }) + session = freshSession( + win, + { + onTabCreated: (webContents) => { + const contents = webContents as unknown as MockView['webContents'] + contents.loadURL.mockImplementation(() => new Promise(() => {})) + contents.stop.mockImplementationOnce(() => { + throw new Error('destroy race') + }) + }, + }, + persistence + ) + + session.withBrowserScope('chat-throwing-stop', () => session.restoreBrowserSession()) + await vi.advanceTimersByTimeAsync(20_000) + + const state = session.withBrowserScope('chat-throwing-stop', () => session.getTabsState()) + expect(state.tabs[0]).toMatchObject({ + url: restoredUrl, + loading: false, + issue: { kind: 'load-error', code: -7, description: 'ERR_TIMED_OUT' }, + }) + const contents = session.withBrowserScope( + 'chat-throwing-stop', + () => session.requireTab().view.webContents + ) + session.withBrowserScope('chat-throwing-stop', () => session.reloadPage(contents)) + expect(contents.loadURL).toHaveBeenLastCalledWith(restoredUrl) + } finally { + vi.useRealTimers() + } + }) + + it('gives a redirected background restore its complete site-decision window', async () => { + vi.useFakeTimers() + try { + const tabs = [ + { url: 'http://127.0.0.1:4601/active', pinned: false }, + { url: 'http://127.0.0.1:4601/background', pinned: false }, + ] + const { persistence } = memoryBrowserPersistence({ + 'chat-stale-restore-prompt': { v: 1, tabs, activeIndex: 0, downloads: [] }, + }) + const createdContents: MockView['webContents'][] = [] + session = freshSession( + win, + { + onTabCreated: (webContents) => { + const contents = webContents as unknown as MockView['webContents'] + createdContents.push(contents) + contents.loadURL.mockImplementation(() => new Promise(() => {})) + }, + }, + persistence + ) + + session.withBrowserScope('chat-stale-restore-prompt', () => session.restoreBrowserSession()) + session.activateBrowserScope('chat-stale-restore-prompt') + panel.setPanelBounds({ x: 100, y: 50, width: 800, height: 600 }) + const background = createdContents[1] + const redirected = beginMainFrameRequest(background, 'http://127.0.0.1:4602/redirect') + await vi.advanceTimersByTimeAsync(0) + expect( + session.withBrowserScope('chat-stale-restore-prompt', () => + session.sitePermissionRequestForScope() + ) + ).toMatchObject({ origin: 'http://127.0.0.1:4602' }) + + await vi.advanceTimersByTimeAsync(15_000) + + expect(background.stop).not.toHaveBeenCalled() + expect( + session.withBrowserScope('chat-stale-restore-prompt', () => + session.sitePermissionRequestForScope() + ) + ).toBeDefined() + + await vi.advanceTimersByTimeAsync(5_000) + + await expect(redirected).resolves.toEqual({ cancel: true }) + expect( + session.withBrowserScope('chat-stale-restore-prompt', () => + session.sitePermissionRequestForScope() + ) + ).toBeUndefined() + expect(background.stop).not.toHaveBeenCalled() + + await vi.advanceTimersByTimeAsync(15_000) + + expect(background.stop).toHaveBeenCalledOnce() + } finally { + vi.useRealTimers() + } + }) + + it('does not let repeated redirect prompts extend a restore without bound', async () => { + vi.useFakeTimers() + try { + const tabs = [ + { url: 'http://127.0.0.1:4611/active', pinned: false }, + { url: 'http://127.0.0.1:4611/background', pinned: false }, + ] + const { persistence } = memoryBrowserPersistence({ + 'chat-bounded-restore-prompt': { v: 1, tabs, activeIndex: 0, downloads: [] }, + }) + const createdContents: MockView['webContents'][] = [] + session = freshSession( + win, + { + onTabCreated: (webContents) => { + const contents = webContents as unknown as MockView['webContents'] + createdContents.push(contents) + contents.loadURL.mockImplementation(() => new Promise(() => {})) + }, + }, + persistence + ) + + session.withBrowserScope('chat-bounded-restore-prompt', () => session.restoreBrowserSession()) + session.activateBrowserScope('chat-bounded-restore-prompt') + panel.setPanelBounds({ x: 100, y: 50, width: 800, height: 600 }) + const background = createdContents[1] + const firstRedirect = beginMainFrameRequest(background, 'http://127.0.0.1:4612/first') + await vi.advanceTimersByTimeAsync(0) + expect( + session.withBrowserScope('chat-bounded-restore-prompt', () => + session.sitePermissionRequestForScope() + ) + ).toMatchObject({ origin: 'http://127.0.0.1:4612' }) + + await vi.advanceTimersByTimeAsync(20_000) + await expect(firstRedirect).resolves.toEqual({ cancel: true }) + panel.setPanelBounds({ x: 100, y: 50, width: 800, height: 600 }) + const secondRedirect = beginMainFrameRequest(background, 'http://127.0.0.1:4613/second', 2) + await vi.advanceTimersByTimeAsync(0) + expect( + session.withBrowserScope('chat-bounded-restore-prompt', () => + session.sitePermissionRequestForScope() + ) + ).toMatchObject({ origin: 'http://127.0.0.1:4613' }) + + await vi.advanceTimersByTimeAsync(15_000) + + expect(background.stop).toHaveBeenCalledOnce() + await expect(secondRedirect).resolves.toEqual({ cancel: true }) + expect( + session.withBrowserScope('chat-bounded-restore-prompt', () => + session.sitePermissionRequestForScope() + ) + ).toBeUndefined() + } finally { + vi.useRealTimers() + } + }) + + it('discards a queued restore before an explicit replacement navigation can race it', async () => { + const tabs = Array.from({ length: 6 }, (_, index) => ({ + url: `https://stale-restore-${index}.example/`, + pinned: false, + })) + const { persistence } = memoryBrowserPersistence({ + 'chat-replace-restore': { v: 1, tabs, activeIndex: 0, downloads: [] }, + }) + const createdContents: MockView['webContents'][] = [] + const resolveLoads: Array<(() => void) | undefined> = [] + session = freshSession( + win, + { + onTabCreated: (webContents) => { + const contents = webContents as unknown as MockView['webContents'] + const index = createdContents.push(contents) - 1 + contents.loadURL.mockImplementation( + () => + new Promise((resolve) => { + resolveLoads[index] = resolve + }) + ) + }, + }, + persistence + ) + + session.withBrowserScope('chat-replace-restore', () => session.restoreBrowserSession()) + const queued = createdContents[5] + const replacement = 'https://fresh.example/' + session.withBrowserScope('chat-replace-restore', () => { + session.prepareExplicitNavigation(queued as unknown as WebContents) + }) + void (queued.loadURL as unknown as (url: string) => Promise)(replacement) + resolveLoads[1]?.() + await Promise.resolve() + await Promise.resolve() + + expect(queued.loadURL).toHaveBeenCalledOnce() + expect(queued.loadURL).toHaveBeenCalledWith(replacement) + expect(queued.loadURL).not.toHaveBeenCalledWith(tabs[5].url) + }) + + it('does not start queued restores after their task browser is suspended', async () => { + const tabs = Array.from({ length: 6 }, (_, index) => ({ + url: `https://suspended-${index}.example/`, + pinned: false, + })) + const { persistence } = memoryBrowserPersistence({ + 'chat-restore-suspended': { + v: 1, + tabs, + activeIndex: 0, + downloads: [], + }, + }) + const createdContents: MockView['webContents'][] = [] + const resolveLoads: Array<(() => void) | undefined> = [] + session = freshSession( + win, + { + onTabCreated: (webContents) => { + const contents = webContents as unknown as MockView['webContents'] + const index = createdContents.push(contents) - 1 + contents.loadURL.mockImplementation( + () => + new Promise((resolve) => { + resolveLoads[index] = resolve + }) + ) + }, + }, + persistence + ) + + session.withBrowserScope('chat-restore-suspended', () => session.restoreBrowserSession()) + expect( + createdContents.filter((contents) => contents.loadURL.mock.calls.length > 0) + ).toHaveLength(4) + + session.suspendBrowserScope('chat-restore-suspended') + resolveLoads[1]?.() + await Promise.resolve() + await Promise.resolve() + + expect( + createdContents.filter((contents) => contents.loadURL.mock.calls.length > 0) + ).toHaveLength(4) + expect(createdContents.every((contents) => contents.close.mock.calls.length === 1)).toBe(true) + }) + it('restores more than eight persisted tabs', () => { const tabs = Array.from({ length: 12 }, (_, index) => ({ url: `https://tab-${index}.example/`, @@ -2074,6 +2870,27 @@ describe('browser-agent session', () => { expect(onTabCreated).toHaveBeenLastCalledWith(userTab?.view.webContents) }) + it('does not treat an untrusted page popup as user authorization for its origin', async () => { + panel.setPanelBounds({ x: 100, y: 50, width: 800, height: 600 }) + const source = (session.ensureTab().view as unknown as MockView).webContents + const openWindow = source.setWindowOpenHandler.mock.calls[0]?.[0] as (details: { + url: string + }) => { action: string } + const destination = 'http://127.0.0.1:4099/private?token=secret' + + openWindow({ url: destination }) + const popup = (session.activeTab()?.view as unknown as MockView).webContents + const request = beginMainFrameRequest(popup, destination) + + await vi.waitFor(() => + expect(session.sitePermissionRequestForScope()).toMatchObject({ + origin: 'http://127.0.0.1:4099', + }) + ) + session.respondToSitePermission(session.sitePermissionRequestForScope()?.requestId ?? '', false) + await expect(request).resolves.toEqual({ cancel: true }) + }) + it('blocks controlled pages from moving or resizing the desktop window', () => { const tab = session.ensureTab() const contents = (tab.view as unknown as MockView).webContents @@ -2359,14 +3176,327 @@ describe('browser-agent session', () => { } }) - it('leaves nothing of the signed-out user behind in the browser profile', async () => { - const clearStorageData = vi.fn(async () => {}) - const clearCache = vi.fn(async () => {}) - vi.mocked(electronSession.fromPartition).mockReturnValue({ - clearStorageData, - clearCache, - } as unknown as ReturnType) - const { persistence, snapshots } = memoryBrowserPersistence() + it('holds a new top-level origin for an exact task-scoped user decision', async () => { + panel.setPanelBounds({ x: 100, y: 50, width: 800, height: 600 }) + const contents = (session.ensureTab().view as unknown as MockView).webContents + const first = beginMainFrameRequest( + contents, + 'http://127.0.0.1:4101/private?token=secret#fragment' + ) + + await vi.waitFor(() => { + expect(session.sitePermissionRequestForScope()).toMatchObject({ + tabId: '1', + origin: 'http://127.0.0.1:4101', + }) + }) + const prompt = session.sitePermissionRequestForScope() + expect(prompt).not.toHaveProperty('url') + expect(win.focus).toHaveBeenCalled() + expect(win.webContents.focus).toHaveBeenCalled() + expect(session.respondToSitePermission(prompt?.requestId ?? '', true)).toBe(true) + await expect(first).resolves.toEqual({ cancel: false }) + + await expect( + beginMainFrameRequest(contents, 'http://127.0.0.1:4101/another?different=secret', 2) + ).resolves.toEqual({ cancel: false }) + expect(session.sitePermissionRequestForScope()).toBeUndefined() + + const otherOrigin = beginMainFrameRequest(contents, 'http://127.0.0.1:4102/', 3) + await vi.waitFor(() => expect(session.sitePermissionRequestForScope()).toBeDefined()) + expect(session.respondToSitePermission('not-the-live-request', true)).toBe(false) + const otherPrompt = session.sitePermissionRequestForScope() + expect(session.respondToSitePermission(otherPrompt?.requestId ?? '', false)).toBe(true) + await expect(otherOrigin).resolves.toEqual({ cancel: true }) + }) + + it('allows an SSRF-checked agent destination without granting a cross-origin redirect', async () => { + panel.setPanelBounds({ x: 100, y: 50, width: 800, height: 600 }) + const contents = (session.ensureTab().view as unknown as MockView).webContents + const destination = 'http://127.0.0.1:4111/agent-path?token=secret' + + expect( + session.grantSiteOriginForAgentNavigation(contents as unknown as WebContents, destination) + ).toBe(true) + await expect(beginMainFrameRequest(contents, destination)).resolves.toEqual({ cancel: false }) + expect(session.sitePermissionRequestForScope()).toBeUndefined() + + const redirect = beginMainFrameRequest(contents, 'http://127.0.0.1:4112/redirected', 2) + await vi.waitFor(() => + expect(session.sitePermissionRequestForScope()).toMatchObject({ + origin: 'http://127.0.0.1:4112', + }) + ) + const prompt = session.sitePermissionRequestForScope() + expect(session.respondToSitePermission(prompt?.requestId ?? '', false)).toBe(true) + await expect(redirect).resolves.toEqual({ cancel: true }) + }) + + it('uses a native exact-origin prompt when the active renderer lacks prompt support', async () => { + session = freshSession(win, { + sitePermissionPromptSupported: vi.fn(() => false), + }) + vi.mocked(dialog.showMessageBox).mockResolvedValueOnce({ + response: 1, + checkboxChecked: false, + }) + panel.setPanelBounds({ x: 100, y: 50, width: 800, height: 600 }) + const contents = (session.ensureTab().view as unknown as MockView).webContents + + const request = beginMainFrameRequest( + contents, + 'http://127.0.0.1:4151/private?token=secret#fragment' + ) + + await expect(request).resolves.toEqual({ cancel: false }) + expect(dialog.showMessageBox).toHaveBeenCalledWith( + win, + expect.objectContaining({ + buttons: ['Block', 'Allow'], + defaultId: 0, + cancelId: 0, + message: 'Allow this browser task to open http://127.0.0.1:4151?', + }) + ) + expect(JSON.stringify(vi.mocked(dialog.showMessageBox).mock.lastCall)).not.toContain('secret') + expect(session.sitePermissionRequestForScope()).toBeUndefined() + }) + + it('attaches the native fallback to the window that owns the visible panel', async () => { + const panelOwner = mainWindowMock() + session = freshSession(win, { + sitePermissionPromptSupported: vi.fn(() => false), + }) + vi.mocked(dialog.showMessageBox).mockResolvedValueOnce({ + response: 0, + checkboxChecked: false, + }) + panel.setPanelBounds({ x: 100, y: 50, width: 800, height: 600 }, panelOwner) + const contents = (session.ensureTab().view as unknown as MockView).webContents + + await expect(beginMainFrameRequest(contents, 'http://127.0.0.1:4155/private')).resolves.toEqual( + { cancel: true } + ) + + expect(dialog.showMessageBox).toHaveBeenCalledWith(panelOwner, expect.any(Object)) + }) + + it('denies a new site prompt immediately when its scope is hidden or inactive', async () => { + const hiddenContents = (session.ensureTab().view as unknown as MockView).webContents + + await expect( + beginMainFrameRequest(hiddenContents, 'http://127.0.0.1:4156/hidden') + ).resolves.toEqual({ cancel: true }) + expect(session.sitePermissionRequestForScope()).toBeUndefined() + + panel.setPanelBounds({ x: 100, y: 50, width: 800, height: 600 }) + const inactiveContents = session.withBrowserScope( + 'chat-inactive', + () => session.ensureTab().view as unknown as MockView + ).webContents + await expect( + beginMainFrameRequest(inactiveContents, 'http://127.0.0.1:4157/inactive') + ).resolves.toEqual({ cancel: true }) + expect( + session.withBrowserScope('chat-inactive', () => session.sitePermissionRequestForScope()) + ).toBeUndefined() + }) + + it('does not show the native fallback when the active renderer owns the prompt', async () => { + vi.mocked(dialog.showMessageBox).mockClear() + panel.setPanelBounds({ x: 100, y: 50, width: 800, height: 600 }) + const contents = (session.ensureTab().view as unknown as MockView).webContents + const request = beginMainFrameRequest(contents, 'http://127.0.0.1:4152/docs') + + await vi.waitFor(() => expect(session.sitePermissionRequestForScope()).toBeDefined()) + + expect(dialog.showMessageBox).not.toHaveBeenCalled() + const prompt = session.sitePermissionRequestForScope() + expect(session.respondToSitePermission(prompt?.requestId ?? '', false)).toBe(true) + await expect(request).resolves.toEqual({ cancel: true }) + }) + + it('revalidates a native allow decision after the held request becomes stale', async () => { + session = freshSession(win, { + sitePermissionPromptSupported: vi.fn(() => false), + }) + vi.mocked(dialog.showMessageBox).mockClear() + let answerPrompt: ((result: { response: number; checkboxChecked: boolean }) => void) | undefined + vi.mocked(dialog.showMessageBox).mockImplementationOnce( + () => + new Promise((resolve) => { + answerPrompt = resolve + }) + ) + panel.setPanelBounds({ x: 100, y: 50, width: 800, height: 600 }) + const contents = (session.ensureTab().view as unknown as MockView).webContents + const request = beginMainFrameRequest(contents, 'http://127.0.0.1:4153/held') + await vi.waitFor(() => expect(dialog.showMessageBox).toHaveBeenCalled()) + const signal = vi.mocked(dialog.showMessageBox).mock.lastCall?.at(-1)?.signal + expect(signal?.aborted).toBe(false) + + mainFrameNavigationStarted(contents, false, 'http://127.0.0.1:4154/replacement') + await expect(request).resolves.toEqual({ cancel: true }) + expect(signal?.aborted).toBe(true) + answerPrompt?.({ response: 1, checkboxChecked: false }) + + const retried = beginMainFrameRequest(contents, 'http://127.0.0.1:4153/retried', 2) + await expect(retried).resolves.toEqual({ cancel: true }) + expect(dialog.showMessageBox).toHaveBeenCalledTimes(2) + }) + + it('keeps the held request alive through its own navigation-start event', async () => { + panel.setPanelBounds({ x: 100, y: 50, width: 800, height: 600 }) + const contents = (session.ensureTab().view as unknown as MockView).webContents + const destination = 'http://127.0.0.1:4201/docs' + const request = beginMainFrameRequest(contents, destination) + await vi.waitFor(() => expect(session.sitePermissionRequestForScope()).toBeDefined()) + + mainFrameNavigationStarted(contents, false, `${destination}#section`) + const prompt = session.sitePermissionRequestForScope() + expect(prompt).toBeDefined() + expect(session.respondToSitePermission(prompt?.requestId ?? '', true)).toBe(true) + await expect(request).resolves.toEqual({ cancel: false }) + + const replaced = beginMainFrameRequest(contents, 'http://127.0.0.1:4202/', 2) + await vi.waitFor(() => expect(session.sitePermissionRequestForScope()).toBeDefined()) + mainFrameNavigationStarted(contents, false, 'http://127.0.0.1:4203/') + await expect(replaced).resolves.toEqual({ cancel: true }) + expect(session.sitePermissionRequestForScope()).toBeUndefined() + }) + + it('invalidates a held site decision before an explicit replacement navigation', async () => { + panel.setPanelBounds({ x: 100, y: 50, width: 800, height: 600 }) + const contents = (session.ensureTab().view as unknown as MockView).webContents + const held = beginMainFrameRequest(contents, 'http://127.0.0.1:4204/held') + await vi.waitFor(() => expect(session.sitePermissionRequestForScope()).toBeDefined()) + const requestId = session.sitePermissionRequestForScope()?.requestId + + session.prepareExplicitNavigation(contents as unknown as WebContents) + + await expect(held).resolves.toEqual({ cancel: true }) + expect(session.sitePermissionRequestForScope()).toBeUndefined() + expect(session.respondToSitePermission(requestId ?? '', true)).toBe(false) + }) + + it('seeds restored origins before loading while still holding a new redirect origin', async () => { + const restoredUrl = 'http://127.0.0.1:4301/restored?private=value' + const { persistence } = memoryBrowserPersistence({ + 'chat-test': { + v: 1, + tabs: [{ url: restoredUrl, pinned: true }], + activeIndex: 0, + downloads: [], + }, + }) + session = freshSession(win, {}, persistence) + panel.setPanelBounds({ x: 100, y: 50, width: 800, height: 600 }) + session.restoreBrowserSession() + const contents = (session.requireTab().view as unknown as MockView).webContents + expect(contents.loadURL).toHaveBeenCalledWith(restoredUrl) + + await expect(beginMainFrameRequest(contents, restoredUrl)).resolves.toEqual({ cancel: false }) + expect(session.sitePermissionRequestForScope()).toBeUndefined() + + const redirected = beginMainFrameRequest(contents, 'http://127.0.0.1:4302/login', 2) + await vi.waitFor(() => + expect(session.sitePermissionRequestForScope()).toMatchObject({ + origin: 'http://127.0.0.1:4302', + }) + ) + const prompt = session.sitePermissionRequestForScope() + session.respondToSitePermission(prompt?.requestId ?? '', false) + await expect(redirected).resolves.toEqual({ cancel: true }) + }) + + it('bounds task grants and fails closed when a main-frame request cannot map to a live tab', async () => { + panel.setPanelBounds({ x: 100, y: 50, width: 800, height: 600 }) + const contents = (session.ensureTab().view as unknown as MockView).webContents + for (let index = 0; index <= 64; index += 1) { + expect( + session.grantSiteOriginForUserNavigation( + contents as unknown as WebContents, + `http://127.0.0.1:${4400 + index}/private` + ) + ).toBe(true) + } + + const evicted = beginMainFrameRequest(contents, 'http://127.0.0.1:4400/again') + await vi.waitFor(() => + expect(session.sitePermissionRequestForScope()).toMatchObject({ + origin: 'http://127.0.0.1:4400', + }) + ) + session.respondToSitePermission(session.sitePermissionRequestForScope()?.requestId ?? '', false) + await expect(evicted).resolves.toEqual({ cancel: true }) + + const handler = contents.session.webRequest.onBeforeRequest.mock.calls[0]?.[0] + const unmapped = new Promise<{ cancel: boolean }>((resolve) => { + handler( + { + id: 99, + url: 'http://127.0.0.1:4499/', + method: 'GET', + resourceType: 'mainFrame', + referrer: '', + timestamp: Date.now(), + uploadData: [], + }, + resolve + ) + }) + await expect(unmapped).resolves.toEqual({ cancel: true }) + }) + + it('blocks an image hostname that resolves to a private address', async () => { + mockLookup.mockResolvedValue([{ address: '10.0.0.5', family: 4 }]) + const contents = (session.ensureTab().view as unknown as MockView).webContents + + await expect( + beginSubresourceRequest(contents, 'https://private-image.evil.example/status.png', 'image') + ).resolves.toEqual({ cancel: true }) + expect(mockLookup).toHaveBeenCalledWith('private-image.evil.example', { + all: true, + verbatim: true, + }) + }) + + it('default-denies pending site requests on timeout, tab close, and stale-document approval', async () => { + vi.useFakeTimers() + try { + panel.setPanelBounds({ x: 100, y: 50, width: 800, height: 600 }) + const tab = session.ensureTab() + const contents = (tab.view as unknown as MockView).webContents + const timedOut = beginMainFrameRequest(contents, 'http://127.0.0.1:4501/') + await vi.advanceTimersByTimeAsync(0) + await vi.advanceTimersByTimeAsync(20_000) + await expect(timedOut).resolves.toEqual({ cancel: true }) + + panel.setPanelBounds({ x: 100, y: 50, width: 800, height: 600 }) + const stale = beginMainFrameRequest(contents, 'http://127.0.0.1:4502/', 2) + await vi.advanceTimersByTimeAsync(0) + const stalePrompt = session.sitePermissionRequestForScope() + contents.getURL.mockReturnValue('https://changed.example/') + expect(session.respondToSitePermission(stalePrompt?.requestId ?? '', true)).toBe(true) + await expect(stale).resolves.toEqual({ cancel: true }) + + const closing = beginMainFrameRequest(contents, 'http://127.0.0.1:4503/', 3) + await vi.advanceTimersByTimeAsync(0) + session.closeTab(tab.id) + await expect(closing).resolves.toEqual({ cancel: true }) + } finally { + vi.useRealTimers() + } + }) + + it('leaves nothing of the signed-out user behind in the browser profile', async () => { + const clearStorageData = vi.fn(async () => {}) + const clearCache = vi.fn(async () => {}) + vi.mocked(electronSession.fromPartition).mockReturnValue({ + clearStorageData, + clearCache, + } as unknown as ReturnType) + const { persistence, snapshots } = memoryBrowserPersistence() session = freshSession(win, {}, persistence) panel.setPanelBounds({ x: 0, y: 0, width: 800, height: 600 }) @@ -2520,12 +3650,13 @@ describe('browser-agent session', () => { } }) - it('saves downloads to the configured folder instead of cancelling them', () => { + it('pauses downloads until the async disk check passes, then saves them', async () => { const directory = mkdtempSync(join(tmpdir(), 'sim-browser-downloads-')) const { persistence, snapshots } = memoryBrowserPersistence() const onDownloadsChanged = vi.fn() session = freshSession(win, { onDownloadsChanged }, persistence, { getDirectory: () => directory, + getFreeDiskBytes: () => Number.MAX_SAFE_INTEGER, }) const contents = (session.ensureTab().view as unknown as MockView).webContents const webSession = contents.session as typeof contents.session & { @@ -2542,6 +3673,8 @@ describe('browser-agent session', () => { getReceivedBytes: vi.fn(() => 20), getTotalBytes: vi.fn(() => 100), setSavePath: vi.fn(), + pause: vi.fn(), + resume: vi.fn(), cancel: vi.fn(), on: vi.fn(), once: vi.fn(), @@ -2549,6 +3682,9 @@ describe('browser-agent session', () => { willDownload?.({}, item, contents) + expect(item.pause).toHaveBeenCalledOnce() + expect(item.resume).not.toHaveBeenCalled() + await vi.waitFor(() => expect(item.resume).toHaveBeenCalledOnce()) expect(item.cancel).not.toHaveBeenCalled() expect(item.setSavePath).toHaveBeenCalledWith(join(directory, 'report.csv')) expect(item.once).toHaveBeenCalledWith('done', expect.any(Function)) @@ -2592,7 +3728,10 @@ describe('browser-agent session', () => { reveal?.() expect(shell.showItemInFolder).toHaveBeenCalledWith(join(directory, 'report.csv')) - session = freshSession(win, {}, persistence, { getDirectory: () => directory }) + session = freshSession(win, {}, persistence, { + getDirectory: () => directory, + getFreeDiskBytes: () => Number.MAX_SAFE_INTEGER, + }) session.restoreBrowserSession() expect(session.getBrowserDownloadsState('chat-test').downloads[0]).toMatchObject({ filename: 'report.csv', @@ -2600,10 +3739,736 @@ describe('browser-agent session', () => { }) }) + it('does not let a pre-allocation progress event consume the admission probe', async () => { + const directory = mkdtempSync(join(tmpdir(), 'sim-browser-downloads-')) + const getFreeDiskBytes = vi.fn(() => Number.MAX_SAFE_INTEGER) + session = freshSession(win, {}, undefined, { + getDirectory: () => directory, + getFreeDiskBytes, + }) + const contents = (session.ensureTab().view as unknown as MockView).webContents + const download = mockDownloadItem({ filename: 'early-progress.bin', totalBytes: 100 }) + + startMockDownload(contents, download) + download.emitUpdated() + + await vi.waitFor(() => expect(download.item.resume).toHaveBeenCalledOnce()) + expect(getFreeDiskBytes).toHaveBeenCalledOnce() + expect(download.item.cancel).not.toHaveBeenCalled() + }) + + it('rejects a declared download above the byte cap with safe visible metadata', () => { + const directory = mkdtempSync(join(tmpdir(), 'sim-browser-downloads-')) + session = freshSession(win, {}, undefined, { + getDirectory: () => directory, + getFreeDiskBytes: () => Number.MAX_SAFE_INTEGER, + }) + const contents = (session.ensureTab().view as unknown as MockView).webContents + const download = mockDownloadItem({ + filename: 'oversized.zip', + totalBytes: 2 * 1024 ** 3 + 1, + }) + + startMockDownload(contents, download) + + expect(download.item.cancel).toHaveBeenCalledOnce() + expect(download.item.setSavePath).not.toHaveBeenCalled() + expect(session.getBrowserDownloadsState('chat-test').downloads).toEqual([ + expect.objectContaining({ filename: 'oversized.zip', state: 'interrupted' }), + ]) + expect(session.getBrowserDownloadsState('chat-test').downloads[0]).not.toHaveProperty( + 'savePath' + ) + expect(session.getBrowserDownloadsState('chat-test').downloads[0]).not.toHaveProperty( + 'interruptionReason' + ) + + vi.mocked(Menu.buildFromTemplate).mockClear() + session.showBrowserDownloadsMenu('chat-test', win, { x: 10, y: 20 }) + const template = vi.mocked(Menu.buildFromTemplate).mock.calls[0]?.[0] as + | MenuItemConstructorOptions[] + | undefined + expect(template?.[0]).toMatchObject({ + label: 'oversized.zip', + enabled: false, + }) + expect(template?.[0]?.sublabel).toContain('2.0 GB download limit') + }) + + it('reserves a known download remaining size above the free-disk floor', async () => { + const directory = mkdtempSync(join(tmpdir(), 'sim-browser-downloads-')) + const getFreeDiskBytes = vi.fn(() => 1.2 * 1024 ** 3) + session = freshSession(win, {}, undefined, { + getDirectory: () => directory, + getFreeDiskBytes, + }) + const contents = (session.ensureTab().view as unknown as MockView).webContents + const download = mockDownloadItem({ + filename: 'known-size.iso', + totalBytes: 1.5 * 1024 ** 3, + }) + + startMockDownload(contents, download) + + expect(download.item.pause).toHaveBeenCalledOnce() + await vi.waitFor(() => expect(getFreeDiskBytes).toHaveBeenCalledOnce()) + await vi.waitFor(() => expect(download.item.cancel).toHaveBeenCalledOnce()) + expect(download.item.resume).not.toHaveBeenCalled() + expect(session.getBrowserDownloadsState('chat-test').downloads[0]).toMatchObject({ + filename: 'known-size.iso', + state: 'interrupted', + }) + }) + + it('fails closed when the asynchronous free-space probe rejects', async () => { + const directory = mkdtempSync(join(tmpdir(), 'sim-browser-downloads-')) + session = freshSession(win, {}, undefined, { + getDirectory: () => directory, + getFreeDiskBytes: () => Promise.reject(new Error('disk unavailable')), + }) + const contents = (session.ensureTab().view as unknown as MockView).webContents + const download = mockDownloadItem({ filename: 'probe-error.bin', totalBytes: 100 }) + + startMockDownload(contents, download) + + expect(download.item.pause).toHaveBeenCalledOnce() + await vi.waitFor(() => expect(download.item.cancel).toHaveBeenCalledOnce()) + expect(download.item.resume).not.toHaveBeenCalled() + expect(session.getBrowserDownloadsState('chat-test').downloads[0]).toMatchObject({ + filename: 'probe-error.bin', + state: 'interrupted', + }) + }) + + it('fails a hung admission probe closed and ignores its late rejection', async () => { + vi.useFakeTimers() + try { + const directory = mkdtempSync(join(tmpdir(), 'sim-browser-downloads-')) + const probe = deferred() + const getFreeDiskBytes = vi.fn(() => probe.promise) + session = freshSession(win, {}, undefined, { + getDirectory: () => directory, + getFreeDiskBytes, + }) + const contents = (session.ensureTab().view as unknown as MockView).webContents + const download = mockDownloadItem({ filename: 'hung-admission.bin', totalBytes: 100 }) + + startMockDownload(contents, download) + await vi.waitFor(() => expect(getFreeDiskBytes).toHaveBeenCalledOnce()) + expect(download.item.resume).not.toHaveBeenCalled() + const timersDuringProbe = vi.getTimerCount() + + await vi.advanceTimersByTimeAsync(5_000) + + expect(download.item.cancel).toHaveBeenCalledOnce() + expect(download.item.resume).not.toHaveBeenCalled() + expect(vi.getTimerCount()).toBe(timersDuringProbe - 1) + probe.reject(new Error('late disk failure')) + await vi.advanceTimersByTimeAsync(0) + expect(download.item.cancel).toHaveBeenCalledOnce() + expect(download.item.resume).not.toHaveBeenCalled() + } finally { + vi.useRealTimers() + } + }) + + it('fails a hung progress probe closed instead of disabling later disk checks', async () => { + vi.useFakeTimers() + try { + const directory = mkdtempSync(join(tmpdir(), 'sim-browser-downloads-')) + const progressProbe = deferred() + const getFreeDiskBytes = vi + .fn<(directory: string) => number | Promise>() + .mockReturnValueOnce(Number.MAX_SAFE_INTEGER) + .mockReturnValueOnce(progressProbe.promise) + session = freshSession(win, {}, undefined, { + getDirectory: () => directory, + getFreeDiskBytes, + }) + const contents = (session.ensureTab().view as unknown as MockView).webContents + const download = mockDownloadItem({ filename: 'hung-progress.bin' }) + + startMockDownload(contents, download) + await vi.waitFor(() => expect(download.item.resume).toHaveBeenCalledOnce()) + await vi.advanceTimersByTimeAsync(1_000) + download.emitUpdated() + expect(getFreeDiskBytes).toHaveBeenCalledTimes(2) + + await vi.advanceTimersByTimeAsync(5_000) + + expect(download.item.cancel).toHaveBeenCalledOnce() + progressProbe.resolve(Number.MAX_SAFE_INTEGER) + await vi.advanceTimersByTimeAsync(0) + expect(download.item.cancel).toHaveBeenCalledOnce() + } finally { + vi.useRealTimers() + } + }) + + it('fails hung path allocation closed without reserving a late destination', async () => { + vi.useFakeTimers() + try { + const directory = mkdtempSync(join(tmpdir(), 'sim-browser-downloads-')) + const firstProbe = deferred() + const secondProbe = deferred() + const pathExists = vi + .fn<(path: string) => boolean | Promise>() + .mockReturnValueOnce(firstProbe.promise) + .mockReturnValueOnce(secondProbe.promise) + .mockReturnValue(false) + session = freshSession(win, {}, undefined, { + getDirectory: () => directory, + getFreeDiskBytes: () => Number.MAX_SAFE_INTEGER, + pathExists, + }) + const contents = (session.ensureTab().view as unknown as MockView).webContents + const first = mockDownloadItem({ filename: 'hung-path.bin', totalBytes: 100 }) + const second = mockDownloadItem({ filename: 'hung-path.bin', totalBytes: 100 }) + + startMockDownload(contents, first) + startMockDownload(contents, second) + expect(pathExists).toHaveBeenCalledTimes(2) + const timersDuringAllocation = vi.getTimerCount() + + await vi.advanceTimersByTimeAsync(5_000) + + expect(first.item.cancel).toHaveBeenCalledOnce() + expect(second.item.cancel).toHaveBeenCalledOnce() + expect(vi.getTimerCount()).toBe(timersDuringAllocation - 2) + + firstProbe.resolve(false) + secondProbe.reject(new Error('late path lookup failure')) + await vi.advanceTimersByTimeAsync(0) + expect(first.item.setSavePath).not.toHaveBeenCalled() + expect(second.item.setSavePath).not.toHaveBeenCalled() + expect(first.item.resume).not.toHaveBeenCalled() + expect(second.item.resume).not.toHaveBeenCalled() + + first.emitDone('cancelled') + second.emitDone('cancelled') + const replacement = mockDownloadItem({ filename: 'hung-path.bin', totalBytes: 100 }) + startMockDownload(contents, replacement) + await vi.waitFor(() => + expect(replacement.item.setSavePath).toHaveBeenCalledWith(join(directory, 'hung-path.bin')) + ) + await vi.waitFor(() => expect(replacement.item.resume).toHaveBeenCalledOnce()) + } finally { + vi.useRealTimers() + } + }) + + it('counts slow pending admissions against the per-task concurrency cap', async () => { + const directory = mkdtempSync(join(tmpdir(), 'sim-browser-downloads-')) + const firstProbe = deferred() + const secondProbe = deferred() + const getFreeDiskBytes = vi + .fn<(directory: string) => Promise>() + .mockReturnValueOnce(firstProbe.promise) + .mockReturnValueOnce(secondProbe.promise) + session = freshSession(win, {}, undefined, { + getDirectory: () => directory, + getFreeDiskBytes, + }) + const contents = (session.ensureTab().view as unknown as MockView).webContents + const first = mockDownloadItem({ filename: 'pending-a.bin', totalBytes: 100 }) + const second = mockDownloadItem({ filename: 'pending-b.bin', totalBytes: 100 }) + const blocked = mockDownloadItem({ filename: 'blocked.bin', totalBytes: 100 }) + + startMockDownload(contents, first) + startMockDownload(contents, second) + startMockDownload(contents, blocked) + + expect(first.item.pause).toHaveBeenCalledOnce() + expect(second.item.pause).toHaveBeenCalledOnce() + expect(blocked.item.pause).not.toHaveBeenCalled() + expect(blocked.item.setSavePath).not.toHaveBeenCalled() + expect(blocked.item.cancel).toHaveBeenCalledOnce() + await vi.waitFor(() => expect(getFreeDiskBytes).toHaveBeenCalledTimes(2)) + + firstProbe.resolve(Number.MAX_SAFE_INTEGER) + secondProbe.resolve(Number.MAX_SAFE_INTEGER) + await vi.waitFor(() => expect(first.item.resume).toHaveBeenCalledOnce()) + await vi.waitFor(() => expect(second.item.resume).toHaveBeenCalledOnce()) + }) + + it('does not construct or probe a save path for a synchronously rejected item', () => { + const unusableDirectory = Symbol('must not reach path construction') as unknown as string + session = freshSession(win, {}, undefined, { + getDirectory: () => unusableDirectory, + getFreeDiskBytes: () => Number.MAX_SAFE_INTEGER, + }) + const contents = (session.ensureTab().view as unknown as MockView).webContents + const download = mockDownloadItem({ + filename: 'too-large.bin', + totalBytes: 2 * 1024 ** 3 + 1, + }) + + expect(() => startMockDownload(contents, download)).not.toThrow() + expect(download.item.cancel).toHaveBeenCalledOnce() + expect(download.item.setSavePath).not.toHaveBeenCalled() + expect(download.item.pause).not.toHaveBeenCalled() + }) + + it('fails closed when the configured download directory cannot contain a file', async () => { + const directory = mkdtempSync(join(tmpdir(), 'sim-browser-downloads-')) + const notDirectory = join(directory, 'ordinary-file') + writeFileSync(notDirectory, 'not a directory') + session = freshSession(win, {}, undefined, { + getDirectory: () => notDirectory, + getFreeDiskBytes: () => Number.MAX_SAFE_INTEGER, + }) + const contents = (session.ensureTab().view as unknown as MockView).webContents + const download = mockDownloadItem({ filename: 'cannot-save.bin', totalBytes: 100 }) + + startMockDownload(contents, download) + + await vi.waitFor(() => expect(download.item.cancel).toHaveBeenCalledOnce()) + expect(download.item.setSavePath).not.toHaveBeenCalled() + expect(download.item.resume).not.toHaveBeenCalled() + expect(session.getBrowserDownloadsState('chat-test').downloads[0]).toMatchObject({ + filename: 'cannot-save.bin', + state: 'interrupted', + }) + }) + + it('reserves active downloads across different folders on the same disk', async () => { + const firstDirectory = mkdtempSync(join(tmpdir(), 'sim-browser-downloads-a-')) + const secondDirectory = mkdtempSync(join(tmpdir(), 'sim-browser-downloads-b-')) + let directory = firstDirectory + session = freshSession(win, {}, undefined, { + getDirectory: () => directory, + getFreeDiskBytes: () => 4 * 1024 ** 3, + }) + const contents = (session.ensureTab().view as unknown as MockView).webContents + const first = mockDownloadItem({ filename: 'first.iso', totalBytes: 2 * 1024 ** 3 }) + const second = mockDownloadItem({ filename: 'second.iso', totalBytes: 2 * 1024 ** 3 }) + + startMockDownload(contents, first) + directory = secondDirectory + startMockDownload(contents, second) + + await vi.waitFor(() => expect(first.item.resume).toHaveBeenCalledOnce()) + await vi.waitFor(() => expect(second.item.cancel).toHaveBeenCalledOnce()) + expect(first.item.cancel).not.toHaveBeenCalled() + expect(second.item.resume).not.toHaveBeenCalled() + }) + + it('stops an unknown-size download immediately when its received bytes cross the cap', async () => { + const directory = mkdtempSync(join(tmpdir(), 'sim-browser-downloads-')) + session = freshSession(win, {}, undefined, { + getDirectory: () => directory, + getFreeDiskBytes: () => Number.MAX_SAFE_INTEGER, + }) + const contents = (session.ensureTab().view as unknown as MockView).webContents + const download = mockDownloadItem({ filename: 'stream.bin' }) + + startMockDownload(contents, download) + await vi.waitFor(() => expect(download.item.resume).toHaveBeenCalledOnce()) + const firstSavePath = download.item.setSavePath.mock.calls[0]?.[0] + download.setReceivedBytes(2 * 1024 ** 3 + 1) + download.emitUpdated() + + expect(download.item.cancel).toHaveBeenCalledOnce() + expect(session.getBrowserDownloadsState('chat-test').downloads[0]).toMatchObject({ + filename: 'stream.bin', + state: 'interrupted', + receivedBytes: 2 * 1024 ** 3 + 1, + }) + + download.emitDone('cancelled') + const replacement = mockDownloadItem({ filename: 'stream.bin', totalBytes: 100 }) + startMockDownload(contents, replacement) + expect(replacement.item.cancel).not.toHaveBeenCalled() + await vi.waitFor(() => expect(replacement.item.setSavePath).toHaveBeenCalledWith(firstSavePath)) + }) + + it('throttles free-disk checks while stopping promptly after the interval', async () => { + vi.useFakeTimers() + try { + vi.setSystemTime(new Date('2026-08-31T00:00:00.000Z')) + const directory = mkdtempSync(join(tmpdir(), 'sim-browser-downloads-')) + let freeDiskBytes = 4 * 1024 ** 3 + let lastProbeAt = 0 + const getFreeDiskBytes = vi.fn(() => { + lastProbeAt = Date.now() + return freeDiskBytes + }) + session = freshSession(win, {}, undefined, { + getDirectory: () => directory, + getFreeDiskBytes, + }) + const contents = (session.ensureTab().view as unknown as MockView).webContents + const download = mockDownloadItem({ filename: 'unknown-size.bin' }) + + startMockDownload(contents, download) + await vi.waitFor(() => expect(download.item.resume).toHaveBeenCalledOnce()) + expect(getFreeDiskBytes).toHaveBeenCalledOnce() + vi.setSystemTime(lastProbeAt) + freeDiskBytes = 512 * 1024 ** 2 + + download.setReceivedBytes(10) + download.emitUpdated() + vi.advanceTimersByTime(999) + download.setReceivedBytes(20) + download.emitUpdated() + expect(getFreeDiskBytes).toHaveBeenCalledOnce() + expect(download.item.cancel).not.toHaveBeenCalled() + + vi.advanceTimersByTime(1) + download.setReceivedBytes(30) + download.emitUpdated() + expect(getFreeDiskBytes).toHaveBeenCalledTimes(2) + await vi.advanceTimersByTimeAsync(0) + expect(download.item.cancel).toHaveBeenCalledOnce() + expect(session.getBrowserDownloadsState('chat-test').downloads[0]).toMatchObject({ + state: 'interrupted', + receivedBytes: 30, + }) + } finally { + vi.useRealTimers() + } + }) + + it('coalesces progress probes while a free-space check is still in flight', async () => { + vi.useFakeTimers() + try { + vi.setSystemTime(new Date('2026-08-31T00:00:00.000Z')) + const directory = mkdtempSync(join(tmpdir(), 'sim-browser-downloads-')) + const progressProbe = deferred() + const getFreeDiskBytes = vi + .fn<(directory: string) => number | Promise>() + .mockReturnValueOnce(Number.MAX_SAFE_INTEGER) + .mockReturnValueOnce(progressProbe.promise) + session = freshSession(win, {}, undefined, { + getDirectory: () => directory, + getFreeDiskBytes, + }) + const contents = (session.ensureTab().view as unknown as MockView).webContents + const download = mockDownloadItem({ filename: 'coalesced.bin' }) + + startMockDownload(contents, download) + await vi.waitFor(() => expect(download.item.resume).toHaveBeenCalledOnce()) + + await vi.advanceTimersByTimeAsync(1_000) + download.emitUpdated() + expect(getFreeDiskBytes).toHaveBeenCalledTimes(2) + await vi.advanceTimersByTimeAsync(2_000) + download.emitUpdated() + download.emitUpdated() + expect(getFreeDiskBytes).toHaveBeenCalledTimes(2) + + progressProbe.resolve(Number.MAX_SAFE_INTEGER) + await vi.advanceTimersByTimeAsync(0) + expect(download.item.cancel).not.toHaveBeenCalled() + } finally { + vi.useRealTimers() + } + }) + + it('ignores a late admission sample after the item reaches a terminal state', async () => { + const directory = mkdtempSync(join(tmpdir(), 'sim-browser-downloads-')) + const probe = deferred() + session = freshSession(win, {}, undefined, { + getDirectory: () => directory, + getFreeDiskBytes: () => probe.promise, + }) + const contents = (session.ensureTab().view as unknown as MockView).webContents + const download = mockDownloadItem({ filename: 'finished-before-probe.bin', totalBytes: 100 }) + + startMockDownload(contents, download) + expect(download.item.pause).toHaveBeenCalledOnce() + download.emitDone('cancelled') + probe.resolve(Number.MAX_SAFE_INTEGER) + await vi.waitFor(() => expect(download.item.resume).not.toHaveBeenCalled()) + + const replacement = mockDownloadItem({ filename: 'replacement.bin', totalBytes: 100 }) + startMockDownload(contents, replacement) + expect(replacement.item.cancel).not.toHaveBeenCalled() + }) + + it('cancels every active download on profile wipe without reviving late work', async () => { + const directory = mkdtempSync(join(tmpdir(), 'sim-browser-downloads-')) + const firstProbe = deferred() + const secondProbe = deferred() + const { persistence, snapshots } = memoryBrowserPersistence() + const getFreeDiskBytes = vi + .fn<(directory: string) => number | Promise>() + .mockReturnValueOnce(firstProbe.promise) + .mockReturnValueOnce(secondProbe.promise) + .mockReturnValue(Number.MAX_SAFE_INTEGER) + session = freshSession(win, {}, persistence, { + getDirectory: () => directory, + getFreeDiskBytes, + }) + const contents = (session.ensureTab().view as unknown as MockView).webContents + const first = mockDownloadItem({ filename: 'same-name.bin', totalBytes: 100 }) + const second = mockDownloadItem({ filename: 'same-name.bin', totalBytes: 100 }) + + startMockDownload(contents, first) + startMockDownload(contents, second) + await vi.waitFor(() => expect(getFreeDiskBytes).toHaveBeenCalledTimes(2)) + const allocatedPaths = [ + first.item.setSavePath.mock.calls[0]?.[0], + second.item.setSavePath.mock.calls[0]?.[0], + ] + expect(new Set(allocatedPaths).size).toBe(2) + expect(allocatedPaths).toContain(join(directory, 'same-name.bin')) + + await session.clearProfileStorage() + + expect(first.item.cancel).toHaveBeenCalledOnce() + expect(second.item.cancel).toHaveBeenCalledOnce() + expect(session.getBrowserDownloadsState('chat-test').downloads).toEqual([]) + + firstProbe.resolve(Number.MAX_SAFE_INTEGER) + secondProbe.reject(new Error('late profile probe rejection')) + await Promise.resolve() + await Promise.resolve() + expect(first.item.resume).not.toHaveBeenCalled() + expect(second.item.resume).not.toHaveBeenCalled() + expect(session.getBrowserDownloadsState('chat-test').downloads).toEqual([]) + expect(snapshots.get('chat-test')?.downloads).toEqual([]) + + const nextContents = (session.ensureTab().view as unknown as MockView).webContents + const replacement = mockDownloadItem({ filename: 'same-name.bin', totalBytes: 100 }) + startMockDownload(nextContents, replacement) + await vi.waitFor(() => + expect(replacement.item.setSavePath).toHaveBeenCalledWith(join(directory, 'same-name.bin')) + ) + await vi.waitFor(() => expect(replacement.item.resume).toHaveBeenCalledOnce()) + expect(replacement.item.cancel).not.toHaveBeenCalled() + + first.emitDone('completed') + second.emitDone('cancelled') + const concurrent = mockDownloadItem({ filename: 'same-name.bin', totalBytes: 100 }) + startMockDownload(nextContents, concurrent) + await vi.waitFor(() => expect(concurrent.item.setSavePath).toHaveBeenCalledOnce()) + expect(concurrent.item.setSavePath).not.toHaveBeenCalledWith(join(directory, 'same-name.bin')) + }) + + it('does not reserve a late filename after profile teardown starts', async () => { + const directory = mkdtempSync(join(tmpdir(), 'sim-browser-downloads-')) + session = freshSession(win, {}, undefined, { + getDirectory: () => directory, + getFreeDiskBytes: () => Number.MAX_SAFE_INTEGER, + }) + const contents = (session.ensureTab().view as unknown as MockView).webContents + const download = mockDownloadItem({ filename: 'teardown-race.bin', totalBytes: 100 }) + + startMockDownload(contents, download) + await session.clearProfileStorage() + await Promise.resolve() + + expect(download.item.cancel).toHaveBeenCalledOnce() + expect(download.item.setSavePath).not.toHaveBeenCalled() + expect(download.item.resume).not.toHaveBeenCalled() + }) + + it('does not let a cancelled allocation release another download path owner', async () => { + const directory = mkdtempSync(join(tmpdir(), 'sim-browser-downloads-')) + const firstPathProbe = deferred() + const secondPathProbe = deferred() + const pathExists = vi + .fn<(path: string) => boolean | Promise>() + .mockReturnValueOnce(firstPathProbe.promise) + .mockReturnValueOnce(secondPathProbe.promise) + .mockReturnValue(false) + session = freshSession(win, {}, undefined, { + getDirectory: () => directory, + getFreeDiskBytes: () => Number.MAX_SAFE_INTEGER, + pathExists, + }) + const firstContents = session.withBrowserScope( + 'chat-first', + () => (session.ensureTab().view as unknown as MockView).webContents + ) + const secondContents = session.withBrowserScope( + 'chat-second', + () => (session.ensureTab().view as unknown as MockView).webContents + ) + const thirdContents = session.withBrowserScope( + 'chat-third', + () => (session.ensureTab().view as unknown as MockView).webContents + ) + const first = mockDownloadItem({ filename: 'shared.bin', totalBytes: 100 }) + const second = mockDownloadItem({ filename: 'shared.bin', totalBytes: 100 }) + + startMockDownload(firstContents, first) + startMockDownload(secondContents, second) + firstPathProbe.resolve(false) + queueMicrotask(() => session.disposeBrowserScope('chat-first')) + secondPathProbe.resolve(false) + + await vi.waitFor(() => + expect(second.item.setSavePath).toHaveBeenCalledWith(join(directory, 'shared.bin')) + ) + expect(first.item.setSavePath).not.toHaveBeenCalled() + + const third = mockDownloadItem({ filename: 'shared.bin', totalBytes: 100 }) + startMockDownload(thirdContents, third) + await vi.waitFor(() => expect(third.item.setSavePath).toHaveBeenCalledOnce()) + expect(third.item.setSavePath).not.toHaveBeenCalledWith(join(directory, 'shared.bin')) + }) + + it('cancels only the disposed scope and ignores its late download callbacks', async () => { + const directory = mkdtempSync(join(tmpdir(), 'sim-browser-downloads-')) + const disposedPathProbe = deferred() + const onDownloadsChanged = vi.fn() + session = freshSession(win, { onDownloadsChanged }, undefined, { + getDirectory: () => directory, + getFreeDiskBytes: () => Number.MAX_SAFE_INTEGER, + pathExists: (path) => + path.endsWith('disposed.bin') ? disposedPathProbe.promise : Promise.resolve(false), + }) + const disposedContents = session.withBrowserScope( + 'chat-disposed', + () => (session.ensureTab().view as unknown as MockView).webContents + ) + const retainedContents = session.withBrowserScope( + 'chat-retained', + () => (session.ensureTab().view as unknown as MockView).webContents + ) + const disposedDownload = mockDownloadItem({ filename: 'disposed.bin', totalBytes: 100 }) + const retainedDownload = mockDownloadItem({ filename: 'retained.bin', totalBytes: 100 }) + + startMockDownload(disposedContents, disposedDownload) + startMockDownload(retainedContents, retainedDownload) + await vi.waitFor(() => expect(retainedDownload.item.resume).toHaveBeenCalledOnce()) + onDownloadsChanged.mockClear() + + session.disposeBrowserScope('chat-disposed') + + expect(disposedDownload.item.cancel).toHaveBeenCalledOnce() + expect(retainedDownload.item.cancel).not.toHaveBeenCalled() + disposedPathProbe.resolve(false) + await Promise.resolve() + await Promise.resolve() + disposedDownload.emitUpdated() + disposedDownload.emitDone('cancelled') + + expect(disposedDownload.item.setSavePath).not.toHaveBeenCalled() + expect(disposedDownload.item.resume).not.toHaveBeenCalled() + expect(session.getBrowserDownloadsState('chat-disposed').downloads).toEqual([]) + expect(onDownloadsChanged).not.toHaveBeenCalledWith( + expect.objectContaining({ scopeId: 'chat-disposed' }) + ) + retainedDownload.emitUpdated() + expect(onDownloadsChanged).toHaveBeenCalledWith( + expect.objectContaining({ scopeId: 'chat-retained' }) + ) + }) + + it('cancels only the suspended scope and cannot republish it after reactivation', async () => { + const directory = mkdtempSync(join(tmpdir(), 'sim-browser-downloads-')) + const suspendedDiskProbe = deferred() + const onDownloadsChanged = vi.fn() + const getFreeDiskBytes = vi + .fn<() => number | Promise>() + .mockReturnValueOnce(suspendedDiskProbe.promise) + .mockReturnValue(Number.MAX_SAFE_INTEGER) + session = freshSession(win, { onDownloadsChanged }, undefined, { + getDirectory: () => directory, + getFreeDiskBytes, + }) + const suspendedContents = session.withBrowserScope( + 'chat-suspended', + () => (session.ensureTab().view as unknown as MockView).webContents + ) + const retainedContents = session.withBrowserScope( + 'chat-retained', + () => (session.ensureTab().view as unknown as MockView).webContents + ) + const suspendedDownload = mockDownloadItem({ filename: 'suspended.bin', totalBytes: 100 }) + const retainedDownload = mockDownloadItem({ filename: 'retained.bin', totalBytes: 100 }) + + startMockDownload(suspendedContents, suspendedDownload) + startMockDownload(retainedContents, retainedDownload) + await vi.waitFor(() => expect(suspendedDownload.item.setSavePath).toHaveBeenCalledOnce()) + await vi.waitFor(() => expect(retainedDownload.item.resume).toHaveBeenCalledOnce()) + onDownloadsChanged.mockClear() + + expect(session.suspendBrowserScope('chat-suspended')).toBe(true) + expect(suspendedDownload.item.cancel).toHaveBeenCalledOnce() + expect(retainedDownload.item.cancel).not.toHaveBeenCalled() + session.activateBrowserScope('chat-suspended') + onDownloadsChanged.mockClear() + + suspendedDiskProbe.resolve(Number.MAX_SAFE_INTEGER) + await Promise.resolve() + await Promise.resolve() + suspendedDownload.emitUpdated() + suspendedDownload.emitDone('cancelled') + + expect(suspendedDownload.item.resume).not.toHaveBeenCalled() + expect(session.getBrowserDownloadsState('chat-suspended').downloads).toEqual([]) + expect(onDownloadsChanged).not.toHaveBeenCalledWith( + expect.objectContaining({ scopeId: 'chat-suspended' }) + ) + retainedDownload.emitUpdated() + expect(onDownloadsChanged).toHaveBeenCalledWith( + expect.objectContaining({ scopeId: 'chat-retained' }) + ) + }) + + it('bounds active downloads per task and releases the slot on completion', async () => { + const directory = mkdtempSync(join(tmpdir(), 'sim-browser-downloads-')) + session = freshSession(win, {}, undefined, { + getDirectory: () => directory, + getFreeDiskBytes: () => Number.MAX_SAFE_INTEGER, + }) + const contents = (session.ensureTab().view as unknown as MockView).webContents + const first = mockDownloadItem({ filename: 'first.txt', totalBytes: 100 }) + const second = mockDownloadItem({ filename: 'second.txt', totalBytes: 100 }) + const rejected = mockDownloadItem({ filename: 'third.txt', totalBytes: 100 }) + + startMockDownload(contents, first) + startMockDownload(contents, second) + startMockDownload(contents, rejected) + expect(first.item.cancel).not.toHaveBeenCalled() + expect(second.item.cancel).not.toHaveBeenCalled() + expect(rejected.item.cancel).toHaveBeenCalledOnce() + + first.emitDone('completed') + const replacement = mockDownloadItem({ filename: 'fourth.txt', totalBytes: 100 }) + startMockDownload(contents, replacement) + expect(replacement.item.cancel).not.toHaveBeenCalled() + await vi.waitFor(() => expect(replacement.item.setSavePath).toHaveBeenCalledOnce()) + }) + + it('bounds active browser downloads across tasks', () => { + const directory = mkdtempSync(join(tmpdir(), 'sim-browser-downloads-')) + session = freshSession(win, {}, undefined, { + getDirectory: () => directory, + getFreeDiskBytes: () => Number.MAX_SAFE_INTEGER, + }) + for (let scopeIndex = 0; scopeIndex < 3; scopeIndex++) { + session.withBrowserScope(`chat-download-${scopeIndex}`, () => { + const contents = (session.ensureTab().view as unknown as MockView).webContents + startMockDownload(contents, mockDownloadItem({ filename: `${scopeIndex}-a.txt` })) + startMockDownload(contents, mockDownloadItem({ filename: `${scopeIndex}-b.txt` })) + }) + } + const blocked = mockDownloadItem({ filename: 'global-overflow.txt' }) + session.withBrowserScope('chat-download-overflow', () => { + const contents = (session.ensureTab().view as unknown as MockView).webContents + startMockDownload(contents, blocked) + }) + + expect(blocked.item.cancel).toHaveBeenCalledOnce() + expect(blocked.item.setSavePath).not.toHaveBeenCalled() + expect(session.getBrowserDownloadsState('chat-download-overflow').downloads[0]).toMatchObject({ + filename: 'global-overflow.txt', + state: 'interrupted', + }) + }) + it('does not recreate a disposed scope when a download finishes later', () => { const directory = mkdtempSync(join(tmpdir(), 'sim-browser-downloads-')) const { persistence, snapshots } = memoryBrowserPersistence() - session = freshSession(win, {}, persistence, { getDirectory: () => directory }) + session = freshSession(win, {}, persistence, { + getDirectory: () => directory, + getFreeDiskBytes: () => Number.MAX_SAFE_INTEGER, + }) const contents = (session.ensureTab().view as unknown as MockView).webContents const webSession = contents.session as typeof contents.session & { on: ReturnType @@ -2619,6 +4484,8 @@ describe('browser-agent session', () => { getReceivedBytes: vi.fn(() => 4), getTotalBytes: vi.fn(() => 4), setSavePath: vi.fn(), + pause: vi.fn(), + resume: vi.fn(), cancel: vi.fn(), on: vi.fn(), once: vi.fn(), diff --git a/apps/desktop/src/main/browser-agent/session.ts b/apps/desktop/src/main/browser-agent/session.ts index 9d5c041a700..9bedd83f938 100644 --- a/apps/desktop/src/main/browser-agent/session.ts +++ b/apps/desktop/src/main/browser-agent/session.ts @@ -1,5 +1,6 @@ import { AsyncLocalStorage } from 'node:async_hooks' import { existsSync } from 'node:fs' +import { statfs } from 'node:fs/promises' import { join } from 'node:path' import type { BrowserDataKind, @@ -9,6 +10,7 @@ import type { BrowserMediaPermissionRequest, BrowserOmniboxFocusMode, BrowserPageIssue, + BrowserSitePermissionRequest, BrowserTabState, BrowserTabsState, BrowserTheme, @@ -34,6 +36,7 @@ import type { } from 'electron' import { app, + dialog, session as electronSession, Menu, nativeTheme, @@ -89,12 +92,14 @@ export interface AgentTab { view: WebContentsView pinned: boolean pendingRestoreUrl?: string + pendingRestore?: PendingTabRestore pageIssue?: BrowserPageIssue syntheticForward?: { url: string; baseHistoryIndex: number } preserveSyntheticForwardOnNextNavigation?: boolean recoveringUnresponsive?: boolean pendingMediaPermission?: PendingMediaPermission mediaPermissionGrant?: MediaPermissionGrant + pendingSitePermission?: PendingSitePermission lastRealUserGestureAt?: number } @@ -110,6 +115,19 @@ interface MediaPermissionGrant { devices: Set } +interface PendingSitePermission { + request: BrowserSitePermissionRequest + /** Exact committed document from which the suspended request originated. */ + documentUrl: string + /** Exact destination retained only in main-process memory for receipt validation. */ + destinationUrl: string + contents: WebContents + networkRequestId: number + resolve: (allowed: boolean) => void + timeout: ReturnType + nativePromptController?: AbortController +} + export interface BrowserSessionPersistence { load: (scopeId: string) => BrowserSessionSnapshot | null save: (scopeId: string, snapshot: BrowserSessionSnapshot) => boolean @@ -120,6 +138,10 @@ export interface BrowserSessionPersistence { export interface BrowserDownloadSettings { /** Resolves the current destination when a download starts. */ getDirectory: () => string + /** Overrides the destination filesystem's available-byte lookup. */ + getFreeDiskBytes?: (directory: string) => number | Promise + /** Overrides asynchronous destination collision checks. */ + pathExists?: (path: string) => boolean | Promise } export interface AgentSessionEvents { @@ -140,6 +162,8 @@ export interface AgentSessionEvents { onActiveTabChanged: (contents: WebContents) => void /** The active tab's recoverable page state changed without a navigation. */ onPageStateChanged: (contents: WebContents) => void + /** Whether the current app renderer can present and answer a site-origin prompt. */ + sitePermissionPromptSupported: (scopeId: string) => boolean /** The tab list or active tab changed. */ onTabsChanged: () => void /** Sim's appearance preference changed for an existing tab. */ @@ -158,8 +182,27 @@ export interface AgentSessionEvents { const MAX_RECENTLY_CLOSED_TABS = 10 const MAX_LIVE_TABS_PER_SCOPE = 32 const MAX_LIVE_TABS_GLOBAL = 96 +/** + * Admission reserves active downloads' worst-case remaining bytes so concurrent + * downloads cannot collectively consume the disk floor; unknown sizes reserve + * the per-file cap. + */ +const MAX_BROWSER_DOWNLOAD_BYTES = 2 * 1024 ** 3 +const MAX_ACTIVE_BROWSER_DOWNLOADS_PER_SCOPE = 2 +const MAX_ACTIVE_BROWSER_DOWNLOADS_GLOBAL = 6 +const MIN_BROWSER_DOWNLOAD_FREE_DISK_BYTES = 1024 ** 3 +const BROWSER_DOWNLOAD_DISK_CHECK_INTERVAL_MS = 1_000 +const BROWSER_DOWNLOAD_DISK_CHECK_TIMEOUT_MS = 5_000 +const BROWSER_DOWNLOAD_PATH_ALLOCATION_TIMEOUT_MS = 5_000 +/** One foreground reservation keeps a selected tab responsive under background restore load. */ +const MAX_TAB_RESTORE_CONCURRENCY = 4 +const MAX_BACKGROUND_TAB_RESTORE_CONCURRENCY = 3 +const BACKGROUND_TAB_RESTORE_TIMEOUT_MS = 15_000 +const FOREGROUND_TAB_RESTORE_TIMEOUT_MS = 20_000 const MEDIA_PERMISSION_GESTURE_WINDOW_MS = 10_000 const MEDIA_PERMISSION_PROMPT_TIMEOUT_MS = 30_000 +const SITE_PERMISSION_PROMPT_TIMEOUT_MS = 20_000 +const MAX_SITE_ORIGIN_GRANTS_PER_SCOPE = 64 export type BrowserShortcut = 'focus-omnibox' | 'new-tab' | 'close-tab' | 'find' @@ -227,6 +270,8 @@ interface BrowserScopeState { */ findingTabId: string | null findingRequestId: number | null + /** Memory-bounded, task-local origins explicitly reached or approved by the user. */ + siteOriginGrants: Map } function createBrowserScopeState(): BrowserScopeState { @@ -247,6 +292,7 @@ function createBrowserScopeState(): BrowserScopeState { automationNeedsAttention: false, findingTabId: null, findingRequestId: null, + siteOriginGrants: new Map(), } } @@ -365,13 +411,51 @@ let browserTheme: BrowserTheme = 'system' let browserAppTheme: BrowserTheme = 'system' let browserAppearanceTheme: DesktopAppearanceTheme = 'app' let browserDefaultZoom: DesktopZoomPercent = 100 -const activeDownloadPaths = new Set() -type TrackedBrowserDownload = BrowserDownloadInfo & { savePath: string } +type TrackedBrowserDownload = BrowserDownloadInfo & { + savePath?: string + interruptionReason?: string +} type BrowserFinishedDownload = Omit & { state: Exclude + savePath: string +} + +interface ActiveBrowserDownload { + directory: string + download: TrackedBrowserDownload + item: DownloadItem + diskCheckInFlight: boolean + lastDiskCheckAt: number + savePath?: string + scopeId: string + terminal: boolean + limitReason?: string +} + +const activeDownloadPaths = new Map() + +interface PendingTabRestore { + generation: number + tab: AgentTab + url: string + priority: 'foreground' | 'background' + ready: Promise + resolveReady: (loaded: boolean) => void + started: boolean + settled: boolean + requeueAfterPreemption: boolean + cancelLoad?: () => void + grantSitePermissionGrace?: () => void + promoteToForeground?: () => void } const browserDownloadsByScope = new Map() +const activeBrowserDownloads = new Set() +const pendingForegroundTabRestores: PendingTabRestore[] = [] +const pendingBackgroundTabRestores: PendingTabRestore[] = [] +const activeTabRestores = new Set() +const activeBackgroundTabRestores = new Set() +let backgroundTabRestoreGeneration = 0 /** Mirrors the compact recent-downloads panel used by mainstream browsers. */ const MAX_RECENT_FINISHED_DOWNLOADS = 5 @@ -381,7 +465,7 @@ function browserDownloadsState(scopeId: string): BrowserDownloadsState { return { scopeId: resolved, downloads: (browserDownloadsByScope.get(resolved) ?? []).map( - ({ savePath: _savePath, ...item }) => ({ ...item }) + ({ savePath: _savePath, interruptionReason: _interruptionReason, ...item }) => ({ ...item }) ), } } @@ -409,10 +493,252 @@ function updateDownloadProgress(download: BrowserDownloadInfo, item: DownloadIte download.totalBytes = Math.max(0, item.getTotalBytes()) } +function activeBrowserDownloadCount(scopeId?: string): number { + if (!scopeId) return activeBrowserDownloads.size + const resolved = resolveBrowserScopeId(scopeId) + let count = 0 + for (const active of activeBrowserDownloads) { + if (resolveBrowserScopeId(active.scopeId) === resolved) count += 1 + } + return count +} + +function withBrowserDownloadTimeout( + operation: Promise, + timeoutMs: number, + timeoutMessage: string, + onTimeout?: () => void +): Promise { + let timeout: ReturnType | undefined + const expiry = new Promise((_resolve, reject) => { + timeout = setTimeout(() => { + onTimeout?.() + reject(new Error(timeoutMessage)) + }, timeoutMs) + }) + return Promise.race([operation, expiry]).finally(() => clearTimeout(timeout)) +} + +async function browserDownloadFreeDiskBytes(directory: string): Promise { + try { + const configured = browserDownloadSettings?.getFreeDiskBytes?.(directory) + const lookup = + configured === undefined + ? statfs(directory).then((stats) => stats.bavail * stats.bsize) + : Promise.resolve(configured) + const available = await withBrowserDownloadTimeout( + lookup, + BROWSER_DOWNLOAD_DISK_CHECK_TIMEOUT_MS, + 'Browser download disk-space check timed out' + ) + if (!Number.isFinite(available) || available < 0) return null + return Math.min(Number.MAX_SAFE_INTEGER, Math.floor(available)) + } catch (error) { + logger.warn('Could not determine free disk space for agent browser download', { + error: getErrorMessage(error), + }) + return null + } +} + +function browserDownloadSizeLimitReason(): string { + return `Stopped: exceeds the ${formatBrowserDownloadBytes(MAX_BROWSER_DOWNLOAD_BYTES)} download limit` +} + +function browserDownloadDiskLimitReason(): string { + return `Stopped: not enough disk space to finish safely while keeping ${formatBrowserDownloadBytes(MIN_BROWSER_DOWNLOAD_FREE_DISK_BYTES)} free` +} + +function downloadRemainingReservation(item: DownloadItem): number { + const receivedBytes = Math.max(0, item.getReceivedBytes()) + const totalBytes = Math.max(0, item.getTotalBytes()) + const targetBytes = totalBytes > 0 ? totalBytes : MAX_BROWSER_DOWNLOAD_BYTES + return Math.max(0, targetBytes - receivedBytes) +} + +function activeDownloadReservations(through?: ActiveBrowserDownload): number { + let reservedBytes = 0 + for (const active of activeBrowserDownloads) { + if (!active.limitReason) { + reservedBytes += downloadRemainingReservation(active.item) + } + if (active === through) break + } + return reservedBytes +} + +function browserDownloadAdmissionReason(scopeId: string, item: DownloadItem): string | null { + if (Math.max(0, item.getTotalBytes()) > MAX_BROWSER_DOWNLOAD_BYTES) { + return browserDownloadSizeLimitReason() + } + if (activeBrowserDownloadCount(scopeId) >= MAX_ACTIVE_BROWSER_DOWNLOADS_PER_SCOPE) { + return `Stopped: this task already has ${MAX_ACTIVE_BROWSER_DOWNLOADS_PER_SCOPE} downloads in progress` + } + if (activeBrowserDownloadCount() >= MAX_ACTIVE_BROWSER_DOWNLOADS_GLOBAL) { + return `Stopped: Sim already has ${MAX_ACTIVE_BROWSER_DOWNLOADS_GLOBAL} browser downloads in progress` + } + return null +} + +function browserDownloadSizeLimitReasonForItem(item: DownloadItem): string | null { + if ( + Math.max(0, item.getReceivedBytes()) > MAX_BROWSER_DOWNLOAD_BYTES || + Math.max(0, item.getTotalBytes()) > MAX_BROWSER_DOWNLOAD_BYTES + ) { + return browserDownloadSizeLimitReason() + } + return null +} + +function createTrackedBrowserDownload( + item: DownloadItem, + state: BrowserDownloadInfo['state'], + interruptionReason?: string +): TrackedBrowserDownload { + const filename = suggestedFilename(item.getFilename(), item.getMimeType()) + return { + id: generateId(), + filename, + state, + receivedBytes: Math.max(0, item.getReceivedBytes()), + totalBytes: Math.max(0, item.getTotalBytes()), + startedAt: new Date().toISOString(), + interruptionReason, + } +} + +function recordBrowserDownload(scopeId: string, download: TrackedBrowserDownload): void { + browserDownloadsByScope.set(scopeId, [download, ...(browserDownloadsByScope.get(scopeId) ?? [])]) + trimBrowserDownloads(scopeId) + publishBrowserDownloads(scopeId) +} + +function cancelBrowserDownloadForLimit(active: ActiveBrowserDownload, reason: string): void { + if (active.limitReason) return + active.limitReason = reason + active.download.interruptionReason = reason + active.download.state = 'interrupted' + try { + active.item.cancel() + } catch (error) { + logger.warn('Could not cancel an agent browser download after a safety limit', { + error: getErrorMessage(error), + }) + } +} + +function publishActiveBrowserDownload(active: ActiveBrowserDownload): void { + const liveScopeId = resolveBrowserScopeId(active.scopeId) + if ( + suspendedBrowserScopes.has(liveScopeId) || + !browserScopeStates.has(liveScopeId) || + !browserDownloadsByScope.get(liveScopeId)?.includes(active.download) + ) { + return + } + publishBrowserDownloads(liveScopeId) +} + +function checkBrowserDownloadDiskSpace( + active: ActiveBrowserDownload, + check: 'admission' | 'progress', + now = Date.now() +): void { + if (active.terminal || active.limitReason || active.diskCheckInFlight) return + if ( + check === 'progress' && + now - active.lastDiskCheckAt < BROWSER_DOWNLOAD_DISK_CHECK_INTERVAL_MS + ) { + return + } + + active.lastDiskCheckAt = now + active.diskCheckInFlight = true + void browserDownloadFreeDiskBytes(active.directory) + .then((freeDiskBytes) => { + if (active.terminal || active.limitReason || !activeBrowserDownloads.has(active)) { + return + } + const requiredFreeDiskBytes = + MIN_BROWSER_DOWNLOAD_FREE_DISK_BYTES + + activeDownloadReservations(check === 'admission' ? active : undefined) + if (freeDiskBytes === null) { + cancelBrowserDownloadForLimit(active, 'Stopped: available disk space could not be checked') + publishActiveBrowserDownload(active) + return + } + if (freeDiskBytes < requiredFreeDiskBytes) { + cancelBrowserDownloadForLimit(active, browserDownloadDiskLimitReason()) + publishActiveBrowserDownload(active) + return + } + if (check === 'admission' && active.download.state === 'progressing') active.item.resume() + }) + .catch((error) => { + if (active.terminal || active.limitReason || !activeBrowserDownloads.has(active)) return + logger.warn('Could not complete an agent browser download disk-space check', { + error: getErrorMessage(error), + }) + cancelBrowserDownloadForLimit(active, 'Stopped: available disk space could not be checked') + publishActiveBrowserDownload(active) + }) + .finally(() => { + active.diskCheckInFlight = false + }) +} + +function releaseActiveBrowserDownload(active: ActiveBrowserDownload): void { + if (active.terminal) return + active.terminal = true + activeBrowserDownloads.delete(active) + releaseActiveBrowserDownloadPath(active) +} + +function releaseActiveBrowserDownloadPath( + active: ActiveBrowserDownload, + savePath = active.savePath +): void { + if (savePath && activeDownloadPaths.get(savePath) === active) { + activeDownloadPaths.delete(savePath) + } +} + +function cancelActiveBrowserDownloads(scopeId?: string): void { + const resolvedScopeId = scopeId === undefined ? null : resolveBrowserScopeId(scopeId) + const downloads = [...activeBrowserDownloads].filter( + (active) => + resolvedScopeId === null || resolveBrowserScopeId(active.scopeId) === resolvedScopeId + ) + for (const active of downloads) releaseActiveBrowserDownload(active) + const cancelledDownloads = new Set(downloads.map((active) => active.download)) + for (const [downloadScopeId, trackedDownloads] of browserDownloadsByScope) { + if (resolvedScopeId !== null && resolveBrowserScopeId(downloadScopeId) !== resolvedScopeId) { + continue + } + const retainedDownloads = trackedDownloads.filter( + (download) => !cancelledDownloads.has(download) + ) + if (retainedDownloads.length > 0) { + browserDownloadsByScope.set(downloadScopeId, retainedDownloads) + } else { + browserDownloadsByScope.delete(downloadScopeId) + } + } + for (const active of downloads) { + try { + active.item.cancel() + } catch (error) { + logger.warn('Could not cancel an active browser download while tearing down the session', { + error: getErrorMessage(error), + }) + } + } +} + function isFinishedBrowserDownload( download: TrackedBrowserDownload ): download is BrowserFinishedDownload { - return download.state !== 'progressing' + return download.state !== 'progressing' && typeof download.savePath === 'string' } /** Returns safe metadata only; local paths stay in the Electron main process. */ @@ -429,14 +755,16 @@ function formatBrowserDownloadBytes(bytes: number): string { return `${value >= 10 || unitIndex === 0 ? value.toFixed(0) : value.toFixed(1)} ${units[unitIndex]}` } -function downloadMenuDetail(download: BrowserDownloadInfo): string { +function downloadMenuDetail(download: TrackedBrowserDownload): string { const received = formatBrowserDownloadBytes(download.receivedBytes) if (download.state === 'progressing') { return download.totalBytes > 0 ? `${received} / ${formatBrowserDownloadBytes(download.totalBytes)}` : `${received} · Downloading` } - if (download.state === 'interrupted') return `${received} · Failed` + if (download.state === 'interrupted') { + return `${received} · ${download.interruptionReason ?? 'Failed'}` + } if (download.state === 'cancelled') return `${received} · Cancelled` return received } @@ -454,7 +782,10 @@ export function showBrowserDownloadsMenu( downloads.length === 0 ? [{ label: 'No downloads yet', enabled: false }] : downloads.map((download) => { - const revealable = download.state === 'completed' && existsSync(download.savePath) + const revealable = + download.state === 'completed' && + typeof download.savePath === 'string' && + existsSync(download.savePath) return { label: download.filename, sublabel: downloadMenuDetail(download), @@ -480,7 +811,14 @@ export function showBrowserDownloadInFolder(scopeId: string, downloadId: string) const download = browserDownloadsByScope .get(resolved) ?.find((candidate) => candidate.id === downloadId) - if (!download || download.state !== 'completed' || !existsSync(download.savePath)) return false + if ( + !download || + download.state !== 'completed' || + typeof download.savePath !== 'string' || + !existsSync(download.savePath) + ) { + return false + } shell.showItemInFolder(download.savePath) return true } @@ -511,8 +849,13 @@ function resetSessionState(): void { browserAppTheme = 'system' browserAppearanceTheme = 'app' browserDefaultZoom = 100 - activeDownloadPaths.clear() browserDownloadsByScope.clear() + cancelActiveBrowserDownloads() + backgroundTabRestoreGeneration += 1 + pendingForegroundTabRestores.length = 0 + pendingBackgroundTabRestores.length = 0 + activeTabRestores.clear() + activeBackgroundTabRestores.clear() activatePanelScope(null) } @@ -674,10 +1017,7 @@ export function migrateBrowserScope(fromScopeId: string, toScopeId: string): boo /** Destroys one chat's live browser state without touching the shared profile. */ export function disposeBrowserScope(scopeId: string): void { const resolved = resolveBrowserScopeId(scopeId) - // A migrated provisional id is only an alias. Disposing that spelling must - // never destroy the durable chat state it now points at. if (resolved !== scopeId) { - browserScopeAliases.delete(scopeId) suspendedBrowserScopes.delete(scopeId) browserDownloadsByScope.delete(scopeId) try { @@ -690,6 +1030,7 @@ export function disposeBrowserScope(scopeId: string): void { return } browserDownloadsByScope.delete(resolved) + cancelActiveBrowserDownloads(resolved) suspendedBrowserScopes.delete(resolved) const state = browserScopeStates.get(resolved) @@ -737,15 +1078,17 @@ export function suspendBrowserScope(scopeId: string): boolean { const state = browserScopeStates.get(resolved) if (!state) { suspendedBrowserScopes.add(resolved) + cancelActiveBrowserDownloads(resolved) return true } withBrowserScope(resolved, () => { if (hasSession()) persistBrowserSession() + suspendedBrowserScopes.add(resolved) + cancelActiveBrowserDownloads(resolved) closeLiveTabs() }) - suspendedBrowserScopes.add(resolved) browserScopeStates.delete(resolved) if (getActiveBrowserScopeId() === resolved) { activeBrowserScopeId = null @@ -782,7 +1125,7 @@ function browserSessionSnapshot(): BrowserSessionSnapshot { const downloads = (browserDownloadsByScope.get(getBrowserScopeId()) ?? []) .filter(isFinishedBrowserDownload) .slice(0, MAX_RECENT_FINISHED_DOWNLOADS) - .map((download) => ({ ...download })) + .map(({ interruptionReason: _interruptionReason, ...download }) => ({ ...download })) return { v: 1, tabs: liveTabs.map((tab) => ({ url: tabUrl(tab), pinned: tab.pinned })), @@ -892,6 +1235,11 @@ function mediaOrigin(candidate: unknown): string | null { } } +function withoutUrlFragment(url: string): string { + const fragmentIndex = url.indexOf('#') + return fragmentIndex < 0 ? url : url.slice(0, fragmentIndex) +} + function requestedMediaDevices(candidate: unknown): BrowserMediaDevice[] | null { if (!Array.isArray(candidate) || candidate.length === 0) return null const devices = new Set() @@ -1006,6 +1354,212 @@ export async function respondToMediaPermission(requestId: string, allowed: boole publishPageIssue(tab) } +function grantSiteOrigin(state: BrowserScopeState, origin: string): void { + state.siteOriginGrants.delete(origin) + state.siteOriginGrants.set(origin, true) + while (state.siteOriginGrants.size > MAX_SITE_ORIGIN_GRANTS_PER_SCOPE) { + const oldest = state.siteOriginGrants.keys().next().value + if (typeof oldest !== 'string') break + state.siteOriginGrants.delete(oldest) + } +} + +function hasSiteOriginGrant(state: BrowserScopeState, origin: string): boolean { + if (!state.siteOriginGrants.has(origin)) return false + grantSiteOrigin(state, origin) + return true +} + +function publishSitePermissionState(scopeId: string): void { + const resolved = resolveBrowserScopeId(scopeId) + const state = browserScopeStates.get(resolved) + if (!state) return + const active = state.tabs.find((tab) => tab.id === state.activeTabId) + if (active && !active.view.webContents.isDestroyed()) { + withBrowserScope(resolved, () => events?.onPageStateChanged(active.view.webContents)) + } +} + +function settleSitePermission(tab: AgentTab, allowed: boolean, publish = true): boolean { + const pending = tab.pendingSitePermission + if (!pending) return false + tab.pendingSitePermission = undefined + clearTimeout(pending.timeout) + pending.nativePromptController?.abort() + pending.resolve(allowed) + if (publish) publishSitePermissionState(tab.scopeId) + return true +} + +function scopedTabForRequest(details: { + webContents?: WebContents + webContentsId?: number +}): { scopeId: string; tab: AgentTab } | null { + if (details.webContents) return scopedTabForContents(details.webContents) + if (typeof details.webContentsId !== 'number') return null + for (const [scopeId, state] of browserScopeStates) { + const tab = state.tabs.find( + (candidate) => candidate.view.webContents.id === details.webContentsId + ) + if (tab) return { scopeId, tab } + } + return null +} + +/** Highest-priority exact site request: visible tab, automation tab, then task tab order. */ +export function sitePermissionRequestForScope(): BrowserSitePermissionRequest | undefined { + const state = browserScopeState() + const active = state.tabs.find((tab) => tab.id === state.activeTabId)?.pendingSitePermission + if (active) return active.request + const automation = state.tabs.find( + (tab) => tab.id === state.automationTabId + )?.pendingSitePermission + if (automation) return automation.request + return state.tabs.find((tab) => tab.pendingSitePermission)?.pendingSitePermission?.request +} + +function grantSiteOriginForExplicitNavigation(contents: WebContents, destination: string): boolean { + const scoped = scopedTabForContents(contents) + const origin = mediaOrigin(destination) + if (!scoped || !origin) return false + const state = browserScopeStates.get(scoped.scopeId) + if (!state || scoped.tab.view.webContents !== contents || contents.isDestroyed()) return false + grantSiteOrigin(state, origin) + return true +} + +/** Grants only the destination origin entered through a native-activation-gated user action. */ +export function grantSiteOriginForUserNavigation( + contents: WebContents, + destination: string +): boolean { + return grantSiteOriginForExplicitNavigation(contents, destination) +} + +/** Grants the exact destination origin after the browser driver has completed its SSRF check. */ +export function grantSiteOriginForAgentNavigation( + contents: WebContents, + destination: string +): boolean { + return grantSiteOriginForExplicitNavigation(contents, destination) +} + +/** Applies a response only to the exact live task, tab, document, and suspended network request. */ +export function respondToSitePermission(requestId: string, allowed: boolean): boolean { + const scopeId = getBrowserScopeId() + const state = browserScopeStates.get(scopeId) + const tab = state?.tabs.find( + (candidate) => candidate.pendingSitePermission?.request.requestId === requestId + ) + const pending = tab?.pendingSitePermission + if (!state || !tab || !pending) return false + + if (!allowed) return settleSitePermission(tab, false) + + const contents = tab.view.webContents + const live = + !contents.isDestroyed() && + pending.contents === contents && + pending.request.tabId === tab.id && + pending.documentUrl === contents.getURL() && + mediaOrigin(pending.destinationUrl) === pending.request.origin && + scopeId === resolveBrowserScopeId(tab.scopeId) && + scopeId === getActiveBrowserScopeId() && + isPanelVisible() + if (!live) return settleSitePermission(tab, false) + + grantSiteOrigin(state, pending.request.origin) + return settleSitePermission(tab, true) +} + +async function requestSitePermission(details: { + id: number + url: string + webContents?: WebContents + webContentsId?: number +}): Promise { + const origin = mediaOrigin(details.url) + const scoped = scopedTabForRequest(details) + if (!origin || !scoped || suspendedBrowserScopes.has(scoped.scopeId)) return false + const state = browserScopeStates.get(scoped.scopeId) + const contents = scoped.tab.view.webContents + if (!state || contents.isDestroyed()) return false + + if (mediaOrigin(contents.getURL()) === origin || hasSiteOriginGrant(state, origin)) return true + if (scoped.scopeId !== getActiveBrowserScopeId() || !isPanelVisible()) return false + const win = panelWindow() + if (!win || win.isDestroyed()) return false + + settleSitePermission(scoped.tab, false, false) + revokeTabMediaPermissions(scoped.tab, false) + const request: BrowserSitePermissionRequest = { + requestId: generateId(), + tabId: scoped.tab.id, + origin, + } + const allowed = new Promise((resolve) => { + scoped.tab.pendingSitePermission = { + request, + documentUrl: contents.getURL(), + destinationUrl: details.url, + contents, + networkRequestId: details.id, + resolve, + timeout: setTimeout( + bindToBrowserScope(scoped.scopeId, () => { + const pending = scoped.tab.pendingSitePermission + if ( + pending?.request.requestId !== request.requestId || + pending.networkRequestId !== details.id + ) { + return + } + settleSitePermission(scoped.tab, false) + }), + SITE_PERMISSION_PROMPT_TIMEOUT_MS + ), + } + }) + scoped.tab.pendingRestore?.grantSitePermissionGrace?.() + if (events?.sitePermissionPromptSupported(scoped.scopeId)) { + win.focus() + win.webContents.focus() + publishSitePermissionState(scoped.scopeId) + } else { + const nativePromptController = new AbortController() + const pending = scoped.tab.pendingSitePermission + if (!pending || pending.request.requestId !== request.requestId) return await allowed + pending.nativePromptController = nativePromptController + void dialog + .showMessageBox(win, { + type: 'warning', + buttons: ['Block', 'Allow'], + defaultId: 0, + cancelId: 0, + noLink: true, + signal: nativePromptController.signal, + message: `Allow this browser task to open ${request.origin}?`, + detail: 'Only allow this site if it is expected for the current task.', + }) + .then(({ response }) => { + withBrowserScope(scoped.scopeId, () => { + respondToSitePermission(request.requestId, response === 1) + }) + }) + .catch((error) => { + if (!nativePromptController.signal.aborted) { + logger.warn('Could not present the native site permission prompt', { + error: getErrorMessage(error), + }) + } + withBrowserScope(scoped.scopeId, () => { + respondToSitePermission(request.requestId, false) + }) + }) + } + return await allowed +} + /** * Default-deny hardening for the agent partition. Site permissions remain * denied apart from ALLOWED_SITE_PERMISSIONS. Media is granted only after a @@ -1111,10 +1665,10 @@ function configureAgentPartition(ses: Session): void { // redirects, link clicks, location.href, meta-refresh) — so an internal host // can't slip in that way. // - // Subresources that come back readable or that execute get the resolving - // check too, cached per host; images and fonts keep the cheap synchronous - // path. See isBlockedSubresourceUrl and subresourceNeedsResolution for why - // each way round. + // Subresources that come back readable, render into screenshots, or execute + // get the resolving check too, cached per host; fonts keep the cheap + // synchronous path. See isBlockedSubresourceUrl and + // subresourceNeedsResolution for why each way round. ses.webRequest.onBeforeRequest((details, callback) => { // Answered exactly once, and never throwing. A throw inside the `then` // below would otherwise land in the `catch` and answer a second time, and @@ -1130,13 +1684,15 @@ function configureAgentPartition(ses: Session): void { logger.warn('Could not answer an agent request', { error: getErrorMessage(error) }) } } - if (details.resourceType === 'mainFrame' || details.resourceType === 'subFrame') { + if (details.resourceType === 'mainFrame') { void checkAgentUrl(details.url) - .then((guard) => { + .then(async (guard) => { if (!guard.ok) { logger.warn('Blocked agent document navigation to a private host') + settle(true) + return } - settle(!guard.ok) + settle(!(await requestSitePermission(details))) }) .catch((error) => { // Fail closed: an unexpected rejection must cancel, never leave the @@ -1146,6 +1702,18 @@ function configureAgentPartition(ses: Session): void { }) return } + if (details.resourceType === 'subFrame') { + void checkAgentUrl(details.url) + .then((guard) => { + if (!guard.ok) logger.warn('Blocked agent document navigation to a private host') + settle(!guard.ok) + }) + .catch((error) => { + logger.error('Agent SSRF check failed; cancelling request', { error }) + settle(true) + }) + return + } if (!subresourceNeedsResolution(details.resourceType)) { settle(isBlockedRequestUrl(details.url)) return @@ -1169,31 +1737,68 @@ function configureAgentPartition(ses: Session): void { item.cancel() return } - const filename = suggestedFilename(item.getFilename(), item.getMimeType()) - const savePath = uniqueDownloadPath( - directory, - filename, - (candidate) => activeDownloadPaths.has(candidate) || existsSync(candidate) - ) - activeDownloadPaths.add(savePath) - item.setSavePath(savePath) - const download: BrowserDownloadInfo & { savePath: string } = { - id: generateId(), - filename, - state: 'progressing', - receivedBytes: Math.max(0, item.getReceivedBytes()), - totalBytes: Math.max(0, item.getTotalBytes()), - startedAt: new Date().toISOString(), - savePath, + const admissionReason = browserDownloadAdmissionReason(scopeId, item) + if (admissionReason) { + item.cancel() + const rejected = createTrackedBrowserDownload(item, 'interrupted', admissionReason) + recordBrowserDownload(scopeId, rejected) + withBrowserScope(scopeId, persistBrowserSession) + logger.warn('Agent browser download rejected by a safety limit', { + filename: rejected.filename, + reason: admissionReason, + }) + return } - browserDownloadsByScope.set(scopeId, [ + + const download = createTrackedBrowserDownload(item, 'progressing') + const { filename } = download + try { + item.pause() + } catch (error) { + const reason = 'Stopped: the download could not be paused for a disk-space safety check' + download.interruptionReason = reason + download.state = 'interrupted' + try { + item.cancel() + } catch (cancelError) { + logger.warn('Could not cancel an agent browser download after pause failed', { + error: getErrorMessage(cancelError), + filename, + }) + } + recordBrowserDownload(scopeId, download) + withBrowserScope(scopeId, persistBrowserSession) + logger.warn('Agent browser download could not be paused for admission', { + error: getErrorMessage(error), + filename, + }) + return + } + const active: ActiveBrowserDownload = { + directory, download, - ...(browserDownloadsByScope.get(scopeId) ?? []), - ]) - trimBrowserDownloads(scopeId) - publishBrowserDownloads(scopeId) + item, + diskCheckInFlight: false, + lastDiskCheckAt: 0, + scopeId, + terminal: false, + } + activeBrowserDownloads.add(active) + recordBrowserDownload(scopeId, download) logger.info('Agent browser download started', { filename }) item.on('updated', (_updatedEvent, state) => { + updateDownloadProgress(download, item) + if (state === 'interrupted') { + download.state = 'interrupted' + } else { + const limitReason = browserDownloadSizeLimitReasonForItem(item) + if (limitReason) cancelBrowserDownloadForLimit(active, limitReason) + else { + download.state = 'progressing' + if (active.savePath) checkBrowserDownloadDiskSpace(active, 'progress') + } + } + const liveScopeId = resolveBrowserScopeId(scopeId) if ( suspendedBrowserScopes.has(liveScopeId) || @@ -1202,12 +1807,10 @@ function configureAgentPartition(ses: Session): void { ) { return } - updateDownloadProgress(download, item) - download.state = state === 'interrupted' ? 'interrupted' : 'progressing' publishBrowserDownloads(liveScopeId) }) item.once('done', (_doneEvent, state) => { - activeDownloadPaths.delete(savePath) + releaseActiveBrowserDownload(active) const liveScopeId = resolveBrowserScopeId(scopeId) if ( suspendedBrowserScopes.has(liveScopeId) || @@ -1217,17 +1820,98 @@ function configureAgentPartition(ses: Session): void { return } updateDownloadProgress(download, item) - download.state = state + download.state = active.limitReason ? 'interrupted' : state trimBrowserDownloads(liveScopeId) publishBrowserDownloads(liveScopeId) withBrowserScope(liveScopeId, persistBrowserSession) - if (state === 'completed') { + if (download.state === 'completed') { logger.info('Agent browser download completed', { filename }) - if (process.platform === 'darwin') app.dock?.downloadFinished(savePath) - } else if (state === 'interrupted') { - logger.warn('Agent browser download interrupted', { filename }) + if (process.platform === 'darwin' && active.savePath) { + app.dock?.downloadFinished(active.savePath) + } + } else if (download.state === 'interrupted') { + logger.warn('Agent browser download interrupted', { + filename, + reason: download.interruptionReason, + }) } }) + let allocationExpired = false + const allocation = uniqueDownloadPath(directory, filename, { + isActive: () => + !allocationExpired && + !active.terminal && + !active.limitReason && + activeBrowserDownloads.has(active), + pathExists: browserDownloadSettings?.pathExists, + reservePath: (candidate) => { + if ( + allocationExpired || + active.terminal || + active.limitReason || + !activeBrowserDownloads.has(active) || + activeDownloadPaths.has(candidate) + ) { + return false + } + activeDownloadPaths.set(candidate, active) + active.savePath = candidate + return true + }, + }) + void withBrowserDownloadTimeout( + allocation, + BROWSER_DOWNLOAD_PATH_ALLOCATION_TIMEOUT_MS, + 'Browser download path allocation timed out', + () => { + allocationExpired = true + } + ) + .then((savePath) => { + if (active.terminal || !activeBrowserDownloads.has(active)) { + releaseActiveBrowserDownloadPath(active, savePath ?? undefined) + return + } + if (!savePath) { + cancelBrowserDownloadForLimit( + active, + 'Stopped: a safe non-conflicting download filename could not be allocated' + ) + publishActiveBrowserDownload(active) + return + } + download.savePath = savePath + try { + item.setSavePath(savePath) + } catch (error) { + releaseActiveBrowserDownloadPath(active, savePath) + active.savePath = undefined + download.savePath = undefined + logger.warn('Could not set the destination for an agent browser download', { + error: getErrorMessage(error), + filename, + }) + cancelBrowserDownloadForLimit( + active, + 'Stopped: the download destination could not be prepared safely' + ) + publishActiveBrowserDownload(active) + return + } + checkBrowserDownloadDiskSpace(active, 'admission') + }) + .catch((error) => { + if (active.terminal || !activeBrowserDownloads.has(active)) return + logger.warn('Could not allocate an agent browser download destination', { + error: getErrorMessage(error), + filename, + }) + cancelBrowserDownloadForLimit( + active, + 'Stopped: the download destination could not be prepared safely' + ) + publishActiveBrowserDownload(active) + }) }) } @@ -1308,6 +1992,7 @@ export function goBack(contents: WebContents): boolean { const tab = tabForContents(contents) if (!tab) return false if (tab.pageIssue?.kind === 'load-error') { + prepareExplicitNavigation(contents) tab.syntheticForward = { url: tab.pageIssue.url, baseHistoryIndex: contents.navigationHistory.getActiveIndex(), @@ -1317,6 +2002,7 @@ export function goBack(contents: WebContents): boolean { return true } if (!contents.navigationHistory.canGoBack()) return false + prepareExplicitNavigation(contents) tab.preserveSyntheticForwardOnNextNavigation = Boolean(tab.syntheticForward) contents.navigationHistory.goBack() return true @@ -1332,15 +2018,18 @@ export function goForward(contents: WebContents): boolean { contents.navigationHistory.getActiveIndex() < syntheticForward.baseHistoryIndex && contents.navigationHistory.canGoForward() ) { + prepareExplicitNavigation(contents) tab.preserveSyntheticForwardOnNextNavigation = true contents.navigationHistory.goForward() return true } + prepareExplicitNavigation(contents) tab.syntheticForward = undefined void contents.loadURL(syntheticForward.url).catch(() => {}) return true } if (!contents.navigationHistory.canGoForward()) return false + prepareExplicitNavigation(contents) contents.navigationHistory.goForward() return true } @@ -1348,6 +2037,7 @@ export function goForward(contents: WebContents): boolean { /** Retries the appropriate recovery path for a failed, crashed, or hung page. */ export function reloadPage(contents: WebContents): void { const tab = tabForContents(contents) + prepareExplicitNavigation(contents) const issue = tab?.pageIssue if (issue?.kind === 'load-error') { void contents.loadURL(issue.url).catch(() => {}) @@ -1485,10 +2175,14 @@ export function stopFindInActiveTab(focusPage: boolean): void { * inside the browser resource rather than spawn a native window, and both are * reached from an untrusted page, so the scheme is checked here once. */ -function openTabWithUrl(url: string, agentOwned: boolean): void { +function openTabWithUrl( + url: string, + { agentOwned, userAuthorized }: { agentOwned: boolean; userAuthorized: boolean } +): void { if (!/^https?:\/\//i.test(url)) return try { const tab = agentOwned ? addAutomationTab() : addTab() + if (userAuthorized) grantSiteOriginForUserNavigation(tab.view.webContents, url) void tab.view.webContents.loadURL(url).catch(() => {}) } catch (error) { logger.warn('Could not open a link in a new browser tab', { @@ -1540,7 +2234,10 @@ function initializeTabView(view: WebContentsView, scopeId: string): WebContentsV contents.setUserAgent(browserUserAgent()) attachAgentContextMenu(contents, { addToChat: (text) => withBrowserScope(scopeId, () => addPageSelectionToChat(contents, text)), - openTab: (url) => withBrowserScope(scopeId, () => openTabWithUrl(url, false)), + openTab: (url) => + withBrowserScope(scopeId, () => + openTabWithUrl(url, { agentOwned: false, userAuthorized: true }) + ), defaultZoomFactor: getBrowserDefaultZoomFactor, }) @@ -1597,7 +2294,12 @@ function initializeTabView(view: WebContentsView, scopeId: string): WebContentsV // Keep popups inside the browser resource: http(s) window.open and // target=_blank requests become a new internal tab, never a native window. contents.setWindowOpenHandler((details) => { - withBrowserScope(scopeId, () => openTabWithUrl(details.url, agentOwnsPopupFrom(contents))) + withBrowserScope(scopeId, () => + openTabWithUrl(details.url, { + agentOwned: agentOwnsPopupFrom(contents), + userAuthorized: false, + }) + ) return { action: 'deny' } }) @@ -1624,6 +2326,7 @@ function initializeTabView(view: WebContentsView, scopeId: string): WebContentsV return } dismissFind(tab.id) + settleSitePermission(tab, false) revokeTabMediaPermissions(tab, false) tab.pageIssue = { kind: 'crashed', @@ -1641,6 +2344,7 @@ function initializeTabView(view: WebContentsView, scopeId: string): WebContentsV const tab = tabs.find((entry) => entry.view === view) if (!tab || tab.pageIssue?.kind === 'crashed') return dismissFind(tab.id) + settleSitePermission(tab, false) revokeTabMediaPermissions(tab, false) tab.pageIssue = { kind: 'unresponsive', @@ -1746,7 +2450,16 @@ function initializeTabView(view: WebContentsView, scopeId: string): WebContentsV bindToBrowserScope(scopeId, (details) => { if (!details.isMainFrame) return const tab = tabs.find((entry) => entry.view === view) - if (tab) revokeTabMediaPermissions(tab) + if (tab) { + if ( + tab.pendingSitePermission && + withoutUrlFragment(tab.pendingSitePermission.destinationUrl) !== + withoutUrlFragment(details.url) + ) { + settleSitePermission(tab, false) + } + revokeTabMediaPermissions(tab) + } notePageNavigationStarted(contents) events?.onTabNavigated(contents, false) }) @@ -1765,7 +2478,10 @@ function initializeTabView(view: WebContentsView, scopeId: string): WebContentsV 'destroyed', bindToBrowserScope(scopeId, () => { const tab = tabs.find((entry) => entry.view === view) - if (tab) revokeTabMediaPermissions(tab, false) + if (tab) { + settleSitePermission(tab, false) + revokeTabMediaPermissions(tab, false) + } events?.onTabClosed(contents) }) ) @@ -2007,6 +2723,257 @@ function closeTabAfterFailedRestore(tab: AgentTab): void { } } +function isPendingTabRestoreLive(pending: PendingTabRestore): boolean { + if (pending.generation !== backgroundTabRestoreGeneration) return false + const state = browserScopeStates.get(resolveBrowserScopeId(pending.tab.scopeId)) + return Boolean( + state?.tabs.includes(pending.tab) && + !pending.tab.view.webContents.isDestroyed() && + pending.tab.pendingRestoreUrl === pending.url + ) +} + +function loadPendingTabRestore(pending: PendingTabRestore, timeoutMs: number): Promise { + if (!isPendingTabRestoreLive(pending) || pending.url === 'about:blank') { + return Promise.resolve(false) + } + const contents = pending.tab.view.webContents + return new Promise((resolve) => { + let settled = false + let timeout: ReturnType | undefined + const startedAt = Date.now() + let hardDeadlineAt = startedAt + timeoutMs + SITE_PERMISSION_PROMPT_TIMEOUT_MS + let deadlineAt = startedAt + timeoutMs + let foregroundDeadlineGranted = pending.priority === 'foreground' + let sitePermissionGraceGranted = false + const finish = (loaded: boolean) => { + if (settled) return + settled = true + if (timeout) clearTimeout(timeout) + pending.cancelLoad = undefined + pending.grantSitePermissionGrace = undefined + pending.promoteToForeground = undefined + if (!loaded) settleSitePermission(pending.tab, false) + if ( + loaded && + isPendingTabRestoreLive(pending) && + pending.tab.pendingRestoreUrl === pending.url + ) { + pending.tab.pendingRestoreUrl = undefined + } + resolve(loaded) + } + const stopLoad = (timedOut: boolean) => { + try { + if (!contents.isDestroyed()) contents.stop() + } catch (error) { + logger.warn('Could not stop a deferred browser tab restore', { + error: getErrorMessage(error), + }) + } finally { + if (timedOut && isPendingTabRestoreLive(pending)) { + withBrowserScope(pending.tab.scopeId, () => { + recordPageLoadFailure(contents, { + kind: 'load-error', + code: -7, + description: 'ERR_TIMED_OUT', + url: pending.url, + }) + }) + } + finish(false) + } + } + pending.cancelLoad = () => stopLoad(false) + const scheduleDeadline = () => { + if (settled) return + if (timeout) clearTimeout(timeout) + timeout = setTimeout(() => stopLoad(true), Math.max(0, deadlineAt - Date.now())) + } + pending.grantSitePermissionGrace = () => { + if (settled || sitePermissionGraceGranted) return + sitePermissionGraceGranted = true + deadlineAt = Math.min(deadlineAt + SITE_PERMISSION_PROMPT_TIMEOUT_MS, hardDeadlineAt) + scheduleDeadline() + } + pending.promoteToForeground = () => { + if (settled || foregroundDeadlineGranted) return + foregroundDeadlineGranted = true + hardDeadlineAt = + startedAt + FOREGROUND_TAB_RESTORE_TIMEOUT_MS + SITE_PERMISSION_PROMPT_TIMEOUT_MS + deadlineAt = Math.min( + Math.max(deadlineAt, Date.now() + FOREGROUND_TAB_RESTORE_TIMEOUT_MS), + hardDeadlineAt + ) + scheduleDeadline() + } + scheduleDeadline() + try { + void Promise.resolve(contents.loadURL(pending.url)).then( + () => finish(true), + () => finish(false) + ) + } catch { + finish(false) + } + }) +} + +function createPendingTabRestore( + tab: AgentTab, + url: string, + priority: PendingTabRestore['priority'] +): PendingTabRestore { + let resolveReady = (_loaded: boolean) => {} + const ready = new Promise((resolve) => { + resolveReady = resolve + }) + const pending: PendingTabRestore = { + generation: backgroundTabRestoreGeneration, + tab, + url, + priority, + ready, + resolveReady, + started: false, + settled: false, + requeueAfterPreemption: false, + } + tab.pendingRestore = pending + return pending +} + +function settlePendingTabRestore(pending: PendingTabRestore, loaded = false): void { + if (pending.settled) return + pending.settled = true + if (pending.tab.pendingRestore === pending) pending.tab.pendingRestore = undefined + pending.resolveReady(loaded) +} + +function startCountedTabRestore(pending: PendingTabRestore): void { + pending.started = true + activeTabRestores.add(pending) + if (pending.priority === 'background') activeBackgroundTabRestores.add(pending) + const timeoutMs = + pending.priority === 'foreground' + ? FOREGROUND_TAB_RESTORE_TIMEOUT_MS + : BACKGROUND_TAB_RESTORE_TIMEOUT_MS + void loadPendingTabRestore(pending, timeoutMs) + .then((loaded) => { + if (pending.requeueAfterPreemption && !loaded && isPendingTabRestoreLive(pending)) { + pending.requeueAfterPreemption = false + pending.started = false + const queue = + pending.priority === 'foreground' + ? pendingForegroundTabRestores + : pendingBackgroundTabRestores + queue.push(pending) + return + } + settlePendingTabRestore(pending, loaded) + }) + .finally(() => { + if (pending.generation !== backgroundTabRestoreGeneration) return + activeTabRestores.delete(pending) + activeBackgroundTabRestores.delete(pending) + drainTabRestores() + }) +} + +/** + * Globally bounds restore work. Foreground entries have priority while one + * process-wide slot remains unavailable to background loads. The queues are + * bounded by the 96-live-tab process invariant. + */ +function drainTabRestores(): void { + while (activeTabRestores.size < MAX_TAB_RESTORE_CONCURRENCY) { + const pending = + pendingForegroundTabRestores.shift() ?? + (activeBackgroundTabRestores.size < MAX_BACKGROUND_TAB_RESTORE_CONCURRENCY + ? pendingBackgroundTabRestores.shift() + : undefined) + if (!pending) break + if (!isPendingTabRestoreLive(pending)) { + settlePendingTabRestore(pending) + continue + } + startCountedTabRestore(pending) + } + if ( + pendingForegroundTabRestores.length > 0 && + activeTabRestores.size >= MAX_TAB_RESTORE_CONCURRENCY + ) { + const preempted = activeBackgroundTabRestores.values().next().value + if (preempted) { + activeBackgroundTabRestores.delete(preempted) + activeTabRestores.delete(preempted) + preempted.requeueAfterPreemption = true + preempted.cancelLoad?.() + drainTabRestores() + } + } +} + +function queueBackgroundTabRestore(tab: AgentTab, url: string): void { + if (url === 'about:blank') return + if (tab.pendingRestore) return + pendingBackgroundTabRestores.push(createPendingTabRestore(tab, url, 'background')) + drainTabRestores() +} + +function promotePendingTabRestore(tab: AgentTab): PendingTabRestore | undefined { + const url = tab.pendingRestoreUrl + if (!url || url === 'about:blank') return undefined + let pending = tab.pendingRestore + if (pending && activeBackgroundTabRestores.has(pending)) { + activeBackgroundTabRestores.delete(pending) + pending.priority = 'foreground' + pending.promoteToForeground?.() + drainTabRestores() + return pending + } + if (!pending) pending = createPendingTabRestore(tab, url, 'foreground') + if (!pending.started) { + const queueIndex = pendingBackgroundTabRestores.indexOf(pending) + if (queueIndex >= 0) pendingBackgroundTabRestores.splice(queueIndex, 1) + pending.priority = 'foreground' + if (!pendingForegroundTabRestores.includes(pending)) pendingForegroundTabRestores.push(pending) + drainTabRestores() + } + return pending +} + +function discardPendingTabRestore(tab: AgentTab): void { + for (let index = pendingForegroundTabRestores.length - 1; index >= 0; index -= 1) { + if (pendingForegroundTabRestores[index]?.tab === tab) { + pendingForegroundTabRestores.splice(index, 1) + } + } + for (let index = pendingBackgroundTabRestores.length - 1; index >= 0; index -= 1) { + if (pendingBackgroundTabRestores[index]?.tab === tab) { + pendingBackgroundTabRestores.splice(index, 1) + } + } + const pending = tab.pendingRestore + if (!pending) return + pending.cancelLoad?.() + settlePendingTabRestore(pending) +} + +export async function waitForPendingTabRestore(tab: AgentTab): Promise { + const pending = promotePendingTabRestore(tab) + return pending ? await pending.ready : true +} + +/** Prevents a delayed restore slot from overwriting a newer explicit navigation. */ +export function prepareExplicitNavigation(contents: WebContents): void { + const tab = tabForContents(contents) + if (!tab) return + settleSitePermission(tab, false) + tab.pendingRestoreUrl = undefined + discardPendingTabRestore(tab) +} + /** Marks the visible page as user-selected without blocking automation on it. */ export function claimActiveTabForUser(): AgentTab | null { const tab = activeTab() @@ -2081,9 +3048,11 @@ export function restoreBrowserSession(): void { nextTabId: state.nextTabId, restored: state.restored, lastPersistedSnapshot: state.lastPersistedSnapshot, + siteOriginGrants: new Map(state.siteOriginGrants), } const previousDownloads = browserDownloadsByScope.get(scopeId) const restoredTabs: AgentTab[] = [] + const restoredLoads: Array<{ tab: AgentTab; url: string }> = [] state.restoring = true try { if (snapshot) { @@ -2094,10 +3063,10 @@ export function restoreBrowserSession(): void { for (const { entry } of selectedEntries) { const tab = addTabInternal({ pinned: entry.pinned, activate: false, notify: false }) tab.pendingRestoreUrl = entry.url + const restoredOrigin = mediaOrigin(entry.url) + if (restoredOrigin) grantSiteOrigin(state, restoredOrigin) restoredTabs.push(tab) - if (entry.url !== 'about:blank') { - void tab.view.webContents.loadURL(entry.url).catch(() => {}) - } + restoredLoads.push({ tab, url: entry.url }) } const restoredActiveIndex = selectedEntries.findIndex( ({ sourceIndex }) => sourceIndex === snapshot.activeIndex @@ -2116,6 +3085,7 @@ export function restoreBrowserSession(): void { state.nextTabId = previousState.nextTabId state.restored = previousState.restored state.lastPersistedSnapshot = previousState.lastPersistedSnapshot + state.siteOriginGrants = previousState.siteOriginGrants if (previousDownloads) browserDownloadsByScope.set(scopeId, previousDownloads) else browserDownloadsByScope.delete(scopeId) applyActiveTabThrottling() @@ -2125,6 +3095,16 @@ export function restoreBrowserSession(): void { } applyActiveTabThrottling() + const restoredActive = restoredLoads.find(({ tab }) => tab.id === state.activeTabId) + if (restoredActive) { + pendingForegroundTabRestores.push( + createPendingTabRestore(restoredActive.tab, restoredActive.url, 'foreground') + ) + drainTabRestores() + } + for (const restore of restoredLoads) { + if (restore !== restoredActive) queueBackgroundTabRestore(restore.tab, restore.url) + } if (snapshot) publishBrowserDownloads(scopeId) const active = activeTab() if (active) { @@ -2204,6 +3184,7 @@ export function reopenClosedTab(): AgentTab | null { // onBeforeRequest still runs the full DNS-resolving SSRF check on the // document load. Pre-checking would only buy a nicer error, and there is // no model to report one to — this path is a user keystroke. + grantSiteOriginForUserNavigation(tab.view.webContents, url) void tab.view.webContents.loadURL(url).catch(() => {}) } return tab @@ -2226,6 +3207,7 @@ export function duplicateTab(tabId: string): AgentTab | null { // Sanitized to http(s) without embedded credentials above, and the // partition's onBeforeRequest still runs the full SSRF check on the load — // same reasoning as reopenClosedTab, and this is likewise a user action. + grantSiteOriginForUserNavigation(tab.view.webContents, url) void tab.view.webContents.loadURL(url).catch(() => {}) } return tab @@ -2248,6 +3230,7 @@ export function switchTab(tabId: string): AgentTab { } currentScope.activeTabId = tab.id currentScope.visibleTabUserSelected = true + promotePendingTabRestore(tab) // Visible selection does not move the automation exemption; the user may // inspect another page while a tool continues in its background tab. applyActiveTabThrottling() @@ -2308,6 +3291,8 @@ export function closeTab(tabId: string): void { dismissFind(tabId) clearAutomationIndicatorsForTab(tabId) const [tab] = tabs.splice(index, 1) + discardPendingTabRestore(tab) + settleSitePermission(tab, false) revokeTabMediaPermissions(tab, false) recentlyClosedTabUrls.unshift(sanitizeRestorableUrl(tabUrl(tab)) ?? 'about:blank') if (recentlyClosedTabUrls.length > MAX_RECENTLY_CLOSED_TABS) { @@ -2341,6 +3326,7 @@ export function closeTab(tabId: string): void { persistBrowserSession() events?.onTabsChanged() if (!hasSession()) { + currentScope.siteOriginGrants.clear() events?.onSessionClosed() } } @@ -2474,9 +3460,10 @@ export function handleFocusedShortcut( focusRendererOmnibox('select') return true case 'reload-or-clear': - shortcutTab.view.webContents.reload() + reloadPage(shortcutTab.view.webContents) return true case 'hard-reload': + prepareExplicitNavigation(shortcutTab.view.webContents) shortcutTab.view.webContents.reloadIgnoringCache() return true } @@ -2540,6 +3527,8 @@ function closeTabFromUser(tabId: string): void { function closeLiveTabs(): void { dismissFind(currentScope.findingTabId) for (const tab of tabs.splice(0)) { + discardPendingTabRestore(tab) + settleSitePermission(tab, false, false) revokeTabMediaPermissions(tab, false) detachIfAttached(tab.view) if (!tab.view.webContents.isDestroyed()) { @@ -2552,6 +3541,7 @@ function closeLiveTabs(): void { currentScope.automationActive = false currentScope.automationNeedsAttention = false currentScope.visibleTabUserSelected = false + currentScope.siteOriginGrants.clear() clearFocusedBrowserTab() } @@ -2622,6 +3612,8 @@ export async function clearProfileStorage(): Promise { events?.onTabsChanged() }) } + browserDownloadsByScope.clear() + cancelActiveBrowserDownloads() layout() const ses = electronSession.fromPartition(AGENT_PARTITION) diff --git a/apps/desktop/src/main/browser-agent/url-guard.test.ts b/apps/desktop/src/main/browser-agent/url-guard.test.ts index 0468ffb799a..80df3e973ca 100644 --- a/apps/desktop/src/main/browser-agent/url-guard.test.ts +++ b/apps/desktop/src/main/browser-agent/url-guard.test.ts @@ -305,8 +305,11 @@ describe('isBlockedSubresourceUrl', () => { }) describe('subresourceNeedsResolution', () => { - it('exempts only the high-volume, non-readable types', () => { - expect(subresourceNeedsResolution('image')).toBe(false) + it('resolves images because their rendered contents are observable in screenshots', () => { + expect(subresourceNeedsResolution('image')).toBe(true) + }) + + it('exempts only fonts from hostname resolution', () => { expect(subresourceNeedsResolution('font')).toBe(false) }) diff --git a/apps/desktop/src/main/browser-agent/url-guard.ts b/apps/desktop/src/main/browser-agent/url-guard.ts index 75a20e2679d..b604c243703 100644 --- a/apps/desktop/src/main/browser-agent/url-guard.ts +++ b/apps/desktop/src/main/browser-agent/url-guard.ts @@ -43,10 +43,9 @@ function guardHost(rawUrl: string): string | null { * * Loopback is deliberately allowed: it is the user's own machine, and opening * a dev server on localhost is one of the most ordinary things to do in this - * panel — the URL bar already assumes `http://` for it. Nothing is given away - * by it either, since the desktop app hands the same agent an unrestricted - * shell on that machine, so a blocked `http://localhost:3000` is one - * `curl http://localhost:3000` away regardless. + * panel — the URL bar already assumes `http://` for it. This is an explicit + * desktop-product capability, independent of whether terminal execution is + * enabled or separately approval-gated. * * Every other private range stays blocked. Those are a different matter: the * LAN is other people's machines, and `169.254.169.254` is link-local rather @@ -121,11 +120,13 @@ export async function checkAgentUrl(rawUrl: string): Promise { /** * Subresource types that keep the cheap synchronous literal-IP check. * - * Images and fonts are the high-volume types and are not readable - * cross-origin, so the residual for them is a load/error timing oracle — a - * documented, accepted trade against a DNS lookup per asset. + * Fonts are high-volume and their response bytes are not exposed to the model, + * so the residual is a load/error timing oracle — a documented, accepted trade + * against a DNS lookup per asset. Images are not exempt: browser screenshots + * make their rendered contents observable even when cross-origin reads are + * otherwise blocked. */ -const LITERAL_ONLY_RESOURCE_TYPES: ReadonlySet = new Set(['image', 'font']) +const LITERAL_ONLY_RESOURCE_TYPES: ReadonlySet = new Set(['font']) /** * Whether a subresource needs the DNS-resolving check rather than the literal-IP diff --git a/apps/desktop/src/main/downloads.test.ts b/apps/desktop/src/main/downloads.test.ts index 92721bfa706..cc6b64c49bf 100644 --- a/apps/desktop/src/main/downloads.test.ts +++ b/apps/desktop/src/main/downloads.test.ts @@ -1,3 +1,6 @@ +import { mkdtempSync, symlinkSync, writeFileSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' import { describe, expect, it, vi } from 'vitest' vi.mock('electron', () => import('@/test/electron-mock')) @@ -40,16 +43,96 @@ describe('suggestedFilename', () => { }) describe('uniqueDownloadPath', () => { - it('keeps the original name when it is available', () => { - expect(uniqueDownloadPath('/Downloads', 'report.csv', () => false)).toBe( - '/Downloads/report.csv' - ) + it('keeps the original name when it is available', async () => { + await expect( + uniqueDownloadPath('/Downloads', 'report.csv', { pathExists: () => false }) + ).resolves.toBe('/Downloads/report.csv') + }) + + it('uses a collision-resistant suffix instead of scanning sequential copy names', async () => { + const occupied = new Set(['/Downloads/report.csv']) + await expect( + uniqueDownloadPath('/Downloads', 'report.csv', { + pathExists: (path) => occupied.has(path), + suffixForAttempt: () => 'safe-id', + }) + ).resolves.toBe('/Downloads/report (safe-id).csv') + }) + + it('checks a pre-existing filesystem entry without blocking the caller', async () => { + const directory = mkdtempSync(join(tmpdir(), 'sim-download-path-')) + writeFileSync(join(directory, 'report.csv'), 'existing') + + const allocation = uniqueDownloadPath(directory, 'report.csv', { + suffixForAttempt: () => 'safe-id', + }) + + await expect(allocation).resolves.toBe(join(directory, 'report (safe-id).csv')) + }) + + it('treats a dangling symlink as occupied', async () => { + const directory = mkdtempSync(join(tmpdir(), 'sim-download-path-')) + symlinkSync(join(directory, 'missing-target'), join(directory, 'report.csv')) + + await expect( + uniqueDownloadPath(directory, 'report.csv', { + suffixForAttempt: () => 'safe-id', + }) + ).resolves.toBe(join(directory, 'report (safe-id).csv')) }) - it('adds a copy number before the extension instead of overwriting', () => { - const occupied = new Set(['/Downloads/report.csv', '/Downloads/report (1).csv']) - expect(uniqueDownloadPath('/Downloads', 'report.csv', (path) => occupied.has(path))).toBe( - '/Downloads/report (2).csv' + it('atomically separates simultaneous allocations of the same name', async () => { + const reservations = new Set() + const options = { + pathExists: async () => false, + reservePath: (path: string) => { + if (reservations.has(path)) return false + reservations.add(path) + return true + }, + suffixForAttempt: (attempt: number) => `copy-${attempt}`, + } + + const [first, second] = await Promise.all([ + uniqueDownloadPath('/Downloads', 'report.csv', options), + uniqueDownloadPath('/Downloads', 'report.csv', options), + ]) + + expect(new Set([first, second])).toEqual( + new Set(['/Downloads/report.csv', '/Downloads/report (copy-1).csv']) ) }) + + it('stops after the configured collision cap', async () => { + const pathExists = vi.fn(async () => true) + + await expect( + uniqueDownloadPath('/Downloads', 'report.csv', { + maxAttempts: 3, + pathExists, + suffixForAttempt: (attempt) => `copy-${attempt}`, + }) + ).resolves.toBeNull() + expect(pathExists).toHaveBeenCalledTimes(3) + }) + + it('abandons an allocation torn down while the filesystem check is pending', async () => { + let resolveExists = (_exists: boolean) => {} + const exists = new Promise((resolve) => { + resolveExists = resolve + }) + let active = true + const reservePath = vi.fn(() => true) + const allocation = uniqueDownloadPath('/Downloads', 'report.csv', { + isActive: () => active, + pathExists: () => exists, + reservePath, + }) + + active = false + resolveExists(false) + + await expect(allocation).resolves.toBeNull() + expect(reservePath).not.toHaveBeenCalled() + }) }) diff --git a/apps/desktop/src/main/downloads.ts b/apps/desktop/src/main/downloads.ts index f571acc675a..1267eeac3ea 100644 --- a/apps/desktop/src/main/downloads.ts +++ b/apps/desktop/src/main/downloads.ts @@ -1,6 +1,7 @@ -import { existsSync } from 'node:fs' +import { lstat } from 'node:fs/promises' import { basename, extname, join } from 'node:path' import { createLogger } from '@sim/logger' +import { generateShortId } from '@sim/utils/id' import type { Session } from 'electron' import { app } from 'electron' import type { EventRecorder } from '@/main/observability' @@ -8,6 +9,8 @@ import type { EventRecorder } from '@/main/observability' const logger = createLogger('DesktopDownloads') const MAX_FILENAME_LENGTH = 200 +const MAX_DOWNLOAD_PATH_ATTEMPTS = 16 +const DOWNLOAD_PATH_SUFFIX_LENGTH = 8 const MIME_EXTENSIONS: Record = { 'text/csv': '.csv', @@ -53,25 +56,78 @@ export function suggestedFilename( return `download-${stamp}${extension}` } +export interface UniqueDownloadPathOptions { + /** Asynchronous filesystem seam used by tests and non-standard storage backends. */ + pathExists?: (path: string) => boolean | Promise + /** Atomically reserves a candidate against other allocations in this process. */ + reservePath?: (path: string) => boolean + /** Stops an allocation whose owning download was torn down while I/O was pending. */ + isActive?: () => boolean + /** Deterministic test seam for collision-resistant copy suffixes. */ + suffixForAttempt?: (attempt: number) => string + maxAttempts?: number +} + +async function downloadPathExists(path: string): Promise { + try { + await lstat(path) + return true + } catch (error) { + if ((error as NodeJS.ErrnoException).code === 'ENOENT') return false + throw error + } +} + +function suffixedFilename(filename: string, suffix: string): string { + const extension = extname(filename) + const stem = basename(filename, extension) + const marker = ` (${suffix})` + const maxStemLength = Math.max(1, MAX_FILENAME_LENGTH - extension.length - marker.length) + return `${stem.slice(0, maxStemLength)}${marker}${extension}` +} + /** - * Picks a Chrome-style non-conflicting destination without overwriting an - * existing download: `report.csv`, `report (1).csv`, and so on. + * Asynchronously reserves a non-conflicting destination without blocking the + * Electron main thread. The original filename remains the first choice; a + * bounded number of collision-resistant alternatives avoids an unbounded scan + * through attacker-controlled pre-existing copy names. */ -export function uniqueDownloadPath( +export async function uniqueDownloadPath( directory: string, rawFilename: string, - pathExists: (path: string) => boolean = existsSync -): string { + options: UniqueDownloadPathOptions = {} +): Promise { const filename = sanitizeFilename(rawFilename) || 'download' - const extension = extname(filename) - const stem = basename(filename, extension) - let candidate = join(directory, filename) - let copy = 1 - while (pathExists(candidate)) { - candidate = join(directory, `${stem} (${copy})${extension}`) - copy += 1 + const pathExists = options.pathExists ?? downloadPathExists + const reservePath = options.reservePath ?? (() => true) + const isActive = options.isActive ?? (() => true) + const suffixForAttempt = + options.suffixForAttempt ?? (() => generateShortId(DOWNLOAD_PATH_SUFFIX_LENGTH)) + const requestedAttempts = options.maxAttempts ?? MAX_DOWNLOAD_PATH_ATTEMPTS + const maxAttempts = Math.max( + 1, + Math.min( + MAX_DOWNLOAD_PATH_ATTEMPTS, + Number.isFinite(requestedAttempts) + ? Math.trunc(requestedAttempts) + : MAX_DOWNLOAD_PATH_ATTEMPTS + ) + ) + + for (let attempt = 0; attempt < maxAttempts; attempt += 1) { + if (!isActive()) return null + const suffix = + attempt === 0 + ? '' + : sanitizeFilename(suffixForAttempt(attempt)).slice(0, 32) || + generateShortId(DOWNLOAD_PATH_SUFFIX_LENGTH) + const candidateFilename = attempt === 0 ? filename : suffixedFilename(filename, suffix) + const candidate = join(directory, candidateFilename) + if (await pathExists(candidate)) continue + if (!isActive()) return null + if (reservePath(candidate)) return candidate } - return candidate + return null } /** diff --git a/apps/desktop/src/main/index.ts b/apps/desktop/src/main/index.ts index 539c2329a5a..fc60d931d18 100644 --- a/apps/desktop/src/main/index.ts +++ b/apps/desktop/src/main/index.ts @@ -18,6 +18,7 @@ import { newChatRoute, settingsRoute } from '@/main/app-routes' import { activateBrowserScope as activateAgentBrowserScope, clearBrowserProfile as clearAgentBrowserProfile, + closeBrowserSession as closeAgentBrowserSession, initDriver as initBrowserAgentDriver, } from '@/main/browser-agent/driver' import { @@ -27,7 +28,6 @@ import { setPanelOccluded as setBrowserAgentPanelOccluded, } from '@/main/browser-agent/panel' import { - closeSession as closeAgentBrowserSession, handleFocusedShortcut as handleFocusedBrowserShortcut, isBrowserScopeSuspended, quiesceBrowserSessions, @@ -720,6 +720,8 @@ function main(): void { onSessionStatus: (alive, scopeId) => { scopeEvents.sendBrowser(scopeId, 'browser-agent:session-status', alive, scopeId) }, + sitePermissionPromptSupported: (scopeId) => + scopeEvents.browserSitePermissionPromptSupported(scopeId), onFillAvailability: (available, scopeId) => { scopeEvents.sendBrowser(scopeId, 'browser-credentials:fill-availability', { available, diff --git a/apps/desktop/src/main/ipc.test.ts b/apps/desktop/src/main/ipc.test.ts index 118288347a2..b75537b14c6 100644 --- a/apps/desktop/src/main/ipc.test.ts +++ b/apps/desktop/src/main/ipc.test.ts @@ -139,6 +139,15 @@ import { findCachedTerminalThemeProfile, listTerminalThemeProfiles } from '@/mai const APP = 'https://sim.ai' const ESC = '\u001b' const BEL = '\u0007' +const CANONICAL_BROWSER_URL_INPUT = + 'HTTPS://B\u00dcCHER.Example:443/docs/../private?query=sim#result' +const CANONICAL_BROWSER_URL = 'https://xn--bcher-kva.example/private?query=sim#result' +const INVALID_BROWSER_URLS = [ + 'https://[', + 'file:///tmp/private', + `https://docs.example/${'a'.repeat(8_192)}`, + `https://docs.example/${'\u00e9'.repeat(1_400)}`, +] as const const DEFAULT_DESKTOP_PREFERENCES: DesktopPreferences = { notificationsEnabled: true, @@ -289,6 +298,7 @@ describe('registerIpcHandlers', () => { scopeEvents: { activateBrowser: vi.fn(), activateTerminal: vi.fn(), + registerBrowserSitePermissionPromptSupport: vi.fn(), sendBrowser: vi.fn(), sendTerminal: vi.fn(), }, @@ -780,6 +790,43 @@ describe('registerIpcHandlers', () => { ) }) + it('rejects malformed browser execution envelopes before admission or authorization', async () => { + const { invoke } = collectHandlers() + const handler = invoke.get('browser-agent:execute-tool') + const fetchAuthorization = vi.fn(async () => + Response.json({ chatId: 'chat-1', toolName: 'browser_snapshot', args: {} }) + ) + const malformedEvent = { + senderFrame: { url: `${APP}/workspace/ws1` }, + sender: { session: { fetch: fetchAuthorization } }, + } + const captureBoundary = vi.spyOn(browserDriver, 'captureBrowserToolQueueBoundary') + const invalidScopeFlood = Array.from( + { length: browserDriver.BROWSER_TOOL_ADMISSION_LIMITS.process * 2 }, + (_, index) => + handler?.(malformedEvent, `tool-${index}`, 'browser_snapshot', {}, `invalid scope ${index}`) + ) + + const results = await Promise.all([ + ...invalidScopeFlood, + handler?.(malformedEvent, '', 'browser_snapshot', {}, 'chat-1'), + handler?.(malformedEvent, 'x'.repeat(257), 'browser_snapshot', {}, 'chat-1'), + handler?.(malformedEvent, 'tool-retired', 'browser_request_takeover', {}, 'chat-1'), + handler?.(malformedEvent, 'tool-non-string', 42, {}, 'chat-1'), + ]) + + expect(results).toHaveLength(browserDriver.BROWSER_TOOL_ADMISSION_LIMITS.process * 2 + 4) + expect(results).toEqual( + results.map(() => ({ + ok: false, + error: 'This browser action is not an authorized pending Copilot tool call.', + })) + ) + expect(captureBoundary).not.toHaveBeenCalled() + expect(fetchAuthorization).not.toHaveBeenCalled() + captureBoundary.mockRestore() + }) + it('routes exact browser-tool cancellation without waiting for authorization', async () => { const { invoke } = collectHandlers() const cancel = vi.spyOn(browserDriver, 'cancelTool').mockReturnValue(true) @@ -996,6 +1043,97 @@ describe('registerIpcHandlers', () => { requestId: 'request-2', allowed: true, }) + panelAction.mockRestore() + }) + + it('requires trusted input for site grants and user-origin navigation', async () => { + const { invoke, on } = collectHandlers() + const panelAction = vi.spyOn(browserDriver, 'handlePanelAction').mockResolvedValue() + const handler = on.get('browser-agent:panel-action') + + await invoke.get('browser-agent:activate-scope')?.(inactiveAppEvent, 'chat-sites') + handler?.( + inactiveAppEvent, + { action: 'respond-site-permission', requestId: 'request-1', allowed: true }, + 'chat-sites' + ) + handler?.( + inactiveAppEvent, + { action: 'respond-site-permission', requestId: 'request-1', allowed: false }, + 'chat-sites' + ) + handler?.( + inactiveAppEvent, + { action: 'navigate', url: 'https://docs.example/private' }, + 'chat-sites' + ) + + expect(panelAction).toHaveBeenCalledOnce() + expect(panelAction).toHaveBeenCalledWith('chat-sites', { + action: 'respond-site-permission', + requestId: 'request-1', + allowed: false, + }) + + await invoke.get('browser-agent:activate-scope')?.(activeAppEvent, 'chat-sites') + handler?.( + activeAppEvent, + { action: 'respond-site-permission', requestId: 'request-2', allowed: true }, + 'chat-sites' + ) + handler?.( + activeAppEvent, + { action: 'navigate', url: 'https://docs.example/private' }, + 'chat-sites' + ) + + expect(panelAction).toHaveBeenNthCalledWith(2, 'chat-sites', { + action: 'respond-site-permission', + requestId: 'request-2', + allowed: true, + }) + expect(panelAction).toHaveBeenNthCalledWith(3, 'chat-sites', { + action: 'navigate', + url: 'https://docs.example/private', + }) + panelAction.mockRestore() + }) + + it('canonicalizes and validates panel navigation URLs before they reach the driver', async () => { + const { invoke, on } = collectHandlers() + const panelAction = vi.spyOn(browserDriver, 'handlePanelAction').mockResolvedValue() + const handler = on.get('browser-agent:panel-action') + + await invoke.get('browser-agent:activate-scope')?.(activeAppEvent, 'chat-navigation') + handler?.( + activeAppEvent, + { action: 'navigate', url: CANONICAL_BROWSER_URL_INPUT }, + 'chat-navigation' + ) + for (const url of INVALID_BROWSER_URLS) { + handler?.(activeAppEvent, { action: 'navigate', url }, 'chat-navigation') + } + + expect(panelAction).toHaveBeenCalledOnce() + expect(panelAction).toHaveBeenCalledWith('chat-navigation', { + action: 'navigate', + url: CANONICAL_BROWSER_URL, + }) + panelAction.mockRestore() + }) + + it('accepts site permission prompt support only from the app renderer', () => { + const { on } = collectHandlers() + const register = on.get('browser-agent:register-site-permission-prompt-support') + + register?.(evilEvent) + expect(deps.scopeEvents.registerBrowserSitePermissionPromptSupport).not.toHaveBeenCalled() + + register?.(appEvent) + expect(deps.scopeEvents.registerBrowserSitePermissionPromptSupport).toHaveBeenCalledOnce() + expect(deps.scopeEvents.registerBrowserSitePermissionPromptSupport).toHaveBeenCalledWith( + appSender + ) }) it('ignores browser-agent panel actions from outside the app origin', () => { @@ -1256,6 +1394,51 @@ describe('registerIpcHandlers', () => { peek.mockRestore() }) + it('atomically creates and navigates a canonical user URL only from trusted input', async () => { + const tabsState = { scopeId: 'chat-links', tabs: [], activeTabId: '2' } + const tabContents = { loadURL: vi.fn(async () => {}) } + const add = vi.spyOn(browserSession, 'addTab').mockReturnValue({ + view: { webContents: tabContents }, + } as never) + const grant = vi.spyOn(browserSession, 'grantSiteOriginForUserNavigation').mockReturnValue(true) + const peek = vi.spyOn(browserSession, 'peekTabsState').mockReturnValue(tabsState) + const { invoke } = collectHandlers() + + await invoke.get('browser-agent:activate-scope')?.(activeAppEvent, 'chat-links') + await expect( + invoke.get('browser-agent:open-url')?.( + activeAppEvent, + CANONICAL_BROWSER_URL_INPUT, + 'chat-links' + ) + ).resolves.toEqual(tabsState) + + expect(add).toHaveBeenCalledOnce() + expect(grant).toHaveBeenCalledWith(tabContents, CANONICAL_BROWSER_URL) + expect(tabContents.loadURL).toHaveBeenCalledWith(CANONICAL_BROWSER_URL) + + for (const url of INVALID_BROWSER_URLS) { + await expect( + invoke.get('browser-agent:open-url')?.(activeAppEvent, url, 'chat-links') + ).resolves.toEqual({ scopeId: '', tabs: [], activeTabId: null }) + } + expect(add).toHaveBeenCalledOnce() + + await invoke.get('browser-agent:activate-scope')?.(inactiveAppEvent, 'chat-inactive-links') + await expect( + invoke.get('browser-agent:open-url')?.( + inactiveAppEvent, + 'https://docs.example/', + 'chat-inactive-links' + ) + ).resolves.toEqual({ scopeId: '', tabs: [], activeTabId: null }) + expect(add).toHaveBeenCalledOnce() + + add.mockRestore() + grant.mockRestore() + peek.mockRestore() + }) + it('routes browser scope events after activation and a valid provisional migration', async () => { const migrate = vi.spyOn(browserDriver, 'migrateBrowserScope').mockReturnValue(true) const { invoke } = collectHandlers() diff --git a/apps/desktop/src/main/ipc.ts b/apps/desktop/src/main/ipc.ts index 8221c022fdf..3befddec165 100644 --- a/apps/desktop/src/main/ipc.ts +++ b/apps/desktop/src/main/ipc.ts @@ -1,6 +1,7 @@ import { normalize } from 'node:path' import { fileURLToPath } from 'node:url' import { + BROWSER_TOOL_AUTHORIZATION_TIMEOUT_MS, type BrowserPanelAction, type BrowserPanelAnchor, type BrowserPanelBounds, @@ -43,6 +44,7 @@ import { getKnownSessions, handlePanelAction, migrateBrowserScope, + releaseBrowserToolQueueBoundary, restoreBrowserScope, showToolbarMenu, suspendBrowserScope, @@ -52,6 +54,7 @@ import { addTab, findInActiveTab, getBrowserDownloadsState, + grantSiteOriginForUserNavigation, peekTabsState, reorderTab, setBrowserAppTheme, @@ -94,7 +97,6 @@ const logger = createLogger('DesktopIpc') /** Workspace/chat ids are opaque tokens; anything else never reaches a URL. */ const ID_PATTERN = /^[A-Za-z0-9_-]{1,128}$/ const TERMINAL_WRITE_CHUNK_CHARACTERS = 64 * 1024 -const DESKTOP_TOOL_AUTHORIZATION_TIMEOUT_MS = 8_000 function writeTerminalText( terminal: TerminalRegistry, @@ -151,6 +153,10 @@ function parseDesktopScope(raw: unknown): string | null { return isDesktopScopeId(raw) ? raw : null } +function isDesktopToolCallId(raw: unknown): raw is string { + return typeof raw === 'string' && raw.length >= 1 && raw.length <= 256 +} + export interface OAuthConnectScope { workspaceId?: string credentialId?: string @@ -333,7 +339,11 @@ export interface IpcDeps { terminal: TerminalRegistry scopeEvents: Pick< ScopedEventRouter, - 'activateBrowser' | 'activateTerminal' | 'sendBrowser' | 'sendTerminal' + | 'activateBrowser' + | 'activateTerminal' + | 'registerBrowserSitePermissionPromptSupport' + | 'sendBrowser' + | 'sendTerminal' > settings: DesktopSettingsService getWindowState: (sender: WebContents) => DesktopWindowState @@ -489,6 +499,18 @@ const PTY_REPLY = new RegExp( ) const MAX_TERMINAL_WRITE_CHARS = 256_000 const MAX_PTY_REPLY_CHARS = 8_192 +const MAX_BROWSER_NAVIGATION_URL_CHARS = 8_192 + +function canonicalHttpNavigationUrl(rawUrl: unknown): string | null { + if (typeof rawUrl !== 'string' || rawUrl.length > MAX_BROWSER_NAVIGATION_URL_CHARS) return null + try { + const url = new URL(rawUrl) + if (url.protocol !== 'https:' && url.protocol !== 'http:') return null + return url.href.length <= MAX_BROWSER_NAVIGATION_URL_CHARS ? url.href : null + } catch { + return null + } +} interface DesktopToolAuthorization { chatId: string @@ -501,9 +523,7 @@ async function fetchDesktopToolAuthorization( deps: IpcDeps, toolCallId: unknown ): Promise { - if (typeof toolCallId !== 'string' || toolCallId.length < 1 || toolCallId.length > 256) { - return null - } + if (!isDesktopToolCallId(toolCallId)) return null const startedAt = Date.now() try { const response = await event.sender.session.fetch( @@ -513,7 +533,7 @@ async function fetchDesktopToolAuthorization( credentials: 'include', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify({ toolCallId }), - signal: AbortSignal.timeout(DESKTOP_TOOL_AUTHORIZATION_TIMEOUT_MS), + signal: AbortSignal.timeout(BROWSER_TOOL_AUTHORIZATION_TIMEOUT_MS), } ) if (!response.ok) { @@ -918,6 +938,39 @@ export function registerIpcHandlers(deps: IpcDeps): void { }) }, }, + 'browser-agent:register-site-permission-prompt-support': { + kind: 'send', + gate: 'app-origin', + requires: 'browser', + passSender: true, + handler: (sender) => { + deps.scopeEvents.registerBrowserSitePermissionPromptSupport(sender as WebContents) + }, + }, + 'browser-agent:open-url': { + kind: 'invoke', + gate: 'app-origin', + requires: 'browser', + passSender: true, + needsUserActivation: true, + denied: { scopeId: '', tabs: [], activeTabId: null }, + handler: (sender, rawUrl, rawScope) => { + const contents = sender as WebContents + const scope = activeRendererScope(browserScopeBySender, contents, rawScope) + const destination = canonicalHttpNavigationUrl(rawUrl) + if (!scope || !destination) { + return { scopeId: '', tabs: [], activeTabId: null } + } + return withBrowserScope(scope, () => { + const tab = addTab() + if (!grantSiteOriginForUserNavigation(tab.view.webContents, destination)) { + return peekTabsState() + } + void tab.view.webContents.loadURL(destination).catch(() => {}) + return peekTabsState() + }) + }, + }, 'browser-agent:activate-scope': { kind: 'invoke', gate: 'app-origin', @@ -1100,10 +1153,15 @@ export function registerIpcHandlers(deps: IpcDeps): void { gate: 'app-origin', requires: 'browser', passSender: true, - needsUserActivation: ([action]) => - isRecordLike(action) && - action.action === 'respond-media-permission' && - action.allowed === true, + needsUserActivation: ([action]) => { + if (!isRecordLike(action)) return false + if (action.action === 'navigate') return true + return ( + (action.action === 'respond-media-permission' || + action.action === 'respond-site-permission') && + action.allowed === true + ) + }, handler: (sender, action, rawScope) => { const scope = activeRendererScope(browserScopeBySender, sender as WebContents, rawScope) if ( @@ -1114,7 +1172,14 @@ export function registerIpcHandlers(deps: IpcDeps): void { ) { return } - void handlePanelAction(scope, action as BrowserPanelAction).catch(() => {}) + const panelAction = action as BrowserPanelAction + if (panelAction.action === 'navigate') { + const destination = canonicalHttpNavigationUrl(panelAction.url) + if (!destination) return + void handlePanelAction(scope, { ...panelAction, url: destination }).catch(() => {}) + return + } + void handlePanelAction(scope, panelAction).catch(() => {}) }, }, 'browser-agent:set-tab-pinned': { @@ -1900,20 +1965,35 @@ export function registerIpcHandlers(deps: IpcDeps): void { } let handlerArgs = args if (channel === 'browser-agent:execute-tool') { + const toolCallId = args[0] const requestedTool = args[1] const requestedScope = parseDesktopScope(args[3]) - const authorizationBoundary = requestedScope - ? captureBrowserToolQueueBoundary(requestedScope) - : undefined - const authorization = await fetchDesktopToolAuthorization(event, deps, args[0]) + if ( + !isDesktopToolCallId(toolCallId) || + typeof requestedTool !== 'string' || + !isCurrentBrowserToolName(requestedTool) || + !requestedScope + ) { + return { + ok: false, + error: 'This browser action is not an authorized pending Copilot tool call.', + } + } + const authorizationBoundary = captureBrowserToolQueueBoundary(requestedScope) + if (!authorizationBoundary) { + return { + ok: false, + error: + 'Sim already has too many browser actions queued. Wait for earlier actions to finish.', + } + } + const authorization = await fetchDesktopToolAuthorization(event, deps, toolCallId) if ( !authorization || - !requestedScope || authorization.chatId !== requestedScope || - typeof requestedTool !== 'string' || - authorization.toolName !== requestedTool || - !isCurrentBrowserToolName(authorization.toolName) + authorization.toolName !== requestedTool ) { + releaseBrowserToolQueueBoundary(authorizationBoundary) return { ok: false, error: 'This browser action is not an authorized pending Copilot tool call.', @@ -1921,7 +2001,7 @@ export function registerIpcHandlers(deps: IpcDeps): void { } handlerArgs = [ authorization.chatId, - args[0], + toolCallId, authorization.toolName, authorization.args, authorizationBoundary, diff --git a/apps/desktop/src/main/observability.test.ts b/apps/desktop/src/main/observability.test.ts index 836d61f8f01..967cd87f634 100644 --- a/apps/desktop/src/main/observability.test.ts +++ b/apps/desktop/src/main/observability.test.ts @@ -1,8 +1,17 @@ -import { existsSync, mkdtempSync, readFileSync } from 'node:fs' +import { chmodSync, existsSync, mkdtempSync, readFileSync, statSync, writeFileSync } from 'node:fs' import { tmpdir } from 'node:os' import { join } from 'node:path' import { beforeEach, describe, expect, it, vi } from 'vitest' +const { mockLogger } = vi.hoisted(() => ({ + mockLogger: { + info: vi.fn(), + warn: vi.fn(), + error: vi.fn(), + }, +})) + +vi.mock('@sim/logger', () => ({ createLogger: () => mockLogger })) vi.mock('electron', () => import('@/test/electron-mock')) import { app, dialog } from 'electron' @@ -21,6 +30,10 @@ describe('scrubUrl', () => { }) describe('createEventLog', () => { + beforeEach(() => { + vi.clearAllMocks() + }) + it('appends JSONL entries', () => { const dir = mkdtempSync(join(tmpdir(), 'sim-desktop-events-')) const events = createEventLog(dir) @@ -42,6 +55,50 @@ describe('createEventLog', () => { events.record('app_launch', { version: '1.0.0' }) events.record('app_launch', { version: '1.0.0' }) expect(existsSync(`${events.filePath}.1`)).toBe(true) + expect(statSync(`${events.filePath}.1`).mode & 0o777).toBe(0o600) + }) + + it('creates its directory and log with private permissions', () => { + const root = mkdtempSync(join(tmpdir(), 'sim-desktop-events-')) + const dir = join(root, 'logs') + const events = createEventLog(dir) + events.record('app_launch') + + expect(statSync(dir).mode & 0o777).toBe(0o700) + expect(statSync(events.filePath).mode & 0o777).toBe(0o600) + }) + + it('tightens permissions on existing logs', () => { + const dir = mkdtempSync(join(tmpdir(), 'sim-desktop-events-')) + const filePath = join(dir, 'desktop-events.log') + const rotatedFilePath = `${filePath}.1` + writeFileSync(filePath, 'current\n') + writeFileSync(rotatedFilePath, 'rotated\n') + chmodSync(dir, 0o755) + chmodSync(filePath, 0o644) + chmodSync(rotatedFilePath, 0o644) + + createEventLog(dir) + + expect(statSync(dir).mode & 0o777).toBe(0o700) + expect(statSync(filePath).mode & 0o777).toBe(0o600) + expect(statSync(rotatedFilePath).mode & 0o777).toBe(0o600) + }) + + it('reports permission failures without exposing local paths or OS errors', () => { + const root = mkdtempSync(join(tmpdir(), 'sim-desktop-events-')) + const overlongDir = join(root, 'x'.repeat(300)) + + const events = createEventLog(overlongDir) + events.record('app_launch') + + expect(mockLogger.warn.mock.calls).toEqual([ + ['Could not apply private desktop event-log permissions', { target: 'directory' }], + ['Could not apply private desktop event-log permissions', { target: 'current-log' }], + ['Could not apply private desktop event-log permissions', { target: 'rotated-log' }], + ]) + expect(JSON.stringify(mockLogger.warn.mock.calls)).not.toContain(root) + expect(JSON.stringify(mockLogger.warn.mock.calls)).not.toContain('ENAMETOOLONG') }) }) diff --git a/apps/desktop/src/main/observability.ts b/apps/desktop/src/main/observability.ts index 60c8e47b36c..142283ef9da 100644 --- a/apps/desktop/src/main/observability.ts +++ b/apps/desktop/src/main/observability.ts @@ -1,4 +1,4 @@ -import { appendFileSync, mkdirSync, renameSync, statSync } from 'node:fs' +import { appendFileSync, chmodSync, mkdirSync, renameSync, statSync } from 'node:fs' import { join } from 'node:path' import { createLogger } from '@sim/logger' import type { BrowserWindow, Details } from 'electron' @@ -7,6 +7,25 @@ import { app, dialog } from 'electron' const logger = createLogger('DesktopEvents') const DEFAULT_MAX_BYTES = 1_000_000 +const PRIVATE_DIRECTORY_MODE = 0o700 +const PRIVATE_FILE_MODE = 0o600 +type EventLogPermissionTarget = 'directory' | 'current-log' | 'rotated-log' + +function applyPrivateMode( + path: string, + mode: number, + target: EventLogPermissionTarget, + allowMissing = false +): boolean { + try { + chmodSync(path, mode) + return true + } catch (error) { + if (allowMissing && (error as NodeJS.ErrnoException).code === 'ENOENT') return true + logger.warn('Could not apply private desktop event-log permissions', { target }) + return false + } +} export type DesktopEventName = | 'app_launch' @@ -151,26 +170,43 @@ export function scrubUrl(raw: string): string { */ export function createEventLog(dir: string, maxBytes: number = DEFAULT_MAX_BYTES): EventRecorder { const filePath = join(dir, 'desktop-events.log') + const rotatedFilePath = `${filePath}.1` + let privateModesEstablished = true try { - mkdirSync(dir, { recursive: true }) - } catch {} - - const rotateIfNeeded = () => { + mkdirSync(dir, { recursive: true, mode: PRIVATE_DIRECTORY_MODE }) + } catch { + privateModesEstablished = false + } + privateModesEstablished = + applyPrivateMode(dir, PRIVATE_DIRECTORY_MODE, 'directory') && privateModesEstablished + privateModesEstablished = + applyPrivateMode(filePath, PRIVATE_FILE_MODE, 'current-log', true) && privateModesEstablished + privateModesEstablished = + applyPrivateMode(rotatedFilePath, PRIVATE_FILE_MODE, 'rotated-log', true) && + privateModesEstablished + + const rotateIfNeeded = (): boolean => { try { if (statSync(filePath).size > maxBytes) { - renameSync(filePath, `${filePath}.1`) + renameSync(filePath, rotatedFilePath) + return applyPrivateMode(rotatedFilePath, PRIVATE_FILE_MODE, 'rotated-log') } } catch {} + return true } return { filePath, record(name, data) { logger.info(`desktop event: ${name}`, data) + if (!privateModesEstablished) return try { - rotateIfNeeded() + if (!rotateIfNeeded()) { + privateModesEstablished = false + return + } const entry = { at: new Date().toISOString(), name, ...(data ? { data } : {}) } - appendFileSync(filePath, `${JSON.stringify(entry)}\n`) + appendFileSync(filePath, `${JSON.stringify(entry)}\n`, { mode: PRIVATE_FILE_MODE }) } catch (error) { logger.warn('Failed to append desktop event', { error }) } diff --git a/apps/desktop/src/main/scoped-event-router.test.ts b/apps/desktop/src/main/scoped-event-router.test.ts index 7020b526b32..700d6dbc3d5 100644 --- a/apps/desktop/src/main/scoped-event-router.test.ts +++ b/apps/desktop/src/main/scoped-event-router.test.ts @@ -37,6 +37,10 @@ class FakeContents { this.emit('destroyed') } + markDestroyed(): void { + this.destroyed = true + } + private emit(channel: string, ...args: unknown[]): void { for (const listener of [...(this.listeners.get(channel) ?? [])]) listener(...args) } @@ -48,6 +52,76 @@ function webContents(): { fake: FakeContents; contents: WebContents } { } describe('ScopedEventRouter', () => { + it('defaults old renderers to no site permission prompt support', () => { + const router = new ScopedEventRouter() + const renderer = webContents() + + router.activateBrowser(renderer.contents, 'chat-a') + + expect(router.browserSitePermissionPromptSupported('chat-a')).toBe(false) + }) + + it('recognizes an active renderer site permission prompt handshake', () => { + const router = new ScopedEventRouter() + const renderer = webContents() + + router.registerBrowserSitePermissionPromptSupport(renderer.contents) + router.activateBrowser(renderer.contents, 'chat-a') + + expect(router.browserSitePermissionPromptSupported('chat-a')).toBe(true) + }) + + it('requires a fresh site permission prompt handshake after renderer reload', () => { + const router = new ScopedEventRouter() + const renderer = webContents() + + router.registerBrowserSitePermissionPromptSupport(renderer.contents) + router.activateBrowser(renderer.contents, 'chat-a') + renderer.fake.navigate() + router.activateBrowser(renderer.contents, 'chat-a') + + expect(router.browserSitePermissionPromptSupported('chat-a')).toBe(false) + + router.registerBrowserSitePermissionPromptSupport(renderer.contents) + + expect(router.browserSitePermissionPromptSupported('chat-a')).toBe(true) + }) + + it('rejects ambiguous site prompts when two live renderers share a scope', () => { + const router = new ScopedEventRouter() + const first = webContents() + const second = webContents() + + router.activateBrowser(first.contents, 'chat-a') + router.activateBrowser(second.contents, 'chat-a') + router.registerBrowserSitePermissionPromptSupport(first.contents) + + expect(router.browserSitePermissionPromptSupported('chat-a')).toBe(false) + + router.registerBrowserSitePermissionPromptSupport(second.contents) + + expect(router.browserSitePermissionPromptSupported('chat-a')).toBe(false) + }) + + it('recovers site prompt support after an extra recipient moves or is destroyed', () => { + const router = new ScopedEventRouter() + const stable = webContents() + const moving = webContents() + + router.registerBrowserSitePermissionPromptSupport(stable.contents) + router.registerBrowserSitePermissionPromptSupport(moving.contents) + router.activateBrowser(stable.contents, 'chat-a') + router.activateBrowser(moving.contents, 'chat-a') + router.activateBrowser(moving.contents, 'chat-b') + + expect(router.browserSitePermissionPromptSupported('chat-a')).toBe(true) + + router.activateBrowser(moving.contents, 'chat-a') + moving.fake.markDestroyed() + + expect(router.browserSitePermissionPromptSupported('chat-a')).toBe(true) + }) + it('sends resource events only to renderers activated for the matching scope', () => { const router = new ScopedEventRouter() const chatA = webContents() diff --git a/apps/desktop/src/main/scoped-event-router.ts b/apps/desktop/src/main/scoped-event-router.ts index 62541b098b2..7d36ca68bc5 100644 --- a/apps/desktop/src/main/scoped-event-router.ts +++ b/apps/desktop/src/main/scoped-event-router.ts @@ -14,6 +14,34 @@ export class ScopedEventRouter { private readonly browser = this.createSurfaceRoutes() private readonly terminal = this.createSurfaceRoutes() private readonly observedContents = new WeakSet() + private readonly sitePermissionPromptRenderers = new WeakSet() + + /** Records an active renderer handshake without trusting a shell-bundled preload flag. */ + registerBrowserSitePermissionPromptSupport(contents: WebContents): void { + this.sitePermissionPromptRenderers.add(contents) + this.observe(contents) + } + + /** True only when exactly one live renderer owns the scope and registered prompt support. */ + browserSitePermissionPromptSupported(scopeId: string): boolean { + const recipients = this.browser.contentsByScope.get(scopeId) + if (!recipients) return false + let liveRecipientCount = 0 + let supported = false + for (const contents of [...recipients]) { + if (contents.isDestroyed()) { + this.forget(contents) + continue + } + if (this.browser.activeByContents.get(contents) !== scopeId) { + this.removeFromScope(this.browser, contents, scopeId) + continue + } + liveRecipientCount++ + supported ||= this.sitePermissionPromptRenderers.has(contents) + } + return liveRecipientCount === 1 && supported + } activateBrowser(contents: WebContents, scopeId: string): void { this.activate(this.browser, contents, scopeId) @@ -66,6 +94,7 @@ export class ScopedEventRouter { } private forget(contents: WebContents): void { + this.sitePermissionPromptRenderers.delete(contents) this.forgetSurface(this.browser, contents) this.forgetSurface(this.terminal, contents) } diff --git a/apps/desktop/src/main/security-guards.ts b/apps/desktop/src/main/security-guards.ts index 22a91554ec9..93d719e5e9b 100644 --- a/apps/desktop/src/main/security-guards.ts +++ b/apps/desktop/src/main/security-guards.ts @@ -24,11 +24,9 @@ export interface GuardDeps { */ export function attachNavigationGuards(contents: WebContents, deps: GuardDeps): void { const handle = (event: { preventDefault(): void }, url: string) => { - // The agent browser's tabs are general-purpose browsing surfaces: any - // http(s) navigation is their job (they run isolated on their own - // partition with no preload). Everything else stays denied. if (isAgentWebContents(contents)) { - if (!/^https?:/i.test(url)) { + const hasAllowedAgentBrowserScheme = /^https?:/i.test(url) + if (!hasAllowedAgentBrowserScheme) { event.preventDefault() logger.warn('Denied non-http navigation in agent browser', { url: scrubUrl(url) }) } diff --git a/apps/desktop/src/preload/index.test.ts b/apps/desktop/src/preload/index.test.ts index 4780e720e1e..5854b1b624b 100644 --- a/apps/desktop/src/preload/index.test.ts +++ b/apps/desktop/src/preload/index.test.ts @@ -1,9 +1,10 @@ import type { SimDesktopApi } from '@sim/desktop-bridge' import { describe, expect, it, vi } from 'vitest' -const { exposeInMainWorld, invoke } = vi.hoisted(() => ({ +const { exposeInMainWorld, invoke, send } = vi.hoisted(() => ({ exposeInMainWorld: vi.fn(), invoke: vi.fn(() => Promise.resolve(true)), + send: vi.fn(), })) vi.mock('electron', () => ({ @@ -12,7 +13,7 @@ vi.mock('electron', () => ({ invoke, on: vi.fn(), removeListener: vi.fn(), - send: vi.fn(), + send, }, })) @@ -27,6 +28,7 @@ describe('desktop preload bridge', () => { if (!exposed) throw new Error('Expected the desktop preload API to be exposed') expect(exposed.browserAgent.supportsAtomicPanelOcclusion).toBe(true) + exposed.browserAgent.registerSitePermissionPromptSupport?.() await exposed.browserAgent.cancelTool?.('tool-1', 'chat-default') await exposed.browserAgent.cancelActiveTool?.('chat-reloaded') await exposed.browserAgent.setPanelOccluded(true, 'chat-default') @@ -44,6 +46,7 @@ describe('desktop preload bridge', () => { ['browser-agent:search-suggestions', 'sim ai'], ['desktop:settings:set-browser-search-suggestions', false], ]) + expect(send).toHaveBeenCalledWith('browser-agent:register-site-permission-prompt-support') }) it('exposes native microphone settings only on supported platforms', async () => { diff --git a/apps/desktop/src/preload/index.ts b/apps/desktop/src/preload/index.ts index b62def2f525..484bc2d26bd 100644 --- a/apps/desktop/src/preload/index.ts +++ b/apps/desktop/src/preload/index.ts @@ -197,6 +197,9 @@ const api: SimDesktopApi = { }, browserAgent: { supportsAtomicPanelOcclusion: true, + registerSitePermissionPromptSupport: (): void => { + ipcRenderer.send('browser-agent:register-site-permission-prompt-support') + }, executeTool: ( toolCallId: string, tool: BrowserToolName, @@ -213,6 +216,8 @@ const api: SimDesktopApi = { }, openTab: (scopeId: string): Promise => ipcRenderer.invoke('browser-agent:open-tab', scopeId), + openUrl: (url: string, scopeId: string): Promise => + ipcRenderer.invoke('browser-agent:open-url', url, scopeId), activateScope: (scopeId: string): Promise => ipcRenderer.invoke('browser-agent:activate-scope', scopeId), restoreScope: (scopeId: string): Promise => diff --git a/apps/desktop/src/test/electron-mock.ts b/apps/desktop/src/test/electron-mock.ts index 264e5342b67..bb4d81ddc57 100644 --- a/apps/desktop/src/test/electron-mock.ts +++ b/apps/desktop/src/test/electron-mock.ts @@ -156,6 +156,7 @@ function createWebContentsMock() { getTitle: vi.fn(() => 'Example'), loadURL: vi.fn(() => Promise.resolve()), reload: vi.fn(), + stop: vi.fn(), print: vi.fn(), focus: vi.fn(), invalidate: vi.fn(), diff --git a/apps/docs/app/global.css b/apps/docs/app/global.css index 4e493df5ce9..aa2e6f740c8 100644 --- a/apps/docs/app/global.css +++ b/apps/docs/app/global.css @@ -54,8 +54,9 @@ body { nominally references; loading a webfont here would make docs the odd one out, not the aligned one. If the app ever wires that font up for real, add the var back in both places at once. */ - --font-mono: ui-monospace, SFMono-Regular, Menlo, Monaco, Consolas, "Liberation Mono", - "Courier New", monospace; + --font-mono: + ui-monospace, SFMono-Regular, Menlo, Monaco, Consolas, "Liberation Mono", "Courier New", + monospace; } /* Pure white light mode background */ @@ -243,14 +244,16 @@ body { /* Font family utilities */ .font-sans { - font-family: var(--font-geist-sans), ui-sans-serif, system-ui, -apple-system, BlinkMacSystemFont, - "Segoe UI", Roboto, "Helvetica Neue", Arial, sans-serif; + font-family: + var(--font-geist-sans), ui-sans-serif, system-ui, -apple-system, BlinkMacSystemFont, "Segoe UI", + Roboto, "Helvetica Neue", Arial, sans-serif; } /* Platform UI font — Season Sans, used by the chip chrome to match the main app */ .font-season { - font-family: var(--font-season), system-ui, "Segoe UI", Roboto, "Helvetica Neue", Arial, - "Noto Sans", sans-serif; + font-family: + var(--font-season), system-ui, "Segoe UI", Roboto, "Helvetica Neue", Arial, "Noto Sans", + sans-serif; } :root { @@ -445,8 +448,9 @@ html #nd-sidebar button:not([aria-label*="ollapse"]):not([aria-label*="xpand"]) padding: 5px 0.5rem !important; /* 30px tall overall — the app's chip pill, at its px-2 */ font-weight: 400 !important; border-radius: 0.5rem !important; /* platform rounded-lg */ - font-family: var(--font-geist-sans), ui-sans-serif, system-ui, -apple-system, BlinkMacSystemFont, - "Segoe UI", Roboto, "Helvetica Neue", Arial, sans-serif !important; + font-family: + var(--font-geist-sans), ui-sans-serif, system-ui, -apple-system, BlinkMacSystemFont, "Segoe UI", + Roboto, "Helvetica Neue", Arial, sans-serif !important; } /* Sidebar text — platform --text-body */ @@ -904,8 +908,9 @@ video { #nd-page:has(.api-page-header) div:not(.font-mono), #nd-page:has(.api-page-header) label:not(.font-mono), #nd-page:has(.api-page-header) button:not(.font-mono) { - font-family: var(--font-geist-sans), ui-sans-serif, system-ui, -apple-system, BlinkMacSystemFont, - "Segoe UI", Roboto, "Helvetica Neue", Arial, sans-serif; + font-family: + var(--font-geist-sans), ui-sans-serif, system-ui, -apple-system, BlinkMacSystemFont, "Segoe UI", + Roboto, "Helvetica Neue", Arial, sans-serif; } /* Method badge pills — shared background colors (page + sidebar) */ diff --git a/apps/docs/components/icons.tsx b/apps/docs/components/icons.tsx index 5c80c285664..0c0c8783e1b 100644 --- a/apps/docs/components/icons.tsx +++ b/apps/docs/components/icons.tsx @@ -5893,6 +5893,17 @@ export function PipedriveIcon(props: SVGProps) { ) } +export function SailPointIcon(props: SVGProps) { + return ( + + + + + + + ) +} + export function SalesforceIcon(props: SVGProps) { return ( diff --git a/apps/docs/components/ui/icon-mapping.ts b/apps/docs/components/ui/icon-mapping.ts index c7feb4ab052..bf4284b3408 100644 --- a/apps/docs/components/ui/icon-mapping.ts +++ b/apps/docs/components/ui/icon-mapping.ts @@ -117,6 +117,7 @@ import { HexIcon, HubspotIcon, HuggingFaceIcon, + HumanInTheLoopIcon, HunterIOIcon, IAMIcon, IcypeasIcon, @@ -207,6 +208,7 @@ import { RootlyIcon, RssIcon, S3Icon, + SailPointIcon, SalesforceIcon, SapConcurIcon, SapS4HanaIcon, @@ -404,6 +406,8 @@ export const blockTypeToIconMap: Record = { hex: HexIcon, hubspot: HubspotIcon, huggingface: HuggingFaceIcon, + human_in_the_loop: HumanInTheLoopIcon, + human_in_the_loop_v2: HumanInTheLoopIcon, hunter: HunterIOIcon, iam: IAMIcon, icypeas: IcypeasIcon, @@ -510,6 +514,7 @@ export const blockTypeToIconMap: Record = { rootly: RootlyIcon, rss: RssIcon, s3: S3Icon, + sailpoint: SailPointIcon, salesforce: SalesforceIcon, sap_concur: SapConcurIcon, sap_s4hana: SapS4HanaIcon, @@ -546,6 +551,7 @@ export const blockTypeToIconMap: Record = { stt_v2: STTIcon, supabase: SupabaseIcon, table: Table, + table_v2: Table, tailscale: TailscaleIcon, tavily: TavilyIcon, telegram: TelegramIcon, diff --git a/apps/docs/components/workflow-preview/block-display-workflows.ts b/apps/docs/components/workflow-preview/block-display-workflows.ts index eb456340eac..0685ef2c4b0 100644 --- a/apps/docs/components/workflow-preview/block-display-workflows.ts +++ b/apps/docs/components/workflow-preview/block-display-workflows.ts @@ -349,14 +349,14 @@ export const BLOCK_DISPLAY_WORKFLOWS: Record = { ], edges: [], }, - table: { - id: 'table', + table_v2: { + id: 'table_v2', name: 'Table', blocks: [ { - id: 'table', + id: 'table_v2', name: 'Table', - type: 'table', + type: 'table_v2', bgColor: '#10B981', position: { x: 0, y: 0 }, hideTargetHandle: true, diff --git a/apps/docs/components/workflow-preview/block-preview.tsx b/apps/docs/components/workflow-preview/block-preview.tsx index 42a8c9c3aba..34ce64ff364 100644 --- a/apps/docs/components/workflow-preview/block-preview.tsx +++ b/apps/docs/components/workflow-preview/block-preview.tsx @@ -1,15 +1,17 @@ 'use client' import { useMemo } from 'react' +import { CANVAS_Z_INDEX_MODE, useCanvasColorMode } from '@sim/workflow-renderer' +import { type NodeTypes, ReactFlow, ReactFlowProvider } from '@xyflow/react' import { domAnimation, LazyMotion } from 'framer-motion' -import ReactFlow, { type NodeTypes, ReactFlowProvider } from 'reactflow' -import 'reactflow/dist/style.css' +import '@xyflow/react/dist/style.css' import { BLOCK_DISPLAY_WORKFLOWS } from '@/components/workflow-preview/block-display-workflows' import { DocsBlockNode } from '@/components/workflow-preview/docs-block-node' +import { FitViewAfterInit } from '@/components/workflow-preview/fit-view-after-init' import { toReactFlowElements } from '@/components/workflow-preview/workflow-data' /** The hero mounts the same node type the canvas uses, so it can never drift. */ -const NODE_TYPES: NodeTypes = { previewBlock: DocsBlockNode } +const NODE_TYPES = { previewBlock: DocsBlockNode } satisfies NodeTypes const PRO_OPTIONS = { hideAttribution: true } /** `maxZoom` mirrors the previous hand-rolled hero's 1.3 scale. */ const FIT_VIEW_OPTIONS = { padding: 0.2, maxZoom: 1.3 } as const @@ -28,6 +30,7 @@ interface BlockPreviewProps { * `block-display-workflows.ts`. */ export function BlockPreview({ type }: BlockPreviewProps) { + const colorMode = useCanvasColorMode() const workflow = BLOCK_DISPLAY_WORKFLOWS[type] const elements = useMemo(() => (workflow ? toReactFlowElements(workflow) : null), [workflow]) @@ -42,12 +45,12 @@ export function BlockPreview({ type }: BlockPreviewProps) { + diff --git a/apps/docs/components/workflow-preview/docs-block-node.tsx b/apps/docs/components/workflow-preview/docs-block-node.tsx index fac9b1e2660..70ea69da784 100644 --- a/apps/docs/components/workflow-preview/docs-block-node.tsx +++ b/apps/docs/components/workflow-preview/docs-block-node.tsx @@ -2,8 +2,8 @@ import { type ComponentType, memo } from 'react' import { SubBlockRowView, WorkflowBlockView } from '@sim/workflow-renderer' +import type { Node, NodeProps } from '@xyflow/react' import { m } from 'framer-motion' -import type { NodeProps } from 'reactflow' import { resolveIcon } from '@/components/workflow-preview/block-icons' import { BLOCK_STAGGER, @@ -16,7 +16,7 @@ const EMPTY_ICON: ComponentType<{ className?: string }> = () => null const RING_STYLES = 'ring-[1.75px] ring-[var(--brand-secondary)]' -interface DocsBlockData { +export interface DocsBlockData extends Record { name: string blockType: string bgColor: string @@ -30,6 +30,8 @@ interface DocsBlockData { isDimmed?: boolean } +export type DocsBlockNodeType = Node + /** * Docs adapter for workflow block nodes: maps the static preview data to the * shared {@link WorkflowBlockView}'s props. Carries no stores, hooks, or @@ -38,7 +40,10 @@ interface DocsBlockData { * `WorkflowPreview` provides the `LazyMotion` feature set). The block's ring is * driven by `hasRing`/`ringStyles` inside the View. */ -export const DocsBlockNode = memo(function DocsBlockNode({ id, data }: NodeProps) { +export const DocsBlockNode = memo(function DocsBlockNode({ + id, + data, +}: NodeProps) { const { name, blockType, diff --git a/apps/docs/components/workflow-preview/docs-container-node.tsx b/apps/docs/components/workflow-preview/docs-container-node.tsx index b0d27d22ed4..1619b099ca8 100644 --- a/apps/docs/components/workflow-preview/docs-container-node.tsx +++ b/apps/docs/components/workflow-preview/docs-container-node.tsx @@ -2,15 +2,17 @@ import { memo } from 'react' import { type SubflowNodeData, SubflowNodeView } from '@sim/workflow-renderer' -import type { NodeProps } from 'reactflow' +import type { Node, NodeProps } from '@xyflow/react' -interface DocsContainerData { +export interface DocsContainerData extends Record { name: string blockType: string size?: { width: number; height: number } parentId?: string } +export type DocsContainerNodeType = Node + /** * Docs adapter for loop/parallel container blocks: maps the static preview data * to {@link SubflowNodeView}'s read-only `isPreview` shape. Carries no stores, @@ -19,7 +21,7 @@ interface DocsContainerData { export const DocsContainerNode = memo(function DocsContainerNode({ id, data, -}: NodeProps) { +}: NodeProps) { const subflowData: SubflowNodeData = { kind: data.blockType === 'parallel' ? 'parallel' : 'loop', name: data.name, diff --git a/apps/docs/components/workflow-preview/examples.ts b/apps/docs/components/workflow-preview/examples.ts index 639de68608e..dcf10ca36d1 100644 --- a/apps/docs/components/workflow-preview/examples.ts +++ b/apps/docs/components/workflow-preview/examples.ts @@ -139,7 +139,7 @@ export const TABLE_ENRICH_WORKFLOW: PreviewWorkflow = { { id: 'table1', name: 'Table 1', - type: 'table', + type: 'table_v2', bgColor: '#10B981', position: { x: 0, y: 0 }, hideTargetHandle: true, @@ -162,7 +162,7 @@ export const TABLE_ENRICH_WORKFLOW: PreviewWorkflow = { { id: 'table2', name: 'Table 2', - type: 'table', + type: 'table_v2', bgColor: '#10B981', position: { x: 660, y: 0 }, rows: [ @@ -1644,7 +1644,7 @@ export const TABLE_ROUNDTRIP_WORKFLOW: PreviewWorkflow = { { id: 'query', name: 'Table', - type: 'table', + type: 'table_v2', bgColor: '#10B981', position: { x: 0, y: 0 }, hideTargetHandle: true, @@ -1664,7 +1664,7 @@ export const TABLE_ROUNDTRIP_WORKFLOW: PreviewWorkflow = { { id: 'update', name: 'Table', - type: 'table', + type: 'table_v2', bgColor: '#10B981', position: { x: 680, y: 0 }, rows: [ diff --git a/apps/docs/components/workflow-preview/fit-view-after-init.tsx b/apps/docs/components/workflow-preview/fit-view-after-init.tsx new file mode 100644 index 00000000000..fda2379f8d2 --- /dev/null +++ b/apps/docs/components/workflow-preview/fit-view-after-init.tsx @@ -0,0 +1,20 @@ +'use client' + +import { useEffect } from 'react' +import { type FitViewOptions, useNodesInitialized, useReactFlow } from '@xyflow/react' + +interface FitViewAfterInitProps { + options: FitViewOptions +} + +/** Fits a v12 canvas only after every node has real measured dimensions. */ +export function FitViewAfterInit({ options }: FitViewAfterInitProps) { + const nodesInitialized = useNodesInitialized() + const { fitView } = useReactFlow() + + useEffect(() => { + if (nodesInitialized) void fitView(options) + }, [fitView, nodesInitialized, options]) + + return null +} diff --git a/apps/docs/components/workflow-preview/workflow-data.ts b/apps/docs/components/workflow-preview/workflow-data.ts index 4148fccaa70..346ef7fc4c2 100644 --- a/apps/docs/components/workflow-preview/workflow-data.ts +++ b/apps/docs/components/workflow-preview/workflow-data.ts @@ -4,7 +4,15 @@ import { getEdgeZIndex, getEdgeZIndexForTarget, } from '@sim/workflow-renderer' -import { type Edge, type Node, Position } from 'reactflow' +import { type Edge, Position } from '@xyflow/react' +import type { + DocsBlockData, + DocsBlockNodeType, +} from '@/components/workflow-preview/docs-block-node' +import type { + DocsContainerData, + DocsContainerNodeType, +} from '@/components/workflow-preview/docs-container-node' /** * Tool entry displayed as a chip on a block (e.g. an Agent's attached tools). @@ -51,6 +59,15 @@ export interface PreviewWorkflow { edges: Array<{ id: string; source: string; target: string; sourceHandle?: string }> } +export type PreviewNode = DocsBlockNodeType | DocsContainerNodeType + +export interface PreviewEdgeData extends Record { + animate: boolean + delay: number +} + +export type PreviewFlowEdge = Edge + export const BLOCK_STAGGER = 0.12 export const EASE_OUT: [number, number, number, number] = [0.16, 1, 0.3, 1] @@ -96,14 +113,14 @@ export function toReactFlowElements( workflow: PreviewWorkflow, animate = false, highlight: HighlightOptions = {} -): { nodes: Node[]; edges: Edge[] } { +): { nodes: PreviewNode[]; edges: PreviewFlowEdge[] } { const { highlightBlock, highlightEdge, selectedBlock } = highlight const hasHighlight = Boolean(highlightBlock || highlightEdge) const blockIndexMap = new Map(workflow.blocks.map((b, i) => [b.id, i])) const blocksById = new Map(workflow.blocks.map((b) => [b.id, b])) - const nodes: Node[] = workflow.blocks.map((block, index) => { + const nodes: PreviewNode[] = workflow.blocks.map((block, index) => { const isContainer = Boolean(block.size) const nestingDepth = getNestingDepth(block, blocksById) // Nested blocks are authored relative to their container; render them at @@ -113,13 +130,20 @@ export function toReactFlowElements( const position = parent ? { x: parent.position.x + block.position.x, y: parent.position.y + block.position.y } : block.position - return { + const commonNode = { id: block.id, - type: isContainer ? 'previewContainer' : 'previewBlock', position, zIndex: isContainer ? nestingDepth : block.parentId ? CONTAINER_CHILD_Z_BASE : BLOCK_Z_BASE, ...(block.size ? { style: { width: block.size.width, height: block.size.height } } : {}), - data: { + draggable: true, + selectable: false, + connectable: false, + sourcePosition: Position.Right, + targetPosition: Position.Left, + } + + if (isContainer) { + const data: DocsContainerData = { name: block.name, blockType: block.type, bgColor: block.bgColor, @@ -133,16 +157,37 @@ export function toReactFlowElements( animate, isHighlighted: highlightBlock === block.id || selectedBlock === block.id, isDimmed: hasHighlight && highlightBlock !== block.id, - }, - draggable: true, - selectable: false, - connectable: false, - sourcePosition: Position.Right, - targetPosition: Position.Left, + } + return { + ...commonNode, + type: 'previewContainer', + data, + } + } + + const data: DocsBlockData = { + name: block.name, + blockType: block.type, + bgColor: block.bgColor, + rows: block.rows, + branches: block.branches, + tools: block.tools, + hideTargetHandle: block.hideTargetHandle, + size: block.size, + parentId: block.parentId, + index, + animate, + isHighlighted: highlightBlock === block.id || selectedBlock === block.id, + isDimmed: hasHighlight && highlightBlock !== block.id, + } + return { + ...commonNode, + type: 'previewBlock', + data, } }) - const edges: Edge[] = workflow.edges.map((e) => { + const edges: PreviewFlowEdge[] = workflow.edges.map((e) => { const sourceIndex = blockIndexMap.get(e.source) ?? 0 const isEdgeHighlight = highlightEdge === e.id const dimmed = hasHighlight && !isEdgeHighlight diff --git a/apps/docs/components/workflow-preview/workflow-preview.tsx b/apps/docs/components/workflow-preview/workflow-preview.tsx index d87a60dabe8..ef3a1a90998 100644 --- a/apps/docs/components/workflow-preview/workflow-preview.tsx +++ b/apps/docs/components/workflow-preview/workflow-preview.tsx @@ -2,28 +2,31 @@ import { useCallback, useEffect, useMemo, useRef, useState } from 'react' import { Expand, X } from '@sim/emcn/icons' -import { domAnimation, LazyMotion, m } from 'framer-motion' -import ReactFlow, { +import { CANVAS_Z_INDEX_MODE, useCanvasColorMode } from '@sim/workflow-renderer' +import { applyEdgeChanges, applyNodeChanges, - type Edge, type EdgeProps, type EdgeTypes, getSmoothStepPath, - type Node, type NodeTypes, type OnEdgesChange, type OnNodesChange, + ReactFlow, ReactFlowProvider, -} from 'reactflow' -import 'reactflow/dist/style.css' +} from '@xyflow/react' +import { domAnimation, LazyMotion, m } from 'framer-motion' +import '@xyflow/react/dist/style.css' import { BLOCK_DISPLAY_WORKFLOWS } from '@/components/workflow-preview/block-display-workflows' import { BlockInspector } from '@/components/workflow-preview/block-inspector' import { DocsBlockNode } from '@/components/workflow-preview/docs-block-node' import { DocsContainerNode } from '@/components/workflow-preview/docs-container-node' +import { FitViewAfterInit } from '@/components/workflow-preview/fit-view-after-init' import { EASE_OUT, type PreviewBlock, + type PreviewFlowEdge, + type PreviewNode, type PreviewWorkflow, toReactFlowElements, } from '@/components/workflow-preview/workflow-data' @@ -50,7 +53,7 @@ function PreviewEdge({ targetPosition, style, data, -}: EdgeProps) { +}: EdgeProps) { const [edgePath] = getSmoothStepPath({ sourceX, sourceY, @@ -89,11 +92,11 @@ function PreviewEdge({ ) } -const NODE_TYPES: NodeTypes = { +const NODE_TYPES = { previewBlock: DocsBlockNode, previewContainer: DocsContainerNode, -} -const EDGE_TYPES: EdgeTypes = { previewEdge: PreviewEdge } +} satisfies NodeTypes +const EDGE_TYPES = { previewEdge: PreviewEdge } satisfies EdgeTypes const PRO_OPTIONS = { hideAttribution: true } const FIT_VIEW_OPTIONS = { padding: 0.25, maxZoom: 1 } as const const LIGHTBOX_FIT_VIEW_OPTIONS = { padding: 0.3, maxZoom: 1.4 } as const @@ -176,8 +179,10 @@ function PreviewFlow({ [workflow, animate, highlightBlock, highlightEdge, selectedBlock] ) - const [nodes, setNodes] = useState(initialNodes) - const [edges, setEdges] = useState(initialEdges) + const colorMode = useCanvasColorMode() + + const [nodes, setNodes] = useState(initialNodes) + const [edges, setEdges] = useState(initialEdges) /** * Apply data changes (highlight/selection) without discarding positions the @@ -194,42 +199,45 @@ function PreviewFlow({ setEdges(initialEdges) }, [initialNodes, initialEdges]) - const onNodesChange: OnNodesChange = useCallback( + const onNodesChange: OnNodesChange = useCallback( (changes) => setNodes((nds) => applyNodeChanges(changes, nds)), [] ) - const onEdgesChange: OnEdgesChange = useCallback( + const onEdgesChange: OnEdgesChange = useCallback( (changes) => setEdges((eds) => applyEdgeChanges(changes, eds)), [] ) return ( - onNodeClick(node.id) : undefined} - onPaneClick={onPaneClick} - nodeTypes={NODE_TYPES} - edgeTypes={EDGE_TYPES} - defaultEdgeOptions={{ type: 'previewEdge' }} - elementsSelectable={false} - nodesDraggable - nodesConnectable={false} - zoomOnScroll={interactive} - zoomOnDoubleClick={interactive} - panOnScroll={false} - zoomOnPinch - panOnDrag - preventScrolling={interactive} - autoPanOnNodeDrag={false} - proOptions={PRO_OPTIONS} - minZoom={0.1} - fitView - fitViewOptions={interactive ? LIGHTBOX_FIT_VIEW_OPTIONS : FIT_VIEW_OPTIONS} - className='h-full w-full' - /> + <> + + colorMode={colorMode} + zIndexMode={CANVAS_Z_INDEX_MODE} + nodes={nodes} + edges={edges} + onNodesChange={onNodesChange} + onEdgesChange={onEdgesChange} + onNodeClick={onNodeClick ? (_, node) => onNodeClick(node.id) : undefined} + onPaneClick={onPaneClick} + nodeTypes={NODE_TYPES} + edgeTypes={EDGE_TYPES} + defaultEdgeOptions={{ type: 'previewEdge' }} + elementsSelectable={false} + nodesDraggable + nodesConnectable={false} + zoomOnScroll={interactive} + zoomOnDoubleClick={interactive} + panOnScroll={false} + zoomOnPinch + panOnDrag + preventScrolling={interactive} + autoPanOnNodeDrag={false} + proOptions={PRO_OPTIONS} + minZoom={0.1} + className='h-full w-full [--xy-background-color:var(--bg)]' + /> + + ) } diff --git a/apps/docs/content/docs/agents/mcp.mdx b/apps/docs/content/docs/agents/mcp.mdx index 6464ed8bb6c..d6d04ee0371 100644 --- a/apps/docs/content/docs/agents/mcp.mdx +++ b/apps/docs/content/docs/agents/mcp.mdx @@ -85,6 +85,10 @@ Tool validation badges appear on servers with issues — for example, if a tool Self-hosted deployments can restrict which MCP server domains are allowed by setting the `ALLOWED_MCP_DOMAINS` environment variable (comma-separated list). When set, only servers on approved domains can be added. When unset, all domains are allowed. +This governs which domains may be used. It is separate from where those domains are allowed to resolve: an MCP server on a private address is reached by naming it in `EGRESS_ALLOWED_HOSTS` or `EGRESS_ALLOWED_IP_RANGES`, described in [Security](/platform/self-hosting/security#the-ssrf-boundary). Both checks apply. + +The allowlist covers the server URL itself. If the server requires OAuth, any endpoint its metadata names on a *different* origin than the server you configured is treated as content rather than as configuration, so that one has to be publicly routable. Endpoints on the server's own origin keep the server's reachability. + ## Using MCP Tools in Agents Once MCP servers are configured, their tools become available within your agent blocks: diff --git a/apps/docs/content/docs/cli/billing.mdx b/apps/docs/content/docs/cli/billing.mdx index 979d99ca7fb..2df8ad9d9dc 100644 --- a/apps/docs/content/docs/cli/billing.mdx +++ b/apps/docs/content/docs/cli/billing.mdx @@ -39,7 +39,7 @@ List credit usage events (a personal API key reports only your own events; a wor | Option | Required | Description | | --- | --- | --- | -| `--source ` | No | Filter by usage source; sim-chat combines Copilot and workspace chat. Accepted values: `workflow`, `wand`, `sim-chat`, `mcp_copilot`, `mothership_block`, `knowledge-base`, `voice-input`, `enrichment`, `voice-output`. | +| `--source ` | No | Filter by usage source; sim-chat combines Copilot and workspace chat. Accepted values: `workflow`, `wand`, `sim-chat`, `mcp_copilot`, `mothership_block`, `knowledge-base`, `voice-input`, `enrichment`, `voice-output`, `api-tool`. | | `--period ` | No | Billing period. Accepted values: `1d`, `7d`, `30d`, `all`, `custom`. | | `--start-date ` | No | Custom period start (ISO 8601). | | `--end-date ` | No | Custom period end (ISO 8601). | diff --git a/apps/docs/content/docs/cli/reference.mdx b/apps/docs/content/docs/cli/reference.mdx index 9bb1d4cf8af..2aacf33181f 100644 --- a/apps/docs/content/docs/cli/reference.mdx +++ b/apps/docs/content/docs/cli/reference.mdx @@ -263,7 +263,7 @@ sim billing logs [options] | Option | Required | Description | | --- | --- | --- | -| `--source ` | No | Filter by usage source; sim-chat combines Copilot and workspace chat. Accepted values: `workflow`, `wand`, `sim-chat`, `mcp_copilot`, `mothership_block`, `knowledge-base`, `voice-input`, `enrichment`, `voice-output`. | +| `--source ` | No | Filter by usage source; sim-chat combines Copilot and workspace chat. Accepted values: `workflow`, `wand`, `sim-chat`, `mcp_copilot`, `mothership_block`, `knowledge-base`, `voice-input`, `enrichment`, `voice-output`, `api-tool`. | | `--period ` | No | Billing period. Accepted values: `1d`, `7d`, `30d`, `all`, `custom`. | | `--start-date ` | No | Custom period start (ISO 8601). | | `--end-date ` | No | Custom period end (ISO 8601). | @@ -4366,6 +4366,36 @@ sim tables mkdir ## sim tools +### sim tools execute + +Run one built-in tool and print what it produced (personal API key required) + +```bash +sim tools execute [options] +``` + +**Arguments** + + + +| Argument | Required | Description | +| --- | --- | --- | +| `toolId` | Yes | Tool identifier. An unversioned name resolves to the newest version, and the response echoes the resolved id. | + + + +**Options** + + + +| Option | Required | Description | +| --- | --- | --- | +| `--input ` | No | Tool arguments as JSON, keyed by the parameter ids `sim tools get <toolId>` lists (JSON, or @path / @- to read a file or stdin). | +| `--credential-id ` | No | Credential to authenticate with, required for OAuth tools. | +| `--timeout ` | No | Seconds to wait before abandoning the call. | + + + ### sim tools get Get Tool @@ -5157,7 +5187,7 @@ sim workflows run [options] | `--input ` | No | Trigger input as JSON (JSON, or @path / @- to read a file or stdin). | | `--async` | No | Queue the run and return immediately. | | `--execution-timeout-seconds ` | No | Requested server-side timeout for an asynchronous run, in seconds. An upper bound, not the effective timeout: the run uses the smaller of this value and the plan's execution timeout, so requesting more than the plan allows silently yields the plan timeout. Rejected with `400` unless `async` is true. | -| `--select-output ` | No | Return blockName.field values from the streamed result (e.g. agent_1.content), requires --follow; missing fields are omitted (space-separated, or @path / @- with one value per line; @@value for a literal leading @). | +| `--select-output ` | No | Return streamed outputs as blockName.path or childWorkflowId.blockName.path; selecting a child workflow applies to every invocation, requires --follow (space-separated, or @path / @- with one value per line; @@value for a literal leading @). | | `--include-file-base64` | No | Inline eligible output files as base64 content. Rejected when `async` is true. | | `--no-include-file-base64` | No | Send --include-file-base64 as false. | | `--base64-max-bytes ` | No | Maximum total bytes of file content to inline as base64, lowering but never raising the server limit of 16 MiB. Rejected when `async` is true. | diff --git a/apps/docs/content/docs/cli/tools.mdx b/apps/docs/content/docs/cli/tools.mdx index bc43a7dd711..83bdb04bbc5 100644 --- a/apps/docs/content/docs/cli/tools.mdx +++ b/apps/docs/content/docs/cli/tools.mdx @@ -7,6 +7,36 @@ import { CommandTable } from '@/components/ui/command-table' Every command below also accepts the [global options](/cli/commands#global-options). +## Run one built-in tool and print what it produced + +```bash +sim tools execute [options] +``` + +Run one built-in tool and print what it produced (personal API key required) + +**Arguments** + + + +| Argument | Required | Description | +| --- | --- | --- | +| `toolId` | Yes | Tool identifier. An unversioned name resolves to the newest version, and the response echoes the resolved id. | + + + +**Options** + + + +| Option | Required | Description | +| --- | --- | --- | +| `--input ` | No | Tool arguments as JSON, keyed by the parameter ids `sim tools get <toolId>` lists (JSON, or @path / @- to read a file or stdin). | +| `--credential-id ` | No | Credential to authenticate with, required for OAuth tools. | +| `--timeout ` | No | Seconds to wait before abandoning the call. | + + + ## Get tool ```bash diff --git a/apps/docs/content/docs/cli/workflows.mdx b/apps/docs/content/docs/cli/workflows.mdx index 18c783a1c31..7169b5b2d58 100644 --- a/apps/docs/content/docs/cli/workflows.mdx +++ b/apps/docs/content/docs/cli/workflows.mdx @@ -532,7 +532,7 @@ sim workflows run [options] | `--input ` | No | Trigger input as JSON (JSON, or @path / @- to read a file or stdin). | | `--async` | No | Queue the run and return immediately. | | `--execution-timeout-seconds ` | No | Requested server-side timeout for an asynchronous run, in seconds. An upper bound, not the effective timeout: the run uses the smaller of this value and the plan's execution timeout, so requesting more than the plan allows silently yields the plan timeout. Rejected with `400` unless `async` is true. | -| `--select-output ` | No | Return blockName.field values from the streamed result (e.g. agent_1.content), requires --follow; missing fields are omitted (space-separated, or @path / @- with one value per line; @@value for a literal leading @). | +| `--select-output ` | No | Return streamed outputs as blockName.path or childWorkflowId.blockName.path; selecting a child workflow applies to every invocation, requires --follow (space-separated, or @path / @- with one value per line; @@value for a literal leading @). | | `--include-file-base64` | No | Inline eligible output files as base64 content. Rejected when `async` is true. | | `--no-include-file-base64` | No | Send --include-file-base64 as false. | | `--base64-max-bytes ` | No | Maximum total bytes of file content to inline as base64, lowering but never raising the server limit of 16 MiB. Rejected when `async` is true. | diff --git a/apps/docs/content/docs/integrations/ashby.mdx b/apps/docs/content/docs/integrations/ashby.mdx index 270efed9ce6..88a0fb3c75b 100644 --- a/apps/docs/content/docs/integrations/ashby.mdx +++ b/apps/docs/content/docs/integrations/ashby.mdx @@ -65,7 +65,7 @@ These are constraints of the Ashby API itself, not of the Sim block: ## Usage Instructions -Integrate Ashby into the workflow. Manage candidates (list, get, create, update, search, tag, anonymize), applications (list, get, create, delete, change stage, change source), jobs (list, get), job postings (list, get), offers (list, get), notes (list, create), interviews (list), custom field values (set one or many), and reference data (sources, tags, archive reasons, custom fields, departments, locations, openings, users). +Integrate Ashby into the workflow. Manage and search candidates, applications, jobs, users, and openings; transfer applications; upload resumes and candidate files; read application history and interview feedback; manage offers, notes, tags, stages, sources, and custom fields; and react to hiring lifecycle webhooks. @@ -80,6 +80,7 @@ Adds a tag to a candidate in Ashby and returns the updated candidate. | Parameter | Type | Required | Description | | --------- | ---- | -------- | ----------- | | `apiKey` | string | Yes | Ashby API Key | +| `onBehalfOfUserId` | string | No | Active Ashby user UUID to attribute this mutation to; the API key must permit on-behalf-of calls | | `candidateId` | string | Yes | The UUID of the candidate to add the tag to | | `tagId` | string | Yes | The UUID of the tag to add | @@ -102,6 +103,10 @@ Adds a tag to a candidate in Ashby and returns the updated candidate. | `openings` | json | List of openings \(id, openedAt, closedAt, isArchived, archivedAt, closeReasonId, openingState, latestVersion with identifier/description/authorId/createdAt/teamId/jobIds\[\]/targetHireDate/targetStartDate/isBackfill/employmentType/locationIds\[\]/hiringTeam\[\]/customFields\[\]\) | | `users` | json | List of users \(id, firstName, lastName, email, globalRole, isEnabled, updatedAt\) | | `interviewSchedules` | json | List of interview schedules \(id, applicationId, interviewStageId, interviewEvents\[\] with interviewerUserIds/startTime/endTime/feedbackLink/location/meetingLink/hasSubmittedFeedback, status, scheduledBy, createdAt, updatedAt\) | +| `interviewPlans` | json | Interview plans \(id, title, isArchived, createdAt, updatedAt\) | +| `interviewStages` | json | Ordered interview stages for a plan | +| `feedback` | json | Submitted application feedback with form definitions and values | +| `history` | json | Application stage history and allowed actions | | `tags` | json | List of candidate tags \(id, title, isArchived\) | | `id` | string | Resource UUID | | `name` | string | Resource name | @@ -119,8 +124,7 @@ Adds a tag to a candidate in Ashby and returns the updated candidate. | `applicationId` | string | UUID of the deleted application | | `moreDataAvailable` | boolean | Whether more pages exist | | `nextCursor` | string | Pagination cursor for next page | -| `syncToken` | string | Sync token for incremental updates | -| `nextSyncCursor` | string | Ashby's syncToken for the next incremental List Jobs run, exposed as a cursor so it stays readable in block output | +| `nextSyncCursor` | string | Ashby's opaque token for the next incremental list run, exposed as a cursor so it remains usable in workflow output | ### Ashby Anonymize Candidate @@ -131,6 +135,7 @@ Strips personally identifiable information from a candidate in Ashby. This does | Parameter | Type | Required | Description | | --------- | ---- | -------- | ----------- | | `apiKey` | string | Yes | Ashby API Key | +| `onBehalfOfUserId` | string | No | Active Ashby user UUID to attribute this mutation to; the API key must permit on-behalf-of calls | | `candidateId` | string | Yes | UUID of the candidate to anonymize | #### Output @@ -152,6 +157,10 @@ Strips personally identifiable information from a candidate in Ashby. This does | `openings` | json | List of openings \(id, openedAt, closedAt, isArchived, archivedAt, closeReasonId, openingState, latestVersion with identifier/description/authorId/createdAt/teamId/jobIds\[\]/targetHireDate/targetStartDate/isBackfill/employmentType/locationIds\[\]/hiringTeam\[\]/customFields\[\]\) | | `users` | json | List of users \(id, firstName, lastName, email, globalRole, isEnabled, updatedAt\) | | `interviewSchedules` | json | List of interview schedules \(id, applicationId, interviewStageId, interviewEvents\[\] with interviewerUserIds/startTime/endTime/feedbackLink/location/meetingLink/hasSubmittedFeedback, status, scheduledBy, createdAt, updatedAt\) | +| `interviewPlans` | json | Interview plans \(id, title, isArchived, createdAt, updatedAt\) | +| `interviewStages` | json | Ordered interview stages for a plan | +| `feedback` | json | Submitted application feedback with form definitions and values | +| `history` | json | Application stage history and allowed actions | | `tags` | json | List of candidate tags \(id, title, isArchived\) | | `id` | string | Resource UUID | | `name` | string | Resource name | @@ -169,8 +178,7 @@ Strips personally identifiable information from a candidate in Ashby. This does | `applicationId` | string | UUID of the deleted application | | `moreDataAvailable` | boolean | Whether more pages exist | | `nextCursor` | string | Pagination cursor for next page | -| `syncToken` | string | Sync token for incremental updates | -| `nextSyncCursor` | string | Ashby's syncToken for the next incremental List Jobs run, exposed as a cursor so it stays readable in block output | +| `nextSyncCursor` | string | Ashby's opaque token for the next incremental list run, exposed as a cursor so it remains usable in workflow output | ### Ashby Change Application Source @@ -181,6 +189,7 @@ Changes the source attributed to an existing application, so programmatically cr | Parameter | Type | Required | Description | | --------- | ---- | -------- | ----------- | | `apiKey` | string | Yes | Ashby API Key | +| `onBehalfOfUserId` | string | No | Active Ashby user UUID to attribute this mutation to; the API key must permit on-behalf-of calls | | `applicationId` | string | Yes | UUID of the application whose source should change | | `sourceId` | string | No | UUID of the source to attribute the application to, as returned by List Sources. Omit only when unsetSource is true. | | `unsetSource` | boolean | No | Set true to deliberately clear the application source. Required to unset, so that a missing or empty sourceId cannot wipe attribution by accident. | @@ -204,6 +213,10 @@ Changes the source attributed to an existing application, so programmatically cr | `openings` | json | List of openings \(id, openedAt, closedAt, isArchived, archivedAt, closeReasonId, openingState, latestVersion with identifier/description/authorId/createdAt/teamId/jobIds\[\]/targetHireDate/targetStartDate/isBackfill/employmentType/locationIds\[\]/hiringTeam\[\]/customFields\[\]\) | | `users` | json | List of users \(id, firstName, lastName, email, globalRole, isEnabled, updatedAt\) | | `interviewSchedules` | json | List of interview schedules \(id, applicationId, interviewStageId, interviewEvents\[\] with interviewerUserIds/startTime/endTime/feedbackLink/location/meetingLink/hasSubmittedFeedback, status, scheduledBy, createdAt, updatedAt\) | +| `interviewPlans` | json | Interview plans \(id, title, isArchived, createdAt, updatedAt\) | +| `interviewStages` | json | Ordered interview stages for a plan | +| `feedback` | json | Submitted application feedback with form definitions and values | +| `history` | json | Application stage history and allowed actions | | `tags` | json | List of candidate tags \(id, title, isArchived\) | | `id` | string | Resource UUID | | `name` | string | Resource name | @@ -221,8 +234,7 @@ Changes the source attributed to an existing application, so programmatically cr | `applicationId` | string | UUID of the deleted application | | `moreDataAvailable` | boolean | Whether more pages exist | | `nextCursor` | string | Pagination cursor for next page | -| `syncToken` | string | Sync token for incremental updates | -| `nextSyncCursor` | string | Ashby's syncToken for the next incremental List Jobs run, exposed as a cursor so it stays readable in block output | +| `nextSyncCursor` | string | Ashby's opaque token for the next incremental list run, exposed as a cursor so it remains usable in workflow output | ### Ashby Change Application Stage @@ -233,9 +245,11 @@ Moves an application to a different interview stage. Requires an archive reason | Parameter | Type | Required | Description | | --------- | ---- | -------- | ----------- | | `apiKey` | string | Yes | Ashby API Key | +| `onBehalfOfUserId` | string | No | Active Ashby user UUID to attribute this mutation to; the API key must permit on-behalf-of calls | | `applicationId` | string | Yes | The UUID of the application to update the stage of | | `interviewStageId` | string | Yes | The UUID of the interview stage to move the application to | | `archiveReasonId` | string | No | Archive reason UUID. Required when moving to an Archived stage, ignored otherwise | +| `archiveEmail` | json | No | Archive email configuration with communicationTemplateId and optional sendAt ISO 8601 timestamp. Pass null or omit to send no archive email. | #### Output @@ -256,6 +270,10 @@ Moves an application to a different interview stage. Requires an archive reason | `openings` | json | List of openings \(id, openedAt, closedAt, isArchived, archivedAt, closeReasonId, openingState, latestVersion with identifier/description/authorId/createdAt/teamId/jobIds\[\]/targetHireDate/targetStartDate/isBackfill/employmentType/locationIds\[\]/hiringTeam\[\]/customFields\[\]\) | | `users` | json | List of users \(id, firstName, lastName, email, globalRole, isEnabled, updatedAt\) | | `interviewSchedules` | json | List of interview schedules \(id, applicationId, interviewStageId, interviewEvents\[\] with interviewerUserIds/startTime/endTime/feedbackLink/location/meetingLink/hasSubmittedFeedback, status, scheduledBy, createdAt, updatedAt\) | +| `interviewPlans` | json | Interview plans \(id, title, isArchived, createdAt, updatedAt\) | +| `interviewStages` | json | Ordered interview stages for a plan | +| `feedback` | json | Submitted application feedback with form definitions and values | +| `history` | json | Application stage history and allowed actions | | `tags` | json | List of candidate tags \(id, title, isArchived\) | | `id` | string | Resource UUID | | `name` | string | Resource name | @@ -273,8 +291,7 @@ Moves an application to a different interview stage. Requires an archive reason | `applicationId` | string | UUID of the deleted application | | `moreDataAvailable` | boolean | Whether more pages exist | | `nextCursor` | string | Pagination cursor for next page | -| `syncToken` | string | Sync token for incremental updates | -| `nextSyncCursor` | string | Ashby's syncToken for the next incremental List Jobs run, exposed as a cursor so it stays readable in block output | +| `nextSyncCursor` | string | Ashby's opaque token for the next incremental list run, exposed as a cursor so it remains usable in workflow output | ### Ashby Create Application @@ -285,13 +302,15 @@ Creates a new application for a candidate on a job. Optionally specify interview | Parameter | Type | Required | Description | | --------- | ---- | -------- | ----------- | | `apiKey` | string | Yes | Ashby API Key | +| `onBehalfOfUserId` | string | No | Active Ashby user UUID to attribute this mutation to; the API key must permit on-behalf-of calls | | `candidateId` | string | Yes | The UUID of the candidate to consider for the job | | `jobId` | string | Yes | The UUID of the job to consider the candidate for | | `interviewPlanId` | string | No | UUID of the interview plan to use \(defaults to the job default plan\) | -| `interviewStageId` | string | No | UUID of the interview stage to place the application in \(defaults to first Lead stage\) | +| `interviewStageId` | string | No | UUID of the interview stage to place the application in, or FirstPreInterviewScreen \(defaults to the first Lead stage\) | | `sourceId` | string | No | UUID of the source to set on the application | | `creditedToUserId` | string | No | UUID of the user the application is credited to | | `createdAt` | string | No | ISO 8601 timestamp to set as the application creation date \(defaults to now\) | +| `applicationHistory` | json | No | Optional documented application history entries to create with the application | #### Output @@ -312,6 +331,10 @@ Creates a new application for a candidate on a job. Optionally specify interview | `openings` | json | List of openings \(id, openedAt, closedAt, isArchived, archivedAt, closeReasonId, openingState, latestVersion with identifier/description/authorId/createdAt/teamId/jobIds\[\]/targetHireDate/targetStartDate/isBackfill/employmentType/locationIds\[\]/hiringTeam\[\]/customFields\[\]\) | | `users` | json | List of users \(id, firstName, lastName, email, globalRole, isEnabled, updatedAt\) | | `interviewSchedules` | json | List of interview schedules \(id, applicationId, interviewStageId, interviewEvents\[\] with interviewerUserIds/startTime/endTime/feedbackLink/location/meetingLink/hasSubmittedFeedback, status, scheduledBy, createdAt, updatedAt\) | +| `interviewPlans` | json | Interview plans \(id, title, isArchived, createdAt, updatedAt\) | +| `interviewStages` | json | Ordered interview stages for a plan | +| `feedback` | json | Submitted application feedback with form definitions and values | +| `history` | json | Application stage history and allowed actions | | `tags` | json | List of candidate tags \(id, title, isArchived\) | | `id` | string | Resource UUID | | `name` | string | Resource name | @@ -329,8 +352,7 @@ Creates a new application for a candidate on a job. Optionally specify interview | `applicationId` | string | UUID of the deleted application | | `moreDataAvailable` | boolean | Whether more pages exist | | `nextCursor` | string | Pagination cursor for next page | -| `syncToken` | string | Sync token for incremental updates | -| `nextSyncCursor` | string | Ashby's syncToken for the next incremental List Jobs run, exposed as a cursor so it stays readable in block output | +| `nextSyncCursor` | string | Ashby's opaque token for the next incremental list run, exposed as a cursor so it remains usable in workflow output | ### Ashby Create Candidate @@ -341,6 +363,7 @@ Creates a new candidate record in Ashby. | Parameter | Type | Required | Description | | --------- | ---- | -------- | ----------- | | `apiKey` | string | Yes | Ashby API Key | +| `onBehalfOfUserId` | string | No | Active Ashby user UUID to attribute this mutation to; the API key must permit on-behalf-of calls | | `name` | string | Yes | The candidate full name | | `email` | string | No | Primary email address for the candidate | | `phoneNumber` | string | No | Primary phone number for the candidate | @@ -351,6 +374,7 @@ Creates a new candidate record in Ashby. | `creditedToUserId` | string | No | UUID of the Ashby user to credit with sourcing this candidate | | `createdAt` | string | No | Backdated creation timestamp in ISO 8601 \(e.g. 2024-01-01T00:00:00Z\). Defaults to now. | | `alternateEmailAddresses` | json | No | Array of additional email address strings to add to the candidate, e.g. \["a@x.com","b@y.com"\] | +| `location` | json | No | Candidate location object with optional city, region, and country | #### Output @@ -371,6 +395,10 @@ Creates a new candidate record in Ashby. | `openings` | json | List of openings \(id, openedAt, closedAt, isArchived, archivedAt, closeReasonId, openingState, latestVersion with identifier/description/authorId/createdAt/teamId/jobIds\[\]/targetHireDate/targetStartDate/isBackfill/employmentType/locationIds\[\]/hiringTeam\[\]/customFields\[\]\) | | `users` | json | List of users \(id, firstName, lastName, email, globalRole, isEnabled, updatedAt\) | | `interviewSchedules` | json | List of interview schedules \(id, applicationId, interviewStageId, interviewEvents\[\] with interviewerUserIds/startTime/endTime/feedbackLink/location/meetingLink/hasSubmittedFeedback, status, scheduledBy, createdAt, updatedAt\) | +| `interviewPlans` | json | Interview plans \(id, title, isArchived, createdAt, updatedAt\) | +| `interviewStages` | json | Ordered interview stages for a plan | +| `feedback` | json | Submitted application feedback with form definitions and values | +| `history` | json | Application stage history and allowed actions | | `tags` | json | List of candidate tags \(id, title, isArchived\) | | `id` | string | Resource UUID | | `name` | string | Resource name | @@ -388,8 +416,7 @@ Creates a new candidate record in Ashby. | `applicationId` | string | UUID of the deleted application | | `moreDataAvailable` | boolean | Whether more pages exist | | `nextCursor` | string | Pagination cursor for next page | -| `syncToken` | string | Sync token for incremental updates | -| `nextSyncCursor` | string | Ashby's syncToken for the next incremental List Jobs run, exposed as a cursor so it stays readable in block output | +| `nextSyncCursor` | string | Ashby's opaque token for the next incremental list run, exposed as a cursor so it remains usable in workflow output | ### Ashby Create Note @@ -400,6 +427,7 @@ Creates a note on a candidate in Ashby. Supports plain text and HTML content (bo | Parameter | Type | Required | Description | | --------- | ---- | -------- | ----------- | | `apiKey` | string | Yes | Ashby API Key | +| `onBehalfOfUserId` | string | No | Active Ashby user UUID to attribute this mutation to; the API key must permit on-behalf-of calls | | `candidateId` | string | Yes | The UUID of the candidate to add the note to | | `note` | string | Yes | The note content. If noteType is text/html, supports: <b>, <i>, <u>, <a>, <ul>, <ol>, <li>, <code>, <pre> | | `noteType` | string | No | Content type of the note: text/plain \(default\) or text/html | @@ -430,6 +458,7 @@ Permanently deletes an application in Ashby. Requires the candidatesDelete permi | Parameter | Type | Required | Description | | --------- | ---- | -------- | ----------- | | `apiKey` | string | Yes | Ashby API Key | +| `onBehalfOfUserId` | string | No | Active Ashby user UUID to attribute this mutation to; the API key must permit on-behalf-of calls | | `applicationId` | string | Yes | UUID of the application to delete | #### Output @@ -447,7 +476,9 @@ Retrieves full details about a single application by its ID. | Parameter | Type | Required | Description | | --------- | ---- | -------- | ----------- | | `apiKey` | string | Yes | Ashby API Key | -| `applicationId` | string | Yes | The UUID of the application to fetch | +| `applicationId` | string | No | The UUID of the application to fetch | +| `submittedFormInstanceId` | string | No | Submitted application-form instance UUID to use instead of applicationId | +| `expand` | json | No | Ashby-supported application expansions to include | #### Output @@ -468,6 +499,10 @@ Retrieves full details about a single application by its ID. | `openings` | json | List of openings \(id, openedAt, closedAt, isArchived, archivedAt, closeReasonId, openingState, latestVersion with identifier/description/authorId/createdAt/teamId/jobIds\[\]/targetHireDate/targetStartDate/isBackfill/employmentType/locationIds\[\]/hiringTeam\[\]/customFields\[\]\) | | `users` | json | List of users \(id, firstName, lastName, email, globalRole, isEnabled, updatedAt\) | | `interviewSchedules` | json | List of interview schedules \(id, applicationId, interviewStageId, interviewEvents\[\] with interviewerUserIds/startTime/endTime/feedbackLink/location/meetingLink/hasSubmittedFeedback, status, scheduledBy, createdAt, updatedAt\) | +| `interviewPlans` | json | Interview plans \(id, title, isArchived, createdAt, updatedAt\) | +| `interviewStages` | json | Ordered interview stages for a plan | +| `feedback` | json | Submitted application feedback with form definitions and values | +| `history` | json | Application stage history and allowed actions | | `tags` | json | List of candidate tags \(id, title, isArchived\) | | `id` | string | Resource UUID | | `name` | string | Resource name | @@ -485,8 +520,7 @@ Retrieves full details about a single application by its ID. | `applicationId` | string | UUID of the deleted application | | `moreDataAvailable` | boolean | Whether more pages exist | | `nextCursor` | string | Pagination cursor for next page | -| `syncToken` | string | Sync token for incremental updates | -| `nextSyncCursor` | string | Ashby's syncToken for the next incremental List Jobs run, exposed as a cursor so it stays readable in block output | +| `nextSyncCursor` | string | Ashby's opaque token for the next incremental list run, exposed as a cursor so it remains usable in workflow output | ### Ashby Get Candidate @@ -497,7 +531,8 @@ Retrieves full details about a single candidate by their ID. | Parameter | Type | Required | Description | | --------- | ---- | -------- | ----------- | | `apiKey` | string | Yes | Ashby API Key | -| `candidateId` | string | Yes | The UUID of the candidate to fetch | +| `candidateId` | string | No | The UUID of the candidate to fetch | +| `externalMappingId` | string | No | External mapping ID to use instead of the Ashby candidate UUID | #### Output @@ -518,6 +553,10 @@ Retrieves full details about a single candidate by their ID. | `openings` | json | List of openings \(id, openedAt, closedAt, isArchived, archivedAt, closeReasonId, openingState, latestVersion with identifier/description/authorId/createdAt/teamId/jobIds\[\]/targetHireDate/targetStartDate/isBackfill/employmentType/locationIds\[\]/hiringTeam\[\]/customFields\[\]\) | | `users` | json | List of users \(id, firstName, lastName, email, globalRole, isEnabled, updatedAt\) | | `interviewSchedules` | json | List of interview schedules \(id, applicationId, interviewStageId, interviewEvents\[\] with interviewerUserIds/startTime/endTime/feedbackLink/location/meetingLink/hasSubmittedFeedback, status, scheduledBy, createdAt, updatedAt\) | +| `interviewPlans` | json | Interview plans \(id, title, isArchived, createdAt, updatedAt\) | +| `interviewStages` | json | Ordered interview stages for a plan | +| `feedback` | json | Submitted application feedback with form definitions and values | +| `history` | json | Application stage history and allowed actions | | `tags` | json | List of candidate tags \(id, title, isArchived\) | | `id` | string | Resource UUID | | `name` | string | Resource name | @@ -535,8 +574,7 @@ Retrieves full details about a single candidate by their ID. | `applicationId` | string | UUID of the deleted application | | `moreDataAvailable` | boolean | Whether more pages exist | | `nextCursor` | string | Pagination cursor for next page | -| `syncToken` | string | Sync token for incremental updates | -| `nextSyncCursor` | string | Ashby's syncToken for the next incremental List Jobs run, exposed as a cursor so it stays readable in block output | +| `nextSyncCursor` | string | Ashby's opaque token for the next incremental list run, exposed as a cursor so it remains usable in workflow output | ### Ashby Get Job @@ -548,6 +586,7 @@ Retrieves full details about a single job by its ID. | --------- | ---- | -------- | ----------- | | `apiKey` | string | Yes | Ashby API Key | | `jobId` | string | Yes | The UUID of the job to fetch | +| `includeUnpublishedJobPostingIds` | boolean | No | Include IDs for unpublished job postings on this job | #### Output @@ -568,6 +607,10 @@ Retrieves full details about a single job by its ID. | `openings` | json | List of openings \(id, openedAt, closedAt, isArchived, archivedAt, closeReasonId, openingState, latestVersion with identifier/description/authorId/createdAt/teamId/jobIds\[\]/targetHireDate/targetStartDate/isBackfill/employmentType/locationIds\[\]/hiringTeam\[\]/customFields\[\]\) | | `users` | json | List of users \(id, firstName, lastName, email, globalRole, isEnabled, updatedAt\) | | `interviewSchedules` | json | List of interview schedules \(id, applicationId, interviewStageId, interviewEvents\[\] with interviewerUserIds/startTime/endTime/feedbackLink/location/meetingLink/hasSubmittedFeedback, status, scheduledBy, createdAt, updatedAt\) | +| `interviewPlans` | json | Interview plans \(id, title, isArchived, createdAt, updatedAt\) | +| `interviewStages` | json | Ordered interview stages for a plan | +| `feedback` | json | Submitted application feedback with form definitions and values | +| `history` | json | Application stage history and allowed actions | | `tags` | json | List of candidate tags \(id, title, isArchived\) | | `id` | string | Resource UUID | | `name` | string | Resource name | @@ -585,8 +628,7 @@ Retrieves full details about a single job by its ID. | `applicationId` | string | UUID of the deleted application | | `moreDataAvailable` | boolean | Whether more pages exist | | `nextCursor` | string | Pagination cursor for next page | -| `syncToken` | string | Sync token for incremental updates | -| `nextSyncCursor` | string | Ashby's syncToken for the next incremental List Jobs run, exposed as a cursor so it stays readable in block output | +| `nextSyncCursor` | string | Ashby's opaque token for the next incremental list run, exposed as a cursor so it remains usable in workflow output | ### Ashby Get Job Posting @@ -600,6 +642,7 @@ Retrieves full details about a single job posting by its ID. | `jobPostingId` | string | Yes | The UUID of the job posting to fetch | | `jobBoardId` | string | No | Optional job board UUID. If omitted, returns posting for the external job board. | | `expandJob` | boolean | No | Whether to expand and include the related job object in the response | +| `includeUnpublishedJobPostings` | boolean | No | Allow retrieval of an unpublished or draft job posting | #### Output @@ -669,6 +712,7 @@ Retrieves full details about a single offer by its ID. | --------- | ---- | -------- | ----------- | | `apiKey` | string | Yes | Ashby API Key | | `offerId` | string | Yes | The UUID of the offer to fetch | +| `excludeFormDefinition` | boolean | No | Omit the offer form definition from the response | #### Output @@ -689,6 +733,10 @@ Retrieves full details about a single offer by its ID. | `openings` | json | List of openings \(id, openedAt, closedAt, isArchived, archivedAt, closeReasonId, openingState, latestVersion with identifier/description/authorId/createdAt/teamId/jobIds\[\]/targetHireDate/targetStartDate/isBackfill/employmentType/locationIds\[\]/hiringTeam\[\]/customFields\[\]\) | | `users` | json | List of users \(id, firstName, lastName, email, globalRole, isEnabled, updatedAt\) | | `interviewSchedules` | json | List of interview schedules \(id, applicationId, interviewStageId, interviewEvents\[\] with interviewerUserIds/startTime/endTime/feedbackLink/location/meetingLink/hasSubmittedFeedback, status, scheduledBy, createdAt, updatedAt\) | +| `interviewPlans` | json | Interview plans \(id, title, isArchived, createdAt, updatedAt\) | +| `interviewStages` | json | Ordered interview stages for a plan | +| `feedback` | json | Submitted application feedback with form definitions and values | +| `history` | json | Application stage history and allowed actions | | `tags` | json | List of candidate tags \(id, title, isArchived\) | | `id` | string | Resource UUID | | `name` | string | Resource name | @@ -706,8 +754,122 @@ Retrieves full details about a single offer by its ID. | `applicationId` | string | UUID of the deleted application | | `moreDataAvailable` | boolean | Whether more pages exist | | `nextCursor` | string | Pagination cursor for next page | -| `syncToken` | string | Sync token for incremental updates | -| `nextSyncCursor` | string | Ashby's syncToken for the next incremental List Jobs run, exposed as a cursor so it stays readable in block output | +| `nextSyncCursor` | string | Ashby's opaque token for the next incremental list run, exposed as a cursor so it remains usable in workflow output | + +### Ashby Get Opening + +Retrieves one Ashby headcount opening by UUID. + +#### Input + +| Parameter | Type | Required | Description | +| --------- | ---- | -------- | ----------- | +| `apiKey` | string | Yes | Ashby API Key | +| `openingId` | string | Yes | Opening UUID | + +#### Output + +| Parameter | Type | Description | +| --------- | ---- | ----------- | +| `candidates` | json | List of candidates with rich fields \(id, name, primaryEmailAddress, primaryPhoneNumber, emailAddresses\[\], phoneNumbers\[\], socialLinks\[\], linkedInUrl, githubUrl, profileUrl, position, company, school, timezone, location with locationComponents\[\], tags\[\], applicationIds\[\], customFields\[\], resumeFileHandle, fileHandles\[\], source with sourceType, creditedToUser, fraudStatus, createdAt, updatedAt\) | +| `jobs` | json | List of jobs \(id, title, confidential, status, employmentType, locationId, departmentId, defaultInterviewPlanId, interviewPlanIds\[\], customFields\[\], jobPostingIds\[\], customRequisitionId, brandId, hiringTeam\[\], author, createdAt, updatedAt, openedAt, closedAt, location with address, openings\[\] with latestVersion\) | +| `applications` | json | List of applications \(id, status, customFields\[\], candidate summary, currentInterviewStage, source with sourceType, archiveReason with customFields\[\], archivedAt, job summary, creditedToUser, hiringTeam\[\], appliedViaJobPostingId, submitterClientIp, submitterUserAgent, createdAt, updatedAt\) | +| `notes` | json | List of notes \(id, content, author, isPrivate, createdAt\) | +| `offers` | json | List of offers \(id, decidedAt, applicationId, acceptanceStatus, offerStatus, latestVersion with id/startDate/salary/createdAt/openingId/customFields\[\]/fileHandles\[\]/author/approvalStatus\) | +| `archiveReasons` | json | List of archive reasons \(id, text, reasonType \[RejectedByCandidate/RejectedByOrg/Other\], isArchived\) | +| `sources` | json | List of sources \(id, title, isArchived, sourceType \{id, title, isArchived\}\) | +| `customFields` | json | For List Custom Fields, the field definitions \(id, title, isPrivate, fieldType, objectType, isArchived, isRequired, selectableValues\[\] \{label, value, isArchived\}\). For Set Custom Field Values, the field values written to the object \(id, title, isPrivate, valueLabel, value\) | +| `customField` | json | A single custom field value after a write \(id, title, isPrivate, valueLabel, value\) | +| `departments` | json | List of departments \(id, name, externalName, isArchived, parentId, createdAt, updatedAt\) | +| `locations` | json | List of locations \(id, name, externalName, isArchived, isRemote, workplaceType, parentLocationId, type, address with addressCountry/Region/Locality/postalCode/streetAddress\) | +| `jobPostings` | json | List of job postings \(id, title, jobId, departmentName, teamName, locationName, locationIds, workplaceType, employmentType, isListed, publishedDate, applicationDeadline, externalLink, applyLink, compensationTierSummary, shouldDisplayCompensationOnJobBoard, updatedAt\) | +| `openings` | json | List of openings \(id, openedAt, closedAt, isArchived, archivedAt, closeReasonId, openingState, latestVersion with identifier/description/authorId/createdAt/teamId/jobIds\[\]/targetHireDate/targetStartDate/isBackfill/employmentType/locationIds\[\]/hiringTeam\[\]/customFields\[\]\) | +| `users` | json | List of users \(id, firstName, lastName, email, globalRole, isEnabled, updatedAt\) | +| `interviewSchedules` | json | List of interview schedules \(id, applicationId, interviewStageId, interviewEvents\[\] with interviewerUserIds/startTime/endTime/feedbackLink/location/meetingLink/hasSubmittedFeedback, status, scheduledBy, createdAt, updatedAt\) | +| `interviewPlans` | json | Interview plans \(id, title, isArchived, createdAt, updatedAt\) | +| `interviewStages` | json | Ordered interview stages for a plan | +| `feedback` | json | Submitted application feedback with form definitions and values | +| `history` | json | Application stage history and allowed actions | +| `tags` | json | List of candidate tags \(id, title, isArchived\) | +| `id` | string | Resource UUID | +| `name` | string | Resource name | +| `title` | string | Job title or job posting title | +| `status` | string | Status | +| `candidate` | json | Candidate summary \(id, name, primaryEmailAddress, primaryPhoneNumber\). For full candidate fields use the candidates list output or the get/create/update candidate operations. | +| `job` | json | Job details \(id, title, status, employmentType, locationId, departmentId, hiringTeam\[\], author, location, openings\[\], createdAt, updatedAt\) | +| `application` | json | Application details \(id, status, customFields\[\], candidate, currentInterviewStage, source, archiveReason, job, hiringTeam\[\], createdAt, updatedAt\) | +| `offer` | json | Offer details \(id, decidedAt, applicationId, acceptanceStatus, offerStatus, latestVersion\) | +| `jobPosting` | json | Job posting details \(id, title, descriptionPlain, descriptionHtml, descriptionSocial, descriptionParts, departmentName, teamName, teamNameHierarchy\[\], jobId, locationName, locationIds, address, isRemote, workplaceType, employmentType, isListed, publishedDate, applicationDeadline, externalLink, applyLink, compensation, updatedAt, job \[included when expandJob=true\]\) | +| `content` | string | Note content | +| `author` | json | Note author \(id, firstName, lastName, email\) | +| `isPrivate` | boolean | Whether the note is private | +| `createdAt` | string | ISO 8601 creation timestamp | +| `applicationId` | string | UUID of the deleted application | +| `moreDataAvailable` | boolean | Whether more pages exist | +| `nextCursor` | string | Pagination cursor for next page | +| `nextSyncCursor` | string | Ashby's opaque token for the next incremental list run, exposed as a cursor so it remains usable in workflow output | + +### Ashby List Application Feedback + +Lists submitted interview feedback, optionally for one application, with pagination and incremental sync. + +#### Input + +| Parameter | Type | Required | Description | +| --------- | ---- | -------- | ----------- | +| `apiKey` | string | Yes | Ashby API Key | +| `applicationId` | string | No | Application UUID | +| `cursor` | string | No | Pagination cursor | +| `perPage` | number | No | Results per page \(1-100\) | +| `syncToken` | string | No | Opaque token from a completed prior sync run | +| `createdAfter` | string | No | Only feedback submitted after this ISO 8601 timestamp | + +#### Output + +| Parameter | Type | Description | +| --------- | ---- | ----------- | +| `feedback` | array | Submitted application feedback | +| ↳ `id` | string | Feedback UUID | +| ↳ `formDefinition` | json | Feedback form sections and documented field definitions | +| ↳ `feedbackFormDefinitionId` | string | Feedback form definition UUID | +| ↳ `applicationId` | string | Application UUID | +| ↳ `submittedValues` | json | Submitted field values keyed by form field path | +| ↳ `interviewId` | string | Interview UUID | +| ↳ `interviewEventId` | string | Interview event UUID | +| ↳ `applicationHistoryId` | string | Application history UUID | +| ↳ `submittedAt` | string | Submission timestamp | +| `moreDataAvailable` | boolean | Whether more pages exist | +| `nextCursor` | string | Next page cursor | +| `nextSyncCursor` | string | Next incremental sync token | + +### Ashby List Application History + +Lists the full stage history and allowed actions for an application. + +#### Input + +| Parameter | Type | Required | Description | +| --------- | ---- | -------- | ----------- | +| `apiKey` | string | Yes | Ashby API Key | +| `applicationId` | string | Yes | Application UUID | +| `cursor` | string | No | Pagination cursor | +| `perPage` | number | No | Results per page \(1-100\) | + +#### Output + +| Parameter | Type | Description | +| --------- | ---- | ----------- | +| `history` | array | Application stage history | +| ↳ `id` | string | History entry UUID | +| ↳ `stageId` | string | Stage UUID | +| ↳ `title` | string | Stage title | +| ↳ `enteredStageAt` | string | Stage entry timestamp | +| ↳ `leftStageAt` | string | Stage exit timestamp | +| ↳ `stageNumber` | number | Stage sequence number | +| ↳ `allowedActions` | array | Actions permitted at this history point | +| ↳ `actorId` | string | Acting user UUID | +| `moreDataAvailable` | boolean | Whether more pages exist | +| `nextCursor` | string | Next page cursor | ### Ashby List Applications @@ -720,9 +882,12 @@ Lists all applications in an Ashby organization with pagination and optional fil | `apiKey` | string | Yes | Ashby API Key | | `cursor` | string | No | Opaque pagination cursor from a previous response nextCursor value | | `perPage` | number | No | Number of results per page \(default 100\) | -| `status` | string | No | Filter by application status: Active, Hired, Archived, or Lead | +| `status` | string | No | Application status to include: Active, Hired, Archived, or Lead | | `jobId` | string | No | Filter applications by a specific job UUID | | `createdAfter` | string | No | Filter to applications created after this ISO 8601 timestamp \(e.g. 2024-01-01T00:00:00Z\) | +| `createdBefore` | string | No | Filter to applications created before this ISO 8601 timestamp | +| `syncToken` | string | No | Opaque token from a completed prior sync run | +| `expand` | json | No | Ashby-supported application expansions to request | #### Output @@ -731,6 +896,7 @@ Lists all applications in an Ashby organization with pagination and optional fil | `applications` | array | List of applications | | `moreDataAvailable` | boolean | Whether more pages of results exist | | `nextCursor` | string | Opaque cursor for fetching the next page | +| `nextSyncCursor` | string | Opaque token for the next incremental sync, returned on the final page | ### Ashby List Archive Reasons @@ -777,7 +943,7 @@ Lists all candidate tags configured in Ashby. | ↳ `isArchived` | boolean | Whether the tag is archived | | `moreDataAvailable` | boolean | Whether more pages of results exist | | `nextCursor` | string | Opaque cursor for fetching the next page | -| `syncToken` | string | Sync token to use for incremental updates in future requests | +| `nextSyncCursor` | string | Sync token to use for incremental updates in future requests | ### Ashby List Candidates @@ -791,6 +957,8 @@ Lists all candidates in an Ashby organization with cursor-based pagination. | `cursor` | string | No | Opaque pagination cursor from a previous response nextCursor value | | `perPage` | number | No | Number of results per page \(default 100\) | | `createdAfter` | string | No | Only return candidates created after this ISO 8601 timestamp \(e.g. 2024-01-01T00:00:00Z\) | +| `createdBefore` | string | No | Only return candidates created before this ISO 8601 timestamp | +| `syncToken` | string | No | Opaque token from a completed prior sync run | #### Output @@ -799,6 +967,7 @@ Lists all candidates in an Ashby organization with cursor-based pagination. | `candidates` | array | List of candidates | | `moreDataAvailable` | boolean | Whether more pages of results exist | | `nextCursor` | string | Opaque cursor for fetching the next page | +| `nextSyncCursor` | string | Opaque token for the next incremental sync, returned on the final page | ### Ashby List Custom Fields @@ -832,7 +1001,7 @@ Lists all custom field definitions configured in Ashby. | ↳ `isArchived` | boolean | Whether archived | | `moreDataAvailable` | boolean | Whether more pages of results exist | | `nextCursor` | string | Opaque cursor for fetching the next page | -| `syncToken` | string | Opaque sync token returned after the last page; pass on next sync | +| `nextSyncCursor` | string | Opaque sync token returned after the last page; pass on next sync | ### Ashby List Departments @@ -863,7 +1032,7 @@ Lists all departments in Ashby. | ↳ `extraData` | json | Free-form key-value metadata | | `moreDataAvailable` | boolean | Whether more pages of results exist | | `nextCursor` | string | Opaque cursor for fetching the next page | -| `syncToken` | string | Opaque sync token returned after the last page; pass on next sync | +| `nextSyncCursor` | string | Opaque sync token returned after the last page; pass on next sync | ### Ashby List Interview Schedules @@ -879,6 +1048,7 @@ Lists interview schedules in Ashby, optionally filtered by application or interv | `cursor` | string | No | Opaque pagination cursor from a previous response nextCursor value | | `perPage` | number | No | Number of results per page \(default 100\) | | `createdAfter` | string | No | Only return interview schedules created after this ISO 8601 timestamp \(e.g. 2024-01-01T00:00:00Z\) | +| `syncToken` | string | No | Opaque token from a completed prior sync run | #### Output @@ -906,6 +1076,58 @@ Lists interview schedules in Ashby, optionally filtered by application or interv | ↳ `hasSubmittedFeedback` | boolean | Whether any feedback has been submitted | | `moreDataAvailable` | boolean | Whether more pages of results exist | | `nextCursor` | string | Opaque cursor for fetching the next page | +| `nextSyncCursor` | string | Opaque token for the next incremental sync | + +### Ashby List Interview Plans + +Lists Ashby interview plans, including optional archived plans and incremental changes. + +#### Input + +| Parameter | Type | Required | Description | +| --------- | ---- | -------- | ----------- | +| `apiKey` | string | Yes | Ashby API Key | +| `includeArchived` | boolean | No | Include archived interview plans | +| `cursor` | string | No | Pagination cursor | +| `perPage` | number | No | Results per page \(1-100\) | +| `syncToken` | string | No | Opaque token from a completed prior sync run | + +#### Output + +| Parameter | Type | Description | +| --------- | ---- | ----------- | +| `interviewPlans` | array | Interview plans | +| ↳ `id` | string | Interview plan UUID | +| ↳ `title` | string | Plan title | +| ↳ `isArchived` | boolean | Whether archived | +| ↳ `createdAt` | string | Creation timestamp | +| ↳ `updatedAt` | string | Last update timestamp | +| `moreDataAvailable` | boolean | Whether more pages exist | +| `nextCursor` | string | Next page cursor | +| `nextSyncCursor` | string | Next incremental sync token | + +### Ashby List Interview Stages + +Lists the ordered stages in an Ashby interview plan. + +#### Input + +| Parameter | Type | Required | Description | +| --------- | ---- | -------- | ----------- | +| `apiKey` | string | Yes | Ashby API Key | +| `interviewPlanId` | string | Yes | Interview plan UUID | + +#### Output + +| Parameter | Type | Description | +| --------- | ---- | ----------- | +| `interviewStages` | array | Interview stages in plan order | +| ↳ `id` | string | Stage UUID | +| ↳ `title` | string | Stage title | +| ↳ `type` | string | Stage type | +| ↳ `interviewPlanId` | string | Parent plan UUID | +| ↳ `orderInInterviewPlan` | number | Zero-based plan order | +| ↳ `interviewStageGroupId` | string | Stage group UUID | ### Ashby List Job Postings @@ -960,12 +1182,13 @@ Lists all jobs in an Ashby organization. By default returns Open, Closed, and Ar | `cursor` | string | No | Opaque pagination cursor from a previous response nextCursor value | | `perPage` | number | No | Number of results per page \(default and max 100\). Ashby silently caps larger values rather than erroring. | | `syncToken` | string | No | Opaque token from a prior sync to fetch only jobs changed since then. Ashby only returns a new syncToken on the last page, so drain moreDataAvailable/nextCursor before persisting it. | -| `status` | string | No | Filter by job status: Open, Closed, Archived, or Draft | +| `status` | array | No | One job status or an array of statuses to include: Open, Closed, Archived, or Draft | | `createdAfter` | string | No | Only return jobs created after this ISO 8601 timestamp \(e.g. 2024-01-01T00:00:00Z\) | | `openedAfter` | string | No | Only return jobs opened after this ISO 8601 timestamp | | `openedBefore` | string | No | Only return jobs opened before this ISO 8601 timestamp | | `closedAfter` | string | No | Only return jobs closed after this ISO 8601 timestamp | | `closedBefore` | string | No | Only return jobs closed before this ISO 8601 timestamp | +| `includeUnpublishedJobPostingsIds` | boolean | No | Include IDs for unpublished job postings on each job | #### Output @@ -1013,7 +1236,7 @@ Lists all locations configured in Ashby. | ↳ `extraData` | json | Free-form key-value metadata | | `moreDataAvailable` | boolean | Whether more pages of results exist | | `nextCursor` | string | Opaque cursor for fetching the next page | -| `syncToken` | string | Opaque sync token returned after the last page; pass on next sync | +| `nextSyncCursor` | string | Opaque sync token returned after the last page; pass on next sync | ### Ashby List Notes @@ -1026,7 +1249,7 @@ Lists all notes on a candidate with pagination support. | `apiKey` | string | Yes | Ashby API Key | | `candidateId` | string | Yes | The UUID of the candidate to list notes for | | `cursor` | string | No | Opaque pagination cursor from a previous response nextCursor value | -| `perPage` | number | No | Number of results per page | +| `perPage` | number | No | Number of results per page \(1-100\) | #### Output @@ -1059,6 +1282,9 @@ Lists all offers with their latest version in an Ashby organization. | `createdAfter` | string | No | Only return offers created after this ISO 8601 timestamp \(e.g. 2024-01-01T00:00:00Z\) | | `syncToken` | string | No | Opaque token from a prior sync to fetch only items changed since then | | `applicationId` | string | No | Return only offers for the specified application UUID | +| `offerStatus` | json | No | Non-empty array of offer process statuses to include | +| `acceptanceStatus` | json | No | Non-empty array of offer acceptance statuses to include | +| `approvalStatus` | json | No | Non-empty array of latest-version approval statuses to include | #### Output @@ -1067,6 +1293,7 @@ Lists all offers with their latest version in an Ashby organization. | `offers` | array | List of offers | | `moreDataAvailable` | boolean | Whether more pages of results exist | | `nextCursor` | string | Opaque cursor for fetching the next page | +| `nextSyncCursor` | string | Opaque token for the next incremental sync, returned on the final page | ### Ashby List Openings @@ -1080,6 +1307,7 @@ Lists all openings in Ashby with pagination. | `cursor` | string | No | Opaque pagination cursor from a previous response nextCursor value | | `perPage` | number | No | Number of results per page \(default 100\) | | `createdAfter` | string | No | Only return openings created after this ISO 8601 timestamp \(e.g. 2024-01-01T00:00:00Z\) | +| `syncToken` | string | No | Opaque token from a completed prior sync run | #### Output @@ -1087,6 +1315,7 @@ Lists all openings in Ashby with pagination. | --------- | ---- | ----------- | | `moreDataAvailable` | boolean | Whether more pages of results exist | | `nextCursor` | string | Opaque cursor for fetching the next page | +| `nextSyncCursor` | string | Opaque token for the next incremental sync | ### Ashby List Sources @@ -1124,6 +1353,7 @@ Lists all users in Ashby with pagination. | `cursor` | string | No | Opaque pagination cursor from a previous response nextCursor value | | `perPage` | number | No | Number of results per page \(default 100\) | | `includeDeactivated` | boolean | No | When true, includes deactivated users in results \(default false\) | +| `syncToken` | string | No | Opaque token from a completed prior sync run | #### Output @@ -1132,6 +1362,7 @@ Lists all users in Ashby with pagination. | `users` | array | List of users | | `moreDataAvailable` | boolean | Whether more pages of results exist | | `nextCursor` | string | Opaque cursor for fetching the next page | +| `nextSyncCursor` | string | Opaque token for the next incremental sync | ### Ashby Remove Candidate Tag @@ -1142,6 +1373,7 @@ Removes a tag from a candidate in Ashby and returns the updated candidate. | Parameter | Type | Required | Description | | --------- | ---- | -------- | ----------- | | `apiKey` | string | Yes | Ashby API Key | +| `onBehalfOfUserId` | string | No | Active Ashby user UUID to attribute this mutation to; the API key must permit on-behalf-of calls | | `candidateId` | string | Yes | The UUID of the candidate to remove the tag from | | `tagId` | string | Yes | The UUID of the tag to remove | @@ -1164,6 +1396,10 @@ Removes a tag from a candidate in Ashby and returns the updated candidate. | `openings` | json | List of openings \(id, openedAt, closedAt, isArchived, archivedAt, closeReasonId, openingState, latestVersion with identifier/description/authorId/createdAt/teamId/jobIds\[\]/targetHireDate/targetStartDate/isBackfill/employmentType/locationIds\[\]/hiringTeam\[\]/customFields\[\]\) | | `users` | json | List of users \(id, firstName, lastName, email, globalRole, isEnabled, updatedAt\) | | `interviewSchedules` | json | List of interview schedules \(id, applicationId, interviewStageId, interviewEvents\[\] with interviewerUserIds/startTime/endTime/feedbackLink/location/meetingLink/hasSubmittedFeedback, status, scheduledBy, createdAt, updatedAt\) | +| `interviewPlans` | json | Interview plans \(id, title, isArchived, createdAt, updatedAt\) | +| `interviewStages` | json | Ordered interview stages for a plan | +| `feedback` | json | Submitted application feedback with form definitions and values | +| `history` | json | Application stage history and allowed actions | | `tags` | json | List of candidate tags \(id, title, isArchived\) | | `id` | string | Resource UUID | | `name` | string | Resource name | @@ -1181,8 +1417,7 @@ Removes a tag from a candidate in Ashby and returns the updated candidate. | `applicationId` | string | UUID of the deleted application | | `moreDataAvailable` | boolean | Whether more pages exist | | `nextCursor` | string | Pagination cursor for next page | -| `syncToken` | string | Sync token for incremental updates | -| `nextSyncCursor` | string | Ashby's syncToken for the next incremental List Jobs run, exposed as a cursor so it stays readable in block output | +| `nextSyncCursor` | string | Ashby's opaque token for the next incremental list run, exposed as a cursor so it remains usable in workflow output | ### Ashby Search Candidates @@ -1195,6 +1430,7 @@ Searches for candidates by name and/or email with AND logic. Results are limited | `apiKey` | string | Yes | Ashby API Key | | `name` | string | No | Candidate name to search for \(combined with email using AND logic\) | | `email` | string | No | Candidate email to search for \(combined with name using AND logic\) | +| `limit` | number | No | Maximum matches to return \(1-100\) | #### Output @@ -1202,6 +1438,96 @@ Searches for candidates by name and/or email with AND logic. Results are limited | --------- | ---- | ----------- | | `candidates` | array | Matching candidates \(max 100 results\) | +### Ashby Search Jobs + +Searches Ashby jobs by title and/or requisition ID. Provide at least one of these filters. + +#### Input + +| Parameter | Type | Required | Description | +| --------- | ---- | -------- | ----------- | +| `apiKey` | string | Yes | Ashby API Key | +| `title` | string | No | Job title search text | +| `requisitionId` | string | No | Custom requisition ID | +| `limit` | number | No | Maximum matches \(1-100\) | + +#### Output + +| Parameter | Type | Description | +| --------- | ---- | ----------- | +| `jobs` | array | Matching jobs | +| `moreDataAvailable` | boolean | Whether more matches exist | + +### Ashby Search Openings + +Searches Ashby headcount openings by human-readable identifier. + +#### Input + +| Parameter | Type | Required | Description | +| --------- | ---- | -------- | ----------- | +| `apiKey` | string | Yes | Ashby API Key | +| `identifier` | string | Yes | Opening identifier | + +#### Output + +| Parameter | Type | Description | +| --------- | ---- | ----------- | +| `candidates` | json | List of candidates with rich fields \(id, name, primaryEmailAddress, primaryPhoneNumber, emailAddresses\[\], phoneNumbers\[\], socialLinks\[\], linkedInUrl, githubUrl, profileUrl, position, company, school, timezone, location with locationComponents\[\], tags\[\], applicationIds\[\], customFields\[\], resumeFileHandle, fileHandles\[\], source with sourceType, creditedToUser, fraudStatus, createdAt, updatedAt\) | +| `jobs` | json | List of jobs \(id, title, confidential, status, employmentType, locationId, departmentId, defaultInterviewPlanId, interviewPlanIds\[\], customFields\[\], jobPostingIds\[\], customRequisitionId, brandId, hiringTeam\[\], author, createdAt, updatedAt, openedAt, closedAt, location with address, openings\[\] with latestVersion\) | +| `applications` | json | List of applications \(id, status, customFields\[\], candidate summary, currentInterviewStage, source with sourceType, archiveReason with customFields\[\], archivedAt, job summary, creditedToUser, hiringTeam\[\], appliedViaJobPostingId, submitterClientIp, submitterUserAgent, createdAt, updatedAt\) | +| `notes` | json | List of notes \(id, content, author, isPrivate, createdAt\) | +| `offers` | json | List of offers \(id, decidedAt, applicationId, acceptanceStatus, offerStatus, latestVersion with id/startDate/salary/createdAt/openingId/customFields\[\]/fileHandles\[\]/author/approvalStatus\) | +| `archiveReasons` | json | List of archive reasons \(id, text, reasonType \[RejectedByCandidate/RejectedByOrg/Other\], isArchived\) | +| `sources` | json | List of sources \(id, title, isArchived, sourceType \{id, title, isArchived\}\) | +| `customFields` | json | For List Custom Fields, the field definitions \(id, title, isPrivate, fieldType, objectType, isArchived, isRequired, selectableValues\[\] \{label, value, isArchived\}\). For Set Custom Field Values, the field values written to the object \(id, title, isPrivate, valueLabel, value\) | +| `customField` | json | A single custom field value after a write \(id, title, isPrivate, valueLabel, value\) | +| `departments` | json | List of departments \(id, name, externalName, isArchived, parentId, createdAt, updatedAt\) | +| `locations` | json | List of locations \(id, name, externalName, isArchived, isRemote, workplaceType, parentLocationId, type, address with addressCountry/Region/Locality/postalCode/streetAddress\) | +| `jobPostings` | json | List of job postings \(id, title, jobId, departmentName, teamName, locationName, locationIds, workplaceType, employmentType, isListed, publishedDate, applicationDeadline, externalLink, applyLink, compensationTierSummary, shouldDisplayCompensationOnJobBoard, updatedAt\) | +| `openings` | json | List of openings \(id, openedAt, closedAt, isArchived, archivedAt, closeReasonId, openingState, latestVersion with identifier/description/authorId/createdAt/teamId/jobIds\[\]/targetHireDate/targetStartDate/isBackfill/employmentType/locationIds\[\]/hiringTeam\[\]/customFields\[\]\) | +| `users` | json | List of users \(id, firstName, lastName, email, globalRole, isEnabled, updatedAt\) | +| `interviewSchedules` | json | List of interview schedules \(id, applicationId, interviewStageId, interviewEvents\[\] with interviewerUserIds/startTime/endTime/feedbackLink/location/meetingLink/hasSubmittedFeedback, status, scheduledBy, createdAt, updatedAt\) | +| `interviewPlans` | json | Interview plans \(id, title, isArchived, createdAt, updatedAt\) | +| `interviewStages` | json | Ordered interview stages for a plan | +| `feedback` | json | Submitted application feedback with form definitions and values | +| `history` | json | Application stage history and allowed actions | +| `tags` | json | List of candidate tags \(id, title, isArchived\) | +| `id` | string | Resource UUID | +| `name` | string | Resource name | +| `title` | string | Job title or job posting title | +| `status` | string | Status | +| `candidate` | json | Candidate summary \(id, name, primaryEmailAddress, primaryPhoneNumber\). For full candidate fields use the candidates list output or the get/create/update candidate operations. | +| `job` | json | Job details \(id, title, status, employmentType, locationId, departmentId, hiringTeam\[\], author, location, openings\[\], createdAt, updatedAt\) | +| `application` | json | Application details \(id, status, customFields\[\], candidate, currentInterviewStage, source, archiveReason, job, hiringTeam\[\], createdAt, updatedAt\) | +| `offer` | json | Offer details \(id, decidedAt, applicationId, acceptanceStatus, offerStatus, latestVersion\) | +| `jobPosting` | json | Job posting details \(id, title, descriptionPlain, descriptionHtml, descriptionSocial, descriptionParts, departmentName, teamName, teamNameHierarchy\[\], jobId, locationName, locationIds, address, isRemote, workplaceType, employmentType, isListed, publishedDate, applicationDeadline, externalLink, applyLink, compensation, updatedAt, job \[included when expandJob=true\]\) | +| `content` | string | Note content | +| `author` | json | Note author \(id, firstName, lastName, email\) | +| `isPrivate` | boolean | Whether the note is private | +| `createdAt` | string | ISO 8601 creation timestamp | +| `applicationId` | string | UUID of the deleted application | +| `moreDataAvailable` | boolean | Whether more pages exist | +| `nextCursor` | string | Pagination cursor for next page | +| `nextSyncCursor` | string | Ashby's opaque token for the next incremental list run, exposed as a cursor so it remains usable in workflow output | + +### Ashby Search Users + +Searches Ashby users by exact email address. + +#### Input + +| Parameter | Type | Required | Description | +| --------- | ---- | -------- | ----------- | +| `apiKey` | string | Yes | Ashby API Key | +| `email` | string | Yes | User email address | + +#### Output + +| Parameter | Type | Description | +| --------- | ---- | ----------- | +| `users` | array | Matching users | + ### Ashby Set Custom Field Value Sets the value of a single custom field on an Ashby Application, Candidate, Job, or Opening. Custom fields are the only way to annotate a job or req, since Ashby has no job notes and no job tags. Requires the candidatesWrite permission. @@ -1211,6 +1537,7 @@ Sets the value of a single custom field on an Ashby Application, Candidate, Job, | Parameter | Type | Required | Description | | --------- | ---- | -------- | ----------- | | `apiKey` | string | Yes | Ashby API Key | +| `onBehalfOfUserId` | string | No | Active Ashby user UUID to attribute this mutation to; the API key must permit on-behalf-of calls | | `objectId` | string | Yes | UUID of the object to set the field on \(application, candidate, job, or opening\) | | `objectType` | string | Yes | Type of the object: Application, Candidate, Job, or Opening | | `fieldId` | string | Yes | UUID of the custom field definition to set, as returned by List Custom Fields. This is the field definition ID, not the ID of a value already on the object. | @@ -1231,6 +1558,7 @@ Sets several custom field values on one Ashby Application, Candidate, Job, or Op | Parameter | Type | Required | Description | | --------- | ---- | -------- | ----------- | | `apiKey` | string | Yes | Ashby API Key | +| `onBehalfOfUserId` | string | No | Active Ashby user UUID to attribute this mutation to; the API key must permit on-behalf-of calls | | `objectId` | string | Yes | UUID of the object to set the fields on \(application, candidate, job, or opening\) | | `objectType` | string | Yes | Type of the object: Application, Candidate, Job, or Opening | | `values` | json | Yes | Array of at least one \{ fieldId, fieldValue \} pair. fieldId is a custom field definition UUID from List Custom Fields. fieldValue matches the field type: boolean, number, string, string array \(MultiValueSelect\), or an object for Currency, NumberRange, CompensationRange, and Location. Pass null as a fieldValue to clear that field. | @@ -1254,6 +1582,10 @@ Sets several custom field values on one Ashby Application, Candidate, Job, or Op | `openings` | json | List of openings \(id, openedAt, closedAt, isArchived, archivedAt, closeReasonId, openingState, latestVersion with identifier/description/authorId/createdAt/teamId/jobIds\[\]/targetHireDate/targetStartDate/isBackfill/employmentType/locationIds\[\]/hiringTeam\[\]/customFields\[\]\) | | `users` | json | List of users \(id, firstName, lastName, email, globalRole, isEnabled, updatedAt\) | | `interviewSchedules` | json | List of interview schedules \(id, applicationId, interviewStageId, interviewEvents\[\] with interviewerUserIds/startTime/endTime/feedbackLink/location/meetingLink/hasSubmittedFeedback, status, scheduledBy, createdAt, updatedAt\) | +| `interviewPlans` | json | Interview plans \(id, title, isArchived, createdAt, updatedAt\) | +| `interviewStages` | json | Ordered interview stages for a plan | +| `feedback` | json | Submitted application feedback with form definitions and values | +| `history` | json | Application stage history and allowed actions | | `tags` | json | List of candidate tags \(id, title, isArchived\) | | `id` | string | Resource UUID | | `name` | string | Resource name | @@ -1271,8 +1603,7 @@ Sets several custom field values on one Ashby Application, Candidate, Job, or Op | `applicationId` | string | UUID of the deleted application | | `moreDataAvailable` | boolean | Whether more pages exist | | `nextCursor` | string | Pagination cursor for next page | -| `syncToken` | string | Sync token for incremental updates | -| `nextSyncCursor` | string | Ashby's syncToken for the next incremental List Jobs run, exposed as a cursor so it stays readable in block output | +| `nextSyncCursor` | string | Ashby's opaque token for the next incremental list run, exposed as a cursor so it remains usable in workflow output | ### Ashby Update Candidate @@ -1283,19 +1614,23 @@ Updates an existing candidate record in Ashby. Only provided fields are changed. | Parameter | Type | Required | Description | | --------- | ---- | -------- | ----------- | | `apiKey` | string | Yes | Ashby API Key | +| `onBehalfOfUserId` | string | No | Active Ashby user UUID to attribute this mutation to; the API key must permit on-behalf-of calls | | `candidateId` | string | Yes | The UUID of the candidate to update | -| `name` | string | No | Updated full name | -| `email` | string | No | Updated primary email address | -| `phoneNumber` | string | No | Updated primary phone number | -| `linkedInUrl` | string | No | LinkedIn profile URL | -| `githubUrl` | string | No | GitHub profile URL | -| `websiteUrl` | string | No | Personal website URL | -| `alternateEmail` | string | No | An additional email address to add to the candidate | +| `name` | string | No | Updated full name, or null | +| `email` | string | No | Updated primary email address, or null | +| `phoneNumber` | string | No | Updated primary phone number, or null | +| `linkedInUrl` | string | No | LinkedIn profile URL, or null | +| `githubUrl` | string | No | GitHub profile URL, or null | +| `websiteUrl` | string | No | Personal website URL, or null | +| `alternateEmail` | string | No | An additional email address to add to the candidate, or null | | `sourceId` | string | No | UUID of the source to attribute the candidate to | | `creditedToUserId` | string | No | UUID of the Ashby user to credit with sourcing this candidate | -| `createdAt` | string | No | Backdated creation timestamp in ISO 8601. Only updatable if originally backdated. | -| `sendNotifications` | boolean | No | Whether to send a notification when the source is updated \(default true\) | -| `socialLinks` | json | No | Array of social link objects to set on the candidate, e.g. \[\{"type":"LinkedIn","url":"https://..."\}\]. Replaces existing social links. | +| `clearSource` | boolean | No | Explicitly clear the candidate source; mutually exclusive with sourceId | +| `clearCreditedToUser` | boolean | No | Explicitly clear the credited Ashby user; mutually exclusive with creditedToUserId | +| `location` | json | No | Candidate location object with optional city, region, and country; the object and its fields accept null | +| `createdAt` | string | No | Backdated creation timestamp in ISO 8601, or null. Only updatable if originally backdated. | +| `sendNotifications` | boolean | No | Whether to send a notification when the source is updated \(default true\), or null | +| `socialLinks` | json | No | Array of social link objects to set on the candidate, e.g. \[\{"type":"LinkedIn","url":"https://..."\}\]. Replaces existing links; pass \[\] to clear them. Null is also accepted. Mutually exclusive with linkedInUrl, githubUrl, and websiteUrl. | #### Output @@ -1316,6 +1651,10 @@ Updates an existing candidate record in Ashby. Only provided fields are changed. | `openings` | json | List of openings \(id, openedAt, closedAt, isArchived, archivedAt, closeReasonId, openingState, latestVersion with identifier/description/authorId/createdAt/teamId/jobIds\[\]/targetHireDate/targetStartDate/isBackfill/employmentType/locationIds\[\]/hiringTeam\[\]/customFields\[\]\) | | `users` | json | List of users \(id, firstName, lastName, email, globalRole, isEnabled, updatedAt\) | | `interviewSchedules` | json | List of interview schedules \(id, applicationId, interviewStageId, interviewEvents\[\] with interviewerUserIds/startTime/endTime/feedbackLink/location/meetingLink/hasSubmittedFeedback, status, scheduledBy, createdAt, updatedAt\) | +| `interviewPlans` | json | Interview plans \(id, title, isArchived, createdAt, updatedAt\) | +| `interviewStages` | json | Ordered interview stages for a plan | +| `feedback` | json | Submitted application feedback with form definitions and values | +| `history` | json | Application stage history and allowed actions | | `tags` | json | List of candidate tags \(id, title, isArchived\) | | `id` | string | Resource UUID | | `name` | string | Resource name | @@ -1333,76 +1672,190 @@ Updates an existing candidate record in Ashby. Only provided fields are changed. | `applicationId` | string | UUID of the deleted application | | `moreDataAvailable` | boolean | Whether more pages exist | | `nextCursor` | string | Pagination cursor for next page | -| `syncToken` | string | Sync token for incremental updates | -| `nextSyncCursor` | string | Ashby's syncToken for the next incremental List Jobs run, exposed as a cursor so it stays readable in block output | - - +| `nextSyncCursor` | string | Ashby's opaque token for the next incremental list run, exposed as a cursor so it remains usable in workflow output | -## Triggers - -A **Trigger** is a block that starts a workflow when an event happens in this service. - -### Ashby Application Submitted +### Ashby Transfer Application -Trigger workflow when a new application is submitted +Transfers an application to another job, interview plan, and stage. -#### Configuration +#### Input | Parameter | Type | Required | Description | | --------- | ---- | -------- | ----------- | -| `apiKey` | string | Yes | API Key | +| `apiKey` | string | Yes | Ashby API Key | +| `onBehalfOfUserId` | string | No | Active Ashby user UUID to attribute this mutation to; the API key must permit on-behalf-of calls | +| `applicationId` | string | Yes | Application UUID | +| `jobId` | string | Yes | Destination job UUID | +| `interviewPlanId` | string | Yes | Destination interview plan UUID | +| `interviewStageId` | string | Yes | Destination interview stage UUID | +| `startAutomaticActivities` | boolean | No | Start automatic activities configured for the destination stage | #### Output | Parameter | Type | Description | | --------- | ---- | ----------- | -| `action` | string | The webhook event type \(e.g., applicationSubmit, candidateHire\) | -| `application` | object | application output from the tool | -| ↳ `id` | string | Application UUID | -| ↳ `createdAt` | string | Application creation timestamp \(ISO 8601\) | -| ↳ `updatedAt` | string | Application last update timestamp \(ISO 8601\) | -| ↳ `status` | string | Application status \(Active, Hired, Archived, Lead\) | -| ↳ `candidate` | object | candidate output from the tool | -| ↳ `id` | string | Candidate UUID | -| ↳ `name` | string | Candidate name | -| ↳ `currentInterviewStage` | object | currentInterviewStage output from the tool | -| ↳ `id` | string | Current interview stage UUID | -| ↳ `title` | string | Current interview stage title | -| ↳ `stageType` | string | Current interview stage type \(e.g., Lead, Applied, Interview, Offer\) | -| ↳ `job` | object | job output from the tool | -| ↳ `id` | string | Job UUID | -| ↳ `title` | string | Job title | - - ---- +| `candidates` | json | List of candidates with rich fields \(id, name, primaryEmailAddress, primaryPhoneNumber, emailAddresses\[\], phoneNumbers\[\], socialLinks\[\], linkedInUrl, githubUrl, profileUrl, position, company, school, timezone, location with locationComponents\[\], tags\[\], applicationIds\[\], customFields\[\], resumeFileHandle, fileHandles\[\], source with sourceType, creditedToUser, fraudStatus, createdAt, updatedAt\) | +| `jobs` | json | List of jobs \(id, title, confidential, status, employmentType, locationId, departmentId, defaultInterviewPlanId, interviewPlanIds\[\], customFields\[\], jobPostingIds\[\], customRequisitionId, brandId, hiringTeam\[\], author, createdAt, updatedAt, openedAt, closedAt, location with address, openings\[\] with latestVersion\) | +| `applications` | json | List of applications \(id, status, customFields\[\], candidate summary, currentInterviewStage, source with sourceType, archiveReason with customFields\[\], archivedAt, job summary, creditedToUser, hiringTeam\[\], appliedViaJobPostingId, submitterClientIp, submitterUserAgent, createdAt, updatedAt\) | +| `notes` | json | List of notes \(id, content, author, isPrivate, createdAt\) | +| `offers` | json | List of offers \(id, decidedAt, applicationId, acceptanceStatus, offerStatus, latestVersion with id/startDate/salary/createdAt/openingId/customFields\[\]/fileHandles\[\]/author/approvalStatus\) | +| `archiveReasons` | json | List of archive reasons \(id, text, reasonType \[RejectedByCandidate/RejectedByOrg/Other\], isArchived\) | +| `sources` | json | List of sources \(id, title, isArchived, sourceType \{id, title, isArchived\}\) | +| `customFields` | json | For List Custom Fields, the field definitions \(id, title, isPrivate, fieldType, objectType, isArchived, isRequired, selectableValues\[\] \{label, value, isArchived\}\). For Set Custom Field Values, the field values written to the object \(id, title, isPrivate, valueLabel, value\) | +| `customField` | json | A single custom field value after a write \(id, title, isPrivate, valueLabel, value\) | +| `departments` | json | List of departments \(id, name, externalName, isArchived, parentId, createdAt, updatedAt\) | +| `locations` | json | List of locations \(id, name, externalName, isArchived, isRemote, workplaceType, parentLocationId, type, address with addressCountry/Region/Locality/postalCode/streetAddress\) | +| `jobPostings` | json | List of job postings \(id, title, jobId, departmentName, teamName, locationName, locationIds, workplaceType, employmentType, isListed, publishedDate, applicationDeadline, externalLink, applyLink, compensationTierSummary, shouldDisplayCompensationOnJobBoard, updatedAt\) | +| `openings` | json | List of openings \(id, openedAt, closedAt, isArchived, archivedAt, closeReasonId, openingState, latestVersion with identifier/description/authorId/createdAt/teamId/jobIds\[\]/targetHireDate/targetStartDate/isBackfill/employmentType/locationIds\[\]/hiringTeam\[\]/customFields\[\]\) | +| `users` | json | List of users \(id, firstName, lastName, email, globalRole, isEnabled, updatedAt\) | +| `interviewSchedules` | json | List of interview schedules \(id, applicationId, interviewStageId, interviewEvents\[\] with interviewerUserIds/startTime/endTime/feedbackLink/location/meetingLink/hasSubmittedFeedback, status, scheduledBy, createdAt, updatedAt\) | +| `interviewPlans` | json | Interview plans \(id, title, isArchived, createdAt, updatedAt\) | +| `interviewStages` | json | Ordered interview stages for a plan | +| `feedback` | json | Submitted application feedback with form definitions and values | +| `history` | json | Application stage history and allowed actions | +| `tags` | json | List of candidate tags \(id, title, isArchived\) | +| `id` | string | Resource UUID | +| `name` | string | Resource name | +| `title` | string | Job title or job posting title | +| `status` | string | Status | +| `candidate` | json | Candidate summary \(id, name, primaryEmailAddress, primaryPhoneNumber\). For full candidate fields use the candidates list output or the get/create/update candidate operations. | +| `job` | json | Job details \(id, title, status, employmentType, locationId, departmentId, hiringTeam\[\], author, location, openings\[\], createdAt, updatedAt\) | +| `application` | json | Application details \(id, status, customFields\[\], candidate, currentInterviewStage, source, archiveReason, job, hiringTeam\[\], createdAt, updatedAt\) | +| `offer` | json | Offer details \(id, decidedAt, applicationId, acceptanceStatus, offerStatus, latestVersion\) | +| `jobPosting` | json | Job posting details \(id, title, descriptionPlain, descriptionHtml, descriptionSocial, descriptionParts, departmentName, teamName, teamNameHierarchy\[\], jobId, locationName, locationIds, address, isRemote, workplaceType, employmentType, isListed, publishedDate, applicationDeadline, externalLink, applyLink, compensation, updatedAt, job \[included when expandJob=true\]\) | +| `content` | string | Note content | +| `author` | json | Note author \(id, firstName, lastName, email\) | +| `isPrivate` | boolean | Whether the note is private | +| `createdAt` | string | ISO 8601 creation timestamp | +| `applicationId` | string | UUID of the deleted application | +| `moreDataAvailable` | boolean | Whether more pages exist | +| `nextCursor` | string | Pagination cursor for next page | +| `nextSyncCursor` | string | Ashby's opaque token for the next incremental list run, exposed as a cursor so it remains usable in workflow output | -### Ashby Candidate Deleted +### Ashby Upload Candidate File -Trigger workflow when a candidate is deleted +Securely uploads a file and attaches it to an Ashby candidate. -#### Configuration +#### Input | Parameter | Type | Required | Description | | --------- | ---- | -------- | ----------- | -| `apiKey` | string | Yes | API Key | +| `apiKey` | string | Yes | Ashby API Key | +| `candidateId` | string | Yes | Candidate UUID | +| `file` | file | Yes | Stored file to attach to the candidate | +| `fileName` | string | No | Optional filename override | +| `onBehalfOfUserId` | string | No | Active Ashby user UUID to attribute this mutation to | #### Output | Parameter | Type | Description | | --------- | ---- | ----------- | -| `action` | string | The webhook event type \(e.g., applicationSubmit, candidateHire\) | -| `candidate` | object | candidate output from the tool | -| ↳ `id` | string | Deleted candidate UUID | - - ---- - -### Ashby Candidate Hired - -Trigger workflow when a candidate is hired - -#### Configuration - +| `candidates` | json | List of candidates with rich fields \(id, name, primaryEmailAddress, primaryPhoneNumber, emailAddresses\[\], phoneNumbers\[\], socialLinks\[\], linkedInUrl, githubUrl, profileUrl, position, company, school, timezone, location with locationComponents\[\], tags\[\], applicationIds\[\], customFields\[\], resumeFileHandle, fileHandles\[\], source with sourceType, creditedToUser, fraudStatus, createdAt, updatedAt\) | +| `jobs` | json | List of jobs \(id, title, confidential, status, employmentType, locationId, departmentId, defaultInterviewPlanId, interviewPlanIds\[\], customFields\[\], jobPostingIds\[\], customRequisitionId, brandId, hiringTeam\[\], author, createdAt, updatedAt, openedAt, closedAt, location with address, openings\[\] with latestVersion\) | +| `applications` | json | List of applications \(id, status, customFields\[\], candidate summary, currentInterviewStage, source with sourceType, archiveReason with customFields\[\], archivedAt, job summary, creditedToUser, hiringTeam\[\], appliedViaJobPostingId, submitterClientIp, submitterUserAgent, createdAt, updatedAt\) | +| `notes` | json | List of notes \(id, content, author, isPrivate, createdAt\) | +| `offers` | json | List of offers \(id, decidedAt, applicationId, acceptanceStatus, offerStatus, latestVersion with id/startDate/salary/createdAt/openingId/customFields\[\]/fileHandles\[\]/author/approvalStatus\) | +| `archiveReasons` | json | List of archive reasons \(id, text, reasonType \[RejectedByCandidate/RejectedByOrg/Other\], isArchived\) | +| `sources` | json | List of sources \(id, title, isArchived, sourceType \{id, title, isArchived\}\) | +| `customFields` | json | For List Custom Fields, the field definitions \(id, title, isPrivate, fieldType, objectType, isArchived, isRequired, selectableValues\[\] \{label, value, isArchived\}\). For Set Custom Field Values, the field values written to the object \(id, title, isPrivate, valueLabel, value\) | +| `customField` | json | A single custom field value after a write \(id, title, isPrivate, valueLabel, value\) | +| `departments` | json | List of departments \(id, name, externalName, isArchived, parentId, createdAt, updatedAt\) | +| `locations` | json | List of locations \(id, name, externalName, isArchived, isRemote, workplaceType, parentLocationId, type, address with addressCountry/Region/Locality/postalCode/streetAddress\) | +| `jobPostings` | json | List of job postings \(id, title, jobId, departmentName, teamName, locationName, locationIds, workplaceType, employmentType, isListed, publishedDate, applicationDeadline, externalLink, applyLink, compensationTierSummary, shouldDisplayCompensationOnJobBoard, updatedAt\) | +| `openings` | json | List of openings \(id, openedAt, closedAt, isArchived, archivedAt, closeReasonId, openingState, latestVersion with identifier/description/authorId/createdAt/teamId/jobIds\[\]/targetHireDate/targetStartDate/isBackfill/employmentType/locationIds\[\]/hiringTeam\[\]/customFields\[\]\) | +| `users` | json | List of users \(id, firstName, lastName, email, globalRole, isEnabled, updatedAt\) | +| `interviewSchedules` | json | List of interview schedules \(id, applicationId, interviewStageId, interviewEvents\[\] with interviewerUserIds/startTime/endTime/feedbackLink/location/meetingLink/hasSubmittedFeedback, status, scheduledBy, createdAt, updatedAt\) | +| `interviewPlans` | json | Interview plans \(id, title, isArchived, createdAt, updatedAt\) | +| `interviewStages` | json | Ordered interview stages for a plan | +| `feedback` | json | Submitted application feedback with form definitions and values | +| `history` | json | Application stage history and allowed actions | +| `tags` | json | List of candidate tags \(id, title, isArchived\) | +| `id` | string | Resource UUID | +| `name` | string | Resource name | +| `title` | string | Job title or job posting title | +| `status` | string | Status | +| `candidate` | json | Candidate summary \(id, name, primaryEmailAddress, primaryPhoneNumber\). For full candidate fields use the candidates list output or the get/create/update candidate operations. | +| `job` | json | Job details \(id, title, status, employmentType, locationId, departmentId, hiringTeam\[\], author, location, openings\[\], createdAt, updatedAt\) | +| `application` | json | Application details \(id, status, customFields\[\], candidate, currentInterviewStage, source, archiveReason, job, hiringTeam\[\], createdAt, updatedAt\) | +| `offer` | json | Offer details \(id, decidedAt, applicationId, acceptanceStatus, offerStatus, latestVersion\) | +| `jobPosting` | json | Job posting details \(id, title, descriptionPlain, descriptionHtml, descriptionSocial, descriptionParts, departmentName, teamName, teamNameHierarchy\[\], jobId, locationName, locationIds, address, isRemote, workplaceType, employmentType, isListed, publishedDate, applicationDeadline, externalLink, applyLink, compensation, updatedAt, job \[included when expandJob=true\]\) | +| `content` | string | Note content | +| `author` | json | Note author \(id, firstName, lastName, email\) | +| `isPrivate` | boolean | Whether the note is private | +| `createdAt` | string | ISO 8601 creation timestamp | +| `applicationId` | string | UUID of the deleted application | +| `moreDataAvailable` | boolean | Whether more pages exist | +| `nextCursor` | string | Pagination cursor for next page | +| `nextSyncCursor` | string | Ashby's opaque token for the next incremental list run, exposed as a cursor so it remains usable in workflow output | + +### Ashby Upload Resume + +Securely uploads a resume and sets it as the Ashby candidate resume. + +#### Input + +| Parameter | Type | Required | Description | +| --------- | ---- | -------- | ----------- | +| `apiKey` | string | Yes | Ashby API Key | +| `candidateId` | string | Yes | Candidate UUID | +| `file` | file | Yes | Stored resume file | +| `fileName` | string | No | Optional filename override | +| `onBehalfOfUserId` | string | No | Active Ashby user UUID to attribute this mutation to | + +#### Output + +| Parameter | Type | Description | +| --------- | ---- | ----------- | +| `candidates` | json | List of candidates with rich fields \(id, name, primaryEmailAddress, primaryPhoneNumber, emailAddresses\[\], phoneNumbers\[\], socialLinks\[\], linkedInUrl, githubUrl, profileUrl, position, company, school, timezone, location with locationComponents\[\], tags\[\], applicationIds\[\], customFields\[\], resumeFileHandle, fileHandles\[\], source with sourceType, creditedToUser, fraudStatus, createdAt, updatedAt\) | +| `jobs` | json | List of jobs \(id, title, confidential, status, employmentType, locationId, departmentId, defaultInterviewPlanId, interviewPlanIds\[\], customFields\[\], jobPostingIds\[\], customRequisitionId, brandId, hiringTeam\[\], author, createdAt, updatedAt, openedAt, closedAt, location with address, openings\[\] with latestVersion\) | +| `applications` | json | List of applications \(id, status, customFields\[\], candidate summary, currentInterviewStage, source with sourceType, archiveReason with customFields\[\], archivedAt, job summary, creditedToUser, hiringTeam\[\], appliedViaJobPostingId, submitterClientIp, submitterUserAgent, createdAt, updatedAt\) | +| `notes` | json | List of notes \(id, content, author, isPrivate, createdAt\) | +| `offers` | json | List of offers \(id, decidedAt, applicationId, acceptanceStatus, offerStatus, latestVersion with id/startDate/salary/createdAt/openingId/customFields\[\]/fileHandles\[\]/author/approvalStatus\) | +| `archiveReasons` | json | List of archive reasons \(id, text, reasonType \[RejectedByCandidate/RejectedByOrg/Other\], isArchived\) | +| `sources` | json | List of sources \(id, title, isArchived, sourceType \{id, title, isArchived\}\) | +| `customFields` | json | For List Custom Fields, the field definitions \(id, title, isPrivate, fieldType, objectType, isArchived, isRequired, selectableValues\[\] \{label, value, isArchived\}\). For Set Custom Field Values, the field values written to the object \(id, title, isPrivate, valueLabel, value\) | +| `customField` | json | A single custom field value after a write \(id, title, isPrivate, valueLabel, value\) | +| `departments` | json | List of departments \(id, name, externalName, isArchived, parentId, createdAt, updatedAt\) | +| `locations` | json | List of locations \(id, name, externalName, isArchived, isRemote, workplaceType, parentLocationId, type, address with addressCountry/Region/Locality/postalCode/streetAddress\) | +| `jobPostings` | json | List of job postings \(id, title, jobId, departmentName, teamName, locationName, locationIds, workplaceType, employmentType, isListed, publishedDate, applicationDeadline, externalLink, applyLink, compensationTierSummary, shouldDisplayCompensationOnJobBoard, updatedAt\) | +| `openings` | json | List of openings \(id, openedAt, closedAt, isArchived, archivedAt, closeReasonId, openingState, latestVersion with identifier/description/authorId/createdAt/teamId/jobIds\[\]/targetHireDate/targetStartDate/isBackfill/employmentType/locationIds\[\]/hiringTeam\[\]/customFields\[\]\) | +| `users` | json | List of users \(id, firstName, lastName, email, globalRole, isEnabled, updatedAt\) | +| `interviewSchedules` | json | List of interview schedules \(id, applicationId, interviewStageId, interviewEvents\[\] with interviewerUserIds/startTime/endTime/feedbackLink/location/meetingLink/hasSubmittedFeedback, status, scheduledBy, createdAt, updatedAt\) | +| `interviewPlans` | json | Interview plans \(id, title, isArchived, createdAt, updatedAt\) | +| `interviewStages` | json | Ordered interview stages for a plan | +| `feedback` | json | Submitted application feedback with form definitions and values | +| `history` | json | Application stage history and allowed actions | +| `tags` | json | List of candidate tags \(id, title, isArchived\) | +| `id` | string | Resource UUID | +| `name` | string | Resource name | +| `title` | string | Job title or job posting title | +| `status` | string | Status | +| `candidate` | json | Candidate summary \(id, name, primaryEmailAddress, primaryPhoneNumber\). For full candidate fields use the candidates list output or the get/create/update candidate operations. | +| `job` | json | Job details \(id, title, status, employmentType, locationId, departmentId, hiringTeam\[\], author, location, openings\[\], createdAt, updatedAt\) | +| `application` | json | Application details \(id, status, customFields\[\], candidate, currentInterviewStage, source, archiveReason, job, hiringTeam\[\], createdAt, updatedAt\) | +| `offer` | json | Offer details \(id, decidedAt, applicationId, acceptanceStatus, offerStatus, latestVersion\) | +| `jobPosting` | json | Job posting details \(id, title, descriptionPlain, descriptionHtml, descriptionSocial, descriptionParts, departmentName, teamName, teamNameHierarchy\[\], jobId, locationName, locationIds, address, isRemote, workplaceType, employmentType, isListed, publishedDate, applicationDeadline, externalLink, applyLink, compensation, updatedAt, job \[included when expandJob=true\]\) | +| `content` | string | Note content | +| `author` | json | Note author \(id, firstName, lastName, email\) | +| `isPrivate` | boolean | Whether the note is private | +| `createdAt` | string | ISO 8601 creation timestamp | +| `applicationId` | string | UUID of the deleted application | +| `moreDataAvailable` | boolean | Whether more pages exist | +| `nextCursor` | string | Pagination cursor for next page | +| `nextSyncCursor` | string | Ashby's opaque token for the next incremental list run, exposed as a cursor so it remains usable in workflow output | + + + +## Triggers + +A **Trigger** is a block that starts a workflow when an event happens in this service. + +### Ashby Application Submitted + +Trigger workflow when a new application is submitted + +#### Configuration + | Parameter | Type | Required | Description | | --------- | ---- | -------- | ----------- | | `apiKey` | string | Yes | API Key | @@ -1412,6 +1865,99 @@ Trigger workflow when a candidate is hired | Parameter | Type | Description | | --------- | ---- | ----------- | | `action` | string | The webhook event type \(e.g., applicationSubmit, candidateHire\) | +| `webhookActionId` | string | Ashby delivery identifier, stable across retries | +| `application` | object | application output from the tool | +| ↳ `id` | string | Application UUID | +| ↳ `createdAt` | string | Application creation timestamp \(ISO 8601\) | +| ↳ `updatedAt` | string | Application last update timestamp \(ISO 8601\) | +| ↳ `status` | string | Application status \(Active, Hired, Archived, Lead\) | +| ↳ `candidate` | object | candidate output from the tool | +| ↳ `id` | string | Candidate UUID | +| ↳ `name` | string | Candidate name | +| ↳ `currentInterviewStage` | object | currentInterviewStage output from the tool | +| ↳ `id` | string | Current interview stage UUID | +| ↳ `title` | string | Current interview stage title | +| ↳ `stageType` | string | Current interview stage type \(e.g., Lead, Applied, Interview, Offer\) | +| ↳ `job` | object | job output from the tool | +| ↳ `id` | string | Job UUID | +| ↳ `title` | string | Job title | + + +--- + +### Ashby Application Updated + +Trigger workflow when an application is updated + +#### Configuration + +| Parameter | Type | Required | Description | +| --------- | ---- | -------- | ----------- | +| `apiKey` | string | Yes | API Key | + +#### Output + +| Parameter | Type | Description | +| --------- | ---- | ----------- | +| `action` | string | The webhook event type \(e.g., applicationSubmit, candidateHire\) | +| `webhookActionId` | string | Ashby delivery identifier, stable across retries | +| `application` | object | application output from the tool | +| ↳ `id` | string | Application UUID | +| ↳ `createdAt` | string | Application creation timestamp \(ISO 8601\) | +| ↳ `updatedAt` | string | Application last update timestamp \(ISO 8601\) | +| ↳ `status` | string | Application status \(Active, Hired, Archived, Lead\) | +| ↳ `candidate` | object | candidate output from the tool | +| ↳ `id` | string | Candidate UUID | +| ↳ `name` | string | Candidate name | +| ↳ `currentInterviewStage` | object | currentInterviewStage output from the tool | +| ↳ `id` | string | Current interview stage UUID | +| ↳ `title` | string | Current interview stage title | +| ↳ `stageType` | string | Current interview stage type \(e.g., Lead, Applied, Interview, Offer\) | +| ↳ `job` | object | job output from the tool | +| ↳ `id` | string | Job UUID | +| ↳ `title` | string | Job title | + + +--- + +### Ashby Candidate Deleted + +Trigger workflow when a candidate is deleted + +#### Configuration + +| Parameter | Type | Required | Description | +| --------- | ---- | -------- | ----------- | +| `apiKey` | string | Yes | API Key | + +#### Output + +| Parameter | Type | Description | +| --------- | ---- | ----------- | +| `action` | string | The webhook event type \(e.g., applicationSubmit, candidateHire\) | +| `webhookActionId` | string | Ashby delivery identifier, stable across retries | +| `candidate` | object | candidate output from the tool | +| ↳ `id` | string | Deleted candidate UUID | + + +--- + +### Ashby Candidate Hired + +Trigger workflow when a candidate is hired + +#### Configuration + +| Parameter | Type | Required | Description | +| --------- | ---- | -------- | ----------- | +| `apiKey` | string | Yes | API Key | + +#### Output + +| Parameter | Type | Description | +| --------- | ---- | ----------- | +| `action` | string | The webhook event type \(e.g., applicationSubmit, candidateHire\) | +| `webhookActionId` | string | Ashby delivery identifier, stable across retries | | `application` | object | application output from the tool | | ↳ `id` | string | Application UUID | | ↳ `createdAt` | string | Application creation timestamp \(ISO 8601\) | @@ -1437,6 +1983,30 @@ Trigger workflow when a candidate is hired | ↳ `id` | string | Latest offer version UUID | +--- + +### Ashby Candidate Merged + +Trigger workflow when two candidate records are merged + +#### Configuration + +| Parameter | Type | Required | Description | +| --------- | ---- | -------- | ----------- | +| `apiKey` | string | Yes | API Key | + +#### Output + +| Parameter | Type | Description | +| --------- | ---- | ----------- | +| `action` | string | The webhook event type \(e.g., applicationSubmit, candidateHire\) | +| `webhookActionId` | string | Ashby delivery identifier, stable across retries | +| `deletedCandidate` | object | deletedCandidate output from the tool | +| ↳ `id` | string | Deleted candidate UUID | +| `mergedCandidate` | object | mergedCandidate output from the tool | +| ↳ `id` | string | Final merged candidate UUID | + + --- ### Ashby Candidate Stage Change @@ -1454,6 +2024,7 @@ Trigger workflow when a candidate changes interview stages | Parameter | Type | Description | | --------- | ---- | ----------- | | `action` | string | The webhook event type \(e.g., applicationSubmit, candidateHire\) | +| `webhookActionId` | string | Ashby delivery identifier, stable across retries | | `application` | object | application output from the tool | | ↳ `id` | string | Application UUID | | ↳ `createdAt` | string | Application creation timestamp \(ISO 8601\) | @@ -1471,6 +2042,65 @@ Trigger workflow when a candidate changes interview stages | ↳ `title` | string | Job title | +--- + +### Ashby Interview Schedule Created + +Trigger workflow when an interview schedule is created + +#### Configuration + +| Parameter | Type | Required | Description | +| --------- | ---- | -------- | ----------- | +| `apiKey` | string | Yes | API Key | + +#### Output + +| Parameter | Type | Description | +| --------- | ---- | ----------- | +| `action` | string | The webhook event type \(e.g., applicationSubmit, candidateHire\) | +| `webhookActionId` | string | Ashby delivery identifier, stable across retries | +| `interviewSchedule` | object | interviewSchedule output from the tool | +| ↳ `id` | string | Interview schedule UUID | +| ↳ `status` | string | Interview schedule status | +| ↳ `applicationId` | string | Application UUID | +| ↳ `interviewStageId` | string | Interview stage UUID | +| ↳ `scheduledBy` | json | Scheduling user | +| ↳ `createdAt` | string | Creation timestamp | +| ↳ `updatedAt` | string | Last update timestamp | +| ↳ `interviewEvents` | json | Scheduled interview events | + + +--- + +### Ashby Interview Schedule Updated + +Trigger workflow when an interview schedule is updated + +#### Configuration + +| Parameter | Type | Required | Description | +| --------- | ---- | -------- | ----------- | +| `apiKey` | string | Yes | API Key | + +#### Output + +| Parameter | Type | Description | +| --------- | ---- | ----------- | +| `action` | string | The webhook event type \(e.g., applicationSubmit, candidateHire\) | +| `webhookActionId` | string | Ashby delivery identifier, stable across retries | +| `interviewSchedule` | object | interviewSchedule output from the tool | +| ↳ `id` | string | Interview schedule UUID | +| ↳ `status` | string | Interview schedule status | +| ↳ `applicationId` | string | Application UUID | +| ↳ `interviewStageId` | string | Interview stage UUID | +| ↳ `candidateId` | string | Candidate UUID | +| ↳ `scheduledBy` | json | Scheduling user | +| ↳ `createdAt` | string | Creation timestamp | +| ↳ `updatedAt` | string | Last update timestamp | +| ↳ `interviewEvents` | json | Scheduled interview events | + + --- ### Ashby Job Created @@ -1488,6 +2118,87 @@ Trigger workflow when a new job is created | Parameter | Type | Description | | --------- | ---- | ----------- | | `action` | string | The webhook event type \(e.g., applicationSubmit, candidateHire\) | +| `webhookActionId` | string | Ashby delivery identifier, stable across retries | +| `job` | object | job output from the tool | +| ↳ `id` | string | Job UUID | +| ↳ `title` | string | Job title | +| ↳ `confidential` | boolean | Whether the job is confidential | +| ↳ `status` | string | Job status \(Open, Closed, Draft, Archived\) | +| ↳ `employmentType` | string | Employment type \(FullTime, PartTime, Intern, Contract, Temporary\) | + + +--- + +### Ashby Job Posting Deleted + +Trigger workflow when a job posting is deleted + +#### Configuration + +| Parameter | Type | Required | Description | +| --------- | ---- | -------- | ----------- | +| `apiKey` | string | Yes | API Key | + +#### Output + +| Parameter | Type | Description | +| --------- | ---- | ----------- | +| `action` | string | The webhook event type \(e.g., applicationSubmit, candidateHire\) | +| `webhookActionId` | string | Ashby delivery identifier, stable across retries | +| `jobPosting` | object | jobPosting output from the tool | +| ↳ `id` | string | Deleted job posting UUID | +| ↳ `jobId` | string | Associated job UUID | + + +--- + +### Ashby Job Posting Updated + +Trigger workflow when a job posting is updated + +#### Configuration + +| Parameter | Type | Required | Description | +| --------- | ---- | -------- | ----------- | +| `apiKey` | string | Yes | API Key | + +#### Output + +| Parameter | Type | Description | +| --------- | ---- | ----------- | +| `action` | string | The webhook event type \(e.g., applicationSubmit, candidateHire\) | +| `webhookActionId` | string | Ashby delivery identifier, stable across retries | +| `jobPosting` | object | jobPosting output from the tool | +| ↳ `id` | string | Job posting UUID | +| ↳ `title` | string | Job posting title | +| ↳ `jobId` | string | Associated job UUID | +| ↳ `departmentName` | string | Department name | +| ↳ `teamName` | string | Team name | +| ↳ `teamNameHierarchy` | json | Department-to-team name hierarchy | +| ↳ `locationName` | string | Location name | +| ↳ `isListed` | boolean | Whether publicly listed | +| ↳ `publishedDate` | string | Publication timestamp | +| ↳ `updatedAt` | string | Last update timestamp | + + +--- + +### Ashby Job Updated + +Trigger workflow when a job is updated + +#### Configuration + +| Parameter | Type | Required | Description | +| --------- | ---- | -------- | ----------- | +| `apiKey` | string | Yes | API Key | + +#### Output + +| Parameter | Type | Description | +| --------- | ---- | ----------- | +| `action` | string | The webhook event type \(e.g., applicationSubmit, candidateHire\) | +| `webhookActionId` | string | Ashby delivery identifier, stable across retries | | `job` | object | job output from the tool | | ↳ `id` | string | Job UUID | | ↳ `title` | string | Job title | @@ -1513,6 +2224,7 @@ Trigger workflow when a new offer is created | Parameter | Type | Description | | --------- | ---- | ----------- | | `action` | string | The webhook event type \(e.g., applicationSubmit, candidateHire\) | +| `webhookActionId` | string | Ashby delivery identifier, stable across retries | | `offer` | object | offer output from the tool | | ↳ `id` | string | Offer UUID | | ↳ `applicationId` | string | Associated application UUID | @@ -1522,3 +2234,108 @@ Trigger workflow when a new offer is created | ↳ `latestVersion` | object | latestVersion output from the tool | | ↳ `id` | string | Latest offer version UUID | + +--- + +### Ashby Offer Deleted + +Trigger workflow when an offer is deleted + +#### Configuration + +| Parameter | Type | Required | Description | +| --------- | ---- | -------- | ----------- | +| `apiKey` | string | Yes | API Key | + +#### Output + +| Parameter | Type | Description | +| --------- | ---- | ----------- | +| `action` | string | The webhook event type \(e.g., applicationSubmit, candidateHire\) | +| `webhookActionId` | string | Ashby delivery identifier, stable across retries | +| `offer` | object | offer output from the tool | +| ↳ `id` | string | Deleted offer UUID | +| ↳ `applicationId` | string | Associated application UUID | + + +--- + +### Ashby Offer Updated + +Trigger workflow when an offer is updated + +#### Configuration + +| Parameter | Type | Required | Description | +| --------- | ---- | -------- | ----------- | +| `apiKey` | string | Yes | API Key | + +#### Output + +| Parameter | Type | Description | +| --------- | ---- | ----------- | +| `action` | string | The webhook event type \(e.g., applicationSubmit, candidateHire\) | +| `webhookActionId` | string | Ashby delivery identifier, stable across retries | +| `offer` | object | offer output from the tool | +| ↳ `id` | string | Offer UUID | +| ↳ `applicationId` | string | Associated application UUID | +| ↳ `acceptanceStatus` | string | Offer acceptance status \(Accepted, Declined, Pending, Created, Cancelled\) | +| ↳ `offerStatus` | string | Offer process status \(WaitingOnApprovalStart, WaitingOnOfferApproval, WaitingOnApprovalDefinition, WaitingOnCandidateResponse, CandidateRejected, CandidateAccepted, OfferCancelled\) | +| ↳ `decidedAt` | string | Offer decision timestamp \(ISO 8601\). Typically null at creation; populated after candidate responds. | +| ↳ `latestVersion` | object | latestVersion output from the tool | +| ↳ `id` | string | Latest offer version UUID | + + +--- + +### Ashby Opening Created + +Trigger workflow when a headcount opening is created + +#### Configuration + +| Parameter | Type | Required | Description | +| --------- | ---- | -------- | ----------- | +| `apiKey` | string | Yes | API Key | + +#### Output + +| Parameter | Type | Description | +| --------- | ---- | ----------- | +| `action` | string | The webhook event type \(e.g., applicationSubmit, candidateHire\) | +| `webhookActionId` | string | Ashby delivery identifier, stable across retries | +| `opening` | object | opening output from the tool | +| ↳ `id` | string | Opening UUID | +| ↳ `openedAt` | string | Open timestamp | +| ↳ `closedAt` | string | Close timestamp | +| ↳ `isArchived` | boolean | Whether archived | +| ↳ `archivedAt` | string | Archive timestamp | +| ↳ `closeReasonId` | string | Close reason UUID | +| ↳ `openingState` | string | Opening state | +| ↳ `latestVersion` | json | Latest opening version | + + +--- + +### Ashby Signature Request Updated + +Trigger workflow when an e-signature request changes state + +#### Configuration + +| Parameter | Type | Required | Description | +| --------- | ---- | -------- | ----------- | +| `apiKey` | string | Yes | API Key | + +#### Output + +| Parameter | Type | Description | +| --------- | ---- | ----------- | +| `action` | string | The webhook event type \(e.g., applicationSubmit, candidateHire\) | +| `webhookActionId` | string | Ashby delivery identifier, stable across retries | +| `relatedEntityType` | string | Related entity type: application or offer | +| `applicationId` | string | Related application UUID | +| `offerId` | string | Related offer UUID | +| `offerVersionId` | string | Related offer version UUID | +| `eventType` | string | Signature request event: sent, cancelled, completed, or deleted | + diff --git a/apps/docs/content/docs/integrations/calcom.mdx b/apps/docs/content/docs/integrations/calcom.mdx index 5009987e2c1..e2db71f94a8 100644 --- a/apps/docs/content/docs/integrations/calcom.mdx +++ b/apps/docs/content/docs/integrations/calcom.mdx @@ -44,7 +44,7 @@ Create a new booking on Cal.com | --------- | ---- | -------- | ----------- | | `eventTypeId` | number | Yes | The ID of the event type to book | | `start` | string | Yes | Start time in UTC ISO 8601 format \(e.g., 2024-01-15T09:00:00Z\) | -| `attendee` | object | Yes | Attendee information object with name, email, timeZone, and optional phoneNumber \(constructed from individual attendee fields\) | +| `attendee` | object | Yes | Attendee information object with name, email, timeZone, and optional phoneNumber. The Cal.com block composes this from its individual attendee fields; a direct caller sends the object. | | `guests` | array | No | Array of guest email addresses | | `lengthInMinutes` | number | No | Duration of the booking in minutes \(overrides event type default\) | | `metadata` | object | No | Custom metadata to attach to the booking | diff --git a/apps/docs/content/docs/integrations/circleback.mdx b/apps/docs/content/docs/integrations/circleback.mdx index a79f410aa03..86b314e75df 100644 --- a/apps/docs/content/docs/integrations/circleback.mdx +++ b/apps/docs/content/docs/integrations/circleback.mdx @@ -1,11 +1,11 @@ --- title: Circleback -description: Circleback triggers for automating workflows +description: AI-powered meeting notes, action items, and transcripts --- import { BlockInfoCard } from "@/components/ui/block-info-card" - @@ -50,6 +50,693 @@ Whether you want to distribute instant summaries, log action items, or build cus {/* MANUAL-CONTENT-END */} +## Usage Instructions + +Integrate Circleback into your workflow to read meetings, notes, transcripts, and insights, search across meetings, manage action items and tags, and browse the people and companies you meet with. Circleback can also trigger workflows when meetings are processed. + + + +## Actions + +### Circleback List Meetings + +Lists meetings from Circleback with optional ownership, status, tag, and attendee filters. + +#### Input + +| Parameter | Type | Required | Description | +| --------- | ---- | -------- | ----------- | +| `apiKey` | string | Yes | Circleback API key | +| `ownership` | string | No | Which meetings to return: All, Mine, or Shared. Defaults to Mine | +| `statuses` | string | No | Comma-separated meeting statuses to filter by | +| `tagIds` | string | No | Comma-separated tag IDs to filter by | +| `attendeeProfileIds` | string | No | Comma-separated profile IDs of attendees to filter by | +| `cursor` | string | No | Pagination cursor from a previous response | + +#### Output + +| Parameter | Type | Description | +| --------- | ---- | ----------- | +| `meetings` | array | The meetings on this page | +| ↳ `id` | string | The Circleback meeting ID | +| ↳ `name` | string | The meeting name | +| ↳ `createdAt` | string | When the meeting was created \(ISO 8601\) | +| ↳ `updatedAt` | string | When the meeting was last updated \(ISO 8601\) | +| ↳ `duration` | number | The meeting duration in seconds | +| ↳ `url` | string | The URL of the virtual meeting \(Zoom, Google Meet, or Microsoft Teams\) | +| ↳ `recordingUrl` | string | The URL of the meeting recording file, valid for 24 hours | +| ↳ `tags` | array | Tags added to the meeting | +| ↳ `id` | number | The unique identifier of the tag | +| ↳ `name` | string | The display name of the tag | +| ↳ `description` | string | A description of the tag | +| ↳ `icalUid` | string | The identifier of the calendar event associated with the meeting | +| ↳ `attendees` | array | The meeting attendees | +| ↳ `profileId` | number | The unique identifier of the attendee profile | +| ↳ `name` | string | The attendee name | +| ↳ `title` | string | The attendee job title | +| ↳ `companyName` | string | The name of the company the attendee belongs to | +| ↳ `email` | string | The attendee email address | +| ↳ `isCalendarEventOrganizer` | boolean | Whether the attendee organized the calendar event | +| ↳ `isCalendarInvitee` | boolean | Whether the attendee was invited on the calendar event | +| ↳ `notes` | string | The meeting notes with Markdown formatting | +| ↳ `privateNotes` | string | The authenticated user private notes for the meeting | +| ↳ `actionItems` | array | Action items created for the meeting | +| ↳ `id` | number | The unique identifier of the action item | +| ↳ `title` | string | The action item title | +| ↳ `description` | string | The action item description | +| ↳ `assignee` | json | The assignee as an object with profileId, name, title, companyName, and email, or null if unassigned | +| ↳ `status` | string | The completion status, PENDING or DONE | +| ↳ `insights` | json | Insight results for the meeting, keyed by the name of the user-created insight | +| ↳ `linkAccess` | string | Who can access the meeting through its shareable link: Editor, Viewer, or LimitedViewer | +| ↳ `calendarEvent` | json | The associated calendar event as an object with id, icalUid, description, platform, and platformId, or null | +| `nextCursor` | string | Pagination cursor for the next page, or null on the last page | +| `hasMore` | boolean | Whether another page of meetings is available | + +### Circleback Get Meeting + +Gets a Circleback meeting by ID, including notes, attendees, action items, insights, and recording details. + +#### Input + +| Parameter | Type | Required | Description | +| --------- | ---- | -------- | ----------- | +| `apiKey` | string | Yes | Circleback API key | +| `meetingId` | string | Yes | The unique identifier of the meeting | + +#### Output + +| Parameter | Type | Description | +| --------- | ---- | ----------- | +| `id` | string | The Circleback meeting ID | +| `name` | string | The meeting name | +| `createdAt` | string | When the meeting was created \(ISO 8601\) | +| `updatedAt` | string | When the meeting was last updated \(ISO 8601\) | +| `duration` | number | The meeting duration in seconds | +| `url` | string | The URL of the virtual meeting \(Zoom, Google Meet, or Microsoft Teams\) | +| `recordingUrl` | string | The URL of the meeting recording file, valid for 24 hours | +| `tags` | array | Tags added to the meeting | +| ↳ `id` | number | The unique identifier of the tag | +| ↳ `name` | string | The display name of the tag | +| ↳ `description` | string | A description of the tag | +| `icalUid` | string | The identifier of the calendar event associated with the meeting | +| `attendees` | array | The meeting attendees | +| ↳ `profileId` | number | The unique identifier of the attendee profile | +| ↳ `name` | string | The attendee name | +| ↳ `title` | string | The attendee job title | +| ↳ `companyName` | string | The name of the company the attendee belongs to | +| ↳ `email` | string | The attendee email address | +| ↳ `isCalendarEventOrganizer` | boolean | Whether the attendee organized the calendar event | +| ↳ `isCalendarInvitee` | boolean | Whether the attendee was invited on the calendar event | +| `notes` | string | The meeting notes with Markdown formatting | +| `privateNotes` | string | The authenticated user private notes for the meeting | +| `actionItems` | array | Action items created for the meeting | +| ↳ `id` | number | The unique identifier of the action item | +| ↳ `title` | string | The action item title | +| ↳ `description` | string | The action item description | +| ↳ `assignee` | json | The assignee as an object with profileId, name, title, companyName, and email, or null if unassigned | +| ↳ `status` | string | The completion status, PENDING or DONE | +| `insights` | json | Insight results for the meeting, keyed by the name of the user-created insight | +| `linkAccess` | string | Who can access the meeting through its shareable link: Editor, Viewer, or LimitedViewer | +| `calendarEvent` | json | The associated calendar event as an object with id, icalUid, description, platform, and platformId, or null | + +### Circleback Search Meetings + +Searches Circleback meetings by name and content, with optional filters for specific meetings, tags, and people. + +#### Input + +| Parameter | Type | Required | Description | +| --------- | ---- | -------- | ----------- | +| `apiKey` | string | Yes | Circleback API key | +| `searchTerm` | string | No | The text to search for across meeting names and content | +| `meetingIds` | string | No | Comma-separated meeting IDs to restrict the search to | +| `tagIds` | string | No | Comma-separated tag IDs to restrict the search to | +| `attendeeProfileIds` | string | No | Comma-separated profile IDs of attendees to restrict the search to | +| `cursor` | string | No | Pagination cursor from a previous response | + +#### Output + +| Parameter | Type | Description | +| --------- | ---- | ----------- | +| `meetings` | array | The matching meetings on this page | +| ↳ `id` | string | The Circleback meeting ID | +| ↳ `name` | string | The meeting name | +| ↳ `createdAt` | string | When the meeting was created \(ISO 8601\) | +| ↳ `updatedAt` | string | When the meeting was last updated \(ISO 8601\) | +| ↳ `duration` | number | The meeting duration in seconds | +| ↳ `url` | string | The URL of the virtual meeting \(Zoom, Google Meet, or Microsoft Teams\) | +| ↳ `recordingUrl` | string | The URL of the meeting recording file, valid for 24 hours | +| ↳ `tags` | array | Tags added to the meeting | +| ↳ `id` | number | The unique identifier of the tag | +| ↳ `name` | string | The display name of the tag | +| ↳ `description` | string | A description of the tag | +| ↳ `icalUid` | string | The identifier of the calendar event associated with the meeting | +| ↳ `attendees` | array | The meeting attendees | +| ↳ `profileId` | number | The unique identifier of the attendee profile | +| ↳ `name` | string | The attendee name | +| ↳ `title` | string | The attendee job title | +| ↳ `companyName` | string | The name of the company the attendee belongs to | +| ↳ `email` | string | The attendee email address | +| ↳ `isCalendarEventOrganizer` | boolean | Whether the attendee organized the calendar event | +| ↳ `isCalendarInvitee` | boolean | Whether the attendee was invited on the calendar event | +| ↳ `notes` | string | The meeting notes with Markdown formatting | +| ↳ `privateNotes` | string | The authenticated user private notes for the meeting | +| ↳ `actionItems` | array | Action items created for the meeting | +| ↳ `id` | number | The unique identifier of the action item | +| ↳ `title` | string | The action item title | +| ↳ `description` | string | The action item description | +| ↳ `assignee` | json | The assignee as an object with profileId, name, title, companyName, and email, or null if unassigned | +| ↳ `status` | string | The completion status, PENDING or DONE | +| ↳ `insights` | json | Insight results for the meeting, keyed by the name of the user-created insight | +| ↳ `linkAccess` | string | Who can access the meeting through its shareable link: Editor, Viewer, or LimitedViewer | +| ↳ `calendarEvent` | json | The associated calendar event as an object with id, icalUid, description, platform, and platformId, or null | +| `nextCursor` | string | Pagination cursor for the next page, or null on the last page | +| `hasMore` | boolean | Whether another page of results is available | + +### Circleback Get Transcript + +Gets the full transcript for a Circleback meeting. + +#### Input + +| Parameter | Type | Required | Description | +| --------- | ---- | -------- | ----------- | +| `apiKey` | string | Yes | Circleback API key | +| `meetingId` | string | Yes | The unique identifier of the meeting | + +#### Output + +| Parameter | Type | Description | +| --------- | ---- | ----------- | +| `transcript` | array | The transcript segments in order | +| ↳ `speaker` | string | The speaker name | +| ↳ `text` | string | The words spoken | +| ↳ `timestamp` | number | The timestamp in seconds that marks the beginning of the segment | + +### Circleback Update Meeting + +Updates the name, notes, or private notes of a Circleback meeting. The API returns only the fields that were updated. + +#### Input + +| Parameter | Type | Required | Description | +| --------- | ---- | -------- | ----------- | +| `apiKey` | string | Yes | Circleback API key | +| `meetingId` | string | Yes | The unique identifier of the meeting | +| `name` | string | No | The new name of the meeting | +| `notes` | string | No | The new meeting notes in Markdown | +| `privateNotes` | string | No | The authenticated user private notes for the meeting | + +#### Output + +| Parameter | Type | Description | +| --------- | ---- | ----------- | +| `id` | string | The Circleback meeting ID | +| `name` | string | The updated meeting name, when the name was updated | +| `notes` | string | The updated meeting notes, when the notes were updated | +| `privateNotes` | string | The updated private notes, when the private notes were updated | +| `updatedAt` | string | When the meeting was last updated \(ISO 8601\) | + +### Circleback Delete Meeting + +Deletes a Circleback meeting. Only the owner of the meeting can delete it. Returns the deleted meeting. + +#### Input + +| Parameter | Type | Required | Description | +| --------- | ---- | -------- | ----------- | +| `apiKey` | string | Yes | Circleback API key | +| `meetingId` | string | Yes | The unique identifier of the meeting | + +#### Output + +| Parameter | Type | Description | +| --------- | ---- | ----------- | +| `id` | string | The Circleback meeting ID | +| `name` | string | The meeting name | +| `createdAt` | string | When the meeting was created \(ISO 8601\) | +| `updatedAt` | string | When the meeting was last updated \(ISO 8601\) | +| `duration` | number | The meeting duration in seconds | +| `url` | string | The URL of the virtual meeting \(Zoom, Google Meet, or Microsoft Teams\) | +| `recordingUrl` | string | The URL of the meeting recording file, valid for 24 hours | +| `tags` | array | Tags added to the meeting | +| ↳ `id` | number | The unique identifier of the tag | +| ↳ `name` | string | The display name of the tag | +| ↳ `description` | string | A description of the tag | +| `icalUid` | string | The identifier of the calendar event associated with the meeting | +| `attendees` | array | The meeting attendees | +| ↳ `profileId` | number | The unique identifier of the attendee profile | +| ↳ `name` | string | The attendee name | +| ↳ `title` | string | The attendee job title | +| ↳ `companyName` | string | The name of the company the attendee belongs to | +| ↳ `email` | string | The attendee email address | +| ↳ `isCalendarEventOrganizer` | boolean | Whether the attendee organized the calendar event | +| ↳ `isCalendarInvitee` | boolean | Whether the attendee was invited on the calendar event | +| `notes` | string | The meeting notes with Markdown formatting | +| `privateNotes` | string | The authenticated user private notes for the meeting | +| `actionItems` | array | Action items created for the meeting | +| ↳ `id` | number | The unique identifier of the action item | +| ↳ `title` | string | The action item title | +| ↳ `description` | string | The action item description | +| ↳ `assignee` | json | The assignee as an object with profileId, name, title, companyName, and email, or null if unassigned | +| ↳ `status` | string | The completion status, PENDING or DONE | +| `insights` | json | Insight results for the meeting, keyed by the name of the user-created insight | +| `linkAccess` | string | Who can access the meeting through its shareable link: Editor, Viewer, or LimitedViewer | +| `calendarEvent` | json | The associated calendar event as an object with id, icalUid, description, platform, and platformId, or null | + +### Circleback List Action Items + +Lists action items across the authenticated user meetings, filtered by assignee, status, tags, and attendees. Defaults to incomplete action items assigned to the API key owner. + +#### Input + +| Parameter | Type | Required | Description | +| --------- | ---- | -------- | ----------- | +| `apiKey` | string | Yes | Circleback API key | +| `assigneeType` | string | No | The assignee scope to filter by: Me, NotMe, Profile, MyWorkspace, OutsideMyWorkspace, Unassigned, or Anyone. Defaults to Me | +| `assigneeProfileId` | string | No | Profile ID of the assignee to filter by | +| `assigneeTeamId` | string | No | Team ID of the assignee to filter by | +| `status` | string | No | Completion status to filter by: PENDING or DONE. Defaults to incomplete action items | +| `attendeeProfileIds` | string | No | Comma-separated profile IDs of meeting attendees to filter by | +| `tagIds` | string | No | Comma-separated tag IDs to filter by | +| `cursor` | string | No | Pagination cursor from a previous response | + +#### Output + +| Parameter | Type | Description | +| --------- | ---- | ----------- | +| `actionItems` | array | The action items on this page. Each also carries canEditActionItem, whether the caller may edit it | +| ↳ `id` | number | The unique identifier of the action item | +| ↳ `title` | string | The action item title | +| ↳ `description` | string | The action item description | +| ↳ `assignee` | json | The assignee as an object with profileId, name, title, companyName, and email, or null if unassigned | +| ↳ `completedAt` | string | When the action item was marked done, or null if not completed | +| ↳ `meetingId` | string | The ID of the meeting the action item belongs to, or null | +| ↳ `status` | string | The completion status, PENDING or DONE | +| ↳ `meetings` | array | The meetings the action item is associated with | +| ↳ `id` | string | The Circleback meeting ID | +| ↳ `name` | string | The meeting name | +| ↳ `createdAt` | string | When the meeting was created \(ISO 8601\) | +| ↳ `canEditActionItem` | boolean | Whether the caller may edit the action item. Returned only by the list operation | +| `nextCursor` | string | Pagination cursor for the next page, or null on the last page | +| `hasMore` | boolean | Whether another page of action items is available | + +### Circleback Update Action Item + +Updates the title, description, status, or assignee of a Circleback action item. Returns the updated action item. + +#### Input + +| Parameter | Type | Required | Description | +| --------- | ---- | -------- | ----------- | +| `apiKey` | string | Yes | Circleback API key | +| `actionItemId` | string | Yes | The unique identifier of the action item | +| `title` | string | No | The new title of the action item | +| `description` | string | No | The new detailed description of the action item | +| `assigneeProfileId` | string | No | The profile ID to assign the action item to, or the literal text null to remove the assignee | +| `status` | string | No | The completion status: PENDING or DONE | + +#### Output + +| Parameter | Type | Description | +| --------- | ---- | ----------- | +| `id` | number | The unique identifier of the action item | +| `title` | string | The action item title | +| `description` | string | The action item description | +| `assignee` | json | The assignee as an object with profileId, name, title, companyName, and email, or null if unassigned | +| `completedAt` | string | When the action item was marked done, or null if not completed | +| `meetingId` | string | The ID of the meeting the action item belongs to, or null | +| `status` | string | The completion status, PENDING or DONE | +| `meetings` | array | The meetings the action item is associated with | +| ↳ `id` | string | The Circleback meeting ID | +| ↳ `name` | string | The meeting name | +| ↳ `createdAt` | string | When the meeting was created \(ISO 8601\) | +| `canEditActionItem` | boolean | Whether the caller may edit the action item. Returned only by the list operation | + +### Circleback Delete Action Item + +Deletes a Circleback action item. Returns the deleted action item. + +#### Input + +| Parameter | Type | Required | Description | +| --------- | ---- | -------- | ----------- | +| `apiKey` | string | Yes | Circleback API key | +| `actionItemId` | string | Yes | The unique identifier of the action item | + +#### Output + +| Parameter | Type | Description | +| --------- | ---- | ----------- | +| `id` | number | The unique identifier of the action item | +| `title` | string | The action item title | +| `description` | string | The action item description | +| `assignee` | json | The assignee as an object with profileId, name, title, companyName, and email, or null if unassigned | +| `completedAt` | string | When the action item was marked done, or null if not completed | +| `meetingId` | string | The ID of the meeting the action item belongs to, or null | +| `status` | string | The completion status, PENDING or DONE | +| `meetings` | array | The meetings the action item is associated with | +| ↳ `id` | string | The Circleback meeting ID | +| ↳ `name` | string | The meeting name | +| ↳ `createdAt` | string | When the meeting was created \(ISO 8601\) | +| `canEditActionItem` | boolean | Whether the caller may edit the action item. Returned only by the list operation | + +### Circleback List Calendar Events + +Lists upcoming calendar events from the authenticated user connected calendars, with pagination and a configurable time window. + +#### Input + +| Parameter | Type | Required | Description | +| --------- | ---- | -------- | ----------- | +| `apiKey` | string | Yes | Circleback API key | +| `startTimeLookbackHours` | string | No | How many hours in the past to include calendar meetings from | +| `startTimeLookaheadHours` | string | No | How many hours in the future to include calendar meetings from | +| `sortDirection` | string | No | The direction calendar meetings are sorted in by start time: ascending or descending. Defaults to ascending | +| `includeOfflineSingleAttendee` | string | No | Set to true to include calendar meetings that have no meeting link and only one attendee | +| `cursor` | string | No | Pagination cursor from a previous response | + +#### Output + +| Parameter | Type | Description | +| --------- | ---- | ----------- | +| `events` | array | The calendar events on this page | +| ↳ `id` | string | The unique identifier of the calendar meeting | +| ↳ `title` | string | The title of the calendar meeting | +| ↳ `icalUid` | string | The iCalendar UID of the calendar event | +| ↳ `attendees` | array | The attendees invited to the calendar meeting | +| ↳ `name` | string | The name of the attendee | +| ↳ `email` | string | The email address of the attendee | +| ↳ `isOrganizer` | boolean | Whether the attendee organized the calendar meeting | +| ↳ `status` | string | The attendee response to the invitation: accepted, declined, tentative, or not_available | +| ↳ `calendarDescription` | string | The description of the calendar event | +| ↳ `calendarPlatform` | string | The calendar platform the meeting was synced from | +| ↳ `startTime` | string | When the calendar meeting starts \(ISO 8601\) | +| ↳ `endTime` | string | When the calendar meeting ends \(ISO 8601\) | +| ↳ `isExternal` | boolean | Whether the meeting includes attendees from more than one email domain | +| ↳ `isHostedByMe` | boolean | Whether the current user organized the calendar meeting | +| ↳ `location` | string | The location of the calendar meeting | +| ↳ `meetingId` | string | The Circleback meeting associated with the calendar meeting, when one exists | +| ↳ `meetingPlatform` | string | The conferencing platform hosting the meeting, when detected | +| ↳ `organizerEmail` | string | The email address of the meeting organizer | +| ↳ `platform` | string | The conferencing platform hosting the meeting, when detected. Alias of meetingPlatform | +| ↳ `overrideShouldRecord` | boolean | Whether the user manually overrode the automatic recording decision, or null when no override is set | +| ↳ `recurringEventId` | string | The identifier of the recurring event series the meeting belongs to | +| ↳ `willRecord` | boolean | Whether the notetaker will join and record the meeting | +| ↳ `willRecordReason` | string | A human-readable explanation of why the meeting will or will not be recorded | +| `nextCursor` | string | Pagination cursor for the next page, or null on the last page | +| `hasMore` | boolean | Whether another page of calendar events is available | + +### Circleback List Companies + +Lists the companies whose people attend the authenticated user meetings, with optional tag filters. + +#### Input + +| Parameter | Type | Required | Description | +| --------- | ---- | -------- | ----------- | +| `apiKey` | string | Yes | Circleback API key | +| `tagIds` | string | No | Comma-separated tag IDs. Filters companies to those with meetings so tagged | +| `cursor` | string | No | Pagination cursor from a previous response | + +#### Output + +| Parameter | Type | Description | +| --------- | ---- | ----------- | +| `companies` | array | The companies on this page | +| ↳ `id` | number | The unique identifier of the company | +| ↳ `name` | string | The company name | +| ↳ `avatarUrl` | string | The URL of the company logo image | +| ↳ `domain` | string | The company website domain | +| `nextCursor` | string | Pagination cursor for the next page, or null on the last page | +| `hasMore` | boolean | Whether another page of companies is available | + +### Circleback Get Company + +Gets a company by its domain from Circleback, including its people and external links. + +#### Input + +| Parameter | Type | Required | Description | +| --------- | ---- | -------- | ----------- | +| `apiKey` | string | Yes | Circleback API key | +| `domain` | string | Yes | The website domain of the company to fetch, such as example.com | + +#### Output + +| Parameter | Type | Description | +| --------- | ---- | ----------- | +| `name` | string | The company name | +| `avatarUrl` | string | The URL of the company logo image | +| `domain` | string | The company website domain | +| `externalLinks` | array | Links to the company on external platforms and connected integrations | +| ↳ `url` | string | The URL of the external resource | +| ↳ `objectType` | string | Whether the link refers to a person or a company | +| ↳ `type` | string | The platform or integration the link points to, such as Attio, HubSpot, Linear, Salesforce, Zoho, linkedin, or website | +| `people` | array | People at the company the authenticated user has met with | +| ↳ `id` | number | The unique identifier of the person | +| ↳ `title` | string | The person job title | +| ↳ `companyId` | number | The unique identifier of the company the person belongs to | +| ↳ `companyName` | string | The name of the company the person belongs to | +| ↳ `email` | string | The person email address | +| ↳ `firstName` | string | The person first name | +| ↳ `lastName` | string | The person last name | + +### Circleback List People + +Lists the people who attend the authenticated user meetings, ordered by most recent meeting, with optional company and tag filters. + +#### Input + +| Parameter | Type | Required | Description | +| --------- | ---- | -------- | ----------- | +| `apiKey` | string | Yes | Circleback API key | +| `domains` | string | No | Comma-separated company domains to filter people by | +| `tagIds` | string | No | Comma-separated tag IDs. Filters people to attendees of meetings so tagged | +| `limit` | string | No | The maximum number of people to return | +| `cursor` | string | No | Pagination cursor from a previous response | + +#### Output + +| Parameter | Type | Description | +| --------- | ---- | ----------- | +| `people` | array | The people on this page | +| ↳ `id` | number | The unique identifier of the person | +| ↳ `title` | string | The person job title | +| ↳ `companyId` | number | The unique identifier of the company the person belongs to | +| ↳ `companyName` | string | The name of the company the person belongs to | +| ↳ `email` | string | The person email address | +| ↳ `firstName` | string | The person first name | +| ↳ `lastName` | string | The person last name | +| `nextCursor` | string | Pagination cursor for the next page, or null on the last page | +| `hasMore` | boolean | Whether another page of people is available | + +### Circleback Get Person + +Gets a person by their profile ID from Circleback, including their profile details and external links. + +#### Input + +| Parameter | Type | Required | Description | +| --------- | ---- | -------- | ----------- | +| `apiKey` | string | Yes | Circleback API key | +| `profileId` | string | Yes | The unique identifier of the person profile | + +#### Output + +| Parameter | Type | Description | +| --------- | ---- | ----------- | +| `id` | number | The unique identifier of the person | +| `title` | string | The person job title | +| `companyId` | number | The unique identifier of the company the person belongs to | +| `companyName` | string | The name of the company the person belongs to | +| `email` | string | The person email address | +| `firstName` | string | The person first name | +| `lastName` | string | The person last name | +| `externalLinks` | array | Links to the person on external platforms and connected integrations | +| ↳ `url` | string | The URL of the external resource | +| ↳ `objectType` | string | Whether the link refers to a person or a company | +| ↳ `type` | string | The platform or integration the link points to, such as Attio, HubSpot, Linear, Salesforce, Zoho, linkedin, or website | + +### Circleback List Tags + +Lists every tag available to the authenticated Circleback user. + +#### Input + +| Parameter | Type | Required | Description | +| --------- | ---- | -------- | ----------- | +| `apiKey` | string | Yes | Circleback API key | + +#### Output + +| Parameter | Type | Description | +| --------- | ---- | ----------- | +| `tags` | array | Every tag available to the authenticated user | +| ↳ `id` | number | The unique identifier of the tag | +| ↳ `name` | string | The display name of the tag | +| ↳ `description` | string | A description of the tag | + +### Circleback Create Tag + +Creates a new tag in Circleback. If a tag with the same name already exists, a conflict error is returned. + +#### Input + +| Parameter | Type | Required | Description | +| --------- | ---- | -------- | ----------- | +| `apiKey` | string | Yes | Circleback API key | +| `tagName` | string | Yes | The display name of the tag | +| `tagDescription` | string | No | A description of the tag | + +#### Output + +| Parameter | Type | Description | +| --------- | ---- | ----------- | +| `id` | number | The unique identifier of the tag | +| `name` | string | The display name of the tag | +| `description` | string | A description of the tag | + +### Circleback Update Tag + +Updates the name or description of a Circleback tag. + +#### Input + +| Parameter | Type | Required | Description | +| --------- | ---- | -------- | ----------- | +| `apiKey` | string | Yes | Circleback API key | +| `tagId` | string | Yes | The unique identifier of the tag | +| `tagName` | string | No | The new display name of the tag | +| `tagDescription` | string | No | The new description of the tag | + +#### Output + +| Parameter | Type | Description | +| --------- | ---- | ----------- | +| `id` | number | The unique identifier of the tag | +| `name` | string | The display name of the tag | +| `description` | string | A description of the tag | + +### Circleback Delete Tag + +Permanently deletes a Circleback tag and removes it from every meeting it was applied to. + +#### Input + +| Parameter | Type | Required | Description | +| --------- | ---- | -------- | ----------- | +| `apiKey` | string | Yes | Circleback API key | +| `tagId` | string | Yes | The unique identifier of the tag | + +#### Output + +| Parameter | Type | Description | +| --------- | ---- | ----------- | +| `id` | number | The unique identifier of the tag | +| `name` | string | The display name of the tag | +| `description` | string | A description of the tag | + +### Circleback Add Tag to Meetings + +Applies an existing Circleback tag to one or more meetings and returns the updated meetings. + +#### Input + +| Parameter | Type | Required | Description | +| --------- | ---- | -------- | ----------- | +| `apiKey` | string | Yes | Circleback API key | +| `tagId` | string | Yes | The unique identifier of the tag to apply | +| `meetingIds` | string | No | Comma-separated IDs of the meetings to tag | + +#### Output + +| Parameter | Type | Description | +| --------- | ---- | ----------- | +| `meetings` | array | The updated meetings | +| ↳ `id` | string | The Circleback meeting ID | +| ↳ `name` | string | The meeting name | +| ↳ `createdAt` | string | When the meeting was created \(ISO 8601\) | +| ↳ `updatedAt` | string | When the meeting was last updated \(ISO 8601\) | +| ↳ `duration` | number | The meeting duration in seconds | +| ↳ `url` | string | The URL of the virtual meeting \(Zoom, Google Meet, or Microsoft Teams\) | +| ↳ `recordingUrl` | string | The URL of the meeting recording file, valid for 24 hours | +| ↳ `tags` | array | Tags added to the meeting | +| ↳ `id` | number | The unique identifier of the tag | +| ↳ `name` | string | The display name of the tag | +| ↳ `description` | string | A description of the tag | +| ↳ `icalUid` | string | The identifier of the calendar event associated with the meeting | +| ↳ `attendees` | array | The meeting attendees | +| ↳ `profileId` | number | The unique identifier of the attendee profile | +| ↳ `name` | string | The attendee name | +| ↳ `title` | string | The attendee job title | +| ↳ `companyName` | string | The name of the company the attendee belongs to | +| ↳ `email` | string | The attendee email address | +| ↳ `isCalendarEventOrganizer` | boolean | Whether the attendee organized the calendar event | +| ↳ `isCalendarInvitee` | boolean | Whether the attendee was invited on the calendar event | +| ↳ `notes` | string | The meeting notes with Markdown formatting | +| ↳ `privateNotes` | string | The authenticated user private notes for the meeting | +| ↳ `actionItems` | array | Action items created for the meeting | +| ↳ `id` | number | The unique identifier of the action item | +| ↳ `title` | string | The action item title | +| ↳ `description` | string | The action item description | +| ↳ `assignee` | json | The assignee as an object with profileId, name, title, companyName, and email, or null if unassigned | +| ↳ `status` | string | The completion status, PENDING or DONE | +| ↳ `insights` | json | Insight results for the meeting, keyed by the name of the user-created insight | +| ↳ `linkAccess` | string | Who can access the meeting through its shareable link: Editor, Viewer, or LimitedViewer | +| ↳ `calendarEvent` | json | The associated calendar event as an object with id, icalUid, description, platform, and platformId, or null | + +### Circleback Remove Tag from Meetings + +Removes a Circleback tag from one or more meetings and returns the updated meetings. + +#### Input + +| Parameter | Type | Required | Description | +| --------- | ---- | -------- | ----------- | +| `apiKey` | string | Yes | Circleback API key | +| `tagId` | string | Yes | The unique identifier of the tag to remove | +| `meetingIds` | string | No | Comma-separated IDs of the meetings to remove the tag from | + +#### Output + +| Parameter | Type | Description | +| --------- | ---- | ----------- | +| `meetings` | array | The updated meetings | +| ↳ `id` | string | The Circleback meeting ID | +| ↳ `name` | string | The meeting name | +| ↳ `createdAt` | string | When the meeting was created \(ISO 8601\) | +| ↳ `updatedAt` | string | When the meeting was last updated \(ISO 8601\) | +| ↳ `duration` | number | The meeting duration in seconds | +| ↳ `url` | string | The URL of the virtual meeting \(Zoom, Google Meet, or Microsoft Teams\) | +| ↳ `recordingUrl` | string | The URL of the meeting recording file, valid for 24 hours | +| ↳ `tags` | array | Tags added to the meeting | +| ↳ `id` | number | The unique identifier of the tag | +| ↳ `name` | string | The display name of the tag | +| ↳ `description` | string | A description of the tag | +| ↳ `icalUid` | string | The identifier of the calendar event associated with the meeting | +| ↳ `attendees` | array | The meeting attendees | +| ↳ `profileId` | number | The unique identifier of the attendee profile | +| ↳ `name` | string | The attendee name | +| ↳ `title` | string | The attendee job title | +| ↳ `companyName` | string | The name of the company the attendee belongs to | +| ↳ `email` | string | The attendee email address | +| ↳ `isCalendarEventOrganizer` | boolean | Whether the attendee organized the calendar event | +| ↳ `isCalendarInvitee` | boolean | Whether the attendee was invited on the calendar event | +| ↳ `notes` | string | The meeting notes with Markdown formatting | +| ↳ `privateNotes` | string | The authenticated user private notes for the meeting | +| ↳ `actionItems` | array | Action items created for the meeting | +| ↳ `id` | number | The unique identifier of the action item | +| ↳ `title` | string | The action item title | +| ↳ `description` | string | The action item description | +| ↳ `assignee` | json | The assignee as an object with profileId, name, title, companyName, and email, or null if unassigned | +| ↳ `status` | string | The completion status, PENDING or DONE | +| ↳ `insights` | json | Insight results for the meeting, keyed by the name of the user-created insight | +| ↳ `linkAccess` | string | Who can access the meeting through its shareable link: Editor, Viewer, or LimitedViewer | +| ↳ `calendarEvent` | json | The associated calendar event as an object with id, icalUid, description, platform, and platformId, or null | + + + ## Triggers A **Trigger** is a block that starts a workflow when an event happens in this service. diff --git a/apps/docs/content/docs/integrations/elasticsearch.mdx b/apps/docs/content/docs/integrations/elasticsearch.mdx index a0fb33108fb..fc520cd52a9 100644 --- a/apps/docs/content/docs/integrations/elasticsearch.mdx +++ b/apps/docs/content/docs/integrations/elasticsearch.mdx @@ -306,7 +306,7 @@ Retrieve index information including settings, mappings, and aliases. | Parameter | Type | Description | | --------- | ---- | ----------- | -| `index` | json | Index information including aliases, mappings, and settings | +| `indices` | json | Matched indices keyed by index name, each with its aliases, mappings, and settings | ### Elasticsearch Cluster Health @@ -324,7 +324,7 @@ Get the health status of the Elasticsearch cluster. | `username` | string | No | Username for basic auth | | `password` | string | No | Password for basic auth | | `waitForStatus` | string | No | Wait until cluster reaches this status: green, yellow, or red | -| `timeout` | string | No | Timeout for the wait operation \(e.g., 30s, 1m\) | +| `clusterTimeout` | string | No | How long Elasticsearch waits for the cluster to reach the requested status, as an Elasticsearch time value \(e.g., 30s, 1m\). Not named "timeout": that name is reserved by the tool transport as a client-side abort deadline in milliseconds. | #### Output @@ -377,12 +377,13 @@ List all indices in the Elasticsearch cluster with their health, status, and sta | `apiKey` | string | No | Elasticsearch API key | | `username` | string | No | Username for basic auth | | `password` | string | No | Password for basic auth | +| `includeSystemIndices` | boolean | No | Include Elasticsearch system indices \(names starting with "."\). Omitted by default. | #### Output | Parameter | Type | Description | | --------- | ---- | ----------- | | `message` | string | Summary message about the indices | -| `indices` | json | Array of index information objects | +| `indices` | json | Array of index information objects \(index, health, status, docsCount, storeSize, primaryShards, replicaShards\). System indices are omitted unless includeSystemIndices is set. | diff --git a/apps/docs/content/docs/integrations/file.mdx b/apps/docs/content/docs/integrations/file.mdx index 4222a602a7d..23cbb7cfdb5 100644 --- a/apps/docs/content/docs/integrations/file.mdx +++ b/apps/docs/content/docs/integrations/file.mdx @@ -1,6 +1,6 @@ --- title: File -description: Read, get content, fetch, write, append, compress, decompress, and manage sharing for files +description: Read, search, get content, fetch, write, append, compress, decompress, and manage sharing for files --- import { BlockInfoCard } from "@/components/ui/block-info-card" @@ -11,23 +11,24 @@ import { BlockInfoCard } from "@/components/ui/block-info-card" /> {/* MANUAL-CONTENT-START:intro */} -The File block is a built-in Sim block for working with files stored in the workspace, or fetched from external URLs. It handles reading, writing, appending, compressing, decompressing, and sharing files as part of a workflow. +The File block is a built-in Sim block for working with files stored in the workspace, or fetched from external URLs. It handles reading, searching, writing, appending, compressing, decompressing, and sharing files as part of a workflow. With the File block, you can: - **Read and extract content**: Load workspace file objects and extract their text content +- **Search workspace content**: Match a regular expression, or an exact piece of text, against the indexed lines of active workspace files with bounded line-level results - **Fetch from URLs**: Retrieve and parse files from external URLs with custom headers - **Write and append**: Create new workspace files or append content to existing ones - **Compress and decompress**: Bundle files into a .zip archive or extract an archive into the workspace - **Manage sharing**: Enable or disable a public share link for a file, with public, password, email, or SSO access modes -In Sim, the File block allows your agents to read and extract text from workspace files, fetch and parse files from URLs, write or append content to files, bundle files into or out of .zip archives, and control public sharing access for a file—all programmatically as steps in a workflow. This makes it possible to move file content into and out of a workflow, package outputs for download or transfer, and expose files to external users through a managed share link. +In Sim, the File block allows your agents to search, read, and extract text from workspace files, fetch and parse files from URLs, write or append content to files, bundle files into or out of .zip archives, and control public sharing access for a file—all programmatically as steps in a workflow. This makes it possible to explore workspace content, move file content into and out of a workflow, package outputs for download or transfer, and expose files to external users through a managed share link. {/* MANUAL-CONTENT-END */} ## Usage Instructions -Read workspace file objects, extract the text content of files, fetch and parse files from URLs with optional headers, write new workspace files, append content to existing files, compress files into a .zip archive, extract a .zip archive into the workspace, or manage the public share link for a file. +Read workspace file objects, search indexed text across all active workspace files, extract the text content of files, fetch and parse files from URLs with optional headers, write new workspace files, append content to existing files, compress files into a .zip archive, extract a .zip archive into the workspace, or manage the public share link for a file. @@ -67,6 +68,36 @@ Extract the text content of one or more workspace files from selected file objec | --------- | ---- | ----------- | | `contents` | array | Array of file text contents, one entry per file in input order | +### File Search + +Search the indexed text of active workspace files for lines matching a regular expression, and return each matching line once with its file ID and line number. Coverage is what the index currently holds, so check "complete" and "indexStatus" before concluding that something is absent. + +#### Input + +| Parameter | Type | Required | Description | +| --------- | ---- | -------- | ----------- | +| `query` | string | Yes | A regular expression matched against each line, 3-512 characters. Supports "." "*" "+" "?" "\{n,m\}" and their lazy forms, character classes such as "\[a-z\]" and "\[^0-9\]", the classes \d \w \s and \D \W \S, alternation "\|", groups "\(...\)" and "\(?:...\)", the anchors "^" and "$", and the word boundary \b. Lookahead, lookbehind, backreferences, named groups, inline flags such as "\(?i\)", \p\{...\} and POSIX "\[\[:alpha:\]\]" classes are not supported, and a pattern cannot span a line break. The pattern must contain at least 3 consecutive literal characters that every match will include — write "error \d+" rather than "\w+ \d+". Escape any metacharacter you mean literally. Matching is case-insensitive until the pattern contains an uppercase letter you are searching for; uppercase inside an escape or a character class, such as \D or \[A-Z\], does not make it case-sensitive. When the workflow builder sets Match to exact instead, the query is matched verbatim and no metacharacter needs escaping. | +| `mode` | string | No | How the query is read, chosen by the workflow builder: "regex" \(default\) as a regular expression, or "exact" as verbatim text. | +| `maxResults` | number | No | Hard result cap configured by the workflow builder \(1-200, default 50\). | + +#### Output + +| Parameter | Type | Description | +| --------- | ---- | ----------- | +| `results` | array | Matching logical lines with their workspace file ID and 1-based line number. | +| ↳ `fileId` | string | Canonical workspace file ID. | +| ↳ `lineNumber` | number | 1-based logical line number. | +| ↳ `text` | string | Matching line or bounded match-centered preview. | +| `count` | number | Number of returned matching lines. | +| `truncated` | boolean | Whether more matching lines exist beyond the configured hard cap. | +| `complete` | boolean | Whether indexing has no pending or failed current revisions; skipped and partial coverage is reported separately. | +| `indexStatus` | object | Current workspace search-index coverage by file status. | +| ↳ `readyFiles` | number | Files whose current revision is searchable. | +| ↳ `pendingFiles` | number | Files still waiting to be indexed. | +| ↳ `failedFiles` | number | Files whose current indexing attempt failed. | +| ↳ `skippedFiles` | number | Files intentionally excluded because they are unsupported or oversized. | +| ↳ `partialFiles` | number | Searchable files whose extracted text was truncated by the parser or cap. | + ### File Fetch Fetch and parse a file from a URL with optional custom headers. @@ -87,15 +118,17 @@ Fetch and parse a file from a URL with optional custom headers. ### File Write -Create a new workspace file. If a file with the same name already exists, a numeric suffix is added (e.g., "data (1).csv"). +Create a new workspace file, either from text content or from an existing file. If a file with the same name already exists, a numeric suffix is added (e.g., "data (1).csv") unless overwrite is enabled. #### Input | Parameter | Type | Required | Description | | --------- | ---- | -------- | ----------- | -| `fileName` | string | Yes | File name \(e.g., "data.csv"\). If a file with this name exists, a numeric suffix is added automatically. | -| `content` | string | Yes | The text content to write to the file. | -| `contentType` | string | No | MIME type for new files \(e.g., "text/plain"\). Auto-detected from file extension if omitted. | +| `fileName` | string | No | File name \(e.g., "data.csv"\). Required when writing text; optional when storing a file, which keeps its own name unless this overrides it. If the name already exists, a numeric suffix is added automatically unless overwrite is enabled. | +| `content` | string | No | The text content to write to the file. Provide exactly one of content or fileInput. | +| `fileInput` | file | No | An existing file to store in the workspace, such as one produced by an earlier tool. Use this for anything that is not text — PDFs, images, audio, archives. Provide exactly one of content or fileInput. | +| `contentType` | string | No | MIME type for new files \(e.g., "text/plain"\). Auto-detected from the file extension, or taken from the stored file, if omitted. | +| `overwrite` | boolean | No | Replace the contents of an existing file at the exact target path \(folder and name\) instead of creating a suffixed copy. Creates the file when that path does not exist yet. | #### Output diff --git a/apps/docs/content/docs/integrations/github.mdx b/apps/docs/content/docs/integrations/github.mdx index 120ad61de2f..daaea6f0ebc 100644 --- a/apps/docs/content/docs/integrations/github.mdx +++ b/apps/docs/content/docs/integrations/github.mdx @@ -1019,10 +1019,10 @@ Update branch protection rules for a specific branch, including status checks, r | `owner` | string | Yes | Repository owner \(user or organization\) | | `repo` | string | Yes | Repository name | | `branch` | string | Yes | Branch name | -| `required_status_checks` | object | Yes | Required status check configuration \(null to disable\). Object with strict \(boolean\) and contexts \(string array\) | -| `enforce_admins` | boolean | Yes | Whether to enforce restrictions for administrators | -| `required_pull_request_reviews` | object | Yes | PR review requirements \(null to disable\). Object with optional required_approving_review_count, dismiss_stale_reviews, require_code_owner_reviews | -| `restrictions` | object | Yes | Push restrictions \(null to disable\). Object with users \(string array\) and teams \(string array\) | +| `required_status_checks` | object | No | Required status check configuration. Object with strict \(boolean\) and contexts \(string array\). Omit to disable status checks — GitHub receives an explicit null. | +| `enforce_admins` | boolean | No | Whether to enforce restrictions for administrators. Omit to disable admin enforcement — GitHub receives an explicit null. | +| `required_pull_request_reviews` | object | No | PR review requirements. Object with optional required_approving_review_count, dismiss_stale_reviews, require_code_owner_reviews. Omit to disable review requirements — GitHub receives an explicit null. | +| `restrictions` | object | No | Push restrictions, available only for organization-owned repositories. Object with users \(string array\), teams \(string array\) and optional apps \(string array\). Omit to disable push restrictions — GitHub receives an explicit null. | | `apiKey` | string | Yes | GitHub Personal Access Token | #### Output @@ -1805,7 +1805,7 @@ Trigger a workflow dispatch event for a GitHub Actions workflow. The workflow mu ### GitHub List Workflow Runs -List workflow runs for a repository. Supports filtering by actor, branch, event, and status. Returns run details including status, conclusion, and links. +List workflow runs for a repository, or for a single workflow when a workflow ID or filename is given. Supports filtering by actor, branch, event, and status. Returns run details including status, conclusion, and links. #### Input @@ -1813,6 +1813,7 @@ List workflow runs for a repository. Supports filtering by actor, branch, event, | --------- | ---- | -------- | ----------- | | `owner` | string | Yes | Repository owner \(user or organization\) | | `repo` | string | Yes | Repository name | +| `workflow_id` | string | No | The ID of the workflow. You can also pass the workflow file name as a string \(e.g., ci.yml\). Omit to list runs across the whole repository. | | `actor` | string | No | Filter by user who triggered the workflow | | `branch` | string | No | Filter by branch name | | `event` | string | No | Filter by event type \(e.g., push, pull_request, workflow_dispatch\) | diff --git a/apps/docs/content/docs/integrations/incidentio.mdx b/apps/docs/content/docs/integrations/incidentio.mdx index ce3d999f927..ad2412852e5 100644 --- a/apps/docs/content/docs/integrations/incidentio.mdx +++ b/apps/docs/content/docs/integrations/incidentio.mdx @@ -62,7 +62,6 @@ List incidents from incident.io. Returns a list of incidents with their details | ↳ `id` | string | Incident ID | | ↳ `name` | string | Incident name/title | | ↳ `summary` | string | Incident summary | -| ↳ `description` | string | Incident description | | ↳ `mode` | string | Incident mode \(standard, retrospective, test\) | | ↳ `call_url` | string | Video call URL | | ↳ `severity` | object | Incident severity | @@ -82,7 +81,7 @@ List incidents from incident.io. Returns a list of incidents with their details | ↳ `is_default` | boolean | Whether this is the default incident type | | ↳ `created_at` | string | When the incident was created \(ISO 8601\) | | ↳ `updated_at` | string | When the incident was last updated \(ISO 8601\) | -| ↳ `incident_url` | string | URL to the incident page | +| ↳ `permalink` | string | Permalink to the incident in incident.io | | ↳ `slack_channel_id` | string | Slack channel ID | | ↳ `slack_channel_name` | string | Slack channel name | | ↳ `visibility` | string | Incident visibility \(public, private\) | @@ -116,7 +115,6 @@ Create a new incident in incident.io. Requires idempotency_key, severity_id, and | ↳ `id` | string | Incident ID | | ↳ `name` | string | Incident name | | ↳ `summary` | string | Brief summary of the incident | -| ↳ `description` | string | Detailed description of the incident | | ↳ `mode` | string | Incident mode \(e.g., standard, retrospective\) | | ↳ `call_url` | string | URL for the incident call/bridge | | ↳ `severity` | object | Severity of the incident | @@ -132,7 +130,7 @@ Create a new incident in incident.io. Requires idempotency_key, severity_id, and | ↳ `name` | string | Type name | | ↳ `created_at` | string | Creation timestamp | | ↳ `updated_at` | string | Last update timestamp | -| ↳ `incident_url` | string | URL to the incident | +| ↳ `permalink` | string | Permalink to the incident in incident.io | | ↳ `slack_channel_id` | string | Associated Slack channel ID | | ↳ `slack_channel_name` | string | Associated Slack channel name | | ↳ `visibility` | string | Incident visibility | @@ -156,10 +154,9 @@ Retrieve detailed information about a specific incident from incident.io by its | ↳ `id` | string | Incident ID | | ↳ `name` | string | Incident name | | ↳ `summary` | string | Brief summary of the incident | -| ↳ `description` | string | Detailed description of the incident | | ↳ `mode` | string | Incident mode \(e.g., standard, retrospective\) | | ↳ `call_url` | string | URL for the incident call/bridge | -| ↳ `permalink` | string | Permanent link to the incident | +| ↳ `permalink` | string | Permalink to the incident in incident.io | | ↳ `severity` | object | Severity of the incident | | ↳ `id` | string | Severity ID | | ↳ `name` | string | Severity name | @@ -173,7 +170,6 @@ Retrieve detailed information about a specific incident from incident.io by its | ↳ `name` | string | Type name | | ↳ `created_at` | string | Creation timestamp | | ↳ `updated_at` | string | Last update timestamp | -| ↳ `incident_url` | string | URL to the incident | | ↳ `slack_channel_id` | string | Associated Slack channel ID | | ↳ `slack_channel_name` | string | Associated Slack channel name | | ↳ `visibility` | string | Incident visibility | @@ -205,7 +201,6 @@ Update an existing incident in incident.io. Can update name, summary, severity, | ↳ `id` | string | Incident ID | | ↳ `name` | string | Incident name | | ↳ `summary` | string | Brief summary of the incident | -| ↳ `description` | string | Detailed description of the incident | | ↳ `mode` | string | Incident mode \(e.g., standard, retrospective\) | | ↳ `call_url` | string | URL for the incident call/bridge | | ↳ `severity` | object | Severity of the incident | @@ -221,7 +216,7 @@ Update an existing incident in incident.io. Can update name, summary, severity, | ↳ `name` | string | Type name | | ↳ `created_at` | string | Creation timestamp | | ↳ `updated_at` | string | Last update timestamp | -| ↳ `incident_url` | string | URL to the incident | +| ↳ `permalink` | string | Permalink to the incident in incident.io | | ↳ `slack_channel_id` | string | Associated Slack channel ID | | ↳ `slack_channel_name` | string | Associated Slack channel name | | ↳ `visibility` | string | Incident visibility | @@ -486,7 +481,7 @@ Create a new workflow in incident.io. | `trigger` | string | No | Trigger type for the workflow \(e.g., "incident.updated", "incident.created"\) | | `steps` | string | No | Array of workflow steps as JSON string. Example: \[\{"label": "Notify team", "name": "slack.post_message"\}\] | | `condition_groups` | string | No | Array of condition groups as JSON string to control when the workflow runs. Example: \[\{"conditions": \[\{"operation": "one_of", "param_bindings": \[\], "subject": "incident.severity"\}\]\}\] | -| `runs_on_incidents` | string | No | When to run the workflow: "newly_created" \(only new incidents\), "newly_created_and_active" \(new and active incidents\), "active" \(only active incidents\), or "all" \(all incidents\) | +| `runs_on_incidents` | string | No | When to run the workflow: "newly_created" \(only newly created incidents\) or "newly_created_and_active" \(newly created and already active incidents\) | | `runs_on_incident_modes` | string | No | Array of incident modes to run on as JSON string. Example: \["standard", "retrospective"\] | | `include_private_incidents` | boolean | No | Whether to include private incidents | | `continue_on_step_error` | boolean | No | Whether to continue executing subsequent steps if a step fails | @@ -569,7 +564,7 @@ Update an existing workflow in incident.io. | `name` | string | Yes | New name for the workflow \(e.g., "Notify on Critical Incidents"\) | | `steps` | string | Yes | Complete array of workflow steps as a JSON string | | `condition_groups` | string | Yes | Complete array of workflow condition groups as a JSON string | -| `runs_on_incidents` | string | Yes | When to run the workflow: newly_created, newly_created_and_active, active, or all | +| `runs_on_incidents` | string | Yes | When to run the workflow: newly_created or newly_created_and_active | | `runs_on_incident_modes` | string | Yes | Complete array of incident modes to run on as a JSON string | | `include_private_incidents` | boolean | Yes | Whether to include private incidents | | `continue_on_step_error` | boolean | Yes | Whether to continue executing subsequent steps if a step fails | @@ -783,11 +778,15 @@ List all escalation policies in incident.io | Parameter | Type | Description | | --------- | ---- | ----------- | -| `escalations` | array | List of escalation policies | -| ↳ `id` | string | The escalation policy ID | -| ↳ `name` | string | The escalation policy name | -| ↳ `created_at` | string | When the escalation policy was created | -| ↳ `updated_at` | string | When the escalation policy was last updated | +| `escalations` | array | List of escalations | +| ↳ `id` | string | The escalation ID | +| ↳ `title` | string | The escalation title | +| ↳ `status` | string | The current escalation status | +| ↳ `description` | string | Additional detail provided with this escalation | +| ↳ `priority` | object | The escalation priority | +| ↳ `name` | string | Priority name | +| ↳ `created_at` | string | When the escalation was created | +| ↳ `updated_at` | string | When the escalation was last updated | | `pagination_meta` | object | Pagination metadata | | ↳ `after` | string | Cursor for next page | | ↳ `page_size` | number | Number of results per page | @@ -810,11 +809,15 @@ Create a new escalation policy in incident.io | Parameter | Type | Description | | --------- | ---- | ----------- | -| `escalation` | object | The created escalation policy | -| ↳ `id` | string | The escalation policy ID | -| ↳ `name` | string | The escalation policy name | -| ↳ `created_at` | string | When the escalation policy was created | -| ↳ `updated_at` | string | When the escalation policy was last updated | +| `escalation` | object | The created escalation | +| ↳ `id` | string | The escalation ID | +| ↳ `title` | string | The escalation title | +| ↳ `status` | string | The current escalation status | +| ↳ `description` | string | Additional detail provided with this escalation | +| ↳ `priority` | object | The escalation priority | +| ↳ `name` | string | Priority name | +| ↳ `created_at` | string | When the escalation was created | +| ↳ `updated_at` | string | When the escalation was last updated | ### Show Escalation @@ -831,11 +834,15 @@ Get details of a specific escalation policy in incident.io | Parameter | Type | Description | | --------- | ---- | ----------- | -| `escalation` | object | The escalation policy details | -| ↳ `id` | string | The escalation policy ID | -| ↳ `name` | string | The escalation policy name | -| ↳ `created_at` | string | When the escalation policy was created | -| ↳ `updated_at` | string | When the escalation policy was last updated | +| `escalation` | object | The escalation details | +| ↳ `id` | string | The escalation ID | +| ↳ `title` | string | The escalation title | +| ↳ `status` | string | The current escalation status | +| ↳ `description` | string | Additional detail provided with this escalation | +| ↳ `priority` | object | The escalation priority | +| ↳ `name` | string | Priority name | +| ↳ `created_at` | string | When the escalation was created | +| ↳ `updated_at` | string | When the escalation was last updated | ### incident.io Custom Fields List @@ -870,7 +877,7 @@ Create a new custom field in incident.io. | `apiKey` | string | Yes | incident.io API Key | | `name` | string | Yes | Name of the custom field \(e.g., "Affected Service"\) | | `description` | string | Yes | Description of the custom field \(required\) | -| `field_type` | string | Yes | Type of the custom field \(e.g., text, single_select, multi_select, numeric, datetime, link, user, team\) | +| `field_type` | string | Yes | Type of the custom field: text, link, numeric, single_select, or multi_select | #### Output @@ -1200,20 +1207,30 @@ List all updates for a specific incident in incident.io | ↳ `id` | string | The update ID | | ↳ `incident_id` | string | The incident ID | | ↳ `message` | string | The update message | +| ↳ `merged_into_incident_id` | string | ID of the incident this incident was merged into | | ↳ `new_severity` | object | New severity if changed | | ↳ `id` | string | Severity ID | | ↳ `name` | string | Severity name | | ↳ `rank` | number | Severity rank | -| ↳ `new_status` | object | New status if changed | +| ↳ `new_incident_status` | object | The incident status after this update | | ↳ `id` | string | Status ID | | ↳ `name` | string | Status name | | ↳ `category` | string | Status category | -| ↳ `updater` | object | User who created the update | -| ↳ `id` | string | User ID | -| ↳ `name` | string | User name | -| ↳ `email` | string | User email | +| ↳ `updater` | object | Actor who created the update | +| ↳ `user` | object | Set when a user made the update | +| ↳ `id` | string | User ID | +| ↳ `name` | string | User name | +| ↳ `email` | string | User email | +| ↳ `api_key` | object | Set when an API key made the update | +| ↳ `id` | string | API key ID | +| ↳ `name` | string | API key name | +| ↳ `workflow` | object | Set when a workflow made the update | +| ↳ `id` | string | Workflow ID | +| ↳ `name` | string | Workflow name | +| ↳ `alert` | object | Set when an alert made the update | +| ↳ `id` | string | Alert ID | +| ↳ `title` | string | Alert title | | ↳ `created_at` | string | When the update was created | -| ↳ `updated_at` | string | When the update was last modified | | `pagination_meta` | object | Pagination information | | ↳ `after` | string | Cursor for next page | | ↳ `page_size` | number | Number of results per page | diff --git a/apps/docs/content/docs/integrations/meta.json b/apps/docs/content/docs/integrations/meta.json index ae31c81a3c8..382216cef33 100644 --- a/apps/docs/content/docs/integrations/meta.json +++ b/apps/docs/content/docs/integrations/meta.json @@ -219,6 +219,7 @@ "rocketlane", "rootly", "s3", + "sailpoint", "salesforce", "salesforce-service-account", "sap_concur", diff --git a/apps/docs/content/docs/integrations/okta.mdx b/apps/docs/content/docs/integrations/okta.mdx index 5cf8c8e7be7..73224f40da0 100644 --- a/apps/docs/content/docs/integrations/okta.mdx +++ b/apps/docs/content/docs/integrations/okta.mdx @@ -300,7 +300,7 @@ Permanently delete a user from your Okta organization. Can only be performed on | --------- | ---- | -------- | ----------- | | `apiKey` | string | Yes | Okta API token for authentication | | `domain` | string | Yes | Okta domain \(e.g., dev-123456.okta.com\) | -| `userId` | string | Yes | User ID to delete | +| `userId` | string | Yes | User ID or login \(email\) to delete | | `sendEmail` | boolean | No | Send deactivation email to admin \(default: false\) | #### Output @@ -690,7 +690,7 @@ List the MFA factors a user has enrolled, with each factor type, provider, and e | --------- | ---- | -------- | ----------- | | `apiKey` | string | Yes | Okta API token for authentication | | `domain` | string | Yes | Okta domain \(e.g., dev-123456.okta.com\) | -| `userId` | string | Yes | User ID or login to list enrolled factors for | +| `userId` | string | Yes | Okta user ID \(not a login or email\) to list enrolled factors for | #### Output @@ -718,7 +718,7 @@ Retrieve a single enrolled MFA factor for a user, including its type, provider, | --------- | ---- | -------- | ----------- | | `apiKey` | string | Yes | Okta API token for authentication | | `domain` | string | Yes | Okta domain \(e.g., dev-123456.okta.com\) | -| `userId` | string | Yes | User ID or login the factor belongs to | +| `userId` | string | Yes | Okta user ID \(not a login or email\) the factor belongs to | | `factorId` | string | Yes | Factor ID to look up | #### Output @@ -745,7 +745,7 @@ Enroll an MFA factor for a user. The profile fields required depend on the facto | --------- | ---- | -------- | ----------- | | `apiKey` | string | Yes | Okta API token for authentication | | `domain` | string | Yes | Okta domain \(e.g., dev-123456.okta.com\) | -| `userId` | string | Yes | User ID or login to enroll the factor for | +| `userId` | string | Yes | Okta user ID \(not a login or email\) to enroll the factor for | | `factorType` | string | Yes | Factor type to enroll \(sms, call, email, question, push, token:software:totp, u2f, webauthn\) | | `provider` | string | Yes | Factor provider \(OKTA, GOOGLE, FIDO, DUO, RSA, SYMANTEC, YUBICO, CUSTOM\). Each provider supports a subset of factor types | | `phoneNumber` | string | No | Phone number in E.164 format. Required for the sms and call factor types | @@ -779,7 +779,7 @@ Unenroll one specific MFA factor for a user so they can re-enroll it. Destructiv | --------- | ---- | -------- | ----------- | | `apiKey` | string | Yes | Okta API token for authentication | | `domain` | string | Yes | Okta domain \(e.g., dev-123456.okta.com\) | -| `userId` | string | Yes | User ID or login the factor belongs to | +| `userId` | string | Yes | Okta user ID \(not a login or email\) the factor belongs to | | `factorId` | string | Yes | Factor ID to unenroll | | `removeRecoveryEnrollment` | boolean | No | Also remove the phone number as a recovery method, not only as a factor. Applies to sms and call factors only \(default: false\) | @@ -822,7 +822,7 @@ Revoke every active Okta session for a user, signing them out of all devices imm | --------- | ---- | -------- | ----------- | | `apiKey` | string | Yes | Okta API token for authentication | | `domain` | string | Yes | Okta domain \(e.g., dev-123456.okta.com\) | -| `userId` | string | Yes | User ID or login whose sessions will be revoked | +| `userId` | string | Yes | Okta user ID \(not a login or email\) whose sessions will be revoked | | `oauthTokens` | boolean | No | Also revoke the user OpenID Connect and OAuth refresh and access tokens \(default: false\) | | `forgetDevices` | boolean | No | Clear the user remembered factors for all devices \(default: true\) | @@ -1136,7 +1136,7 @@ List the administrator roles assigned to a user. Returns both standard roles and | --------- | ---- | -------- | ----------- | | `apiKey` | string | Yes | Okta API token for authentication | | `domain` | string | Yes | Okta domain \(e.g., dev-123456.okta.com\) | -| `userId` | string | Yes | User ID or login to list admin roles for | +| `userId` | string | Yes | Okta user ID \(not a login or email\) to list admin roles for | #### Output @@ -1165,7 +1165,7 @@ Grant a user an administrator role. Use a standard role type such as USER_ADMIN | --------- | ---- | -------- | ----------- | | `apiKey` | string | Yes | Okta API token for authentication | | `domain` | string | Yes | Okta domain \(e.g., dev-123456.okta.com\) | -| `userId` | string | Yes | User ID or login to assign the admin role to | +| `userId` | string | Yes | Okta user ID \(not a login or email\) to assign the admin role to | | `roleType` | string | Yes | Role type to assign: SUPER_ADMIN, ORG_ADMIN, APP_ADMIN, USER_ADMIN, HELP_DESK_ADMIN, READ_ONLY_ADMIN, API_ACCESS_MANAGEMENT_ADMIN, GROUP_MEMBERSHIP_ADMIN, REPORT_ADMIN, WORKFLOWS_ADMIN, ACCESS_CERTIFICATIONS_ADMIN, ACCESS_REQUESTS_ADMIN, or CUSTOM | | `customRoleId` | string | No | Custom role ID. Required when the role type is CUSTOM | | `resourceSetId` | string | No | Resource set ID the custom role applies to. Required when the role type is CUSTOM | @@ -1197,7 +1197,7 @@ Revoke an administrator role from a user. Destructive: the user immediately lose | --------- | ---- | -------- | ----------- | | `apiKey` | string | Yes | Okta API token for authentication | | `domain` | string | Yes | Okta domain \(e.g., dev-123456.okta.com\) | -| `userId` | string | Yes | User ID or login to revoke the admin role from | +| `userId` | string | Yes | Okta user ID \(not a login or email\) to revoke the admin role from | | `roleAssignmentId` | string | Yes | Role assignment ID to revoke, as returned by List User Roles. For a custom role this is the resource set binding ID | #### Output diff --git a/apps/docs/content/docs/integrations/sailpoint.mdx b/apps/docs/content/docs/integrations/sailpoint.mdx new file mode 100644 index 00000000000..332e4c9d30e --- /dev/null +++ b/apps/docs/content/docs/integrations/sailpoint.mdx @@ -0,0 +1,1785 @@ +--- +title: SailPoint +description: Govern identities and access in SailPoint Identity Security Cloud +--- + +import { BlockInfoCard } from "@/components/ui/block-info-card" + + + +{/* MANUAL-CONTENT-START:intro */} +## MANUAL DESCRIPTION + +The SailPoint integration connects Sim workflows to SailPoint Identity Security Cloud (ISC) with a Personal Access Token (PAT). Enter the tenant name from your ISC URL—or the full `*.api.identitynow.com` / `*.api.identitynowgov.com` host—plus the PAT client ID and client secret. Sim exchanges those credentials at the tenant's `/oauth/token` endpoint and calls SailPoint's current service-versioned endpoints, such as `/identities/v1`, `/access-requests/v1`, and `/certifications/v1`. There is no global API-version setting. + +Create the PAT with the least-privileged scopes required by the actions in your workflow. Common read scopes include `sp:search:read`, `idn:identity:read`, `idn:accounts:read`, `idn:entitlement:read`, `idn:role-unchecked:read` or `idn:role-checked:read`, `idn:access-profile:read`, `idn:sources:read`, `idn:campaign:read`, `idn:access-request-status:read`, `idn:access-request-config:read`, `idn:task-management:read`, and `idn:access-request-approvals:read`. SailPoint lists `idn:access-request:manage` and `idn:access-request-self:manage` for access-request submission, `idn:access-request:create` for account-selection discovery, `idn:access-request:manage` for cancellation, `idn:campaign:manage` for certification decisions and sign-off, `idn:access-request-approvals:manage` for approval actions, `idn:sources:manage` for account import, and `idn:entitlement:manage` for entitlement import and entitlement request configuration. Some identity-governance endpoints require a user-context PAT and an appropriate SailPoint user authority in addition to an OAuth scope; scopes never grant authority beyond the PAT owner's ISC permissions. + +List actions return one bounded page. Standard collections accept up to 250 records per call; role collections accept up to 50; Search accepts up to 10,000. Use `offset`, `sorters`, or Search's `searchAfter` cursor to continue. Enable `count` only when you need the provider's `X-Total-Count` header. Omitting Search `indices` searches every index allowed by SailPoint; complex Search request fields are available as structured JSON inputs. + +Access requests are asynchronous. A successful submission returns SailPoint's `newRequests` and `existingRequests` tracking records, including the access-request IDs needed by the status tools. The standard request form applies the same requested items to every identity; use `requestedForWithRequestedItems` when identities need different items, dates, forms, or account selections. Use **Get Account Selections** before a machine grant/modify or a human multi-account request, then copy the returned source/account selection into **Request Access**. Account-selection discovery accepts at most 25 flat requested items. An entitlement revoke is limited to one entitlement per request, while entitlement grants are limited to 25 entitlements and 10 identities. Sim also caps other request recipient/item arrays at 250 to keep execution payloads bounded. + +Use **Get Access Request Config** to inspect the tenant's request-on-behalf-of and machine-identity settings. Use **Get Entitlement Request Config** to inspect one entitlement's grant, revocation, duration, approval, and form requirements before constructing a request. These configuration reads help a workflow avoid offering a request shape the tenant or entitlement does not permit. + +Account and entitlement imports upload a CSV to a source and return a task that can be followed with **Get Task Status**. Sim caps each uploaded CSV at 25 MiB and does not automatically poll the task. The file must be available to the workflow owner, and the source must support the corresponding import operation. + +The 40 actions cover six connected workflows: search and entity lookup; account, entitlement, role, access-profile, and source inventory; access-request configuration and account-selection discovery; access request submission, cancellation, approval, rejection, and status; campaign and certification review, decision, and sign-off; and CSV import plus task monitoring. Provider-defined objects such as account attributes and Search documents remain JSON because their fields depend on the tenant, source, index, and field projection. +{/* MANUAL-CONTENT-END */} + + +## Usage Instructions + +Read and act on identity-governance data in SailPoint Identity Security Cloud (ISC) with a Personal Access Token (PAT) exchanged through OAuth2 client credentials at https://TENANT.api.identitynow.com/oauth/token. SailPoint versions each service independently, so the integration uses current service paths such as /search/v1, /identities/v1, and /access-requests/v1; there is no shared annual API-version setting. Use a PAT whose owner has the ISC user level required by each endpoint because many identity, role, access-profile, certification, approval, and access-request operations require user context in addition to scopes. Common read scopes include sp:search:read, idn:identity:read, idn:accounts:read, idn:entitlement:read, idn:role-unchecked:read or idn:role-checked:read, idn:access-profile:read, idn:sources:read, idn:campaign:read, idn:access-request-status:read, idn:access-request-config:read, idn:task-management:read, and idn:access-request-approvals:read. Mutations additionally use idn:sources:manage for account aggregation, idn:entitlement:manage for entitlement aggregation and entitlement request configuration, idn:campaign:manage for certification decisions and sign-off, the access-request scopes listed by SailPoint for request submission, idn:access-request:create for account-selection discovery, and idn:access-request-approvals:manage for approval actions. A scope alone does not grant authority beyond the PAT owner's ISC permissions, and authorization failures may be returned as provider errors or filtered visibility depending on the endpoint and tenant policy. + + + +## Actions + +### SailPoint Approve Access Request + +Approve one pending access-request approval. + +#### Input + +| Parameter | Type | Required | Description | +| --------- | ---- | -------- | ----------- | +| `clientId` | string | Yes | SailPoint Personal Access Token client ID | +| `clientSecret` | string | Yes | SailPoint Personal Access Token client secret | +| `tenant` | string | Yes | SailPoint tenant name or full *.api.identitynow.com / *.api.identitynowgov.com host | +| `approvalId` | string | Yes | Approval ID | +| `comment` | string | No | Optional reviewer comment | + +#### Output + +| Parameter | Type | Description | +| --------- | ---- | ----------- | +| `accepted` | boolean | Whether SailPoint accepted the asynchronous action | +| `status` | number | Provider response status \(normally 202\) | + +### SailPoint Cancel Access Request + +Cancel an access request that has not passed approval. + +#### Input + +| Parameter | Type | Required | Description | +| --------- | ---- | -------- | ----------- | +| `clientId` | string | Yes | SailPoint Personal Access Token client ID | +| `clientSecret` | string | Yes | SailPoint Personal Access Token client secret | +| `tenant` | string | Yes | SailPoint tenant name or full *.api.identitynow.com / *.api.identitynowgov.com host | +| `accountActivityId` | string | Yes | Account activity / identity request ID | +| `comment` | string | Yes | Cancellation reason | + +#### Output + +| Parameter | Type | Description | +| --------- | ---- | ----------- | +| `accepted` | boolean | Whether SailPoint accepted the asynchronous action | +| `status` | number | Provider response status \(normally 202\) | + +### SailPoint Decide Certification Review Items + +Approve or revoke 1-250 review items in an identity certification. + +#### Input + +| Parameter | Type | Required | Description | +| --------- | ---- | -------- | ----------- | +| `clientId` | string | Yes | SailPoint Personal Access Token client ID | +| `clientSecret` | string | Yes | SailPoint Personal Access Token client secret | +| `tenant` | string | Yes | SailPoint tenant name or full *.api.identitynow.com / *.api.identitynowgov.com host | +| `id` | string | Yes | Certification ID | +| `decisions` | array | Yes | Array of \{id, decision: APPROVE\|REVOKE, bulk, proposedEndDate?, recommendation?, comments?\} | + +#### Output + +| Parameter | Type | Description | +| --------- | ---- | ----------- | +| `certification` | object | Updated identity certification | +| ↳ `id` | string | Certification ID | +| ↳ `name` | string | Certification name | +| ↳ `campaign` | json | Campaign reference | +| ↳ `completed` | boolean | Whether all decisions are complete | +| ↳ `identitiesCompleted` | number | Identities fully reviewed | +| ↳ `identitiesTotal` | number | Total identities | +| ↳ `created` | string | Creation timestamp | +| ↳ `modified` | string | Last modification timestamp | +| ↳ `decisionsMade` | number | Decisions made | +| ↳ `decisionsTotal` | number | Total decisions | +| ↳ `due` | string | Certification due timestamp | +| ↳ `signed` | string | Sign-off timestamp | +| ↳ `reviewer` | json | Reviewer reference | +| ↳ `reassignment` | json | Reassignment details | +| ↳ `hasErrors` | boolean | Whether the certification has errors | +| ↳ `errorMessage` | string | Certification error message | +| ↳ `phase` | string | Certification phase | + +### SailPoint Get Access Profile + +Get an access profile by ID. + +#### Input + +| Parameter | Type | Required | Description | +| --------- | ---- | -------- | ----------- | +| `clientId` | string | Yes | SailPoint Personal Access Token client ID | +| `clientSecret` | string | Yes | SailPoint Personal Access Token client secret | +| `tenant` | string | Yes | SailPoint tenant name or full *.api.identitynow.com / *.api.identitynowgov.com host | +| `id` | string | Yes | Access profile ID | + +#### Output + +| Parameter | Type | Description | +| --------- | ---- | ----------- | +| `accessProfile` | object | SailPoint access profile | +| ↳ `id` | string | Access profile ID | +| ↳ `name` | string | Access profile name | +| ↳ `description` | string | Access profile description | +| ↳ `created` | string | Creation timestamp | +| ↳ `modified` | string | Last modification timestamp | +| ↳ `enabled` | boolean | Whether the access profile is enabled | +| ↳ `owner` | json | Primary owner reference | +| ↳ `source` | json | Source reference | +| ↳ `entitlements` | array | Entitlement references | +| ↳ `requestable` | boolean | Whether the access profile is requestable | +| ↳ `accessRequestConfig` | json | Access-request configuration | +| ↳ `revocationRequestConfig` | json | Revocation-request configuration | +| ↳ `segments` | array | Segment IDs | +| ↳ `accessModelMetadata` | json | Access-model metadata | +| ↳ `provisioningCriteria` | json | Multi-account provisioning criteria | +| ↳ `additionalOwners` | array | Additional owner references | + +### SailPoint Get Access Profile Entitlements + +List entitlements in one access profile. + +#### Input + +| Parameter | Type | Required | Description | +| --------- | ---- | -------- | ----------- | +| `clientId` | string | Yes | SailPoint Personal Access Token client ID | +| `clientSecret` | string | Yes | SailPoint Personal Access Token client secret | +| `tenant` | string | Yes | SailPoint tenant name or full *.api.identitynow.com / *.api.identitynowgov.com host | +| `id` | string | Yes | Access profile ID | +| `filters` | string | No | SailPoint standard collection filter expression for this operation | +| `sorters` | string | No | Comma-separated supported sort fields, prefixed with - for descending order | +| `limit` | number | No | Maximum records for this page \(0-250; default 250\) | +| `offset` | number | No | Zero-based record offset \(default 0\) | +| `count` | boolean | No | Return the total matching count in X-Total-Count \(default false\) | + +#### Output + +| Parameter | Type | Description | +| --------- | ---- | ----------- | +| `items` | array | Entitlements in this access profile | +| ↳ `id` | string | Entitlement ID | +| ↳ `name` | string | Entitlement name | +| ↳ `attribute` | string | Source entitlement attribute | +| ↳ `value` | string | Source entitlement value | +| ↳ `sourceSchemaObjectType` | string | Source schema object type | +| ↳ `description` | string | Entitlement description | +| ↳ `privileged` | boolean | Whether the entitlement is privileged | +| ↳ `cloudGoverned` | boolean | Whether SailPoint governs the entitlement | +| ↳ `requestable` | boolean | Whether the entitlement is requestable | +| ↳ `owner` | object | Primary owner reference | +| ↳ `id` | string | Identity ID | +| ↳ `type` | string | IDENTITY | +| ↳ `name` | string | Identity display name | +| ↳ `additionalOwners` | array | Additional owner references | +| ↳ `type` | string | IDENTITY or GOVERNANCE_GROUP | +| ↳ `id` | string | Identity or governance-group ID | +| ↳ `name` | string | Display name | +| ↳ `manuallyUpdatedFields` | json | Fields manually updated in SailPoint | +| ↳ `accessModelMetadata` | object | Access-model metadata | +| ↳ `attributes` | array | Access-model metadata attributes | +| ↳ `key` | string | Metadata type identifier | +| ↳ `name` | string | Metadata type display name | +| ↳ `multiselect` | boolean | Whether the metadata accepts multiple values | +| ↳ `status` | string | Metadata item status | +| ↳ `type` | string | Metadata item type | +| ↳ `objectTypes` | array | Applicable object types | +| ↳ `description` | string | Metadata item description | +| ↳ `values` | array | Metadata values | +| ↳ `value` | string | Metadata value | +| ↳ `name` | string | Metadata value display name | +| ↳ `status` | string | Metadata value status | +| ↳ `created` | string | Creation timestamp | +| ↳ `modified` | string | Last modification timestamp | +| ↳ `source` | object | Source reference | +| ↳ `id` | string | Source ID | +| ↳ `type` | string | SOURCE | +| ↳ `name` | string | Source name | +| ↳ `attributes` | json | Source-defined entitlement attributes | +| ↳ `segments` | array | Segment IDs | +| ↳ `directPermissions` | array | Direct permissions | +| ↳ `rights` | array | Rights granted on the target | +| ↳ `target` | string | Permission target | +| `count` | number | Number of records returned in this page | +| `totalCount` | number | Total matching records when count=true | + +### SailPoint Get Access Request Config + +Get tenant access-request, request-on-behalf-of, and machine-identity configuration. + +#### Input + +| Parameter | Type | Required | Description | +| --------- | ---- | -------- | ----------- | +| `clientId` | string | Yes | SailPoint Personal Access Token client ID | +| `clientSecret` | string | Yes | SailPoint Personal Access Token client secret | +| `tenant` | string | Yes | SailPoint tenant name or full *.api.identitynow.com / *.api.identitynowgov.com host | + +#### Output + +| Parameter | Type | Description | +| --------- | ---- | ----------- | +| `accessRequestConfig` | object | Tenant access-request configuration | +| ↳ `approvalsMustBeExternal` | boolean | Whether approvals must be handled externally | +| ↳ `reauthorizationEnabled` | boolean | Whether reauthorization is enabled | +| ↳ `requestOnBehalfOfConfig` | object | Request-on-behalf-of policy | +| ↳ `allowRequestOnBehalfOfAnyoneByAnyone` | boolean | Whether anyone may request for anyone | +| ↳ `allowRequestOnBehalfOfEmployeeByManager` | boolean | Whether managers may request for their employees | +| ↳ `allowRequestOnBehalfOfForMachineIdentity` | boolean | Whether anyone may request for a machine identity | +| ↳ `allowRequestForMachineByOwner` | boolean | Whether machine owners may request for their machines | +| ↳ `entitlementRequestConfig` | object | Tenant entitlement request configuration | +| ↳ `accessRequestConfig` | object | Entitlement grant request configuration | +| ↳ `approvalSchemes` | array | Ordered approval schemes | +| ↳ `approverType` | string | ENTITLEMENT_OWNER, SOURCE_OWNER, MANAGER, GOVERNANCE_GROUP, or WORKFLOW | +| ↳ `approverId` | string | Governance group or workflow approver ID | +| ↳ `requestCommentRequired` | boolean | Whether a request comment is required | +| ↳ `denialCommentRequired` | boolean | Whether a denial comment is required | +| ↳ `reauthorizationRequired` | boolean | Whether reauthorization is required | +| ↳ `requireEndDate` | boolean | Whether an end date is required | +| ↳ `maxPermittedAccessDuration` | object | Maximum permitted access duration | +| ↳ `value` | number | Duration value | +| ↳ `timeUnit` | string | HOURS, DAYS, WEEKS, or MONTHS | +| ↳ `formDefinitionId` | string | Request form definition ID | +| ↳ `revocationRequestConfig` | object | Entitlement revocation request configuration | +| ↳ `approvalSchemes` | array | Ordered revocation approval schemes | +| ↳ `approverType` | string | ENTITLEMENT_OWNER, SOURCE_OWNER, MANAGER, GOVERNANCE_GROUP, or WORKFLOW | +| ↳ `approverId` | string | Governance group or workflow approver ID | +| ↳ `govGroupVisibilityEnabled` | boolean | Whether governance group visibility is enabled | +| ↳ `machineIdentityAccessRequestEnabled` | boolean | Whether machine identity access requests are enabled | + +### SailPoint Get Access Request Status + +List requested-item status records for access requests. + +#### Input + +| Parameter | Type | Required | Description | +| --------- | ---- | -------- | ----------- | +| `clientId` | string | Yes | SailPoint Personal Access Token client ID | +| `clientSecret` | string | Yes | SailPoint Personal Access Token client secret | +| `tenant` | string | Yes | SailPoint tenant name or full *.api.identitynow.com / *.api.identitynowgov.com host | +| `requestedFor` | string | No | Identity ID for whom the access was requested | +| `requestedBy` | string | No | Identity ID that submitted the access request | +| `regardingIdentity` | string | No | Identity ID that is either the requester or the request target | +| `assignedTo` | string | No | Identity ID assigned to the access-request work item | +| `requestState` | string | No | EXECUTING | +| `filters` | string | No | SailPoint standard collection filter expression for this operation | +| `sorters` | string | No | Comma-separated supported sort fields, prefixed with - for descending order | +| `limit` | number | No | Maximum records for this page \(0-250; default 250\) | +| `offset` | number | No | Zero-based record offset \(default 0\) | +| `count` | boolean | No | Return the total matching count in X-Total-Count \(default false\) | + +#### Output + +| Parameter | Type | Description | +| --------- | ---- | ----------- | +| `items` | array | Requested item status records in this page | +| ↳ `id` | string | Requested item status ID | +| ↳ `name` | string | Requested item name | +| ↳ `type` | string | Requested item type | +| ↳ `cancelledRequestDetails` | json | Cancellation details | +| ↳ `errorMessages` | array | Localized request errors | +| ↳ `state` | string | Request state | +| ↳ `approvalDetails` | array | Approval details | +| ↳ `approvalIds` | array | Approval IDs | +| ↳ `manualWorkItemDetails` | array | Manual provisioning work items | +| ↳ `accountActivityItemId` | string | Account activity item ID | +| ↳ `requestType` | string | Access request type | +| ↳ `modified` | string | Last modification timestamp | +| ↳ `created` | string | Creation timestamp | +| ↳ `requester` | json | Requester reference | +| ↳ `requestedFor` | json | Requested-for identity reference | +| ↳ `identityType` | string | HUMAN or MACHINE | +| ↳ `requesterComment` | json | Requester comment | +| ↳ `sodViolationContext` | json | Separation-of-duties violation context | +| ↳ `provisioningDetails` | json | Provisioning details | +| ↳ `preApprovalTriggerDetails` | json | Pre-approval trigger details | +| ↳ `accessRequestPhases` | array | Request lifecycle phases | +| ↳ `description` | string | Requested object description | +| ↳ `startDate` | string | Requested start date | +| ↳ `removeDate` | string | Requested removal date | +| ↳ `cancelable` | boolean | Whether the request can be cancelled | +| ↳ `accessRequestId` | string | Access request ID | +| ↳ `clientMetadata` | json | Caller-provided string metadata | +| ↳ `requestedAccounts` | array | Selected account references | +| ↳ `privilegeLevel` | string | Requested object privilege level | +| ↳ `jitDetails` | array | Just-in-time access details | +| ↳ `form` | json | Completed request form | +| `count` | number | Number of records returned in this page | +| `totalCount` | number | Total matching records when count=true | + +### SailPoint Get Account + +Get an account from the current /accounts/v1 service by ID. + +#### Input + +| Parameter | Type | Required | Description | +| --------- | ---- | -------- | ----------- | +| `clientId` | string | Yes | SailPoint Personal Access Token client ID | +| `clientSecret` | string | Yes | SailPoint Personal Access Token client secret | +| `tenant` | string | Yes | SailPoint tenant name or full *.api.identitynow.com / *.api.identitynowgov.com host | +| `id` | string | Yes | Account ID | + +#### Output + +| Parameter | Type | Description | +| --------- | ---- | ----------- | +| `account` | object | SailPoint account | +| ↳ `id` | string | Account ID | +| ↳ `name` | string | Account name | +| ↳ `created` | string | Creation timestamp | +| ↳ `modified` | string | Last modification timestamp | +| ↳ `sourceId` | string | Source ID | +| ↳ `sourceName` | string | Source name | +| ↳ `identityId` | string | Correlated identity ID | +| ↳ `cloudLifecycleState` | string | Cloud lifecycle state | +| ↳ `identityState` | string | Identity state | +| ↳ `connectionType` | string | Source connection type | +| ↳ `isMachine` | boolean | Whether this is a machine account | +| ↳ `recommendation` | json | Correlation recommendation | +| ↳ `attributes` | json | Source-defined account attributes | +| ↳ `authoritative` | boolean | Whether the account is authoritative | +| ↳ `description` | string | Account description | +| ↳ `disabled` | boolean | Whether the account is disabled | +| ↳ `locked` | boolean | Whether the account is locked | +| ↳ `nativeIdentity` | string | Native account identifier | +| ↳ `systemAccount` | boolean | Whether this is a system account | +| ↳ `uncorrelated` | boolean | Whether the account is uncorrelated | +| ↳ `uuid` | string | Account UUID | +| ↳ `manuallyCorrelated` | boolean | Whether the account was manually correlated | +| ↳ `hasEntitlements` | boolean | Whether the account has entitlements | +| ↳ `identity` | json | Correlated identity reference | +| ↳ `sourceOwner` | json | Source owner reference | +| ↳ `features` | string | Account features | +| ↳ `origin` | string | Account origin | +| ↳ `ownerIdentity` | json | Owner identity reference | + +### SailPoint Get Account Activity + +Get an account activity by ID. + +#### Input + +| Parameter | Type | Required | Description | +| --------- | ---- | -------- | ----------- | +| `clientId` | string | Yes | SailPoint Personal Access Token client ID | +| `clientSecret` | string | Yes | SailPoint Personal Access Token client secret | +| `tenant` | string | Yes | SailPoint tenant name or full *.api.identitynow.com / *.api.identitynowgov.com host | +| `id` | string | Yes | Account activity ID | + +#### Output + +| Parameter | Type | Description | +| --------- | ---- | ----------- | +| `accountActivity` | object | SailPoint account activity | +| ↳ `id` | string | Account activity ID | +| ↳ `name` | string | Account activity name | +| ↳ `created` | string | Creation timestamp | +| ↳ `modified` | string | Last modification timestamp | +| ↳ `completed` | string | Completion timestamp | +| ↳ `completionStatus` | string | Completion status | +| ↳ `type` | string | Activity type | +| ↳ `requesterIdentitySummary` | json | Requester identity summary | +| ↳ `targetIdentitySummary` | json | Target identity summary | +| ↳ `errors` | array | Provisioning errors | +| ↳ `warnings` | array | Provisioning warnings | +| ↳ `items` | array | Account activity items | +| ↳ `executionStatus` | string | Execution status | +| ↳ `clientMetadata` | json | Caller-provided string metadata | + +### SailPoint Get Account Entitlements + +List entitlements granted to one account. + +#### Input + +| Parameter | Type | Required | Description | +| --------- | ---- | -------- | ----------- | +| `clientId` | string | Yes | SailPoint Personal Access Token client ID | +| `clientSecret` | string | Yes | SailPoint Personal Access Token client secret | +| `tenant` | string | Yes | SailPoint tenant name or full *.api.identitynow.com / *.api.identitynowgov.com host | +| `id` | string | Yes | Account ID | +| `limit` | number | No | Maximum records for this page \(0-250; default 250\) | +| `offset` | number | No | Zero-based record offset \(default 0\) | +| `count` | boolean | No | Return the total matching count in X-Total-Count \(default false\) | + +#### Output + +| Parameter | Type | Description | +| --------- | ---- | ----------- | +| `items` | array | Entitlements on this account | +| ↳ `id` | string | Entitlement ID | +| ↳ `name` | string | Entitlement name | +| ↳ `attribute` | string | Source entitlement attribute | +| ↳ `value` | string | Source entitlement value | +| ↳ `sourceSchemaObjectType` | string | Source schema object type | +| ↳ `description` | string | Entitlement description | +| ↳ `privileged` | boolean | Whether the entitlement is privileged | +| ↳ `cloudGoverned` | boolean | Whether SailPoint governs the entitlement | +| ↳ `requestable` | boolean | Whether the entitlement is requestable | +| ↳ `owner` | object | Primary owner reference | +| ↳ `id` | string | Identity ID | +| ↳ `type` | string | IDENTITY | +| ↳ `name` | string | Identity display name | +| ↳ `additionalOwners` | array | Additional owner references | +| ↳ `type` | string | IDENTITY or GOVERNANCE_GROUP | +| ↳ `id` | string | Identity or governance-group ID | +| ↳ `name` | string | Display name | +| ↳ `manuallyUpdatedFields` | json | Fields manually updated in SailPoint | +| ↳ `accessModelMetadata` | object | Access-model metadata | +| ↳ `attributes` | array | Access-model metadata attributes | +| ↳ `key` | string | Metadata type identifier | +| ↳ `name` | string | Metadata type display name | +| ↳ `multiselect` | boolean | Whether the metadata accepts multiple values | +| ↳ `status` | string | Metadata item status | +| ↳ `type` | string | Metadata item type | +| ↳ `objectTypes` | array | Applicable object types | +| ↳ `description` | string | Metadata item description | +| ↳ `values` | array | Metadata values | +| ↳ `value` | string | Metadata value | +| ↳ `name` | string | Metadata value display name | +| ↳ `status` | string | Metadata value status | +| ↳ `created` | string | Creation timestamp | +| ↳ `modified` | string | Last modification timestamp | +| ↳ `source` | object | Source reference | +| ↳ `id` | string | Source ID | +| ↳ `type` | string | SOURCE | +| ↳ `name` | string | Source name | +| ↳ `attributes` | json | Source-defined entitlement attributes | +| ↳ `segments` | array | Segment IDs | +| ↳ `directPermissions` | array | Direct permissions | +| ↳ `rights` | array | Rights granted on the target | +| ↳ `target` | string | Permission target | +| `count` | number | Number of records returned in this page | +| `totalCount` | number | Total matching records when count=true | + +### SailPoint Get Account Selections + +Resolve eligible source accounts before submitting a machine or multi-account access request. + +#### Input + +| Parameter | Type | Required | Description | +| --------- | ---- | -------- | ----------- | +| `clientId` | string | Yes | SailPoint Personal Access Token client ID | +| `clientSecret` | string | Yes | SailPoint Personal Access Token client secret | +| `tenant` | string | Yes | SailPoint tenant name or full *.api.identitynow.com / *.api.identitynowgov.com host | +| `requestType` | string | No | GRANT_ACCESS \(default\), REVOKE_ACCESS, or MODIFY_ACCESS | +| `requestedFor` | array | No | Human identity IDs for the flat request shape | +| `requestedItems` | array | No | Flat human request items | +| `requestedForWithRequestedItems` | array | No | Per-identity request items for account selection and all machine identity requests | +| `clientMetadata` | json | No | Arbitrary string-to-string metadata returned by related APIs | + +#### Output + +| Parameter | Type | Description | +| --------- | ---- | ----------- | +| `accountSelections` | object | Eligible account selections grouped by identity and requested item | +| ↳ `identities` | array | Identity-specific eligible account selections | +| ↳ `requestedItems` | array | Requested items and their eligible accounts | +| ↳ `description` | string | Requested item description | +| ↳ `accountsSelectionBlocked` | boolean | Whether account selection is blocked | +| ↳ `accountsSelectionBlockedReason` | string | Provider reason account selection is blocked | +| ↳ `type` | string | ACCESS_PROFILE, ROLE, or ENTITLEMENT | +| ↳ `id` | string | Requested item ID | +| ↳ `name` | string | Requested item name | +| ↳ `sources` | array | Sources and eligible accounts for this item | +| ↳ `type` | string | SOURCE or provider reference type | +| ↳ `id` | string | Source ID | +| ↳ `name` | string | Source name | +| ↳ `accounts` | array | Eligible accounts on this source | +| ↳ `uuid` | string | Account UUID | +| ↳ `nativeIdentity` | string | Native account identifier | +| ↳ `type` | string | ACCOUNT or provider reference type | +| ↳ `id` | string | Account reference ID | +| ↳ `name` | string | Account name | +| ↳ `accountsSelectionRequired` | boolean | Whether this identity requires account selection | +| ↳ `type` | string | IDENTITY, MACHINE_IDENTITY, or provider reference type | +| ↳ `id` | string | Identity ID | +| ↳ `name` | string | Identity name | + +### SailPoint Get Campaign + +Get a certification campaign by ID. + +#### Input + +| Parameter | Type | Required | Description | +| --------- | ---- | -------- | ----------- | +| `clientId` | string | Yes | SailPoint Personal Access Token client ID | +| `clientSecret` | string | Yes | SailPoint Personal Access Token client secret | +| `tenant` | string | Yes | SailPoint tenant name or full *.api.identitynow.com / *.api.identitynowgov.com host | +| `id` | string | Yes | Campaign ID | +| `detail` | string | No | SLIM or FULL | + +#### Output + +| Parameter | Type | Description | +| --------- | ---- | ----------- | +| `campaign` | object | SailPoint certification campaign | +| ↳ `id` | string | Campaign ID | +| ↳ `name` | string | Campaign name | +| ↳ `description` | string | Campaign description | +| ↳ `deadline` | string | Campaign deadline | +| ↳ `type` | string | Campaign type | +| ↳ `status` | string | Campaign status | +| ↳ `correlatedStatus` | string | Campaign correlation status | +| ↳ `mandatoryCommentRequirement` | string | Decision comment requirement | +| ↳ `created` | string | Creation timestamp | +| ↳ `modified` | string | Last modification timestamp | +| ↳ `recommendationsEnabled` | boolean | Whether recommendations are enabled | +| ↳ `emailNotificationEnabled` | boolean | Whether email notifications are enabled | +| ↳ `autoRevokeAllowed` | boolean | Whether automatic revocation is allowed | +| ↳ `totalCertifications` | number | Total certifications | +| ↳ `completedCertifications` | number | Completed certifications | +| ↳ `alerts` | array | Campaign alerts | +| ↳ `filter` | json | Campaign filter reference | +| ↳ `sunsetCommentsRequired` | boolean | Whether sunset-date changes require comments | +| ↳ `sourceOwnerCampaignInfo` | json | Source-owner campaign configuration | +| ↳ `searchCampaignInfo` | json | Search campaign configuration | +| ↳ `roleCompositionCampaignInfo` | json | Role-composition campaign configuration | +| ↳ `machineAccountCampaignInfo` | json | Machine-account campaign configuration | +| ↳ `sourcesWithOrphanEntitlements` | array | Sources containing orphan entitlements | + +### SailPoint Get Certification + +Get an identity certification by ID. + +#### Input + +| Parameter | Type | Required | Description | +| --------- | ---- | -------- | ----------- | +| `clientId` | string | Yes | SailPoint Personal Access Token client ID | +| `clientSecret` | string | Yes | SailPoint Personal Access Token client secret | +| `tenant` | string | Yes | SailPoint tenant name or full *.api.identitynow.com / *.api.identitynowgov.com host | +| `id` | string | Yes | Certification ID | + +#### Output + +| Parameter | Type | Description | +| --------- | ---- | ----------- | +| `certification` | object | SailPoint identity certification | +| ↳ `id` | string | Certification ID | +| ↳ `name` | string | Certification name | +| ↳ `campaign` | json | Campaign reference | +| ↳ `completed` | boolean | Whether all decisions are complete | +| ↳ `identitiesCompleted` | number | Identities fully reviewed | +| ↳ `identitiesTotal` | number | Total identities | +| ↳ `created` | string | Creation timestamp | +| ↳ `modified` | string | Last modification timestamp | +| ↳ `decisionsMade` | number | Decisions made | +| ↳ `decisionsTotal` | number | Total decisions | +| ↳ `due` | string | Certification due timestamp | +| ↳ `signed` | string | Sign-off timestamp | +| ↳ `reviewer` | json | Reviewer reference | +| ↳ `reassignment` | json | Reassignment details | +| ↳ `hasErrors` | boolean | Whether the certification has errors | +| ↳ `errorMessage` | string | Certification error message | +| ↳ `phase` | string | Certification phase | + +### SailPoint Get Entitlement + +Get an entitlement by ID. + +#### Input + +| Parameter | Type | Required | Description | +| --------- | ---- | -------- | ----------- | +| `clientId` | string | Yes | SailPoint Personal Access Token client ID | +| `clientSecret` | string | Yes | SailPoint Personal Access Token client secret | +| `tenant` | string | Yes | SailPoint tenant name or full *.api.identitynow.com / *.api.identitynowgov.com host | +| `id` | string | Yes | Entitlement ID | + +#### Output + +| Parameter | Type | Description | +| --------- | ---- | ----------- | +| `entitlement` | object | SailPoint entitlement | +| ↳ `id` | string | Entitlement ID | +| ↳ `name` | string | Entitlement name | +| ↳ `attribute` | string | Source entitlement attribute | +| ↳ `value` | string | Source entitlement value | +| ↳ `sourceSchemaObjectType` | string | Source schema object type | +| ↳ `description` | string | Entitlement description | +| ↳ `privilegeLevel` | object | Privilege-level details | +| ↳ `direct` | string | Direct privilege level assigned to the entitlement | +| ↳ `setBy` | string | User or process that set the privilege level | +| ↳ `setByType` | string | Method by which the privilege level was set | +| ↳ `inherited` | string | Inherited privilege level on the entitlement | +| ↳ `effective` | string | Effective privilege level assigned to the entitlement | +| ↳ `tags` | array | Entitlement tags | +| ↳ `cloudGoverned` | boolean | Whether SailPoint governs the entitlement | +| ↳ `requestable` | boolean | Whether the entitlement is requestable | +| ↳ `owner` | object | Primary owner reference | +| ↳ `id` | string | Identity ID | +| ↳ `type` | string | IDENTITY | +| ↳ `name` | string | Identity display name | +| ↳ `manuallyUpdatedFields` | json | Fields manually updated in SailPoint | +| ↳ `accessModelMetadata` | object | Access-model metadata | +| ↳ `attributes` | array | Access-model metadata attributes | +| ↳ `key` | string | Metadata type identifier | +| ↳ `name` | string | Metadata type display name | +| ↳ `multiselect` | boolean | Whether the metadata accepts multiple values | +| ↳ `status` | string | Metadata item status | +| ↳ `type` | string | Metadata item type | +| ↳ `objectTypes` | array | Applicable object types | +| ↳ `description` | string | Metadata item description | +| ↳ `values` | array | Metadata values | +| ↳ `value` | string | Metadata value | +| ↳ `name` | string | Metadata value display name | +| ↳ `status` | string | Metadata value status | +| ↳ `created` | string | Creation timestamp | +| ↳ `modified` | string | Last modification timestamp | +| ↳ `source` | object | Source reference | +| ↳ `id` | string | Source ID | +| ↳ `type` | string | SOURCE | +| ↳ `name` | string | Source name | +| ↳ `attributes` | json | Source-defined entitlement attributes | +| ↳ `segments` | array | Segment IDs | +| ↳ `directPermissions` | array | Direct permissions | +| ↳ `rights` | array | Rights granted on the target | +| ↳ `target` | string | Permission target | + +### SailPoint Get Entitlement Request Config + +Get grant, revocation, duration, approval, and form settings for an entitlement. + +#### Input + +| Parameter | Type | Required | Description | +| --------- | ---- | -------- | ----------- | +| `clientId` | string | Yes | SailPoint Personal Access Token client ID | +| `clientSecret` | string | Yes | SailPoint Personal Access Token client secret | +| `tenant` | string | Yes | SailPoint tenant name or full *.api.identitynow.com / *.api.identitynowgov.com host | +| `id` | string | Yes | Entitlement ID | + +#### Output + +| Parameter | Type | Description | +| --------- | ---- | ----------- | +| `entitlementRequestConfig` | object | Entitlement request configuration | +| ↳ `accessRequestConfig` | object | Entitlement grant request configuration | +| ↳ `approvalSchemes` | array | Ordered approval schemes | +| ↳ `approverType` | string | ENTITLEMENT_OWNER, SOURCE_OWNER, MANAGER, GOVERNANCE_GROUP, or WORKFLOW | +| ↳ `approverId` | string | Governance group or workflow approver ID | +| ↳ `requestCommentRequired` | boolean | Whether a request comment is required | +| ↳ `denialCommentRequired` | boolean | Whether a denial comment is required | +| ↳ `reauthorizationRequired` | boolean | Whether reauthorization is required | +| ↳ `requireEndDate` | boolean | Whether an end date is required | +| ↳ `maxPermittedAccessDuration` | object | Maximum permitted access duration | +| ↳ `value` | number | Duration value | +| ↳ `timeUnit` | string | HOURS, DAYS, WEEKS, or MONTHS | +| ↳ `formDefinitionId` | string | Request form definition ID | +| ↳ `revocationRequestConfig` | object | Entitlement revocation request configuration | +| ↳ `approvalSchemes` | array | Ordered revocation approval schemes | +| ↳ `approverType` | string | ENTITLEMENT_OWNER, SOURCE_OWNER, MANAGER, GOVERNANCE_GROUP, or WORKFLOW | +| ↳ `approverId` | string | Governance group or workflow approver ID | + +### SailPoint Get Identity + +Get an identity from the current /identities/v1 service by ID. + +#### Input + +| Parameter | Type | Required | Description | +| --------- | ---- | -------- | ----------- | +| `clientId` | string | Yes | SailPoint Personal Access Token client ID | +| `clientSecret` | string | Yes | SailPoint Personal Access Token client secret | +| `tenant` | string | Yes | SailPoint tenant name or full *.api.identitynow.com / *.api.identitynowgov.com host | +| `id` | string | Yes | Identity ID | + +#### Output + +| Parameter | Type | Description | +| --------- | ---- | ----------- | +| `identity` | object | SailPoint identity | +| ↳ `id` | string | Identity ID | +| ↳ `name` | string | Identity name | +| ↳ `created` | string | Creation timestamp | +| ↳ `modified` | string | Last modification timestamp | +| ↳ `alias` | string | Identity alias | +| ↳ `emailAddress` | string | Identity email address | +| ↳ `processingState` | string | Identity processing state | +| ↳ `identityStatus` | string | Identity status | +| ↳ `managerRef` | json | Manager reference | +| ↳ `isManager` | boolean | Whether the identity manages other identities | +| ↳ `lastRefresh` | string | Last identity refresh timestamp | +| ↳ `attributes` | json | Tenant-defined identity attributes | +| ↳ `lifecycleState` | json | Lifecycle-state reference | + +### SailPoint Get Role + +Get a role by ID. + +#### Input + +| Parameter | Type | Required | Description | +| --------- | ---- | -------- | ----------- | +| `clientId` | string | Yes | SailPoint Personal Access Token client ID | +| `clientSecret` | string | Yes | SailPoint Personal Access Token client secret | +| `tenant` | string | Yes | SailPoint tenant name or full *.api.identitynow.com / *.api.identitynowgov.com host | +| `id` | string | Yes | Role ID | + +#### Output + +| Parameter | Type | Description | +| --------- | ---- | ----------- | +| `role` | object | SailPoint role | +| ↳ `id` | string | Role ID | +| ↳ `name` | string | Role name | +| ↳ `created` | string | Creation timestamp | +| ↳ `modified` | string | Last modification timestamp | +| ↳ `description` | string | Role description | +| ↳ `owner` | json | Primary owner reference | +| ↳ `additionalOwners` | array | Additional owner references | +| ↳ `accessProfiles` | array | Access profile references | +| ↳ `entitlements` | array | Entitlement references | +| ↳ `membership` | json | Role membership selector | +| ↳ `legacyMembershipInfo` | json | Legacy membership information | +| ↳ `enabled` | boolean | Whether the role is enabled | +| ↳ `requestable` | boolean | Whether the role is requestable | +| ↳ `accessRequestConfig` | json | Access-request configuration | +| ↳ `revocationRequestConfig` | json | Revocation-request configuration | +| ↳ `segments` | array | Segment IDs | +| ↳ `dimensional` | boolean | Whether the role is dimensional | +| ↳ `dimensionRefs` | array | Dimension references | +| ↳ `accessModelMetadata` | json | Access-model metadata | +| ↳ `privilegeLevel` | string | Role privilege level | + +### SailPoint Get Role Entitlements + +List entitlements in one role using the current non-experimental roles service. + +#### Input + +| Parameter | Type | Required | Description | +| --------- | ---- | -------- | ----------- | +| `clientId` | string | Yes | SailPoint Personal Access Token client ID | +| `clientSecret` | string | Yes | SailPoint Personal Access Token client secret | +| `tenant` | string | Yes | SailPoint tenant name or full *.api.identitynow.com / *.api.identitynowgov.com host | +| `id` | string | Yes | Role ID | +| `filters` | string | No | SailPoint standard collection filter expression for this operation | +| `sorters` | string | No | Comma-separated supported sort fields, prefixed with - for descending order | +| `limit` | number | No | Maximum roles for this page \(0-50; default 50\) | +| `offset` | number | No | Zero-based record offset \(default 0\) | +| `count` | boolean | No | Return the total matching count in X-Total-Count \(default false\) | + +#### Output + +| Parameter | Type | Description | +| --------- | ---- | ----------- | +| `items` | array | Entitlements in this role | +| ↳ `id` | string | Entitlement ID | +| ↳ `name` | string | Entitlement name | +| ↳ `attribute` | string | Source entitlement attribute | +| ↳ `value` | string | Source entitlement value | +| ↳ `sourceSchemaObjectType` | string | Source schema object type | +| ↳ `description` | string | Entitlement description | +| ↳ `privileged` | boolean | Whether the entitlement is privileged | +| ↳ `cloudGoverned` | boolean | Whether SailPoint governs the entitlement | +| ↳ `requestable` | boolean | Whether the entitlement is requestable | +| ↳ `owner` | object | Primary owner reference | +| ↳ `id` | string | Identity ID | +| ↳ `type` | string | IDENTITY | +| ↳ `name` | string | Identity display name | +| ↳ `additionalOwners` | array | Additional owner references | +| ↳ `type` | string | IDENTITY or GOVERNANCE_GROUP | +| ↳ `id` | string | Identity or governance-group ID | +| ↳ `name` | string | Display name | +| ↳ `manuallyUpdatedFields` | json | Fields manually updated in SailPoint | +| ↳ `accessModelMetadata` | object | Access-model metadata | +| ↳ `attributes` | array | Access-model metadata attributes | +| ↳ `key` | string | Metadata type identifier | +| ↳ `name` | string | Metadata type display name | +| ↳ `multiselect` | boolean | Whether the metadata accepts multiple values | +| ↳ `status` | string | Metadata item status | +| ↳ `type` | string | Metadata item type | +| ↳ `objectTypes` | array | Applicable object types | +| ↳ `description` | string | Metadata item description | +| ↳ `values` | array | Metadata values | +| ↳ `value` | string | Metadata value | +| ↳ `name` | string | Metadata value display name | +| ↳ `status` | string | Metadata value status | +| ↳ `created` | string | Creation timestamp | +| ↳ `modified` | string | Last modification timestamp | +| ↳ `source` | object | Source reference | +| ↳ `id` | string | Source ID | +| ↳ `type` | string | SOURCE | +| ↳ `name` | string | Source name | +| ↳ `attributes` | json | Source-defined entitlement attributes | +| ↳ `segments` | array | Segment IDs | +| ↳ `directPermissions` | array | Direct permissions | +| ↳ `rights` | array | Rights granted on the target | +| ↳ `target` | string | Permission target | +| `count` | number | Number of records returned in this page | +| `totalCount` | number | Total matching records when count=true | + +### SailPoint Get Source + +Get an identity source by ID. + +#### Input + +| Parameter | Type | Required | Description | +| --------- | ---- | -------- | ----------- | +| `clientId` | string | Yes | SailPoint Personal Access Token client ID | +| `clientSecret` | string | Yes | SailPoint Personal Access Token client secret | +| `tenant` | string | Yes | SailPoint tenant name or full *.api.identitynow.com / *.api.identitynowgov.com host | +| `id` | string | Yes | Source ID | + +#### Output + +| Parameter | Type | Description | +| --------- | ---- | ----------- | +| `source` | object | SailPoint identity source | +| ↳ `id` | string | Source ID | +| ↳ `name` | string | Source name | +| ↳ `description` | string | Source description | +| ↳ `owner` | json | Source owner reference | +| ↳ `cluster` | json | Virtual appliance cluster reference | +| ↳ `accountCorrelationConfig` | json | Account correlation configuration | +| ↳ `accountCorrelationRule` | json | Account correlation rule reference | +| ↳ `managerCorrelationMapping` | json | Manager correlation mapping | +| ↳ `managerCorrelationRule` | json | Manager correlation rule reference | +| ↳ `beforeProvisioningRule` | json | Before-provisioning rule reference | +| ↳ `schemas` | array | Source schemas | +| ↳ `passwordPolicies` | array | Password policy references | +| ↳ `features` | array | Source features | +| ↳ `type` | string | Source type | +| ↳ `connector` | string | Connector name | +| ↳ `connectorClass` | string | Connector implementation class | +| ↳ `connectorAttributes` | json | Connector-specific attributes | +| ↳ `deleteThreshold` | number | Account deletion threshold | +| ↳ `authoritative` | boolean | Whether the source is authoritative | +| ↳ `managementWorkgroup` | json | Management workgroup reference | +| ↳ `healthy` | boolean | Whether the source is healthy | +| ↳ `status` | string | Source status | +| ↳ `since` | string | Status start timestamp | +| ↳ `connectorId` | string | Connector ID | +| ↳ `connectorName` | string | Connector display name | +| ↳ `connectionType` | string | Connection type | +| ↳ `connectorImplementationId` | string | Connector implementation ID | +| ↳ `created` | string | Creation timestamp | +| ↳ `modified` | string | Last modification timestamp | +| ↳ `credentialProviderEnabled` | boolean | Whether a credential provider is enabled | +| ↳ `category` | string | Source category | + +### SailPoint Get Task Status + +Get the current status of a SailPoint background task by ID. + +#### Input + +| Parameter | Type | Required | Description | +| --------- | ---- | -------- | ----------- | +| `clientId` | string | Yes | SailPoint Personal Access Token client ID | +| `clientSecret` | string | Yes | SailPoint Personal Access Token client secret | +| `tenant` | string | Yes | SailPoint tenant name or full *.api.identitynow.com / *.api.identitynowgov.com host | +| `id` | string | Yes | Task ID | + +#### Output + +| Parameter | Type | Description | +| --------- | ---- | ----------- | +| `task` | object | SailPoint task status | +| ↳ `id` | string | Task ID | +| ↳ `type` | string | Task type | +| ↳ `uniqueName` | string | Task unique name | +| ↳ `description` | string | Task description | +| ↳ `parentName` | string | Parent task name | +| ↳ `launcher` | string | Task launcher | +| ↳ `target` | object | Task target | +| ↳ `id` | string | Target ID | +| ↳ `type` | string | APPLICATION or IDENTITY | +| ↳ `name` | string | Target name | +| ↳ `created` | string | Creation timestamp | +| ↳ `modified` | string | Last modification timestamp | +| ↳ `launched` | string | Launch timestamp | +| ↳ `completed` | string | Completion timestamp | +| ↳ `completionStatus` | string | Task completion status | +| ↳ `messages` | array | Task messages | +| ↳ `type` | string | INFO, WARN, or ERROR | +| ↳ `localizedText` | object | Localized task message | +| ↳ `locale` | string | Message locale | +| ↳ `message` | string | Message text | +| ↳ `key` | string | Message key | +| ↳ `parameters` | array | Internationalization parameters | +| ↳ `returns` | array | Task return descriptors | +| ↳ `name` | string | Return value display name | +| ↳ `attributeName` | string | Task attribute name | +| ↳ `attributes` | json | Task-specific attributes | +| ↳ `progress` | string | Human-readable progress | +| ↳ `percentComplete` | number | Completion percentage | +| ↳ `taskDefinitionSummary` | object | Task definition summary | +| ↳ `id` | string | Task-definition ID | +| ↳ `uniqueName` | string | Task-definition unique name | +| ↳ `description` | string | Task-definition description | +| ↳ `parentName` | string | Parent task-definition name | +| ↳ `executor` | string | Task-definition executor | +| ↳ `arguments` | json | Task-definition arguments | + +### SailPoint List Access Profiles + +List access profiles with current visibility and segmentation controls. + +#### Input + +| Parameter | Type | Required | Description | +| --------- | ---- | -------- | ----------- | +| `clientId` | string | Yes | SailPoint Personal Access Token client ID | +| `clientSecret` | string | Yes | SailPoint Personal Access Token client secret | +| `tenant` | string | Yes | SailPoint tenant name or full *.api.identitynow.com / *.api.identitynowgov.com host | +| `forSubadmin` | string | No | Subadmin identity ID or 'me' whose visible resources should be returned | +| `forSegmentIds` | string | No | Comma-separated segment IDs used to restrict the returned resources | +| `includeUnsegmented` | boolean | No | Include resources not assigned to a segment \(default true\) | +| `filters` | string | No | SailPoint standard collection filter expression for this operation | +| `sorters` | string | No | Comma-separated supported sort fields, prefixed with - for descending order | +| `limit` | number | No | Maximum records for this page \(0-250; default 250\) | +| `offset` | number | No | Zero-based record offset \(default 0\) | +| `count` | boolean | No | Return the total matching count in X-Total-Count \(default false\) | + +#### Output + +| Parameter | Type | Description | +| --------- | ---- | ----------- | +| `items` | array | Access profiles in this page | +| ↳ `id` | string | Access profile ID | +| ↳ `name` | string | Access profile name | +| ↳ `description` | string | Access profile description | +| ↳ `created` | string | Creation timestamp | +| ↳ `modified` | string | Last modification timestamp | +| ↳ `enabled` | boolean | Whether the access profile is enabled | +| ↳ `owner` | json | Primary owner reference | +| ↳ `source` | json | Source reference | +| ↳ `entitlements` | array | Entitlement references | +| ↳ `requestable` | boolean | Whether the access profile is requestable | +| ↳ `accessRequestConfig` | json | Access-request configuration | +| ↳ `revocationRequestConfig` | json | Revocation-request configuration | +| ↳ `segments` | array | Segment IDs | +| ↳ `accessModelMetadata` | json | Access-model metadata | +| ↳ `provisioningCriteria` | json | Multi-account provisioning criteria | +| ↳ `additionalOwners` | array | Additional owner references | +| `count` | number | Number of records returned in this page | +| `totalCount` | number | Total matching records when count=true | + +### SailPoint List Account Activities + +List provisioning activities with identity, filter, sort, and page controls. + +#### Input + +| Parameter | Type | Required | Description | +| --------- | ---- | -------- | ----------- | +| `clientId` | string | Yes | SailPoint Personal Access Token client ID | +| `clientSecret` | string | Yes | SailPoint Personal Access Token client secret | +| `tenant` | string | Yes | SailPoint tenant name or full *.api.identitynow.com / *.api.identitynowgov.com host | +| `requestedFor` | string | No | Target identity ID or 'me'; mutually exclusive with regardingIdentity | +| `requestedBy` | string | No | Requester identity ID or 'me'; mutually exclusive with regardingIdentity | +| `regardingIdentity` | string | No | Requester-or-target identity ID or 'me'; excludes requestedFor/requestedBy | +| `filters` | string | No | SailPoint standard collection filter expression for this operation | +| `sorters` | string | No | Comma-separated supported sort fields, prefixed with - for descending order | +| `limit` | number | No | Maximum records for this page \(0-250; default 250\) | +| `offset` | number | No | Zero-based record offset \(default 0\) | +| `count` | boolean | No | Return the total matching count in X-Total-Count \(default false\) | + +#### Output + +| Parameter | Type | Description | +| --------- | ---- | ----------- | +| `items` | array | Account activities in this page | +| ↳ `id` | string | Account activity ID | +| ↳ `name` | string | Account activity name | +| ↳ `created` | string | Creation timestamp | +| ↳ `modified` | string | Last modification timestamp | +| ↳ `completed` | string | Completion timestamp | +| ↳ `completionStatus` | string | Completion status | +| ↳ `type` | string | Activity type | +| ↳ `requesterIdentitySummary` | json | Requester identity summary | +| ↳ `targetIdentitySummary` | json | Target identity summary | +| ↳ `errors` | array | Provisioning errors | +| ↳ `warnings` | array | Provisioning warnings | +| ↳ `items` | array | Account activity items | +| ↳ `executionStatus` | string | Execution status | +| ↳ `clientMetadata` | json | Caller-provided string metadata | +| `count` | number | Number of records returned in this page | +| `totalCount` | number | Total matching records when count=true | + +### SailPoint List Accounts + +List accounts with documented filtering, sorting, detail, and pagination. + +#### Input + +| Parameter | Type | Required | Description | +| --------- | ---- | -------- | ----------- | +| `clientId` | string | Yes | SailPoint Personal Access Token client ID | +| `clientSecret` | string | Yes | SailPoint Personal Access Token client secret | +| `tenant` | string | Yes | SailPoint tenant name or full *.api.identitynow.com / *.api.identitynowgov.com host | +| `filters` | string | No | SailPoint standard collection filter expression for this operation | +| `sorters` | string | No | Comma-separated supported sort fields, prefixed with - for descending order | +| `detailLevel` | string | No | SLIM or FULL \(default FULL\) | +| `limit` | number | No | Maximum records for this page \(0-250; default 250\) | +| `offset` | number | No | Zero-based record offset \(default 0\) | +| `count` | boolean | No | Return the total matching count in X-Total-Count \(default false\) | + +#### Output + +| Parameter | Type | Description | +| --------- | ---- | ----------- | +| `items` | array | Accounts in this page | +| ↳ `id` | string | Account ID | +| ↳ `name` | string | Account name | +| ↳ `created` | string | Creation timestamp | +| ↳ `modified` | string | Last modification timestamp | +| ↳ `sourceId` | string | Source ID | +| ↳ `sourceName` | string | Source name | +| ↳ `identityId` | string | Correlated identity ID | +| ↳ `cloudLifecycleState` | string | Cloud lifecycle state | +| ↳ `identityState` | string | Identity state | +| ↳ `connectionType` | string | Source connection type | +| ↳ `isMachine` | boolean | Whether this is a machine account | +| ↳ `recommendation` | json | Correlation recommendation | +| ↳ `attributes` | json | Source-defined account attributes | +| ↳ `authoritative` | boolean | Whether the account is authoritative | +| ↳ `description` | string | Account description | +| ↳ `disabled` | boolean | Whether the account is disabled | +| ↳ `locked` | boolean | Whether the account is locked | +| ↳ `nativeIdentity` | string | Native account identifier | +| ↳ `systemAccount` | boolean | Whether this is a system account | +| ↳ `uncorrelated` | boolean | Whether the account is uncorrelated | +| ↳ `uuid` | string | Account UUID | +| ↳ `manuallyCorrelated` | boolean | Whether the account was manually correlated | +| ↳ `hasEntitlements` | boolean | Whether the account has entitlements | +| ↳ `identity` | json | Correlated identity reference | +| ↳ `sourceOwner` | json | Source owner reference | +| ↳ `features` | string | Account features | +| ↳ `origin` | string | Account origin | +| ↳ `ownerIdentity` | json | Owner identity reference | +| `count` | number | Number of records returned in this page | +| `totalCount` | number | Total matching records when count=true | + +### SailPoint List Campaigns + +List certification campaigns with detail, filtering, sorting, and pagination. + +#### Input + +| Parameter | Type | Required | Description | +| --------- | ---- | -------- | ----------- | +| `clientId` | string | Yes | SailPoint Personal Access Token client ID | +| `clientSecret` | string | Yes | SailPoint Personal Access Token client secret | +| `tenant` | string | Yes | SailPoint tenant name or full *.api.identitynow.com / *.api.identitynowgov.com host | +| `detail` | string | No | SLIM \(default\) or FULL | +| `filters` | string | No | SailPoint standard collection filter expression for this operation | +| `sorters` | string | No | Comma-separated supported sort fields, prefixed with - for descending order | +| `limit` | number | No | Maximum records for this page \(0-250; default 250\) | +| `offset` | number | No | Zero-based record offset \(default 0\) | +| `count` | boolean | No | Return the total matching count in X-Total-Count \(default false\) | + +#### Output + +| Parameter | Type | Description | +| --------- | ---- | ----------- | +| `items` | array | Certification campaigns in this page | +| ↳ `id` | string | Campaign ID | +| ↳ `name` | string | Campaign name | +| ↳ `description` | string | Campaign description | +| ↳ `deadline` | string | Campaign deadline | +| ↳ `type` | string | Campaign type | +| ↳ `status` | string | Campaign status | +| ↳ `correlatedStatus` | string | Campaign correlation status | +| ↳ `mandatoryCommentRequirement` | string | Decision comment requirement | +| ↳ `created` | string | Creation timestamp | +| ↳ `modified` | string | Last modification timestamp | +| ↳ `recommendationsEnabled` | boolean | Whether recommendations are enabled | +| ↳ `emailNotificationEnabled` | boolean | Whether email notifications are enabled | +| ↳ `autoRevokeAllowed` | boolean | Whether automatic revocation is allowed | +| ↳ `totalCertifications` | number | Total certifications | +| ↳ `completedCertifications` | number | Completed certifications | +| ↳ `alerts` | array | Campaign alerts | +| ↳ `filter` | json | Campaign filter reference | +| ↳ `sunsetCommentsRequired` | boolean | Whether sunset-date changes require comments | +| ↳ `sourceOwnerCampaignInfo` | json | Source-owner campaign configuration | +| ↳ `searchCampaignInfo` | json | Search campaign configuration | +| ↳ `roleCompositionCampaignInfo` | json | Role-composition campaign configuration | +| ↳ `machineAccountCampaignInfo` | json | Machine-account campaign configuration | +| ↳ `sourcesWithOrphanEntitlements` | array | Sources containing orphan entitlements | +| `count` | number | Number of records returned in this page | +| `totalCount` | number | Total matching records when count=true | + +### SailPoint List Certification Review Items + +List access-review items in one identity certification. + +#### Input + +| Parameter | Type | Required | Description | +| --------- | ---- | -------- | ----------- | +| `clientId` | string | Yes | SailPoint Personal Access Token client ID | +| `clientSecret` | string | Yes | SailPoint Personal Access Token client secret | +| `tenant` | string | Yes | SailPoint tenant name or full *.api.identitynow.com / *.api.identitynowgov.com host | +| `id` | string | Yes | Certification ID | +| `filters` | string | No | SailPoint standard collection filter expression for this operation | +| `sorters` | string | No | Comma-separated supported sort fields, prefixed with - for descending order | +| `entitlements` | string | No | Comma-separated entitlement IDs | +| `accessProfiles` | string | No | Comma-separated access profile IDs | +| `roles` | string | No | Comma-separated role IDs | +| `limit` | number | No | Maximum records for this page \(0-250; default 250\) | +| `offset` | number | No | Zero-based record offset \(default 0\) | +| `count` | boolean | No | Return the total matching count in X-Total-Count \(default false\) | + +#### Output + +| Parameter | Type | Description | +| --------- | ---- | ----------- | +| `items` | array | Certification access-review items in this page | +| ↳ `accessSummary` | json | Reviewed access summary | +| ↳ `identitySummary` | json | Reviewed identity summary | +| ↳ `id` | string | Review item ID | +| ↳ `completed` | boolean | Whether review is complete | +| ↳ `newAccess` | boolean | Whether this is newly granted access | +| ↳ `decision` | string | Current certification decision | +| ↳ `comments` | string | Reviewer comments | +| `count` | number | Number of records returned in this page | +| `totalCount` | number | Total matching records when count=true | + +### SailPoint List Certifications + +List identity certifications assigned to a reviewer. + +#### Input + +| Parameter | Type | Required | Description | +| --------- | ---- | -------- | ----------- | +| `clientId` | string | Yes | SailPoint Personal Access Token client ID | +| `clientSecret` | string | Yes | SailPoint Personal Access Token client secret | +| `tenant` | string | Yes | SailPoint tenant name or full *.api.identitynow.com / *.api.identitynowgov.com host | +| `reviewerIdentity` | string | No | Reviewer identity ID or 'me' | +| `filters` | string | No | SailPoint standard collection filter expression for this operation | +| `sorters` | string | No | Comma-separated supported sort fields, prefixed with - for descending order | +| `limit` | number | No | Maximum records for this page \(0-250; default 250\) | +| `offset` | number | No | Zero-based record offset \(default 0\) | +| `count` | boolean | No | Return the total matching count in X-Total-Count \(default false\) | + +#### Output + +| Parameter | Type | Description | +| --------- | ---- | ----------- | +| `items` | array | Identity certifications in this page | +| ↳ `id` | string | Certification ID | +| ↳ `name` | string | Certification name | +| ↳ `campaign` | json | Campaign reference | +| ↳ `completed` | boolean | Whether all decisions are complete | +| ↳ `identitiesCompleted` | number | Identities fully reviewed | +| ↳ `identitiesTotal` | number | Total identities | +| ↳ `created` | string | Creation timestamp | +| ↳ `modified` | string | Last modification timestamp | +| ↳ `decisionsMade` | number | Decisions made | +| ↳ `decisionsTotal` | number | Total decisions | +| ↳ `due` | string | Certification due timestamp | +| ↳ `signed` | string | Sign-off timestamp | +| ↳ `reviewer` | json | Reviewer reference | +| ↳ `reassignment` | json | Reassignment details | +| ↳ `hasErrors` | boolean | Whether the certification has errors | +| ↳ `errorMessage` | string | Certification error message | +| ↳ `phase` | string | Certification phase | +| `count` | number | Number of records returned in this page | +| `totalCount` | number | Total matching records when count=true | + +### SailPoint List Entitlements + +List entitlements with current segmentation, cursor, filter, and page controls. + +#### Input + +| Parameter | Type | Required | Description | +| --------- | ---- | -------- | ----------- | +| `clientId` | string | Yes | SailPoint Personal Access Token client ID | +| `clientSecret` | string | Yes | SailPoint Personal Access Token client secret | +| `tenant` | string | Yes | SailPoint tenant name or full *.api.identitynow.com / *.api.identitynowgov.com host | +| `segmentedForIdentity` | string | No | Identity ID whose visible segments restrict the results | +| `forSegmentIds` | string | No | Comma-separated segment IDs used to restrict the returned resources | +| `includeUnsegmented` | boolean | No | Include resources not assigned to a segment \(default true\) | +| `searchAfter` | string | No | Opaque search-after cursor from the previous entitlement page | +| `filters` | string | No | SailPoint standard collection filter expression for this operation | +| `sorters` | string | No | Comma-separated supported sort fields, prefixed with - for descending order | +| `limit` | number | No | Maximum records for this page \(0-250; default 250\) | +| `offset` | number | No | Zero-based record offset \(default 0\) | +| `count` | boolean | No | Return the total matching count in X-Total-Count \(default false\) | + +#### Output + +| Parameter | Type | Description | +| --------- | ---- | ----------- | +| `items` | array | Entitlements in this page | +| ↳ `id` | string | Entitlement ID | +| ↳ `name` | string | Entitlement name | +| ↳ `attribute` | string | Source entitlement attribute | +| ↳ `value` | string | Source entitlement value | +| ↳ `sourceSchemaObjectType` | string | Source schema object type | +| ↳ `description` | string | Entitlement description | +| ↳ `privilegeLevel` | object | Privilege-level details | +| ↳ `direct` | string | Direct privilege level assigned to the entitlement | +| ↳ `setBy` | string | User or process that set the privilege level | +| ↳ `setByType` | string | Method by which the privilege level was set | +| ↳ `inherited` | string | Inherited privilege level on the entitlement | +| ↳ `effective` | string | Effective privilege level assigned to the entitlement | +| ↳ `tags` | array | Entitlement tags | +| ↳ `cloudGoverned` | boolean | Whether SailPoint governs the entitlement | +| ↳ `requestable` | boolean | Whether the entitlement is requestable | +| ↳ `owner` | object | Primary owner reference | +| ↳ `id` | string | Identity ID | +| ↳ `type` | string | IDENTITY | +| ↳ `name` | string | Identity display name | +| ↳ `manuallyUpdatedFields` | json | Fields manually updated in SailPoint | +| ↳ `accessModelMetadata` | object | Access-model metadata | +| ↳ `attributes` | array | Access-model metadata attributes | +| ↳ `key` | string | Metadata type identifier | +| ↳ `name` | string | Metadata type display name | +| ↳ `multiselect` | boolean | Whether the metadata accepts multiple values | +| ↳ `status` | string | Metadata item status | +| ↳ `type` | string | Metadata item type | +| ↳ `objectTypes` | array | Applicable object types | +| ↳ `description` | string | Metadata item description | +| ↳ `values` | array | Metadata values | +| ↳ `value` | string | Metadata value | +| ↳ `name` | string | Metadata value display name | +| ↳ `status` | string | Metadata value status | +| ↳ `created` | string | Creation timestamp | +| ↳ `modified` | string | Last modification timestamp | +| ↳ `source` | object | Source reference | +| ↳ `id` | string | Source ID | +| ↳ `type` | string | SOURCE | +| ↳ `name` | string | Source name | +| ↳ `attributes` | json | Source-defined entitlement attributes | +| ↳ `segments` | array | Segment IDs | +| ↳ `directPermissions` | array | Direct permissions | +| ↳ `rights` | array | Rights granted on the target | +| ↳ `target` | string | Permission target | +| `count` | number | Number of records returned in this page | +| `totalCount` | number | Total matching records when count=true | + +### SailPoint List Identities + +List identities with documented filtering, sorting, and pagination. + +#### Input + +| Parameter | Type | Required | Description | +| --------- | ---- | -------- | ----------- | +| `clientId` | string | Yes | SailPoint Personal Access Token client ID | +| `clientSecret` | string | Yes | SailPoint Personal Access Token client secret | +| `tenant` | string | Yes | SailPoint tenant name or full *.api.identitynow.com / *.api.identitynowgov.com host | +| `filters` | string | No | SailPoint standard collection filter expression for this operation | +| `sorters` | string | No | Comma-separated supported sort fields, prefixed with - for descending order | +| `defaultFilter` | string | No | CORRELATED_ONLY \(default\) or NONE | +| `limit` | number | No | Maximum records for this page \(0-250; default 250\) | +| `offset` | number | No | Zero-based record offset \(default 0\) | +| `count` | boolean | No | Return the total matching count in X-Total-Count \(default false\) | + +#### Output + +| Parameter | Type | Description | +| --------- | ---- | ----------- | +| `items` | array | Identities in this page | +| ↳ `id` | string | Identity ID | +| ↳ `name` | string | Identity name | +| ↳ `created` | string | Creation timestamp | +| ↳ `modified` | string | Last modification timestamp | +| ↳ `alias` | string | Identity alias | +| ↳ `emailAddress` | string | Identity email address | +| ↳ `processingState` | string | Identity processing state | +| ↳ `identityStatus` | string | Identity status | +| ↳ `managerRef` | json | Manager reference | +| ↳ `isManager` | boolean | Whether the identity manages other identities | +| ↳ `lastRefresh` | string | Last identity refresh timestamp | +| ↳ `attributes` | json | Tenant-defined identity attributes | +| ↳ `lifecycleState` | json | Lifecycle-state reference | +| `count` | number | Number of records returned in this page | +| `totalCount` | number | Total matching records when count=true | + +### SailPoint List Identity Entitlements + +List tagged entitlement references held by one identity. + +#### Input + +| Parameter | Type | Required | Description | +| --------- | ---- | -------- | ----------- | +| `clientId` | string | Yes | SailPoint Personal Access Token client ID | +| `clientSecret` | string | Yes | SailPoint Personal Access Token client secret | +| `tenant` | string | Yes | SailPoint tenant name or full *.api.identitynow.com / *.api.identitynowgov.com host | +| `id` | string | Yes | Identity ID | +| `limit` | number | No | Maximum records for this page \(0-250; default 250\) | +| `offset` | number | No | Zero-based record offset \(default 0\) | +| `count` | boolean | No | Return the total matching count in X-Total-Count \(default false\) | + +#### Output + +| Parameter | Type | Description | +| --------- | ---- | ----------- | +| `items` | array | Entitlements held by this identity | +| ↳ `objectRef` | json | Tagged entitlement reference | +| ↳ `tags` | array | Tags applied to the entitlement | +| `count` | number | Number of records returned in this page | +| `totalCount` | number | Total matching records when count=true | + +### SailPoint List Pending Access Request Approvals + +List pending access-request approvals visible to the caller. + +#### Input + +| Parameter | Type | Required | Description | +| --------- | ---- | -------- | ----------- | +| `clientId` | string | Yes | SailPoint Personal Access Token client ID | +| `clientSecret` | string | Yes | SailPoint Personal Access Token client secret | +| `tenant` | string | Yes | SailPoint tenant name or full *.api.identitynow.com / *.api.identitynowgov.com host | +| `ownerId` | string | No | Approval owner identity ID or 'me'; admins may omit it for all approvals | +| `filters` | string | No | SailPoint standard collection filter expression for this operation | +| `sorters` | string | No | Comma-separated supported sort fields, prefixed with - for descending order | +| `limit` | number | No | Maximum records for this page \(0-250; default 250\) | +| `offset` | number | No | Zero-based record offset \(default 0\) | +| `count` | boolean | No | Return the total matching count in X-Total-Count \(default false\) | + +#### Output + +| Parameter | Type | Description | +| --------- | ---- | ----------- | +| `items` | array | Pending access-request approvals in this page | +| ↳ `id` | string | Approval ID | +| ↳ `accessRequestId` | string | Access request ID | +| ↳ `name` | string | Approval name | +| ↳ `created` | string | Creation timestamp | +| ↳ `modified` | string | Last modification timestamp | +| ↳ `requestCreated` | string | Access-request creation timestamp | +| ↳ `requestType` | string | GRANT_ACCESS, REVOKE_ACCESS, or MODIFY_ACCESS | +| ↳ `identityType` | string | HUMAN or MACHINE | +| ↳ `requester` | json | Requester reference | +| ↳ `requestedFor` | json | Requested-for identity reference | +| ↳ `owner` | json | Access item owner | +| ↳ `requestedObject` | json | Requested access object | +| ↳ `requesterComment` | json | Requester comment | +| ↳ `previousReviewersComments` | array | Previous reviewer comments | +| ↳ `forwardHistory` | array | Approval forwarding history | +| ↳ `commentRequiredWhenRejected` | boolean | Whether rejection requires a comment | +| ↳ `actionInProcess` | string | Asynchronous action in progress | +| ↳ `removeDate` | string | Requested removal date | +| ↳ `removeDateUpdateRequested` | boolean | Whether this request changes the removal date | +| ↳ `currentRemoveDate` | string | Removal date at request time | +| ↳ `startDate` | string | Requested start date | +| ↳ `startUpdateRequested` | boolean | Whether this request changes the start date | +| ↳ `currentStartDate` | string | Start date at request time | +| ↳ `sodViolationContext` | json | Separation-of-duties violation context | +| ↳ `clientMetadata` | json | Caller-provided metadata | +| ↳ `requestedAccounts` | array | Selected account references | +| ↳ `privilegeLevel` | string | Requested object privilege level | +| ↳ `maxPermittedAccessDuration` | json | Maximum allowed access duration | +| ↳ `jitDetails` | array | Just-in-time access details | +| ↳ `form` | json | Completed request form | +| `count` | number | Number of records returned in this page | +| `totalCount` | number | Total matching records when count=true | + +### SailPoint List Roles + +List roles with current visibility, segmentation, filtering, and pagination controls. + +#### Input + +| Parameter | Type | Required | Description | +| --------- | ---- | -------- | ----------- | +| `clientId` | string | Yes | SailPoint Personal Access Token client ID | +| `clientSecret` | string | Yes | SailPoint Personal Access Token client secret | +| `tenant` | string | Yes | SailPoint tenant name or full *.api.identitynow.com / *.api.identitynowgov.com host | +| `forSubadmin` | string | No | Subadmin identity ID or 'me' whose visible resources should be returned | +| `forSegmentIds` | string | No | Comma-separated segment IDs used to restrict the returned resources | +| `includeUnsegmented` | boolean | No | Include resources not assigned to a segment \(default true\) | +| `filters` | string | No | SailPoint standard collection filter expression for this operation | +| `sorters` | string | No | Comma-separated supported sort fields, prefixed with - for descending order | +| `limit` | number | No | Maximum roles for this page \(0-50; default 50\) | +| `offset` | number | No | Zero-based record offset \(default 0\) | +| `count` | boolean | No | Return the total matching count in X-Total-Count \(default false\) | + +#### Output + +| Parameter | Type | Description | +| --------- | ---- | ----------- | +| `items` | array | Roles in this page | +| ↳ `id` | string | Role ID | +| ↳ `name` | string | Role name | +| ↳ `created` | string | Creation timestamp | +| ↳ `modified` | string | Last modification timestamp | +| ↳ `description` | string | Role description | +| ↳ `owner` | json | Primary owner reference | +| ↳ `additionalOwners` | array | Additional owner references | +| ↳ `accessProfiles` | array | Access profile references | +| ↳ `entitlements` | array | Entitlement references | +| ↳ `membership` | json | Role membership selector | +| ↳ `legacyMembershipInfo` | json | Legacy membership information | +| ↳ `enabled` | boolean | Whether the role is enabled | +| ↳ `requestable` | boolean | Whether the role is requestable | +| ↳ `accessRequestConfig` | json | Access-request configuration | +| ↳ `revocationRequestConfig` | json | Revocation-request configuration | +| ↳ `segments` | array | Segment IDs | +| ↳ `dimensional` | boolean | Whether the role is dimensional | +| ↳ `dimensionRefs` | array | Dimension references | +| ↳ `accessModelMetadata` | json | Access-model metadata | +| ↳ `privilegeLevel` | string | Role privilege level | +| `count` | number | Number of records returned in this page | +| `totalCount` | number | Total matching records when count=true | + +### SailPoint List Sources + +List identity sources with visibility, filtering, sorting, and pagination controls. + +#### Input + +| Parameter | Type | Required | Description | +| --------- | ---- | -------- | ----------- | +| `clientId` | string | Yes | SailPoint Personal Access Token client ID | +| `clientSecret` | string | Yes | SailPoint Personal Access Token client secret | +| `tenant` | string | Yes | SailPoint tenant name or full *.api.identitynow.com / *.api.identitynowgov.com host | +| `filters` | string | No | SailPoint standard collection filter expression for this operation | +| `sorters` | string | No | Comma-separated supported sort fields, prefixed with - for descending order | +| `forSubadmin` | string | No | Subadmin identity ID or 'me' whose visible resources should be returned | +| `includeIDNSource` | boolean | No | Include the built-in IdentityNow source \(default false\) | +| `limit` | number | No | Maximum records for this page \(0-250; default 250\) | +| `offset` | number | No | Zero-based record offset \(default 0\) | +| `count` | boolean | No | Return the total matching count in X-Total-Count \(default false\) | + +#### Output + +| Parameter | Type | Description | +| --------- | ---- | ----------- | +| `items` | array | Sources in this page | +| ↳ `id` | string | Source ID | +| ↳ `name` | string | Source name | +| ↳ `description` | string | Source description | +| ↳ `owner` | json | Source owner reference | +| ↳ `cluster` | json | Virtual appliance cluster reference | +| ↳ `accountCorrelationConfig` | json | Account correlation configuration | +| ↳ `accountCorrelationRule` | json | Account correlation rule reference | +| ↳ `managerCorrelationMapping` | json | Manager correlation mapping | +| ↳ `managerCorrelationRule` | json | Manager correlation rule reference | +| ↳ `beforeProvisioningRule` | json | Before-provisioning rule reference | +| ↳ `schemas` | array | Source schemas | +| ↳ `passwordPolicies` | array | Password policy references | +| ↳ `features` | array | Source features | +| ↳ `type` | string | Source type | +| ↳ `connector` | string | Connector name | +| ↳ `connectorClass` | string | Connector implementation class | +| ↳ `connectorAttributes` | json | Connector-specific attributes | +| ↳ `deleteThreshold` | number | Account deletion threshold | +| ↳ `authoritative` | boolean | Whether the source is authoritative | +| ↳ `managementWorkgroup` | json | Management workgroup reference | +| ↳ `healthy` | boolean | Whether the source is healthy | +| ↳ `status` | string | Source status | +| ↳ `since` | string | Status start timestamp | +| ↳ `connectorId` | string | Connector ID | +| ↳ `connectorName` | string | Connector display name | +| ↳ `connectionType` | string | Connection type | +| ↳ `connectorImplementationId` | string | Connector implementation ID | +| ↳ `created` | string | Creation timestamp | +| ↳ `modified` | string | Last modification timestamp | +| ↳ `credentialProviderEnabled` | boolean | Whether a credential provider is enabled | +| ↳ `category` | string | Source category | +| `count` | number | Number of records returned in this page | +| `totalCount` | number | Total matching records when count=true | + +### SailPoint Load Accounts + +Start account aggregation for a source, optionally using a CSV file. + +#### Input + +| Parameter | Type | Required | Description | +| --------- | ---- | -------- | ----------- | +| `clientId` | string | Yes | SailPoint Personal Access Token client ID | +| `clientSecret` | string | Yes | SailPoint Personal Access Token client secret | +| `tenant` | string | Yes | SailPoint tenant name or full *.api.identitynow.com / *.api.identitynowgov.com host | +| `sourceId` | string | Yes | Source ID | +| `file` | file | No | Delimited-file source account CSV | +| `disableOptimization` | boolean | No | Reprocess every account instead of using optimized aggregation | + +#### Output + +| Parameter | Type | Description | +| --------- | ---- | ----------- | +| `success` | boolean | Whether SailPoint successfully created the task | +| `task` | object | Account aggregation task | +| ↳ `id` | string | Task ID | +| ↳ `type` | string | Task type | +| ↳ `name` | string | Task name | +| ↳ `description` | string | Task description | +| ↳ `launcher` | string | Task launcher | +| ↳ `created` | string | Creation timestamp | +| ↳ `launched` | string | Launch timestamp | +| ↳ `completed` | string | Completion timestamp | +| ↳ `completionStatus` | string | Task completion status | +| ↳ `parentName` | string | Parent task name | +| ↳ `messages` | array | Task messages | +| ↳ `type` | string | INFO, WARN, or ERROR | +| ↳ `error` | boolean | Whether the message is an error | +| ↳ `warning` | boolean | Whether the message is a warning | +| ↳ `key` | string | Message key | +| ↳ `localizedText` | string | Localized message text | +| ↳ `progress` | string | Human-readable progress | +| ↳ `attributes` | json | Task-specific attributes | +| ↳ `returns` | array | Task return descriptors | +| ↳ `displayLabel` | string | Return value display label | +| ↳ `attributeName` | string | Task attribute name | + +### SailPoint Load Entitlements + +Start entitlement aggregation for a source, optionally using a CSV file. + +#### Input + +| Parameter | Type | Required | Description | +| --------- | ---- | -------- | ----------- | +| `clientId` | string | Yes | SailPoint Personal Access Token client ID | +| `clientSecret` | string | Yes | SailPoint Personal Access Token client secret | +| `tenant` | string | Yes | SailPoint tenant name or full *.api.identitynow.com / *.api.identitynowgov.com host | +| `sourceId` | string | Yes | Source ID | +| `file` | file | No | Delimited-file source entitlement CSV | + +#### Output + +| Parameter | Type | Description | +| --------- | ---- | ----------- | +| `task` | object | Entitlement aggregation task | +| ↳ `id` | string | Task ID | +| ↳ `type` | string | Task type | +| ↳ `uniqueName` | string | Task unique name | +| ↳ `description` | string | Task description | +| ↳ `launcher` | string | Task launcher | +| ↳ `created` | string | Creation timestamp | +| ↳ `returns` | array | Task return descriptors | +| ↳ `displayLabel` | string | Return value display label | +| ↳ `attributeName` | string | Task attribute name | + +### SailPoint Reject Access Request + +Reject one pending access-request approval with a reviewer comment. + +#### Input + +| Parameter | Type | Required | Description | +| --------- | ---- | -------- | ----------- | +| `clientId` | string | Yes | SailPoint Personal Access Token client ID | +| `clientSecret` | string | Yes | SailPoint Personal Access Token client secret | +| `tenant` | string | Yes | SailPoint tenant name or full *.api.identitynow.com / *.api.identitynowgov.com host | +| `approvalId` | string | Yes | Approval ID | +| `comment` | string | Yes | Reviewer rejection comment | + +#### Output + +| Parameter | Type | Description | +| --------- | ---- | ----------- | +| `accepted` | boolean | Whether SailPoint accepted the asynchronous action | +| `status` | number | Provider response status \(normally 202\) | + +### SailPoint Request Access + +Submit a current human or machine identity access request. + +#### Input + +| Parameter | Type | Required | Description | +| --------- | ---- | -------- | ----------- | +| `clientId` | string | Yes | SailPoint Personal Access Token client ID | +| `clientSecret` | string | Yes | SailPoint Personal Access Token client secret | +| `tenant` | string | Yes | SailPoint tenant name or full *.api.identitynow.com / *.api.identitynowgov.com host | +| `requestType` | string | No | GRANT_ACCESS \(default\), REVOKE_ACCESS, or MODIFY_ACCESS | +| `requestedFor` | array | No | Human identity IDs for the flat request shape | +| `requestedItems` | array | No | Flat human request items | +| `requestedForWithRequestedItems` | array | No | Per-identity request items for account selection and all machine identity requests | +| `clientMetadata` | json | No | Arbitrary string-to-string metadata returned by related APIs | + +#### Output + +| Parameter | Type | Description | +| --------- | ---- | ----------- | +| `accepted` | boolean | Whether SailPoint accepted the asynchronous action | +| `status` | number | Provider response status \(normally 202\) | +| `newRequests` | array | New access request tracking records | +| ↳ `requestedFor` | string | Requested-for identity ID | +| ↳ `requestedItemsDetails` | array | Requested item references | +| ↳ `type` | string | ACCESS_PROFILE, ROLE, or ENTITLEMENT | +| ↳ `id` | string | Requested item ID | +| ↳ `attributesHash` | number | Stable request attributes hash | +| ↳ `accessRequestIds` | array | Access request tracking IDs | +| `existingRequests` | array | Already-existing request tracking records | +| ↳ `requestedFor` | string | Requested-for identity ID | +| ↳ `requestedItemsDetails` | array | Requested item references | +| ↳ `type` | string | ACCESS_PROFILE, ROLE, or ENTITLEMENT | +| ↳ `id` | string | Requested item ID | +| ↳ `attributesHash` | number | Stable request attributes hash | +| ↳ `accessRequestIds` | array | Access request tracking IDs | + +### SailPoint Search + +Search current SailPoint indices with every documented search query mode. + +#### Input + +| Parameter | Type | Required | Description | +| --------- | ---- | -------- | ----------- | +| `clientId` | string | Yes | SailPoint Personal Access Token client ID | +| `clientSecret` | string | Yes | SailPoint Personal Access Token client secret | +| `tenant` | string | Yes | SailPoint tenant name or full *.api.identitynow.com / *.api.identitynowgov.com host | +| `indices` | array | No | Indices to search: accessprofiles, accountactivities, entitlements, events, identities, roles, or *. Omit to search all. | +| `queryType` | string | No | SAILPOINT \(default\), DSL, TEXT, or TYPEAHEAD | +| `queryVersion` | string | No | Elasticsearch query language version \(default 5.2\) | +| `query` | object | No | SAILPOINT query object: \{query?, fields?, timeZone?, innerHit?\} | +| `queryDsl` | json | No | Elasticsearch Query DSL object used with queryType=DSL | +| `textQuery` | object | No | TEXT query object with required terms\[\] and fields\[\] | +| `typeAheadQuery` | object | No | TYPEAHEAD query with query, field, optional nestedType, maxExpansions \(1-1000\), size, sort, and sortByValue | +| `includeNested` | boolean | No | Include nested objects in search results \(default true\) | +| `queryResultFilter` | object | No | Result projection object with includes\[\] and/or excludes\[\] | +| `aggregationType` | string | No | Aggregation query language: DSL \(default\) or SAILPOINT | +| `aggregationsVersion` | string | No | Elasticsearch aggregation language version \(default 5.2\) | +| `aggregationsDsl` | json | No | Dynamic Elasticsearch aggregations DSL object | +| `aggregations` | json | No | Typed SailPoint aggregation specification | +| `sort` | array | No | Ordered search fields; prefix + or - for direction | +| `searchAfter` | array | No | String values from the final sorted record of the previous search page | +| `filters` | json | No | Map of result field names to filter objects | +| `limit` | number | No | Maximum search documents for this page \(0-10,000; default 250\) | +| `offset` | number | No | Zero-based record offset \(default 0\) | +| `count` | boolean | No | Return the total matching count in X-Total-Count \(default false\) | + +#### Output + +| Parameter | Type | Description | +| --------- | ---- | ----------- | +| `results` | array | Index-dependent search documents | +| `count` | number | Documents returned in this page | +| `totalCount` | number | Total matching documents when count=true | + +### SailPoint Search Aggregate + +Run an Elasticsearch DSL or SailPoint aggregation over current search indices. + +#### Input + +| Parameter | Type | Required | Description | +| --------- | ---- | -------- | ----------- | +| `clientId` | string | Yes | SailPoint Personal Access Token client ID | +| `clientSecret` | string | Yes | SailPoint Personal Access Token client secret | +| `tenant` | string | Yes | SailPoint tenant name or full *.api.identitynow.com / *.api.identitynowgov.com host | +| `indices` | array | No | Indices to search: accessprofiles, accountactivities, entitlements, events, identities, roles, or *. Omit to search all. | +| `queryType` | string | No | SAILPOINT \(default\), DSL, TEXT, or TYPEAHEAD | +| `queryVersion` | string | No | Elasticsearch query language version \(default 5.2\) | +| `query` | object | No | SAILPOINT query object: \{query?, fields?, timeZone?, innerHit?\} | +| `queryDsl` | json | No | Elasticsearch Query DSL object used with queryType=DSL | +| `textQuery` | object | No | TEXT query object with required terms\[\] and fields\[\] | +| `typeAheadQuery` | object | No | TYPEAHEAD query with query, field, optional nestedType, maxExpansions \(1-1000\), size, sort, and sortByValue | +| `includeNested` | boolean | No | Include nested objects in search results \(default true\) | +| `queryResultFilter` | object | No | Result projection object with includes\[\] and/or excludes\[\] | +| `aggregationType` | string | No | Aggregation query language: DSL \(default\) or SAILPOINT | +| `aggregationsVersion` | string | No | Elasticsearch aggregation language version \(default 5.2\) | +| `aggregationsDsl` | json | No | Dynamic Elasticsearch aggregations DSL object | +| `aggregations` | json | No | Typed SailPoint aggregation specification | +| `sort` | array | No | Ordered search fields; prefix + or - for direction | +| `searchAfter` | array | No | String values from the final sorted record of the previous search page | +| `filters` | json | No | Map of result field names to filter objects | +| `limit` | number | No | Maximum records for this page \(0-250; default 250\) | +| `offset` | number | No | Zero-based record offset \(default 0\) | +| `count` | boolean | No | Return the total matching count in X-Total-Count \(default false\) | + +#### Output + +| Parameter | Type | Description | +| --------- | ---- | ----------- | +| `aggregations` | json | Dynamic Elasticsearch aggregation result document | +| `hits` | array | Index-dependent aggregation hits | +| `totalCount` | number | Total matching documents when count=true | + +### SailPoint Search Count + +Count documents matching a complete SailPoint search body. + +#### Input + +| Parameter | Type | Required | Description | +| --------- | ---- | -------- | ----------- | +| `clientId` | string | Yes | SailPoint Personal Access Token client ID | +| `clientSecret` | string | Yes | SailPoint Personal Access Token client secret | +| `tenant` | string | Yes | SailPoint tenant name or full *.api.identitynow.com / *.api.identitynowgov.com host | +| `indices` | array | No | Indices to search: accessprofiles, accountactivities, entitlements, events, identities, roles, or *. Omit to search all. | +| `queryType` | string | No | SAILPOINT \(default\), DSL, TEXT, or TYPEAHEAD | +| `queryVersion` | string | No | Elasticsearch query language version \(default 5.2\) | +| `query` | object | No | SAILPOINT query object: \{query?, fields?, timeZone?, innerHit?\} | +| `queryDsl` | json | No | Elasticsearch Query DSL object used with queryType=DSL | +| `textQuery` | object | No | TEXT query object with required terms\[\] and fields\[\] | +| `typeAheadQuery` | object | No | TYPEAHEAD query with query, field, optional nestedType, maxExpansions \(1-1000\), size, sort, and sortByValue | +| `includeNested` | boolean | No | Include nested objects in search results \(default true\) | +| `queryResultFilter` | object | No | Result projection object with includes\[\] and/or excludes\[\] | +| `aggregationType` | string | No | Aggregation query language: DSL \(default\) or SAILPOINT | +| `aggregationsVersion` | string | No | Elasticsearch aggregation language version \(default 5.2\) | +| `aggregationsDsl` | json | No | Dynamic Elasticsearch aggregations DSL object | +| `aggregations` | json | No | Typed SailPoint aggregation specification | +| `sort` | array | No | Ordered search fields; prefix + or - for direction | +| `searchAfter` | array | No | String values from the final sorted record of the previous search page | +| `filters` | json | No | Map of result field names to filter objects | + +#### Output + +| Parameter | Type | Description | +| --------- | ---- | ----------- | +| `total` | number | Number of matching documents | + +### SailPoint Sign Off Certification + +Sign off a completed identity certification. + +#### Input + +| Parameter | Type | Required | Description | +| --------- | ---- | -------- | ----------- | +| `clientId` | string | Yes | SailPoint Personal Access Token client ID | +| `clientSecret` | string | Yes | SailPoint Personal Access Token client secret | +| `tenant` | string | Yes | SailPoint tenant name or full *.api.identitynow.com / *.api.identitynowgov.com host | +| `id` | string | Yes | Certification ID | + +#### Output + +| Parameter | Type | Description | +| --------- | ---- | ----------- | +| `certification` | object | Signed-off identity certification | +| ↳ `id` | string | Certification ID | +| ↳ `name` | string | Certification name | +| ↳ `campaign` | json | Campaign reference | +| ↳ `completed` | boolean | Whether all decisions are complete | +| ↳ `identitiesCompleted` | number | Identities fully reviewed | +| ↳ `identitiesTotal` | number | Total identities | +| ↳ `created` | string | Creation timestamp | +| ↳ `modified` | string | Last modification timestamp | +| ↳ `decisionsMade` | number | Decisions made | +| ↳ `decisionsTotal` | number | Total decisions | +| ↳ `due` | string | Certification due timestamp | +| ↳ `signed` | string | Sign-off timestamp | +| ↳ `reviewer` | json | Reviewer reference | +| ↳ `reassignment` | json | Reassignment details | +| ↳ `hasErrors` | boolean | Whether the certification has errors | +| ↳ `errorMessage` | string | Certification error message | +| ↳ `phase` | string | Certification phase | + + diff --git a/apps/docs/content/docs/integrations/slack.mdx b/apps/docs/content/docs/integrations/slack.mdx index 56af83e283d..47656dc9aa9 100644 --- a/apps/docs/content/docs/integrations/slack.mdx +++ b/apps/docs/content/docs/integrations/slack.mdx @@ -1,6 +1,6 @@ --- title: Slack -description: Send, update, delete messages, manage views and modals, add or remove reactions, manage canvases, get channel info and user presence in Slack +description: Send and manage Slack messages, Agent Sessions, streamed replies, views, reactions, conversations, and canvases --- import { BlockInfoCard } from "@/components/ui/block-info-card" @@ -24,9 +24,23 @@ With the Slack integration in Sim, you can: - **Read messages**: Retrieve recent messages from channels or DMs, with filtering by time range - **Manage channels and users**: List channels, members, and users in your Slack workspace - **Download files**: Retrieve files shared in Slack channels for processing within a workflow +- **Build Slack agents**: Manage Agent Sessions, stream replies, handle stop/title/context events, and configure Agent View actions and suggested prompts for custom bots In Sim, the Slack integration enables your agents to programmatically interact with Slack as part of their workflows. This allows for automation scenarios such as sending notifications with dynamic updates, managing conversational flows with editable status messages, acknowledging important messages with reactions, and maintaining clean channels by removing outdated bot messages. The integration can also be used in trigger mode to start a workflow when a message is sent to a channel. +## Stream Trigger Responses + +Custom-bot Slack triggers can stream workflow outputs directly back into the conversation that started a run. Enable **Stream response to Slack** on a Message, App Mention, or Assistant Thread Started trigger, then select the outputs to deliver. + +- A selected Agent output streams immediately as it is generated. If the Agent later calls a tool, any pre-tool commentary already streamed remains visible. +- A selected non-streaming block output is sent when that block invocation completes. +- Loop and parallel invocations each create their own Slack response. +- The response status label defaults to `Running` and can be customized in the trigger's advanced settings. +- Optional thinking and tool-call updates appear as Slack tasks in a timeline or plan. +- Slack Agent Sessions remain in processing state for the run, return to active when it finishes, and the native Slack stop button cancels active workflow executions. + +Automatic trigger responses require a custom bot created by the Slack setup wizard. They are not available with the shared Sim Slack app. + ## AI-Generated Content Sim workflows may use AI models to generate messages and responses sent to Slack. AI-generated content may be inaccurate or contain errors. Always review automated outputs, especially for critical communications. @@ -39,7 +53,7 @@ If you encounter issues with the Slack integration, contact us at [help@sim.ai]( ## Usage Instructions -Integrate Slack into the workflow. Can send, update, and delete messages, send ephemeral messages visible only to a specific user, open/update/push modal views, publish Home tab views, create canvases, read messages, and add or remove reactions. Requires Bot Token instead of OAuth in advanced mode. Can be used in trigger mode to trigger a workflow when a message is sent to a channel. +Integrate Slack messaging and administration into a workflow. Custom Slack bots can manage Agent Sessions, stream incremental Markdown or structured chunks, react to Agent Session events, and configure Agent View suggested prompts. Standard messaging and management operations support both the Sim app and custom bot credentials. @@ -870,9 +884,79 @@ Set the clickable suggested prompts shown in a Slack assistant thread (the promp | `channel` | string | Channel ID the prompts were set on | | `threadTs` | string | Thread timestamp the prompts were set on | +### Slack Set Agent Suggested Prompts + +Set suggested prompts in Slack Agent View, optionally scoped to a specific thread. + +#### Input + +| Parameter | Type | Required | Description | +| --------- | ---- | -------- | ----------- | +| `authMethod` | string | No | Slack authentication method | +| `botToken` | string | No | Custom Slack bot token | +| `channel` | string | Yes | Agent direct-message channel ID | +| `threadTs` | string | No | Optional thread timestamp for legacy thread-scoped prompts | +| `prompts` | json | Yes | One to four prompt objects with title and message fields | +| `promptsTitle` | string | No | Optional heading displayed above the prompt chips | + +#### Output + +| Parameter | Type | Description | +| --------- | ---- | ----------- | +| `ok` | boolean | Whether Slack updated the suggested prompts | + +### Slack Set Agent Session Status + +Create or update the state of a Slack agent session associated with a thread. + +#### Input + +| Parameter | Type | Required | Description | +| --------- | ---- | -------- | ----------- | +| `authMethod` | string | No | Slack authentication method | +| `botToken` | string | No | Custom Slack bot token | +| `channel` | string | Yes | Channel ID containing the agent session thread | +| `threadTs` | string | Yes | Timestamp of the thread associated with the agent session | +| `status` | string | Yes | Agent session state: active, processing, suspended, or closed | +| `title` | string | No | Title used when creating the agent session, up to 200 characters | +| `initiatorUserId` | string | No | Slack user ID that initiated the session | +| `iconEmoji` | string | No | Emoji used to customize the agent identity | +| `iconUrl` | string | No | Image URL used to customize the agent identity | +| `username` | string | No | Display name used to customize the agent identity | + +#### Output + +| Parameter | Type | Description | +| --------- | ---- | ----------- | +| `ok` | boolean | Whether Slack updated the agent session | +| `status` | string | Requested agent session status | +| `agentStatus` | string | Agent status recorded by Slack | +| `title` | string | Current agent session title, or null when the session has no title | + +### Slack Rename Agent Session + +Rename the Slack agent session associated with a thread. + +#### Input + +| Parameter | Type | Required | Description | +| --------- | ---- | -------- | ----------- | +| `authMethod` | string | No | Slack authentication method | +| `botToken` | string | No | Custom Slack bot token | +| `channel` | string | Yes | Channel ID containing the agent session thread | +| `threadTs` | string | Yes | Timestamp of the thread associated with the agent session | +| `title` | string | Yes | New agent session title, from 1 to 200 characters | + +#### Output + +| Parameter | Type | Description | +| --------- | ---- | ----------- | +| `ok` | boolean | Whether Slack renamed the agent session | +| `title` | string | Updated agent session title | + ### Slack List Channels -List all channels in a Slack workspace. Returns public and private channels the bot has access to. +List accessible Slack conversations. Credential-group user tokens also return one-to-one and group direct messages. #### Input @@ -889,26 +973,33 @@ List all channels in a Slack workspace. Returns public and private channels the | Parameter | Type | Description | | --------- | ---- | ----------- | -| `channels` | array | Array of channel objects from the workspace | -| ↳ `id` | string | Channel ID \(e.g., C1234567890\) | -| ↳ `name` | string | Channel name without # prefix | +| `channels` | array | Accessible public and private channels, plus direct and group DMs for credential-group user tokens | +| ↳ `id` | string | Conversation ID \(for example, C123, D123, or G123\) | +| ↳ `name` | string | Channel or group-DM name; omitted for one-to-one direct messages | | ↳ `is_channel` | boolean | Whether this is a channel | -| ↳ `is_private` | boolean | Whether channel is private | -| ↳ `is_archived` | boolean | Whether channel is archived | +| ↳ `is_private` | boolean | Whether the conversation is private | +| ↳ `is_archived` | boolean | Whether the conversation is archived | | ↳ `is_general` | boolean | Whether this is the general channel | -| ↳ `is_member` | boolean | Whether the bot/user is a member | +| ↳ `is_member` | boolean | Whether the credential owner is a member | | ↳ `is_shared` | boolean | Whether channel is shared across workspaces | | ↳ `is_ext_shared` | boolean | Whether channel is externally shared | | ↳ `is_org_shared` | boolean | Whether channel is org-wide shared | | ↳ `num_members` | number | Number of members in the channel | -| ↳ `topic` | string | Channel topic | -| ↳ `purpose` | string | Channel purpose/description | +| ↳ `topic` | string | Conversation topic | +| ↳ `purpose` | string | Conversation purpose | | ↳ `created` | number | Unix timestamp when channel was created | | ↳ `creator` | string | User ID of channel creator | | ↳ `updated` | number | Unix timestamp of last update | -| `ids` | array | Array of channel IDs for easy access | -| `names` | array | Array of channel names for easy access | -| `count` | number | Total number of channels returned | +| ↳ `is_group` | boolean | Whether this is a legacy private channel or group direct message | +| ↳ `is_im` | boolean | Whether this is a one-to-one direct message | +| ↳ `is_mpim` | boolean | Whether this is a group direct message | +| ↳ `user` | string | Other participant user ID for a one-to-one direct message | +| ↳ `is_user_deleted` | boolean | Whether the other participant in a direct message is deactivated | +| ↳ `is_open` | boolean | Whether a direct or group-direct-message conversation is open | +| ↳ `priority` | number | Slack sidebar sort priority | +| `ids` | array | Conversation IDs for every returned channel or DM | +| `names` | array | Names of returned channels and group DMs; one-to-one DMs have no name | +| `count` | number | Total number of conversations returned | | `nextCursor` | string | Cursor for the next page; null if no more pages | ### Slack List Channel Members @@ -1873,7 +1964,7 @@ A **Trigger** is a block that starts a workflow when an event happens in this se ### Slack -Trigger from Slack events (mentions, messages, reactions) +Trigger from Slack events, interactions, and slash commands #### Configuration @@ -1886,9 +1977,16 @@ Trigger from Slack events (mentions, messages, reactions) | `channelFilter` | channel-selector | No | Restrict to specific channels. Leave empty to trigger on any channel the bot has been added to. | | `manualChannelFilter` | string | No | Comma-separated channel IDs to restrict to. Set IDs directly here. | | `threads` | string | No | Include thread replies, exclude them \(top-level only\), or fire only on thread replies. | +| `streamResponse` | boolean | No | Create a Slack agent session and stream selected workflow outputs into the conversation that started this run. Custom bots only. | +| `streamOutputs` | workflow-output-selector | No | Use `<blockName>.<outputPath>` for this workflow or `<childWorkflowId>.<blockName>.<outputPath>` for a child workflow. Selecting a child workflow applies to every invocation of it. Agent outputs stream live; other outputs are sent when the block completes. | +| `streamTaskTitle` | string | No | Optional status Slack shows while each selected response is being produced. Leave empty to use Running. | +| `streamTaskDisplayMode` | string | No | Choose how Slack displays thinking and tool progress. | +| `streamIncludeThinking` | boolean | No | Show agent thinking as Slack task updates while the response is generated. | +| `streamIncludeToolCalls` | boolean | No | Show tool execution lifecycle as Slack task updates. | | `emoji` | string | No | Comma-separated emoji names to restrict to. Leave empty to match any emoji. | | `nameContains` | string | No | Only fire when the created channel name contains this text. | | `interactionFilter` | string | No | Comma-separated action_ids \(buttons/selects\) or callback_ids \(modals\) to restrict to. Leave empty to fire on any interaction. | +| `commandFilter` | string | No | Restrict this trigger to one slash command. Leave empty to fire for every command configured on the bot. | | `filterBotMessages` | boolean | No | Ignore messages sent by other bots. This app's own output is always ignored. | | `includeOwnMessages` | boolean | No | Also fire on this app's own messages and reactions. Can cause loops — use with care. | | `includeFiles` | boolean | No | Download and include file attachments from messages. Requires files:read. | @@ -1909,7 +2007,14 @@ Trigger from Slack events (mentions, messages, reactions) | ↳ `text` | string | Message text content. For slash commands, the text after the command. For interactivity, the source message text \(falls back to the triggering action value\) | | ↳ `timestamp` | string | Message timestamp from the triggering event | | ↳ `thread_ts` | string | Parent thread timestamp \(if message is in a thread\) | +| ↳ `streaming_message_ts` | array | Message timestamps streamed during a stopped agent session | +| ↳ `title` | string | Current agent session title | +| ↳ `previous_title` | string | Previous agent session title | +| ↳ `tab` | string | App Home tab that was opened, including messages for Agent View | +| ↳ `context` | json | Current Agent View context. Normalized from context on app_context_changed/app_home_opened or app_context on message.im | | ↳ `team_id` | string | Slack workspace/team ID | +| ↳ `user_team_id` | string | Slack workspace/team ID of the user who triggered the event. Used for Slack Connect response streaming. | +| ↳ `enterprise_id` | string | Slack Enterprise Grid organization ID | | ↳ `event_id` | string | Unique event identifier | | ↳ `reaction` | string | Emoji reaction name \(e.g., thumbsup\). Present for reaction_added/reaction_removed events | | ↳ `item_user` | string | User ID of the original message author. Present for reaction_added/reaction_removed events | diff --git a/apps/docs/content/docs/integrations/table.mdx b/apps/docs/content/docs/integrations/table.mdx index 744cbe6aa71..096574ad428 100644 --- a/apps/docs/content/docs/integrations/table.mdx +++ b/apps/docs/content/docs/integrations/table.mdx @@ -6,7 +6,7 @@ description: User-defined data tables import { BlockInfoCard } from "@/components/ui/block-info-card" @@ -17,7 +17,7 @@ Tables allow you to create and manage custom data tables directly within Sim. St - **No external setup**: Create tables instantly without configuring external databases - **Workflow-native**: Data persists across workflow executions and is accessible from any workflow in your workspace - **Flexible schema**: Define columns with types (string, number, currency, boolean, date, json, select) and constraints (required, unique) -- **Powerful querying**: Filter, sort, and paginate data using MongoDB-style operators +- **Powerful querying**: Filter, sort, and paginate data using a typed predicate grammar - **Agent-friendly**: Tables can be used as tools by AI agents for dynamic data storage and retrieval **Key Features:** @@ -54,7 +54,7 @@ Tables are created from the **Tables** section in the sidebar. Each table requir ## Usage Instructions -Create and manage custom data tables. Store, query, and manipulate structured data within workflows. Query Rows returns every matching row when Limit is omitted and fails if the result exceeds 5MB. +Create and manage custom data tables. Store, query, and manipulate structured data within workflows. Query Rows accepts a plain predicate — `{"field":"wins","op":"gte","value":10}` — for one condition. Use `all` (AND) or `any` (OR) groups for multiple or nested conditions. Operators: eq, ne, gt, gte, lt, lte, in, nin, like, ilike, nlike, nilike, contains, ncontains, startsWith, endsWith, isNull, isNotNull, isEmpty, isNotEmpty. Order is a sort spec `[{"field":"wins","direction":"desc"}]`. Query Rows returns every matching row when Limit is omitted (fails if the result exceeds 5MB — add a filter or a Limit). With a Limit, responses page: a non-null nextCursor means more rows exist — pass it back as the cursor. Columns to Return narrows each row to the selected columns (by stable id or name; one that no longer exists is skipped); leave it empty for every column. @@ -204,29 +204,29 @@ Delete multiple rows that match filter criteria. Use with caution - supports opt ### Query Rows -Query rows from a table with filtering, sorting, and pagination +Query rows with a typed predicate filter and cursor pagination. A single filter can be a plain condition: `\{"field":"wins","op":"gte","value":10\}`. Use `all` (AND) or `any` (OR) groups for multiple or nested conditions. Operators: eq, ne, gt, gte, lt, lte, in, nin, like, ilike, nlike, nilike, contains, ncontains, startsWith, endsWith, isNull, isNotNull, isEmpty, isNotEmpty. Order is a sort spec, e.g. `[\{"field":"wins","direction":"desc"\}]`. Omit limit to return the entire result — the query fails if it exceeds the 5MB budget (narrow with a filter or set a limit). With a limit, a page can end early at the byte budget: a non-null nextCursor means more rows exist — pass it back as cursor to continue; never infer completion from page size. #### Input | Parameter | Type | Required | Description | | --------- | ---- | -------- | ----------- | | `tableId` | string | Yes | Table ID | -| `filter` | object | No | Filter conditions \(MongoDB-style operators: $eq, $ne, $gt, $gte, $lt, $lte, $in, $nin, $contains, $ncontains, $startsWith, $endsWith, $empty\) | -| `sort` | object | No | Sort order as \{field: "asc"\|"desc"\} | -| `limit` | number | No | Maximum rows to return. Omit to return every matching row; the query fails if the result exceeds the 5MB response budget. | -| `offset` | number | No | Number of rows to skip \(default: 0\) | +| `filter` | json | No | Predicate condition, e.g. `\{"field":"wins","op":"gte","value":10\}`. Use `all` or `any` for multiple conditions; omit to match all rows. | +| `columns` | array | No | Stable column IDs or table column names to include in each row data object. Omit or pass an empty array to return all columns. A reference that matches no column is ignored. | +| `order` | json | No | Sort spec, e.g. `\[\{"field":"wins","direction":"desc"\}\]`. | +| `limit` | number | No | Maximum rows per page. Omit to return the entire matching result — fails if it exceeds the 5MB budget. With a limit, pages may byte-cut early and set nextCursor when more remain. | +| `cursor` | string | No | Opaque pagination cursor returned by a prior query. Omit for the first page. | #### Output | Parameter | Type | Description | | --------- | ---- | ----------- | -| `success` | boolean | Whether query succeeded | +| `success` | boolean | Whether the query succeeded | | `rows` | array | Query result rows | | `rowCount` | number | Number of rows returned | -| `totalCount` | number | Total rows matching filter | -| `limit` | number | Limit used in query | -| `offset` | number | Offset used in query | -| `nextCursor` | string | Non-null when more rows match past this page. A page can end early at the byte budget, so this — not a short rowCount — is what says whether more remain. To page, advance offset by rowCount and stop when this is null. | +| `totalCount` | number | Total rows matching the predicate \(computed on the first page only\) | +| `limit` | number | Limit used in the query | +| `nextCursor` | string | Cursor to fetch the next page, or null on the last page | ### Get Row @@ -272,65 +272,104 @@ Get the schema configuration of a table {/* MANUAL-CONTENT-START:notes */} ## Filter Operators -Filters use MongoDB-style operators for flexible querying: +A filter is a predicate. One condition is an object naming a column, an operator, and a value: + +```json +{"field": "status", "op": "eq", "value": "active"} +``` | Operator | Description | Example | |----------|-------------|---------| -| `$eq` | Equals | `{"status": {"$eq": "active"}}` or `{"status": "active"}` | -| `$ne` | Not equals | `{"status": {"$ne": "deleted"}}` | -| `$gt` | Greater than | `{"age": {"$gt": 18}}` | -| `$gte` | Greater than or equal | `{"score": {"$gte": 80}}` | -| `$lt` | Less than | `{"price": {"$lt": 100}}` | -| `$lte` | Less than or equal | `{"quantity": {"$lte": 10}}` | -| `$in` | In array | `{"status": {"$in": ["active", "pending"]}}` | -| `$nin` | Not in array | `{"type": {"$nin": ["spam", "blocked"]}}` | -| `$contains` | String contains (case-insensitive) | `{"email": {"$contains": "@gmail.com"}}` | -| `$ncontains` | Does not contain (case-insensitive; matches empty cells) | `{"email": {"$ncontains": "@spam.com"}}` | -| `$startsWith` | Starts with (case-insensitive) | `{"name": {"$startsWith": "Dr."}}` | -| `$endsWith` | Ends with (case-insensitive) | `{"file": {"$endsWith": ".pdf"}}` | -| `$empty` | Cell is empty (`true`) or non-empty (`false`) | `{"phone": {"$empty": true}}` | +| `eq` | Equals | `{"field": "status", "op": "eq", "value": "active"}` | +| `ne` | Not equals | `{"field": "status", "op": "ne", "value": "deleted"}` | +| `gt` | Greater than | `{"field": "age", "op": "gt", "value": 18}` | +| `gte` | Greater than or equal | `{"field": "score", "op": "gte", "value": 80}` | +| `lt` | Less than | `{"field": "price", "op": "lt", "value": 100}` | +| `lte` | Less than or equal | `{"field": "quantity", "op": "lte", "value": 10}` | +| `in` | In array | `{"field": "status", "op": "in", "value": ["active", "pending"]}` | +| `nin` | Not in array | `{"field": "type", "op": "nin", "value": ["spam", "blocked"]}` | +| `contains` / `ncontains` | Contains, or does not contain (case-insensitive) | `{"field": "email", "op": "contains", "value": "@gmail.com"}` | +| `like` / `nlike` | Pattern match, `*` wildcard (case-sensitive) | `{"field": "name", "op": "like", "value": "Dr.*"}` | +| `ilike` / `nilike` | Pattern match, `*` wildcard (case-insensitive) | `{"field": "name", "op": "ilike", "value": "*jo*"}` | +| `startsWith` | Starts with (case-insensitive) | `{"field": "name", "op": "startsWith", "value": "Dr."}` | +| `endsWith` | Ends with (case-insensitive) | `{"field": "file", "op": "endsWith", "value": ".pdf"}` | +| `isNull` / `isNotNull` | Cell is (not) null | `{"field": "phone", "op": "isNull"}` | +| `isEmpty` / `isNotEmpty` | Cell is (not) empty | `{"field": "phone", "op": "isEmpty"}` | + +Most columns are scalar (string, number, boolean, date) or opaque JSON; use `ilike` with `*value*` for substring matching on text. + +**Select columns accept only a subset of these operators**, and a query using any other operator on one is rejected rather than returning no rows: + +| Column | Allowed operators | +|--------|-------------------| +| Single-select | `eq`, `ne`, `in`, `nin`, `isEmpty`, `isNotEmpty` | +| Multi-select | `contains`, `ncontains`, `isEmpty`, `isNotEmpty` | + +A multi-select cell holds a list of options, so match it with `contains` (by option name) rather than `ilike`. ### Combining Filters -Multiple field conditions are combined with AND logic: +Wrap conditions in `all` for AND: ```json { - "status": "active", - "age": {"$gte": 18} + "all": [ + {"field": "status", "op": "eq", "value": "active"}, + {"field": "age", "op": "gte", "value": 18} + ] } ``` -Use `$or` for OR logic: +Use `any` for OR: ```json { - "$or": [ - {"status": "active"}, - {"status": "pending"} + "any": [ + {"field": "status", "op": "eq", "value": "active"}, + {"field": "status", "op": "eq", "value": "pending"} ] } ``` -## Sort Specification - -Specify sort order with column names and direction: +Groups nest, so mixed logic is a group inside a group: ```json { - "createdAt": "desc" + "all": [ + {"field": "status", "op": "eq", "value": "active"}, + {"any": [ + {"field": "plan", "op": "eq", "value": "pro"}, + {"field": "score", "op": "gte", "value": 90} + ]} + ] } ``` +Omit the filter entirely to match every row. + +## Sort Specification + +Order is a list of column/direction pairs, applied in order: + +```json +[{"field": "createdAt", "direction": "desc"}] +``` + Multi-column sorting: ```json -{ - "priority": "desc", - "name": "asc" -} +[ + {"field": "priority", "direction": "desc"}, + {"field": "name", "direction": "asc"} +] ``` +## Pagination + +Omit **Limit** to return every matching row in one response; the query fails if the result exceeds 5MB, so narrow with a filter rather than guessing a limit. + +With a **Limit**, results page. A page can end at the limit *or* at the 5MB byte budget, whichever comes first, so a short page does not mean the end. Pass the returned `nextCursor` back as **Cursor** to fetch the next page and stop only when `nextCursor` is null — never infer completion from the row count. + ## Built-in Columns Every row automatically includes: diff --git a/apps/docs/content/docs/integrations/webflow.mdx b/apps/docs/content/docs/integrations/webflow.mdx index 32b968c82ef..db906102450 100644 --- a/apps/docs/content/docs/integrations/webflow.mdx +++ b/apps/docs/content/docs/integrations/webflow.mdx @@ -36,7 +36,7 @@ Integrates Webflow CMS into the workflow. Can create, get, list, update, or dele ### Webflow List Items -List all items from a Webflow CMS collection +List items from a Webflow CMS collection #### Input @@ -64,6 +64,7 @@ List all items from a Webflow CMS collection | ↳ `itemCount` | number | Number of items returned | | ↳ `offset` | number | Pagination offset | | ↳ `limit` | number | Maximum items per page | +| ↳ `total` | number | Total number of matching items | ### Webflow Get Item diff --git a/apps/docs/content/docs/platform/credentials.mdx b/apps/docs/content/docs/platform/credentials.mdx index f91b41c6630..922080c7a98 100644 --- a/apps/docs/content/docs/platform/credentials.mdx +++ b/apps/docs/content/docs/platform/credentials.mdx @@ -177,11 +177,13 @@ When a workflow runs, secrets resolve in this order: | Run started by | Personal secrets come from | | --- | --- | | Clicking Run, or a personal API key | The person running it | -| A workspace API key, schedule, or webhook | The workflow owner | +| A workspace API key, schedule, webhook, or deployed chat | The workflow owner | | A public API URL with no authentication | Nobody — personal secrets do not resolve | The workflow owner is the fallback only where nobody can be identified but somebody in the workspace set the trigger up, since those workflows are usually built against the owner's own keys. A public URL can be called by anyone, so it never borrows a person's keys at all — put every secret such a workflow needs in **Workspace**. +If the workflow owner later leaves the workspace, the run keeps working: it resolves workspace secrets as normal and simply resolves no personal ones, so any block that needed a personal secret fails on its own with the missing key named. Move that secret to **Workspace** to fix it for good. + ## Best Practices - **Use workspace secrets for production** so workflows work regardless of who triggers them @@ -193,7 +195,8 @@ The workflow owner is the fallback only where nobody can be identified but someb { question: "Are my secrets encrypted at rest?", answer: "Yes. Values saved under Secrets are encrypted before being stored in the database." }, { question: "Can a saved secret still appear in a workflow result?", answer: "Yes. Functional workflow data is not rewritten, so the raw value can still reach downstream blocks and tools and can appear in workflow execution responses, streams, or callbacks if your workflow deliberately returns or prints it. Log-facing views and read APIs receive a protected copy after a successful {{KEY}} resolution. Before content is sent to a model, exact values from the run's authorized secret catalog are replaced with placeholders, but encoded or otherwise transformed values remain outside that protection." }, { question: "What happens if both a workspace secret and a personal secret have the same key name?", answer: "Among secrets available to the execution actor, the workspace secret takes precedence and the personal secret is the fallback. An inaccessible workspace secret does not shadow an authorized personal value." }, - { question: "Who determines which personal secret is used for automated runs?", answer: "Whoever is running it, when that can be identified. Clicking Run or calling with a personal API key uses that person's personal secrets. A workspace API key, schedule, or webhook has no identifiable caller, so it falls back to the workflow owner's — those triggers are set up inside the workspace and the workflow is usually built against the owner's own keys. A public API URL with no authentication can be called by anyone, so no personal secrets resolve at all — those workflows run on workspace secrets only." }, + { question: "Who determines which personal secret is used for automated runs?", answer: "Whoever is running it, when that can be identified. Clicking Run or calling with a personal API key uses that person's personal secrets. A workspace API key, schedule, webhook, or deployed chat has no identifiable caller, so it falls back to the workflow owner's — those triggers are set up inside the workspace and the workflow is usually built against the owner's own keys. A public API URL with no authentication can be called by anyone, so no personal secrets resolve at all — those workflows run on workspace secrets only." }, + { question: "What happens to automated runs if the workflow owner leaves the workspace?", answer: "They keep running. Workspace secrets resolve as normal, because they are checked against the workspace's billing account rather than the owner. Personal secrets stop resolving, so any block that referenced one fails with that key named — move it to Workspace to fix it permanently." }, { question: "Can I import secrets from a .env file?", answer: "Yes. Paste .env-style content (KEY=VALUE format) into any key or value field and the secrets will be auto-populated. The parser supports export KEY=VALUE, quoted values, and inline comments." }, { question: "What happens if I delete a secret that is used in a workflow?", answer: "The workflow will fail at any block that references the deleted secret during execution because the value cannot be resolved. Update any references before deleting a secret." }, ]} /> diff --git a/apps/docs/content/docs/platform/enterprise/sso.mdx b/apps/docs/content/docs/platform/enterprise/sso.mdx index a79851039ac..731ac1d305b 100644 --- a/apps/docs/content/docs/platform/enterprise/sso.mdx +++ b/apps/docs/content/docs/platform/enterprise/sso.mdx @@ -48,6 +48,7 @@ Go to **Settings → Security → Single sign-on** in your organization settings | **Provider ID** | A short slug identifying this connection. Letters, numbers, and dashes only. It must be **unique across every Sim organization**, so include something specific to you — `azure-ad-acme`, not `azure-ad`. If the ID is taken, Sim tells you and suggests a free one. | | **Issuer URL** | The identity provider's issuer URL. Must be HTTPS. | | **Domain** | Your organization's email domain, e.g. `company.com`. Users with this domain will be routed through SSO at sign-in. | +| **Member provisioning** | **Automatic** adds a user authenticated through this verified SSO connection to the organization as a Member and consumes a billed seat. Team seat counts grow with membership; fixed-seat plans require available capacity. **Invite only** authenticates the user without creating organization membership. Neither mode grants workspace access automatically. | **OIDC additional fields:** @@ -267,16 +268,17 @@ Once SSO is configured, users with your domain (`company.com`) can sign in throu 1. User goes to `sim.ai` and clicks **Sign in with SSO** 2. They enter their work email (e.g. `alice@company.com`) 3. Sim redirects them to your identity provider -4. After authenticating, they are returned to Sim and added to your organization automatically -5. They land in the workspace +4. After authenticating, they are returned to Sim +5. If **Member provisioning** is **Automatic**, Sim adds them to the organization as a Member, growing a Team seat count or validating available fixed-seat capacity +6. They land in an accessible workspace, or see a clear no-access state until an admin grants workspace access -Users who sign in via SSO for the first time are automatically provisioned and added to your organization — no manual invite required. +With **Automatic** provisioning, no invitation is required for organization membership. The join follows the organization's seat policy and does not infer a role from IdP claims: every newly provisioned user starts as a Member. Team subscriptions grow their billed seat count with membership; fixed-seat plans reject the join when capacity is full. With **Invite only**, SSO proves identity but does not create new membership or workspace access; new access must be granted separately, while existing organization membership and workspace access remain available. Sign-in must start from Sim. Launching from your identity provider's app portal (Microsoft's **My Apps**, Okta's dashboard tile) sends an unsolicited assertion, which Sim rejects. This is deliberate — accepting them would let anyone replay an assertion into your tenant — but it means an IdP-initiated test fails even when the configuration is correct. -SSO provisioning creates internal organization members. External workspace members are different: they are invited to a specific workspace without joining your organization or consuming one of your seats. +SSO provisioning creates internal organization members but does not grant workspace access. External workspace members are different: they are invited to a specific workspace without joining your organization or consuming one of your seats. Existing invitations and external access take precedence over automatic provisioning so their intended role and workspace grants are preserved. Password-based login remains available. Forcing all organization members to use SSO exclusively is not yet supported. @@ -299,7 +301,11 @@ SSO provisioning creates internal organization members. External workspace membe }, { question: "What happens when a user signs in with SSO for the first time?", - answer: "Sim creates an account for them automatically and adds them to your organization. No manual invite is needed. They are assigned the member role by default. External workspace members are not provisioned through SSO into your organization; they are invited directly to a workspace and remain outside your org roster." + answer: "Sim creates or links their account. If Member provisioning is Automatic and a seat is available, Sim adds them to your organization as a Member; no manual organization invite is needed. Workspace access is always granted separately. If provisioning is Invite only, or the user already has a pending invitation or external workspace access, Sim preserves that flow instead of creating membership automatically." + }, + { + question: "Does disabling someone in the identity provider remove their Sim access?", + answer: "No. Disabling the IdP account blocks future SSO authentication, but Sim does not currently receive SCIM deprovisioning or IdP logout events to remove membership or revoke active Sim sessions. Remove or suspend the user in Sim as part of offboarding." }, { question: "Can I still use email/password login after enabling SSO?", diff --git a/apps/docs/content/docs/platform/self-hosting/environment-variables.mdx b/apps/docs/content/docs/platform/self-hosting/environment-variables.mdx index 378d6080f8d..2111c649095 100644 --- a/apps/docs/content/docs/platform/self-hosting/environment-variables.mdx +++ b/apps/docs/content/docs/platform/self-hosting/environment-variables.mdx @@ -147,7 +147,11 @@ Without a remote provider, user code runs in an in-process V8 isolate inside the | `WEBHOOK_EXECUTION_CONCURRENCY_LIMIT` | `75` | Webhook-triggered executions in parallel | | `SCHEDULE_EXECUTION_CONCURRENCY_LIMIT` | `30` | Scheduled executions in parallel | | `RESUME_EXECUTION_CONCURRENCY_LIMIT` | `50` | Resumed executions in parallel | -| `ALLOW_PRIVATE_DATABASE_HOSTS` | unset | Let database/connector tools reach private, reserved, and loopback hosts. Loosens the SSRF boundary | +| `EGRESS_ALLOWED_HOSTS` | unset | Comma-separated hostnames outbound requests may reach on a private network. Leading wildcard allowed, e.g. `host.docker.internal,*.svc.cluster.local`. Not honored for URLs harvested from content or a third-party response, nor for an HTTP block's `proxyUrl` | +| `EGRESS_ALLOWED_IP_RANGES` | unset | Comma-separated CIDRs or IPs outbound requests may reach on a private network, e.g. `10.0.0.0/8`. Same exclusions | +| `ALLOW_PRIVATE_DATABASE_HOSTS` | unset | **Deprecated.** Vouches for the entire private address space, for database, cache, and mail connector hosts only. Replace it with the two settings above naming specific destinations | + +A malformed entry in either allowlist stops the app at startup with a message naming the setting. See [the SSRF boundary](/platform/self-hosting/security#the-ssrf-boundary) for the accepted syntax and for what the allowlist does and does not reach. Your reverse proxy's body-size limit must be at least as large as the app limits above. See [Networking](/platform/self-hosting/networking). diff --git a/apps/docs/content/docs/platform/self-hosting/security.mdx b/apps/docs/content/docs/platform/self-hosting/security.mdx index 0b67762d142..f2c600160c7 100644 --- a/apps/docs/content/docs/platform/self-hosting/security.mdx +++ b/apps/docs/content/docs/platform/self-hosting/security.mdx @@ -134,18 +134,56 @@ Resource ceilings for the in-process path: ## The SSRF boundary -Sim blocks outbound requests from database and connector tools to private, reserved, and loopback addresses. This stops a workflow from being used to scan your internal network. +By default Sim blocks outbound requests to private, reserved, and loopback addresses. This stops a workflow from being used to scan your internal network. Two things soften it on a self-hosted deployment: the provenances marked **Yes** below reach whatever you allowlist, and a configured endpoint, self-hosted service, or request target written as `localhost` or a loopback literal is reachable without any allowlist at all — a local Ollama or Jupyter is the ordinary case. That second carve-out stops short in two places: it does not lift the blocked-port list, and it does not extend to a database, cache, or mail connector on `localhost` — loopback is where Sim's own database and Redis listen, so reaching them has to be asked for. Neither softening applies on Sim Cloud. Every outbound request is classified by where its URL came from: -Self-hosted deployments often legitimately need to reach an internal database by service name. That is opt-in: +| Provenance | Examples | Reaches allowlisted private destinations | +|---|---|---| +| Configured endpoint | GitHub Enterprise, Grafana, a data-drain destination, a connector's host | Yes | +| Self-hosted service | vLLM, Jupyter, 1Password Connect, ClickHouse, an MCP server — software usually run on-prem without TLS, so plain HTTP is expected | Yes | +| Request target | The HTTP block's URL, an A2A agent, an RSS feed, a Function block's `fetch` | Yes | +| Database host | A database, cache, or mail connector's host | Yes | +| Content fetch | An image URL, a file imported by URL, a link from a third-party API response | **No** | +| Proxy | The outbound HTTP proxy itself | **No** | + +Content fetches never reach a private destination, allowlist or not — that is the class where SSRF is actually exploited. Nor does the proxy: it is the component deciding where everything else may go, so it is held to public destinations regardless of what the allowlist says. + +Deployments frequently need to reach an internal service by name or address. Name the destinations: ```bash -ALLOW_PRIVATE_DATABASE_HOSTS=true +EGRESS_ALLOWED_HOSTS=host.docker.internal,vllm.ai.svc.cluster.local +EGRESS_ALLOWED_IP_RANGES=10.4.2.17/32,10.4.9.0/24 +``` + +A wildcard (`*.svc.cluster.local`) and a broad range (`10.0.0.0/8`) are accepted, but they hand every workflow author the whole namespace or network. Name the hosts you actually use. + +Both lists are validated when Sim starts, and a malformed entry stops it with a message naming the setting. `EGRESS_ALLOWED_HOSTS` takes hostnames only — a URL or a CIDR is rejected — and a wildcard has to be a leading `*.` covering at least two labels, so `*.local` is refused and `*.svc.cluster.local` matches `vllm.ai.svc.cluster.local` but not the bare `svc.cluster.local`. `EGRESS_ALLOWED_IP_RANGES` takes CIDRs and bare addresses; a range shorter than `/8` (such as `0.0.0.0/1`) is refused as a near-catch-all, so `10.0.0.0/8` is the broadest a single entry can name. + +On Sim Cloud plain HTTP is refused for every provenance except the proxy, whose scheme is fixed by protocol — self-hosted-service ones included: nothing is vouched there, so a credential would cross the wire in the clear. + +Naming a destination permits plain HTTP to it and lifts the blocked-port list for it, since those are the same decision about the same host. The loopback carve-out does not: it is granted without being asked for, so `http://localhost:5432` stays refused until `localhost` is named. A database, cache, or mail connector's host carries no scheme or port of its own, so naming one of those only lifts the private-address block. Cloud metadata endpoints (`169.254.169.254` and equivalents) stay blocked no matter how broad the allowlist is, and both variables are ignored entirely on Sim Cloud. + +The allowlist reaches the four provenances marked **Yes** above. It does not reach a content fetch, and it does not reach a proxy: an HTTP block's `proxyUrl` must be a public address, because the proxy is what decides where every other request may go. Adding an internal proxy to the allowlist will not make it work. + +To reach a service on the Docker host, pair the allowlist with the host alias that Compose already sets up: + +```bash +EGRESS_ALLOWED_HOSTS=host.docker.internal ``` - This loosens the SSRF boundary for every workflow author on the instance. Enable it only on a trusted private network, and prefer pairing it with a NetworkPolicy that constrains what the app can actually reach. + An allowlist widens what every workflow author on the instance can reach. Name specific hosts and narrow ranges rather than whole private networks, and pair it with a NetworkPolicy that constrains what the app can actually reach. When `networkPolicy.enabled` is true the chart permits broad egress on port 443 only, so an allowlisted in-cluster target on another port also needs a `networkPolicy.egress` rule — or `networkPolicy.allowExternalEgress: true` for unrestricted egress. +### Upgrading from an earlier release + +The allowlist replaces four separate escape hatches, so a few deployments that worked before now need a destination named: + +- **`ALLOW_PRIVATE_DATABASE_HOSTS`** still works, but it is deprecated and logs a warning at startup. It vouches for the whole private address space, loopback included, for database, cache, and mail connector hosts. Replace it with `EGRESS_ALLOWED_HOSTS` or `EGRESS_ALLOWED_IP_RANGES` naming the hosts you actually use. +- **1Password Connect** on a private, non-loopback address, and an **MCP server** on a private address or reached through a DNS name that points at loopback, are no longer reachable implicitly. Name them. +- **`ALLOWED_MCP_DOMAINS`** governs which domains may be used; it no longer disables the address check, so an MCP server on a private address needs the allowlist too. +- **Content fetches** — an image URL, a file imported by URL, an MCP OAuth endpoint on a different origin than the MCP server itself — never use the allowlist. Those destinations have to be publicly routable. (An SSO OIDC *discovery* URL is a configured endpoint and does use the allowlist; the endpoints named *inside* the discovery document are validated as content when Sim registers the provider, so an internal IdP endpoint is refused there and cannot be allowlisted — provide the endpoints explicitly to have them validated as configured endpoints instead.) +- **Redirects** are re-judged at every hop under the request's own provenance, so a redirect that lands on a blocked port is refused, and one that downgrades to plain HTTP is refused for every provenance except the self-hosted-service and proxy classes, which expect plain HTTP by design. Only 301, 302, 303, 307 and 308 are followed; 300, 305 and 306 are not. When a redirect crosses origins and the caller supplied no redirect policy, every header is dropped, so no credential — a standard `Authorization`/`Cookie` or a custom one like `PRIVATE-TOKEN` — reaches the new origin; a caller that supplies a policy keeps its non-credential headers and drops the ones it marked sensitive. A cross-origin redirect that would carry a request body to the new origin is refused outright rather than replayed: a body-preserving hop — a 307 or 308, or any hop in a legacy-replay workflow — that crosses origins fails with a message saying so, whereas a standard 301, 302 or 303 drops the body to a GET first and continues. + ## Client IP and forwarded headers Behind a load balancer, `X-Forwarded-For` is client-controllable. Set `AUTH_TRUSTED_PROXIES` to your proxies' actual addresses so Better Auth resolves the real client IP, and `TRUSTED_ORIGINS` if users reach Sim from more than one origin. Both are covered in [Authentication](/platform/self-hosting/authentication#behind-a-load-balancer). @@ -192,5 +230,5 @@ The service bundles ~2.2 GB of spaCy models, so first start takes around three m { question: "Can I rotate ENCRYPTION_KEY?", answer: "Not without re-encrypting everything it protects. Changing it makes workspace environment variables, stored provider API keys, MCP OAuth credentials, and deployment secrets permanently unreadable. Treat it as a permanent, backed-up value rather than a rotating secret."}, { question: "Where does user-authored code run?", answer: "By default in an in-process V8 isolate inside the app container, which isolates at the JS-engine level but shares the container's network and filesystem context. For untrusted authors, or to run Python at all, use E2B or Daytona so each execution runs in a remote sandbox."}, { question: "Why does the chart's NetworkPolicy allow traffic from any pod?", answer: "networkPolicy.ingressFrom defaults to an empty peer selector as a simple default that works on any cluster. On a shared cluster you should scope it to your ingress controller's namespace."}, - { question: "What does ALLOW_PRIVATE_DATABASE_HOSTS change?", answer: "It lets database and connector tools reach private, reserved, and loopback addresses — needed to connect to an internal database by Kubernetes service name. It also widens the SSRF boundary for every workflow author, so enable it only on a trusted network."}, + { question: "How do I reach an internal service from a workflow?", answer: "Name it in EGRESS_ALLOWED_HOSTS (hostnames, leading wildcard allowed) or EGRESS_ALLOWED_IP_RANGES (CIDRs). That permits plain HTTP to it and lifts the blocked-port list for HTTP destinations; a database, cache, or mail host carries no scheme or port of its own, so naming it only lifts the private-address block. Cloud metadata endpoints stay blocked regardless, content fetches never use the allowlist, and both variables are ignored on Sim Cloud."}, ]} /> diff --git a/apps/docs/content/docs/platform/self-hosting/troubleshooting.mdx b/apps/docs/content/docs/platform/self-hosting/troubleshooting.mdx index 6e00170b57f..8451dc7e593 100644 --- a/apps/docs/content/docs/platform/self-hosting/troubleshooting.mdx +++ b/apps/docs/content/docs/platform/self-hosting/troubleshooting.mdx @@ -25,6 +25,24 @@ OLLAMA_URL=http://host.docker.internal:11434 # macOS/Windows OLLAMA_URL=http://192.168.1.x:11434 # Linux (use actual IP) ``` +## A Workflow Cannot Reach a Service on Your Network + +Outbound requests to private, reserved, and loopback addresses are blocked by default, so a workflow pointed at your Docker host, a LAN service, or a Kubernetes service name fails with a message naming the blocker — the private or loopback address it resolved to, a blocked port, or `must use https:// to a public destination` when the URL is plain HTTP — and pointing at the allowlist variables. + +Name the destination: + +```bash +EGRESS_ALLOWED_HOSTS=host.docker.internal,*.svc.cluster.local +EGRESS_ALLOWED_IP_RANGES=10.0.0.0/8 +``` + +Naming a destination also permits plain HTTP to it and lifts the blocked-port list for it. A database, cache, or mail connector's host carries no scheme or port of its own, so naming one of those only lifts the private-address block. Cloud metadata endpoints (`169.254.169.254` and equivalents) stay blocked however broad the list is, and both variables are ignored on Sim Cloud. + +Two things this does not cover: + +- Inside a container `localhost` is the container itself, so it will never reach a service on your host. Use `host.docker.internal` (the Compose files map it) and name it above. +- URLs harvested from content or from a third-party API response — an image URL, a file imported by URL, an MCP OAuth endpoint on a different origin than the MCP server itself — never reach a private network, allowlist or not. Nor does an HTTP block's `proxyUrl`. + ## LM Studio Requests Route to Ollama Sim identifies dynamically discovered LM Studio and vLLM models by their `vllm/` prefix. If the endpoint is unavailable and you manually enter the raw LM Studio model identifier, Sim treats that unknown identifier as an Ollama model. diff --git a/apps/docs/content/docs/tables/index.mdx b/apps/docs/content/docs/tables/index.mdx index e44ca483464..41b821d995e 100644 --- a/apps/docs/content/docs/tables/index.mdx +++ b/apps/docs/content/docs/tables/index.mdx @@ -24,6 +24,7 @@ Every column has a type, which decides how its values are stored and validated. | **Currency** | An amount in a currency you pick per column | `$1,234.56` | | **Boolean** | `true` or `false` | `true` | | **Date** | A date | `2026-03-16` | +| **Expiration** | An absolute row expiration time, stored as Unix epoch seconds (seconds since January 1, 1970 UTC) | `1773671400` | | **JSON** | An object or array | `{ "tier": "pro" }` | | **Select** | One of a fixed set of options, or several | `Pro` | @@ -31,6 +32,8 @@ Types are enforced as you enter values, so a Number column only takes numbers. A Currency column stores a plain number and renders it in the currency you choose for that column, so filters, sorts, and exports all see the amount itself. Changing a column's currency relabels it — it does not convert the amounts. +A table can have one Expiration column. Adding it enables row expiration; rows with a non-empty expiration value become eligible for deletion after that time passes. Cleanup runs periodically, so actual row removal may happen after the expiration timestamp rather than exactly at it. Deleting the Expiration column disables expiration for the table. Expiration cells use the date editor, while APIs and workflows read and write integer Unix epoch seconds. + ## Editing a table Open the **Tables** section in the sidebar and click **New table** to create one. Add columns from the column header, type into a cell to edit it, and paste rows from a spreadsheet to bulk-load. Filter and sort from the toolbar without changing the underlying data. The editor has full keyboard support; see [keyboard shortcuts](/keyboard-shortcuts). diff --git a/apps/docs/content/docs/tables/using-in-workflows.mdx b/apps/docs/content/docs/tables/using-in-workflows.mdx index b690f3a6a70..1c9fe27fbb1 100644 --- a/apps/docs/content/docs/tables/using-in-workflows.mdx +++ b/apps/docs/content/docs/tables/using-in-workflows.mdx @@ -19,7 +19,7 @@ Throughout this page the running example is a `leads` table with columns `compan A **Table block** performs one operation against one table. The **Operation** dropdown picks the action; the **Table** selector picks the target. The fields below those two change based on the operation you choose. -{/* VISUAL: Table block UI showing the Operation dropdown open, plus the conditional fields that appear for Query Rows (Filter Conditions, Sort Order, Limit, Offset). */} +{/* VISUAL: Table block UI showing the Operation dropdown open, plus the conditional fields that appear for Query Rows (Filter, Order, Columns to Return, Limit, Cursor). */} The operations fall into three groups: @@ -43,7 +43,7 @@ Later blocks read these by name: `` is the array, `` is the array, ` -**Filter Conditions** narrow the result. In the default **Builder** input mode you add rules visually: pick a column, an operator, and a value. Switch the **Input Mode** to **Editor** to write the filter as an object instead, using operators like `$eq`, `$gt`, `$contains`, and `$in`: +**Filter** narrows the result. You can build rules visually - pick a column, an operator, and a value - or write the filter directly as a predicate. One condition names a field, an operator, and a value: ``` -{ status: "unprocessed", createdAt: { $gte: "2026-06-01" } } +{"field": "status", "op": "eq", "value": "unprocessed"} ``` -**Sort Order** orders the result, again visually in Builder mode or as an object in Editor mode, for example `{ createdAt: "desc" }`. **Limit** caps how many rows come back (default 100, max 1000) and **Offset** skips rows for pagination. +Combine conditions with `all` (AND) or `any` (OR), and nest the groups for mixed logic: -{/* VISUAL: Filter Conditions and Sort Order builders, showing a status = unprocessed rule and a createdAt descending sort, with the equivalent Editor-mode object beside them. */} +``` +{"all": [ + {"field": "status", "op": "eq", "value": "unprocessed"}, + {"field": "createdAt", "op": "gte", "value": "2026-06-01"} +]} +``` + +**Order** sorts the result as a list of column/direction pairs, for example `[{"field": "createdAt", "direction": "desc"}]`. **Columns to Return** narrows each row to the fields a downstream step actually needs. **Limit** caps how many rows come back per page, and **Cursor** continues a previous page - see [Paginate large reads](#variations) below. + +{/* VISUAL: Filter and Order builders, showing a status = unprocessed rule and a createdAt descending sort, with the equivalent predicate JSON beside them. */} For a one-off point lookup, use **Get Row by ID** with a single `Row ID`. **Get Schema** returns the table's column definitions, useful when a workflow needs to inspect structure before writing. The full operator list lives in the [Table block reference](/integrations/table). @@ -125,11 +134,11 @@ After the run, the table holds the enriched rows. The next run queries them agai **Iterate row by row.** Wrap a Query → process → update cycle in a [Loop block](/workflows/blocks/loop) to handle one row at a time. This runs sequentially, slower than a batch update but useful when each row needs its own multi-step logic. Inside the loop the Agent reads the current row and an Update Row by ID writes its result. -**Paginate large reads.** Query Rows returns at most 1000 rows, and a page can also end early once its rows reach the response size budget — so a page may come back shorter than your **Limit** even when more rows match. Advance **Offset** by the `rowCount` you actually received, not by the Limit you asked for, and keep going while `nextCursor` is set. Stop when `nextCursor` is null. Stepping by the Limit instead skips whatever a short page left behind. +**Paginate large reads.** Omit **Limit** to get every matching row in one response; the query fails if the result exceeds 5MB, so narrow with a filter rather than guessing a limit. With a **Limit**, a page can end at the limit *or* early once its rows reach the 5MB budget — so a short page does not mean the end. Pass the returned `nextCursor` back as **Cursor** and keep going while it is non-null. Stop only when `nextCursor` is null; never infer completion from the row count. ## Inspecting reads and writes -Every Table block's input and output is recorded in [logs](/logs-debugging). For a Query block, the log shows the filter and sort it sent and the rows it received. For an Update or Insert, it shows the row data written and the count affected. When a write does nothing or a query comes back empty, the log is where you check the filter and the data shape before looking anywhere else. +Every Table block's input and output is recorded in [logs](/logs-debugging). For a Query block, the log shows the filter and order it sent and the rows it received. For an Update or Insert, it shows the row data written and the count affected. When a write does nothing or a query comes back empty, the log is where you check the filter and the data shape before looking anywhere else. ## Next diff --git a/apps/docs/content/docs/workflows/blocks/agent.mdx b/apps/docs/content/docs/workflows/blocks/agent.mdx index 18241d3a3ca..960bd7bab02 100644 --- a/apps/docs/content/docs/workflows/blocks/agent.mdx +++ b/apps/docs/content/docs/workflows/blocks/agent.mdx @@ -109,7 +109,7 @@ Live tool-call chips stream for **OpenAI, Anthropic, Azure Anthropic, Google, Ve | Provider | Streamed thinking | Models | |----------|-------------------|--------| | OpenAI | Summaries only — Requires OpenAI organization verification; falls back to no summaries. | `gpt-5.6-sol`, `gpt-5.6-terra`, `gpt-5.6-luna`, `gpt-5.5-pro`, `gpt-5.5`, `gpt-5.4-pro`, `gpt-5.4`, `gpt-5.4-mini`, `gpt-5.4-nano`, `gpt-5.2-pro`, `gpt-5.2`, `gpt-5.1`, `gpt-5-pro`, `gpt-5`, `gpt-5-mini`, `gpt-5-nano`, `o4-mini`, `o3`, `o3-mini`, `o1` | -| Anthropic | Summaries only — These generations omit full thinking; Sim requests summarized thinking on streaming runs. | `claude-fable-5`, `claude-sonnet-5`, `claude-opus-5`, `claude-opus-4-8`, `claude-opus-4-7`, `claude-opus-4-6`, `claude-sonnet-4-6`, `claude-opus-4-5`, `claude-opus-4-1`, `claude-sonnet-4-5`, `claude-haiku-4-5` | +| Anthropic | Summaries only — These generations omit full thinking; Sim requests summarized thinking on streaming runs. | `claude-fable-5-1`, `claude-fable-5`, `claude-sonnet-5`, `claude-opus-5`, `claude-opus-4-8`, `claude-opus-4-7`, `claude-opus-4-6`, `claude-sonnet-4-6`, `claude-opus-4-5`, `claude-opus-4-1`, `claude-sonnet-4-5`, `claude-haiku-4-5` | | Azure OpenAI | Summaries only — Requires OpenAI organization verification; falls back to no summaries. | `azure/gpt-5.4`, `azure/gpt-5.4-mini`, `azure/gpt-5.4-nano`, `azure/gpt-5.2`, `azure/gpt-5.1`, `azure/gpt-5.1-codex`, `azure/gpt-5`, `azure/gpt-5-mini`, `azure/gpt-5-nano`, `azure/o3`, `azure/o4-mini` | | Azure Anthropic | Summaries only — These generations omit full thinking; Sim requests summarized thinking on streaming runs. | `azure-anthropic/claude-opus-4-6`, `azure-anthropic/claude-opus-4-5`, `azure-anthropic/claude-sonnet-4-5`, `azure-anthropic/claude-opus-4-1`, `azure-anthropic/claude-haiku-4-5` | | Google | Summaries only | `gemini-3.6-flash`, `gemini-3.5-flash-lite`, `gemini-3.5-flash`, `gemini-3.1-pro-preview`, `gemini-3.1-flash-lite`, `gemini-3-flash-preview`, `gemini-2.5-pro`, `gemini-2.5-flash`, `gemini-2.5-flash-lite` | diff --git a/apps/docs/content/docs/workflows/blocks/function.mdx b/apps/docs/content/docs/workflows/blocks/function.mdx index 2b130f2ad7b..b79425b698e 100644 --- a/apps/docs/content/docs/workflows/blocks/function.mdx +++ b/apps/docs/content/docs/workflows/blocks/function.mdx @@ -102,6 +102,49 @@ Sim supplies the rendered heredoc privately while preserving the quoted delimite | --- | --- | | `` | The value your code returns (object, array, string, number, …) | | `` | Anything printed with `console.log()` or `print()` | +| `` | Files your code wrote to `/tmp/sim/outputs`, ready to attach or upload | + +## Files + +**Reading.** Reference a file's `path` and it is mounted for you: + +```python +import pandas as pd + +frame = pd.read_csv() +frame.describe().to_csv('/tmp/sim/outputs/summary.csv') +``` + +`.path` resolves to the file's location on the sandbox filesystem, so any language +can open it — pandas, ffmpeg, a CLI. It is the counterpart to `.base64`, which +inlines the contents instead and works only in JavaScript. Both appear in the +reference dropdown next to `.name` and `.size`. + +**Writing.** Anything your code writes to `/tmp/sim/outputs` comes back as +``, a list of file objects any file-accepting block takes directly — +attach them to an email, upload them to storage, or save them to the workspace with +the File block. There is nothing to turn on. + +The one exception is a call that names an explicit `outputSandboxPath`. That asks +for particular paths to be exported and answers with that export's own result, so +the output directory is not harvested alongside it — choose one or the other rather +than expecting both in the same run. + + +Referencing `.path` runs the block in the remote sandbox, since the local +JavaScript VM has no filesystem — expect the slower start of a remote run even for +plain JavaScript. Referencing the file itself (``, `.name`, +`.url`) does not, and stays local. Up to 20 files come back per run, 50MB total, +nested no more than 11 directories deep; a run that exceeds any of these fails +rather than returning part of what your code wrote. + + + +Returned files live with the execution rather than in your workspace, and a text +file containing a resolved secret value is refused rather than returned — there is +nowhere on an execution file to record that it carries one. Write such a file to a +workspace path instead, or keep the secret out of the output. + ## Language @@ -401,8 +444,8 @@ The lazy `sim.files` and `sim.values` helpers are available only in JavaScript f { question: "What languages does the Function block support?", answer: "JavaScript, Python, and Shell. JavaScript is the default. Python remains a stable saved language choice; Shell and custom Sandbox controls appear when a remote sandbox provider is enabled. Python and Shell execution require that provider." }, { question: "When does code run locally vs. in a sandbox?", answer: "JavaScript without external imports runs in a local isolated sandbox for speed. JavaScript that uses import or require, Python, and Shell run in the configured remote sandbox." }, { question: "Does JavaScript still work without E2B or Daytona?", answer: "Yes. JavaScript without import or require runs in Sim's local isolated VM and does not require a remote provider. JavaScript with external imports, Python, Shell, and custom Sandboxes require E2B or Daytona and fail explicitly when it is unavailable." }, - { question: "How do I reference outputs from other blocks inside my code?", answer: "Use angle-bracket syntax directly, like or , with no quotes around the tag — Sim replaces it with the real value before execution. For environment variables, use double curly braces: {{API_KEY}}." }, - { question: "What does the Function block return?", answer: "Two outputs: result and stdout. Use return in JavaScript, assign __sim_result__ in Python, or print an __SIM_RESULT__= marker in Shell to set result. Ordinary console, print, and command output goes to stdout." }, + { question: "How do I reference outputs from other blocks inside my code?", answer: "Use angle-bracket syntax directly, like or , with no quotes around the tag — Sim replaces it with the real value before execution. For environment variables, use double curly braces: {{API_KEY}}. To read a file, reference its path — mounts it and resolves to a location any language can open." }, + { question: "What does the Function block return?", answer: "Three outputs: result, stdout, and files. Use return in JavaScript, assign __sim_result__ in Python, or print an __SIM_RESULT__= marker in Shell to set result. Ordinary console, print, and command output goes to stdout. Anything your code writes to /tmp/sim/outputs comes back in files as a file object later blocks can accept directly." }, { question: "Can I make HTTP requests from a Function block?", answer: "Yes. fetch() is available in JavaScript with async/await. In Python, use requests or httpx. In Shell, use curl or a CLI available on the selected sandbox." }, { question: "Is there a timeout for Function block execution?", answer: "Yes, a configurable execution timeout. If your code exceeds it, the run is terminated and the block reports an error. Keep this in mind for external calls or heavy processing." }, ]} /> diff --git a/apps/docs/content/docs/workflows/triggers/table.mdx b/apps/docs/content/docs/workflows/triggers/table.mdx index 6217056cfc2..92612212dcd 100644 --- a/apps/docs/content/docs/workflows/triggers/table.mdx +++ b/apps/docs/content/docs/workflows/triggers/table.mdx @@ -7,7 +7,7 @@ import { BlockPreview } from '@/components/workflow-preview' The **Table trigger** runs a workflow when a row is inserted or updated in a [Sim table](/tables). Use it to react to data changes — enrich a row when it's added, or send a follow-up when a status column flips. - + ## Configuration diff --git a/apps/docs/openapi-v2-billing.json b/apps/docs/openapi-v2-billing.json index 8649896cb0e..024a60372e0 100644 --- a/apps/docs/openapi-v2-billing.json +++ b/apps/docs/openapi-v2-billing.json @@ -122,7 +122,8 @@ "knowledge-base", "voice-input", "enrichment", - "voice-output" + "voice-output", + "api-tool" ] } }, @@ -452,7 +453,7 @@ "description": "Human-readable explanation of the error." }, "details": { - "description": "Structured error details. On a `403` whose cause a caller can act on, this carries a `code` from a closed set:\n- `INSUFFICIENT_WORKSPACE_ROLE` — The caller has access to the workspace but its role is below the one this operation requires.\n- `PERSONAL_API_KEYS_DISABLED` — The workspace's organization does not allow personal API keys. Use a workspace API key.\n- `WORKSPACE_KEY_OPERATION_NOT_PERMITTED` — This operation is not available to a workspace-scoped API key. Use a personal API key.\n- `PRINCIPAL_KIND_NOT_PERMITTED` — This operation does not accept the caller’s kind of API key.\n- `ORGANIZATION_MEMBERSHIP_REQUIRED` — The caller is not a member of the organization it named.\n- `ORGANIZATION_ADMIN_REQUIRED` — The caller is a member of the organization but not an admin or owner.\n- `ENTERPRISE_PLAN_REQUIRED` — The organization has no active enterprise subscription.\n- `ORGANIZATION_PLAN_REQUIRED` — The organization has no active organization subscription (Pro for Teams, Max for Teams, or Enterprise).\n- `AUDIT_LOGS_DISABLED` — Audit logging is not enabled for this deployment.\n- `SKILL_EDITOR_ACCESS_REQUIRED` — The caller can write in the workspace but is not an editor of this skill.\n- `SECRET_ADMIN_ACCESS_REQUIRED` — The caller can write in the workspace but is not an admin of this secret. Ask a workspace admin, or someone holding admin on the secret, to grant access or set the value.\n- `WORKSPACE_RESOURCE_LIMIT_REACHED` — The workspace already holds the maximum number of resources of this kind. Delete one, or contact Sim to raise the limit; the message names the ceiling.\n- `PUBLIC_SHARING_NOT_ALLOWED` — The workspace's organization does not permit sharing this resource publicly. An organization admin controls the policy.\n- `CREDENTIAL_ADMIN_ACCESS_REQUIRED` — The caller can reach the workspace but cannot administer this credential.\n- `MCP_SERVER_URL_NOT_ALLOWED` — The supplied MCP server URL is outside the allowed domains or resolves to an internal address.\n- `WORKSPACE_PLAN_CAPABILITY_REQUIRED` — The workspace's plan does not include a capability this request depends on. The message names the capability; upgrading the workspace's plan is the remedy.\n- `CHAT_AUTH_MODE_NOT_PERMITTED` — The workspace's permission group does not allow the chat authentication mode the request selected. A mode already saved on the deployment may still be re-saved; changing to a disallowed one cannot.\n- `CONNECTOR_MANAGED_RESOURCE_READ_ONLY` — This resource is managed by a knowledge base connector and cannot be edited directly. Change it at the source and re-sync, or exclude the document from the connector." + "description": "Structured error details. On a `403` whose cause a caller can act on, this carries a `code` from a closed set:\n- `INSUFFICIENT_WORKSPACE_ROLE` — The caller has access to the workspace but its role is below the one this operation requires.\n- `PERSONAL_API_KEYS_DISABLED` — The workspace's organization does not allow personal API keys. Use a workspace API key.\n- `WORKSPACE_KEY_OPERATION_NOT_PERMITTED` — This operation is not available to a workspace-scoped API key. Use a personal API key.\n- `PRINCIPAL_KIND_NOT_PERMITTED` — This operation does not accept the caller’s kind of API key.\n- `ORGANIZATION_MEMBERSHIP_REQUIRED` — The caller is not a member of the organization it named.\n- `ORGANIZATION_ADMIN_REQUIRED` — The caller is a member of the organization but not an admin or owner.\n- `ENTERPRISE_PLAN_REQUIRED` — The organization has no active enterprise subscription.\n- `ORGANIZATION_PLAN_REQUIRED` — The organization has no active organization subscription (Pro for Teams, Max for Teams, or Enterprise).\n- `AUDIT_LOGS_DISABLED` — Audit logging is not enabled for this deployment.\n- `SKILL_EDITOR_ACCESS_REQUIRED` — The caller can write in the workspace but is not an editor of this skill.\n- `SECRET_ADMIN_ACCESS_REQUIRED` — The caller can write in the workspace but is not an admin of this secret. Ask a workspace admin, or someone holding admin on the secret, to grant access or set the value.\n- `WORKSPACE_RESOURCE_LIMIT_REACHED` — The workspace already holds the maximum number of resources of this kind. Delete one, or contact Sim to raise the limit; the message names the ceiling.\n- `PUBLIC_SHARING_NOT_ALLOWED` — The workspace's organization does not permit sharing this resource publicly. An organization admin controls the policy.\n- `CREDENTIAL_ADMIN_ACCESS_REQUIRED` — The caller can reach the workspace but cannot administer this credential.\n- `MCP_SERVER_URL_NOT_ALLOWED` — The supplied MCP server URL is outside the allowed domains or resolves to an internal address.\n- `WORKSPACE_PLAN_CAPABILITY_REQUIRED` — The workspace's plan does not include a capability this request depends on. The message names the capability; upgrading the workspace's plan is the remedy.\n- `CHAT_AUTH_MODE_NOT_PERMITTED` — The workspace's permission group does not allow the chat authentication mode the request selected. A mode already saved on the deployment may still be re-saved; changing to a disallowed one cannot.\n- `CONNECTOR_MANAGED_RESOURCE_READ_ONLY` — This resource is managed by a knowledge base connector and cannot be edited directly. Change it at the source and re-sync, or exclude the document from the connector.\n- `PERMISSION_GROUP_CAPABILITY_BLOCKED` — The caller's permission group does not allow this capability. The message names it; an organization admin controls the group.\n- `INTEGRATION_NOT_ALLOWED` — The integration this request names is outside the workspace's allowed set. An organization admin controls the permission group's integration allowlist, and a self-hosted deployment can narrow it further with ALLOWED_INTEGRATIONS." } }, "required": ["code", "message"], @@ -636,7 +637,8 @@ "knowledge-base", "voice-input", "enrichment", - "voice-output" + "voice-output", + "api-tool" ], "description": "Product surface that consumed the credits." }, diff --git a/apps/docs/openapi-v2-files-audit.json b/apps/docs/openapi-v2-files-audit.json index 696ff4e3623..651d2d3fe41 100644 --- a/apps/docs/openapi-v2-files-audit.json +++ b/apps/docs/openapi-v2-files-audit.json @@ -2793,7 +2793,7 @@ "description": "Human-readable explanation of the error." }, "details": { - "description": "Structured error details. On a `403` whose cause a caller can act on, this carries a `code` from a closed set:\n- `INSUFFICIENT_WORKSPACE_ROLE` — The caller has access to the workspace but its role is below the one this operation requires.\n- `PERSONAL_API_KEYS_DISABLED` — The workspace's organization does not allow personal API keys. Use a workspace API key.\n- `WORKSPACE_KEY_OPERATION_NOT_PERMITTED` — This operation is not available to a workspace-scoped API key. Use a personal API key.\n- `PRINCIPAL_KIND_NOT_PERMITTED` — This operation does not accept the caller’s kind of API key.\n- `ORGANIZATION_MEMBERSHIP_REQUIRED` — The caller is not a member of the organization it named.\n- `ORGANIZATION_ADMIN_REQUIRED` — The caller is a member of the organization but not an admin or owner.\n- `ENTERPRISE_PLAN_REQUIRED` — The organization has no active enterprise subscription.\n- `ORGANIZATION_PLAN_REQUIRED` — The organization has no active organization subscription (Pro for Teams, Max for Teams, or Enterprise).\n- `AUDIT_LOGS_DISABLED` — Audit logging is not enabled for this deployment.\n- `SKILL_EDITOR_ACCESS_REQUIRED` — The caller can write in the workspace but is not an editor of this skill.\n- `SECRET_ADMIN_ACCESS_REQUIRED` — The caller can write in the workspace but is not an admin of this secret. Ask a workspace admin, or someone holding admin on the secret, to grant access or set the value.\n- `WORKSPACE_RESOURCE_LIMIT_REACHED` — The workspace already holds the maximum number of resources of this kind. Delete one, or contact Sim to raise the limit; the message names the ceiling.\n- `PUBLIC_SHARING_NOT_ALLOWED` — The workspace's organization does not permit sharing this resource publicly. An organization admin controls the policy.\n- `CREDENTIAL_ADMIN_ACCESS_REQUIRED` — The caller can reach the workspace but cannot administer this credential.\n- `MCP_SERVER_URL_NOT_ALLOWED` — The supplied MCP server URL is outside the allowed domains or resolves to an internal address.\n- `WORKSPACE_PLAN_CAPABILITY_REQUIRED` — The workspace's plan does not include a capability this request depends on. The message names the capability; upgrading the workspace's plan is the remedy.\n- `CHAT_AUTH_MODE_NOT_PERMITTED` — The workspace's permission group does not allow the chat authentication mode the request selected. A mode already saved on the deployment may still be re-saved; changing to a disallowed one cannot.\n- `CONNECTOR_MANAGED_RESOURCE_READ_ONLY` — This resource is managed by a knowledge base connector and cannot be edited directly. Change it at the source and re-sync, or exclude the document from the connector." + "description": "Structured error details. On a `403` whose cause a caller can act on, this carries a `code` from a closed set:\n- `INSUFFICIENT_WORKSPACE_ROLE` — The caller has access to the workspace but its role is below the one this operation requires.\n- `PERSONAL_API_KEYS_DISABLED` — The workspace's organization does not allow personal API keys. Use a workspace API key.\n- `WORKSPACE_KEY_OPERATION_NOT_PERMITTED` — This operation is not available to a workspace-scoped API key. Use a personal API key.\n- `PRINCIPAL_KIND_NOT_PERMITTED` — This operation does not accept the caller’s kind of API key.\n- `ORGANIZATION_MEMBERSHIP_REQUIRED` — The caller is not a member of the organization it named.\n- `ORGANIZATION_ADMIN_REQUIRED` — The caller is a member of the organization but not an admin or owner.\n- `ENTERPRISE_PLAN_REQUIRED` — The organization has no active enterprise subscription.\n- `ORGANIZATION_PLAN_REQUIRED` — The organization has no active organization subscription (Pro for Teams, Max for Teams, or Enterprise).\n- `AUDIT_LOGS_DISABLED` — Audit logging is not enabled for this deployment.\n- `SKILL_EDITOR_ACCESS_REQUIRED` — The caller can write in the workspace but is not an editor of this skill.\n- `SECRET_ADMIN_ACCESS_REQUIRED` — The caller can write in the workspace but is not an admin of this secret. Ask a workspace admin, or someone holding admin on the secret, to grant access or set the value.\n- `WORKSPACE_RESOURCE_LIMIT_REACHED` — The workspace already holds the maximum number of resources of this kind. Delete one, or contact Sim to raise the limit; the message names the ceiling.\n- `PUBLIC_SHARING_NOT_ALLOWED` — The workspace's organization does not permit sharing this resource publicly. An organization admin controls the policy.\n- `CREDENTIAL_ADMIN_ACCESS_REQUIRED` — The caller can reach the workspace but cannot administer this credential.\n- `MCP_SERVER_URL_NOT_ALLOWED` — The supplied MCP server URL is outside the allowed domains or resolves to an internal address.\n- `WORKSPACE_PLAN_CAPABILITY_REQUIRED` — The workspace's plan does not include a capability this request depends on. The message names the capability; upgrading the workspace's plan is the remedy.\n- `CHAT_AUTH_MODE_NOT_PERMITTED` — The workspace's permission group does not allow the chat authentication mode the request selected. A mode already saved on the deployment may still be re-saved; changing to a disallowed one cannot.\n- `CONNECTOR_MANAGED_RESOURCE_READ_ONLY` — This resource is managed by a knowledge base connector and cannot be edited directly. Change it at the source and re-sync, or exclude the document from the connector.\n- `PERMISSION_GROUP_CAPABILITY_BLOCKED` — The caller's permission group does not allow this capability. The message names it; an organization admin controls the group.\n- `INTEGRATION_NOT_ALLOWED` — The integration this request names is outside the workspace's allowed set. An organization admin controls the permission group's integration allowlist, and a self-hosted deployment can narrow it further with ALLOWED_INTEGRATIONS." } }, "required": ["code", "message"], diff --git a/apps/docs/openapi-v2-knowledge.json b/apps/docs/openapi-v2-knowledge.json index 38eecd23994..15357b66a8b 100644 --- a/apps/docs/openapi-v2-knowledge.json +++ b/apps/docs/openapi-v2-knowledge.json @@ -4499,7 +4499,7 @@ "description": "Human-readable explanation of the error." }, "details": { - "description": "Structured error details. On a `403` whose cause a caller can act on, this carries a `code` from a closed set:\n- `INSUFFICIENT_WORKSPACE_ROLE` — The caller has access to the workspace but its role is below the one this operation requires.\n- `PERSONAL_API_KEYS_DISABLED` — The workspace's organization does not allow personal API keys. Use a workspace API key.\n- `WORKSPACE_KEY_OPERATION_NOT_PERMITTED` — This operation is not available to a workspace-scoped API key. Use a personal API key.\n- `PRINCIPAL_KIND_NOT_PERMITTED` — This operation does not accept the caller’s kind of API key.\n- `ORGANIZATION_MEMBERSHIP_REQUIRED` — The caller is not a member of the organization it named.\n- `ORGANIZATION_ADMIN_REQUIRED` — The caller is a member of the organization but not an admin or owner.\n- `ENTERPRISE_PLAN_REQUIRED` — The organization has no active enterprise subscription.\n- `ORGANIZATION_PLAN_REQUIRED` — The organization has no active organization subscription (Pro for Teams, Max for Teams, or Enterprise).\n- `AUDIT_LOGS_DISABLED` — Audit logging is not enabled for this deployment.\n- `SKILL_EDITOR_ACCESS_REQUIRED` — The caller can write in the workspace but is not an editor of this skill.\n- `SECRET_ADMIN_ACCESS_REQUIRED` — The caller can write in the workspace but is not an admin of this secret. Ask a workspace admin, or someone holding admin on the secret, to grant access or set the value.\n- `WORKSPACE_RESOURCE_LIMIT_REACHED` — The workspace already holds the maximum number of resources of this kind. Delete one, or contact Sim to raise the limit; the message names the ceiling.\n- `PUBLIC_SHARING_NOT_ALLOWED` — The workspace's organization does not permit sharing this resource publicly. An organization admin controls the policy.\n- `CREDENTIAL_ADMIN_ACCESS_REQUIRED` — The caller can reach the workspace but cannot administer this credential.\n- `MCP_SERVER_URL_NOT_ALLOWED` — The supplied MCP server URL is outside the allowed domains or resolves to an internal address.\n- `WORKSPACE_PLAN_CAPABILITY_REQUIRED` — The workspace's plan does not include a capability this request depends on. The message names the capability; upgrading the workspace's plan is the remedy.\n- `CHAT_AUTH_MODE_NOT_PERMITTED` — The workspace's permission group does not allow the chat authentication mode the request selected. A mode already saved on the deployment may still be re-saved; changing to a disallowed one cannot.\n- `CONNECTOR_MANAGED_RESOURCE_READ_ONLY` — This resource is managed by a knowledge base connector and cannot be edited directly. Change it at the source and re-sync, or exclude the document from the connector." + "description": "Structured error details. On a `403` whose cause a caller can act on, this carries a `code` from a closed set:\n- `INSUFFICIENT_WORKSPACE_ROLE` — The caller has access to the workspace but its role is below the one this operation requires.\n- `PERSONAL_API_KEYS_DISABLED` — The workspace's organization does not allow personal API keys. Use a workspace API key.\n- `WORKSPACE_KEY_OPERATION_NOT_PERMITTED` — This operation is not available to a workspace-scoped API key. Use a personal API key.\n- `PRINCIPAL_KIND_NOT_PERMITTED` — This operation does not accept the caller’s kind of API key.\n- `ORGANIZATION_MEMBERSHIP_REQUIRED` — The caller is not a member of the organization it named.\n- `ORGANIZATION_ADMIN_REQUIRED` — The caller is a member of the organization but not an admin or owner.\n- `ENTERPRISE_PLAN_REQUIRED` — The organization has no active enterprise subscription.\n- `ORGANIZATION_PLAN_REQUIRED` — The organization has no active organization subscription (Pro for Teams, Max for Teams, or Enterprise).\n- `AUDIT_LOGS_DISABLED` — Audit logging is not enabled for this deployment.\n- `SKILL_EDITOR_ACCESS_REQUIRED` — The caller can write in the workspace but is not an editor of this skill.\n- `SECRET_ADMIN_ACCESS_REQUIRED` — The caller can write in the workspace but is not an admin of this secret. Ask a workspace admin, or someone holding admin on the secret, to grant access or set the value.\n- `WORKSPACE_RESOURCE_LIMIT_REACHED` — The workspace already holds the maximum number of resources of this kind. Delete one, or contact Sim to raise the limit; the message names the ceiling.\n- `PUBLIC_SHARING_NOT_ALLOWED` — The workspace's organization does not permit sharing this resource publicly. An organization admin controls the policy.\n- `CREDENTIAL_ADMIN_ACCESS_REQUIRED` — The caller can reach the workspace but cannot administer this credential.\n- `MCP_SERVER_URL_NOT_ALLOWED` — The supplied MCP server URL is outside the allowed domains or resolves to an internal address.\n- `WORKSPACE_PLAN_CAPABILITY_REQUIRED` — The workspace's plan does not include a capability this request depends on. The message names the capability; upgrading the workspace's plan is the remedy.\n- `CHAT_AUTH_MODE_NOT_PERMITTED` — The workspace's permission group does not allow the chat authentication mode the request selected. A mode already saved on the deployment may still be re-saved; changing to a disallowed one cannot.\n- `CONNECTOR_MANAGED_RESOURCE_READ_ONLY` — This resource is managed by a knowledge base connector and cannot be edited directly. Change it at the source and re-sync, or exclude the document from the connector.\n- `PERMISSION_GROUP_CAPABILITY_BLOCKED` — The caller's permission group does not allow this capability. The message names it; an organization admin controls the group.\n- `INTEGRATION_NOT_ALLOWED` — The integration this request names is outside the workspace's allowed set. An organization admin controls the permission group's integration allowlist, and a self-hosted deployment can narrow it further with ALLOWED_INTEGRATIONS." } }, "required": ["code", "message"], diff --git a/apps/docs/openapi-v2-logs.json b/apps/docs/openapi-v2-logs.json index 3f69f6dd896..2b65611a2fb 100644 --- a/apps/docs/openapi-v2-logs.json +++ b/apps/docs/openapi-v2-logs.json @@ -787,7 +787,7 @@ "description": "Human-readable explanation of the error." }, "details": { - "description": "Structured error details. On a `403` whose cause a caller can act on, this carries a `code` from a closed set:\n- `INSUFFICIENT_WORKSPACE_ROLE` — The caller has access to the workspace but its role is below the one this operation requires.\n- `PERSONAL_API_KEYS_DISABLED` — The workspace's organization does not allow personal API keys. Use a workspace API key.\n- `WORKSPACE_KEY_OPERATION_NOT_PERMITTED` — This operation is not available to a workspace-scoped API key. Use a personal API key.\n- `PRINCIPAL_KIND_NOT_PERMITTED` — This operation does not accept the caller’s kind of API key.\n- `ORGANIZATION_MEMBERSHIP_REQUIRED` — The caller is not a member of the organization it named.\n- `ORGANIZATION_ADMIN_REQUIRED` — The caller is a member of the organization but not an admin or owner.\n- `ENTERPRISE_PLAN_REQUIRED` — The organization has no active enterprise subscription.\n- `ORGANIZATION_PLAN_REQUIRED` — The organization has no active organization subscription (Pro for Teams, Max for Teams, or Enterprise).\n- `AUDIT_LOGS_DISABLED` — Audit logging is not enabled for this deployment.\n- `SKILL_EDITOR_ACCESS_REQUIRED` — The caller can write in the workspace but is not an editor of this skill.\n- `SECRET_ADMIN_ACCESS_REQUIRED` — The caller can write in the workspace but is not an admin of this secret. Ask a workspace admin, or someone holding admin on the secret, to grant access or set the value.\n- `WORKSPACE_RESOURCE_LIMIT_REACHED` — The workspace already holds the maximum number of resources of this kind. Delete one, or contact Sim to raise the limit; the message names the ceiling.\n- `PUBLIC_SHARING_NOT_ALLOWED` — The workspace's organization does not permit sharing this resource publicly. An organization admin controls the policy.\n- `CREDENTIAL_ADMIN_ACCESS_REQUIRED` — The caller can reach the workspace but cannot administer this credential.\n- `MCP_SERVER_URL_NOT_ALLOWED` — The supplied MCP server URL is outside the allowed domains or resolves to an internal address.\n- `WORKSPACE_PLAN_CAPABILITY_REQUIRED` — The workspace's plan does not include a capability this request depends on. The message names the capability; upgrading the workspace's plan is the remedy.\n- `CHAT_AUTH_MODE_NOT_PERMITTED` — The workspace's permission group does not allow the chat authentication mode the request selected. A mode already saved on the deployment may still be re-saved; changing to a disallowed one cannot.\n- `CONNECTOR_MANAGED_RESOURCE_READ_ONLY` — This resource is managed by a knowledge base connector and cannot be edited directly. Change it at the source and re-sync, or exclude the document from the connector." + "description": "Structured error details. On a `403` whose cause a caller can act on, this carries a `code` from a closed set:\n- `INSUFFICIENT_WORKSPACE_ROLE` — The caller has access to the workspace but its role is below the one this operation requires.\n- `PERSONAL_API_KEYS_DISABLED` — The workspace's organization does not allow personal API keys. Use a workspace API key.\n- `WORKSPACE_KEY_OPERATION_NOT_PERMITTED` — This operation is not available to a workspace-scoped API key. Use a personal API key.\n- `PRINCIPAL_KIND_NOT_PERMITTED` — This operation does not accept the caller’s kind of API key.\n- `ORGANIZATION_MEMBERSHIP_REQUIRED` — The caller is not a member of the organization it named.\n- `ORGANIZATION_ADMIN_REQUIRED` — The caller is a member of the organization but not an admin or owner.\n- `ENTERPRISE_PLAN_REQUIRED` — The organization has no active enterprise subscription.\n- `ORGANIZATION_PLAN_REQUIRED` — The organization has no active organization subscription (Pro for Teams, Max for Teams, or Enterprise).\n- `AUDIT_LOGS_DISABLED` — Audit logging is not enabled for this deployment.\n- `SKILL_EDITOR_ACCESS_REQUIRED` — The caller can write in the workspace but is not an editor of this skill.\n- `SECRET_ADMIN_ACCESS_REQUIRED` — The caller can write in the workspace but is not an admin of this secret. Ask a workspace admin, or someone holding admin on the secret, to grant access or set the value.\n- `WORKSPACE_RESOURCE_LIMIT_REACHED` — The workspace already holds the maximum number of resources of this kind. Delete one, or contact Sim to raise the limit; the message names the ceiling.\n- `PUBLIC_SHARING_NOT_ALLOWED` — The workspace's organization does not permit sharing this resource publicly. An organization admin controls the policy.\n- `CREDENTIAL_ADMIN_ACCESS_REQUIRED` — The caller can reach the workspace but cannot administer this credential.\n- `MCP_SERVER_URL_NOT_ALLOWED` — The supplied MCP server URL is outside the allowed domains or resolves to an internal address.\n- `WORKSPACE_PLAN_CAPABILITY_REQUIRED` — The workspace's plan does not include a capability this request depends on. The message names the capability; upgrading the workspace's plan is the remedy.\n- `CHAT_AUTH_MODE_NOT_PERMITTED` — The workspace's permission group does not allow the chat authentication mode the request selected. A mode already saved on the deployment may still be re-saved; changing to a disallowed one cannot.\n- `CONNECTOR_MANAGED_RESOURCE_READ_ONLY` — This resource is managed by a knowledge base connector and cannot be edited directly. Change it at the source and re-sync, or exclude the document from the connector.\n- `PERMISSION_GROUP_CAPABILITY_BLOCKED` — The caller's permission group does not allow this capability. The message names it; an organization admin controls the group.\n- `INTEGRATION_NOT_ALLOWED` — The integration this request names is outside the workspace's allowed set. An organization admin controls the permission group's integration allowlist, and a self-hosted deployment can narrow it further with ALLOWED_INTEGRATIONS." } }, "required": ["code", "message"], @@ -1344,6 +1344,19 @@ ], "description": "Files the run produced, or null when none are recorded. Only the run's own output files appear; input attachments a caller supplied are addressed through the files API instead." }, + "executedByEmail": { + "anyOf": [ + { + "type": "string", + "format": "email", + "pattern": "^(?!\\.)(?!.*\\.\\.)([A-Za-z0-9_'+\\-\\.]*)[A-Za-z0-9_+-]@([A-Za-z0-9][A-Za-z0-9\\-]*\\.)+[A-Za-z]{2,}$" + }, + { + "type": "null" + } + ], + "description": "Email of the identity the run executed as: the caller for an interactive or personal-API-key run, and the workspace billing account for a schedule, webhook, deployed chat, or public API call. Null when the run failed before an identity was resolved." + }, "workflow": { "type": "object", "properties": { @@ -1398,7 +1411,8 @@ "type": "null" } ], - "description": "Workflow owner email, or null when unavailable." + "description": "Deprecated — use the run-level `executedByEmail` instead. Email of the workflow's current owner, or null when unavailable. This is a property of the workflow as it stands today, not of the run: it changes when workflow ownership is reassigned, and the owner is not the identity a background run executes as.", + "deprecated": true }, "workspaceId": { "anyOf": [ @@ -1577,6 +1591,7 @@ "endedAt", "totalDurationMs", "files", + "executedByEmail", "workflow", "workflowState", "traceSpans", @@ -1614,6 +1629,7 @@ "endedAt": "2026-01-15T10:30:01.250Z", "totalDurationMs": 1250, "files": null, + "executedByEmail": "billing@example.com", "workflow": { "id": "3b1f7c92-8d4e-4a6b-9c0d-5e2f8a714b36", "name": "Customer Support Agent", diff --git a/apps/docs/openapi-v2-resources.json b/apps/docs/openapi-v2-resources.json index 402e5d1dfdd..b94dd9b6a0d 100644 --- a/apps/docs/openapi-v2-resources.json +++ b/apps/docs/openapi-v2-resources.json @@ -4039,6 +4039,89 @@ } } }, + "/api/v2/tools/{toolId}/execute": { + "post": { + "operationId": "executeTool", + "summary": "Run Tool", + "description": "Run one built-in tool and return what it produced. Supply `input` using the parameter ids `GET /api/v2/tools/{toolId}` publishes; Sim resolves the credential named by `credentialId`, injects a hosted API key for the tools it supplies one for, and substitutes environment-variable references, so the request carries arguments rather than secrets. A parameter the tool marks `user-only` also accepts `{{VAR_NAME}}` as its whole value, resolved server-side against the workspace environment; every other value is sent verbatim, so a literal secret passes through untouched. A tool that runs and refuses is a `200` carrying `status: \"failed\"` and the reason — the error envelope is reserved for failures of this API, not of the third party. A tool the workspace's visible blocks do not expose answers `404` identically to one that does not exist; one whose integration the workspace does not permit answers `403` with `error.details.code` `INTEGRATION_NOT_ALLOWED`. Hosted-key spend this call incurs is billed to the workspace. A workspace API key is rejected with `403`; use a personal API key.", + "tags": ["Catalog"], + "parameters": [ + { + "name": "toolId", + "in": "path", + "required": true, + "description": "Tool identifier. An unversioned name resolves to the newest version, and the response echoes the resolved id.", + "schema": { + "type": "string", + "minLength": 1, + "maxLength": 255, + "description": "Tool identifier. An unversioned name resolves to the newest version, and the response echoes the resolved id." + } + } + ], + "requestBody": { + "required": true, + "description": "Workspace, arguments, and the credential to authenticate with.", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ExecuteToolRequest" + } + } + } + }, + "responses": { + "200": { + "description": "The outcome of the tool call.", + "headers": { + "X-RateLimit-Limit": { + "$ref": "#/components/headers/X-RateLimit-Limit" + }, + "X-RateLimit-Remaining": { + "$ref": "#/components/headers/X-RateLimit-Remaining" + }, + "X-RateLimit-Reset": { + "$ref": "#/components/headers/X-RateLimit-Reset" + } + }, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ExecuteToolResponse" + } + } + } + }, + "400": { + "$ref": "#/components/responses/BadRequest" + }, + "401": { + "$ref": "#/components/responses/Unauthorized" + }, + "403": { + "$ref": "#/components/responses/Forbidden" + }, + "404": { + "$ref": "#/components/responses/NotFound" + }, + "413": { + "$ref": "#/components/responses/PayloadTooLarge" + }, + "415": { + "$ref": "#/components/responses/UnsupportedMediaType" + }, + "429": { + "$ref": "#/components/responses/RateLimited" + }, + "500": { + "$ref": "#/components/responses/InternalError" + }, + "503": { + "$ref": "#/components/responses/ServiceUnavailable" + } + } + } + }, "/api/v2/connector-types": { "get": { "operationId": "listConnectorTypes", @@ -4372,7 +4455,7 @@ "description": "Human-readable explanation of the error." }, "details": { - "description": "Structured error details. On a `403` whose cause a caller can act on, this carries a `code` from a closed set:\n- `INSUFFICIENT_WORKSPACE_ROLE` — The caller has access to the workspace but its role is below the one this operation requires.\n- `PERSONAL_API_KEYS_DISABLED` — The workspace's organization does not allow personal API keys. Use a workspace API key.\n- `WORKSPACE_KEY_OPERATION_NOT_PERMITTED` — This operation is not available to a workspace-scoped API key. Use a personal API key.\n- `PRINCIPAL_KIND_NOT_PERMITTED` — This operation does not accept the caller’s kind of API key.\n- `ORGANIZATION_MEMBERSHIP_REQUIRED` — The caller is not a member of the organization it named.\n- `ORGANIZATION_ADMIN_REQUIRED` — The caller is a member of the organization but not an admin or owner.\n- `ENTERPRISE_PLAN_REQUIRED` — The organization has no active enterprise subscription.\n- `ORGANIZATION_PLAN_REQUIRED` — The organization has no active organization subscription (Pro for Teams, Max for Teams, or Enterprise).\n- `AUDIT_LOGS_DISABLED` — Audit logging is not enabled for this deployment.\n- `SKILL_EDITOR_ACCESS_REQUIRED` — The caller can write in the workspace but is not an editor of this skill.\n- `SECRET_ADMIN_ACCESS_REQUIRED` — The caller can write in the workspace but is not an admin of this secret. Ask a workspace admin, or someone holding admin on the secret, to grant access or set the value.\n- `WORKSPACE_RESOURCE_LIMIT_REACHED` — The workspace already holds the maximum number of resources of this kind. Delete one, or contact Sim to raise the limit; the message names the ceiling.\n- `PUBLIC_SHARING_NOT_ALLOWED` — The workspace's organization does not permit sharing this resource publicly. An organization admin controls the policy.\n- `CREDENTIAL_ADMIN_ACCESS_REQUIRED` — The caller can reach the workspace but cannot administer this credential.\n- `MCP_SERVER_URL_NOT_ALLOWED` — The supplied MCP server URL is outside the allowed domains or resolves to an internal address.\n- `WORKSPACE_PLAN_CAPABILITY_REQUIRED` — The workspace's plan does not include a capability this request depends on. The message names the capability; upgrading the workspace's plan is the remedy.\n- `CHAT_AUTH_MODE_NOT_PERMITTED` — The workspace's permission group does not allow the chat authentication mode the request selected. A mode already saved on the deployment may still be re-saved; changing to a disallowed one cannot.\n- `CONNECTOR_MANAGED_RESOURCE_READ_ONLY` — This resource is managed by a knowledge base connector and cannot be edited directly. Change it at the source and re-sync, or exclude the document from the connector." + "description": "Structured error details. On a `403` whose cause a caller can act on, this carries a `code` from a closed set:\n- `INSUFFICIENT_WORKSPACE_ROLE` — The caller has access to the workspace but its role is below the one this operation requires.\n- `PERSONAL_API_KEYS_DISABLED` — The workspace's organization does not allow personal API keys. Use a workspace API key.\n- `WORKSPACE_KEY_OPERATION_NOT_PERMITTED` — This operation is not available to a workspace-scoped API key. Use a personal API key.\n- `PRINCIPAL_KIND_NOT_PERMITTED` — This operation does not accept the caller’s kind of API key.\n- `ORGANIZATION_MEMBERSHIP_REQUIRED` — The caller is not a member of the organization it named.\n- `ORGANIZATION_ADMIN_REQUIRED` — The caller is a member of the organization but not an admin or owner.\n- `ENTERPRISE_PLAN_REQUIRED` — The organization has no active enterprise subscription.\n- `ORGANIZATION_PLAN_REQUIRED` — The organization has no active organization subscription (Pro for Teams, Max for Teams, or Enterprise).\n- `AUDIT_LOGS_DISABLED` — Audit logging is not enabled for this deployment.\n- `SKILL_EDITOR_ACCESS_REQUIRED` — The caller can write in the workspace but is not an editor of this skill.\n- `SECRET_ADMIN_ACCESS_REQUIRED` — The caller can write in the workspace but is not an admin of this secret. Ask a workspace admin, or someone holding admin on the secret, to grant access or set the value.\n- `WORKSPACE_RESOURCE_LIMIT_REACHED` — The workspace already holds the maximum number of resources of this kind. Delete one, or contact Sim to raise the limit; the message names the ceiling.\n- `PUBLIC_SHARING_NOT_ALLOWED` — The workspace's organization does not permit sharing this resource publicly. An organization admin controls the policy.\n- `CREDENTIAL_ADMIN_ACCESS_REQUIRED` — The caller can reach the workspace but cannot administer this credential.\n- `MCP_SERVER_URL_NOT_ALLOWED` — The supplied MCP server URL is outside the allowed domains or resolves to an internal address.\n- `WORKSPACE_PLAN_CAPABILITY_REQUIRED` — The workspace's plan does not include a capability this request depends on. The message names the capability; upgrading the workspace's plan is the remedy.\n- `CHAT_AUTH_MODE_NOT_PERMITTED` — The workspace's permission group does not allow the chat authentication mode the request selected. A mode already saved on the deployment may still be re-saved; changing to a disallowed one cannot.\n- `CONNECTOR_MANAGED_RESOURCE_READ_ONLY` — This resource is managed by a knowledge base connector and cannot be edited directly. Change it at the source and re-sync, or exclude the document from the connector.\n- `PERMISSION_GROUP_CAPABILITY_BLOCKED` — The caller's permission group does not allow this capability. The message names it; an organization admin controls the group.\n- `INTEGRATION_NOT_ALLOWED` — The integration this request names is outside the workspace's allowed set. An organization admin controls the permission group's integration allowlist, and a self-hosted deployment can narrow it further with ALLOWED_INTEGRATIONS." } }, "required": ["code", "message"], @@ -9470,6 +9553,119 @@ } ] }, + "V2ToolExecution": { + "type": "object", + "properties": { + "toolId": { + "type": "string", + "description": "Tool that ran. An unversioned name resolves to the newest version visible in the workspace, so this can differ from the id in the path." + }, + "status": { + "type": "string", + "enum": ["succeeded", "failed"], + "description": "Whether the tool reported success. A failed tool call is still a 200." + }, + "output": { + "description": "Whatever the tool produced, shaped by its declared outputs." + }, + "error": { + "anyOf": [ + { + "type": "object", + "properties": { + "message": { + "type": "string", + "description": "Why the tool call did not succeed." + } + }, + "required": ["message"], + "additionalProperties": false + }, + { + "type": "null" + } + ], + "description": "Populated only when `status` is `failed`." + } + }, + "required": ["toolId", "status", "output", "error"], + "additionalProperties": false, + "title": "Tool execution", + "description": "The result of running one built-in tool." + }, + "ExecuteToolResponse": { + "type": "object", + "properties": { + "data": { + "description": "Response data.", + "$ref": "#/components/schemas/V2ToolExecution" + } + }, + "required": ["data"], + "additionalProperties": false, + "title": "Run tool response", + "description": "What the tool produced, or why it did not succeed.", + "examples": [ + { + "data": { + "toolId": "slack_message", + "status": "succeeded", + "output": { + "ts": "1718191234.004500" + }, + "error": null + } + } + ] + }, + "ExecuteToolRequest": { + "type": "object", + "properties": { + "workspaceId": { + "type": "string", + "minLength": 1, + "maxLength": 128, + "description": "Workspace whose integration allowlist, credentials, and environment variables govern this call." + }, + "input": { + "default": {}, + "description": "Arguments for the tool, keyed by the parameter ids the tool catalog publishes for it. A parameter whose visibility is `user-only` also accepts an environment-variable reference written as the whole value, `{{VAR_NAME}}`, resolved server-side against the workspace environment; any other value is sent verbatim.", + "type": "object", + "propertyNames": { + "type": "string" + }, + "additionalProperties": { + "description": "One argument value. Its shape is declared by the tool parameter." + } + }, + "credentialId": { + "description": "Credential to authenticate with. Required when the tool declares an OAuth requirement; the workspace credentials list names the candidates.", + "type": "string", + "minLength": 1, + "maxLength": 255 + }, + "timeoutSeconds": { + "description": "How long to wait for the tool before abandoning the call.", + "type": "integer", + "minimum": 1, + "maximum": 300 + } + }, + "required": ["workspaceId"], + "additionalProperties": false, + "title": "Run tool request", + "description": "Workspace, arguments, and the credential to authenticate with.", + "examples": [ + { + "workspaceId": "a91c4b2e-6d3f-4e8a-b5c7-0d9e2f1a8c64", + "input": { + "channel": "C0123456789", + "text": "Deploy finished." + }, + "credentialId": "cred_01J8ZK3QW4M6X2R9T7B5C0V2" + } + ] + }, "V2ConnectorType": { "type": "object", "properties": { diff --git a/apps/docs/openapi-v2-tables.json b/apps/docs/openapi-v2-tables.json index 359c8e1d4b9..d91748c097d 100644 --- a/apps/docs/openapi-v2-tables.json +++ b/apps/docs/openapi-v2-tables.json @@ -4897,7 +4897,7 @@ "description": "Human-readable explanation of the error." }, "details": { - "description": "Structured error details. On a `403` whose cause a caller can act on, this carries a `code` from a closed set:\n- `INSUFFICIENT_WORKSPACE_ROLE` — The caller has access to the workspace but its role is below the one this operation requires.\n- `PERSONAL_API_KEYS_DISABLED` — The workspace's organization does not allow personal API keys. Use a workspace API key.\n- `WORKSPACE_KEY_OPERATION_NOT_PERMITTED` — This operation is not available to a workspace-scoped API key. Use a personal API key.\n- `PRINCIPAL_KIND_NOT_PERMITTED` — This operation does not accept the caller’s kind of API key.\n- `ORGANIZATION_MEMBERSHIP_REQUIRED` — The caller is not a member of the organization it named.\n- `ORGANIZATION_ADMIN_REQUIRED` — The caller is a member of the organization but not an admin or owner.\n- `ENTERPRISE_PLAN_REQUIRED` — The organization has no active enterprise subscription.\n- `ORGANIZATION_PLAN_REQUIRED` — The organization has no active organization subscription (Pro for Teams, Max for Teams, or Enterprise).\n- `AUDIT_LOGS_DISABLED` — Audit logging is not enabled for this deployment.\n- `SKILL_EDITOR_ACCESS_REQUIRED` — The caller can write in the workspace but is not an editor of this skill.\n- `SECRET_ADMIN_ACCESS_REQUIRED` — The caller can write in the workspace but is not an admin of this secret. Ask a workspace admin, or someone holding admin on the secret, to grant access or set the value.\n- `WORKSPACE_RESOURCE_LIMIT_REACHED` — The workspace already holds the maximum number of resources of this kind. Delete one, or contact Sim to raise the limit; the message names the ceiling.\n- `PUBLIC_SHARING_NOT_ALLOWED` — The workspace's organization does not permit sharing this resource publicly. An organization admin controls the policy.\n- `CREDENTIAL_ADMIN_ACCESS_REQUIRED` — The caller can reach the workspace but cannot administer this credential.\n- `MCP_SERVER_URL_NOT_ALLOWED` — The supplied MCP server URL is outside the allowed domains or resolves to an internal address.\n- `WORKSPACE_PLAN_CAPABILITY_REQUIRED` — The workspace's plan does not include a capability this request depends on. The message names the capability; upgrading the workspace's plan is the remedy.\n- `CHAT_AUTH_MODE_NOT_PERMITTED` — The workspace's permission group does not allow the chat authentication mode the request selected. A mode already saved on the deployment may still be re-saved; changing to a disallowed one cannot.\n- `CONNECTOR_MANAGED_RESOURCE_READ_ONLY` — This resource is managed by a knowledge base connector and cannot be edited directly. Change it at the source and re-sync, or exclude the document from the connector." + "description": "Structured error details. On a `403` whose cause a caller can act on, this carries a `code` from a closed set:\n- `INSUFFICIENT_WORKSPACE_ROLE` — The caller has access to the workspace but its role is below the one this operation requires.\n- `PERSONAL_API_KEYS_DISABLED` — The workspace's organization does not allow personal API keys. Use a workspace API key.\n- `WORKSPACE_KEY_OPERATION_NOT_PERMITTED` — This operation is not available to a workspace-scoped API key. Use a personal API key.\n- `PRINCIPAL_KIND_NOT_PERMITTED` — This operation does not accept the caller’s kind of API key.\n- `ORGANIZATION_MEMBERSHIP_REQUIRED` — The caller is not a member of the organization it named.\n- `ORGANIZATION_ADMIN_REQUIRED` — The caller is a member of the organization but not an admin or owner.\n- `ENTERPRISE_PLAN_REQUIRED` — The organization has no active enterprise subscription.\n- `ORGANIZATION_PLAN_REQUIRED` — The organization has no active organization subscription (Pro for Teams, Max for Teams, or Enterprise).\n- `AUDIT_LOGS_DISABLED` — Audit logging is not enabled for this deployment.\n- `SKILL_EDITOR_ACCESS_REQUIRED` — The caller can write in the workspace but is not an editor of this skill.\n- `SECRET_ADMIN_ACCESS_REQUIRED` — The caller can write in the workspace but is not an admin of this secret. Ask a workspace admin, or someone holding admin on the secret, to grant access or set the value.\n- `WORKSPACE_RESOURCE_LIMIT_REACHED` — The workspace already holds the maximum number of resources of this kind. Delete one, or contact Sim to raise the limit; the message names the ceiling.\n- `PUBLIC_SHARING_NOT_ALLOWED` — The workspace's organization does not permit sharing this resource publicly. An organization admin controls the policy.\n- `CREDENTIAL_ADMIN_ACCESS_REQUIRED` — The caller can reach the workspace but cannot administer this credential.\n- `MCP_SERVER_URL_NOT_ALLOWED` — The supplied MCP server URL is outside the allowed domains or resolves to an internal address.\n- `WORKSPACE_PLAN_CAPABILITY_REQUIRED` — The workspace's plan does not include a capability this request depends on. The message names the capability; upgrading the workspace's plan is the remedy.\n- `CHAT_AUTH_MODE_NOT_PERMITTED` — The workspace's permission group does not allow the chat authentication mode the request selected. A mode already saved on the deployment may still be re-saved; changing to a disallowed one cannot.\n- `CONNECTOR_MANAGED_RESOURCE_READ_ONLY` — This resource is managed by a knowledge base connector and cannot be edited directly. Change it at the source and re-sync, or exclude the document from the connector.\n- `PERMISSION_GROUP_CAPABILITY_BLOCKED` — The caller's permission group does not allow this capability. The message names it; an organization admin controls the group.\n- `INTEGRATION_NOT_ALLOWED` — The integration this request names is outside the workspace's allowed set. An organization admin controls the permission group's integration allowlist, and a self-hosted deployment can narrow it further with ALLOWED_INTEGRATIONS." } }, "required": ["code", "message"], @@ -4979,7 +4979,16 @@ }, "type": { "type": "string", - "enum": ["string", "number", "currency", "boolean", "date", "json", "select"], + "enum": [ + "string", + "number", + "currency", + "boolean", + "date", + "ttl", + "json", + "select" + ], "description": "Data type of values stored in the column." }, "required": { @@ -5257,7 +5266,16 @@ }, "type": { "type": "string", - "enum": ["string", "number", "currency", "boolean", "date", "json", "select"], + "enum": [ + "string", + "number", + "currency", + "boolean", + "date", + "ttl", + "json", + "select" + ], "description": "Column data type." }, "required": { @@ -5436,7 +5454,16 @@ }, "type": { "type": "string", - "enum": ["string", "number", "currency", "boolean", "date", "json", "select"], + "enum": [ + "string", + "number", + "currency", + "boolean", + "date", + "ttl", + "json", + "select" + ], "description": "Data type of values stored in the column." }, "required": { @@ -5536,7 +5563,16 @@ }, "type": { "type": "string", - "enum": ["string", "number", "currency", "boolean", "date", "json", "select"], + "enum": [ + "string", + "number", + "currency", + "boolean", + "date", + "ttl", + "json", + "select" + ], "description": "Column data type." }, "required": { @@ -5633,7 +5669,7 @@ "type": { "description": "Replacement column data type.", "type": "string", - "enum": ["string", "number", "currency", "boolean", "date", "json", "select"] + "enum": ["string", "number", "currency", "boolean", "date", "ttl", "json", "select"] }, "required": { "description": "Whether inserts must supply a value for this column.", @@ -7397,7 +7433,16 @@ }, "type": { "type": "string", - "enum": ["string", "number", "currency", "boolean", "date", "json", "select"], + "enum": [ + "string", + "number", + "currency", + "boolean", + "date", + "ttl", + "json", + "select" + ], "description": "Data type of values stored in the column." }, "required": { @@ -7597,7 +7642,16 @@ }, "type": { "type": "string", - "enum": ["string", "number", "currency", "boolean", "date", "json", "select"], + "enum": [ + "string", + "number", + "currency", + "boolean", + "date", + "ttl", + "json", + "select" + ], "description": "Output column data type." }, "required": { @@ -7738,7 +7792,16 @@ }, "type": { "type": "string", - "enum": ["string", "number", "currency", "boolean", "date", "json", "select"], + "enum": [ + "string", + "number", + "currency", + "boolean", + "date", + "ttl", + "json", + "select" + ], "description": "Output column data type." }, "required": { @@ -7856,7 +7919,16 @@ }, "type": { "type": "string", - "enum": ["string", "number", "currency", "boolean", "date", "json", "select"], + "enum": [ + "string", + "number", + "currency", + "boolean", + "date", + "ttl", + "json", + "select" + ], "description": "Data type of values stored in the column." }, "required": { diff --git a/apps/docs/openapi-v2-workflows.json b/apps/docs/openapi-v2-workflows.json index ca61a2adaca..265634bc17a 100644 --- a/apps/docs/openapi-v2-workflows.json +++ b/apps/docs/openapi-v2-workflows.json @@ -3908,7 +3908,7 @@ "description": "Human-readable explanation of the error." }, "details": { - "description": "Structured error details. On a `403` whose cause a caller can act on, this carries a `code` from a closed set:\n- `INSUFFICIENT_WORKSPACE_ROLE` — The caller has access to the workspace but its role is below the one this operation requires.\n- `PERSONAL_API_KEYS_DISABLED` — The workspace's organization does not allow personal API keys. Use a workspace API key.\n- `WORKSPACE_KEY_OPERATION_NOT_PERMITTED` — This operation is not available to a workspace-scoped API key. Use a personal API key.\n- `PRINCIPAL_KIND_NOT_PERMITTED` — This operation does not accept the caller’s kind of API key.\n- `ORGANIZATION_MEMBERSHIP_REQUIRED` — The caller is not a member of the organization it named.\n- `ORGANIZATION_ADMIN_REQUIRED` — The caller is a member of the organization but not an admin or owner.\n- `ENTERPRISE_PLAN_REQUIRED` — The organization has no active enterprise subscription.\n- `ORGANIZATION_PLAN_REQUIRED` — The organization has no active organization subscription (Pro for Teams, Max for Teams, or Enterprise).\n- `AUDIT_LOGS_DISABLED` — Audit logging is not enabled for this deployment.\n- `SKILL_EDITOR_ACCESS_REQUIRED` — The caller can write in the workspace but is not an editor of this skill.\n- `SECRET_ADMIN_ACCESS_REQUIRED` — The caller can write in the workspace but is not an admin of this secret. Ask a workspace admin, or someone holding admin on the secret, to grant access or set the value.\n- `WORKSPACE_RESOURCE_LIMIT_REACHED` — The workspace already holds the maximum number of resources of this kind. Delete one, or contact Sim to raise the limit; the message names the ceiling.\n- `PUBLIC_SHARING_NOT_ALLOWED` — The workspace's organization does not permit sharing this resource publicly. An organization admin controls the policy.\n- `CREDENTIAL_ADMIN_ACCESS_REQUIRED` — The caller can reach the workspace but cannot administer this credential.\n- `MCP_SERVER_URL_NOT_ALLOWED` — The supplied MCP server URL is outside the allowed domains or resolves to an internal address.\n- `WORKSPACE_PLAN_CAPABILITY_REQUIRED` — The workspace's plan does not include a capability this request depends on. The message names the capability; upgrading the workspace's plan is the remedy.\n- `CHAT_AUTH_MODE_NOT_PERMITTED` — The workspace's permission group does not allow the chat authentication mode the request selected. A mode already saved on the deployment may still be re-saved; changing to a disallowed one cannot.\n- `CONNECTOR_MANAGED_RESOURCE_READ_ONLY` — This resource is managed by a knowledge base connector and cannot be edited directly. Change it at the source and re-sync, or exclude the document from the connector." + "description": "Structured error details. On a `403` whose cause a caller can act on, this carries a `code` from a closed set:\n- `INSUFFICIENT_WORKSPACE_ROLE` — The caller has access to the workspace but its role is below the one this operation requires.\n- `PERSONAL_API_KEYS_DISABLED` — The workspace's organization does not allow personal API keys. Use a workspace API key.\n- `WORKSPACE_KEY_OPERATION_NOT_PERMITTED` — This operation is not available to a workspace-scoped API key. Use a personal API key.\n- `PRINCIPAL_KIND_NOT_PERMITTED` — This operation does not accept the caller’s kind of API key.\n- `ORGANIZATION_MEMBERSHIP_REQUIRED` — The caller is not a member of the organization it named.\n- `ORGANIZATION_ADMIN_REQUIRED` — The caller is a member of the organization but not an admin or owner.\n- `ENTERPRISE_PLAN_REQUIRED` — The organization has no active enterprise subscription.\n- `ORGANIZATION_PLAN_REQUIRED` — The organization has no active organization subscription (Pro for Teams, Max for Teams, or Enterprise).\n- `AUDIT_LOGS_DISABLED` — Audit logging is not enabled for this deployment.\n- `SKILL_EDITOR_ACCESS_REQUIRED` — The caller can write in the workspace but is not an editor of this skill.\n- `SECRET_ADMIN_ACCESS_REQUIRED` — The caller can write in the workspace but is not an admin of this secret. Ask a workspace admin, or someone holding admin on the secret, to grant access or set the value.\n- `WORKSPACE_RESOURCE_LIMIT_REACHED` — The workspace already holds the maximum number of resources of this kind. Delete one, or contact Sim to raise the limit; the message names the ceiling.\n- `PUBLIC_SHARING_NOT_ALLOWED` — The workspace's organization does not permit sharing this resource publicly. An organization admin controls the policy.\n- `CREDENTIAL_ADMIN_ACCESS_REQUIRED` — The caller can reach the workspace but cannot administer this credential.\n- `MCP_SERVER_URL_NOT_ALLOWED` — The supplied MCP server URL is outside the allowed domains or resolves to an internal address.\n- `WORKSPACE_PLAN_CAPABILITY_REQUIRED` — The workspace's plan does not include a capability this request depends on. The message names the capability; upgrading the workspace's plan is the remedy.\n- `CHAT_AUTH_MODE_NOT_PERMITTED` — The workspace's permission group does not allow the chat authentication mode the request selected. A mode already saved on the deployment may still be re-saved; changing to a disallowed one cannot.\n- `CONNECTOR_MANAGED_RESOURCE_READ_ONLY` — This resource is managed by a knowledge base connector and cannot be edited directly. Change it at the source and re-sync, or exclude the document from the connector.\n- `PERMISSION_GROUP_CAPABILITY_BLOCKED` — The caller's permission group does not allow this capability. The message names it; an organization admin controls the group.\n- `INTEGRATION_NOT_ALLOWED` — The integration this request names is outside the workspace's allowed set. An organization admin controls the permission group's integration allowlist, and a self-hosted deployment can narrow it further with ALLOWED_INTEGRATIONS." } }, "required": ["code", "message"], @@ -8593,6 +8593,10 @@ "StoredChatDeploymentOutputConfig": { "type": "object", "properties": { + "workflowId": { + "description": "Child workflow containing the selected block. Omitted for the deployed workflow.", + "type": "string" + }, "blockId": { "type": "string", "description": "Block whose output the chat streams." @@ -8902,6 +8906,11 @@ "ChatDeploymentOutputConfig": { "type": "object", "properties": { + "workflowId": { + "description": "Child workflow containing the selected block. Omit for the deployed workflow.", + "type": "string", + "minLength": 1 + }, "blockId": { "type": "string", "minLength": 1, @@ -9305,7 +9314,7 @@ "type": "boolean" }, "selectedOutputs": { - "description": "Block output references to include in a streamed response, as `blockId`, `blockId.path`, or `BlockName.path` (resolved against the live workflow). Requires `stream: true` — it shapes the streamed envelope only, so it is rejected on a sync request and when `async` is true. To narrow a finished run, pass `selectedOutputs` to the run resource instead.", + "description": "Block output references to include in a streamed response. Use `.` for the executed workflow or `..` for a child workflow; block names are normalized workflow reference names. Selecting a child workflow applies to every invocation of it. Requires `stream: true` — it shapes the streamed envelope only, so it is rejected on a sync request and when `async` is true. To narrow a finished run, pass `selectedOutputs` to the run resource instead.", "maxItems": 100, "type": "array", "items": { @@ -9997,7 +10006,7 @@ "description": "Whether a paused execution was cancelled." }, "reason": { - "description": "Machine-readable cancellation outcome, present on every cancellation including full successes. `recorded` is the success value. `already_cancelled`, `already_completed`, and `already_failed` mean the run had already reached that terminal state, so nothing was cancelled and `durablyRecorded` is false. `redis_unavailable` and `redis_write_failed` mean the distributed cancellation signal was not written, so an already-running execution may not observe the cancellation. `paused_event_publish_failed` and `paused_database_cancel_failed` name the failing step for a paused run.", + "description": "Machine-readable cancellation outcome, present on every cancellation including full successes. `recorded` and `queue_cancelled` are successful cancellation values. `already_cancelled`, `already_completed`, and `already_failed` mean the run had already reached that terminal state, so nothing was cancelled and `durablyRecorded` is false. The remaining values identify a degraded or incomplete cancellation step.", "type": "string", "enum": [ "recorded", @@ -10007,7 +10016,10 @@ "redis_unavailable", "redis_write_failed", "paused_event_publish_failed", - "paused_database_cancel_failed" + "paused_database_cancel_failed", + "queue_cancelled", + "active_resume_signal_failed", + "cancellation_not_finalized" ] } }, diff --git a/apps/docs/package.json b/apps/docs/package.json index 700fdfecbc6..66ba9567c52 100644 --- a/apps/docs/package.json +++ b/apps/docs/package.json @@ -21,6 +21,7 @@ "@sim/db": "workspace:*", "@sim/emcn": "workspace:*", "@sim/workflow-renderer": "workspace:*", + "@xyflow/react": "12.11.3", "class-variance-authority": "^0.7.1", "clsx": "^2.1.1", "drizzle-orm": "^0.45.2", @@ -35,14 +36,12 @@ "remark-breaks": "^4.0.0", "shiki": "4.3.1", "tailwind-merge": "^3.0.2", - "reactflow": "^11.11.4", "framer-motion": "^12.5.0", "zod": "4.3.6" }, "devDependencies": { "@sim/tsconfig": "workspace:*", "@tailwindcss/postcss": "^4.0.12", - "@types/mdx": "^2.0.13", "@types/node": "24.2.1", "@types/react": "^19.1.2", "@types/react-dom": "^19.0.4", diff --git a/apps/realtime/src/handlers/index.ts b/apps/realtime/src/handlers/index.ts index 8dd71093673..cda7c032d62 100644 --- a/apps/realtime/src/handlers/index.ts +++ b/apps/realtime/src/handlers/index.ts @@ -1,4 +1,4 @@ -import { ROOM_TYPES } from '@sim/realtime-protocol/rooms' +import { WORKSPACE_LIST_ROOM_TYPES } from '@sim/realtime-protocol/rooms' import { setupConnectionHandlers } from '@/handlers/connection' import { setupWorkspaceFileDocHandlers } from '@/handlers/file-doc' import { setupOperationsHandlers } from '@/handlers/operations' @@ -18,8 +18,9 @@ export function setupAllHandlers(socket: AuthenticatedSocket, roomManager: IRoom setupVariablesHandlers(socket, roomManager) setupPresenceHandlers(socket, roomManager) // Presence-free, workspace-scoped live-list rooms (share one implementation). - setupWorkspaceInvalidationRoom(socket, roomManager, ROOM_TYPES.WORKSPACE_FILES) - setupWorkspaceInvalidationRoom(socket, roomManager, ROOM_TYPES.WORKSPACE_TABLES) + for (const roomType of WORKSPACE_LIST_ROOM_TYPES) { + setupWorkspaceInvalidationRoom(socket, roomManager, roomType) + } setupWorkspaceFileDocHandlers(socket, roomManager) setupTablesHandlers(socket, roomManager) setupConnectionHandlers(socket, roomManager) diff --git a/apps/realtime/src/handlers/workspace-invalidation-room.test.ts b/apps/realtime/src/handlers/workspace-invalidation-room.test.ts index e487edd0a8e..46c5b210efb 100644 --- a/apps/realtime/src/handlers/workspace-invalidation-room.test.ts +++ b/apps/realtime/src/handlers/workspace-invalidation-room.test.ts @@ -1,7 +1,7 @@ /** * @vitest-environment node */ -import { ROOM_TYPES } from '@sim/realtime-protocol/rooms' +import { WORKSPACE_LIST_ROOM_TYPES } from '@sim/realtime-protocol/rooms' import { sleep } from '@sim/utils/helpers' import { beforeEach, describe, expect, it, vi } from 'vitest' import type { IRoomManager } from '@/rooms' @@ -73,229 +73,226 @@ function createRoomManager(overrides?: Partial): IRoomManager { } as unknown as IRoomManager } -// The two presence-free live-list rooms share one implementation; run the whole suite against both -// so files and tables can never drift. Event names and room names derive from the room type. -describe.each([ROOM_TYPES.WORKSPACE_FILES, ROOM_TYPES.WORKSPACE_TABLES] as const)( - 'setupWorkspaceInvalidationRoom(%s)', - (roomType) => { - const joinEvent = `join-${roomType}` - const successEvent = `${joinEvent}-success` - const errorEvent = `${joinEvent}-error` - const leaveEvent = `leave-${roomType}` - const roomOf = (workspaceId: string) => `${roomType}:${workspaceId}` - - const setup = (socket: ReturnType['socket'], roomManager: IRoomManager) => - setupWorkspaceInvalidationRoom( - socket as unknown as Parameters[0], - roomManager, - roomType - ) - - beforeEach(() => { - vi.clearAllMocks() - mockAuthorizeRoom.mockResolvedValue({ - allowed: true, - status: 200, - workspaceId: 'ws-1', - workspacePermission: 'admin', - }) - }) - - it('rejects join when the socket is not authenticated', async () => { - const { socket, handlers } = createSocket({ userId: undefined, userName: undefined }) - setup(socket, createRoomManager()) - - await handlers[joinEvent]({ workspaceId: 'ws-1' }) - - expect(socket.emit).toHaveBeenCalledWith(errorEvent, { - workspaceId: 'ws-1', - error: 'Authentication required', - code: 'AUTHENTICATION_REQUIRED', - retryable: false, - }) +// The presence-free live-list rooms share one implementation; run the whole suite against each +// so they can never drift. Event names and room names derive from the room type. +describe.each(WORKSPACE_LIST_ROOM_TYPES)('setupWorkspaceInvalidationRoom(%s)', (roomType) => { + const joinEvent = `join-${roomType}` + const successEvent = `${joinEvent}-success` + const errorEvent = `${joinEvent}-error` + const leaveEvent = `leave-${roomType}` + const roomOf = (workspaceId: string) => `${roomType}:${workspaceId}` + + const setup = (socket: ReturnType['socket'], roomManager: IRoomManager) => + setupWorkspaceInvalidationRoom( + socket as unknown as Parameters[0], + roomManager, + roomType + ) + + beforeEach(() => { + vi.clearAllMocks() + mockAuthorizeRoom.mockResolvedValue({ + allowed: true, + status: 200, + workspaceId: 'ws-1', + workspacePermission: 'admin', }) + }) - it('rejects join with a retryable error when realtime is unavailable', async () => { - const { socket, handlers } = createSocket() - setup(socket, createRoomManager({ isReady: vi.fn().mockReturnValue(false) })) + it('rejects join when the socket is not authenticated', async () => { + const { socket, handlers } = createSocket({ userId: undefined, userName: undefined }) + setup(socket, createRoomManager()) - await handlers[joinEvent]({ workspaceId: 'ws-1' }) + await handlers[joinEvent]({ workspaceId: 'ws-1' }) - expect(socket.emit).toHaveBeenCalledWith( - errorEvent, - expect.objectContaining({ code: 'ROOM_MANAGER_UNAVAILABLE', retryable: true }) - ) + expect(socket.emit).toHaveBeenCalledWith(errorEvent, { + workspaceId: 'ws-1', + error: 'Authentication required', + code: 'AUTHENTICATION_REQUIRED', + retryable: false, }) - - it('rejects join when workspace access is denied', async () => { - mockAuthorizeRoom.mockResolvedValue({ - allowed: false, - status: 403, - workspaceId: 'ws-1', - workspacePermission: null, - }) - const { socket, handlers } = createSocket() - setup(socket, createRoomManager()) - - await handlers[joinEvent]({ workspaceId: 'ws-1' }) - - expect(socket.emit).toHaveBeenCalledWith( - errorEvent, - expect.objectContaining({ code: 'ACCESS_DENIED', retryable: false }) - ) + }) + + it('rejects join with a retryable error when realtime is unavailable', async () => { + const { socket, handlers } = createSocket() + setup(socket, createRoomManager({ isReady: vi.fn().mockReturnValue(false) })) + + await handlers[joinEvent]({ workspaceId: 'ws-1' }) + + expect(socket.emit).toHaveBeenCalledWith( + errorEvent, + expect.objectContaining({ code: 'ROOM_MANAGER_UNAVAILABLE', retryable: true }) + ) + }) + + it('rejects join when workspace access is denied', async () => { + mockAuthorizeRoom.mockResolvedValue({ + allowed: false, + status: 403, + workspaceId: 'ws-1', + workspacePermission: null, }) - - it('joins the room on success without any presence bookkeeping', async () => { - const { socket, handlers } = createSocket() - const roomManager = createRoomManager() - setup(socket, roomManager) - - await handlers[joinEvent]({ workspaceId: 'ws-1' }) - - expect(socket.join).toHaveBeenCalledWith(roomOf('ws-1')) - expect(socket.emit).toHaveBeenCalledWith(successEvent, { workspaceId: 'ws-1' }) - // The room is live-list-only: no room-manager presence is tracked or broadcast. - expect(roomManager.addUserToRoom).not.toHaveBeenCalled() - expect(roomManager.broadcastPresenceUpdate).not.toHaveBeenCalled() - }) - - it('aborts a join superseded during the access re-check await', async () => { - // The access re-resolve is an await like any other: a leave landing during it must - // still cancel this join, or the stale join would leave the room the client - // switched to and commit the abandoned one. Forced down the re-resolve's DB path - // by expiring the cached decision mid-join, so the interleaving is deterministic - // rather than dependent on microtask ordering. - vi.useFakeTimers() - try { - const { handlers, socket } = createSocket({ id: 'socket-sup', userId: 'user-sup' }) - setupWorkspaceInvalidationRoom( - socket as unknown as Parameters[0], - createRoomManager(), - roomType - ) - - let call = 0 - mockAuthorizeRoom.mockImplementation(async () => { - call += 1 - if (call === 1) { - // A later-started read commits, so this join's own decision is dropped; then - // the join stalls past the TTL so that decision is expired by re-check time. - commitRoomPermission( - 'user-sup', - { type: roomType, id: 'ws-sup' }, - 'admin', - beginRoomPermissionRead() - ) - await sleep(31_000) - } else { - // Second call is the re-check's re-resolve: the client leaves during it. - handlers[leaveEvent]({ workspaceId: 'ws-sup' }) - } - return { allowed: true, status: 200, workspaceId: 'ws-sup', workspacePermission: 'admin' } - }) - - const joining = handlers[joinEvent]({ workspaceId: 'ws-sup' }) - await vi.advanceTimersByTimeAsync(31_000) - await joining - - expect(call).toBe(2) - expect(socket.join).not.toHaveBeenCalled() - expect(socket.emit).not.toHaveBeenCalledWith(successEvent, expect.anything()) - } finally { - vi.useRealTimers() - } - }) - - it('does not join when access was revoked while the join was in flight', async () => { - // The sweep records a revocation before it evicts, so a join whose authorize - // completed just before that must not put the socket back in the room. - const { handlers, socket } = createSocket({ id: 'socket-race', userId: 'user-race' }) + const { socket, handlers } = createSocket() + setup(socket, createRoomManager()) + + await handlers[joinEvent]({ workspaceId: 'ws-1' }) + + expect(socket.emit).toHaveBeenCalledWith( + errorEvent, + expect.objectContaining({ code: 'ACCESS_DENIED', retryable: false }) + ) + }) + + it('joins the room on success without any presence bookkeeping', async () => { + const { socket, handlers } = createSocket() + const roomManager = createRoomManager() + setup(socket, roomManager) + + await handlers[joinEvent]({ workspaceId: 'ws-1' }) + + expect(socket.join).toHaveBeenCalledWith(roomOf('ws-1')) + expect(socket.emit).toHaveBeenCalledWith(successEvent, { workspaceId: 'ws-1' }) + // The room is live-list-only: no room-manager presence is tracked or broadcast. + expect(roomManager.addUserToRoom).not.toHaveBeenCalled() + expect(roomManager.broadcastPresenceUpdate).not.toHaveBeenCalled() + }) + + it('aborts a join superseded during the access re-check await', async () => { + // The access re-resolve is an await like any other: a leave landing during it must + // still cancel this join, or the stale join would leave the room the client + // switched to and commit the abandoned one. Forced down the re-resolve's DB path + // by expiring the cached decision mid-join, so the interleaving is deterministic + // rather than dependent on microtask ordering. + vi.useFakeTimers() + try { + const { handlers, socket } = createSocket({ id: 'socket-sup', userId: 'user-sup' }) setupWorkspaceInvalidationRoom( socket as unknown as Parameters[0], createRoomManager(), roomType ) + let call = 0 mockAuthorizeRoom.mockImplementation(async () => { - commitRoomPermission( - 'user-race', - { type: roomType, id: 'ws-race' }, - null, - beginRoomPermissionRead() - ) - return { allowed: true, status: 200, workspaceId: 'ws-race', workspacePermission: 'admin' } + call += 1 + if (call === 1) { + // A later-started read commits, so this join's own decision is dropped; then + // the join stalls past the TTL so that decision is expired by re-check time. + commitRoomPermission( + 'user-sup', + { type: roomType, id: 'ws-sup' }, + 'admin', + beginRoomPermissionRead() + ) + await sleep(31_000) + } else { + // Second call is the re-check's re-resolve: the client leaves during it. + handlers[leaveEvent]({ workspaceId: 'ws-sup' }) + } + return { allowed: true, status: 200, workspaceId: 'ws-sup', workspacePermission: 'admin' } }) - await handlers[joinEvent]({ workspaceId: 'ws-race' }) + const joining = handlers[joinEvent]({ workspaceId: 'ws-sup' }) + await vi.advanceTimersByTimeAsync(31_000) + await joining - expect(socket.emit).toHaveBeenCalledWith( - errorEvent, - expect.objectContaining({ code: 'ACCESS_DENIED', retryable: false }) - ) + expect(call).toBe(2) expect(socket.join).not.toHaveBeenCalled() + expect(socket.emit).not.toHaveBeenCalledWith(successEvent, expect.anything()) + } finally { + vi.useRealTimers() + } + }) + + it('does not join when access was revoked while the join was in flight', async () => { + // The sweep records a revocation before it evicts, so a join whose authorize + // completed just before that must not put the socket back in the room. + const { handlers, socket } = createSocket({ id: 'socket-race', userId: 'user-race' }) + setupWorkspaceInvalidationRoom( + socket as unknown as Parameters[0], + createRoomManager(), + roomType + ) + + mockAuthorizeRoom.mockImplementation(async () => { + commitRoomPermission( + 'user-race', + { type: roomType, id: 'ws-race' }, + null, + beginRoomPermissionRead() + ) + return { allowed: true, status: 200, workspaceId: 'ws-race', workspacePermission: 'admin' } }) - it('leaves a previously-joined room when switching workspaces', async () => { - const { socket, handlers, rooms } = createSocket() - rooms.add(roomOf('ws-old')) - setup(socket, createRoomManager()) + await handlers[joinEvent]({ workspaceId: 'ws-race' }) - await handlers[joinEvent]({ workspaceId: 'ws-1' }) + expect(socket.emit).toHaveBeenCalledWith( + errorEvent, + expect.objectContaining({ code: 'ACCESS_DENIED', retryable: false }) + ) + expect(socket.join).not.toHaveBeenCalled() + }) - expect(socket.leave).toHaveBeenCalledWith(roomOf('ws-old')) - expect(socket.join).toHaveBeenCalledWith(roomOf('ws-1')) - }) + it('leaves a previously-joined room when switching workspaces', async () => { + const { socket, handlers, rooms } = createSocket() + rooms.add(roomOf('ws-old')) + setup(socket, createRoomManager()) - it('leaves the scoped room on leave', () => { - const { socket, handlers, rooms } = createSocket() - rooms.add(roomOf('ws-1')) - setup(socket, createRoomManager()) + await handlers[joinEvent]({ workspaceId: 'ws-1' }) - handlers[leaveEvent]({ workspaceId: 'ws-1' }) + expect(socket.leave).toHaveBeenCalledWith(roomOf('ws-old')) + expect(socket.join).toHaveBeenCalledWith(roomOf('ws-1')) + }) - expect(socket.leave).toHaveBeenCalledWith(roomOf('ws-1')) - }) + it('leaves the scoped room on leave', () => { + const { socket, handlers, rooms } = createSocket() + rooms.add(roomOf('ws-1')) + setup(socket, createRoomManager()) - it('cancels an in-flight join when the user leaves that workspace mid-authorize', async () => { - const { socket, handlers } = createSocket() - let resolveAuth: (value: unknown) => void = () => {} - mockAuthorizeRoom.mockReturnValue( - new Promise((resolve) => { - resolveAuth = resolve - }) - ) - setup(socket, createRoomManager()) + handlers[leaveEvent]({ workspaceId: 'ws-1' }) - // Join ws-1 is awaiting authorization when the view unmounts and leaves ws-1. - const joinPromise = handlers[joinEvent]({ workspaceId: 'ws-1' }) - handlers[leaveEvent]({ workspaceId: 'ws-1' }) - resolveAuth({ allowed: true, status: 200, workspaceId: 'ws-1', workspacePermission: 'admin' }) - await joinPromise + expect(socket.leave).toHaveBeenCalledWith(roomOf('ws-1')) + }) - // The stale join must NOT join the room the client has since left (no stranded membership). - expect(socket.join).not.toHaveBeenCalled() - expect(socket.emit).not.toHaveBeenCalledWith(successEvent, { workspaceId: 'ws-1' }) - }) - - it('does not cancel an in-flight join when a deferred leave targets a different workspace', async () => { - const { socket, handlers } = createSocket() - let resolveAuth: (value: unknown) => void = () => {} - mockAuthorizeRoom.mockReturnValue( - new Promise((resolve) => { - resolveAuth = resolve - }) - ) - setup(socket, createRoomManager()) - - // The client has switched to ws-2 (join in-flight) when a stale leave for the prior ws-1 lands. - const joinPromise = handlers[joinEvent]({ workspaceId: 'ws-2' }) - handlers[leaveEvent]({ workspaceId: 'ws-1' }) - resolveAuth({ allowed: true, status: 200, workspaceId: 'ws-2', workspacePermission: 'admin' }) - await joinPromise - - // The deferred leave for ws-1 must not abort the join the client actually wants (ws-2). - expect(socket.join).toHaveBeenCalledWith(roomOf('ws-2')) - expect(socket.emit).toHaveBeenCalledWith(successEvent, { workspaceId: 'ws-2' }) - }) - } -) + it('cancels an in-flight join when the user leaves that workspace mid-authorize', async () => { + const { socket, handlers } = createSocket() + let resolveAuth: (value: unknown) => void = () => {} + mockAuthorizeRoom.mockReturnValue( + new Promise((resolve) => { + resolveAuth = resolve + }) + ) + setup(socket, createRoomManager()) + + // Join ws-1 is awaiting authorization when the view unmounts and leaves ws-1. + const joinPromise = handlers[joinEvent]({ workspaceId: 'ws-1' }) + handlers[leaveEvent]({ workspaceId: 'ws-1' }) + resolveAuth({ allowed: true, status: 200, workspaceId: 'ws-1', workspacePermission: 'admin' }) + await joinPromise + + // The stale join must NOT join the room the client has since left (no stranded membership). + expect(socket.join).not.toHaveBeenCalled() + expect(socket.emit).not.toHaveBeenCalledWith(successEvent, { workspaceId: 'ws-1' }) + }) + + it('does not cancel an in-flight join when a deferred leave targets a different workspace', async () => { + const { socket, handlers } = createSocket() + let resolveAuth: (value: unknown) => void = () => {} + mockAuthorizeRoom.mockReturnValue( + new Promise((resolve) => { + resolveAuth = resolve + }) + ) + setup(socket, createRoomManager()) + + // The client has switched to ws-2 (join in-flight) when a stale leave for the prior ws-1 lands. + const joinPromise = handlers[joinEvent]({ workspaceId: 'ws-2' }) + handlers[leaveEvent]({ workspaceId: 'ws-1' }) + resolveAuth({ allowed: true, status: 200, workspaceId: 'ws-2', workspacePermission: 'admin' }) + await joinPromise + + // The deferred leave for ws-1 must not abort the join the client actually wants (ws-2). + expect(socket.join).toHaveBeenCalledWith(roomOf('ws-2')) + expect(socket.emit).toHaveBeenCalledWith(successEvent, { workspaceId: 'ws-2' }) + }) +}) diff --git a/apps/realtime/src/routes/http.ts b/apps/realtime/src/routes/http.ts index aed7d1a58a9..19d2401e37e 100644 --- a/apps/realtime/src/routes/http.ts +++ b/apps/realtime/src/routes/http.ts @@ -1,5 +1,5 @@ import type { IncomingMessage, ServerResponse } from 'http' -import { ROOM_TYPES } from '@sim/realtime-protocol/rooms' +import { WORKSPACE_LIST_ROOM_TYPES } from '@sim/realtime-protocol/rooms' import { safeCompare } from '@sim/security/compare' import { env } from '@/env' import { applyMarkdownToLiveFileDoc } from '@/handlers/file-doc' @@ -164,43 +164,27 @@ export function createHttpHandler(roomManager: IRoomManager, logger: Logger) { return } - // Fan out a file-tree change to everyone viewing a workspace's files, so their - // browser refetches. File mutations happen over the HTTP API (not the socket); - // this is the lossy liveness signal — a missed one only means stale-until-refetch. - if (req.method === 'POST' && req.url === '/api/workspace-files-changed') { + // Fan out a workspace list change (files tree, tables list, workflow registry) to everyone in + // that workspace's live-list room, so their browser refetches. These mutations happen over the + // HTTP API (not the socket); this is the lossy liveness signal — a missed one only means + // stale-until-refetch. Endpoint and event names derive from the room type, mirroring the socket + // handler and the client hook. + const listRoomType = WORKSPACE_LIST_ROOM_TYPES.find( + (type) => req.url === `/api/${type}-changed` + ) + if (req.method === 'POST' && listRoomType) { try { const body = await readRequestBody(req) const { workspaceId } = JSON.parse(body) if (!isNonEmptyString(workspaceId)) return sendError(res, 'Invalid workspaceId', 400) - roomManager.emitToRoom( - { type: ROOM_TYPES.WORKSPACE_FILES, id: workspaceId }, - 'workspace-files-changed', - { workspaceId, timestamp: Date.now() } - ) - sendSuccess(res) - } catch (error) { - logger.error('Error handling workspace files changed notification:', error) - sendError(res, 'Failed to process files change notification') - } - return - } - - // Fan out a table-list change to everyone viewing a workspace's tables, so their browser - // refetches. The list-level counterpart to workspace-files-changed; same lossy-signal contract. - if (req.method === 'POST' && req.url === '/api/workspace-tables-changed') { - try { - const body = await readRequestBody(req) - const { workspaceId } = JSON.parse(body) - if (!isNonEmptyString(workspaceId)) return sendError(res, 'Invalid workspaceId', 400) - roomManager.emitToRoom( - { type: ROOM_TYPES.WORKSPACE_TABLES, id: workspaceId }, - 'workspace-tables-changed', - { workspaceId, timestamp: Date.now() } - ) + roomManager.emitToRoom({ type: listRoomType, id: workspaceId }, `${listRoomType}-changed`, { + workspaceId, + timestamp: Date.now(), + }) sendSuccess(res) } catch (error) { - logger.error('Error handling workspace tables changed notification:', error) - sendError(res, 'Failed to process tables change notification') + logger.error(`Error handling ${listRoomType} changed notification:`, error) + sendError(res, 'Failed to process list change notification') } return } diff --git a/apps/sim/.env.example b/apps/sim/.env.example index 443ff1d2da9..1c9c6cbd024 100644 --- a/apps/sim/.env.example +++ b/apps/sim/.env.example @@ -12,8 +12,9 @@ BETTER_AUTH_URL=http://localhost:3000 # Authentication Bypass (Optional - for self-hosted deployments behind private networks) # DISABLE_AUTH=true # Uncomment to bypass authentication entirely. Creates an anonymous session for all requests. -# Private Database Hosts (Optional - for self-hosted deployments only) -# ALLOW_PRIVATE_DATABASE_HOSTS=true # Uncomment to let database/connector tools reach private/reserved/loopback hosts (e.g. Docker/K8s service names, localhost). Loosens the SSRF boundary; only enable on a trusted private network. +# Private-network egress allowlist (Optional - self-hosted only; ignored on Sim Cloud) +# EGRESS_ALLOWED_HOSTS=host.docker.internal,*.svc.cluster.local # Uncomment to let outbound requests reach these hosts on a private network. Widens the SSRF boundary; only use on a trusted private network. +# EGRESS_ALLOWED_IP_RANGES=10.0.0.0/8 # Same, by CIDR. Cloud metadata endpoints stay blocked regardless, and neither setting is honored for URLs harvested from content or for a proxy. # NextJS (Required) NEXT_PUBLIC_APP_URL=http://localhost:3000 @@ -201,6 +202,7 @@ CRON_SECRET=your_cron_secret # Use `openssl rand -hex 32` to generate. Authentic # DATA_DRAINS_ENABLED= / NEXT_PUBLIC_DATA_DRAINS_ENABLED= # Export streams # FORKING_ENABLED= # Workspace forks # CREDENTIAL_GROUPS= # Enterprise managed OAuth collections +# TABLE_ROW_TTL= # Table TTL columns and expired-row cleanup # ORGANIZATIONS_ENABLED= / NEXT_PUBLIC_ORGANIZATIONS_ENABLED= # Organizations only # Instance organization (Optional). Most enterprise features read their settings from the diff --git a/apps/sim/app/(interfaces)/chat/hooks/use-chat-streaming.ts b/apps/sim/app/(interfaces)/chat/hooks/use-chat-streaming.ts index d4e9b8d1f09..8dbac2525a3 100644 --- a/apps/sim/app/(interfaces)/chat/hooks/use-chat-streaming.ts +++ b/apps/sim/app/(interfaces)/chat/hooks/use-chat-streaming.ts @@ -21,6 +21,7 @@ import { isChatThinkingFrame, isChatToolFrame, } from '@/lib/workflows/streaming/agent-stream-protocol' +import { scopeOutputBlockId } from '@/lib/workflows/streaming/output-selector' import type { ChatFile, ChatMessage, @@ -70,7 +71,7 @@ function extractFilesFromData( } export interface StreamingOptions { - outputConfigs?: Array<{ blockId: string; path?: string }> + outputConfigs?: Array<{ workflowId?: string; blockId: string; path?: string }> /** * Shared AbortController for fetch + SSE body reads. When provided (preferred), * Stop aborts the in-flight request server-side as well as the reader. @@ -430,7 +431,10 @@ export function useChatStreaming() { if (outputConfigs?.length && finalData.output) { for (const config of outputConfigs) { - const blockOutputs = finalData.output[config.blockId] + const outputBlockId = config.workflowId + ? scopeOutputBlockId(config.workflowId, config.blockId) + : config.blockId + const blockOutputs = finalData.output[outputBlockId] if (!blockOutputs) continue const value = getOutputValue(blockOutputs, config.path) diff --git a/apps/sim/app/(interfaces)/resume/[workflowId]/[executionId]/page.test.tsx b/apps/sim/app/(interfaces)/resume/[workflowId]/[executionId]/page.test.tsx new file mode 100644 index 00000000000..873da140c5c --- /dev/null +++ b/apps/sim/app/(interfaces)/resume/[workflowId]/[executionId]/page.test.tsx @@ -0,0 +1,109 @@ +/** + * @vitest-environment node + */ + +import { beforeEach, describe, expect, it, vi } from 'vitest' +import { OrchestrationError } from '@/lib/core/orchestration/types' + +const mocks = vi.hoisted(() => ({ + authorize: vi.fn(), + getSession: vi.fn(), + resumePage: vi.fn(() => null), + unavailablePage: vi.fn(() => null), + redirect: vi.fn((url: string) => { + throw new Error(`NEXT_REDIRECT:${url}`) + }), +})) + +vi.mock('@/lib/auth', () => ({ + auth: { api: { getSession: vi.fn() } }, + getSession: mocks.getSession, +})) + +vi.mock('next/navigation', () => ({ + redirect: mocks.redirect, +})) + +vi.mock('@/lib/workflows/application/read-paused-workflow-execution', () => ({ + readPausedWorkflowExecution: { authorize: mocks.authorize }, +})) + +vi.mock('@/app/(interfaces)/resume/[workflowId]/[executionId]/resume-page-client', () => ({ + default: mocks.resumePage, +})) + +vi.mock( + '@/app/(interfaces)/resume/[workflowId]/[executionId]/resume-execution-unavailable', + () => ({ + ResumeExecutionUnavailable: mocks.unavailablePage, + }) +) + +import ResumeExecutionPageWrapper from '@/app/(interfaces)/resume/[workflowId]/[executionId]/page' + +const PAGE_PARAMS = { workflowId: 'workflow-1', executionId: 'execution-1' } + +function pageProps(contextId?: string) { + return { + params: Promise.resolve(PAGE_PARAMS), + searchParams: Promise.resolve(contextId ? { contextId } : {}), + } +} + +describe('ResumeExecutionPageWrapper', () => { + beforeEach(() => { + vi.clearAllMocks() + mocks.getSession.mockResolvedValue({ + user: { id: 'user-1' }, + session: { id: 'session-1' }, + }) + mocks.authorize.mockResolvedValue(undefined) + }) + + it('redirects an unauthenticated visitor before any protected lookup', async () => { + mocks.getSession.mockResolvedValueOnce(null) + const callbackPath = '/resume/workflow-1/execution-1?contextId=context-1' + + await expect(ResumeExecutionPageWrapper(pageProps('context-1'))).rejects.toThrow( + `NEXT_REDIRECT:/login?callbackUrl=${encodeURIComponent(callbackPath)}` + ) + expect(mocks.authorize).not.toHaveBeenCalled() + }) + + it('authorizes the session without serializing paused execution detail into the page', async () => { + const result = await ResumeExecutionPageWrapper(pageProps('context-1')) + + expect(mocks.authorize).toHaveBeenCalledWith({ + principal: { kind: 'session', userId: 'user-1', sessionId: 'session-1' }, + input: PAGE_PARAMS, + }) + expect(result.props).toMatchObject({ + params: PAGE_PARAMS, + initialContextId: 'context-1', + }) + expect(result.type).toBe(mocks.resumePage) + expect(result.key).toBe('workflow-1:execution-1:context-1') + expect(result.props).not.toHaveProperty('initialExecutionDetail') + expect(result.props).not.toHaveProperty('canLoadExecution') + }) + + it.each([ + new OrchestrationError('forbidden', 'Insufficient workspace permissions'), + new OrchestrationError('not_found', 'Workflow not found'), + ])('renders a data-free concealed state after authorization refusal: %s', async (error) => { + mocks.authorize.mockRejectedValueOnce(error) + + const result = await ResumeExecutionPageWrapper(pageProps()) + + expect(result.type).toBe(mocks.unavailablePage) + expect(result.type).not.toBe(mocks.resumePage) + expect(result.props).toEqual({}) + }) + + it('propagates authorization infrastructure failures', async () => { + const infrastructureError = new Error('database unavailable') + mocks.authorize.mockRejectedValueOnce(infrastructureError) + + await expect(ResumeExecutionPageWrapper(pageProps())).rejects.toBe(infrastructureError) + }) +}) diff --git a/apps/sim/app/(interfaces)/resume/[workflowId]/[executionId]/page.tsx b/apps/sim/app/(interfaces)/resume/[workflowId]/[executionId]/page.tsx index 7a965893e1f..edb0e262df2 100644 --- a/apps/sim/app/(interfaces)/resume/[workflowId]/[executionId]/page.tsx +++ b/apps/sim/app/(interfaces)/resume/[workflowId]/[executionId]/page.tsx @@ -1,5 +1,9 @@ import type { Metadata } from 'next' -import { PauseResumeManager } from '@/lib/workflows/executor/human-in-the-loop-manager' +import { redirect } from 'next/navigation' +import { getSession } from '@/lib/auth' +import { asOrchestrationError } from '@/lib/core/orchestration/types' +import { readPausedWorkflowExecution } from '@/lib/workflows/application/read-paused-workflow-execution' +import { ResumeExecutionUnavailable } from '@/app/(interfaces)/resume/[workflowId]/[executionId]/resume-execution-unavailable' import ResumeExecutionPage from '@/app/(interfaces)/resume/[workflowId]/[executionId]/resume-page-client' export const metadata: Metadata = { @@ -30,16 +34,37 @@ export default async function ResumeExecutionPageWrapper({ const initialContextId = Array.isArray(initialContextIdParam) ? initialContextIdParam[0] : initialContextIdParam + const resumePath = `/resume/${encodeURIComponent(workflowId)}/${encodeURIComponent(executionId)}${ + initialContextId ? `?${new URLSearchParams({ contextId: initialContextId })}` : '' + }` + const session = await getSession() + if (!session?.user?.id) { + redirect(`/login?callbackUrl=${encodeURIComponent(resumePath)}`) + } + if (!session.session?.id) throw new Error('Authenticated session is missing its session ID') - const detail = await PauseResumeManager.getPausedExecutionDetail({ - workflowId, - executionId, - }) + try { + if (!readPausedWorkflowExecution.authorize) { + throw new Error('Paused execution read use case does not expose authorization') + } + await readPausedWorkflowExecution.authorize({ + principal: { + kind: 'session', + userId: session.user.id, + sessionId: session.session.id, + }, + input: { workflowId, executionId }, + }) + } catch (error) { + const classified = asOrchestrationError(error) + if (classified?.code !== 'forbidden' && classified?.code !== 'not_found') throw error + return + } return ( ) diff --git a/apps/sim/app/(interfaces)/resume/[workflowId]/[executionId]/resume-execution-unavailable.tsx b/apps/sim/app/(interfaces)/resume/[workflowId]/[executionId]/resume-execution-unavailable.tsx new file mode 100644 index 00000000000..52c9d065127 --- /dev/null +++ b/apps/sim/app/(interfaces)/resume/[workflowId]/[executionId]/resume-execution-unavailable.tsx @@ -0,0 +1,17 @@ +import { ChipLink } from '@sim/emcn' + +export function ResumeExecutionUnavailable() { + return ( +
+
+

Execution Not Found

+

+ This execution could not be located or has already completed. +

+ + Return Home + +
+
+ ) +} diff --git a/apps/sim/app/(interfaces)/resume/[workflowId]/[executionId]/resume-page-client.test.tsx b/apps/sim/app/(interfaces)/resume/[workflowId]/[executionId]/resume-page-client.test.tsx new file mode 100644 index 00000000000..47993d91f1b --- /dev/null +++ b/apps/sim/app/(interfaces)/resume/[workflowId]/[executionId]/resume-page-client.test.tsx @@ -0,0 +1,180 @@ +/** + * @vitest-environment jsdom + */ +import { act } from 'react' +import { QueryClient, QueryClientProvider } from '@tanstack/react-query' +import { createRoot, type Root } from 'react-dom/client' +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import { ApiClientError } from '@/lib/api/client/errors' +import type { PausePointWithQueue } from '@/hooks/queries/resume-execution' + +const mocks = vi.hoisted(() => ({ + pauseContextDetail: vi.fn(), + refetch: vi.fn(), + replace: vi.fn(), + resumeContext: vi.fn(), + resumeExecutionDetail: vi.fn(), +})) + +vi.mock('next/navigation', () => ({ + useRouter: () => ({ replace: mocks.replace }), +})) + +vi.mock('@/hooks/queries/resume-execution', () => ({ + resumeKeys: { + execution: (workflowId: string, executionId: string) => [ + 'resume-execution', + 'execution', + workflowId, + executionId, + ], + context: (workflowId: string, executionId: string, contextId: string) => [ + 'resume-execution', + 'context', + workflowId, + executionId, + contextId, + ], + }, + usePauseContextDetail: mocks.pauseContextDetail, + useResumeContext: mocks.resumeContext, + useResumeExecutionDetail: mocks.resumeExecutionDetail, +})) + +import ResumeExecutionPage, { + selectInitialResumeContextId, +} from '@/app/(interfaces)/resume/[workflowId]/[executionId]/resume-page-client' + +const params = { workflowId: 'workflow-1', executionId: 'execution-1' } + +let container: HTMLDivElement +let queryClient: QueryClient +let root: Root + +function apiError(status: number): ApiClientError { + return new ApiClientError({ + status, + message: status === 404 ? 'Workflow not found' : 'Request failed', + body: { error: 'Request failed' }, + }) +} + +function renderPage(initialContextId?: string) { + act(() => { + root.render( + + + + ) + }) +} + +describe('ResumeExecutionPage', () => { + beforeEach(() => { + ;(globalThis as { IS_REACT_ACT_ENVIRONMENT?: boolean }).IS_REACT_ACT_ENVIRONMENT = true + container = document.createElement('div') + document.body.appendChild(container) + root = createRoot(container) + queryClient = new QueryClient({ defaultOptions: { queries: { retry: false } } }) + mocks.pauseContextDetail.mockReturnValue({ data: undefined, isLoading: false }) + mocks.resumeContext.mockReturnValue({ mutateAsync: vi.fn() }) + mocks.resumeExecutionDetail.mockReturnValue({ + data: undefined, + error: null, + isError: false, + isFetching: true, + isLoading: true, + refetch: mocks.refetch, + }) + }) + + afterEach(() => { + act(() => root.unmount()) + queryClient.clear() + container.remove() + vi.clearAllMocks() + }) + + it('renders a concealed state for an absent or newly inaccessible execution', () => { + mocks.resumeExecutionDetail.mockReturnValue({ + data: undefined, + error: apiError(404), + isError: true, + isFetching: false, + isLoading: false, + refetch: mocks.refetch, + }) + + renderPage('context-1') + + expect(container.textContent).toContain('Execution Not Found') + expect(container.textContent).not.toContain('Could Not Load Execution') + expect(mocks.pauseContextDetail).toHaveBeenLastCalledWith( + params.workflowId, + params.executionId, + undefined + ) + }) + + it('redirects an expired session back through login', () => { + mocks.resumeExecutionDetail.mockReturnValue({ + data: undefined, + error: apiError(401), + isError: true, + isFetching: false, + isLoading: false, + refetch: mocks.refetch, + }) + + renderPage('context-1') + + const callbackPath = '/resume/workflow-1/execution-1?contextId=context-1' + expect(mocks.replace).toHaveBeenCalledWith( + `/login?callbackUrl=${encodeURIComponent(callbackPath)}` + ) + expect(container.textContent).toContain('Redirecting to sign in') + }) + + it('shows a retryable error instead of mislabeling infrastructure failure', () => { + mocks.resumeExecutionDetail.mockReturnValue({ + data: undefined, + error: apiError(500), + isError: true, + isFetching: false, + isLoading: false, + refetch: mocks.refetch, + }) + + renderPage() + + expect(container.textContent).toContain('Could Not Load Execution') + expect(container.textContent).not.toContain('Execution Not Found') + const retryButton = Array.from(container.querySelectorAll('button')).find( + (button) => button.textContent === 'Try again' + ) + expect(retryButton).toBeDefined() + act(() => retryButton?.dispatchEvent(new MouseEvent('click', { bubbles: true }))) + expect(mocks.refetch).toHaveBeenCalledOnce() + }) +}) + +describe('selectInitialResumeContextId', () => { + const pausePoints = [ + { contextId: 'resumed-context', resumeStatus: 'resumed' }, + { contextId: 'paused-context', resumeStatus: 'paused' }, + ] as PausePointWithQueue[] + + it('uses a requested context only when the authorized execution contains it', () => { + expect(selectInitialResumeContextId(pausePoints, 'paused-context')).toBe('paused-context') + expect(selectInitialResumeContextId(pausePoints, 'unknown-context')).toBe('paused-context') + }) + + it('falls back to the first context when none is paused', () => { + expect( + selectInitialResumeContextId( + [{ contextId: 'first-context', resumeStatus: 'resumed' }] as PausePointWithQueue[], + null + ) + ).toBe('first-context') + }) +}) diff --git a/apps/sim/app/(interfaces)/resume/[workflowId]/[executionId]/resume-page-client.tsx b/apps/sim/app/(interfaces)/resume/[workflowId]/[executionId]/resume-page-client.tsx index 5576c035050..2cc1050d9b0 100644 --- a/apps/sim/app/(interfaces)/resume/[workflowId]/[executionId]/resume-page-client.tsx +++ b/apps/sim/app/(interfaces)/resume/[workflowId]/[executionId]/resume-page-client.tsx @@ -4,6 +4,7 @@ import { useCallback, useEffect, useMemo, useRef, useState } from 'react' import { Badge, Button, + Chip, ChipInput, ChipSelect, ChipTextarea, @@ -22,6 +23,8 @@ import { RefreshCw } from '@sim/emcn/icons' import { formatDateTime } from '@sim/utils/formatting' import { useQueryClient } from '@tanstack/react-query' import { useRouter } from 'next/navigation' +import { isApiClientError } from '@/lib/api/client/errors' +import { ResumeExecutionUnavailable } from '@/app/(interfaces)/resume/[workflowId]/[executionId]/resume-execution-unavailable' import { type PauseContextDetail, type PausedExecutionDetail, @@ -54,7 +57,6 @@ interface ResponseStructureRow { interface ResumeExecutionPageProps { params: { workflowId: string; executionId: string } - initialExecutionDetail: PausedExecutionDetail | null initialContextId?: string | null } @@ -66,6 +68,22 @@ const STATUS_BADGE_VARIANT: Record pausePoint.contextId === requestedContextId) + ) { + return requestedContextId + } + return ( + pausePoints.find((pausePoint) => pausePoint.resumeStatus === 'paused')?.contextId ?? + pausePoints[0]?.contextId + ) +} + function formatDate(value: string | null): string { if (!value) return '—' try { @@ -155,7 +173,6 @@ function renderStructuredValuePreview(value: unknown) { export default function ResumeExecutionPage({ params, - initialExecutionDetail, initialContextId, }: ResumeExecutionPageProps) { const { workflowId, executionId } = params @@ -164,22 +181,23 @@ export default function ResumeExecutionPage({ const { data: executionDetail, + error: executionLoadError, + isError: executionLoadFailed, + isLoading: loadingExecution, isFetching: refreshingExecution, refetch: refetchExecutionDetail, - } = useResumeExecutionDetail(workflowId, executionId, initialExecutionDetail ?? undefined) + } = useResumeExecutionDetail(workflowId, executionId) const pausePoints = executionDetail?.pausePoints ?? [] - const defaultContextId = useMemo(() => { - if (initialContextId) return initialContextId - return ( - pausePoints.find((point) => point.resumeStatus === 'paused')?.contextId ?? - pausePoints[0]?.contextId - ) - }, [initialContextId, pausePoints]) + const defaultContextId = executionDetail + ? selectInitialResumeContextId(pausePoints, initialContextId) + : undefined + const [selectedContextIdOverride, setSelectedContextIdOverride] = useState< + string | null | undefined + >(undefined) + const selectedContextId = + selectedContextIdOverride === undefined ? (defaultContextId ?? null) : selectedContextIdOverride - const [selectedContextId, setSelectedContextId] = useState( - defaultContextId ?? null - ) const { data: selectedDetail, isLoading: loadingDetail } = usePauseContextDetail( workflowId, executionId, @@ -201,6 +219,18 @@ export default function ResumeExecutionPage({ const resumeMutation = useResumeContext() + const executionErrorStatus = isApiClientError(executionLoadError) + ? executionLoadError.status + : null + + useEffect(() => { + if (executionErrorStatus !== 401) return + const resumePath = `/resume/${encodeURIComponent(workflowId)}/${encodeURIComponent(executionId)}${ + initialContextId ? `?${new URLSearchParams({ contextId: initialContextId })}` : '' + }` + router.replace(`/login?callbackUrl=${encodeURIComponent(resumePath)}`) + }, [executionErrorStatus, executionId, initialContextId, router, workflowId]) + const normalizeInputFormatFields = useCallback((raw: any): NormalizedInputField[] => { if (!Array.isArray(raw)) return [] return raw @@ -529,7 +559,7 @@ export default function ResumeExecutionPage({ if (!selectedContextId) { const firstPaused = data?.pausePoints.find((point) => point.resumeStatus === 'paused')?.contextId ?? null - setSelectedContextId(firstPaused) + setSelectedContextIdOverride(firstPaused) } }, [refetchExecutionDetail, selectedContextId]) @@ -635,7 +665,10 @@ export default function ResumeExecutionPage({ } } ) - setSelectedContextId((prev) => (prev !== selectedContextId ? prev : fallbackContextId)) + setSelectedContextIdOverride((override) => { + const currentContextId = override === undefined ? (defaultContextId ?? null) : override + return currentContextId !== selectedContextId ? override : fallbackContextId + }) setMessage( payload.status === 'queued' ? 'Resume request queued.' : 'Resume started successfully.' ) @@ -691,25 +724,50 @@ export default function ResumeExecutionPage({ ) } - // Not found state - if (!executionDetail) { + if (loadingExecution) { return (
-
-

Execution Not Found

-

- This execution could not be located or has already completed. -

- -
+ Loading…
) } + if (executionLoadFailed) { + if (executionErrorStatus === 401) { + return ( +
+ Redirecting to sign in… +
+ ) + } + if (executionErrorStatus === 403 || executionErrorStatus === 404) { + return + } + return ( +
+
+

Could Not Load Execution

+

+ An unexpected error occurred while loading this execution. Please try again. +

+ void refetchExecutionDetail()} + > + {refreshingExecution ? 'Trying again…' : 'Try again'} + +
+
+ ) + } + + if (!executionDetail) { + return + } + return (
@@ -757,7 +815,7 @@ export default function ResumeExecutionPage({ key={pause.contextId} variant={pause.contextId === selectedContextId ? 'active' : 'ghost'} onClick={() => { - setSelectedContextId(pause.contextId) + setSelectedContextIdOverride(pause.contextId) setError(null) setMessage(null) }} diff --git a/apps/sim/app/(landing)/demo/components/demo-scheduler/demo-scheduler.test.tsx b/apps/sim/app/(landing)/demo/components/demo-scheduler/demo-scheduler.test.tsx new file mode 100644 index 00000000000..2f0711f0091 --- /dev/null +++ b/apps/sim/app/(landing)/demo/components/demo-scheduler/demo-scheduler.test.tsx @@ -0,0 +1,174 @@ +/** + * @vitest-environment jsdom + */ +import { act } from 'react' +import { createRoot, type Root } from 'react-dom/client' +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' + +const { mockCal, mockCalComponent, mockConsent, mockGetCalApi, mockTrackGoogleEvent } = vi.hoisted( + () => ({ + mockCal: vi.fn(), + mockCalComponent: vi.fn(() => null), + mockConsent: { marketing: true, measurement: true }, + mockGetCalApi: vi.fn(), + mockTrackGoogleEvent: vi.fn(), + }) +) + +vi.mock('@calcom/embed-react', () => ({ + default: mockCalComponent, + getCalApi: mockGetCalApi, +})) +vi.mock('@/lib/analytics/google', () => ({ trackGoogleEvent: mockTrackGoogleEvent })) +vi.mock('@/lib/consent/scripts', () => ({ X_DEMO_BOOKED_EVENT_ID: 'demo-booked' })) +vi.mock('@/lib/consent/tracking-consent', () => ({ + useTrackingConsent: () => mockConsent, +})) + +import { + DemoScheduler, + preloadCalEmbed, + resolveCalEmbedConfig, +} from '@/app/(landing)/demo/components/demo-scheduler/demo-scheduler' + +const LEAD = { + name: 'Ada Lovelace', + email: 'ada@example.com', + notes: 'Company: Analytical Engines\nTopic: Demo', +} + +describe('DemoScheduler', () => { + let container: HTMLDivElement + let root: Root + + beforeEach(() => { + ;(globalThis as { IS_REACT_ACT_ENVIRONMENT?: boolean }).IS_REACT_ACT_ENVIRONMENT = true + vi.clearAllMocks() + mockConsent.marketing = true + mockConsent.measurement = true + mockGetCalApi.mockResolvedValue(mockCal) + container = document.createElement('div') + document.body.append(container) + root = createRoot(container) + }) + + afterEach(async () => { + await act(async () => { + root.unmount() + await Promise.resolve() + }) + container.remove() + window.twq = undefined + }) + + it('passes the main-branch presentation and lead config to the official embed', async () => { + await act(async () => { + root.render() + await Promise.resolve() + }) + + expect(mockCalComponent).toHaveBeenCalledWith( + expect.objectContaining({ + namespace: 'demo', + calLink: 'team/sim/demo', + calOrigin: 'https://app.cal.com', + embedJsUrl: 'https://app.cal.com/embed/embed.js', + className: 'size-full overflow-auto', + config: { + name: LEAD.name, + email: LEAD.email, + notes: LEAD.notes, + theme: 'light', + 'ui.color-scheme': 'light', + layout: 'month_view', + useSlotsViewOnSmallScreen: 'true', + }, + }), + undefined + ) + expect(mockCal).toHaveBeenCalledWith('ui', { + hideEventTypeDetails: true, + styles: { branding: { brandColor: '#6f3dfa' } }, + }) + }) + + it('registers consent-aware booking analytics and removes the listener on unmount', async () => { + const trackXEvent = vi.fn() + window.twq = trackXEvent + + await act(async () => { + root.render() + await Promise.resolve() + }) + + const registration = mockCal.mock.calls.find(([method]) => method === 'on')?.[1] as + | { action: string; callback: () => void } + | undefined + expect(registration?.action).toBe('bookingSuccessfulV2') + + registration?.callback() + expect(mockTrackGoogleEvent).toHaveBeenCalledWith('get_a_demo', { + page_path: '/demo', + form_name: 'sim_demo', + booking_status: 'scheduled', + }) + expect(trackXEvent).toHaveBeenCalledWith('event', 'demo-booked', {}) + + await act(async () => { + root.unmount() + await Promise.resolve() + }) + expect(mockCal).toHaveBeenCalledWith('off', { + action: 'bookingSuccessfulV2', + callback: registration?.callback, + }) + root = createRoot(container) + }) + + it('does not register booking analytics without measurement or marketing consent', async () => { + mockConsent.marketing = false + mockConsent.measurement = false + + await act(async () => { + root.render() + await Promise.resolve() + }) + + expect(mockCal).toHaveBeenCalledWith('ui', { + hideEventTypeDetails: true, + styles: { branding: { brandColor: '#6f3dfa' } }, + }) + expect(mockCal.mock.calls.some(([method]) => method === 'on')).toBe(false) + }) + + it('preloads the configured booker only once', async () => { + await act(async () => { + preloadCalEmbed() + preloadCalEmbed() + await Promise.resolve() + }) + + expect(mockGetCalApi).toHaveBeenCalledOnce() + expect(mockGetCalApi).toHaveBeenCalledWith({ + namespace: 'demo', + embedJsUrl: 'https://app.cal.com/embed/embed.js', + }) + expect(mockCal).toHaveBeenCalledOnce() + expect(mockCal).toHaveBeenCalledWith('preload', { calLink: 'team/sim/demo' }) + }) + + it('falls back from malformed Cal configuration and preserves valid custom origins', () => { + expect(resolveCalEmbedConfig('javascript:alert(1)')).toEqual({ + calLink: 'team/sim/demo', + calOrigin: 'https://app.cal.com', + embedJsUrl: 'https://app.cal.com/embed/embed.js', + }) + expect(resolveCalEmbedConfig('https://book.example.com/team/demo?theme=light#ignored')).toEqual( + { + calLink: 'team/demo?theme=light', + calOrigin: 'https://book.example.com', + embedJsUrl: 'https://book.example.com/embed/embed.js', + } + ) + }) +}) diff --git a/apps/sim/app/(landing)/demo/components/demo-scheduler/demo-scheduler.tsx b/apps/sim/app/(landing)/demo/components/demo-scheduler/demo-scheduler.tsx index c09e1a266f2..682c1990987 100644 --- a/apps/sim/app/(landing)/demo/components/demo-scheduler/demo-scheduler.tsx +++ b/apps/sim/app/(landing)/demo/components/demo-scheduler/demo-scheduler.tsx @@ -7,9 +7,42 @@ import { X_DEMO_BOOKED_EVENT_ID } from '@/lib/consent/scripts' import { useTrackingConsent } from '@/lib/consent/tracking-consent' import type { DemoLead } from '@/app/(landing)/demo/components/demo-form' -/** The Cal.com event the demo books - set `NEXT_PUBLIC_CAL_LINK` to override. */ const CAL_NAMESPACE = 'demo' -const CAL_LINK = process.env.NEXT_PUBLIC_CAL_LINK ?? 'team/sim/demo' +const DEFAULT_CAL_ORIGIN = 'https://app.cal.com' +const DEFAULT_CAL_LINK = 'team/sim/demo' + +interface CalEmbedConfig { + calLink: string + calOrigin: string + embedJsUrl: string +} + +function parseCalEmbedConfig(link: string): CalEmbedConfig { + const url = new URL(link.replace(/^\/+/, ''), `${DEFAULT_CAL_ORIGIN}/`) + if (!['http:', 'https:'].includes(url.protocol) || url.username || url.password) { + throw new Error('Cal link must use HTTP(S) without embedded credentials') + } + + const calLink = `${url.pathname.replace(/^\/+/, '')}${url.search}` + if (!calLink) throw new Error('Cal link must include an event path') + + return { + calLink, + calOrigin: url.origin, + embedJsUrl: `${url.origin}/embed/embed.js`, + } +} + +/** Resolves the configured booker, falling back safely when the environment value is invalid. */ +export function resolveCalEmbedConfig(configuredLink?: string): CalEmbedConfig { + try { + return parseCalEmbedConfig(configuredLink?.trim() || DEFAULT_CAL_LINK) + } catch { + return parseCalEmbedConfig(DEFAULT_CAL_LINK) + } +} + +const CAL_EMBED = resolveCalEmbedConfig(process.env.NEXT_PUBLIC_CAL_LINK) /** * Sim's brand color, matching the `--brand-agent` token. The embed renders in a @@ -37,9 +70,9 @@ let calEmbedPreloaded = false export function preloadCalEmbed(): void { if (calEmbedPreloaded) return calEmbedPreloaded = true - getCalApi({ namespace: CAL_NAMESPACE }) + getCalApi({ namespace: CAL_NAMESPACE, embedJsUrl: CAL_EMBED.embedJsUrl }) .then((cal) => { - cal('preload', { calLink: CAL_LINK }) + cal('preload', { calLink: CAL_EMBED.calLink }) }) .catch(() => { calEmbedPreloaded = false @@ -71,7 +104,7 @@ export function DemoScheduler({ lead }: DemoSchedulerProps) { } if (marketing) window.twq?.('event', X_DEMO_BOOKED_EVENT_ID, {}) } - const api = getCalApi({ namespace: CAL_NAMESPACE }) + const api = getCalApi({ namespace: CAL_NAMESPACE, embedJsUrl: CAL_EMBED.embedJsUrl }) api .then((cal) => { if (cancelled) return @@ -104,8 +137,10 @@ export function DemoScheduler({ lead }: DemoSchedulerProps) {
+

+ {comparison.heading} +

+

+ {comparison.intro} +

+ +
+ + + + + {comparison.columns.map((column) => ( + + ))} + + + + + {comparison.columns.map((column, index) => ( + + ))} + + + + {comparison.rows.map((row) => ( + + + {row.values.map((value, index) => ( + + ))} + + ))} + +
{comparison.heading}
+ Capability + + {column} +
+ {row.label} + + {value.href ? ( + + {value.text} + + ) : ( + value.text + )} +
+
+ +

+ {comparison.conclusion} +

+ + ) +} diff --git a/apps/sim/app/(landing)/integrations/(shell)/[slug]/page.tsx b/apps/sim/app/(landing)/integrations/(shell)/[slug]/page.tsx index 3b373807a1b..7cced85a754 100644 --- a/apps/sim/app/(landing)/integrations/(shell)/[slug]/page.tsx +++ b/apps/sim/app/(landing)/integrations/(shell)/[slug]/page.tsx @@ -18,6 +18,7 @@ import { BackLink } from '@/app/(landing)/components' import { JsonLd } from '@/app/(landing)/components/json-ld' import { LandingFAQ } from '@/app/(landing)/components/landing-faq' import { ShareButton } from '@/app/(landing)/components/share-button' +import { IntegrationComparisonSection } from '@/app/(landing)/integrations/(shell)/[slug]/components/integration-comparison-section/integration-comparison-section' import { IntegrationCtaButton } from '@/app/(landing)/integrations/(shell)/[slug]/components/integration-cta-button' import { TemplateCardButton } from '@/app/(landing)/integrations/(shell)/[slug]/components/template-card-button' import { IntegrationIcon } from '@/app/(landing)/integrations/components/integration-icon' @@ -391,10 +392,12 @@ export default async function IntegrationPage({ params }: { params: Promise<{ sl const relatedIntegrations = relatedSlugs .map((s) => bySlug.get(s)) .filter((i): i is Integration => i !== undefined) - const faqs = buildFAQs( - integration, - relatedIntegrations.map((i) => i.name) - ) + const faqs = + seo?.faqs ?? + buildFAQs( + integration, + relatedIntegrations.map((i) => i.name) + ) const matchingTemplates = getTemplatesForBlock(integration.type) .sort( (a, b) => @@ -685,6 +688,13 @@ export default async function IntegrationPage({ params }: { params: Promise<{ sl
+ {seo?.comparison && ( + <> + +
+ + )} + {/* Triggers - rows */} {triggers.length > 0 && (
@@ -883,6 +893,34 @@ export default async function IntegrationPage({ params }: { params: Promise<{ sl
)} + {seo?.narrativeComparison && ( + <> +
+

+ {seo.narrativeComparison.heading} +

+
+ {seo.narrativeComparison.paragraphs.map((paragraph) => ( +

+ {paragraph} +

+ ))} +
+
+
+ + )} + {/* FAQ - full width */}

= { 'slack workflow automation', 'slack integration', ], - h1: 'Slack Integrations for Workflow Automation', + h1: 'Slack Workflow Automation with Sim', tagline: 'Build Slack workflow automation in Sim. Send, update, delete, and read messages; manage channels, users, canvases, and modals; and trigger AI agents from mentions, messages, and reactions in real time.', overview: - 'Use Sim as your Slack integration for team communication and operations. Build Slack automation that routes requests, posts alerts, summarises threads, updates tickets, and keeps work moving. Sim supports messages, reactions, canvases, views, channel and user lookups, file downloads, and real-time Slack workflows in one workspace.', + 'Sim automates Slack workflows that route messages, trigger alerts, summarize threads, update tickets, and manage incident response. Slack messages and events start agent workflows that interpret what was said and choose the next action in Slack or a connected tool, so routine coordination and time-sensitive operations keep moving without anyone relaying details by hand.', triggersIntro: - 'Connect the Slack Webhook trigger to Sim and run Slack workflow automation the moment a mention, message, or reaction happens, no polling, no delay.', + 'Sim supports one real-time Slack trigger. Select the Slack events you care about, such as mentions, messages, and reactions, and Sim starts the connected workflow the moment one arrives instead of waiting for a scheduled check. A monitoring alert posted in Slack can open an incident-response workflow, and a ticketing update posted in Slack can be summarised and passed to another connected tool.', templatesIntro: - 'Ready-to-use Slack automation templates for Q&A bots, sales alerts, incident response, standups, digests, and CRM updates. Click any template to launch a workflow faster.', + 'Pre-built agent templates turn common Slack workflows into editable starting points: routing templates classify messages and send them to the right channel or owner, summarisation templates condense long threads into updates that preserve decisions and action items, and ticket sync and incident response templates update connected records and coordinate follow-up. Every template is editable, so you can adapt its channels, routing rules, data sources, and approval requirements.', + toolsSubtitleSuffix: + ' across messaging, channels, threads, users, reactions and files, and canvases and views. Combine multiple Slack actions in one workflow to summarise a message, route it, update a ticket, and post the ticket update back in Slack', + narrativeComparison: { + id: 'slack-automation-comparison', + heading: 'Slack automation with Sim vs. Zapier, Make, and n8n', + paragraphs: [ + 'Sim is built for Slack workflows that need to interpret conversation context before choosing an action. A Sim agent can summarize a thread and determine whether it describes an incident before using a configured Slack or ticketing action.', + 'Zapier, Make, and n8n are useful substitutes for predefined trigger-action sequences with known conditions and branches. Choose that model when each event should produce a predictable result. For example, a simple rule that posts every new message from channel X to channel Y works well as a fixed chain because it does not require interpretation. Choose Sim when the next action depends on the meaning of a Slack conversation rather than fixed keywords or predetermined branches.', + ], + }, + faqs: [ + { + question: 'What Slack workflows can Sim automate?', + answer: + 'Slack workflow automation uses messages and channel events to start work in connected tools. Sim can route Slack messages, trigger alerts, summarize threads, update tickets, and coordinate incident response. You can use Sim to turn Slack conversations into tracked work without manually copying details.', + }, + { + question: 'How does routing and alerting work in Sim?', + answer: + 'Routing and alerting send selected Slack messages to the people or channels responsible for responding. Sim uses Slack events and workflow logic to route a message or send an alert based on its content. This directs requests and incidents without requiring someone to monitor every channel.', + }, + { + question: 'Can Sim summarize Slack threads?', + answer: + 'Thread summarization condenses a Slack conversation into its main points and relevant context. Sim can read a Slack thread and send the resulting summary to another Slack channel or a workflow that uses connected tools.', + }, + { + question: 'How does Sim update tickets from Slack?', + answer: + 'A Slack-to-ticket workflow transfers conversation details into a connected ticketing tool. Sim can pass information from a Slack message or thread to the ticketing tool and update the matching record. You can use Sim to keep tickets current without entering the same information twice.', + }, + { + question: 'Does Sim support incident response in Slack?', + answer: + 'Incident response workflows use Slack events to start response steps and coordinate follow-up. Sim can trigger an incident workflow from Slack and pass relevant context to connected response tools. This keeps responders informed and incident records current as the conversation develops.', + }, + { + question: 'What is the difference between Sim and Zapier for Slack?', + answer: + 'Zapier supports trigger-action automation, while Sim centers Slack automation on workflows that can include AI agents. Sim workflows can interpret a Slack incident thread before routing the relevant information or updating a ticket. You can use Sim when the workflow needs to understand conversation context before taking action.', + }, + ], + }, + 'twilio-sms': { + tagline: + 'Sim connects to Twilio to let an AI agent send SMS updates when an order status changes. Connect Twilio SMS to Sim to send outbound messages and start AI workflows when Twilio reports an inbound SMS, inbound MMS, or message status change.', + overview: + 'The Twilio SMS block sends outbound messages. Twilio SMS Received and Twilio Message Status triggers start Sim workflows from supported webhook events.', + comparison: { + id: 'twilio-sms-comparison', + heading: 'Sim vs. Zapier, Make, and n8n for Twilio SMS', + intro: + "Zapier, Make, and n8n provide Twilio integrations for general-purpose automation. Sim focuses on AI workflows that use Agent blocks to draft or transform messages with order or customer data before a Twilio SMS block sends them. Supported Twilio webhooks can start Sim workflows without scheduled polling. Zapier, Make, or n8n may suit a Twilio workflow that does not require Sim's Agent blocks.", + columns: ['Sim', 'Zapier', 'Make', 'n8n'], + rows: [ + { + label: 'Trigger model', + values: [ + { text: 'Real-time webhook' }, + { + text: 'Polling for new SMS', + href: 'https://zapier.com/apps/twilio/integrations', + }, + { + text: 'Instant webhook', + href: 'https://apps.make.com/twilio', + }, + { + text: 'Webhook', + href: 'https://docs.n8n.io/integrations/builtin/trigger-nodes/n8n-nodes-base.twiliotrigger/', + }, + ], + }, + { + label: 'AI drafting before send', + values: [ + { text: 'Agent block, built in' }, + { + text: 'Requires a separate AI step or app', + href: 'https://zapier.com/apps/twilio/integrations/chatgpt', + }, + { + text: 'Requires a separate AI step or app', + href: 'https://www.make.com/en/integrations/twilio/openai-gpt-3', + }, + { + text: 'Requires a separate AI node', + href: 'https://docs.n8n.io/integrations/builtin/cluster-nodes/root-nodes/n8n-nodes-langchain.agent/', + }, + ], + }, + { + label: 'Free to start', + values: [ + { text: 'Yes' }, + { + text: 'Yes, with a limited free tier', + href: 'https://zapier.com/pricing', + }, + { + text: 'Yes, with a limited free tier', + href: 'https://www.make.com/en/pricing', + }, + { + text: 'Yes, when self-hosted', + href: 'https://docs.n8n.io/privacy-and-security/sustainable-use-license/', + }, + ], + }, + ], + conclusion: + 'Choose Sim when the workflow needs an AI agent to read order or customer context and write the message, rather than relay a fixed template.', + }, + triggersIntro: + 'A supported Twilio SMS webhook starts a Sim workflow when Sim receives the corresponding event. Webhook delivery removes the need for scheduled polling, but Twilio processing and network conditions can affect when Sim receives an event.', + faqs: [ + { + question: 'What can I do with the Twilio SMS integration?', + answer: + "Sim's Twilio SMS integration lets an AI agent send SMS messages and receive SMS or MMS messages inside a workflow. You can combine Twilio SMS with Agent blocks and integrations such as AgentPhone and Discord. Combining these blocks lets you generate messages and coordinate responses across supported tools in one workflow.", + }, + { + question: 'How do I connect Twilio SMS to Sim?', + answer: + 'A Twilio SMS connection authorizes Sim to use the Twilio actions and triggers you configure. Create an account at sim.ai, add a Twilio SMS block to a workflow, and enter the Twilio credentials requested in the connection settings. After you choose an action or trigger and test the workflow, you can deploy it to process supported Twilio events.', + }, + { + question: 'How do Twilio SMS triggers work?', + answer: + 'A Twilio SMS trigger starts a Sim workflow when its webhook receives a supported Twilio event. Add a trigger block to the workflow, then configure the generated webhook URL in the relevant Twilio settings for Twilio SMS Received or Twilio Message Status events. The configured webhooks start matching workflows without scheduled polling.', + }, + { + question: 'What data does a Twilio SMS trigger provide?', + answer: + 'A Twilio SMS trigger provides the fields in the webhook payload that Twilio sends to Sim. Twilio SMS Received events include data about an inbound SMS or MMS, which the workflow can pass to Agent blocks or later steps. Available fields vary by event and payload, so inspect a test event before mapping them.', + }, + ], }, airtable: { title: 'Airtable Automation with Sim', diff --git a/apps/sim/app/(landing)/integrations/data/types.ts b/apps/sim/app/(landing)/integrations/data/types.ts index b64d7120db4..c68812d79e3 100644 --- a/apps/sim/app/(landing)/integrations/data/types.ts +++ b/apps/sim/app/(landing)/integrations/data/types.ts @@ -29,6 +29,36 @@ export interface IntegrationLandingContent { aiDisclaimer?: string } +export interface IntegrationComparisonRow { + label: string + values: IntegrationComparisonValue[] +} + +export interface IntegrationComparisonValue { + text: string + href?: string +} + +export interface IntegrationComparisonContent { + id: string + heading: string + intro: string + columns: string[] + rows: IntegrationComparisonRow[] + conclusion: string +} + +export interface IntegrationFaqContent { + question: string + answer: string +} + +export interface IntegrationNarrativeSectionContent { + id: string + heading: string + paragraphs: string[] +} + /** * Hand-authored, per-integration SEO/GEO overrides keyed by slug. Unlike * {@link IntegrationLandingContent}, this is consumed at render time directly by @@ -57,6 +87,12 @@ export interface IntegrationSeoContent { triggersIntro?: string /** Agent-templates intro paragraph, overriding the generated default. */ templatesIntro?: string + /** Optional comparison section rendered immediately before real-time triggers. */ + comparison?: IntegrationComparisonContent + /** Optional prose comparison rendered after the supported-tools list. */ + narrativeComparison?: IntegrationNarrativeSectionContent + /** Full FAQ replacement for integrations with hand-authored answers. */ + faqs?: IntegrationFaqContent[] /** * Text appended to the `"{n} {name} tool(s) available in Sim"` subtitle (e.g. * `" for Confluence automation across pages, blog posts, …"`). Keeps the tool diff --git a/apps/sim/app/_shell/paste-admission-guard.test.tsx b/apps/sim/app/_shell/paste-admission-guard.test.tsx index d10ad45b2c0..50b3d9dc0dc 100644 --- a/apps/sim/app/_shell/paste-admission-guard.test.tsx +++ b/apps/sim/app/_shell/paste-admission-guard.test.tsx @@ -17,6 +17,18 @@ import { PasteAdmissionGuard } from '@/app/_shell/paste-admission-guard' let host: HTMLDivElement let root: Root +const selectionContext = { + kind: 'table_selection', + tableId: 'table-1', + tableName: 'Large table', + rowIds: ['row-1'], + label: 'Large table (1 row)', +} + +function selectionPayload(sourceWorkspaceId = 'ws-1'): string { + return JSON.stringify({ version: 1, sourceWorkspaceId, context: selectionContext }) +} + function dispatchPaste( target: Element, text: string, @@ -116,32 +128,33 @@ describe('PasteAdmissionGuard', () => { it('lets a prompt consume a compact Sim selection reference before its large plain text', () => { const input = document.createElement('textarea') input.dataset.pasteMaxBytes = '4' - input.dataset.pasteSelectionContext = 'reference' + input.dataset.pasteSelectionContext = 'ws-1' + host.appendChild(input) + + expect( + dispatchPaste(input, '12345', { selectionContext: selectionPayload() }).defaultPrevented + ).toBe(false) + }) + + it('still bounds a cross-workspace selection plain-text representation', () => { + const input = document.createElement('textarea') + input.dataset.pasteMaxBytes = '4' + input.dataset.pasteSelectionContext = 'ws-2' host.appendChild(input) - const selectionContext = JSON.stringify({ - kind: 'table_selection', - tableId: 'table-1', - tableName: 'Large table', - rowIds: ['row-1'], - label: 'Large table (1 row)', - }) - expect(dispatchPaste(input, '12345', { selectionContext }).defaultPrevented).toBe(false) + expect( + dispatchPaste(input, '12345', { selectionContext: selectionPayload() }).defaultPrevented + ).toBe(true) }) it('still bounds a Sim selection plain-text representation outside the prompt', () => { const input = document.createElement('textarea') input.dataset.pasteMaxBytes = '4' host.appendChild(input) - const selectionContext = JSON.stringify({ - kind: 'table_selection', - tableId: 'table-1', - tableName: 'Large table', - rowIds: ['row-1'], - label: 'Large table (1 row)', - }) - expect(dispatchPaste(input, '12345', { selectionContext }).defaultPrevented).toBe(true) + expect( + dispatchPaste(input, '12345', { selectionContext: selectionPayload() }).defaultPrevented + ).toBe(true) }) it('bounds rich HTML separately from its smaller plain-text representation', () => { diff --git a/apps/sim/app/_shell/paste-admission-guard.tsx b/apps/sim/app/_shell/paste-admission-guard.tsx index 1a12cb1f3a7..2ee15ab9c06 100644 --- a/apps/sim/app/_shell/paste-admission-guard.tsx +++ b/apps/sim/app/_shell/paste-admission-guard.tsx @@ -43,7 +43,15 @@ export function PasteAdmissionGuard() { } const acceptsSelectionContext = event.target.closest('[data-paste-selection-context]') - if (acceptsSelectionContext && readSelectionContextFromClipboard(event.clipboardData)) return + const destinationWorkspaceId = acceptsSelectionContext?.getAttribute( + 'data-paste-selection-context' + ) + if ( + destinationWorkspaceId && + readSelectionContextFromClipboard(event.clipboardData, destinationWorkspaceId) + ) { + return + } const handlesImageFiles = event.target.closest('[data-paste-handles-images="true"]') if (handlesImageFiles && clipboardHasImageFile(event.clipboardData)) return diff --git a/apps/sim/app/_shell/public-env-script.test.tsx b/apps/sim/app/_shell/public-env-script.test.tsx index efe90cb73d1..96262f09916 100644 --- a/apps/sim/app/_shell/public-env-script.test.tsx +++ b/apps/sim/app/_shell/public-env-script.test.tsx @@ -4,10 +4,19 @@ import { renderToStaticMarkup } from 'react-dom/server' import { describe, expect, it, vi } from 'vitest' import { PUBLIC_ENV_ATTRIBUTE } from '@/lib/core/config/env' -import { PublicEnvScript, publicEnvHtmlAttributes } from '@/app/_shell/public-env-script' +import { + PublicEnvScript, + publicEnvHtmlAttributes, + RuntimePublicEnvScript, + serializePublicEnv, +} from '@/app/_shell/public-env-script' vi.unmock('@/lib/core/config/env') +const { mockConnection } = vi.hoisted(() => ({ mockConnection: vi.fn() })) + +vi.mock('next/server', () => ({ connection: mockConnection })) + /** * Guards the one property that matters: the emitted tag assigns `window.__ENV` * itself. Next's `beforeInteractive` strategy instead pushes the assignment onto @@ -29,11 +38,35 @@ describe('PublicEnvScript', () => { expect(markup).not.toContain('__next_s') }) + it('cannot be terminated by a script-like public value', () => { + const serialized = serializePublicEnv({ + NEXT_PUBLIC_SCRIPT_ESCAPE_TEST: '', + }) + + expect(serialized).not.toContain('') + expect(serialized).toContain('\\u003c/script>') + }) + it('passes only NEXT_PUBLIC_ variables through to the browser', () => { const keys = Object.keys(PublicEnvScript().props.env) expect(keys.every((key) => /^NEXT_PUBLIC_/i.test(key))).toBe(true) }) + + it('reads self-hosted values at request time after opting into dynamic rendering', async () => { + const previous = process.env.NEXT_PUBLIC_RUNTIME_ENV_TEST + process.env.NEXT_PUBLIC_RUNTIME_ENV_TEST = 'runtime-value' + + try { + const script = await RuntimePublicEnvScript() + expect(mockConnection).toHaveBeenCalledOnce() + expect(script.props.env.NEXT_PUBLIC_RUNTIME_ENV_TEST).toBe('runtime-value') + } finally { + if (previous === undefined) + Reflect.deleteProperty(process.env, 'NEXT_PUBLIC_RUNTIME_ENV_TEST') + else process.env.NEXT_PUBLIC_RUNTIME_ENV_TEST = previous + } + }) }) /** diff --git a/apps/sim/app/_shell/public-env-script.tsx b/apps/sim/app/_shell/public-env-script.tsx index a1a97765c7a..ccc39d0f30c 100644 --- a/apps/sim/app/_shell/public-env-script.tsx +++ b/apps/sim/app/_shell/public-env-script.tsx @@ -1,9 +1,8 @@ -import { EnvScript } from 'next-runtime-env' +import { connection } from 'next/server' import { PUBLIC_ENV_ATTRIBUTE } from '@/lib/core/config/env' /** - * Every `NEXT_PUBLIC_*` value currently in `process.env`. Filter matches - * `next-runtime-env`'s own `getPublicEnv()` exactly. + * Every `NEXT_PUBLIC_*` value currently in `process.env`. */ function readPublicEnv(): Record { return Object.fromEntries( @@ -36,33 +35,44 @@ const HOSTED_PUBLIC_ENV = readPublicEnv() * * Read fresh rather than from {@link HOSTED_PUBLIC_ENV} so the one helper serves * both deployment modes: self-hosted images re-inject env per deploy without a - * rebuild, and `next-runtime-env`'s script reads `process.env` per request for - * exactly that reason. On hosted the two reads are the same values, because - * nothing mutates `process.env` after boot. + * rebuild. On hosted the two reads are the same values, because nothing mutates + * `process.env` after boot. */ export function publicEnvHtmlAttributes(): Record { return { [PUBLIC_ENV_ATTRIBUTE]: JSON.stringify(readPublicEnv()) } } /** - * Static equivalent of `next-runtime-env`'s `` for the hosted - * deployment. It renders the library's own ``, so the emitted markup - * is identical to the self-hosted path - only the env read differs. - * `` additionally calls `unstable_noStore()`, which opts the - * entire app into dynamic rendering; that only pays off for self-hosted Docker - * images that re-inject env per deploy without a rebuild, so hosted reads the - * env once here and stays static. - * - * `disableNextScript` is load-bearing. Without it, `` defaults to - * Next's `') - expect(result.isValid).toBe(false) - expect(result.error).toContain('format is invalid') - }) - }) -}) - describe('validateAirtableId', () => { describe('valid base IDs (app prefix)', () => { it.concurrent('should accept valid base ID', () => { @@ -1867,264 +1680,6 @@ describe('validateMondayNumericId', () => { }) }) -describe('validateMondayGroupId', () => { - describe('valid inputs', () => { - it.concurrent('should accept simple group IDs', () => { - const result = validateMondayGroupId('topics') - expect(result.isValid).toBe(true) - expect(result.sanitized).toBe('topics') - }) - - it.concurrent('should accept group IDs with underscores', () => { - const result = validateMondayGroupId('new_group') - expect(result.isValid).toBe(true) - }) - - it.concurrent('should accept group IDs with spaces', () => { - const result = validateMondayGroupId('test group id') - expect(result.isValid).toBe(true) - }) - - it.concurrent('should accept group IDs with uppercase letters', () => { - const result = validateMondayGroupId('Group One') - expect(result.isValid).toBe(true) - }) - - it.concurrent('should accept group IDs with digits', () => { - const result = validateMondayGroupId('group123') - expect(result.isValid).toBe(true) - }) - - it.concurrent('should accept auto-generated group IDs', () => { - const result = validateMondayGroupId('group_title') - expect(result.isValid).toBe(true) - }) - }) - - describe('invalid inputs', () => { - it.concurrent('should reject null', () => { - const result = validateMondayGroupId(null) - expect(result.isValid).toBe(false) - }) - - it.concurrent('should reject empty string', () => { - const result = validateMondayGroupId('') - expect(result.isValid).toBe(false) - }) - - it.concurrent('should reject strings with brackets', () => { - const result = validateMondayGroupId('group"]){id}#') - expect(result.isValid).toBe(false) - }) - - it.concurrent('should reject strings with quotes', () => { - const result = validateMondayGroupId('group")') - expect(result.isValid).toBe(false) - }) - - it.concurrent('should reject control characters', () => { - const result = validateMondayGroupId('group\x00id') - expect(result.isValid).toBe(false) - }) - - it.concurrent('should reject strings exceeding max length', () => { - const result = validateMondayGroupId('a'.repeat(256)) - expect(result.isValid).toBe(false) - }) - - it.concurrent('should reject strings with special characters', () => { - const result = validateMondayGroupId('group;DROP') - expect(result.isValid).toBe(false) - }) - }) -}) - -describe('validateMondayColumnId', () => { - describe('valid inputs', () => { - it.concurrent('should accept simple column IDs', () => { - const result = validateMondayColumnId('status') - expect(result.isValid).toBe(true) - expect(result.sanitized).toBe('status') - }) - - it.concurrent('should accept column IDs with digits', () => { - const result = validateMondayColumnId('date4') - expect(result.isValid).toBe(true) - }) - - it.concurrent('should accept auto-generated column IDs', () => { - const result = validateMondayColumnId('email_mksr9hcd') - expect(result.isValid).toBe(true) - }) - - it.concurrent('should accept column IDs with underscores', () => { - const result = validateMondayColumnId('color_mksreyj6') - expect(result.isValid).toBe(true) - }) - - it.concurrent('should accept single character column IDs', () => { - const result = validateMondayColumnId('a') - expect(result.isValid).toBe(true) - }) - }) - - describe('invalid inputs', () => { - it.concurrent('should reject null', () => { - const result = validateMondayColumnId(null) - expect(result.isValid).toBe(false) - }) - - it.concurrent('should reject empty string', () => { - const result = validateMondayColumnId('') - expect(result.isValid).toBe(false) - }) - - it.concurrent('should reject uppercase letters', () => { - const result = validateMondayColumnId('Status') - expect(result.isValid).toBe(false) - }) - - it.concurrent('should reject spaces', () => { - const result = validateMondayColumnId('my column') - expect(result.isValid).toBe(false) - }) - - it.concurrent('should reject hyphens', () => { - const result = validateMondayColumnId('my-column') - expect(result.isValid).toBe(false) - }) - - it.concurrent('should reject special characters', () => { - const result = validateMondayColumnId('col;DROP') - expect(result.isValid).toBe(false) - }) - - it.concurrent('should reject strings exceeding max length', () => { - const result = validateMondayColumnId('a'.repeat(256)) - expect(result.isValid).toBe(false) - }) - }) - - describe('validateSupabaseProjectId', () => { - describe('valid inputs', () => { - it.concurrent('should accept a typical 20-char lowercase alphanumeric project ID', () => { - const result = validateSupabaseProjectId('jdrkgepadsdopsntdlom') - expect(result.isValid).toBe(true) - expect(result.sanitized).toBe('jdrkgepadsdopsntdlom') - }) - - it.concurrent('should accept project IDs with digits', () => { - const result = validateSupabaseProjectId('abc123def456ghi789jk') - expect(result.isValid).toBe(true) - }) - - it.concurrent('should accept IDs at the minimum length boundary (10)', () => { - const result = validateSupabaseProjectId('abcdefghij') - expect(result.isValid).toBe(true) - }) - - it.concurrent('should accept IDs at the maximum length boundary (40)', () => { - const result = validateSupabaseProjectId('a'.repeat(40)) - expect(result.isValid).toBe(true) - }) - }) - - describe('SSRF attack vectors', () => { - it.concurrent('should reject fragment injection (#)', () => { - const result = validateSupabaseProjectId('evil#attacker.com') - expect(result.isValid).toBe(false) - }) - - it.concurrent('should reject @ for authority injection', () => { - const result = validateSupabaseProjectId('evil@attacker.com') - expect(result.isValid).toBe(false) - }) - - it.concurrent('should reject path traversal with slashes', () => { - const result = validateSupabaseProjectId('evil/../../etc/passwd') - expect(result.isValid).toBe(false) - }) - - it.concurrent('should reject dots (subdomain manipulation)', () => { - const result = validateSupabaseProjectId('evil.attacker.com') - expect(result.isValid).toBe(false) - }) - - it.concurrent('should reject backslashes', () => { - const result = validateSupabaseProjectId('evil\\path') - expect(result.isValid).toBe(false) - }) - - it.concurrent('should reject colons (port injection)', () => { - const result = validateSupabaseProjectId('evil:8080') - expect(result.isValid).toBe(false) - }) - - it.concurrent('should reject URL-encoded characters', () => { - const result = validateSupabaseProjectId('evil%23attacker') - expect(result.isValid).toBe(false) - }) - - it.concurrent('should reject spaces', () => { - const result = validateSupabaseProjectId('evil host') - expect(result.isValid).toBe(false) - }) - - it.concurrent('should reject newlines (header injection)', () => { - const result = validateSupabaseProjectId('evil\r\nHost: attacker.com') - expect(result.isValid).toBe(false) - }) - }) - - describe('invalid formats', () => { - it.concurrent('should reject null', () => { - const result = validateSupabaseProjectId(null) - expect(result.isValid).toBe(false) - }) - - it.concurrent('should reject undefined', () => { - const result = validateSupabaseProjectId(undefined) - expect(result.isValid).toBe(false) - }) - - it.concurrent('should reject empty string', () => { - const result = validateSupabaseProjectId('') - expect(result.isValid).toBe(false) - }) - - it.concurrent('should reject uppercase letters', () => { - const result = validateSupabaseProjectId('JDRKGEPADSDOPSNTDLOM') - expect(result.isValid).toBe(false) - }) - - it.concurrent('should reject mixed case', () => { - const result = validateSupabaseProjectId('jdrkGEPadsdOPSntdlom') - expect(result.isValid).toBe(false) - }) - - it.concurrent('should reject hyphens', () => { - const result = validateSupabaseProjectId('jdrk-gepa-dsdo-psnt') - expect(result.isValid).toBe(false) - }) - - it.concurrent('should reject underscores', () => { - const result = validateSupabaseProjectId('jdrk_gepa_dsdo_psnt') - expect(result.isValid).toBe(false) - }) - - it.concurrent('should reject IDs shorter than 10 characters', () => { - const result = validateSupabaseProjectId('abcdefghi') - expect(result.isValid).toBe(false) - }) - - it.concurrent('should reject IDs longer than 40 characters', () => { - const result = validateSupabaseProjectId('a'.repeat(41)) - expect(result.isValid).toBe(false) - }) - }) - }) -}) - describe('validateCallbackUrl', () => { const ORIGIN = 'https://sim.app' const originalWindow = (globalThis as { window?: unknown }).window @@ -2291,7 +1846,7 @@ describe('validateServiceNowInstanceUrl', () => { it.concurrent('should reject private IPs', () => { const result = validateServiceNowInstanceUrl('https://192.168.1.1') expect(result.isValid).toBe(false) - expect(result.error).toContain('private IP') + expect(result.error).toContain('private or reserved address') }) it.concurrent('should reject link-local metadata IP', () => { @@ -2389,7 +1944,7 @@ describe('validateWorkdayTenantUrl', () => { it.concurrent('should reject private IPs', () => { const result = validateWorkdayTenantUrl('https://192.168.1.1') expect(result.isValid).toBe(false) - expect(result.error).toContain('private IP') + expect(result.error).toContain('private or reserved address') }) it.concurrent('should reject link-local metadata IP (SSRF classic)', () => { @@ -2409,3 +1964,122 @@ describe('validateWorkdayTenantUrl', () => { }) }) }) + +describe('validateSupabaseProjectId', () => { + describe('valid inputs', () => { + it.concurrent('should accept a typical 20-char lowercase alphanumeric project ID', () => { + const result = validateSupabaseProjectId('jdrkgepadsdopsntdlom') + expect(result.isValid).toBe(true) + expect(result.sanitized).toBe('jdrkgepadsdopsntdlom') + }) + + it.concurrent('should accept project IDs with digits', () => { + const result = validateSupabaseProjectId('abc123def456ghi789jk') + expect(result.isValid).toBe(true) + }) + + it.concurrent('should accept IDs at the minimum length boundary (10)', () => { + const result = validateSupabaseProjectId('abcdefghij') + expect(result.isValid).toBe(true) + }) + + it.concurrent('should accept IDs at the maximum length boundary (40)', () => { + const result = validateSupabaseProjectId('a'.repeat(40)) + expect(result.isValid).toBe(true) + }) + }) + + describe('SSRF attack vectors', () => { + it.concurrent('should reject fragment injection (#)', () => { + const result = validateSupabaseProjectId('evil#attacker.com') + expect(result.isValid).toBe(false) + }) + + it.concurrent('should reject @ for authority injection', () => { + const result = validateSupabaseProjectId('evil@attacker.com') + expect(result.isValid).toBe(false) + }) + + it.concurrent('should reject path traversal with slashes', () => { + const result = validateSupabaseProjectId('evil/../../etc/passwd') + expect(result.isValid).toBe(false) + }) + + it.concurrent('should reject dots (subdomain manipulation)', () => { + const result = validateSupabaseProjectId('evil.attacker.com') + expect(result.isValid).toBe(false) + }) + + it.concurrent('should reject backslashes', () => { + const result = validateSupabaseProjectId('evil\\path') + expect(result.isValid).toBe(false) + }) + + it.concurrent('should reject colons (port injection)', () => { + const result = validateSupabaseProjectId('evil:8080') + expect(result.isValid).toBe(false) + }) + + it.concurrent('should reject URL-encoded characters', () => { + const result = validateSupabaseProjectId('evil%23attacker') + expect(result.isValid).toBe(false) + }) + + it.concurrent('should reject spaces', () => { + const result = validateSupabaseProjectId('evil host') + expect(result.isValid).toBe(false) + }) + + it.concurrent('should reject newlines (header injection)', () => { + const result = validateSupabaseProjectId('evil\r\nHost: attacker.com') + expect(result.isValid).toBe(false) + }) + }) + + describe('invalid formats', () => { + it.concurrent('should reject null', () => { + const result = validateSupabaseProjectId(null) + expect(result.isValid).toBe(false) + }) + + it.concurrent('should reject undefined', () => { + const result = validateSupabaseProjectId(undefined) + expect(result.isValid).toBe(false) + }) + + it.concurrent('should reject empty string', () => { + const result = validateSupabaseProjectId('') + expect(result.isValid).toBe(false) + }) + + it.concurrent('should reject uppercase letters', () => { + const result = validateSupabaseProjectId('JDRKGEPADSDOPSNTDLOM') + expect(result.isValid).toBe(false) + }) + + it.concurrent('should reject mixed case', () => { + const result = validateSupabaseProjectId('jdrkGEPadsdOPSntdlom') + expect(result.isValid).toBe(false) + }) + + it.concurrent('should reject hyphens', () => { + const result = validateSupabaseProjectId('jdrk-gepa-dsdo-psnt') + expect(result.isValid).toBe(false) + }) + + it.concurrent('should reject underscores', () => { + const result = validateSupabaseProjectId('jdrk_gepa_dsdo_psnt') + expect(result.isValid).toBe(false) + }) + + it.concurrent('should reject IDs shorter than 10 characters', () => { + const result = validateSupabaseProjectId('abcdefghi') + expect(result.isValid).toBe(false) + }) + + it.concurrent('should reject IDs longer than 40 characters', () => { + const result = validateSupabaseProjectId('a'.repeat(41)) + expect(result.isValid).toBe(false) + }) + }) +}) diff --git a/apps/sim/lib/core/security/input-validation.ts b/apps/sim/lib/core/security/input-validation.ts index fb448e04a56..21f88bef1e1 100644 --- a/apps/sim/lib/core/security/input-validation.ts +++ b/apps/sim/lib/core/security/input-validation.ts @@ -1,7 +1,11 @@ import { createLogger } from '@sim/logger' -import { isLoopbackIp, isPrivateIp, unwrapIpv6Brackets } from '@sim/security/ssrf' -import * as ipaddr from 'ipaddr.js' -import { isHosted } from '@/lib/core/config/env-flags' +import { evaluateUrl, isLiftableByVouching, policyDefersToAddress } from '@sim/security/egress' +import { isIpLiteral, unwrapIpv6Brackets } from '@sim/security/ssrf' +import { + describeEgressDenial, + type EgressProfile, + resolveEgressPolicy, +} from '@/lib/core/security/egress/profiles' import { getBaseUrl } from '@/lib/core/utils/urls' const logger = createLogger('InputValidation') @@ -12,7 +16,8 @@ export interface ValidationResult { sanitized?: string } -export interface PathSegmentOptions { +/** Options for {@link validatePathSegment}. */ +interface PathSegmentOptions { /** Name of the parameter for error messages */ paramName?: string /** Maximum length allowed (default: 255) */ @@ -246,80 +251,6 @@ export function validateNumericId( return { isValid: true, sanitized: num.toString() } } -/** - * Validates an integer value (from JSON body or other sources) - * - * This is stricter than validateNumericId - it requires: - * - Value must already be a number type (not string) - * - Must be an integer (no decimals) - * - Must be finite (not NaN or Infinity) - * - * @param value - The value to validate - * @param paramName - Name of the parameter for error messages - * @param options - Additional options (min, max) - * @returns ValidationResult - * - * @example - * ```typescript - * const result = validateInteger(failedCount, 'failedCount', { min: 0 }) - * if (!result.isValid) { - * return NextResponse.json({ error: result.error }, { status: 400 }) - * } - * ``` - */ -export function validateInteger( - value: unknown, - paramName = 'value', - options: { min?: number; max?: number } = {} -): ValidationResult { - if (value === null || value === undefined) { - return { - isValid: false, - error: `${paramName} is required`, - } - } - - if (typeof value !== 'number') { - logger.warn('Value is not a number', { paramName, valueType: typeof value }) - return { - isValid: false, - error: `${paramName} must be a number`, - } - } - - if (Number.isNaN(value) || !Number.isFinite(value)) { - logger.warn('Invalid number value', { paramName, value }) - return { - isValid: false, - error: `${paramName} must be a valid number`, - } - } - - if (!Number.isInteger(value)) { - logger.warn('Value is not an integer', { paramName, value }) - return { - isValid: false, - error: `${paramName} must be an integer`, - } - } - - if (options.min !== undefined && value < options.min) { - return { - isValid: false, - error: `${paramName} must be at least ${options.min}`, - } - } - - if (options.max !== undefined && value > options.max) { - return { - isValid: false, - error: `${paramName} must be at most ${options.max}`, - } - } - - return { isValid: true } -} - /** * Validates that a value is in an allowed list (enum validation) * @@ -363,121 +294,6 @@ export function validateEnum( return { isValid: true, sanitized: value } } -/** - * Validates a hostname to prevent SSRF attacks - * - * This function checks that a hostname is not a private IP, localhost, or other reserved address. - * It complements the validateProxyUrl function by providing hostname-specific validation. - * - * @param hostname - The hostname to validate - * @param paramName - Name of the parameter for error messages - * @returns ValidationResult - * - * @example - * ```typescript - * const result = validateHostname(webhookDomain, 'webhook domain') - * if (!result.isValid) { - * return NextResponse.json({ error: result.error }, { status: 400 }) - * } - * ``` - */ -export function validateHostname( - hostname: string | null | undefined, - paramName = 'hostname' -): ValidationResult { - if (hostname === null || hostname === undefined || hostname === '') { - return { - isValid: false, - error: `${paramName} is required`, - } - } - - const lowerHostname = hostname.toLowerCase() - - if (lowerHostname === 'localhost') { - logger.warn('Hostname is localhost', { paramName }) - return { - isValid: false, - error: `${paramName} cannot be a private IP address or localhost`, - } - } - - if (ipaddr.isValid(lowerHostname)) { - if (isPrivateIp(lowerHostname)) { - logger.warn('Hostname matches blocked IP range', { - paramName, - hostname: hostname.substring(0, 100), - }) - return { - isValid: false, - error: `${paramName} cannot be a private IP address or localhost`, - } - } - } - - const hostnamePattern = - /^[a-z0-9]([a-z0-9-]{0,61}[a-z0-9])?(\.[a-z0-9]([a-z0-9-]{0,61}[a-z0-9])?)*$/i - - if (!hostnamePattern.test(hostname)) { - logger.warn('Invalid hostname format', { - paramName, - hostname: hostname.substring(0, 100), - }) - return { - isValid: false, - error: `${paramName} is not a valid hostname`, - } - } - - return { isValid: true, sanitized: hostname } -} - -/** - * Validates a file extension - * - * @param extension - The file extension (with or without leading dot) - * @param allowedExtensions - Array of allowed extensions (without dots) - * @param paramName - Name of the parameter for error messages - * @returns ValidationResult - * - * @example - * ```typescript - * const result = validateFileExtension(ext, ['jpg', 'png', 'gif'], 'file extension') - * if (!result.isValid) { - * return NextResponse.json({ error: result.error }, { status: 400 }) - * } - * ``` - */ -export function validateFileExtension( - extension: string | null | undefined, - allowedExtensions: readonly string[], - paramName = 'file extension' -): ValidationResult { - if (extension === null || extension === undefined || extension === '') { - return { - isValid: false, - error: `${paramName} is required`, - } - } - - const ext = extension.startsWith('.') ? extension.slice(1) : extension - const normalizedExt = ext.toLowerCase() - - if (!allowedExtensions.map((e) => e.toLowerCase()).includes(normalizedExt)) { - logger.warn('File extension not in allowed list', { - paramName, - extension: ext, - allowedExtensions, - }) - return { - isValid: false, - error: `${paramName} must be one of: ${allowedExtensions.join(', ')}`, - } - } - - return { isValid: true, sanitized: normalizedExt } -} - /** * Validates Microsoft Graph API resource IDs * @@ -598,6 +414,13 @@ export function validateSharePointSiteId( } } + if (value === '.' || value === '..') { + return { + isValid: false, + error: `${paramName} cannot be a dot segment`, + } + } + return { isValid: true, sanitized: value } } @@ -629,27 +452,6 @@ export function validateJiraCloudId( }) } -/** - * Validates an Atlassian Assets workspace ID (a UUID-shaped, hyphenated - * alphanumeric identifier) before it is interpolated into an API path. - * - * @param value - The Assets workspace ID to validate - * @param paramName - Name of the parameter for error messages - * @returns ValidationResult - */ -export function validateAssetsWorkspaceId( - value: string | null | undefined, - paramName = 'workspaceId' -): ValidationResult { - return validatePathSegment(value, { - paramName, - allowHyphens: true, - allowUnderscores: false, - allowDots: false, - maxLength: 100, - }) -} - /** * Validates Jira issue keys (format: PROJECT-123 or PROJECT-KEY-123) * @@ -679,20 +481,25 @@ export function validateJiraIssueKey( } /** - * Validates a URL to prevent SSRF attacks + * Synchronous, pre-DNS egress check for a URL. + * + * This is the cheap half of the guard: it rejects a bad scheme, a denied port, + * and a disallowed IP literal without a lookup. A hostname it accepts is NOT + * cleared to be dialed — only {@link validateUrlWithDNS} can do that, because + * only a resolved address can be classified. Use this for form/contract + * validation; use the DNS-resolving variant before connecting. * - * This function checks that URLs: - * - Use https:// protocol only - * - Do not point to private IP ranges or localhost - * - Do not use suspicious ports + * It also declines to refuse anything the resolved address could permit, so a + * destination allowlisted by IP range is still configurable. * * @param url - The URL to validate * @param paramName - Name of the parameter for error messages + * @param profile - Where this URL came from; see {@link EgressProfile} * @returns ValidationResult * * @example * ```typescript - * const result = validateExternalUrl(url, 'fileUrl') + * const result = validateExternalUrl(url, 'fileUrl', 'configuredEndpoint') * if (!result.isValid) { * return NextResponse.json({ error: result.error }, { status: 400 }) * } @@ -700,100 +507,41 @@ export function validateJiraIssueKey( */ export function validateExternalUrl( url: string | null | undefined, - paramName = 'url', - options: { allowHttp?: boolean } = {} + paramName: string, + profile: EgressProfile ): ValidationResult { if (!url || typeof url !== 'string') { - return { - isValid: false, - error: `${paramName} is required and must be a string`, - } + return { isValid: false, error: `${paramName} is required and must be a string` } } - let parsedUrl: URL + let parsed: URL try { - parsedUrl = new URL(url) + parsed = new URL(url) } catch { - return { - isValid: false, - error: `${paramName} must be a valid URL`, - } - } - - const protocol = parsedUrl.protocol - const hostname = parsedUrl.hostname.toLowerCase() - - const cleanHostname = unwrapIpv6Brackets(hostname) - - // The whole loopback range, not just 127.0.0.1: 127.0.0.2 is the same - // machine, and matching two literals made this validator disagree with MCP's - // domain-check about what "localhost" means. Both directions stay coherent — - // hosted rejects the wider set, self-hosted permits http on it. - const isLocalhost = cleanHostname === 'localhost' || isLoopbackIp(cleanHostname) - - if (isLocalhost && isHosted) { - return { - isValid: false, - error: `${paramName} cannot point to localhost`, - } - } - - if (options.allowHttp) { - if (protocol !== 'https:' && protocol !== 'http:') { - return { - isValid: false, - error: `${paramName} must use http:// or https:// protocol`, - } - } - } else if (protocol !== 'https:' && !(protocol === 'http:' && isLocalhost && !isHosted)) { - return { - isValid: false, - error: `${paramName} must use https:// protocol`, - } - } - - if (!isLocalhost && ipaddr.isValid(cleanHostname)) { - if (isPrivateIp(cleanHostname)) { - return { - isValid: false, - error: `${paramName} cannot point to private IP addresses`, - } - } - } - - const port = parsedUrl.port - const blockedPorts = ['22', '23', '25', '3306', '5432', '6379', '27017', '9200'] - - if (port && blockedPorts.includes(port)) { - return { - isValid: false, - error: `${paramName} uses a blocked port`, - } - } - - return { isValid: true } -} - -/** - * Validates an image URL to prevent SSRF attacks - * Alias for validateExternalUrl for backward compatibility - */ -export function validateImageUrl( - url: string | null | undefined, - paramName = 'imageUrl' -): ValidationResult { - return validateExternalUrl(url, paramName) -} - -/** - * Validates a proxy URL to prevent SSRF attacks - * Alias for validateExternalUrl for backward compatibility - */ -export function validateProxyUrl( - url: string | null | undefined, - paramName = 'proxyUrl' -): ValidationResult { - return validateExternalUrl(url, paramName) + return { isValid: false, error: `${paramName} must be a valid URL` } + } + + const policy = resolveEgressPolicy(profile) + const decision = evaluateUrl(parsed, policy) + if (decision.allowed) return { isValid: true } + + // A refusal the resolved address could lift is not this check's to make: a + // host permitted only by EGRESS_ALLOWED_IP_RANGES cannot be recognised until + // DNS runs, and refusing here would stop it being configured at all. + // validateUrlWithDNS makes the authoritative call before anything is dialled. + // + // Only for a hostname. A literal was judged against its own address, so there + // is nothing a lookup could add and deferring would accept a literal outside + // every configured range. + if ( + !isIpLiteral(unwrapIpv6Brackets(parsed.hostname)) && + policyDefersToAddress(policy) && + isLiftableByVouching(decision.reason) + ) { + return { isValid: true } + } + + return { isValid: false, error: describeEgressDenial(decision, paramName, profile) } } /** @@ -1054,115 +802,6 @@ export function validateS3BucketName( return { isValid: true, sanitized: value } } -/** - * Validates a Google Calendar ID - * - * Google Calendar IDs can be: - * - "primary" (literal string for the user's primary calendar) - * - Email addresses (for user calendars) - * - Alphanumeric strings with hyphens, underscores, and dots (for other calendars) - * - * This validator allows these legitimate formats while blocking path traversal and injection attempts. - * - * @param value - The calendar ID to validate - * @param paramName - Name of the parameter for error messages - * @returns ValidationResult - * - * @example - * ```typescript - * const result = validateGoogleCalendarId(calendarId, 'calendarId') - * if (!result.isValid) { - * return NextResponse.json({ error: result.error }, { status: 400 }) - * } - * ``` - */ -export function validateGoogleCalendarId( - value: string | null | undefined, - paramName = 'calendarId' -): ValidationResult { - if (value === null || value === undefined || value === '') { - return { - isValid: false, - error: `${paramName} is required`, - } - } - - if (value === 'primary') { - return { isValid: true, sanitized: value } - } - - const pathTraversalPatterns = [ - '../', - '..\\', - '%2e%2e%2f', - '%2e%2e/', - '..%2f', - '%2e%2e%5c', - '%2e%2e\\', - '..%5c', - '%252e%252e%252f', - ] - - const lowerValue = value.toLowerCase() - for (const pattern of pathTraversalPatterns) { - if (lowerValue.includes(pattern)) { - logger.warn('Path traversal attempt in Google Calendar ID', { - paramName, - value: value.substring(0, 100), - }) - return { - isValid: false, - error: `${paramName} contains invalid path traversal sequence`, - } - } - } - - if (/[\x00-\x1f\x7f]/.test(value) || value.includes('%00')) { - logger.warn('Control characters in Google Calendar ID', { paramName }) - return { - isValid: false, - error: `${paramName} contains invalid control characters`, - } - } - - if (value.includes('\n') || value.includes('\r')) { - return { - isValid: false, - error: `${paramName} contains invalid newline characters`, - } - } - - const emailPattern = /^[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}$/ - if (emailPattern.test(value)) { - return { isValid: true, sanitized: value } - } - - const calendarIdPattern = /^[a-zA-Z0-9._@%#+-]+$/ - if (!calendarIdPattern.test(value)) { - logger.warn('Invalid Google Calendar ID format', { - paramName, - value: value.substring(0, 100), - }) - return { - isValid: false, - error: `${paramName} format is invalid. Must be "primary", an email address, or an alphanumeric ID`, - } - } - - if (value.length > 255) { - logger.warn('Google Calendar ID exceeds maximum length', { - paramName, - length: value.length, - }) - return { - isValid: false, - error: `${paramName} exceeds maximum length of 255 characters`, - } - } - - return { isValid: true, sanitized: value } -} - /** * Validates a pagination cursor token * @@ -1401,125 +1040,6 @@ export function validateMondayNumericId( return { isValid: true, sanitized: str } } -/** - * Validates a Monday.com group ID. - * - * Monday.com group IDs are strings that can contain lowercase/uppercase letters, - * digits, underscores, and spaces. They are user-visible identifiers like - * "topics", "new_group", or "test group id". Auto-generated IDs may also - * include "group_title" patterns. - * - * @param value - The group ID to validate - * @param paramName - Name of the parameter for error messages - * @returns ValidationResult - * - * @example - * ```typescript - * const result = validateMondayGroupId(groupId, 'groupId') - * if (!result.isValid) { - * return NextResponse.json({ error: result.error }, { status: 400 }) - * } - * ``` - */ -export function validateMondayGroupId( - value: string | null | undefined, - paramName = 'groupId' -): ValidationResult { - if (value === null || value === undefined || value === '') { - return { - isValid: false, - error: `${paramName} is required`, - } - } - - if (value.length > 255) { - logger.warn('Monday.com group ID exceeds maximum length', { - paramName, - length: value.length, - }) - return { - isValid: false, - error: `${paramName} exceeds maximum length of 255 characters`, - } - } - - if (/[\x00-\x1f\x7f]/.test(value) || value.includes('%00')) { - logger.warn('Monday.com group ID contains control characters', { paramName }) - return { - isValid: false, - error: `${paramName} contains invalid control characters`, - } - } - - if (!/^[a-zA-Z0-9_ ]+$/.test(value)) { - logger.warn('Monday.com group ID contains disallowed characters', { - paramName, - value: value.substring(0, 100), - }) - return { - isValid: false, - error: `${paramName} can only contain letters, digits, underscores, and spaces`, - } - } - - return { isValid: true, sanitized: value } -} - -/** - * Validates a Monday.com column ID. - * - * Column IDs are strings containing lowercase letters (a-z), digits (0-9), - * and underscores. User-specified IDs are 1-20 characters of [a-z_]. - * Auto-generated IDs follow patterns like "status", "date4", "email_mksr9hcd". - * - * @param value - The column ID to validate - * @param paramName - Name of the parameter for error messages - * @returns ValidationResult - * - * @example - * ```typescript - * const result = validateMondayColumnId(columnId, 'columnId') - * if (!result.isValid) { - * return NextResponse.json({ error: result.error }, { status: 400 }) - * } - * ``` - */ -export function validateMondayColumnId( - value: string | null | undefined, - paramName = 'columnId' -): ValidationResult { - if (value === null || value === undefined || value === '') { - return { - isValid: false, - error: `${paramName} is required`, - } - } - - if (value.length > 255) { - logger.warn('Monday.com column ID exceeds maximum length', { - paramName, - length: value.length, - }) - return { - isValid: false, - error: `${paramName} exceeds maximum length of 255 characters`, - } - } - - if (!/^[a-z0-9_]+$/.test(value)) { - logger.warn('Monday.com column ID contains disallowed characters', { - paramName, - value: value.substring(0, 100), - }) - return { - isValid: false, - error: `${paramName} can only contain lowercase letters, digits, and underscores`, - } - } - - return { isValid: true, sanitized: value } -} - /** * Validates a Supabase project ID. * @@ -1587,6 +1107,82 @@ const SERVICENOW_ALLOWED_HOST_SUFFIXES = [ '.servicenowservices.com', ] as const +/** + * Validates a vendor-hosted URL: an ordinary egress check, then a hostname + * allowlist that pins it to the vendor's own domains. + * + * The allowlist is what makes these connectors safe to point at a + * customer-supplied tenant: egress validation alone would accept any public + * host, so a tenant field would otherwise be an open redirect for credentials + * scoped to that vendor. + * + * @param url - The URL or bare host to validate + * @param options.suffixes - Permitted host suffixes, each written with a leading dot + * @param options.vendor - Vendor name, used in the error message + * @param options.paramName - Name of the parameter for error messages + * @param options.assumeHttps - Accept a bare host by prepending `https://` + * @param options.sanitize - What to return as `sanitized`: the input, or the parsed origin + * @param options.allowBareSuffix - Also accept the suffix itself as a hostname + */ +function validateVendorHostedUrl( + url: string | null | undefined, + options: { + suffixes: readonly string[] + vendor: string + paramName: string + assumeHttps?: boolean + sanitize?: 'input' | 'origin' + allowBareSuffix?: boolean + } +): ValidationResult { + const { + suffixes, + vendor, + paramName, + assumeHttps = false, + sanitize = 'input', + allowBareSuffix = true, + } = options + + const raw = typeof url === 'string' ? url.trim() : '' + if (!raw) { + return { isValid: false, error: `${paramName} is required` } + } + + const candidate = assumeHttps && !/^https?:\/\//i.test(raw) ? `https://${raw}` : raw + + // These vendors are public SaaS reached over TLS. Enforced here rather than + // left to the egress policy, which would permit plain HTTP to a host an + // operator happened to put in their allowlist. + if (/^http:\/\//i.test(candidate)) { + return { isValid: false, error: `${paramName} must use https://` } + } + + const urlResult = validateExternalUrl(candidate, paramName, 'configuredEndpoint') + if (!urlResult.isValid) return urlResult + + const parsed = new URL(candidate) + const hostname = parsed.hostname.toLowerCase() + const allowed = suffixes.some( + (suffix) => (allowBareSuffix && hostname === suffix.slice(1)) || hostname.endsWith(suffix) + ) + + if (!allowed) { + logger.warn(`${vendor} host not on allowlist`, { + paramName, + hostname: hostname.substring(0, 100), + }) + return { + isValid: false, + error: `${paramName} must be a ${vendor}-hosted domain (e.g., ${suffixes + .map((suffix) => `*${suffix}`) + .join(', ')})`, + } + } + + return { isValid: true, sanitized: sanitize === 'origin' ? parsed.origin : candidate } +} + /** * Validates a ServiceNow instance URL to prevent SSRF attacks. * @@ -1620,26 +1216,11 @@ export function validateServiceNowInstanceUrl( url: string | null | undefined, paramName = 'instanceUrl' ): ValidationResult { - const urlResult = validateExternalUrl(url, paramName) - if (!urlResult.isValid) return urlResult - - const hostname = new URL(url as string).hostname.toLowerCase() - const isAllowedHost = SERVICENOW_ALLOWED_HOST_SUFFIXES.some( - (suffix) => hostname === suffix.slice(1) || hostname.endsWith(suffix) - ) - - if (!isAllowedHost) { - logger.warn('ServiceNow instance URL hostname not on allowlist', { - paramName, - hostname: hostname.substring(0, 100), - }) - return { - isValid: false, - error: `${paramName} must be a ServiceNow-hosted domain (e.g., *.service-now.com, *.servicenow.com, or *.servicenowservices.com)`, - } - } - - return { isValid: true, sanitized: url as string } + return validateVendorHostedUrl(url, { + suffixes: SERVICENOW_ALLOWED_HOST_SUFFIXES, + vendor: 'ServiceNow', + paramName, + }) } const WORKDAY_ALLOWED_HOST_SUFFIXES = ['.workday.com', '.myworkday.com'] as const @@ -1673,26 +1254,11 @@ export function validateWorkdayTenantUrl( url: string | null | undefined, paramName = 'tenantUrl' ): ValidationResult { - const urlResult = validateExternalUrl(url, paramName) - if (!urlResult.isValid) return urlResult - - const hostname = new URL(url as string).hostname.toLowerCase() - const isAllowedHost = WORKDAY_ALLOWED_HOST_SUFFIXES.some( - (suffix) => hostname === suffix.slice(1) || hostname.endsWith(suffix) - ) - - if (!isAllowedHost) { - logger.warn('Workday tenant URL hostname not on allowlist', { - paramName, - hostname: hostname.substring(0, 100), - }) - return { - isValid: false, - error: `${paramName} must be a Workday-hosted domain (e.g., *.workday.com or *.myworkday.com)`, - } - } - - return { isValid: true, sanitized: url as string } + return validateVendorHostedUrl(url, { + suffixes: WORKDAY_ALLOWED_HOST_SUFFIXES, + vendor: 'Workday', + paramName, + }) } /** @@ -1750,32 +1316,14 @@ export function validateDatabricksWorkspaceHost( host: string | null | undefined, paramName = 'workspaceHost' ): ValidationResult { - const raw = typeof host === 'string' ? host.trim() : '' - if (!raw) { - return { isValid: false, error: `${paramName} is required` } - } - - const withScheme = /^https?:\/\//i.test(raw) ? raw : `https://${raw}` - - const urlResult = validateExternalUrl(withScheme, paramName) - if (!urlResult.isValid) return urlResult - - const parsed = new URL(withScheme) - const hostname = parsed.hostname.toLowerCase() - const isAllowedHost = DATABRICKS_ALLOWED_HOST_SUFFIXES.some((suffix) => hostname.endsWith(suffix)) - - if (!isAllowedHost) { - logger.warn('Databricks workspace host not on allowlist', { - paramName, - hostname: hostname.substring(0, 100), - }) - return { - isValid: false, - error: `${paramName} must be a Databricks-hosted domain (e.g., *.cloud.databricks.com, *.azuredatabricks.net, or *.gcp.databricks.com)`, - } - } - - return { isValid: true, sanitized: parsed.origin } + return validateVendorHostedUrl(host, { + suffixes: DATABRICKS_ALLOWED_HOST_SUFFIXES, + vendor: 'Databricks', + paramName, + assumeHttps: true, + sanitize: 'origin', + allowBareSuffix: false, + }) } /** diff --git a/apps/sim/lib/core/security/pinned-fetch.server.test.ts b/apps/sim/lib/core/security/pinned-fetch.server.test.ts index a1eca0eb09e..ded5a698a08 100644 --- a/apps/sim/lib/core/security/pinned-fetch.server.test.ts +++ b/apps/sim/lib/core/security/pinned-fetch.server.test.ts @@ -2,7 +2,8 @@ * @vitest-environment node */ import { Readable } from 'node:stream' -import { beforeEach, describe, expect, it, vi } from 'vitest' +import { resetEnvFlagsMock, setEnvFlags } from '@sim/testing' +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' const { mockAgent, mockUndiciRequest, capturedAgentOptions } = vi.hoisted(() => { const capturedAgentOptions: unknown[] = [] @@ -47,6 +48,8 @@ function undiciReply(statusCode: number, headers: Record, body: return { statusCode, headers, body, trailers: {}, opaque: null, context: {} } } +afterEach(resetEnvFlagsMock) + describe('createPinnedFetch', () => { beforeEach(() => { vi.clearAllMocks() @@ -55,7 +58,7 @@ describe('createPinnedFetch', () => { }) it('builds an undici Agent whose pinned lookup always resolves to the validated IP', async () => { - createPinnedFetch('203.0.113.10') + createPinnedFetch('93.184.216.34', { profile: 'configuredEndpoint' }) expect(capturedAgentOptions).toHaveLength(1) const { connect } = capturedAgentOptions[0] as { connect: { lookup: PinnedLookup } } @@ -66,23 +69,23 @@ describe('createPinnedFetch', () => { resolve({ address, family }) ) }) - expect(resolved).toEqual({ address: '203.0.113.10', family: 4 }) + expect(resolved).toEqual({ address: '93.184.216.34', family: 4 }) }) it('defaults allowH2 to false so existing consumers are unchanged', () => { - createPinnedFetch('203.0.113.10') + createPinnedFetch('93.184.216.34', { profile: 'configuredEndpoint' }) const opts = capturedAgentOptions[0] as { allowH2?: boolean } expect(opts.allowH2).toBe(false) }) it('opts the Agent into HTTP/2 when allowH2 is requested', () => { - createPinnedFetch('203.0.113.10', { allowH2: true }) + createPinnedFetch('93.184.216.34', { profile: 'configuredEndpoint', allowH2: true }) const opts = capturedAgentOptions[0] as { allowH2?: boolean } expect(opts.allowH2).toBe(true) }) it('uses IPv6 family when the validated IP is IPv6', async () => { - createPinnedFetch('2606:4700:4700::1111') + createPinnedFetch('2606:4700:4700::1111', { profile: 'configuredEndpoint' }) const { connect } = capturedAgentOptions[0] as { connect: { lookup: PinnedLookup } } const resolved = await new Promise<{ address: string; family: number }>((resolve) => { connect.lookup('example.com', {}, (_err, address, family) => resolve({ address, family })) @@ -91,7 +94,7 @@ describe('createPinnedFetch', () => { }) it('dispatches through the pinned Agent, preserving init', async () => { - const pinned = createPinnedFetch('203.0.113.10') + const pinned = createPinnedFetch('93.184.216.34', { profile: 'configuredEndpoint' }) const controller = new AbortController() await pinned('https://myresource.openai.azure.com/openai/v1/responses', { @@ -115,7 +118,7 @@ describe('createPinnedFetch', () => { mockUndiciRequest.mockResolvedValueOnce( undiciReply(302, { location: 'https://login.example.com/' }, byteStream('')) ) - const pinned = createPinnedFetch('203.0.113.10') + const pinned = createPinnedFetch('93.184.216.34', { profile: 'configuredEndpoint' }) const response = await pinned('https://mcp.example.com/', { redirect: 'manual' }) @@ -128,7 +131,7 @@ describe('createPinnedFetch', () => { mockUndiciRequest.mockResolvedValueOnce( undiciReply(302, { location: 'https://login.example.com/' }, byteStream('')) ) - const pinned = createPinnedFetch('203.0.113.10') + const pinned = createPinnedFetch('93.184.216.34', { profile: 'configuredEndpoint' }) const response = await pinned(new Request('https://mcp.example.com/', { redirect: 'manual' })) @@ -142,7 +145,7 @@ describe('createPinnedFetch', () => { undiciReply(307, { location: 'https://other-origin.example/final' }, byteStream('')) ) .mockResolvedValueOnce(undiciReply(200, {}, byteStream('done'))) - const pinned = createPinnedFetch('203.0.113.10') + const pinned = createPinnedFetch('93.184.216.34', { profile: 'configuredEndpoint' }) const response = await pinned('https://azure.example.com/v1/responses', { method: 'GET', @@ -163,12 +166,13 @@ describe('createPinnedFetch', () => { expect(await response.text()).toBe('done') }) - it('does NOT block a private IP-literal URL (self-hosted-private MCP carve-out)', async () => { + it('reaches a private IP-literal URL the operator allowlisted', async () => { + setEnvFlags({ egressAllowedIpRanges: '10.0.0.0/8' }) mockUndiciRequest.mockResolvedValueOnce(undiciReply(200, {}, byteStream('mcp'))) - const pinned = createPinnedFetch('10.0.0.5') + const pinned = createPinnedFetch('10.0.0.5', { profile: 'configuredEndpoint' }) - // A self-hosted MCP configured with a private IP-literal URL must still connect — the old - // undici.fetch path never ran the SSRF initial-target check that would otherwise block it. + // A self-hosted MCP on a private address connects because the deployment + // named that range, not because the address happened to be the pinned one. const response = await pinned('http://10.0.0.5:3000/mcp', { method: 'POST', body: '{}' }) expect(mockUndiciRequest).toHaveBeenCalledTimes(1) @@ -176,13 +180,14 @@ describe('createPinnedFetch', () => { expect(await response.text()).toBe('mcp') }) - it('follows a redirect that stays on the pinned private IP (self-hosted MCP alias)', async () => { + it('follows a redirect that stays inside the allowlisted range', async () => { + setEnvFlags({ egressAllowedIpRanges: '10.0.0.0/8' }) mockUndiciRequest .mockResolvedValueOnce( undiciReply(301, { location: 'http://10.0.0.5:3000/mcp/' }, byteStream('')) ) .mockResolvedValueOnce(undiciReply(200, {}, byteStream('mcp'))) - const pinned = createPinnedFetch('10.0.0.5') + const pinned = createPinnedFetch('10.0.0.5', { profile: 'configuredEndpoint' }) const response = await pinned('http://10.0.0.5:3000/mcp', { method: 'GET' }) @@ -191,21 +196,37 @@ describe('createPinnedFetch', () => { expect(await response.text()).toBe('mcp') }) - it('STILL blocks a redirect to a different private IP (no metadata-IP escape)', async () => { + it('still blocks a redirect to a private IP outside the allowlist', async () => { + setEnvFlags({ egressAllowedIpRanges: '10.0.0.0/8' }) + // A genuine private address outside the allowlisted range, not the metadata + // endpoint — that one is refused unconditionally, so it would pass here even + // if the allowlist had regressed to permitting all private addresses. mockUndiciRequest.mockResolvedValueOnce( - undiciReply(302, { location: 'http://169.254.169.254/latest/meta-data/' }, byteStream('')) + undiciReply(302, { location: 'https://192.168.1.5/internal' }, byteStream('')) ) - const pinned = createPinnedFetch('10.0.0.5') + const pinned = createPinnedFetch('10.0.0.5', { profile: 'configuredEndpoint' }) await expect(pinned('http://10.0.0.5:3000/mcp', { method: 'GET' })).rejects.toThrow( - /private or reserved/ + /private or reserved address/ ) - // The initial request happened; the redirect to the metadata IP was refused. + // The initial request happened; the redirect out of the range was refused. expect(mockUndiciRequest).toHaveBeenCalledTimes(1) }) + it('still blocks a redirect to the metadata endpoint from inside the allowlist', async () => { + setEnvFlags({ egressAllowedIpRanges: '10.0.0.0/8,169.254.0.0/16' }) + mockUndiciRequest.mockResolvedValueOnce( + undiciReply(302, { location: 'http://169.254.169.254/latest/meta-data/' }, byteStream('')) + ) + const pinned = createPinnedFetch('10.0.0.5', { profile: 'configuredEndpoint' }) + + await expect(pinned('http://10.0.0.5:3000/mcp', { method: 'GET' })).rejects.toThrow( + /cloud metadata endpoint/ + ) + }) + it('reuses one dispatcher across all calls of a single instance', async () => { - const pinned = createPinnedFetch('203.0.113.10') + const pinned = createPinnedFetch('93.184.216.34', { profile: 'configuredEndpoint' }) await pinned('https://example.com/a') await pinned('https://example.com/b') @@ -216,8 +237,8 @@ describe('createPinnedFetch', () => { }) it('creates an independent dispatcher per instance', async () => { - const a = createPinnedFetch('203.0.113.10') - const b = createPinnedFetch('203.0.113.10') + const a = createPinnedFetch('93.184.216.34', { profile: 'configuredEndpoint' }) + const b = createPinnedFetch('93.184.216.34', { profile: 'configuredEndpoint' }) await a('https://example.com/a') await b('https://example.com/b') @@ -229,7 +250,7 @@ describe('createPinnedFetch', () => { it('returns a streaming Response built from the undici.request body', async () => { mockUndiciRequest.mockResolvedValueOnce(undiciReply(201, {}, byteStream('pong'))) - const pinned = createPinnedFetch('203.0.113.10') + const pinned = createPinnedFetch('93.184.216.34', { profile: 'configuredEndpoint' }) const response = await pinned('https://example.com') expect(response.status).toBe(201) expect(await response.text()).toBe('pong') diff --git a/apps/sim/lib/core/security/pinned-redirect-replay.server.test.ts b/apps/sim/lib/core/security/pinned-redirect-replay.server.test.ts index cccf9ddfa8d..07670f51f87 100644 --- a/apps/sim/lib/core/security/pinned-redirect-replay.server.test.ts +++ b/apps/sim/lib/core/security/pinned-redirect-replay.server.test.ts @@ -15,7 +15,9 @@ vi.mock('@sim/security/dns', () => ({ vi.mock('@/lib/core/config/env-flags', () => ({ isHosted: false, - isPrivateDatabaseHostsAllowed: false, + getEgressAllowedHosts: () => undefined, + getEgressAllowedIpRanges: () => undefined, + isLegacyPrivateDatabaseAccessAllowed: () => false, getProxyUrl: () => undefined, })) @@ -70,7 +72,7 @@ describe('secureFetchWithPinnedIP redirect replay', () => { await expect( secureFetchWithPinnedIP(origin, '127.0.0.1', { - allowHttp: true, + profile: 'configuredEndpoint', assertRedirectTarget, }) ).rejects.toThrow('redirect target rejected') @@ -79,7 +81,40 @@ describe('secureFetchWithPinnedIP redirect replay', () => { expect(hops).toEqual([]) }) - it('preserves historical replay when no redirect policy is present', async () => { + it('returns a 305 Use Proxy rather than following it', async () => { + const hops: RecordedHop[] = [] + const target = await startRecordingServer(hops) + const origin = await startServer((req, res) => { + req.resume() + res.writeHead(305, { location: `${target}/after` }) + res.end() + }) + + const response = await secureFetchWithPinnedIP(origin, '127.0.0.1', { + profile: 'configuredEndpoint', + }) + + // 305 is the one redirect a guard must never follow (it names a proxy); it + // is handed back to the caller, not chased. + expect(response.status).toBe(305) + expect(hops).toEqual([]) + }) + + it("re-judges a redirect hop under the request's own policy and refuses metadata", async () => { + const origin = await startServer((req, res) => { + req.resume() + res.writeHead(302, { location: 'http://169.254.169.254/latest/meta-data/' }) + res.end() + }) + + await expect( + secureFetchWithPinnedIP(origin, '127.0.0.1', { + profile: 'requestTarget', + }) + ).rejects.toThrow(/Redirect blocked/i) + }) + + it('drops every header on a cross-origin hop when no policy is supplied', async () => { const hops: RecordedHop[] = [] const target = await startRecordingServer(hops) const origin = await startServer((req, res) => { @@ -89,22 +124,97 @@ describe('secureFetchWithPinnedIP redirect replay', () => { }) const response = await secureFetchWithPinnedIP(origin, '127.0.0.1', { - method: 'POST', - body: '{"message":"legacy"}', + method: 'GET', headers: { Authorization: 'Bearer legacy-token', - 'Content-Type': 'application/json', + 'Private-Token': 'glpat-secret', + 'X-Trace': 'keep-me', Host: 'legacy.example', }, - allowHttp: true, + profile: 'configuredEndpoint', }) expect(response.status).toBe(200) expect(hops).toHaveLength(1) - expect(hops[0].method).toBe('POST') - expect(hops[0].body).toBe('{"message":"legacy"}') - expect(hops[0].headers.authorization).toBe('Bearer legacy-token') - expect(hops[0].headers.host).toBe('legacy.example') + // Without a policy declaring which headers are sensitive, none survive the + // cross-origin hop — a custom credential header cannot leak. + expect(hops[0].headers.authorization).toBeUndefined() + expect(hops[0].headers['private-token']).toBeUndefined() + expect(hops[0].headers['x-trace']).toBeUndefined() + expect(hops[0].headers.host).not.toBe('legacy.example') + }) + + it('refuses to replay a body to another origin', async () => { + const hops: RecordedHop[] = [] + const target = await startRecordingServer(hops) + // 307 preserves the method and body verbatim, which is exactly the case + // that would hand an Agiloft-style `$password` form to the redirect target. + const origin = await startServer((req, res) => { + req.resume() + res.writeHead(307, { location: `${target}/after` }) + res.end() + }) + + await expect( + secureFetchWithPinnedIP(origin, '127.0.0.1', { + method: 'POST', + body: '$login=admin&$password=hunter2', + headers: { 'Content-Type': 'application/x-www-form-urlencoded' }, + profile: 'configuredEndpoint', + }) + ).rejects.toThrow('cross-origin redirect would forward a request body') + + expect(hops).toHaveLength(0) + }) + + it('replays a body to another origin only when a policy opts in', async () => { + const hops: RecordedHop[] = [] + const target = await startRecordingServer(hops) + const origin = await startServer((req, res) => { + req.resume() + res.writeHead(307, { location: `${target}/after` }) + res.end() + }) + + await secureFetchWithPinnedIP(origin, '127.0.0.1', { + method: 'POST', + body: '{"intentional":true}', + headers: { 'Content-Type': 'application/json' }, + redirectPolicy: { + mode: 'legacy', + sendCredentialsOnCrossOriginRedirect: false, + allowCrossOriginBody: true, + }, + profile: 'configuredEndpoint', + }) + + expect(hops).toHaveLength(1) + expect(hops[0].body).toBe('{"intentional":true}') + }) + + it('keeps credentials cross-origin only when a policy explicitly opts in', async () => { + const hops: RecordedHop[] = [] + const target = await startRecordingServer(hops) + const origin = await startServer((req, res) => { + req.resume() + res.writeHead(303, { location: `${target}/after` }) + res.end() + }) + + await secureFetchWithPinnedIP(origin, '127.0.0.1', { + method: 'POST', + body: '{"message":"opt-in"}', + headers: { Authorization: 'Bearer keep-me', 'Content-Type': 'application/json' }, + redirectPolicy: { + mode: 'legacy', + sendCredentialsOnCrossOriginRedirect: true, + allowCrossOriginBody: true, + }, + profile: 'configuredEndpoint', + }) + + expect(hops).toHaveLength(1) + expect(hops[0].headers.authorization).toBe('Bearer keep-me') }) it('lets a legacy block withhold credentials without changing its replay semantics', async () => { @@ -128,9 +238,10 @@ describe('secureFetchWithPinnedIP redirect replay', () => { redirectPolicy: { mode: 'legacy', sendCredentialsOnCrossOriginRedirect: false, + allowCrossOriginBody: true, sensitiveHeaders: ['x-api-key'], }, - allowHttp: true, + profile: 'configuredEndpoint', }) expect(hops).toHaveLength(1) @@ -164,9 +275,10 @@ describe('secureFetchWithPinnedIP redirect replay', () => { redirectPolicy: { mode: 'standard', sendCredentialsOnCrossOriginRedirect: false, + allowCrossOriginBody: true, sensitiveHeaders: ['x-api-key'], }, - allowHttp: true, + profile: 'configuredEndpoint', }) expect(response.status).toBe(200) @@ -201,8 +313,9 @@ describe('secureFetchWithPinnedIP redirect replay', () => { redirectPolicy: { mode: 'standard', sendCredentialsOnCrossOriginRedirect: false, + allowCrossOriginBody: true, }, - allowHttp: true, + profile: 'configuredEndpoint', }) expect(hops).toHaveLength(1) @@ -226,8 +339,9 @@ describe('secureFetchWithPinnedIP redirect replay', () => { redirectPolicy: { mode: 'standard', sendCredentialsOnCrossOriginRedirect: false, + allowCrossOriginBody: true, }, - allowHttp: true, + profile: 'configuredEndpoint', }) expect(hops).toHaveLength(1) @@ -254,8 +368,9 @@ describe('secureFetchWithPinnedIP redirect replay', () => { redirectPolicy: { mode: 'standard', sendCredentialsOnCrossOriginRedirect: false, + allowCrossOriginBody: true, }, - allowHttp: true, + profile: 'configuredEndpoint', }) expect(hops).toHaveLength(1) @@ -282,8 +397,9 @@ describe('secureFetchWithPinnedIP redirect replay', () => { redirectPolicy: { mode: 'standard', sendCredentialsOnCrossOriginRedirect: true, + allowCrossOriginBody: true, }, - allowHttp: true, + profile: 'configuredEndpoint', }) expect(hops).toHaveLength(1) @@ -320,8 +436,9 @@ describe('secureFetchWithPinnedIP redirect replay', () => { redirectPolicy: { mode: 'standard', sendCredentialsOnCrossOriginRedirect: false, + allowCrossOriginBody: true, }, - allowHttp: true, + profile: 'configuredEndpoint', }) expect(response.status).toBe(200) @@ -354,7 +471,7 @@ describe('secureFetchWithPinnedIP redirect replay', () => { method: 'GET', headers: { Authorization: 'Bearer strip-me', 'X-Trace': 'keep-me' }, stripAuthOnRedirect: true, - allowHttp: true, + profile: 'configuredEndpoint', }) expect(hops).toHaveLength(1) diff --git a/apps/sim/lib/core/security/secure-fetch-response-cap.server.test.ts b/apps/sim/lib/core/security/secure-fetch-response-cap.server.test.ts index 78d6f21805d..ce7e2b7d6d6 100644 --- a/apps/sim/lib/core/security/secure-fetch-response-cap.server.test.ts +++ b/apps/sim/lib/core/security/secure-fetch-response-cap.server.test.ts @@ -12,7 +12,9 @@ vi.mock('@sim/security/dns', () => ({ vi.mock('@/lib/core/config/env-flags', () => ({ isHosted: false, - isPrivateDatabaseHostsAllowed: false, + getEgressAllowedHosts: () => undefined, + getEgressAllowedIpRanges: () => undefined, + isLegacyPrivateDatabaseAccessAllowed: () => false, getProxyUrl: () => undefined, })) @@ -45,6 +47,7 @@ describe('secureFetchWithPinnedIP response cap', () => { }) const response = await secureFetchWithPinnedIP(origin, '127.0.0.1', { + profile: 'configuredEndpoint', maxResponseBytes: 64 * 1024, }) @@ -60,7 +63,9 @@ describe('secureFetchWithPinnedIP response cap', () => { res.end('{}') }) - await expect(secureFetchWithPinnedIP(origin, '127.0.0.1', {})).rejects.toThrow(/response body/i) + await expect( + secureFetchWithPinnedIP(origin, '127.0.0.1', { profile: 'configuredEndpoint' }) + ).rejects.toThrow(/response body/i) }) it('reads a body that fits under the default cap', async () => { @@ -69,7 +74,9 @@ describe('secureFetchWithPinnedIP response cap', () => { res.end('{"ok":true}') }) - const response = await secureFetchWithPinnedIP(origin, '127.0.0.1', {}) + const response = await secureFetchWithPinnedIP(origin, '127.0.0.1', { + profile: 'configuredEndpoint', + }) expect(await response.text()).toBe('{"ok":true}') }) @@ -83,7 +90,10 @@ describe('secureFetchWithPinnedIP response cap', () => { res.end() }) - const response = await secureFetchWithPinnedIP(origin, '127.0.0.1', { method: 'HEAD' }) + const response = await secureFetchWithPinnedIP(origin, '127.0.0.1', { + profile: 'configuredEndpoint', + method: 'HEAD', + }) expect(response.status).toBe(200) expect(response.headers.get('content-type')).toBe('video/mp4') @@ -95,7 +105,9 @@ describe('secureFetchWithPinnedIP response cap', () => { res.end() }) - const response = await secureFetchWithPinnedIP(origin, '127.0.0.1', {}) + const response = await secureFetchWithPinnedIP(origin, '127.0.0.1', { + profile: 'configuredEndpoint', + }) expect(response.status).toBe(304) }) diff --git a/apps/sim/lib/core/security/ssrf-guarded-lookup.test.ts b/apps/sim/lib/core/security/ssrf-guarded-lookup.test.ts index 7a59c48727e..12e553a225f 100644 --- a/apps/sim/lib/core/security/ssrf-guarded-lookup.test.ts +++ b/apps/sim/lib/core/security/ssrf-guarded-lookup.test.ts @@ -21,6 +21,7 @@ declare module '@/lib/core/security/input-validation.server?ssrf-guarded-lookup- export * from '@/lib/core/security/input-validation.server' } +import type { EgressProfile } from '@/lib/core/security/egress/profiles' import { createSsrfGuardedLookup, followRedirectsGuarded, @@ -30,10 +31,11 @@ type LookupResult = { address: string; family: number } function runLookup( hostname: string, - options: { all?: boolean } = {} + options: { all?: boolean } = {}, + profile: EgressProfile = 'contentFetch' ): Promise<{ err: Error | null; address?: string | LookupResult[]; family?: number }> { return new Promise((resolve) => { - const lookup = createSsrfGuardedLookup() + const lookup = createSsrfGuardedLookup(profile) type LookupCb = (err: Error | null, address?: string | LookupResult[], family?: number) => void // double-cast-allowed: net.LookupFunction's overloaded callback shapes collapse to this in practice ;(lookup as unknown as (h: string, o: object, cb: LookupCb) => void)( @@ -120,7 +122,7 @@ function redirectTo(location: string, status = 302): Response { describe('followRedirectsGuarded', () => { it('returns a non-redirect response as-is', async () => { const raw = vi.fn(async () => new Response('ok', { status: 200 })) - const res = await followRedirectsGuarded(raw, 'https://a.example/x', {}) + const res = await followRedirectsGuarded(raw, 'https://a.example/x', {}, 'contentFetch') expect(res.status).toBe(200) expect(raw).toHaveBeenCalledTimes(1) }) @@ -130,9 +132,12 @@ describe('followRedirectsGuarded', () => { .fn() .mockResolvedValueOnce(redirectTo('https://a.example/y')) .mockResolvedValueOnce(new Response('ok', { status: 200 })) - const res = await followRedirectsGuarded(raw, 'https://a.example/x', { - headers: { 'x-api-key': 'secret' }, - }) + const res = await followRedirectsGuarded( + raw, + 'https://a.example/x', + { headers: { 'x-api-key': 'secret' } }, + 'contentFetch' + ) expect(res.status).toBe(200) expect(raw.mock.calls[1][0]).toBe('https://a.example/y') expect(raw.mock.calls[1][1].headers).toEqual({ 'x-api-key': 'secret' }) @@ -143,39 +148,42 @@ describe('followRedirectsGuarded', () => { .fn() .mockResolvedValueOnce(redirectTo('https://b.example/harvest')) .mockResolvedValueOnce(new Response('ok', { status: 200 })) - await followRedirectsGuarded(raw, 'https://a.example/x', { - headers: { 'x-api-key': 'secret' }, - }) + await followRedirectsGuarded( + raw, + 'https://a.example/x', + { headers: { 'x-api-key': 'secret' } }, + 'contentFetch' + ) expect(raw.mock.calls[1][1].headers).toBeUndefined() }) it('blocks a redirect to a private IP literal (metadata endpoint)', async () => { const raw = vi.fn(async () => redirectTo('http://169.254.169.254/latest/meta-data/')) - await expect(followRedirectsGuarded(raw, 'https://a.example/x', {})).rejects.toThrow( - /Blocked by SSRF policy/ - ) + await expect( + followRedirectsGuarded(raw, 'https://a.example/x', {}, 'contentFetch') + ).rejects.toThrow(/Blocked by SSRF policy/) expect(raw).toHaveBeenCalledTimes(1) }) it('blocks a redirect to a bracketed private IPv6 literal', async () => { const raw = vi.fn(async () => redirectTo('http://[::1]/admin')) - await expect(followRedirectsGuarded(raw, 'https://a.example/x', {})).rejects.toThrow( - /Blocked by SSRF policy/ - ) + await expect( + followRedirectsGuarded(raw, 'https://a.example/x', {}, 'contentFetch') + ).rejects.toThrow(/Blocked by SSRF policy/) }) it('blocks non-http(s) redirect protocols', async () => { const raw = vi.fn(async () => redirectTo('file:///etc/passwd')) - await expect(followRedirectsGuarded(raw, 'https://a.example/x', {})).rejects.toThrow( - /unsupported protocol/ - ) + await expect( + followRedirectsGuarded(raw, 'https://a.example/x', {}, 'contentFetch') + ).rejects.toThrow(/unsupported protocol/) }) it('caps the number of hops', async () => { const raw = vi.fn(async () => redirectTo('https://a.example/loop')) - await expect(followRedirectsGuarded(raw, 'https://a.example/x', {})).rejects.toThrow( - /more than \d+ redirects/ - ) + await expect( + followRedirectsGuarded(raw, 'https://a.example/x', {}, 'contentFetch') + ).rejects.toThrow(/more than \d+ redirects/) }) it('switches POST to a bodyless GET on 303', async () => { @@ -183,7 +191,12 @@ describe('followRedirectsGuarded', () => { .fn() .mockResolvedValueOnce(redirectTo('https://a.example/next', 303)) .mockResolvedValueOnce(new Response('ok', { status: 200 })) - await followRedirectsGuarded(raw, 'https://a.example/x', { method: 'POST', body: 'data' }) + await followRedirectsGuarded( + raw, + 'https://a.example/x', + { method: 'POST', body: 'data' }, + 'contentFetch' + ) expect(raw.mock.calls[1][1].method).toBe('GET') expect(raw.mock.calls[1][1].body).toBeUndefined() }) @@ -193,7 +206,7 @@ describe('followRedirectsGuarded', () => { .fn() .mockResolvedValueOnce(redirectTo('https://a.example/next', 303)) .mockResolvedValueOnce(new Response(null, { status: 200 })) - await followRedirectsGuarded(raw, 'https://a.example/x', { method: 'HEAD' }) + await followRedirectsGuarded(raw, 'https://a.example/x', { method: 'HEAD' }, 'contentFetch') expect(raw.mock.calls[1][1].method).toBe('HEAD') }) @@ -202,7 +215,12 @@ describe('followRedirectsGuarded', () => { .fn() .mockResolvedValueOnce(redirectTo('https://a.example/next', 307)) .mockResolvedValueOnce(new Response('ok', { status: 200 })) - await followRedirectsGuarded(raw, 'https://a.example/x', { method: 'POST', body: 'data' }) + await followRedirectsGuarded( + raw, + 'https://a.example/x', + { method: 'POST', body: 'data' }, + 'contentFetch' + ) expect(raw.mock.calls[1][1].method).toBe('POST') expect(raw.mock.calls[1][1].body).toBe('data') }) @@ -212,7 +230,7 @@ describe('followRedirectsGuarded — hardening', () => { it('blocks a private IP-literal as the INITIAL url (guard is self-contained)', async () => { const raw = vi.fn(async () => new Response('ok')) await expect( - followRedirectsGuarded(raw, 'http://169.254.169.254/latest/meta-data/', {}) + followRedirectsGuarded(raw, 'http://169.254.169.254/latest/meta-data/', {}, 'contentFetch') ).rejects.toThrow(/Blocked by SSRF policy/) expect(raw).not.toHaveBeenCalled() }) @@ -222,11 +240,16 @@ describe('followRedirectsGuarded — hardening', () => { .fn() .mockResolvedValueOnce(redirectTo('https://a.example/next', 303)) .mockResolvedValueOnce(new Response('ok', { status: 200 })) - await followRedirectsGuarded(raw, 'https://a.example/x', { - method: 'POST', - body: '{"a":1}', - headers: { 'content-type': 'application/json', 'content-length': '7', 'x-keep': 'yes' }, - }) + await followRedirectsGuarded( + raw, + 'https://a.example/x', + { + method: 'POST', + body: '{"a":1}', + headers: { 'content-type': 'application/json', 'content-length': '7', 'x-keep': 'yes' }, + }, + 'contentFetch' + ) const hopHeaders = new Headers(raw.mock.calls[1][1].headers) expect(hopHeaders.get('content-type')).toBeNull() expect(hopHeaders.get('content-length')).toBeNull() @@ -238,10 +261,12 @@ describe('followRedirectsGuarded — cross-origin body protection', () => { it('refuses a cross-origin 307 that would forward a request body', async () => { const raw = vi.fn(async () => redirectTo('https://b.example/steal', 307)) await expect( - followRedirectsGuarded(raw, 'https://a.example/token', { - method: 'POST', - body: 'client_secret=shh', - }) + followRedirectsGuarded( + raw, + 'https://a.example/token', + { method: 'POST', body: 'client_secret=shh' }, + 'contentFetch' + ) ).rejects.toThrow(/cross-origin redirect would forward a request body/) }) @@ -250,7 +275,7 @@ describe('followRedirectsGuarded — cross-origin body protection', () => { .fn() .mockResolvedValueOnce(redirectTo('https://b.example/next', 302)) .mockResolvedValueOnce(new Response('ok', { status: 200 })) - const res = await followRedirectsGuarded(raw, 'https://a.example/x', {}) + const res = await followRedirectsGuarded(raw, 'https://a.example/x', {}, 'contentFetch') expect(res.status).toBe(200) }) }) diff --git a/apps/sim/lib/core/utils/theme.test.ts b/apps/sim/lib/core/utils/theme.test.ts new file mode 100644 index 00000000000..2d374e74eac --- /dev/null +++ b/apps/sim/lib/core/utils/theme.test.ts @@ -0,0 +1,39 @@ +/** + * @vitest-environment jsdom + */ +import { afterEach, describe, expect, it, vi } from 'vitest' +import { syncThemeToNextThemes } from '@/lib/core/utils/theme' + +describe('syncThemeToNextThemes', () => { + afterEach(() => { + localStorage.clear() + document.documentElement.classList.remove('light', 'dark') + vi.restoreAllMocks() + }) + + it('does not dispatch or rewrite classes when the requested theme is already applied', () => { + localStorage.setItem('sim-theme', 'dark') + document.documentElement.classList.add('dark') + const dispatchEvent = vi.spyOn(window, 'dispatchEvent') + const remove = vi.spyOn(document.documentElement.classList, 'remove') + const add = vi.spyOn(document.documentElement.classList, 'add') + + syncThemeToNextThemes('dark') + + expect(dispatchEvent).not.toHaveBeenCalled() + expect(remove).not.toHaveBeenCalled() + expect(add).not.toHaveBeenCalled() + }) + + it('repairs the document class without emitting a redundant storage event', () => { + localStorage.setItem('sim-theme', 'dark') + document.documentElement.classList.add('light') + const dispatchEvent = vi.spyOn(window, 'dispatchEvent') + + syncThemeToNextThemes('dark') + + expect(dispatchEvent).not.toHaveBeenCalled() + expect(document.documentElement.classList.contains('dark')).toBe(true) + expect(document.documentElement.classList.contains('light')).toBe(false) + }) +}) diff --git a/apps/sim/lib/core/utils/theme.ts b/apps/sim/lib/core/utils/theme.ts index 46035f4ce53..29a77542d6e 100644 --- a/apps/sim/lib/core/utils/theme.ts +++ b/apps/sim/lib/core/utils/theme.ts @@ -11,25 +11,30 @@ export function syncThemeToNextThemes(theme: 'system' | 'light' | 'dark') { if (typeof window === 'undefined') return const oldValue = localStorage.getItem('sim-theme') - localStorage.setItem('sim-theme', theme) + if (oldValue !== theme) { + localStorage.setItem('sim-theme', theme) - window.dispatchEvent( - new StorageEvent('storage', { - key: 'sim-theme', - newValue: theme, - oldValue: oldValue, - storageArea: localStorage, - url: window.location.href, - }) - ) + window.dispatchEvent( + new StorageEvent('storage', { + key: 'sim-theme', + newValue: theme, + oldValue, + storageArea: localStorage, + url: window.location.href, + }) + ) + } const root = document.documentElement - root.classList.remove('light', 'dark') + const appliedTheme = + theme === 'system' + ? window.matchMedia('(prefers-color-scheme: dark)').matches + ? 'dark' + : 'light' + : theme + const oppositeTheme = appliedTheme === 'dark' ? 'light' : 'dark' + if (root.classList.contains(appliedTheme) && !root.classList.contains(oppositeTheme)) return - if (theme === 'system') { - const systemTheme = window.matchMedia('(prefers-color-scheme: dark)').matches ? 'dark' : 'light' - root.classList.add(systemTheme) - } else { - root.classList.add(theme) - } + root.classList.remove('light', 'dark') + root.classList.add(appliedTheme) } diff --git a/apps/sim/lib/core/utils/timezone.test.ts b/apps/sim/lib/core/utils/timezone.test.ts index 935a9061cee..64ce3f793d4 100644 --- a/apps/sim/lib/core/utils/timezone.test.ts +++ b/apps/sim/lib/core/utils/timezone.test.ts @@ -1,11 +1,72 @@ import { describe, expect, it } from 'vitest' import { + formatInstantInTimeZone, getSupportedTimezones, getTimezoneOptions, + getWallClockParts, wallClockNow, zonedClockDate, + zonedWallClock, zonedWallClockToUtc, -} from './timezone' + zonedWallClockWithOffset, +} from '@/lib/core/utils/timezone' + +describe('formatInstantInTimeZone', () => { + it.each([ + ['UTC', '0050-01-15T12:00:00Z', '0050-01-15T12:00:00Z'], + ['UTC', '2026-06-15T00:15:30Z', '2026-06-15T00:15:30Z'], + ['America/Los_Angeles', '2026-06-15T00:15:30Z', '2026-06-14T17:15:30-07:00'], + ['Asia/Tokyo', '2026-06-15T00:15:30Z', '2026-06-15T09:15:30+09:00'], + ['Asia/Kathmandu', '2026-06-15T00:15:30Z', '2026-06-15T06:00:30+05:45'], + ['Australia/Lord_Howe', '2026-06-15T00:15:30Z', '2026-06-15T10:45:30+10:30'], + ])('formats an instant in %s with its exact offset', (timeZone, iso, expected) => { + expect(formatInstantInTimeZone(new Date(iso), timeZone)).toBe(expected) + }) + + it('distinguishes both copies of an autumn daylight-saving hour', () => { + expect(formatInstantInTimeZone(new Date('2026-11-01T05:30:00Z'), 'America/New_York')).toBe( + '2026-11-01T01:30:00-04:00' + ) + expect(formatInstantInTimeZone(new Date('2026-11-01T06:30:00Z'), 'America/New_York')).toBe( + '2026-11-01T01:30:00-05:00' + ) + }) + + it('round-trips the same instant after changing display timezones', () => { + const instant = new Date('2026-11-01T06:30:00Z') + for (const timeZone of [ + 'UTC', + 'America/Los_Angeles', + 'America/New_York', + 'Asia/Kathmandu', + 'Australia/Lord_Howe', + ]) { + const editable = formatInstantInTimeZone(instant, timeZone) + expect(new Date(editable).getTime()).toBe(instant.getTime()) + } + }) + + it('preserves a four-digit low year in naive wall-clock output', () => { + expect(zonedWallClock(new Date('0050-01-15T12:00:00Z'), 'UTC')).toBe('0050-01-15T12:00') + }) +}) + +describe('getWallClockParts', () => { + it('returns the calendar fields of an instant in the requested timezone', () => { + expect(getWallClockParts(new Date('2026-06-15T00:15:30Z'), 'America/Los_Angeles')).toEqual({ + year: 2026, + month: 6, + day: 14, + hour: 17, + minute: 15, + second: 30, + }) + }) + + it('rejects an empty timezone instead of using the runtime local timezone', () => { + expect(() => getWallClockParts(new Date('2026-06-15T00:15:30Z'), '')).toThrow(RangeError) + }) +}) describe('zonedWallClockToUtc', () => { it('treats a UTC wall-clock as the same instant', () => { @@ -14,6 +75,26 @@ describe('zonedWallClockToUtc', () => { ) }) + it.each(['0000', '0001', '0050', '0099'])( + 'preserves the full year %s when resolving a wall-clock', + (year) => { + expect(zonedWallClockToUtc(`${year}-01-15T12:00`, 'UTC').toISOString()).toBe( + `${year}-01-15T12:00:00.000Z` + ) + } + ) + + it('uses the requested low year when resolving historical timezone rules', () => { + const wallClock = '0050-01-15T12:00:00' + + expect(zonedWallClockToUtc(wallClock, 'America/New_York').toISOString()).toBe( + '0050-01-15T16:56:02.000Z' + ) + expect(zonedWallClockWithOffset(wallClock, 'America/New_York')).toBe( + '0050-01-15T12:00:00-04:56' + ) + }) + it('applies a positive (east-of-UTC) offset (Asia/Kolkata, UTC+5:30)', () => { expect(zonedWallClockToUtc('2026-06-15T09:00', 'Asia/Kolkata').toISOString()).toBe( '2026-06-15T03:30:00.000Z' @@ -48,11 +129,104 @@ describe('zonedWallClockToUtc', () => { }) it('resolves a spring-forward gap wall-clock forward by the DST shift', () => { - // 2026-03-08 02:00–02:59 does not exist in America/New_York (EST→EDT). - expect(zonedWallClockToUtc('2026-03-08T02:30', 'America/New_York').toISOString()).toBe( - '2026-03-08T07:30:00.000Z' + const instant = zonedWallClockToUtc('2026-03-08T02:30', 'America/New_York') + const stampedWallClock = zonedWallClockWithOffset('2026-03-08T02:30', 'America/New_York') + + expect(instant.toISOString()).toBe('2026-03-08T07:30:00.000Z') + expect(stampedWallClock).toBe('2026-03-08T02:30-05:00') + expect(new Date(stampedWallClock).toISOString()).toBe(instant.toISOString()) + }) + + it.each([ + [ + 'Europe/Berlin', + '2026-03-29T02:30', + '2026-03-29T01:30:00.000Z', + '2026-03-29T03:30:00+02:00', + '2026-03-29T02:30+01:00', + ], + [ + 'Australia/Lord_Howe', + '2026-10-04T02:15', + '2026-10-03T15:45:00.000Z', + '2026-10-04T02:45:00+11:00', + '2026-10-04T02:15+10:30', + ], + ])( + 'resolves an east-of-UTC spring-forward gap in %s to the first compatible wall-clock', + (timeZone, wallClock, expectedInstant, expectedRenderedWallClock, expectedStampedWallClock) => { + const instant = zonedWallClockToUtc(wallClock, timeZone) + const stampedWallClock = zonedWallClockWithOffset(wallClock, timeZone) + + expect(instant.toISOString()).toBe(expectedInstant) + expect(formatInstantInTimeZone(instant, timeZone)).toBe(expectedRenderedWallClock) + expect(stampedWallClock).toBe(expectedStampedWallClock) + expect(new Date(stampedWallClock).toISOString()).toBe(expectedInstant) + } + ) + + it.each([ + ['America/New_York', '2026-11-01T01:30', '2026-11-01T06:30:00.000Z', '-05:00'], + ['Europe/Berlin', '2026-10-25T02:30', '2026-10-25T01:30:00.000Z', '+01:00'], + ['Australia/Lord_Howe', '2026-04-05T01:45', '2026-04-04T15:15:00.000Z', '+10:30'], + ])( + 'chooses the later post-transition instant for an ambiguous fall-back wall-clock in %s', + (timeZone, wallClock, expectedInstant, expectedOffset) => { + const instant = zonedWallClockToUtc(wallClock, timeZone) + const stampedWallClock = zonedWallClockWithOffset(wallClock, timeZone) + + expect(instant.toISOString()).toBe(expectedInstant) + expect(stampedWallClock).toBe(`${wallClock}${expectedOffset}`) + expect(new Date(stampedWallClock).toISOString()).toBe(expectedInstant) + } + ) + + it.each([ + ['America/New_York', '2026-11-01T01:30', '2026-11-01T05:30:00.000Z', '-04:00'], + ['Europe/Berlin', '2026-10-25T02:30', '2026-10-25T00:30:00.000Z', '+02:00'], + ['Australia/Lord_Howe', '2026-04-05T01:45', '2026-04-04T14:45:00.000Z', '+11:00'], + ])( + 'can choose the earlier instant for an ambiguous fall-back wall-clock in %s', + (timeZone, wallClock, expectedInstant, expectedOffset) => { + const options = { ambiguousTime: 'earlier' as const } + const instant = zonedWallClockToUtc(wallClock, timeZone, options) + const stampedWallClock = zonedWallClockWithOffset(wallClock, timeZone, options) + + expect(instant.toISOString()).toBe(expectedInstant) + expect(stampedWallClock).toBe(`${wallClock}${expectedOffset}`) + expect(new Date(stampedWallClock).toISOString()).toBe(expectedInstant) + } + ) + + it('does not retain timezone state between consecutive resolutions', () => { + const wallClock = '2026-06-15T09:00:30' + + expect(zonedWallClockToUtc(wallClock, 'America/New_York').toISOString()).toBe( + '2026-06-15T13:00:30.000Z' + ) + expect(zonedWallClockToUtc(wallClock, 'Asia/Kathmandu').toISOString()).toBe( + '2026-06-15T03:15:30.000Z' + ) + expect(zonedWallClockToUtc(wallClock, 'America/New_York').toISOString()).toBe( + '2026-06-15T13:00:30.000Z' ) }) + + it('can serialize historical sub-minute offsets toward a later instant', () => { + const wallClock = '1970-01-01T00:00:00' + const timezone = 'Africa/Monrovia' + const exactInstant = zonedWallClockToUtc(wallClock, timezone) + const options = { offsetMinuteRounding: 'floor' as const } + + expect(exactInstant.toISOString()).toBe('1970-01-01T00:44:30.000Z') + expect(zonedWallClockWithOffset(wallClock, timezone, options)).toBe('1970-01-01T00:00:00-00:45') + expect(formatInstantInTimeZone(exactInstant, timezone, options)).toBe( + '1970-01-01T00:00:00-00:45' + ) + expect( + Date.parse(zonedWallClockWithOffset(wallClock, timezone, options)) + ).toBeGreaterThanOrEqual(exactInstant.getTime()) + }) }) describe('wallClockNow', () => { diff --git a/apps/sim/lib/core/utils/timezone.ts b/apps/sim/lib/core/utils/timezone.ts index 297c0a65ec8..e4d2ffea799 100644 --- a/apps/sim/lib/core/utils/timezone.ts +++ b/apps/sim/lib/core/utils/timezone.ts @@ -23,6 +23,51 @@ const COMMON_TIMEZONES = [ 'Australia/Sydney', ] +/** A wall-clock reading of an instant in some timezone. */ +export interface WallClockParts { + year: number + /** 1-based month. */ + month: number + day: number + hour: number + minute: number + second: number +} + +function pad(value: number): string { + return String(value).padStart(2, '0') +} + +/** Formats years 0–9999 using ISO's four-digit representation. */ +export function formatIsoYear(year: number): string { + const serialized = String(year) + return year >= 0 && year <= 9999 ? serialized.padStart(4, '0') : serialized +} + +/** Builds a UTC timestamp without `Date.UTC` remapping years 0–99 to 1900–1999. */ +function utcTimestamp(wall: WallClockParts): number { + if (wall.year < 0 || wall.year > 99) { + return Date.UTC(wall.year, wall.month - 1, wall.day, wall.hour, wall.minute, wall.second) + } + const date = new Date(0) + date.setUTCFullYear(wall.year, wall.month - 1, wall.day) + date.setUTCHours(wall.hour, wall.minute, wall.second, 0) + return date.getTime() +} + +/** RFC 3339 offset suffix: `Z` for zero, else `±HH:MM`. */ +export function formatUtcOffsetSuffix(offsetMinutes: number): string { + if (offsetMinutes === 0) return 'Z' + const sign = offsetMinutes > 0 ? '+' : '-' + const absoluteMinutes = Math.abs(offsetMinutes) + return `${sign}${pad(Math.floor(absoluteMinutes / 60))}:${pad(absoluteMinutes % 60)}` +} + +function offsetMsFromWallClock(instant: Date, wall: WallClockParts): number { + const wallAsUtc = utcTimestamp(wall) + return wallAsUtc - instant.getTime() +} + /** The IANA timezone the current runtime resolves to (e.g. `America/New_York`). */ export function getBrowserTimezone(): string { return Intl.DateTimeFormat().resolvedOptions().timeZone @@ -38,6 +83,11 @@ export function isValidTimezone(timezone: string): boolean { } } +/** Removes control characters and bounds an untrusted timezone before displaying it. */ +export function sanitizeTimezoneForDisplay(timezone: string, maxLength = 64): string { + return truncate(timezone.replace(/[\p{Cc}\p{Zl}\p{Zp}]/gu, ' '), maxLength) +} + /** * Rejects a timezone that is not an IANA name. * @@ -54,7 +104,7 @@ export function assertValidTimezone(timezone: string): void { // Echoed back trimmed and stripped of line breaks: the rejected value came off // a query string, and a raw one carrying newlines or U+2028/U+2029 would forge // extra lines in whatever log or error surface renders the message. - const safe = truncate(timezone.replace(/[\p{Cc}\p{Zl}\p{Zp}]/gu, ' '), 64) + const safe = sanitizeTimezoneForDisplay(timezone) throw new Error(`Invalid timezone: ${safe}. Use an IANA name like "America/Los_Angeles".`) } } @@ -116,22 +166,66 @@ export function getTimezoneOptions(): TimezoneOption[] { } /** - * An instant's wall-clock time in `timeZone` as a naive `yyyy-MM-ddTHH:mm` - * string. Lets callers reason about a user's local date/time without UTC — e.g. - * to recover the local date/time a stored task instant represents in its zone. + * The wall-clock fields of `instant` in `timeZone`, or in the runtime's local + * timezone when omitted. */ -export function zonedWallClock(instant: Date, timeZone: string): string { - const parts = new Intl.DateTimeFormat('en-CA', { +export function getWallClockParts(instant: Date, timeZone?: string): WallClockParts { + if (timeZone === undefined) { + return { + year: instant.getFullYear(), + month: instant.getMonth() + 1, + day: instant.getDate(), + hour: instant.getHours(), + minute: instant.getMinutes(), + second: instant.getSeconds(), + } + } + + const parts = new Intl.DateTimeFormat('en-US', { timeZone, + hourCycle: 'h23', + era: 'short', year: 'numeric', month: '2-digit', day: '2-digit', hour: '2-digit', minute: '2-digit', - hourCycle: 'h23', + second: '2-digit', }).formatToParts(instant) - const get = (type: string) => parts.find((p) => p.type === type)?.value ?? '00' - return `${get('year')}-${get('month')}-${get('day')}T${get('hour')}:${get('minute')}` + const get = (type: string) => Number(parts.find((part) => part.type === type)?.value) + const year = get('year') + const era = parts.find((part) => part.type === 'era')?.value + return { + year: era === 'BC' ? 1 - year : year, + month: get('month'), + day: get('day'), + hour: get('hour'), + minute: get('minute'), + second: get('second'), + } +} + +/** Formats an instant as an RFC 3339 wall time in an IANA timezone. */ +export function formatInstantInTimeZone( + instant: Date, + timeZone: string, + options?: ZonedWallClockOptions +): string { + const wall = getWallClockParts(instant, timeZone) + const wholeSecondInstant = new Date(Math.floor(instant.getTime() / 1000) * 1000) + const exactOffsetMinutes = offsetMsFromWallClock(wholeSecondInstant, wall) / 60_000 + const offsetMinutes = roundOffsetMinutes(exactOffsetMinutes, options) + return `${formatIsoYear(wall.year)}-${pad(wall.month)}-${pad(wall.day)}T${pad(wall.hour)}:${pad(wall.minute)}:${pad(wall.second)}${formatUtcOffsetSuffix(offsetMinutes)}` +} + +/** + * An instant's wall-clock time in `timeZone` as a naive `yyyy-MM-ddTHH:mm` + * string. Lets callers reason about a user's local date/time without UTC — e.g. + * to recover the local date/time a stored task instant represents in its zone. + */ +export function zonedWallClock(instant: Date, timeZone: string): string { + const wall = getWallClockParts(instant, timeZone) + return `${formatIsoYear(wall.year)}-${pad(wall.month)}-${pad(wall.day)}T${pad(wall.hour)}:${pad(wall.minute)}` } /** The current wall-clock time in `timeZone` as a naive `yyyy-MM-ddTHH:mm` string. */ @@ -156,26 +250,59 @@ export function zonedClockDate(instant: Date, timeZone: string): Date { /** The UTC offset (ms, east-positive) of `timeZone` at a given instant. */ function timezoneOffsetMs(instant: Date, timeZone: string): number { - const parts = new Intl.DateTimeFormat('en-US', { - timeZone, - hourCycle: 'h23', - year: 'numeric', - month: '2-digit', - day: '2-digit', - hour: '2-digit', - minute: '2-digit', - second: '2-digit', - }).formatToParts(instant) - const get = (type: string) => Number(parts.find((p) => p.type === type)?.value) - const asUtc = Date.UTC( - get('year'), - get('month') - 1, - get('day'), - get('hour'), - get('minute'), - get('second') + return offsetMsFromWallClock(instant, getWallClockParts(instant, timeZone)) +} + +interface ZonedWallClockResolution { + instant: Date + offsetMinutes: number +} + +export interface ZonedWallClockOptions { + /** Which real instant to use when the wall clock occurs twice during a DST fall-back. */ + ambiguousTime?: 'earlier' | 'later' + /** How to serialize rare historical offsets containing seconds into RFC 3339 minutes. */ + offsetMinuteRounding?: 'nearest' | 'floor' +} + +function roundOffsetMinutes(exactOffsetMinutes: number, options?: ZonedWallClockOptions): number { + return options?.offsetMinuteRounding === 'floor' + ? Math.floor(exactOffsetMinutes) + : Math.round(exactOffsetMinutes) +} + +function resolveZonedWallClock( + wallClock: string, + timeZone: string, + options?: ZonedWallClockOptions +): ZonedWallClockResolution { + const [datePart, timePart] = wallClock.split('T') + const [year, month, day] = datePart.split('-').map(Number) + const [hour, minute, second = 0] = timePart.split(':').map(Number) + const utcGuess = utcTimestamp({ year, month, day, hour, minute, second }) + const dayMs = 24 * 60 * 60 * 1000 + const offsets = new Set( + [-dayMs, 0, dayMs].map((distance) => timezoneOffsetMs(new Date(utcGuess + distance), timeZone)) ) - return asUtc - instant.getTime() + const candidates = [...offsets].map((offset) => { + const instantMs = utcGuess - offset + const actualOffset = timezoneOffsetMs(new Date(instantMs), timeZone) + return { instantMs, wallClockMs: instantMs + actualOffset } + }) + const exactCandidate = candidates + .filter(({ wallClockMs }) => wallClockMs === utcGuess) + .sort((a, b) => + options?.ambiguousTime === 'earlier' ? a.instantMs - b.instantMs : b.instantMs - a.instantMs + )[0] + const compatibleCandidate = candidates + .filter(({ wallClockMs }) => wallClockMs > utcGuess) + .sort((a, b) => a.wallClockMs - b.wallClockMs || a.instantMs - b.instantMs)[0] + const chosenCandidate = exactCandidate ?? compatibleCandidate ?? candidates[0] + const instantMs = chosenCandidate.instantMs + return { + instant: new Date(instantMs), + offsetMinutes: (utcGuess - instantMs) / 60_000, + } } /** @@ -184,23 +311,28 @@ function timezoneOffsetMs(instant: Date, timeZone: string): number { * whose own offset reproduces the requested wall-clock, which is correct for any * date (including future ones whose offset differs from today's) and across DST: * a naive single pass reads the offset on the wrong side of a same-day boundary - * — notably the autumn fall-back hour — and lands an hour off. For an ambiguous - * fall-back wall-clock the later (post-transition) instant is chosen; a + * — notably the autumn fall-back hour — and lands an hour off. An ambiguous + * fall-back wall-clock defaults to the later, post-transition instant, but + * callers preserving earlier semantics may request the earlier instant. A * wall-clock in the spring-forward gap (a nonexistent local hour) has no * self-consistent instant and resolves forward by the DST shift, matching how * calendar apps treat that once-a-year hour. */ -export function zonedWallClockToUtc(wallClock: string, timeZone: string): Date { - const [datePart, timePart] = wallClock.split('T') - const [year, month, day] = datePart.split('-').map(Number) - const [hour, minute, second = 0] = timePart.split(':').map(Number) - const utcGuess = Date.UTC(year, month - 1, day, hour, minute, second) - const guessOffset = timezoneOffsetMs(new Date(utcGuess), timeZone) - const candidate = utcGuess - guessOffset - const candidateOffset = timezoneOffsetMs(new Date(candidate), timeZone) - if (candidateOffset === guessOffset) return new Date(candidate) - const adjusted = utcGuess - candidateOffset - return timezoneOffsetMs(new Date(adjusted), timeZone) === candidateOffset - ? new Date(adjusted) - : new Date(candidate) +export function zonedWallClockToUtc( + wallClock: string, + timeZone: string, + options?: ZonedWallClockOptions +): Date { + return resolveZonedWallClock(wallClock, timeZone, options).instant +} + +/** Stamps a naive wall-clock with the offset selected by the shared timezone resolver. */ +export function zonedWallClockWithOffset( + wallClock: string, + timeZone: string, + options?: ZonedWallClockOptions +): string { + const resolution = resolveZonedWallClock(wallClock, timeZone, options) + const offsetMinutes = roundOffsetMinutes(resolution.offsetMinutes, options) + return `${wallClock}${formatUtcOffsetSuffix(offsetMinutes)}` } diff --git a/apps/sim/lib/core/utils/with-route-handler.ts b/apps/sim/lib/core/utils/with-route-handler.ts index 86e41967ada..3e225c3b9d4 100644 --- a/apps/sim/lib/core/utils/with-route-handler.ts +++ b/apps/sim/lib/core/utils/with-route-handler.ts @@ -5,6 +5,7 @@ import { NextResponse } from 'next/server' import { getRateLimitHeaders } from '@/lib/api/server/rate-limit-context' import { HttpError } from '@/lib/core/utils/http-error' import { generateRequestId } from '@/lib/core/utils/request' +import { withPermissionGroupScope } from '@/lib/permission-groups/request-scope.server' const logger = createLogger('RouteHandler') @@ -114,7 +115,7 @@ export function withRouteHandler( return runWithRequestContext({ requestId, method, path, traceId }, async () => { let response: NextResponse | Response try { - response = await handler(request, context) + response = await withPermissionGroupScope(() => handler(request, context)) } catch (error) { const duration = Date.now() - startTime const message = getErrorMessage(error, 'Unknown error') diff --git a/apps/sim/lib/credential-groups/application/authorization.test.ts b/apps/sim/lib/credential-groups/application/authorization.test.ts index 052dfba31f9..fec8b1a7bbb 100644 --- a/apps/sim/lib/credential-groups/application/authorization.test.ts +++ b/apps/sim/lib/credential-groups/application/authorization.test.ts @@ -20,7 +20,10 @@ vi.mock('@/lib/resource-policies/repository', () => ({ requireResourcePolicy: mocks.requirePolicy, })) -import { requireCredentialGroupCredentialAccess } from '@/lib/credential-groups/application/authorization' +import { + requireCredentialGroupCredentialAccess, + requireCredentialGroupWorkflowActor, +} from '@/lib/credential-groups/application/authorization' const context = { workspaceId: 'workspace-1', @@ -216,3 +219,60 @@ describe('requireCredentialGroupCredentialAccess', () => { expect(mocks.loadEnrollmentAccess).not.toHaveBeenCalled() }) }) + +describe('requireCredentialGroupWorkflowActor', () => { + it('returns the external subject a Slack-triggered run acts as', () => { + expect(requireCredentialGroupWorkflowActor(executorPrincipal())).toEqual({ + kind: 'external_user', + provider: 'slack', + tenantId: 'T123', + subjectId: 'U123', + }) + }) + + it('returns no subject for an actorless deployed run', () => { + const principal = executorPrincipal() + principal.delegationContext!.principal = { + kind: 'system', + serviceId: 'schedule', + workspaceId: 'workspace-1', + workflowId: 'root-workflow', + } + + expect(requireCredentialGroupWorkflowActor(principal)).toBeNull() + }) + + it('returns the Sim subject a session-actor run acts as', () => { + const principal = executorPrincipal() + principal.subjectUserId = 'user-1' + principal.delegationContext!.principal = { + kind: 'session', + userId: 'user-1', + sessionId: 'session-1', + } + + expect(requireCredentialGroupWorkflowActor(principal)).toEqual({ + kind: 'sim_user', + userId: 'user-1', + }) + }) + + it('rejects a delegation whose asserted subject contradicts its run', () => { + const invented = executorPrincipal() + invented.subjectUserId = 'invented-user' + expect(() => requireCredentialGroupWorkflowActor(invented)).toThrow( + 'Credential Group actor access required' + ) + + const mismatched = executorPrincipal() + mismatched.subjectUserId = 'user-2' + mismatched.delegationContext!.principal = { + kind: 'session', + userId: 'user-1', + sessionId: 'session-1', + } + expect(() => requireCredentialGroupWorkflowActor(mismatched)).toThrow( + 'Credential Group actor access required' + ) + }) +}) diff --git a/apps/sim/lib/credential-groups/application/authorization.ts b/apps/sim/lib/credential-groups/application/authorization.ts index 2cde1459083..13cc444e606 100644 --- a/apps/sim/lib/credential-groups/application/authorization.ts +++ b/apps/sim/lib/credential-groups/application/authorization.ts @@ -1,5 +1,6 @@ import { type Principal, + type PrincipalSubject, resolvePrincipalSubject, type WorkflowExecutionAuthority, type WorkflowExecutionPrincipal, @@ -67,16 +68,19 @@ function requireConsistentWorkflowSubject( return subject } -export function requireCredentialGroupWorkflowSubject(principal: Principal): string { - const subject = resolvePrincipalSubject(requireWorkflowExecutionPrincipal(principal)) - if ( - subject?.kind !== 'sim_user' || - principal.kind !== 'delegated' || - principal.subjectUserId !== subject.userId - ) { - throw new OrchestrationError('forbidden', 'Credential Group user access required') - } - return subject.userId +/** + * Asserts the delegation still names the subject its run was minted for, without + * requiring that subject to be a Sim user. + * + * A Slack-triggered run's subject is the external Slack user, and a scheduled, + * public-API, or subject-less webhook run has no subject at all. Neither is + * representable as a Sim user, and neither is what authorizes the call — for an + * actorless caller that is the deployment the workspace layer already checked. + * Whoever the run acts as is attribution only; an invitation issued with no Sim + * user simply records none. + */ +export function requireCredentialGroupWorkflowActor(principal: Principal): PrincipalSubject | null { + return requireConsistentWorkflowSubject(principal, requireWorkflowExecutionPrincipal(principal)) } export async function requireCredentialGroupCredentialAccess( diff --git a/apps/sim/lib/credential-groups/application/enrollment-operations.ts b/apps/sim/lib/credential-groups/application/enrollment-operations.ts index b53788bda45..eb17a758bca 100644 --- a/apps/sim/lib/credential-groups/application/enrollment-operations.ts +++ b/apps/sim/lib/credential-groups/application/enrollment-operations.ts @@ -1,4 +1,5 @@ import type { ApplicationOperation } from '@/lib/core/application' +import { assertOperationCapability } from '@/lib/core/application' export interface CredentialGroupEnrollmentOperation extends ApplicationOperation { @@ -9,24 +10,33 @@ function defineCredentialGroupEnrollmentOperation( operation: CredentialGroupEnrollmentOperation ): CredentialGroupEnrollmentOperation { if (!operation.id.trim()) throw new Error('Credential Group enrollment operation ID is required') + assertOperationCapability(operation) return Object.freeze(operation) } export const credentialGroupEnrollmentOperations = { + // permission-group-exempt: the enrollment principal is a one-time credential-connect token, not a workspace member, so no permission group governs it read: defineCredentialGroupEnrollmentOperation({ id: 'credential_groups.enrollment.read', + capability: 'none', principalKind: 'credential_group_enrollment', }), + // permission-group-exempt: the enrollment principal is a one-time credential-connect token, not a workspace member, so no permission group governs it startOAuth: defineCredentialGroupEnrollmentOperation({ id: 'credential_groups.enrollment.oauth.start', + capability: 'none', principalKind: 'credential_group_enrollment', }), + // permission-group-exempt: the enrollment principal is a one-time credential-connect token, not a workspace member, so no permission group governs it completeOAuth: defineCredentialGroupEnrollmentOperation({ id: 'credential_groups.enrollment.oauth.complete', + capability: 'none', principalKind: 'credential_group_enrollment', }), + // permission-group-exempt: the enrollment principal is a one-time credential-connect token, not a workspace member, so no permission group governs it complete: defineCredentialGroupEnrollmentOperation({ id: 'credential_groups.enrollment.complete', + capability: 'none', principalKind: 'credential_group_enrollment', }), } as const diff --git a/apps/sim/lib/credential-groups/application/list-groups.ts b/apps/sim/lib/credential-groups/application/list-groups.ts index d33938db34e..75599f6734e 100644 --- a/apps/sim/lib/credential-groups/application/list-groups.ts +++ b/apps/sim/lib/credential-groups/application/list-groups.ts @@ -2,7 +2,7 @@ import { defineAuthorizedWorkspaceUseCase } from '@/lib/core/application' import { OrchestrationError } from '@/lib/core/orchestration/types' import { credentialGroupWorkspaceDelegationPolicy, - requireCredentialGroupWorkflowSubject, + requireCredentialGroupWorkflowActor, } from '@/lib/credential-groups/application/authorization' import { requireCredentialGroupsAvailable, @@ -35,7 +35,7 @@ export const listCredentialGroupsForWorkflow = defineAuthorizedWorkspaceUseCase( resolveCredentialGroupWorkspaceContext(input.workspaceId), authorizationOptions: { delegation: credentialGroupWorkspaceDelegationPolicy }, authorizeResource({ principal }) { - requireCredentialGroupWorkflowSubject(principal) + requireCredentialGroupWorkflowActor(principal) }, execute: async ({ input, context }): Promise => { if ( diff --git a/apps/sim/lib/credential-groups/application/list-people.ts b/apps/sim/lib/credential-groups/application/list-people.ts index 10338d8defe..6d5d2ae9043 100644 --- a/apps/sim/lib/credential-groups/application/list-people.ts +++ b/apps/sim/lib/credential-groups/application/list-people.ts @@ -3,7 +3,7 @@ import { defineAuthorizedWorkspaceUseCase } from '@/lib/core/application' import { OrchestrationError } from '@/lib/core/orchestration/types' import { credentialGroupDelegationPolicy, - requireCredentialGroupWorkflowSubject, + requireCredentialGroupWorkflowActor, } from '@/lib/credential-groups/application/authorization' import { requireCredentialGroupsAvailable, @@ -38,7 +38,7 @@ export const listCredentialGroupPeople = defineAuthorizedWorkspaceUseCase({ resolveCredentialGroupContext(input.credentialGroupId), authorizationOptions: { delegation: credentialGroupDelegationPolicy }, authorizeResource({ principal }) { - requireCredentialGroupWorkflowSubject(principal) + requireCredentialGroupWorkflowActor(principal) }, execute: async ({ input, context }) => { if (context.status !== 'active') { diff --git a/apps/sim/lib/credential-groups/application/operations.ts b/apps/sim/lib/credential-groups/application/operations.ts index 9339f5dfd9f..409920b039e 100644 --- a/apps/sim/lib/credential-groups/application/operations.ts +++ b/apps/sim/lib/credential-groups/application/operations.ts @@ -1,111 +1,159 @@ import { defineWorkspaceOperation } from '@/lib/core/application' +/** + * Credential groups collect OAuth credentials from people outside the workspace + * so a workflow can act as them — a distinct, entitlement-gated settings + * section, not part of the Integrations tab. + * + * None of them declares a capability. `integrations.manage` names the + * Integrations tab, where a member connects their own accounts, and + * `credentials.personal` withholds exactly that; both describe a member acting + * for themselves, which is the opposite of this section. Every operation here + * already requires workspace `admin`, and the executor-delegated reads run + * inside a workflow whose credential access is decided by the group's own + * enrollment rows. Borrowing a key that names a different surface would make + * hiding the Integrations tab silently disable an unrelated admin section. + */ export const credentialGroupOperations = { + // permission-group-exempt: workspace admin already decides this, and no group key names the credential-groups section listSettings: defineWorkspaceOperation({ id: 'credential_groups.settings.list', minimumRole: 'admin', workspaceApiKey: 'deny', + capability: 'none', principalKinds: ['session'], }), + // permission-group-exempt: workspace admin already decides this, and no group key names the credential-groups section create: defineWorkspaceOperation({ id: 'credential_groups.create', minimumRole: 'admin', workspaceApiKey: 'deny', + capability: 'none', principalKinds: ['session'], }), + // permission-group-exempt: workspace admin already decides this, and no group key names the credential-groups section readSettings: defineWorkspaceOperation({ id: 'credential_groups.settings.read', minimumRole: 'admin', workspaceApiKey: 'deny', + capability: 'none', principalKinds: ['session'], }), + // permission-group-exempt: workspace admin already decides this, and no group key names the credential-groups section update: defineWorkspaceOperation({ id: 'credential_groups.update', minimumRole: 'admin', workspaceApiKey: 'deny', + capability: 'none', principalKinds: ['session'], }), + // permission-group-exempt: workspace admin already decides this, and no group key names the credential-groups section readAccess: defineWorkspaceOperation({ id: 'credential_groups.access.read', minimumRole: 'admin', workspaceApiKey: 'deny', + capability: 'none', principalKinds: ['session'], }), + // permission-group-exempt: workspace admin already decides this, and no group key names the credential-groups section updateAccess: defineWorkspaceOperation({ id: 'credential_groups.access.update', minimumRole: 'admin', workspaceApiKey: 'deny', + capability: 'none', principalKinds: ['session'], }), + // permission-group-exempt: workspace admin already decides this, and no group key names the credential-groups section delete: defineWorkspaceOperation({ id: 'credential_groups.delete', minimumRole: 'admin', workspaceApiKey: 'deny', + capability: 'none', principalKinds: ['session'], }), + // permission-group-exempt: workspace admin already decides this, and no group key names the credential-groups section inviteBatch: defineWorkspaceOperation({ id: 'credential_groups.invites.send_batch', minimumRole: 'admin', workspaceApiKey: 'deny', + capability: 'none', principalKinds: ['session'], }), + // permission-group-exempt: workspace admin already decides this, and no group key names the credential-groups section resendEnrollment: defineWorkspaceOperation({ id: 'credential_groups.enrollments.resend', minimumRole: 'admin', workspaceApiKey: 'deny', + capability: 'none', principalKinds: ['session'], }), + // permission-group-exempt: workspace admin already decides this, and no group key names the credential-groups section deleteEnrollment: defineWorkspaceOperation({ id: 'credential_groups.enrollments.delete', minimumRole: 'admin', workspaceApiKey: 'deny', + capability: 'none', principalKinds: ['session'], }), + // permission-group-exempt: read by the executor to resolve an enrolled person's credential; the group's enrollment rows are the gate, and no group key names them listCredentials: defineWorkspaceOperation({ id: 'credential_groups.credentials.list', minimumRole: 'read', workspaceApiKey: 'deny', + capability: 'none', principalKinds: ['delegated'], delegatedServices: ['executor'], }), + // permission-group-exempt: read by the executor to resolve an enrolled person's credential; the group's enrollment rows are the gate, and no group key names them listGroups: defineWorkspaceOperation({ id: 'credential_groups.list', minimumRole: 'read', workspaceApiKey: 'deny', + capability: 'none', principalKinds: ['delegated'], delegatedServices: ['executor'], }), + // permission-group-exempt: read by the executor to resolve an enrolled person's credential; the group's enrollment rows are the gate, and no group key names them listPeople: defineWorkspaceOperation({ id: 'credential_groups.people.list', minimumRole: 'read', workspaceApiKey: 'deny', + capability: 'none', principalKinds: ['delegated'], delegatedServices: ['executor'], }), + // permission-group-exempt: enrolls an outside person in a credential group, not a member in a workspace, so invitations.send does not name it sendInvite: defineWorkspaceOperation({ id: 'credential_groups.invites.send', minimumRole: 'admin', workspaceApiKey: 'deny', + capability: 'none', principalKinds: ['delegated'], delegatedServices: ['executor'], }), + // permission-group-exempt: mints an enrollment link for an outside person, not a workspace invitation, so invitations.send does not name it createInviteLink: defineWorkspaceOperation({ id: 'credential_groups.invites.link.create', minimumRole: 'admin', workspaceApiKey: 'deny', + capability: 'none', principalKinds: ['delegated'], delegatedServices: ['executor'], }), + // permission-group-exempt: workspace admin already decides this, and no group key names the credential-groups section startSlackConfiguration: defineWorkspaceOperation({ id: 'credential_groups.slack_configuration.start', minimumRole: 'admin', workspaceApiKey: 'deny', + capability: 'none', principalKinds: ['session'], }), + // permission-group-exempt: workspace admin already decides this, and no group key names the credential-groups section completeSlackConfiguration: defineWorkspaceOperation({ id: 'credential_groups.slack_configuration.complete', minimumRole: 'admin', workspaceApiKey: 'deny', + capability: 'none', principalKinds: ['session'], }), } as const diff --git a/apps/sim/lib/credential-groups/application/public-enrollment.test.ts b/apps/sim/lib/credential-groups/application/public-enrollment.test.ts index 9fae003f2de..f6cc790ef10 100644 --- a/apps/sim/lib/credential-groups/application/public-enrollment.test.ts +++ b/apps/sim/lib/credential-groups/application/public-enrollment.test.ts @@ -7,6 +7,8 @@ import { beforeEach, describe, expect, it, vi } from 'vitest' const mocks = vi.hoisted(() => ({ completeEnrollment: vi.fn(), + completeOAuth: vi.fn(), + fireTrigger: vi.fn(), getEnrollment: vi.fn(), getOAuthContext: vi.fn(), startOAuth: vi.fn(), @@ -19,11 +21,17 @@ vi.mock('@/lib/credential-groups/enrollments', () => ({ })) vi.mock('@/lib/credential-groups/oauth', () => ({ - completeCredentialGroupOAuth: vi.fn(), + completeCredentialGroupOAuth: mocks.completeOAuth, startCredentialGroupOAuth: mocks.startOAuth, })) +vi.mock('@/lib/credential-groups/trigger', () => ({ + fireCredentialGroupTrigger: mocks.fireTrigger, +})) + import { + completePublicCredentialGroupEnrollment, + completePublicCredentialGroupOAuth, readPublicCredentialGroupEnrollment, startPublicCredentialGroupOAuth, } from '@/lib/credential-groups/application/public-enrollment' @@ -44,15 +52,43 @@ const identity = { email: principal.email, invitationTokenHash: principal.invitationTokenHash, } +const oauthAttempt = { + state: 'state-1', + provider: 'gmail' as const, + nonceHash: 'nonce-hash', + enrollmentId: principal.enrollmentId, + credentialGroupId: principal.credentialGroupId, + optionId: 'option-1', + authorizationAppId: 'google:client', + scopeVersion: 1, + requiredScopes: ['openid'], + redirectUri: 'https://sim.ai/api/auth/oauth2/callback/google-email', + invitationToken, + createdAt: Date.now(), +} describe('public Credential Group enrollment application operations', () => { beforeEach(() => { vi.clearAllMocks() - mocks.getEnrollment.mockResolvedValue({ status: 'invited', options: [] }) + mocks.getEnrollment.mockResolvedValue({ + status: 'invited', + credentialGroupName: 'Credential Group', + options: [], + }) mocks.getOAuthContext.mockResolvedValue({ enrollmentId: 'enrollment-1', credentialGroupId: 'group-1', - option: { id: 'option-1' }, + credentialGroupName: 'Credential Group', + option: { id: 'option-1', provider: 'gmail' }, + }) + mocks.completeOAuth.mockResolvedValue({ + created: true, + credentialId: 'credential-1', + credentialGroupOptionId: 'option-1', + provider: 'gmail', + providerId: 'google-email', + displayName: 'person@example.com', + enrollmentStatus: 'in_progress', }) mocks.startOAuth.mockResolvedValue('https://accounts.example/authorize') }) @@ -74,7 +110,13 @@ describe('public Credential Group enrollment application operations', () => { const result = await readPublicCredentialGroupEnrollment.execute({ principal, input: {} }) expect(mocks.getEnrollment).toHaveBeenCalledWith(identity) - expect(result).toEqual({ enrollment: { status: 'invited', options: [] } }) + expect(result).toEqual({ + enrollment: { + status: 'invited', + credentialGroupName: 'Credential Group', + options: [], + }, + }) }) it('fails closed when the current invitation no longer resolves', async () => { @@ -104,4 +146,78 @@ describe('public Credential Group enrollment application operations', () => { expect(mocks.getOAuthContext).toHaveBeenCalledWith(identity, 'option-1') expect(result).toEqual({ authorizationUrl: 'https://accounts.example/authorize' }) }) + + it('fires form submitted only for the first completion transition', async () => { + mocks.completeEnrollment.mockResolvedValue({ completed: true, transitioned: true }) + + const result = await completePublicCredentialGroupEnrollment.execute({ + principal, + input: {}, + }) + + expect(result).toEqual({ completed: true }) + expect(mocks.fireTrigger).toHaveBeenCalledWith({ + event: 'form_submitted', + workspaceId: 'workspace-1', + credentialGroupId: 'group-1', + credentialGroupName: 'Credential Group', + enrollmentId: 'enrollment-1', + email: 'person@example.com', + enrollmentStatus: 'completed', + }) + + vi.clearAllMocks() + mocks.getEnrollment.mockResolvedValue({ + status: 'completed', + credentialGroupName: 'Credential Group', + options: [], + }) + mocks.completeEnrollment.mockResolvedValue({ completed: true, transitioned: false }) + + await completePublicCredentialGroupEnrollment.execute({ principal, input: {} }) + + expect(mocks.fireTrigger).not.toHaveBeenCalled() + }) + + it('distinguishes a new credential from a reconnection', async () => { + await completePublicCredentialGroupOAuth.execute({ + principal, + input: { attempt: oauthAttempt, code: 'authorization-code' }, + }) + + expect(mocks.fireTrigger).toHaveBeenCalledWith( + expect.objectContaining({ + event: 'credential_added', + credentialGroupId: 'group-1', + enrollmentId: 'enrollment-1', + credential: expect.objectContaining({ credentialId: 'credential-1' }), + }) + ) + + vi.clearAllMocks() + mocks.getOAuthContext.mockResolvedValue({ + enrollmentId: 'enrollment-1', + credentialGroupId: 'group-1', + credentialGroupName: 'Credential Group', + option: { id: 'option-1', provider: 'gmail' }, + }) + mocks.completeOAuth.mockResolvedValue({ + created: false, + credentialId: 'credential-1', + credentialGroupOptionId: 'option-1', + provider: 'gmail', + providerId: 'google-email', + displayName: 'person@example.com', + enrollmentStatus: 'completed', + }) + + await completePublicCredentialGroupOAuth.execute({ + principal, + input: { attempt: oauthAttempt, code: 'authorization-code' }, + }) + + expect(mocks.fireTrigger).toHaveBeenCalledWith( + expect.objectContaining({ event: 'credential_reconnected', enrollmentStatus: 'completed' }) + ) + }) }) diff --git a/apps/sim/lib/credential-groups/application/public-enrollment.ts b/apps/sim/lib/credential-groups/application/public-enrollment.ts index ccda73d5984..5b3023aad8f 100644 --- a/apps/sim/lib/credential-groups/application/public-enrollment.ts +++ b/apps/sim/lib/credential-groups/application/public-enrollment.ts @@ -15,6 +15,7 @@ import { startCredentialGroupOAuth, } from '@/lib/credential-groups/oauth' import type { CredentialGroupOAuthAttempt } from '@/lib/credential-groups/oauth-state' +import { fireCredentialGroupTrigger } from '@/lib/credential-groups/trigger' interface AuthorizedCredentialGroupEnrollmentUseCaseDefinition { operation: O @@ -125,8 +126,19 @@ export const completePublicCredentialGroupEnrollment = operation: credentialGroupEnrollmentOperations.complete, resolveContext: ({ principal }) => resolvePublicEnrollmentContext(principal), async execute({ context }) { - const completed = await completeAuthorizedCredentialGroupEnrollment(context) - return { completed } + const completion = await completeAuthorizedCredentialGroupEnrollment(context) + if (completion?.transitioned) { + await fireCredentialGroupTrigger({ + event: 'form_submitted', + workspaceId: context.workspaceId, + credentialGroupId: context.credentialGroupId, + credentialGroupName: context.enrollment.credentialGroupName, + enrollmentId: context.enrollmentId, + email: context.email, + enrollmentStatus: 'completed', + }) + } + return { completed: completion?.completed ?? null } }, }) @@ -182,7 +194,23 @@ export const completePublicCredentialGroupOAuth = defineAuthorizedCredentialGrou }) => resolvePublicOAuthContext(principal, input.attempt.optionId), async execute({ principal, input, context }) { requireInvitationToken(principal, input.attempt.invitationToken) - await completeCredentialGroupOAuth(context.oauth, input.attempt, input.code) + const completion = await completeCredentialGroupOAuth(context.oauth, input.attempt, input.code) + await fireCredentialGroupTrigger({ + event: completion.created ? 'credential_added' : 'credential_reconnected', + workspaceId: context.workspaceId, + credentialGroupId: context.credentialGroupId, + credentialGroupName: context.oauth.credentialGroupName, + enrollmentId: context.enrollmentId, + email: context.email, + enrollmentStatus: completion.enrollmentStatus, + credential: { + credentialId: completion.credentialId, + credentialGroupOptionId: completion.credentialGroupOptionId, + provider: completion.provider, + providerId: completion.providerId, + displayName: completion.displayName, + }, + }) return { connectedOptionId: context.oauth.option.id } }, }) diff --git a/apps/sim/lib/credential-groups/application/send-invite.test.ts b/apps/sim/lib/credential-groups/application/send-invite.test.ts new file mode 100644 index 00000000000..d72495678fa --- /dev/null +++ b/apps/sim/lib/credential-groups/application/send-invite.test.ts @@ -0,0 +1,199 @@ +/** + * @vitest-environment node + */ +import type { WorkflowExecutionDelegatedPrincipal } from '@sim/auth/principal' +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const mocks = vi.hoisted(() => ({ + inviteEnrollment: vi.fn(), + loadInviter: vi.fn(), + requireAvailable: vi.fn(), + resolveGroup: vi.fn(), + resolvePermission: vi.fn(), +})) + +vi.mock('@/lib/credential-groups/application/context', () => ({ + requireCredentialGroupsAvailable: mocks.requireAvailable, + resolveCredentialGroupContext: mocks.resolveGroup, +})) + +vi.mock('@/lib/credential-groups/enrollments', () => ({ + inviteCredentialGroupEnrollment: mocks.inviteEnrollment, + loadCredentialGroupInviterIdentity: mocks.loadInviter, + CredentialGroupEnrollmentError: class CredentialGroupEnrollmentError extends Error { + constructor( + message: string, + readonly status: 400 | 404 | 409 | 502 + ) { + super(message) + } + }, +})) + +vi.mock('@sim/platform-authz/workspace', () => ({ + permissionSatisfies: (permission: string | null, required: string) => + permission === 'admin' || permission === required, + resolveEffectiveWorkspacePermission: mocks.resolvePermission, +})) + +import { sendCredentialGroupInvite } from '@/lib/credential-groups/application/send-invite' + +const context = { + workspaceId: 'workspace-1', + workspaceOrganizationId: null, + allowPersonalApiKeys: true, + billedAccountUserId: 'billing-owner-1', + credentialGroupId: 'group-1', + name: 'Support', + status: 'active' as const, + options: [], +} + +function executorPrincipal(): WorkflowExecutionDelegatedPrincipal { + return { + kind: 'delegated', + serviceId: 'executor', + subjectUserId: 'admin-1', + workspaceId: 'workspace-1', + delegationId: 'delegation-1', + audience: 'sim:credential-groups', + issuedAt: new Date(Date.now() - 1_000), + expiresAt: new Date(Date.now() + 60_000), + resourceScope: { credentialGroupId: 'group-1' }, + delegationContext: { + kind: 'workflow_execution', + workflowId: 'workflow-1', + principal: { kind: 'session', userId: 'admin-1', sessionId: 'session-1' }, + currentWorkflow: { workflowId: 'workflow-1', mode: 'draft' }, + }, + } +} + +/** A deployed run whose only actor is the external identity that triggered it. */ +function unattendedPrincipal( + principal: NonNullable['principal'] +): WorkflowExecutionDelegatedPrincipal { + const { subjectUserId: _subject, ...base } = executorPrincipal() + return { + ...base, + delegationContext: { + kind: 'workflow_execution', + workflowId: 'workflow-1', + principal, + currentWorkflow: { + workflowId: 'workflow-1', + mode: 'deployment', + deploymentVersionId: 'version-1', + }, + }, + } +} + +function slackPrincipal(): WorkflowExecutionDelegatedPrincipal { + return unattendedPrincipal({ + kind: 'system', + serviceId: 'webhook', + workspaceId: 'workspace-1', + workflowId: 'workflow-1', + webhookId: 'webhook-1', + provider: 'slack', + subject: { kind: 'external_user', provider: 'slack', tenantId: 'T123', subjectId: 'U123' }, + }) +} + +function invite(principal: WorkflowExecutionDelegatedPrincipal) { + return sendCredentialGroupInvite.execute({ + principal, + input: { credentialGroupId: 'group-1', email: 'person@example.com' }, + }) +} + +describe('sendCredentialGroupInvite', () => { + beforeEach(() => { + vi.clearAllMocks() + mocks.resolveGroup.mockResolvedValue(context) + mocks.resolvePermission.mockResolvedValue('admin') + mocks.requireAvailable.mockResolvedValue(undefined) + mocks.loadInviter.mockResolvedValue({ name: 'Ada Lovelace', email: 'ada@example.com' }) + mocks.inviteEnrollment.mockResolvedValue({ + id: 'enrollment-1', + email: 'person@example.com', + status: 'invited', + }) + }) + + it('invites without naming an inviter on a Slack-triggered run', async () => { + const result = await invite(slackPrincipal()) + + expect(result.enrollment.id).toBe('enrollment-1') + expect(mocks.loadInviter).not.toHaveBeenCalled() + expect(mocks.inviteEnrollment).toHaveBeenCalledWith( + 'workspace-1', + 'group-1', + undefined, + undefined, + 'person@example.com' + ) + }) + + it('invites without naming an inviter on an actorless run', async () => { + await invite( + unattendedPrincipal({ + kind: 'system', + serviceId: 'schedule', + workspaceId: 'workspace-1', + workflowId: 'workflow-1', + }) + ) + + expect(mocks.inviteEnrollment).toHaveBeenCalledWith( + 'workspace-1', + 'group-1', + undefined, + undefined, + 'person@example.com' + ) + }) + + it('names the human a session-actor run acts as', async () => { + await invite(executorPrincipal()) + + expect(mocks.loadInviter).toHaveBeenCalledWith('admin-1') + expect(mocks.inviteEnrollment).toHaveBeenCalledWith( + 'workspace-1', + 'group-1', + 'admin-1', + 'Ada Lovelace', + 'person@example.com' + ) + }) + + it('falls back to the inviter email when they have no name', async () => { + mocks.loadInviter.mockResolvedValue({ name: ' ', email: 'ada@example.com' }) + + await invite(executorPrincipal()) + + expect(mocks.inviteEnrollment).toHaveBeenCalledWith( + 'workspace-1', + 'group-1', + 'admin-1', + 'ada@example.com', + 'person@example.com' + ) + }) + + it('requires the current subject to remain a workspace admin', async () => { + mocks.resolvePermission.mockResolvedValue('write') + + await expect(invite(executorPrincipal())).rejects.toMatchObject({ code: 'forbidden' }) + expect(mocks.inviteEnrollment).not.toHaveBeenCalled() + }) + + it('rejects a delegation asserting a subject its run never had', async () => { + const spoofed = slackPrincipal() + spoofed.subjectUserId = 'invented-user' + + await expect(invite(spoofed)).rejects.toMatchObject({ code: 'forbidden' }) + expect(mocks.inviteEnrollment).not.toHaveBeenCalled() + }) +}) diff --git a/apps/sim/lib/credential-groups/application/send-invite.ts b/apps/sim/lib/credential-groups/application/send-invite.ts index 31ed72e9330..3e970b81675 100644 --- a/apps/sim/lib/credential-groups/application/send-invite.ts +++ b/apps/sim/lib/credential-groups/application/send-invite.ts @@ -1,10 +1,11 @@ import { AuditAction, AuditResourceType } from '@sim/audit' +import { resolvePrincipalSubjectUserId } from '@sim/auth/principal' import { isValidEmailSyntax, normalizeEmail } from '@sim/utils/string' import { defineAuthorizedWorkspaceUseCase } from '@/lib/core/application' import { OrchestrationError } from '@/lib/core/orchestration/types' import { credentialGroupDelegationPolicy, - requireCredentialGroupWorkflowSubject, + requireCredentialGroupWorkflowActor, } from '@/lib/credential-groups/application/authorization' import { requireCredentialGroupsAvailable, @@ -28,7 +29,7 @@ export const sendCredentialGroupInvite = defineAuthorizedWorkspaceUseCase({ resolveCredentialGroupContext(input.credentialGroupId), authorizationOptions: { delegation: credentialGroupDelegationPolicy }, authorizeResource({ principal }) { - requireCredentialGroupWorkflowSubject(principal) + requireCredentialGroupWorkflowActor(principal) }, execute: async ({ principal, input, context }) => { if (context.status !== 'active') { @@ -40,12 +41,11 @@ export const sendCredentialGroupInvite = defineAuthorizedWorkspaceUseCase({ } await requireCredentialGroupsAvailable(context.workspaceId) - const userId = requireCredentialGroupWorkflowSubject(principal) - const inviter = await loadCredentialGroupInviterIdentity(userId) + // Attribution, not authority. An actorless or Slack-triggered run names no + // inviter rather than borrowing its actor, so the email claims no one invited. + const userId = resolvePrincipalSubjectUserId(principal) + const inviter = userId ? await loadCredentialGroupInviterIdentity(userId) : null const inviterName = inviter?.name?.trim() || inviter?.email - if (!inviterName) { - throw new OrchestrationError('conflict', 'Inviting user has no display identity') - } try { const enrollment = await inviteCredentialGroupEnrollment( diff --git a/apps/sim/lib/credential-groups/enrollments.ts b/apps/sim/lib/credential-groups/enrollments.ts index 91e661c7238..f4579c77026 100644 --- a/apps/sim/lib/credential-groups/enrollments.ts +++ b/apps/sim/lib/credential-groups/enrollments.ts @@ -72,7 +72,8 @@ interface IssuedInvitation { } export interface PublicCredentialGroupEnrollment { - inviterName: string + /** Null when the invitation was issued by a workflow or a since-deleted user. */ + inviterName: string | null workspaceName: string credentialGroupName: string options: Array< @@ -93,6 +94,7 @@ export interface PublicCredentialGroupEnrollment { export interface CredentialGroupOAuthContext { enrollmentId: string credentialGroupId: string + credentialGroupName: string workspaceId: string workspaceName: string workspaceOwnerId: string @@ -110,6 +112,11 @@ export interface PublicCredentialGroupEnrollmentIdentity { invitationTokenHash: string } +export interface CredentialGroupEnrollmentCompletion { + completed: true + transitioned: boolean +} + /** Serializes OAuth grant persistence and administrative revocation for one enrollment. */ export async function lockCredentialGroupEnrollmentLifecycle( executor: DbOrTx, @@ -426,8 +433,10 @@ async function issueInvitation( async function sendInvitation( context: InvitationContext, - userId: string, - inviterName: string, + /** See {@link issueInvitation}: the issuer is attribution, never the authority. */ + userId: string | undefined, + /** Absent when a workflow issued the invitation — the copy drops the inviter. */ + inviterName: string | undefined, email: string, options: SendInvitationOptions ): Promise { @@ -658,8 +667,10 @@ export async function loadCredentialGroupInviterIdentity( export async function inviteCredentialGroupEnrollment( workspaceId: string, groupId: string, - userId: string, - inviterName: string, + /** See {@link issueInvitation}: the issuer is attribution, never the authority. */ + userId: string | undefined, + /** See {@link sendInvitation}: absent for a workflow-issued invitation. */ + inviterName: string | undefined, email: string ): Promise { const context = await getInvitationContext(workspaceId, groupId) @@ -789,7 +800,7 @@ async function buildPublicCredentialGroupEnrollment( ) return { - inviterName: row.inviterName ?? 'A workspace admin', + inviterName: row.inviterName, workspaceName: row.workspaceName, credentialGroupName: row.groupName, options: await Promise.all( @@ -849,12 +860,16 @@ export async function completeCredentialGroupEnrollment(token: string): Promise< invitationTokenHash: hashInvitationToken(token), }) if (!row) return null - return completeResolvedCredentialGroupEnrollment(row, identityForPublicEnrollmentRow(row)) + const result = await completeResolvedCredentialGroupEnrollment( + row, + identityForPublicEnrollmentRow(row) + ) + return result?.completed ?? null } export async function completeAuthorizedCredentialGroupEnrollment( identity: PublicCredentialGroupEnrollmentIdentity -): Promise { +): Promise { const row = await resolveAuthorizedPublicEnrollmentRow(identity) if (!row) return null return completeResolvedCredentialGroupEnrollment(row, identity) @@ -863,7 +878,7 @@ export async function completeAuthorizedCredentialGroupEnrollment( async function completeResolvedCredentialGroupEnrollment( row: NonNullable>>, identity: PublicCredentialGroupEnrollmentIdentity -): Promise { +): Promise { return db.transaction(async (tx) => { await lockCredentialGroupEnrollmentLifecycle(tx, row.enrollment.id) const now = new Date() @@ -901,9 +916,15 @@ async function completeResolvedCredentialGroupEnrollment( .for('update') if (!group || group.status !== 'active') return null + const transitioned = current.status !== 'completed' + const [completed] = await tx .update(credentialGroupEnrollment) - .set({ status: 'completed', completedAt: now, updatedAt: now }) + .set({ + status: 'completed', + ...(transitioned ? { completedAt: now } : {}), + updatedAt: now, + }) .where( and( eq(credentialGroupEnrollment.id, row.enrollment.id), @@ -912,7 +933,7 @@ async function completeResolvedCredentialGroupEnrollment( ) .returning({ id: credentialGroupEnrollment.id }) if (!completed) throw new Error('Credential group enrollment completion returned no row') - return true + return { completed: true, transitioned } }) } @@ -948,6 +969,7 @@ function credentialGroupOAuthContextFromRow( return { enrollmentId: row.enrollment.id, credentialGroupId: row.groupId, + credentialGroupName: row.groupName, workspaceId: row.workspaceId, workspaceName: row.workspaceName, workspaceOwnerId: row.workspaceOwnerId, diff --git a/apps/sim/lib/credential-groups/oauth.test.ts b/apps/sim/lib/credential-groups/oauth.test.ts index 798a7a951c2..d448f751314 100644 --- a/apps/sim/lib/credential-groups/oauth.test.ts +++ b/apps/sim/lib/credential-groups/oauth.test.ts @@ -46,6 +46,7 @@ const POLICY = { const CONTEXT = { enrollmentId: 'enrollment-1', credentialGroupId: 'group-1', + credentialGroupName: 'Credential Group', workspaceId: 'workspace-1', workspaceName: 'Workspace', workspaceOwnerId: 'owner-1', @@ -122,6 +123,46 @@ describe('credential group OAuth persistence', () => { expect(dbChainMockFns.insert).not.toHaveBeenCalled() }) + it('returns a created event result after inserting a first credential', async () => { + dbChainMockFns.limit.mockResolvedValueOnce([{ status: 'invited' }]) + queueTableRows(schemaMock.credentialGroup, [GROUP]) + queueTableRows(schemaMock.credential, []) + dbChainMockFns.returning + .mockResolvedValueOnce([{ id: 'credential-1' }]) + .mockResolvedValueOnce([{ id: CONTEXT.enrollmentId }]) + + const result = await completeCredentialGroupOAuth( + CONTEXT, + { + state: 'state-1', + provider: 'gmail', + nonceHash: 'nonce-hash', + enrollmentId: CONTEXT.enrollmentId, + credentialGroupId: CONTEXT.credentialGroupId, + optionId: CONTEXT.option.id, + authorizationAppId: POLICY.authorizationAppId, + scopeVersion: POLICY.scopeVersion, + requiredScopes: POLICY.requiredScopes, + redirectUri: 'https://sim.ai/api/auth/oauth2/callback/google-email', + codeVerifier: 'verifier', + invitationToken: 'invitation-token', + createdAt: Date.now(), + }, + 'authorization-code' + ) + + expect(result).toEqual({ + created: true, + credentialId: 'credential-1', + credentialGroupOptionId: 'option-1', + provider: 'gmail', + providerId: 'google-email', + displayName: 'person@example.com', + enrollmentStatus: 'in_progress', + }) + expect(dbChainMockFns.insert).toHaveBeenCalledWith(schemaMock.credential) + }) + it('preserves completed enrollment state when an account reconnects', async () => { dbChainMockFns.limit.mockResolvedValueOnce([{ status: 'completed' }]) queueTableRows(schemaMock.credentialGroup, [GROUP]) @@ -137,7 +178,7 @@ describe('credential group OAuth persistence', () => { .mockResolvedValueOnce([{ id: 'credential-1' }]) .mockResolvedValueOnce([{ id: CONTEXT.enrollmentId }]) - await completeCredentialGroupOAuth( + const result = await completeCredentialGroupOAuth( { ...CONTEXT, enrollmentStatus: 'completed' }, { state: 'state-1', @@ -162,6 +203,15 @@ describe('credential group OAuth persistence', () => { expect.objectContaining({ status: 'completed', updatedAt: expect.any(Date) }) ) expect(enrollmentUpdate).not.toHaveProperty('completedAt') + expect(result).toEqual({ + created: false, + credentialId: 'credential-1', + credentialGroupOptionId: 'option-1', + provider: 'gmail', + providerId: 'google-email', + displayName: 'person@example.com', + enrollmentStatus: 'completed', + }) }) it('rejects an exchanged grant when the group policy changed before persistence', async () => { diff --git a/apps/sim/lib/credential-groups/oauth.ts b/apps/sim/lib/credential-groups/oauth.ts index 0aea1addc2a..107ba18a767 100644 --- a/apps/sim/lib/credential-groups/oauth.ts +++ b/apps/sim/lib/credential-groups/oauth.ts @@ -21,6 +21,7 @@ import { } from '@/lib/credential-groups/provider-adapter' import { getCredentialGroupProviderAdapter } from '@/lib/credential-groups/provider-registry' import { + type CredentialGroupProvider, getCredentialGroupProviderService, isCredentialGroupProvider, } from '@/lib/credential-groups/providers' @@ -58,6 +59,16 @@ function getOptionAdapter(context: CredentialGroupOAuthContext): CredentialGroup return getCredentialGroupProviderAdapter(context.option.provider) } +export interface CredentialGroupOAuthCompletion { + created: boolean + credentialId: string + credentialGroupOptionId: string + provider: CredentialGroupProvider + providerId: string + displayName: string + enrollmentStatus: 'in_progress' | 'completed' +} + async function assertCurrentPolicy( context: CredentialGroupOAuthContext, adapter: CredentialGroupProviderAdapter, @@ -111,12 +122,12 @@ async function persistGrant( adapter: CredentialGroupProviderAdapter, policy: CredentialGroupProviderPolicy, grant: VerifiedCredentialGroupGrant -): Promise { +): Promise { if (grant.providerId !== policy.providerId) { throw new CredentialGroupOAuthError('Provider returned a credential for another app.', 502) } - await db.transaction(async (tx) => { + return db.transaction(async (tx) => { await lockCredentialGroupEnrollmentLifecycle(tx, context.enrollmentId) await tx.execute( sql`SELECT pg_advisory_xact_lock(hashtextextended(${`credential-group-oauth:${context.enrollmentId}:${context.option.id}`}, 0))` @@ -231,6 +242,7 @@ async function persistGrant( updatedAt: now, } + let credentialId: string if (existing) { const [updated] = await tx .update(credential) @@ -238,6 +250,7 @@ async function persistGrant( .where(eq(credential.id, existing.id)) .returning({ id: credential.id }) if (!updated) throw new Error('Managed OAuth credential update returned no row') + credentialId = updated.id } else { const [inserted] = await tx .insert(credential) @@ -249,12 +262,14 @@ async function persistGrant( }) .returning({ id: credential.id }) if (!inserted) throw new Error('Managed OAuth credential insert returned no row') + credentialId = inserted.id } + const enrollmentStatus = enrollment.status === 'completed' ? 'completed' : 'in_progress' const [updatedEnrollment] = await tx .update(credentialGroupEnrollment) .set({ - status: enrollment.status === 'completed' ? 'completed' : 'in_progress', + status: enrollmentStatus, ...(enrollment.status === 'completed' ? {} : { completedAt: null }), updatedAt: now, }) @@ -268,6 +283,15 @@ async function persistGrant( if (!updatedEnrollment) { throw new CredentialGroupInvitationUnavailableError() } + return { + created: !existing, + credentialId, + credentialGroupOptionId: context.option.id, + provider: adapter.provider, + providerId: policy.providerId, + displayName: grant.displayName, + enrollmentStatus, + } }) } @@ -276,7 +300,7 @@ export async function completeCredentialGroupOAuth( context: CredentialGroupOAuthContext, attempt: CredentialGroupOAuthAttempt, code: string -): Promise { +): Promise { if ( attempt.enrollmentId !== context.enrollmentId || attempt.credentialGroupId !== context.credentialGroupId || @@ -288,5 +312,5 @@ export async function completeCredentialGroupOAuth( const adapter = getOptionAdapter(context) const policy = await assertCurrentPolicy(context, adapter, attempt) const grant = await adapter.exchangeAndVerify({ context, attempt, code, policy }) - await persistGrant(context, adapter, policy, grant) + return persistGrant(context, adapter, policy, grant) } diff --git a/apps/sim/lib/credential-groups/standard-oauth-provider.test.ts b/apps/sim/lib/credential-groups/standard-oauth-provider.test.ts index f1c568f6d3a..0e40b14acc1 100644 --- a/apps/sim/lib/credential-groups/standard-oauth-provider.test.ts +++ b/apps/sim/lib/credential-groups/standard-oauth-provider.test.ts @@ -87,6 +87,7 @@ function buildContext(): CredentialGroupOAuthContext { return { enrollmentId: 'enrollment-1', credentialGroupId: 'group-1', + credentialGroupName: 'Credential Group', workspaceId: 'workspace-1', workspaceName: 'Workspace', workspaceOwnerId: 'owner-1', diff --git a/apps/sim/lib/credential-groups/standard-oauth-provider.ts b/apps/sim/lib/credential-groups/standard-oauth-provider.ts index c9e7c85c2e7..a87fdae1ed3 100644 --- a/apps/sim/lib/credential-groups/standard-oauth-provider.ts +++ b/apps/sim/lib/credential-groups/standard-oauth-provider.ts @@ -1,11 +1,11 @@ import { randomBytes } from 'node:crypto' +import { normalizeEmail } from '@sim/utils/string' import { applyDefaultAccessTokenExpiry, createAuthorizationURL, type OAuth2Tokens, validateAuthorizationCode, -} from '@better-auth/core/oauth2' -import { normalizeEmail } from '@sim/utils/string' +} from 'better-auth/oauth2' import { type ConnectorProviderConfig, getManagedOAuthConnectorProviderConfig, diff --git a/apps/sim/lib/credential-groups/trigger-constants.ts b/apps/sim/lib/credential-groups/trigger-constants.ts new file mode 100644 index 00000000000..446b2b878b0 --- /dev/null +++ b/apps/sim/lib/credential-groups/trigger-constants.ts @@ -0,0 +1,16 @@ +export const CREDENTIAL_GROUP_TRIGGER_PROVIDER = 'credential-group' + +export const CREDENTIAL_GROUP_EVENT_TRIGGER_ID = 'credential_group_event' + +export const CREDENTIAL_GROUP_TRIGGER_EVENT_TYPES = [ + 'credential_added', + 'credential_reconnected', + 'form_submitted', +] as const + +export type CredentialGroupTriggerEventType = (typeof CREDENTIAL_GROUP_TRIGGER_EVENT_TYPES)[number] + +export const CREDENTIAL_GROUP_CREDENTIAL_EVENT_TYPES = [ + 'credential_added', + 'credential_reconnected', +] as const satisfies readonly CredentialGroupTriggerEventType[] diff --git a/apps/sim/lib/credential-groups/trigger-subscriptions.ts b/apps/sim/lib/credential-groups/trigger-subscriptions.ts new file mode 100644 index 00000000000..9c27581f6ac --- /dev/null +++ b/apps/sim/lib/credential-groups/trigger-subscriptions.ts @@ -0,0 +1,44 @@ +import { db } from '@sim/db' +import { webhook, workflow, workflowDeploymentVersion } from '@sim/db/schema' +import { and, eq, inArray, isNull, or } from 'drizzle-orm' +import { CREDENTIAL_GROUP_TRIGGER_PROVIDER } from '@/lib/credential-groups/trigger-constants' +import { deliverableWebhookPredicate } from '@/lib/webhooks/delivery-predicate' +import type { WebhookRecord, WorkflowRecord } from '@/lib/webhooks/polling/types' + +export interface CredentialGroupTriggerSubscription { + webhook: WebhookRecord + workflow: WorkflowRecord +} + +/** Loads only deployed subscriptions in the source workspace that may read this group. */ +export async function fetchCredentialGroupTriggerSubscriptions( + workspaceId: string, + allowedWorkflowIds: string[] +): Promise { + if (allowedWorkflowIds.length === 0) return [] + return db + .select({ webhook, workflow }) + .from(webhook) + .innerJoin(workflow, eq(webhook.workflowId, workflow.id)) + .leftJoin( + workflowDeploymentVersion, + and( + eq(workflowDeploymentVersion.workflowId, workflow.id), + eq(workflowDeploymentVersion.isActive, true) + ) + ) + .where( + and( + eq(webhook.provider, CREDENTIAL_GROUP_TRIGGER_PROVIDER), + deliverableWebhookPredicate(webhook), + eq(workflow.workspaceId, workspaceId), + inArray(workflow.id, allowedWorkflowIds), + eq(workflow.isDeployed, true), + isNull(workflow.archivedAt), + or( + eq(webhook.deploymentVersionId, workflowDeploymentVersion.id), + and(isNull(workflowDeploymentVersion.id), isNull(webhook.deploymentVersionId)) + ) + ) + ) +} diff --git a/apps/sim/lib/credential-groups/trigger.test.ts b/apps/sim/lib/credential-groups/trigger.test.ts new file mode 100644 index 00000000000..e4ef96731de --- /dev/null +++ b/apps/sim/lib/credential-groups/trigger.test.ts @@ -0,0 +1,141 @@ +/** + * @vitest-environment node + */ +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const mocks = vi.hoisted(() => ({ + decodePolicy: vi.fn(), + fetchSubscriptions: vi.fn(), + processEvent: vi.fn(), + requirePolicy: vi.fn(), +})) + +vi.mock('@/lib/credential-groups/application/workflow-access-policy', () => ({ + credentialGroupWorkflowAccessPolicyCodec: { + resourceType: 'credential_group', + parse: (value: unknown) => value, + }, + decodeCredentialGroupWorkflowAccessPolicy: mocks.decodePolicy, +})) + +vi.mock('@/lib/resource-policies/repository', () => ({ + requireResourcePolicy: mocks.requirePolicy, +})) + +vi.mock('@/lib/credential-groups/trigger-subscriptions', () => ({ + fetchCredentialGroupTriggerSubscriptions: mocks.fetchSubscriptions, +})) + +vi.mock('@/lib/webhooks/processor', () => ({ + processPolledWebhookEvent: mocks.processEvent, +})) + +import { + buildCredentialGroupTriggerPayload, + fireCredentialGroupTrigger, +} from '@/lib/credential-groups/trigger' + +const EVENT = { + event: 'credential_added' as const, + workspaceId: 'workspace-1', + credentialGroupId: 'group-1', + credentialGroupName: 'Credential Group', + enrollmentId: 'enrollment-1', + email: 'person@example.com', + enrollmentStatus: 'in_progress' as const, + credential: { + credentialId: 'credential-1', + credentialGroupOptionId: 'option-1', + provider: 'gmail' as const, + providerId: 'google-email', + displayName: 'person@example.com', + }, +} + +function subscription(params: { + workflowId: string + workspaceId?: string + eventType?: string + credentialGroupId?: string +}) { + return { + webhook: { + id: `webhook-${params.workflowId}`, + providerConfig: { + triggerId: 'credential_group_event', + credentialGroupId: params.credentialGroupId ?? 'group-1', + eventType: params.eventType ?? 'credential_added', + }, + }, + workflow: { + id: params.workflowId, + workspaceId: params.workspaceId ?? 'workspace-1', + }, + } +} + +describe('Credential Group trigger delivery', () => { + beforeEach(() => { + vi.clearAllMocks() + mocks.requirePolicy.mockResolvedValue({ document: {} }) + mocks.decodePolicy.mockReturnValue(['workflow-allowed']) + mocks.processEvent.mockResolvedValue({ success: true }) + }) + + it('delivers only to an allowed workflow watching the exact group and event', async () => { + const allowed = subscription({ workflowId: 'workflow-allowed' }) + mocks.fetchSubscriptions.mockResolvedValue([ + allowed, + subscription({ workflowId: 'workflow-denied' }), + subscription({ workflowId: 'workflow-allowed', eventType: 'form_submitted' }), + subscription({ workflowId: 'workflow-allowed', credentialGroupId: 'group-2' }), + subscription({ workflowId: 'workflow-allowed', workspaceId: 'workspace-2' }), + ]) + + await fireCredentialGroupTrigger(EVENT) + + expect(mocks.processEvent).toHaveBeenCalledOnce() + expect(mocks.processEvent).toHaveBeenCalledWith( + allowed.webhook, + allowed.workflow, + expect.objectContaining({ + event: 'credential_added', + credentialGroupId: 'group-1', + credentialId: 'credential-1', + }), + expect.any(String) + ) + }) + + it('does not scan subscriptions when no workflow has group access', async () => { + mocks.decodePolicy.mockReturnValue([]) + + await fireCredentialGroupTrigger(EVENT) + + expect(mocks.fetchSubscriptions).not.toHaveBeenCalled() + expect(mocks.processEvent).not.toHaveBeenCalled() + }) + + it('uses null credential fields for form submissions', () => { + expect( + buildCredentialGroupTriggerPayload({ + event: 'form_submitted', + workspaceId: 'workspace-1', + credentialGroupId: 'group-1', + credentialGroupName: 'Credential Group', + enrollmentId: 'enrollment-1', + email: 'person@example.com', + enrollmentStatus: 'completed', + }) + ).toEqual( + expect.objectContaining({ + event: 'form_submitted', + credentialId: null, + credentialGroupOptionId: null, + provider: null, + providerId: null, + displayName: null, + }) + ) + }) +}) diff --git a/apps/sim/lib/credential-groups/trigger.ts b/apps/sim/lib/credential-groups/trigger.ts new file mode 100644 index 00000000000..31504ed1526 --- /dev/null +++ b/apps/sim/lib/credential-groups/trigger.ts @@ -0,0 +1,167 @@ +import { createLogger } from '@sim/logger' +import { generateShortId } from '@sim/utils/id' +import { isRecordLike } from '@sim/utils/object' +import { + credentialGroupWorkflowAccessPolicyCodec, + decodeCredentialGroupWorkflowAccessPolicy, +} from '@/lib/credential-groups/application/workflow-access-policy' +import type { CredentialGroupProvider } from '@/lib/credential-groups/providers' +import { + CREDENTIAL_GROUP_EVENT_TRIGGER_ID, + CREDENTIAL_GROUP_TRIGGER_EVENT_TYPES, + type CredentialGroupTriggerEventType, +} from '@/lib/credential-groups/trigger-constants' +import { fetchCredentialGroupTriggerSubscriptions } from '@/lib/credential-groups/trigger-subscriptions' +import { requireResourcePolicy } from '@/lib/resource-policies/repository' + +const logger = createLogger('CredentialGroupTrigger') + +interface CredentialGroupTriggerEventBase { + workspaceId: string + credentialGroupId: string + credentialGroupName: string + enrollmentId: string + email: string + enrollmentStatus: 'in_progress' | 'completed' +} + +interface CredentialGroupTriggerCredential { + credentialId: string + credentialGroupOptionId: string + provider: CredentialGroupProvider + providerId: string + displayName: string +} + +export type CredentialGroupTriggerEvent = + | (CredentialGroupTriggerEventBase & { + event: 'credential_added' | 'credential_reconnected' + credential: CredentialGroupTriggerCredential + }) + | (CredentialGroupTriggerEventBase & { + event: 'form_submitted' + credential?: never + }) + +export interface CredentialGroupTriggerPayload { + event: CredentialGroupTriggerEventType + timestamp: string + credentialGroupId: string + credentialGroupName: string + enrollmentId: string + email: string + enrollmentStatus: 'in_progress' | 'completed' + credentialId: string | null + credentialGroupOptionId: string | null + provider: CredentialGroupProvider | null + providerId: string | null + displayName: string | null +} + +interface CredentialGroupTriggerConfig { + triggerId: typeof CREDENTIAL_GROUP_EVENT_TRIGGER_ID + credentialGroupId: string + eventType: CredentialGroupTriggerEventType +} + +function parseCredentialGroupTriggerConfig(value: unknown): CredentialGroupTriggerConfig { + if (!isRecordLike(value)) throw new Error('Credential Group trigger config must be an object') + if (value.triggerId !== CREDENTIAL_GROUP_EVENT_TRIGGER_ID) { + throw new Error('Credential Group trigger ID is invalid') + } + if ( + typeof value.credentialGroupId !== 'string' || + !value.credentialGroupId.trim() || + value.credentialGroupId !== value.credentialGroupId.trim() + ) { + throw new Error('Credential Group trigger requires a canonical Credential Group ID') + } + if ( + typeof value.eventType !== 'string' || + !(CREDENTIAL_GROUP_TRIGGER_EVENT_TYPES as readonly string[]).includes(value.eventType) + ) { + throw new Error('Credential Group trigger event type is invalid') + } + return { + triggerId: CREDENTIAL_GROUP_EVENT_TRIGGER_ID, + credentialGroupId: value.credentialGroupId, + eventType: value.eventType as CredentialGroupTriggerEventType, + } +} + +export function buildCredentialGroupTriggerPayload( + event: CredentialGroupTriggerEvent +): CredentialGroupTriggerPayload { + const credential = event.event === 'form_submitted' ? null : event.credential + return { + event: event.event, + timestamp: new Date().toISOString(), + credentialGroupId: event.credentialGroupId, + credentialGroupName: event.credentialGroupName, + enrollmentId: event.enrollmentId, + email: event.email, + enrollmentStatus: event.enrollmentStatus, + credentialId: credential?.credentialId ?? null, + credentialGroupOptionId: credential?.credentialGroupOptionId ?? null, + provider: credential?.provider ?? null, + providerId: credential?.providerId ?? null, + displayName: credential?.displayName ?? null, + } +} + +/** + * Fires deployed Credential Group triggers after the source mutation commits. + * Delivery is restricted to workflows explicitly allowed by the group's resource policy. + */ +export async function fireCredentialGroupTrigger( + event: CredentialGroupTriggerEvent +): Promise { + try { + const policy = await requireResourcePolicy({ + workspaceId: event.workspaceId, + resourceType: 'credential_group', + resourceId: event.credentialGroupId, + codec: credentialGroupWorkflowAccessPolicyCodec, + }) + const allowedWorkflowIds = new Set( + decodeCredentialGroupWorkflowAccessPolicy(policy.document, event.credentialGroupId) + ) + if (allowedWorkflowIds.size === 0) return + + const subscriptions = await fetchCredentialGroupTriggerSubscriptions(event.workspaceId, [ + ...allowedWorkflowIds, + ]) + const matchingSubscriptions = subscriptions.filter(({ webhook, workflow }) => { + if (workflow.workspaceId !== event.workspaceId) return false + if (!allowedWorkflowIds.has(workflow.id)) return false + const config = parseCredentialGroupTriggerConfig(webhook.providerConfig) + return ( + config.credentialGroupId === event.credentialGroupId && config.eventType === event.event + ) + }) + if (matchingSubscriptions.length === 0) return + + const payload = buildCredentialGroupTriggerPayload(event) + const { processPolledWebhookEvent } = await import('@/lib/webhooks/processor') + for (const { webhook, workflow } of matchingSubscriptions) { + const requestId = generateShortId() + const result = await processPolledWebhookEvent(webhook, workflow, payload, requestId) + if (!result.success) { + logger.error(`[${requestId}] Failed to fire Credential Group trigger`, { + event: event.event, + credentialGroupId: event.credentialGroupId, + subscriberWorkflowId: workflow.id, + statusCode: result.statusCode, + error: result.error, + }) + } + } + } catch (error) { + logger.error('Failed to emit Credential Group event', { + error, + event: event.event, + credentialGroupId: event.credentialGroupId, + enrollmentId: event.enrollmentId, + }) + } +} diff --git a/apps/sim/lib/credentials/application/authorized-credential-use-case.test.ts b/apps/sim/lib/credentials/application/authorized-credential-use-case.test.ts index 65a3b7a9133..ea01ec344ed 100644 --- a/apps/sim/lib/credentials/application/authorized-credential-use-case.test.ts +++ b/apps/sim/lib/credentials/application/authorized-credential-use-case.test.ts @@ -31,6 +31,7 @@ const memberOperation = defineCredentialOperation( minimumRole: 'read', workspaceApiKey: 'deny', principalKinds: ['session'], + capability: 'integrations.manage', }), 'member' ) @@ -40,6 +41,7 @@ const adminOperation = defineCredentialOperation( minimumRole: 'read', workspaceApiKey: 'deny', principalKinds: ['session'], + capability: 'integrations.manage', }), 'admin' ) diff --git a/apps/sim/lib/credentials/application/authorized-user-use-case.ts b/apps/sim/lib/credentials/application/authorized-user-use-case.ts index 95fd6e08712..77010182ee9 100644 --- a/apps/sim/lib/credentials/application/authorized-user-use-case.ts +++ b/apps/sim/lib/credentials/application/authorized-user-use-case.ts @@ -1,9 +1,12 @@ import { type AuditActionType, type AuditResourceTypeValue, recordAudit } from '@sim/audit' import { resolvePrincipalAuditAttribution, type SessionPrincipal } from '@sim/auth/principal' +import { getUserOrganization } from '@/lib/billing/organizations/membership' import type { OperationUseCase } from '@/lib/core/application' import type { OrchestrationRequestContext } from '@/lib/core/orchestration/types' import { OrchestrationError } from '@/lib/core/orchestration/types' import type { CredentialUserOperation } from '@/lib/credentials/application/operations' +import { refuseCapability } from '@/lib/permission-groups/capabilities' +import { isOrganizationCapabilityWithheld } from '@/lib/permission-groups/capability-assertions' export interface CredentialUserAuditEntry { workspaceId: string | null @@ -65,6 +68,38 @@ function recordCredentialUserAudit( } } +/** + * Refuses when the group governing the acting user withholds the operation's + * capability. + * + * permission-group-enforced: integrations.manage — these operations have no + * workspace, so `authorizeWorkspaceOperation` never sees them and the capability + * is applied here instead, at the one place every current-user credential + * operation passes through. + * + * The user's own OAuth connections belong to no workspace, so this resolves the + * organization's default group — the same resolution personal API keys and + * invitations use for an organization-level action. A no-op when the user is in + * no organization or no group governs them, which is the personal-workspace and + * non-enterprise case. + * + * Runs after the session-principal check above, never before: the principal kind + * is this operation's whole access story, and answering the capability question + * first would tell a caller who is not a session about the organization's + * configuration. + */ +async function assertCurrentUserCapability( + userId: string, + operation: CredentialUserOperation +): Promise { + if (operation.capability === 'none') return + const membership = await getUserOrganization(userId) + if (!membership?.organizationId) return + if (await isOrganizationCapabilityWithheld(membership.organizationId, operation.capability)) { + refuseCapability(operation.capability) + } +} + /** Defines a current-user credential operation that cannot borrow workspace identity. */ export function defineAuthorizedCredentialUserUseCase< const O extends CredentialUserOperation, @@ -77,6 +112,7 @@ export function defineAuthorizedCredentialUserUseCase< if (principal.kind !== 'session') { throw new OrchestrationError('forbidden', 'Session authentication required') } + await assertCurrentUserCapability(principal.userId, definition.operation) try { const result = await definition.execute({ principal, input, request }) recordCredentialUserAudit( diff --git a/apps/sim/lib/credentials/application/capability-gate.test.ts b/apps/sim/lib/credentials/application/capability-gate.test.ts new file mode 100644 index 00000000000..247a3ae1260 --- /dev/null +++ b/apps/sim/lib/credentials/application/capability-gate.test.ts @@ -0,0 +1,192 @@ +/** + * @vitest-environment node + * + * The current-user credential operations govern listing and disconnecting a + * user's OAuth connections. They are minted by `defineCredentialUserOperation`, + * which does not call `defineWorkspaceOperation`, so `authorizeWorkspaceOperation` + * never sees them and they shipped with no capability at all — a member whose + * group revokes Integrations could still enumerate and disconnect every + * connection. These pin the gate through the real routes, so the refusal the + * caller actually receives is what is asserted. + * + * They have no workspace, so the gate resolves the organization's default group + * — the same resolution personal API keys use for an organization-level action. + */ +import { authMockFns, createMockRequest } from '@sim/testing' +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const { + mockGetUserOrganization, + mockGetOrgPermissionConfig, + mockListOAuthConnectionsForUser, + mockListConnectedAccountsForUser, + mockDisconnectOAuthAccounts, +} = vi.hoisted(() => ({ + mockGetUserOrganization: vi.fn(), + mockGetOrgPermissionConfig: vi.fn(), + mockListOAuthConnectionsForUser: vi.fn(), + mockListConnectedAccountsForUser: vi.fn(), + mockDisconnectOAuthAccounts: vi.fn(), +})) + +vi.mock('@/lib/billing/organizations/membership', () => ({ + getUserOrganization: mockGetUserOrganization, +})) + +vi.mock('@/lib/permission-groups/resolve.server', () => ({ + getUserPermissionConfig: vi.fn(), + getUserPermissionConfigForOrganization: mockGetOrgPermissionConfig, + resolveVerifiedUserAccessControlContext: vi.fn(), +})) + +vi.mock('@/lib/credentials/oauth-accounts', () => ({ + listOAuthConnectionsForUser: mockListOAuthConnectionsForUser, + listConnectedAccountsForUser: mockListConnectedAccountsForUser, + disconnectOAuthAccounts: mockDisconnectOAuthAccounts, + OAuthDisconnectPartialFailureError: class OAuthDisconnectPartialFailureError extends Error { + credentials: unknown[] = [] + }, +})) + +import { capabilityRefusal } from '@/lib/permission-groups/capability-assertions' +import { DEFAULT_PERMISSION_GROUP_CONFIG } from '@/lib/permission-groups/fields' +import { GET as listConnectedAccounts } from '@/app/api/auth/accounts/route' +import { GET as listConnections } from '@/app/api/auth/oauth/connections/route' +import { POST as disconnect } from '@/app/api/auth/oauth/disconnect/route' + +const USER_ID = 'user-1' +const ORGANIZATION_ID = 'org-1' + +const mockGetSession = authMockFns.mockGetSession + +function callListConnections() { + return listConnections(createMockRequest('GET'), { params: Promise.resolve({}) }) +} + +function callListConnectedAccounts() { + return listConnectedAccounts( + createMockRequest('GET', undefined, {}, 'http://localhost/api/auth/accounts'), + { params: Promise.resolve({}) } + ) +} + +function callDisconnect() { + return disconnect(createMockRequest('POST', { provider: 'google' }), { + params: Promise.resolve({}), + }) +} + +const INTEGRATIONS_REFUSAL = capabilityRefusal('integrations.manage') + +describe('integrations.manage gate on the current-user credential operations', () => { + beforeEach(() => { + vi.clearAllMocks() + mockGetSession.mockResolvedValue({ + user: { id: USER_ID }, + session: { id: 'session-1' }, + }) + mockGetUserOrganization.mockResolvedValue({ + organizationId: ORGANIZATION_ID, + role: 'member', + memberId: 'member-1', + }) + mockGetOrgPermissionConfig.mockResolvedValue(null) + mockListOAuthConnectionsForUser.mockResolvedValue([]) + mockListConnectedAccountsForUser.mockResolvedValue([]) + mockDisconnectOAuthAccounts.mockResolvedValue({ + credentials: [], + provider: 'google', + providerId: undefined, + }) + }) + + describe('when the group withholds Integrations', () => { + beforeEach(() => { + mockGetOrgPermissionConfig.mockResolvedValue({ + ...DEFAULT_PERMISSION_GROUP_CONFIG, + hideIntegrationsTab: true, + }) + }) + + it('refuses to enumerate the OAuth connections, and never reads them', async () => { + const response = await callListConnections() + + expect(response.status).toBe(403) + expect(await response.json()).toEqual({ error: INTEGRATIONS_REFUSAL }) + expect(mockListOAuthConnectionsForUser).not.toHaveBeenCalled() + }) + + it('refuses to list the connected accounts, and never reads them', async () => { + const response = await callListConnectedAccounts() + + expect(response.status).toBe(403) + expect(await response.json()).toEqual({ error: INTEGRATIONS_REFUSAL }) + expect(mockListConnectedAccountsForUser).not.toHaveBeenCalled() + }) + + it('refuses the disconnect, and never deletes a credential', async () => { + const response = await callDisconnect() + + expect(response.status).toBe(403) + expect(await response.json()).toEqual({ error: INTEGRATIONS_REFUSAL }) + expect(mockDisconnectOAuthAccounts).not.toHaveBeenCalled() + }) + + /** + * Concealment: authentication still runs first, so an unauthenticated + * caller is told to authenticate rather than told how someone else's + * organization is configured. + */ + it('still answers an unauthenticated caller with 401, not the capability', async () => { + mockGetSession.mockResolvedValue(null) + + const response = await callListConnections() + + expect(response.status).toBe(401) + expect(mockGetOrgPermissionConfig).not.toHaveBeenCalled() + }) + }) + + describe('when no group governs the caller', () => { + it('lists the OAuth connections', async () => { + const response = await callListConnections() + + expect(response.status).toBe(200) + expect(mockListOAuthConnectionsForUser).toHaveBeenCalledWith(USER_ID) + }) + + it('disconnects', async () => { + const response = await callDisconnect() + + expect(response.status).toBe(200) + expect(mockDisconnectOAuthAccounts).toHaveBeenCalledTimes(1) + }) + + /** + * The personal-workspace case: a user in no organization has no group to + * resolve, so the gate is a no-op and never asks. + */ + it('does not even resolve a group for a user in no organization', async () => { + mockGetUserOrganization.mockResolvedValue(null) + + const response = await callListConnections() + + expect(response.status).toBe(200) + expect(mockGetOrgPermissionConfig).not.toHaveBeenCalled() + }) + }) + + describe('when a group governs the caller but permits Integrations', () => { + it('lets the disconnect through', async () => { + mockGetOrgPermissionConfig.mockResolvedValue({ + ...DEFAULT_PERMISSION_GROUP_CONFIG, + hideSecretsTab: true, + }) + + const response = await callDisconnect() + + expect(response.status).toBe(200) + expect(mockDisconnectOAuthAccounts).toHaveBeenCalledTimes(1) + }) + }) +}) diff --git a/apps/sim/lib/credentials/application/connection-target.test.ts b/apps/sim/lib/credentials/application/connection-target.test.ts index de2a9cd71c9..fc6cef25692 100644 --- a/apps/sim/lib/credentials/application/connection-target.test.ts +++ b/apps/sim/lib/credentials/application/connection-target.test.ts @@ -7,6 +7,11 @@ const mocks = vi.hoisted(() => ({ listCatalog: vi.fn(), getWorkspaceCredential: vi.fn(), getCredentialActorContext: vi.fn(), + assertWorkspaceCapability: vi.fn(), +})) + +vi.mock('@/lib/permission-groups/capability-assertions', () => ({ + assertWorkspaceCapability: mocks.assertWorkspaceCapability, })) vi.mock('@/lib/credentials/application/provider-catalog', () => ({ @@ -85,6 +90,28 @@ describe('resolveCredentialConnectionTarget', () => { mocks.listCatalog.mockResolvedValue([salesforceProvider]) mocks.getWorkspaceCredential.mockResolvedValue(credential) mocks.getCredentialActorContext.mockResolvedValue({ credential, isAdmin: true }) + mocks.assertWorkspaceCapability.mockResolvedValue(undefined) + }) + + /** + * `disablePersonalCredentials` leaves members "only workspace-shared ones", so + * it withholds connecting an account and not re-authorizing a credential the + * workspace already holds. Declaring it on the operation refused both. + */ + it('asserts the personal-credential capability only when connecting an account', async () => { + await resolveCredentialConnectionTarget({ principal, context, providerId: 'salesforce' }) + + expect(mocks.assertWorkspaceCapability).toHaveBeenCalledWith( + 'user-1', + 'workspace-1', + 'credentials.personal', + null + ) + + mocks.assertWorkspaceCapability.mockClear() + await resolveCredentialConnectionTarget({ principal, context, credentialId: 'credential-1' }) + + expect(mocks.assertWorkspaceCapability).not.toHaveBeenCalled() }) it('accepts an exact authorization option for a new connection', async () => { diff --git a/apps/sim/lib/credentials/application/connection-target.ts b/apps/sim/lib/credentials/application/connection-target.ts index 6c98ac540ce..c30d685d38a 100644 --- a/apps/sim/lib/credentials/application/connection-target.ts +++ b/apps/sim/lib/credentials/application/connection-target.ts @@ -1,4 +1,5 @@ import type { Principal } from '@sim/auth/principal' +import { capabilityGovernedPrincipalUserId } from '@/lib/core/application' import { ForbiddenOperationError } from '@/lib/core/application/forbidden' import { OrchestrationError } from '@/lib/core/orchestration/types' import { getCredentialActorContext } from '@/lib/credentials/access' @@ -10,6 +11,7 @@ import { } from '@/lib/credentials/application/provider-catalog' import { getWorkspaceCredential } from '@/lib/credentials/queries' import { credentialProviderMatchesService } from '@/lib/oauth/utils' +import { assertWorkspaceCapability } from '@/lib/permission-groups/capability-assertions' import type { ActiveWorkspaceApplicationContext } from '@/lib/workspaces/application/workspace-context' export interface ResolvedCredentialConnectionTarget { @@ -40,6 +42,28 @@ export async function resolveCredentialConnectionTarget(params: { const catalog = await listCredentialProviderCatalog(principal, context) if (providerId) { + /** + * permission-group-enforced: credentials.personal — scope is the request's + * target, not a property of the operation, exactly as it is for + * `credentials.create`. + * + * Only this branch connects a personal account. Reconnecting re-authorizes a + * credential the workspace already holds, which is the very thing + * `disablePersonalCredentials` leaves members ("leaving only workspace-shared + * ones"), so gating the reconnect on it would withhold the credentials that + * setting mandates. The operations declare `integrations.manage`, which + * governs both branches; this narrower one is asserted where the act + * actually is personal. + */ + const governedUserId = capabilityGovernedPrincipalUserId(principal) + if (governedUserId) { + await assertWorkspaceCapability( + governedUserId, + context.workspaceId, + 'credentials.personal', + context.workspaceOrganizationId + ) + } return { provider: requireAvailableOAuthCredentialProvider(catalog, providerId), providerId, diff --git a/apps/sim/lib/credentials/application/create-credential-connection.test.ts b/apps/sim/lib/credentials/application/create-credential-connection.test.ts index c89fc68718c..1f095959a97 100644 --- a/apps/sim/lib/credentials/application/create-credential-connection.test.ts +++ b/apps/sim/lib/credentials/application/create-credential-connection.test.ts @@ -31,6 +31,7 @@ vi.mock('@/lib/credentials/connect-draft', () => ({ vi.mock('@/lib/core/utils/urls', () => ({ getBaseUrl: mocks.getBaseUrl, + SITE_URL: 'http://localhost:3000', })) import { createCredentialConnection } from '@/lib/credentials/application/create-credential-connection' diff --git a/apps/sim/lib/credentials/application/credential-crud.test.ts b/apps/sim/lib/credentials/application/credential-crud.test.ts index 83e98bee4f7..c90980159b9 100644 --- a/apps/sim/lib/credentials/application/credential-crud.test.ts +++ b/apps/sim/lib/credentials/application/credential-crud.test.ts @@ -1,7 +1,12 @@ /** * @vitest-environment node */ -import { auditMock, auditMockFns } from '@sim/testing' +import { + auditMock, + auditMockFns, + permissionGroupScopeMock, + permissionGroupScopeMockFns, +} from '@sim/testing' import { beforeEach, describe, expect, it, vi } from 'vitest' const mocks = vi.hoisted(() => ({ @@ -11,8 +16,11 @@ const mocks = vi.hoisted(() => ({ getCredentialById: vi.fn(), getActor: vi.fn(), updateRecord: vi.fn(), + createRecord: vi.fn(), })) +const resolveGroupConfigMock = permissionGroupScopeMockFns.mockResolvePermissionGroupConfig + vi.mock('@sim/audit', () => auditMock) vi.mock('@/lib/workspaces/application/workspace-context', () => ({ loadActiveWorkspaceApplicationContext: mocks.loadWorkspace, @@ -32,17 +40,22 @@ vi.mock('@/lib/credentials/access', () => ({ })) vi.mock('@/lib/credentials/orchestration', () => ({ updateCredentialRecord: mocks.updateRecord, - createCredentialRecord: vi.fn(), + createCredentialRecord: mocks.createRecord, isProviderOutageCode: () => false, })) +vi.mock('@/lib/permission-groups/config-scope.server', () => permissionGroupScopeMock) vi.mock('@/lib/credentials/oauth', () => ({ syncWorkspaceOAuthCredentialsForUser: vi.fn() })) vi.mock('@/lib/posthog/server', () => ({ captureServerEvent: vi.fn() })) vi.mock('@/lib/workspaces/permissions/utils', () => ({ checkWorkspaceAccess: vi.fn() })) +import { PermissionGroupCapabilityError } from '@/lib/core/application' import { CredentialProviderOperationError, + createWorkspaceCredential, updateWorkspaceCredentialUseCase, } from '@/lib/credentials/application/credential-crud' +import { credentialOperations } from '@/lib/credentials/application/operations' +import { DEFAULT_PERMISSION_GROUP_CONFIG } from '@/lib/permission-groups/fields' const WORKSPACE_ID = 'workspace-1' const OTHER_WORKSPACE_ID = 'workspace-2' @@ -299,3 +312,104 @@ describe('updateWorkspaceCredentialUseCase', () => { expect(error.code).toBe('validation') }) }) + +describe('personal-credential capability', () => { + const ORGANIZATION_ID = 'organization-1' + const governedWorkspace = { ...workspace, workspaceOrganizationId: ORGANIZATION_ID } + + function createdCredential(type: 'env_personal' | 'env_workspace') { + return { ...credential, type, envKey: 'OPENAI_API_KEY', encryptedServiceAccountKey: null } + } + + beforeEach(() => { + vi.clearAllMocks() + mocks.loadWorkspace.mockResolvedValue(governedWorkspace) + mocks.resolvePermission.mockResolvedValue('admin') + resolveGroupConfigMock.mockResolvedValue({ + ...DEFAULT_PERMISSION_GROUP_CONFIG, + disablePersonalCredentials: true, + }) + }) + + /** + * A connection operation takes a target, not a scope: the same operation + * connects an account and re-authorizes a workspace-shared credential. + * `credentials.personal` belongs to the first branch only and is asserted + * there; the operation carries the capability that governs both. Pinned + * because declaring the narrower one here compiles just as well, and it + * refused the shared credentials that setting exists to mandate. + */ + it.each(['createConnection', 'prepareConnection', 'launchConnection'] as const)( + 'declares the capability on %s that governs both of its targets', + (operationName) => { + expect(credentialOperations[operationName].capability).toBe('integrations.manage') + } + ) + + it('refuses a personal environment secret before it reaches the manager', async () => { + await expect( + createWorkspaceCredential.execute({ + principal: sessionPrincipal, + input: { + workspaceId: WORKSPACE_ID, + type: 'env_personal', + displayName: 'My OpenAI key', + envKey: 'OPENAI_API_KEY', + }, + }) + ).rejects.toBeInstanceOf(PermissionGroupCapabilityError) + + expect(mocks.createRecord).not.toHaveBeenCalled() + }) + + /** + * Scope is the request's `type`, so the same operation must still serve the + * workspace-shared secret the organization is steering members toward. + */ + it('still creates a workspace-shared secret under the same restriction', async () => { + const created = createdCredential('env_workspace') + mocks.createRecord.mockResolvedValue({ success: true, created: true, credential: created }) + mocks.getActor.mockResolvedValue({ + credential: created, + member: { role: 'admin', status: 'active' }, + hasWorkspaceAccess: true, + isAdmin: true, + }) + + const result = await createWorkspaceCredential.execute({ + principal: sessionPrincipal, + input: { + workspaceId: WORKSPACE_ID, + type: 'env_workspace', + displayName: 'Shared OpenAI key', + envKey: 'OPENAI_API_KEY', + }, + }) + + expect(result.credential).toEqual(created) + }) + + it('creates the personal secret when no group withholds it', async () => { + resolveGroupConfigMock.mockResolvedValue(null) + const created = createdCredential('env_personal') + mocks.createRecord.mockResolvedValue({ success: true, created: true, credential: created }) + mocks.getActor.mockResolvedValue({ + credential: created, + member: { role: 'admin', status: 'active' }, + hasWorkspaceAccess: true, + isAdmin: true, + }) + + const result = await createWorkspaceCredential.execute({ + principal: sessionPrincipal, + input: { + workspaceId: WORKSPACE_ID, + type: 'env_personal', + displayName: 'My OpenAI key', + envKey: 'OPENAI_API_KEY', + }, + }) + + expect(result.credential).toEqual(created) + }) +}) diff --git a/apps/sim/lib/credentials/application/credential-crud.ts b/apps/sim/lib/credentials/application/credential-crud.ts index c28bb5302a0..e0ba3090979 100644 --- a/apps/sim/lib/credentials/application/credential-crud.ts +++ b/apps/sim/lib/credentials/application/credential-crud.ts @@ -29,6 +29,7 @@ import { } from '@/lib/credentials/queries' import { getServiceAccountGatingBlockType } from '@/lib/credentials/service-account-provider-ids' import { createIntegrationCredentialVisibility } from '@/lib/integrations/credential-visibility.server' +import { assertWorkspaceCapability } from '@/lib/permission-groups/capability-assertions' import { captureServerEvent } from '@/lib/posthog/server' import { loadActiveWorkspaceApplicationContext } from '@/lib/workspaces/application/workspace-context' import { checkWorkspaceAccess } from '@/lib/workspaces/permissions/utils' @@ -166,6 +167,16 @@ export interface CreateWorkspaceCredentialResult { auditMetadata: Record } +/** + * The credential types that belong to one person rather than to the workspace: + * a personal environment secret, and an OAuth grant bound to the connecting + * user's own linked account. `env_workspace`, `service_account` and + * `managed_oauth` are workspace-shared and stay available. + */ +const PERSONAL_SCOPE_CREDENTIAL_TYPES: ReadonlySet = new Set( + ['env_personal', 'oauth'] +) + export const createWorkspaceCredential = defineAuthorizedWorkspaceUseCase({ operation: credentialOperations.create, resolveContext: async ({ input }: { input: CreateWorkspaceCredentialInput }) => { @@ -174,8 +185,23 @@ export const createWorkspaceCredential = defineAuthorizedWorkspaceUseCase({ return context }, authorizationOptions: {}, - async execute({ principal, input }): Promise { + async execute({ principal, input, context }): Promise { const userId = requirePrincipalSubjectUserId(principal) + /** + * permission-group-enforced: credentials.personal — scope is the request's + * `type`, not a property of the operation: the same `credentials.create` + * makes a personal secret and a workspace-shared one. Declaring the + * capability on the operation would refuse both, so it is asserted here + * against the type actually being created. + */ + if (PERSONAL_SCOPE_CREDENTIAL_TYPES.has(input.type)) { + await assertWorkspaceCapability( + userId, + context.workspaceId, + 'credentials.personal', + context.workspaceOrganizationId + ) + } const result = await createCredentialRecord({ ...input, userId }, { authorizeWorkspace: false }) if (!result.success) throwCredentialMutationFailure(result) if (!result.credential) throw new Error('Credential creation succeeded without a credential') diff --git a/apps/sim/lib/credentials/application/credential-members.ts b/apps/sim/lib/credentials/application/credential-members.ts index 561ae6a130d..58a2262ce60 100644 --- a/apps/sim/lib/credentials/application/credential-members.ts +++ b/apps/sim/lib/credentials/application/credential-members.ts @@ -15,6 +15,7 @@ import { removeCredentialMember, upsertCredentialMember, } from '@/lib/credentials/members' +import { isWorkspaceCapabilityWithheld } from '@/lib/permission-groups/capability-assertions' import { captureServerEvent } from '@/lib/posthog/server' interface CredentialMemberResourceInput { @@ -124,10 +125,46 @@ export const removeCredentialMemberUseCase = defineAuthorizedCredentialUseCase({ }, }) +/** + * Every credential names a workspace (`credential.workspace_id` is NOT NULL), so + * the rows this user-global listing returns are workspace resources reached + * without naming a workspace. The endpoint's own gate resolves the caller's + * organization default group, which is right for the *act* — it belongs to the + * person, not to any one workspace — but it cannot answer per row. + * + * So each row is projected against the group governing **this same user** in the + * workspace holding that credential. That is the person's own group, not a + * bystander's: `credentials.list` withholds exactly these rows from them in that + * workspace under `integrations.manage`, and a listing that names no workspace + * must not be the way back to what the workspace-scoped listing hides. + * + * Only the projection. Leaving a membership stays ungoverned by the workspace + * group on purpose: it revokes the caller's own access and grants nothing, so + * gating it would strand a member inside a credential share they can no longer + * see — the same reasoning that keeps pausing a knowledge connector available + * after its type leaves the allowlist. + * + * The capability is read off the operation rather than spelled out here, so the + * projection follows the declaration if it is ever renamed. + */ export const listCredentialMembershipsUseCase = defineAuthorizedCredentialUserUseCase({ operation: credentialUserOperations.listMemberships, async execute({ principal }) { - return { memberships: await listCredentialMembershipsForUser(principal.userId) } + const memberships = await listCredentialMembershipsForUser(principal.userId) + const capability = credentialUserOperations.listMemberships.capability + if (capability === 'none') return { memberships } + const workspaceIds = [...new Set(memberships.map((membership) => membership.workspaceId))] + const withheld = new Set() + await Promise.all( + workspaceIds.map(async (workspaceId) => { + if (await isWorkspaceCapabilityWithheld(principal.userId, workspaceId, capability)) { + withheld.add(workspaceId) + } + }) + ) + return { + memberships: memberships.filter((membership) => !withheld.has(membership.workspaceId)), + } }, }) diff --git a/apps/sim/lib/credentials/application/managed-oauth-delegation.test.ts b/apps/sim/lib/credentials/application/managed-oauth-delegation.test.ts new file mode 100644 index 00000000000..c6d950036b0 --- /dev/null +++ b/apps/sim/lib/credentials/application/managed-oauth-delegation.test.ts @@ -0,0 +1,99 @@ +/** + * @vitest-environment node + */ +import { beforeEach, describe, expect, it, vi } from 'vitest' +import type { ExecutorDelegationOrigin } from '@/executor/types' + +const { mockBindInternalExecutorDelegation } = vi.hoisted(() => ({ + mockBindInternalExecutorDelegation: vi.fn(), +})) + +vi.mock('@/lib/auth/internal-delegation', () => ({ + bindInternalExecutorDelegation: mockBindInternalExecutorDelegation, + InvalidInternalDelegationBindingError: class InvalidInternalDelegationBindingError extends Error {}, +})) + +vi.mock('@/lib/auth/internal', () => ({ + InvalidInternalDelegationTokenError: class InvalidInternalDelegationTokenError extends Error {}, + verifyInternalDelegationToken: vi.fn(), +})) + +import { InvalidInternalDelegationBindingError } from '@/lib/auth/internal-delegation' +import { + bindExecutorManagedOAuthDelegation, + InvalidManagedOAuthDelegationError, +} from '@/lib/credentials/application/managed-oauth-delegation' + +function delegationOrigin( + overrides: Partial = {} +): ExecutorDelegationOrigin { + return { + subjectUserId: 'user-origin', + workflowId: 'workflow-origin', + executionId: 'execution-origin', + currentWorkflow: { workflowId: 'workflow-origin' }, + ...overrides, + } as ExecutorDelegationOrigin +} + +describe('bindExecutorManagedOAuthDelegation', () => { + beforeEach(() => { + vi.clearAllMocks() + mockBindInternalExecutorDelegation.mockImplementation(async (claims, options) => ({ + kind: 'delegated', + serviceId: 'executor', + subjectUserId: claims.subjectUserId, + workspaceId: 'workspace-canonical', + delegationId: claims.delegationId, + audience: options.audience, + resourceScope: options.resourceScope, + })) + }) + + it('requires current workflow authority before binding', async () => { + await expect( + bindExecutorManagedOAuthDelegation(delegationOrigin({ currentWorkflow: undefined }), 'cred-1') + ).rejects.toThrow('Managed credential delegation is missing current workflow authority') + expect(mockBindInternalExecutorDelegation).not.toHaveBeenCalled() + }) + + it('binds the origin to the managed-OAuth audience scoped to one credential', async () => { + const principal = await bindExecutorManagedOAuthDelegation(delegationOrigin(), 'cred-1') + + expect(mockBindInternalExecutorDelegation).toHaveBeenCalledWith( + expect.objectContaining({ + serviceId: 'executor', + subjectUserId: 'user-origin', + workflowId: 'workflow-origin', + executionId: 'execution-origin', + currentWorkflow: { workflowId: 'workflow-origin' }, + }), + expect.objectContaining({ + audience: 'sim:managed-oauth-credentials', + resourceScope: { credentialId: 'cred-1' }, + }) + ) + expect(principal).toMatchObject({ + audience: 'sim:managed-oauth-credentials', + resourceScope: { credentialId: 'cred-1' }, + }) + }) + + it('wraps binding rejections into the managed-OAuth delegation error', async () => { + mockBindInternalExecutorDelegation.mockRejectedValue( + new InvalidInternalDelegationBindingError('stale workflow context') + ) + + await expect( + bindExecutorManagedOAuthDelegation(delegationOrigin(), 'cred-1') + ).rejects.toBeInstanceOf(InvalidManagedOAuthDelegationError) + }) + + it('rethrows unexpected binding failures unchanged', async () => { + mockBindInternalExecutorDelegation.mockRejectedValue(new Error('db unavailable')) + + await expect(bindExecutorManagedOAuthDelegation(delegationOrigin(), 'cred-1')).rejects.toThrow( + 'db unavailable' + ) + }) +}) diff --git a/apps/sim/lib/credentials/application/managed-oauth-delegation.ts b/apps/sim/lib/credentials/application/managed-oauth-delegation.ts index 95c85426c0d..d5c68994745 100644 --- a/apps/sim/lib/credentials/application/managed-oauth-delegation.ts +++ b/apps/sim/lib/credentials/application/managed-oauth-delegation.ts @@ -8,6 +8,8 @@ import { InvalidInternalDelegationBindingError, } from '@/lib/auth/internal-delegation' import { MANAGED_OAUTH_DELEGATION_AUDIENCE } from '@/lib/credentials/application/authorization' +import { createExecutorPrincipalFromDelegationOrigin } from '@/lib/internal/principals/executor' +import type { ExecutorDelegationOrigin } from '@/executor/types' export class InvalidManagedOAuthDelegationError extends Error { constructor() { @@ -39,3 +41,31 @@ export async function authenticateManagedOAuthDelegation( throw error } } + +/** + * In-process sibling of {@link authenticateManagedOAuthDelegation}: binds the + * executor's own delegation origin to one managed credential without minting and + * re-verifying a delegation JWT — see {@link createExecutorPrincipalFromDelegationOrigin} + * for why that loses nothing. + */ +export async function bindExecutorManagedOAuthDelegation( + origin: ExecutorDelegationOrigin, + credentialId: string +): Promise { + if (!origin.currentWorkflow) { + throw new Error('Managed credential delegation is missing current workflow authority') + } + + try { + return await createExecutorPrincipalFromDelegationOrigin( + origin, + MANAGED_OAUTH_DELEGATION_AUDIENCE, + { credentialId } + ) + } catch (error) { + if (error instanceof InvalidInternalDelegationBindingError) { + throw new InvalidManagedOAuthDelegationError() + } + throw error + } +} diff --git a/apps/sim/lib/credentials/application/membership-projection.test.ts b/apps/sim/lib/credentials/application/membership-projection.test.ts new file mode 100644 index 00000000000..9f92845b680 --- /dev/null +++ b/apps/sim/lib/credentials/application/membership-projection.test.ts @@ -0,0 +1,158 @@ +/** + * @vitest-environment node + * + * `GET /api/credentials/memberships` names no workspace, so its own gate reads + * the caller's organization default group. Every credential it returns does name + * one (`credential.workspace_id` is NOT NULL), and `credentials.list` withholds + * those same rows inside the workspace under `integrations.manage`. These pin + * that the user-global listing is not the way back to what the workspace-scoped + * listing hides — projected against this user's own group in each workspace, + * never a bystander's — and that leaving a membership stays available. + */ +import { authMockFns, createMockRequest, permissionGroupScopeMock } from '@sim/testing' +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const { mockGetUserOrganization, mockGetOrgPermissionConfig, mockList, mockLeave } = vi.hoisted( + () => ({ + mockGetUserOrganization: vi.fn(), + mockGetOrgPermissionConfig: vi.fn(), + mockList: vi.fn(), + mockLeave: vi.fn(), + }) +) + +vi.mock('@/lib/billing/organizations/membership', () => ({ + getUserOrganization: mockGetUserOrganization, +})) + +vi.mock('@/lib/permission-groups/resolve.server', () => ({ + getUserPermissionConfig: vi.fn(), + getUserPermissionConfigForOrganization: mockGetOrgPermissionConfig, + resolveVerifiedUserAccessControlContext: vi.fn(), +})) + +vi.mock('@/lib/permission-groups/config-scope.server', () => permissionGroupScopeMock) + +vi.mock('@/lib/credentials/members', () => ({ + leaveCredentialMembership: mockLeave, + listCredentialMembers: vi.fn(), + listCredentialMembershipsForUser: mockList, + removeCredentialMember: vi.fn(), + upsertCredentialMember: vi.fn(), +})) + +import { DEFAULT_PERMISSION_GROUP_CONFIG } from '@/lib/permission-groups/fields' +import { DELETE, GET } from '@/app/api/credentials/memberships/route' + +const USER_ID = 'user-1' +const GOVERNED_WORKSPACE = 'workspace-governed' +const OPEN_WORKSPACE = 'workspace-open' + +const mockGetSession = authMockFns.mockGetSession +const mockResolveConfig = permissionGroupScopeMock.resolvePermissionGroupConfig + +function membership(id: string, workspaceId: string) { + return { + membershipId: `membership-${id}`, + credentialId: id, + workspaceId, + type: 'oauth' as const, + displayName: id, + providerId: 'google', + role: 'member' as const, + status: 'active' as const, + joinedAt: null, + } +} + +function callList() { + return GET( + createMockRequest('GET', undefined, {}, 'http://localhost/api/credentials/memberships'), + { params: Promise.resolve({}) } + ) +} + +function callLeave(credentialId: string) { + return DELETE( + createMockRequest( + 'DELETE', + undefined, + {}, + `http://localhost/api/credentials/memberships?credentialId=${credentialId}` + ), + { params: Promise.resolve({}) } + ) +} + +describe('credential membership listing under a workspace group', () => { + beforeEach(() => { + vi.clearAllMocks() + mockResolveConfig.mockReset() + mockGetSession.mockResolvedValue({ user: { id: USER_ID }, session: { id: 'session-1' } }) + mockGetUserOrganization.mockResolvedValue({ + organizationId: 'org-1', + role: 'member', + memberId: 'member-1', + }) + mockGetOrgPermissionConfig.mockResolvedValue(null) + mockList.mockResolvedValue([ + membership('cred-governed', GOVERNED_WORKSPACE), + membership('cred-open', OPEN_WORKSPACE), + ]) + mockLeave.mockResolvedValue(undefined) + mockResolveConfig.mockImplementation(async (_userId: string, workspaceId: string) => + workspaceId === GOVERNED_WORKSPACE + ? { ...DEFAULT_PERMISSION_GROUP_CONFIG, hideIntegrationsTab: true } + : null + ) + }) + + it('drops the rows whose workspace withholds Integrations from this user', async () => { + const response = await callList() + + expect(response.status).toBe(200) + const body = await response.json() + expect(body.memberships.map((row: { credentialId: string }) => row.credentialId)).toEqual([ + 'cred-open', + ]) + }) + + it('resolves the group as the caller themself, in each credential’s workspace', async () => { + await callList() + + expect(mockResolveConfig).toHaveBeenCalledWith(USER_ID, GOVERNED_WORKSPACE, undefined) + expect(mockResolveConfig).toHaveBeenCalledWith(USER_ID, OPEN_WORKSPACE, undefined) + }) + + it('asks once per workspace, not once per credential', async () => { + mockList.mockResolvedValue([ + membership('cred-a', GOVERNED_WORKSPACE), + membership('cred-b', GOVERNED_WORKSPACE), + membership('cred-c', OPEN_WORKSPACE), + ]) + + await callList() + + expect(mockResolveConfig).toHaveBeenCalledTimes(2) + }) + + it('returns every row when no group governs the caller anywhere', async () => { + mockResolveConfig.mockResolvedValue(null) + + const response = await callList() + + const body = await response.json() + expect(body.memberships).toHaveLength(2) + }) + + /** + * Leaving revokes the caller's own access and grants nothing, so a workspace + * that hides the module must not strand them inside the share. + */ + it('still lets the member leave a credential in the withholding workspace', async () => { + const response = await callLeave('cred-governed') + + expect(response.status).toBe(200) + expect(mockLeave).toHaveBeenCalledWith({ userId: USER_ID, credentialId: 'cred-governed' }) + }) +}) diff --git a/apps/sim/lib/credentials/application/operations.test.ts b/apps/sim/lib/credentials/application/operations.test.ts index 77737952366..975e3b14668 100644 --- a/apps/sim/lib/credentials/application/operations.test.ts +++ b/apps/sim/lib/credentials/application/operations.test.ts @@ -44,6 +44,7 @@ describe('credential operations', () => { minimumRole: 'read', workspaceApiKey: 'allow', principalKinds: ['workspace_api_key'], + capability: 'integrations.manage', }) expect(() => defineCredentialOperation(workspaceKeyOperation, 'admin')).toThrow( diff --git a/apps/sim/lib/credentials/application/operations.ts b/apps/sim/lib/credentials/application/operations.ts index 4753c3136a8..4bda143e590 100644 --- a/apps/sim/lib/credentials/application/operations.ts +++ b/apps/sim/lib/credentials/application/operations.ts @@ -1,5 +1,6 @@ -import type { ApplicationOperation } from '@/lib/core/application' +import type { ApplicationOperation, OperationDeclarableCapability } from '@/lib/core/application' import { defineWorkspaceOperation, type WorkspaceOperation } from '@/lib/core/application' +import { CAPABILITY_RULES } from '@/lib/permission-groups/capabilities' import { CREDENTIAL_GROUP_CREDENTIAL_USE_ACTION } from '@/lib/resource-policies/registry' export type CredentialRole = 'member' | 'admin' @@ -34,30 +35,46 @@ export const credentialOperations = { id: 'credentials.list', minimumRole: 'read', workspaceApiKey: 'deny', + capability: 'integrations.manage', principalKinds: ['session'], }), listProviders: defineWorkspaceOperation({ id: 'credentials.providers.list', minimumRole: 'read', workspaceApiKey: 'allow', + capability: 'integrations.manage', principalKinds: ['session', 'personal_api_key', 'workspace_api_key'], }), listConnections: defineWorkspaceOperation({ id: 'credentials.connections.list', minimumRole: 'read', workspaceApiKey: 'allow', + capability: 'integrations.manage', principalKinds: ['session', 'personal_api_key', 'workspace_api_key'], }), + /** + * `integrations.manage`, like every other credential operation — these three + * take a *target*, not a scope: `providerId` connects a personal account, + * `credentialId` re-authorizes a credential the workspace already holds. Only + * the first is what `disablePersonalCredentials` withholds, so the narrower + * `credentials.personal` is asserted on that branch inside + * `resolveCredentialConnectionTarget` rather than declared here. Declaring it + * here withheld the reconnect too — refusing the workspace-shared credentials + * that same setting exists to mandate — and, because an operation declares one + * capability, let a group that hid the whole Integrations module still connect. + */ createConnection: defineWorkspaceOperation({ id: 'credentials.connections.create', minimumRole: 'write', workspaceApiKey: 'deny', + capability: 'integrations.manage', principalKinds: ['session', 'personal_api_key'], }), prepareConnection: defineWorkspaceOperation({ id: 'credentials.connections.prepare', minimumRole: 'write', workspaceApiKey: 'deny', + capability: 'integrations.manage', principalKinds: ['delegated'], delegatedServices: ['copilot'], }), @@ -65,6 +82,7 @@ export const credentialOperations = { id: 'credentials.service_accounts.create', minimumRole: 'write', workspaceApiKey: 'deny', + capability: 'integrations.manage', principalKinds: ['session', 'personal_api_key'], }), read: defineCredentialOperation( @@ -72,6 +90,7 @@ export const credentialOperations = { id: 'credentials.read', minimumRole: 'read', workspaceApiKey: 'deny', + capability: 'integrations.manage', principalKinds: ['session'], }), 'member' @@ -80,6 +99,7 @@ export const credentialOperations = { id: 'credentials.create', minimumRole: 'write', workspaceApiKey: 'deny', + capability: 'integrations.manage', principalKinds: ['session'], }), update: defineCredentialOperation( @@ -87,6 +107,7 @@ export const credentialOperations = { id: 'credentials.update', minimumRole: 'read', workspaceApiKey: 'deny', + capability: 'integrations.manage', ...HUMAN_AND_COPILOT_PRINCIPALS, }), 'admin' @@ -96,6 +117,7 @@ export const credentialOperations = { id: 'credentials.delete', minimumRole: 'read', workspaceApiKey: 'deny', + capability: 'integrations.manage', ...HUMAN_AND_COPILOT_PRINCIPALS, }), 'admin' @@ -104,6 +126,7 @@ export const credentialOperations = { id: 'credentials.delete_many', minimumRole: 'read', workspaceApiKey: 'deny', + capability: 'integrations.manage', principalKinds: ['delegated'], delegatedServices: ['copilot'], }), @@ -111,12 +134,14 @@ export const credentialOperations = { id: 'credentials.drafts.save', minimumRole: 'write', workspaceApiKey: 'deny', + capability: 'integrations.manage', principalKinds: ['session'], }), listMembers: defineWorkspaceOperation({ id: 'credentials.members.list', minimumRole: 'read', workspaceApiKey: 'deny', + capability: 'integrations.manage', principalKinds: ['session'], }), upsertMember: defineCredentialOperation( @@ -124,6 +149,7 @@ export const credentialOperations = { id: 'credentials.members.upsert', minimumRole: 'read', workspaceApiKey: 'deny', + capability: 'integrations.manage', principalKinds: ['session'], }), 'admin' @@ -133,6 +159,7 @@ export const credentialOperations = { id: 'credentials.members.remove', minimumRole: 'read', workspaceApiKey: 'deny', + capability: 'integrations.manage', principalKinds: ['session'], }), 'admin' @@ -141,12 +168,14 @@ export const credentialOperations = { id: 'credentials.connections.launch', minimumRole: 'write', workspaceApiKey: 'deny', + capability: 'integrations.manage', principalKinds: ['session'], }), useManagedOAuth: defineWorkspaceOperation({ id: 'credentials.managed_oauth.use', minimumRole: 'read', workspaceApiKey: 'deny', + capability: 'integrations.manage', principalKinds: ['delegated'], delegatedServices: ['executor'], resourcePolicy: { @@ -156,22 +185,81 @@ export const credentialOperations = { }), } as const +/** + * A credential operation whose resource is the acting user's own account rather + * than a workspace, so it carries no role and no workspace-key policy. + * + * It still carries a `capability`, and required rather than optional for the + * same reason `defineWorkspaceOperation` requires one: an absent field cannot be + * told apart from an unreviewed one. Listing and disconnecting a user's OAuth + * connections is exactly what `hideIntegrationsTab` claims to revoke, and these + * operations shipped with no capability at all — invisibly, because this factory + * does not call `defineWorkspaceOperation` and so is read by neither that + * builder's definition-time guard nor `check:permission-group-enforcement`, + * which parses `defineWorkspaceOperation` call sites out of the source text. + */ export interface CredentialUserOperation extends ApplicationOperation { readonly principalKinds: readonly ['session'] + readonly capability: OperationDeclarableCapability | 'none' } function defineCredentialUserOperation( - id: Id + id: Id, + capability: OperationDeclarableCapability | 'none' ): CredentialUserOperation { if (!id.trim()) throw new Error('Credential user operation ID must not be empty') - return Object.freeze({ id, principalKinds: Object.freeze(['session'] as const) }) + if (capability === undefined) { + throw new Error( + `Credential user operation ${id} declares no capability; name one, or 'none' with a reason` + ) + } + if (capability !== 'none') { + const rule = CAPABILITY_RULES[capability] + if (!rule) + throw new Error(`Credential user operation ${id} names unknown capability ${capability}`) + /** + * A parameterized rule reads a value only the request carries, which + * `defineAuthorizedCredentialUserUseCase` never sees; declared here it would + * read as enforced while nothing applied it. + */ + if (rule.kind !== 'static') { + throw new Error( + `Credential user operation ${id} declares parameterized capability ${capability}; assert it from the use case instead` + ) + } + } + return Object.freeze({ id, capability, principalKinds: Object.freeze(['session'] as const) }) } +/** + * All five take `integrations.manage`, matching every other credential + * operation that is not personal-scope by construction — `credentials.list`, + * `credentials.connections.list`, `credentials.members.list` and the rest above. + * Not `credentials.personal`: that one is reserved for the OAuth *connect* flow, + * whose credential is personal by construction, and an organization that + * revokes the Integrations module means members cannot see or remove a + * connection either. + */ export const credentialUserOperations = { - listMemberships: defineCredentialUserOperation('credentials.memberships.list'), - leaveMembership: defineCredentialUserOperation('credentials.memberships.leave'), - listOAuthConnections: defineCredentialUserOperation('credentials.oauth_connections.list'), - listConnectedAccounts: defineCredentialUserOperation('credentials.accounts.list'), - disconnectOAuth: defineCredentialUserOperation('credentials.oauth_connections.disconnect'), + listMemberships: defineCredentialUserOperation( + 'credentials.memberships.list', + 'integrations.manage' + ), + leaveMembership: defineCredentialUserOperation( + 'credentials.memberships.leave', + 'integrations.manage' + ), + listOAuthConnections: defineCredentialUserOperation( + 'credentials.oauth_connections.list', + 'integrations.manage' + ), + listConnectedAccounts: defineCredentialUserOperation( + 'credentials.accounts.list', + 'integrations.manage' + ), + disconnectOAuth: defineCredentialUserOperation( + 'credentials.oauth_connections.disconnect', + 'integrations.manage' + ), } as const diff --git a/apps/sim/lib/credentials/application/provider-catalog.test.ts b/apps/sim/lib/credentials/application/provider-catalog.test.ts index f485a80465d..73fc59914fc 100644 --- a/apps/sim/lib/credentials/application/provider-catalog.test.ts +++ b/apps/sim/lib/credentials/application/provider-catalog.test.ts @@ -20,20 +20,38 @@ vi.mock('@/lib/core/config/env-flags', () => ({ getAllowedIntegrationsFromEnv: mocks.getAllowedIntegrationsFromEnv, })) -vi.mock('@/ee/access-control/utils/permission-check', () => ({ +vi.mock('@/lib/permission-groups/resolve.server', () => ({ getUserPermissionConfig: mocks.getUserPermissionConfig, })) -vi.mock('@/lib/permission-groups/integration-allowlist', () => ({ - intersectIntegrationAllowlists: ( +/** + * The real helpers canonicalize each side through the generated successor map; + * this stub keeps the intersection semantics without the map, because the ids + * used here are fixtures rather than real block types. + */ +vi.mock('@/lib/permission-groups/integration-allowlist', () => { + const intersect = ( permissionGroup: readonly string[] | null, deployment: readonly string[] | null ) => { if (!permissionGroup) return deployment if (!deployment) return permissionGroup return permissionGroup.filter((type) => deployment.includes(type)) - }, -})) + } + return { + intersectIntegrationAllowlists: intersect, + intersectAccessControlAllowlists: ( + permissionGroup: readonly string[] | null, + deployment: readonly string[] | null + ) => { + const result = intersect(permissionGroup, deployment) + return result === null ? null : new Set(result) + }, + resolveAccessControlBlockType: (blockType: string) => blockType, + toAccessControlAllowlist: (allowlist: readonly string[] | null) => + allowlist ? new Set(allowlist) : null, + } +}) vi.mock('@/lib/integrations/credential-visibility.server', () => ({ createIntegrationCredentialVisibility: mocks.createVisibility, @@ -92,9 +110,15 @@ describe('listCredentialProviderCatalog', () => { beforeEach(() => { vi.clearAllMocks() mocks.getAllOAuthServices.mockReturnValue(services) - mocks.getAllowedIntegrationsFromEnv.mockReturnValue(['salesforce']) + /** + * The permission group is the NARROWER half on purpose. With the deployment + * allowlist narrower, the intersection is the same set whether or not the + * group is read at all, and every assertion below passes against a catalog + * that never consulted it — which is what this fixture used to look like. + */ + mocks.getAllowedIntegrationsFromEnv.mockReturnValue(['salesforce', 'trello']) mocks.getUserPermissionConfig.mockResolvedValue({ - allowedIntegrations: ['salesforce', 'trello'], + allowedIntegrations: ['salesforce'], }) mocks.getBlockVisibility.mockResolvedValue({ revealed: new Set(), @@ -186,8 +210,13 @@ describe('listCredentialProviderCatalog', () => { ) expect(mocks.getUserPermissionConfig).not.toHaveBeenCalled() + /** + * The deployment allowlist alone, not the personal caller's narrower group: + * a workspace API key has no user and therefore no group, and borrowing the + * key creator's would hide Trello from every caller of a shared credential. + */ expect(mocks.createVisibility).toHaveBeenCalledWith( - expect.objectContaining({ allowedIntegrationTypes: new Set(['salesforce']) }) + expect.objectContaining({ allowedIntegrationTypes: new Set(['salesforce', 'trello']) }) ) }) diff --git a/apps/sim/lib/credentials/client-credential-accounts/minters/zoho-desk.test.ts b/apps/sim/lib/credentials/client-credential-accounts/minters/zoho-desk.test.ts index 6bf48fd65c1..bfaa0b47f5c 100644 --- a/apps/sim/lib/credentials/client-credential-accounts/minters/zoho-desk.test.ts +++ b/apps/sim/lib/credentials/client-credential-accounts/minters/zoho-desk.test.ts @@ -274,11 +274,10 @@ describe('mintZohoDeskServiceAccountToken', () => { expect(result.apiDomain).toBe('https://desk.zoho.in') expect(mockLoggerWarn).toHaveBeenCalledWith( 'Zoho api_domain disagrees with the selected data center', - expect.objectContaining({ + { selectedDataCenter: 'eu', - selectedDeskBase: 'https://desk.zoho.eu', - reportedDeskBase: 'https://desk.zoho.in', - }) + usedProviderReportedDomain: true, + } ) }) diff --git a/apps/sim/lib/credentials/client-credential-accounts/minters/zoho-desk.ts b/apps/sim/lib/credentials/client-credential-accounts/minters/zoho-desk.ts index 32f776ff33d..4113ce36120 100644 --- a/apps/sim/lib/credentials/client-credential-accounts/minters/zoho-desk.ts +++ b/apps/sim/lib/credentials/client-credential-accounts/minters/zoho-desk.ts @@ -242,10 +242,8 @@ export async function mintZohoDeskServiceAccountToken( const apiDomain = reportedDeskBase ?? dataCenter.deskBase if (reportedDeskBase && reportedDeskBase !== dataCenter.deskBase) { logger.warn('Zoho api_domain disagrees with the selected data center', { - soid, selectedDataCenter: dataCenter.id, - selectedDeskBase: dataCenter.deskBase, - reportedDeskBase, + usedProviderReportedDomain: true, }) } const expiresInSeconds = diff --git a/apps/sim/lib/credentials/draft-hooks.test.ts b/apps/sim/lib/credentials/draft-hooks.test.ts index 5737c26ac23..dbbd887f51f 100644 --- a/apps/sim/lib/credentials/draft-hooks.test.ts +++ b/apps/sim/lib/credentials/draft-hooks.test.ts @@ -23,6 +23,7 @@ import { handleCreateCredentialFromDraft, handleReconnectCredential, } from '@/lib/credentials/draft-hooks' +import { getOAuthRefreshCoordinationIdentity } from '@/lib/oauth/refresh-coordination' describe('handleCreateCredentialFromDraft', () => { beforeEach(() => { @@ -55,7 +56,9 @@ describe('handleCreateCredentialFromDraft', () => { expect(dbChainMockFns.update).toHaveBeenCalledWith(schemaMock.credential) expect(dbChainMockFns.set).toHaveBeenCalledWith({ updatedAt: now }) - expect(mocks.clearDeadFlag).toHaveBeenCalledWith('account-1') + expect(mocks.clearDeadFlag).toHaveBeenCalledWith( + getOAuthRefreshCoordinationIdentity('account-1') + ) expect(auditMockFns.mockRecordAudit).toHaveBeenCalledWith( expect.objectContaining({ action: 'credential.reconnected', @@ -109,6 +112,9 @@ describe('handleReconnectCredential', () => { now: new Date('2026-08-14T18:00:00.000Z'), }) + expect(mocks.clearDeadFlag).toHaveBeenCalledWith( + getOAuthRefreshCoordinationIdentity('account-new') + ) expect(auditMockFns.mockRecordAudit).toHaveBeenCalledWith( expect.objectContaining({ resourceId: 'credential-1', diff --git a/apps/sim/lib/credentials/draft-hooks.ts b/apps/sim/lib/credentials/draft-hooks.ts index 79852f7824a..6cf0b4c9f1f 100644 --- a/apps/sim/lib/credentials/draft-hooks.ts +++ b/apps/sim/lib/credentials/draft-hooks.ts @@ -6,7 +6,7 @@ import { getPostgresConstraintName, getPostgresErrorCode } from '@sim/utils/erro import { generateId } from '@sim/utils/id' import { and, eq, sql } from 'drizzle-orm' import { deleteOrphanedOAuthAccount } from '@/lib/credentials/deletion' -import { clearDeadFlag } from '@/lib/oauth/terminal-errors' +import { clearOAuthRefreshDeadFlag } from '@/lib/oauth/refresh-coordination' import { captureServerEvent } from '@/lib/posthog/server' const logger = createLogger('CredentialDraftHooks') @@ -74,7 +74,7 @@ export async function handleCreateCredentialFromDraft(params: { .set({ updatedAt: now }) .where(eq(schema.credential.id, existingCredential.id)) - await clearDeadFlag(accountId) + await clearOAuthRefreshDeadFlag(accountId) recordAudit({ workspaceId: draft.workspaceId, @@ -108,7 +108,7 @@ export async function handleCreateCredentialFromDraft(params: { accountId, }) - await clearDeadFlag(accountId) + await clearOAuthRefreshDeadFlag(accountId) recordAudit({ workspaceId: draft.workspaceId, @@ -208,7 +208,7 @@ export async function handleReconnectCredential(params: { } ) - await clearDeadFlag(newAccountId) + await clearOAuthRefreshDeadFlag(newAccountId) recordAudit({ workspaceId, diff --git a/apps/sim/lib/custom-tools/application/operations.ts b/apps/sim/lib/custom-tools/application/operations.ts index fbeace41633..1de8b666ec4 100644 --- a/apps/sim/lib/custom-tools/application/operations.ts +++ b/apps/sim/lib/custom-tools/application/operations.ts @@ -9,29 +9,39 @@ const HUMAN_PRINCIPAL_POLICY = { delegatedServices: ['copilot'], } as const +/** + * Every operation declares `custom_tools.use`, reads included. A custom tool is + * a user-authored function an agent calls; a group that withholds them has no + * use for the definitions either, and gating only execution would leave the + * authoring surface open to a member who can never run what it produces. + */ export const customToolOperations = { list: defineWorkspaceOperation({ id: 'custom_tools.list', minimumRole: 'read', workspaceApiKey: 'allow', + capability: 'custom_tools.use', ...ALL_PRINCIPAL_POLICY, }), listAvailable: defineWorkspaceOperation({ id: 'custom_tools.list_available', minimumRole: 'read', workspaceApiKey: 'deny', + capability: 'custom_tools.use', ...HUMAN_PRINCIPAL_POLICY, }), read: defineWorkspaceOperation({ id: 'custom_tools.read', minimumRole: 'read', workspaceApiKey: 'allow', + capability: 'custom_tools.use', ...ALL_PRINCIPAL_POLICY, }), readAvailableByIdOrTitle: defineWorkspaceOperation({ id: 'custom_tools.read_available_by_id_or_title', minimumRole: 'read', workspaceApiKey: 'deny', + capability: 'custom_tools.use', principalKinds: ['delegated'], delegatedServices: ['copilot', 'executor'], }), @@ -39,36 +49,42 @@ export const customToolOperations = { id: 'custom_tools.create', minimumRole: 'write', workspaceApiKey: 'allow', + capability: 'custom_tools.use', ...ALL_PRINCIPAL_POLICY, }), save: defineWorkspaceOperation({ id: 'custom_tools.save', minimumRole: 'write', workspaceApiKey: 'allow', + capability: 'custom_tools.use', ...ALL_PRINCIPAL_POLICY, }), update: defineWorkspaceOperation({ id: 'custom_tools.update', minimumRole: 'write', workspaceApiKey: 'allow', + capability: 'custom_tools.use', ...ALL_PRINCIPAL_POLICY, }), updateAvailable: defineWorkspaceOperation({ id: 'custom_tools.update_available', minimumRole: 'write', workspaceApiKey: 'deny', + capability: 'custom_tools.use', ...HUMAN_PRINCIPAL_POLICY, }), delete: defineWorkspaceOperation({ id: 'custom_tools.delete', minimumRole: 'write', workspaceApiKey: 'allow', + capability: 'custom_tools.use', ...ALL_PRINCIPAL_POLICY, }), deleteAvailable: defineWorkspaceOperation({ id: 'custom_tools.delete_available', minimumRole: 'write', workspaceApiKey: 'deny', + capability: 'custom_tools.use', ...HUMAN_PRINCIPAL_POLICY, }), } as const diff --git a/apps/sim/lib/data-drains/access.ts b/apps/sim/lib/data-drains/access.ts index 5a1db375444..8399011a73a 100644 --- a/apps/sim/lib/data-drains/access.ts +++ b/apps/sim/lib/data-drains/access.ts @@ -24,7 +24,13 @@ export type DrainAccessResult = /** * Auth + membership + role + enterprise-plan gate shared by every data-drain * route. Owner/admin role is required for reads as well as writes since drain - * configs expose customer bucket names and webhook URLs. On Sim Cloud the + * configs expose customer bucket names and webhook URLs. + * + * Deliberately above member permission groups, like the audit-log surface: + * the reader of a drained record is the organization, not a member, so the + * per-member log projections (`hideCostInfo`, `hideTraceSpans`) do not apply + * to what a drain serializes, and no group capability gates configuring one — + * the owner/admin requirement is the whole access story. On Sim Cloud the * gate is the Enterprise plan; on self-hosted it's `DATA_DRAINS_ENABLED`, * which 404s when unset so a newer image doesn't silently expose drains. */ diff --git a/apps/sim/lib/data-drains/destinations/s3.ts b/apps/sim/lib/data-drains/destinations/s3.ts index 32727bfb16c..7d464626547 100644 --- a/apps/sim/lib/data-drains/destinations/s3.ts +++ b/apps/sim/lib/data-drains/destinations/s3.ts @@ -85,7 +85,7 @@ const s3ConfigSchema = z.object({ .string() .url() .refine((v) => v.startsWith('https://'), { message: 'endpoint must use https://' }) - .refine((value) => validateExternalUrl(value, 'endpoint').isValid, { + .refine((value) => validateExternalUrl(value, 'endpoint', 'configuredEndpoint').isValid, { message: 'endpoint must be HTTPS and not point at a private, loopback, or metadata address', }) .optional(), @@ -128,9 +128,9 @@ function isS3ServiceException(error: unknown): error is S3ServiceException { /** DNS-aware SSRF check: catches hostnames that resolve to internal IPs (the schema check only catches IP literals). */ async function assertEndpointIsPublic(endpoint: string | undefined): Promise { if (!endpoint) return - const result = await validateUrlWithDNS(endpoint, 'endpoint') + const result = await validateUrlWithDNS(endpoint, 'endpoint', 'configuredEndpoint') if (!result.isValid) { - throw new Error(result.error ?? 'S3 endpoint failed SSRF validation') + throw new Error(result.error) } } diff --git a/apps/sim/lib/data-drains/destinations/webhook.ts b/apps/sim/lib/data-drains/destinations/webhook.ts index 4e9cd289550..54c480f6dd0 100644 --- a/apps/sim/lib/data-drains/destinations/webhook.ts +++ b/apps/sim/lib/data-drains/destinations/webhook.ts @@ -46,9 +46,9 @@ const RESERVED_SIGNATURE_HEADER_NAMES = new Set([ const HEADER_INJECTION_PATTERN = /[\r\n\0]/ async function resolvePublicTarget(url: string): Promise { - const result = await validateUrlWithDNS(url, 'url') - if (!result.isValid || !result.resolvedIP) { - throw new Error(result.error ?? 'Webhook URL failed SSRF validation') + const result = await validateUrlWithDNS(url, 'url', 'configuredEndpoint') + if (!result.isValid) { + throw new Error(result.error) } return result.resolvedIP } @@ -58,7 +58,7 @@ const webhookConfigSchema = z.object({ .string() .url('url must be a valid URL') .max(2048, 'url must be at most 2048 characters') - .refine((value) => validateExternalUrl(value, 'url').isValid, { + .refine((value) => validateExternalUrl(value, 'url', 'configuredEndpoint').isValid, { message: 'url must be HTTPS and not point at a private, loopback, or metadata address', }), signatureHeader: z @@ -165,6 +165,7 @@ export const webhookDestination: DrainDestination< isProbe: true, }) const response = await secureFetchWithPinnedIP(config.url, resolvedIP, { + profile: 'configuredEndpoint', method: 'POST', body: new Uint8Array(probe), headers, @@ -193,6 +194,7 @@ export const webhookDestination: DrainDestination< let response: Awaited> | undefined try { response = await secureFetchWithPinnedIP(config.url, resolvedIP, { + profile: 'configuredEndpoint', method: 'POST', body: new Uint8Array(body), headers, diff --git a/apps/sim/lib/embeddings/client.test.ts b/apps/sim/lib/embeddings/client.test.ts index cd19f32f156..054a83c8568 100644 --- a/apps/sim/lib/embeddings/client.test.ts +++ b/apps/sim/lib/embeddings/client.test.ts @@ -459,14 +459,17 @@ describe('embed', () => { }) it('retries a rate-limited request and succeeds on a later attempt', async () => { + vi.useFakeTimers() fetchMock .mockResolvedValueOnce(jsonResponse({ error: 'slow down' }, 429)) .mockResolvedValueOnce(jsonResponse(openAIBody([[7, 8]]))) - const result = await embed(['hello'], { + const pending = embed(['hello'], { model: 'text-embedding-3-small', apiKey: 'sk-test', }) + await vi.runAllTimersAsync() + const result = await pending expect(fetchMock).toHaveBeenCalledTimes(2) expect(result.embeddings[0].slice(0, 2)).toEqual([7, 8]) @@ -620,16 +623,19 @@ describe('embed', () => { }) it('projects once even when the request is retried', async () => { + vi.useFakeTimers() const projectInputs = vi.fn((values: readonly string[]) => values.map(() => 'projected')) fetchMock .mockResolvedValueOnce(jsonResponse({ error: 'rate limited' }, 429)) .mockResolvedValueOnce(jsonResponse(openAIBody([[1]]))) - await embed(['secret'], { + const pending = embed(['secret'], { model: 'text-embedding-3-small', apiKey: 'sk-test', projectInputs, }) + await vi.runAllTimersAsync() + await pending expect(fetchMock).toHaveBeenCalledTimes(2) expect(projectInputs).toHaveBeenCalledTimes(1) diff --git a/apps/sim/lib/environment/api.ts b/apps/sim/lib/environment/api.ts index af78dc71c26..3fe45abf5d4 100644 --- a/apps/sim/lib/environment/api.ts +++ b/apps/sim/lib/environment/api.ts @@ -5,7 +5,7 @@ import { getPersonalEnvironmentContract, getWorkspaceEnvironmentContract, type workspaceEnvironmentDataSchema, -} from '@/lib/api/contracts' +} from '@/lib/api/contracts/environment' export type EnvironmentVariable = z.output diff --git a/apps/sim/lib/environment/utils.test.ts b/apps/sim/lib/environment/utils.test.ts index 0d56b57d9b7..c3c7d211c19 100644 --- a/apps/sim/lib/environment/utils.test.ts +++ b/apps/sim/lib/environment/utils.test.ts @@ -18,6 +18,7 @@ const { mockGetUserEntityPermissions, mockGetWorkspaceEnvKeyAdminAccess, mockRecordAudit, + mockGetActivelyBannedUserIds, } = vi.hoisted(() => ({ mockCreateWorkspaceEnvCredentials: vi.fn(), mockCheckWorkspaceAccess: vi.fn(), @@ -25,6 +26,7 @@ const { mockGetUserEntityPermissions: vi.fn(), mockGetWorkspaceEnvKeyAdminAccess: vi.fn(), mockRecordAudit: vi.fn(), + mockGetActivelyBannedUserIds: vi.fn().mockResolvedValue([]), })) // vitest.setup.ts mocks this module globally; this suite tests the real one. @@ -42,6 +44,9 @@ vi.mock('@/lib/credentials/environment', () => ({ getWorkspaceEnvKeyAdminAccess: mockGetWorkspaceEnvKeyAdminAccess, syncPersonalEnvCredentialsForUser: vi.fn(), })) +vi.mock('@/lib/auth/ban', () => ({ + getActivelyBannedUserIds: mockGetActivelyBannedUserIds, +})) vi.mock('@/lib/workspaces/permissions/utils', () => ({ checkWorkspaceAccess: mockCheckWorkspaceAccess, getUserEntityPermissions: mockGetUserEntityPermissions, @@ -50,13 +55,298 @@ vi.mock('@/lib/workspaces/permissions/utils', () => ({ import { getEffectiveDecryptedEnv, getEffectiveEnvironmentSnapshot, + getEffectiveEnvironmentVariableNames, getExecutionEnvironment, getPersonalAndWorkspaceEnv, invalidateEffectiveDecryptedEnvCache, + resolveEffectiveEnvironmentVariables, upsertWorkspaceEnvVars, WorkspaceEnvAccessError, } from '@/lib/environment/utils' +describe('getEffectiveEnvironmentVariableNames', () => { + beforeEach(() => { + vi.clearAllMocks() + resetDbChainMock() + invalidateEffectiveDecryptedEnvCache({ userId: 'names-user' }) + mockCheckWorkspaceAccess.mockResolvedValue({ + exists: true, + hasAccess: true, + canWrite: true, + canAdmin: false, + }) + mockGetAccessibleEnvCredentials.mockResolvedValue([]) + encryptionMockFns.mockDecryptSecret.mockReset() + }) + + it('lists only stored, accessible names across personal and workspace scopes without decryption', async () => { + mockGetAccessibleEnvCredentials.mockResolvedValue([ + { + type: 'env_workspace', + envKey: 'WORKSPACE_VISIBLE', + envOwnerUserId: null, + updatedAt: new Date('2026-01-01T00:00:00.000Z'), + unredacted: false, + }, + { + type: 'env_workspace', + envKey: 'DUPLICATE', + envOwnerUserId: null, + updatedAt: new Date('2026-01-01T00:00:00.000Z'), + unredacted: false, + }, + { + type: 'env_workspace', + envKey: 'MISSING_WORKSPACE', + envOwnerUserId: null, + updatedAt: new Date('2026-01-01T00:00:00.000Z'), + unredacted: false, + }, + { + type: 'env_personal', + envKey: 'SHARED_PRESENT', + envOwnerUserId: 'owner-2', + updatedAt: new Date('2026-01-02T00:00:00.000Z'), + }, + { + type: 'env_personal', + envKey: 'SHARED_MISSING', + envOwnerUserId: 'owner-3', + updatedAt: new Date('2026-01-01T00:00:00.000Z'), + }, + ]) + queueTableRows(environment, [ + { variables: { OWN_ONLY: 'own-cipher', DUPLICATE: 'duplicate-cipher' } }, + ]) + queueTableRows(workspaceEnvironment, [ + { + variables: { + WORKSPACE_VISIBLE: 'workspace-cipher', + DUPLICATE: 'duplicate-workspace-cipher', + WORKSPACE_HIDDEN: 'hidden-cipher', + }, + }, + ]) + queueTableRows(environment, [ + { userId: 'owner-2', variables: { SHARED_PRESENT: 'shared-cipher' } }, + { userId: 'owner-3', variables: { UNRELATED: 'unrelated-cipher' } }, + ]) + + await expect( + getEffectiveEnvironmentVariableNames('names-user', 'workspace-1') + ).resolves.toEqual(['DUPLICATE', 'OWN_ONLY', 'SHARED_PRESENT', 'WORKSPACE_VISIBLE']) + expect(encryptionMockFns.mockDecryptSecret).not.toHaveBeenCalled() + + // A later snapshot performs a fresh lookup, proving the names read did not warm its LRU. + queueTableRows(environment, [{ variables: { FRESH_PERSONAL: 'fresh-personal-cipher' } }]) + queueTableRows(workspaceEnvironment, [ + { variables: { WORKSPACE_VISIBLE: 'fresh-workspace-cipher' } }, + ]) + queueTableRows(environment, [ + { userId: 'owner-2', variables: { SHARED_PRESENT: 'fresh-shared-cipher' } }, + { userId: 'owner-3', variables: {} }, + ]) + encryptionMockFns.mockDecryptSecret.mockImplementation(async (encryptedValue: string) => ({ + decrypted: `plain:${encryptedValue}`, + })) + + await expect( + getEffectiveEnvironmentSnapshot('names-user', 'workspace-1') + ).resolves.toMatchObject({ + personalEncrypted: { + FRESH_PERSONAL: 'fresh-personal-cipher', + SHARED_PRESENT: 'fresh-shared-cipher', + }, + workspaceEncrypted: { WORKSPACE_VISIBLE: 'fresh-workspace-cipher' }, + }) + expect(encryptionMockFns.mockDecryptSecret).toHaveBeenCalledTimes(3) + }) + + it('includes stored legacy workspace names for a workspace admin', async () => { + mockCheckWorkspaceAccess.mockResolvedValue({ + exists: true, + hasAccess: true, + canWrite: true, + canAdmin: true, + }) + queueTableRows(environment, [{ variables: {} }]) + queueTableRows(workspaceEnvironment, [ + { variables: { LEGACY_KEY: 'legacy-cipher', CURRENT_KEY: 'current-cipher' } }, + ]) + + await expect( + getEffectiveEnvironmentVariableNames('names-user', 'workspace-1') + ).resolves.toEqual(['CURRENT_KEY', 'LEGACY_KEY']) + expect(encryptionMockFns.mockDecryptSecret).not.toHaveBeenCalled() + }) +}) + +describe('resolveEffectiveEnvironmentVariables', () => { + beforeEach(() => { + vi.clearAllMocks() + resetDbChainMock() + invalidateEffectiveDecryptedEnvCache({ userId: 'resolver-user' }) + mockCheckWorkspaceAccess.mockResolvedValue({ + exists: true, + hasAccess: true, + canWrite: true, + canAdmin: false, + }) + mockGetAccessibleEnvCredentials.mockResolvedValue([]) + encryptionMockFns.mockDecryptSecret.mockReset() + }) + + it('decrypts only unique requested accessible values with workspace precedence', async () => { + mockGetAccessibleEnvCredentials.mockResolvedValue([ + { + type: 'env_workspace', + envKey: 'VISIBLE_SHARED', + envOwnerUserId: null, + updatedAt: new Date('2026-01-01T00:00:00.000Z'), + unredacted: true, + }, + { + type: 'env_workspace', + envKey: 'HIDDEN_SHARED', + envOwnerUserId: null, + updatedAt: new Date('2026-01-01T00:00:00.000Z'), + unredacted: false, + }, + { + type: 'env_workspace', + envKey: 'DUPLICATE', + envOwnerUserId: null, + updatedAt: new Date('2026-01-01T00:00:00.000Z'), + unredacted: false, + }, + { + type: 'env_workspace', + envKey: 'BROKEN', + envOwnerUserId: null, + updatedAt: new Date('2026-01-01T00:00:00.000Z'), + unredacted: false, + }, + { + type: 'env_personal', + envKey: 'SHARED_PERSONAL', + envOwnerUserId: 'owner-2', + updatedAt: new Date('2026-01-01T00:00:00.000Z'), + }, + ]) + queueTableRows(environment, [ + { + variables: { + OWN_PERSONAL: 'own-cipher', + DUPLICATE: 'personal-shadow-cipher', + UNREQUESTED_PERSONAL: 'unrequested-personal-cipher', + }, + }, + ]) + queueTableRows(workspaceEnvironment, [ + { + variables: { + VISIBLE_SHARED: 'visible-cipher', + HIDDEN_SHARED: 'hidden-cipher', + DUPLICATE: 'workspace-cipher', + BROKEN: 'broken-cipher', + INACCESSIBLE: 'inaccessible-cipher', + UNREQUESTED_WORKSPACE: 'unrequested-workspace-cipher', + }, + }, + ]) + queueTableRows(environment, [ + { userId: 'owner-2', variables: { SHARED_PERSONAL: 'shared-personal-cipher' } }, + ]) + encryptionMockFns.mockDecryptSecret.mockImplementation(async (encryptedValue: string) => { + if (encryptedValue === 'broken-cipher') throw new Error('cannot decrypt') + return { decrypted: `plain:${encryptedValue}` } + }) + + await expect( + resolveEffectiveEnvironmentVariables('resolver-user', 'workspace-1', [ + 'OWN_PERSONAL', + 'SHARED_PERSONAL', + 'VISIBLE_SHARED', + 'HIDDEN_SHARED', + 'DUPLICATE', + 'DUPLICATE', + 'BROKEN', + 'MISSING', + 'INACCESSIBLE', + 'constructor', + ]) + ).resolves.toEqual({ + OWN_PERSONAL: { + value: 'plain:own-cipher', + scope: 'personal', + visible: true, + }, + SHARED_PERSONAL: { + value: 'plain:shared-personal-cipher', + scope: 'personal', + visible: false, + }, + VISIBLE_SHARED: { + value: 'plain:visible-cipher', + scope: 'workspace', + visible: true, + }, + HIDDEN_SHARED: { + value: 'plain:hidden-cipher', + scope: 'workspace', + visible: false, + }, + DUPLICATE: { + value: 'plain:workspace-cipher', + scope: 'workspace', + visible: false, + }, + }) + expect(encryptionMockFns.mockDecryptSecret.mock.calls.map(([value]) => value)).toEqual([ + 'own-cipher', + 'shared-personal-cipher', + 'visible-cipher', + 'hidden-cipher', + 'workspace-cipher', + 'broken-cipher', + ]) + }) + + it('performs a fresh lookup without reading or warming the snapshot cache', async () => { + encryptionMockFns.mockDecryptSecret.mockImplementation(async (encryptedValue: string) => ({ + decrypted: `plain:${encryptedValue}`, + })) + + queueTableRows(environment, [{ variables: { ROTATING: 'first-cipher' } }]) + queueTableRows(workspaceEnvironment, [{ variables: {} }]) + await expect( + resolveEffectiveEnvironmentVariables('resolver-user', 'workspace-1', ['ROTATING']) + ).resolves.toEqual({ + ROTATING: { value: 'plain:first-cipher', scope: 'personal', visible: true }, + }) + + queueTableRows(environment, [{ variables: { ROTATING: 'snapshot-cipher' } }]) + queueTableRows(workspaceEnvironment, [{ variables: {} }]) + await expect( + getEffectiveEnvironmentSnapshot('resolver-user', 'workspace-1') + ).resolves.toMatchObject({ personalDecrypted: { ROTATING: 'plain:snapshot-cipher' } }) + + queueTableRows(environment, [{ variables: { ROTATING: 'fresh-cipher' } }]) + queueTableRows(workspaceEnvironment, [{ variables: {} }]) + await expect( + resolveEffectiveEnvironmentVariables('resolver-user', 'workspace-1', ['ROTATING']) + ).resolves.toEqual({ + ROTATING: { value: 'plain:fresh-cipher', scope: 'personal', visible: true }, + }) + + await expect( + getEffectiveEnvironmentSnapshot('resolver-user', 'workspace-1') + ).resolves.toMatchObject({ personalDecrypted: { ROTATING: 'plain:snapshot-cipher' } }) + expect(encryptionMockFns.mockDecryptSecret).toHaveBeenCalledTimes(3) + expect(mockCheckWorkspaceAccess).toHaveBeenCalledTimes(3) + }) +}) + describe('getPersonalAndWorkspaceEnv access filtering', () => { beforeEach(() => { vi.clearAllMocks() @@ -159,6 +449,7 @@ describe('getExecutionEnvironment', () => { vi.clearAllMocks() resetDbChainMock() mockGetAccessibleEnvCredentials.mockResolvedValue([]) + mockGetActivelyBannedUserIds.mockResolvedValue([]) encryptionMockFns.mockDecryptSecret.mockImplementation(async (encryptedValue: string) => ({ decrypted: `plain:${encryptedValue}`, })) @@ -177,15 +468,17 @@ describe('getExecutionEnvironment', () => { it('resolves each slice against its own identity', async () => { grantAdminTo('actor-1') /** - * Queued rows are FIFO per table, and the actor resolves first: its access was - * already decided, so it skips the `checkWorkspaceAccess` await the personal - * resolution still performs. Only the actor is a workspace admin, so the owner's - * own workspace slice resolves empty and could not be the one that lands. + * Queued rows are FIFO per table, and the personal slice resolves first because + * it is the first element of the implementation's `Promise.all` — both accesses + * are now decided up front and handed in, so neither resolution awaits before + * issuing its queries and the order is plain argument evaluation rather than a + * race between interleaved awaits. Only the actor is a workspace admin, so the + * owner's own workspace slice resolves empty and could not be the one that lands. */ - queueTableRows(environment, [{ variables: { ACTOR_ONLY: 'actor-cipher' } }]) - queueTableRows(workspaceEnvironment, [{ variables: { WORKSPACE_KEY: 'workspace-cipher' } }]) queueTableRows(environment, [{ variables: { PERSONAL_KEY: 'personal-cipher' } }]) queueTableRows(workspaceEnvironment, [{ variables: { WORKSPACE_KEY: 'workspace-cipher' } }]) + queueTableRows(environment, [{ variables: { ACTOR_ONLY: 'actor-cipher' } }]) + queueTableRows(workspaceEnvironment, [{ variables: { WORKSPACE_KEY: 'workspace-cipher' } }]) const snapshot = await getExecutionEnvironment('owner-1', 'actor-1', 'workspace-1') @@ -275,6 +568,148 @@ describe('getExecutionEnvironment', () => { expect(snapshot.personalDecrypted).toEqual({ PERSONAL_KEY: 'plain:personal-cipher' }) expect(snapshot.workspaceDecrypted).toEqual({ WORKSPACE_KEY: 'plain:workspace-cipher' }) }) + + /** + * A deployed chat, schedule, or webhook keeps running after the identity its + * personal-variable fallback points at leaves the workspace. That pointer is + * stored state, not a permission the run holds, so it must not fail the run + * before any block has started. + */ + it('resolves workspace variables only when the personal identity cannot reach the workspace', async () => { + mockCheckWorkspaceAccess.mockImplementation(async (_workspaceId: string, userId: string) => ({ + exists: true, + hasAccess: userId === 'actor-1', + canWrite: true, + canAdmin: true, + })) + queueTableRows(environment, [{ variables: { PERSONAL_KEY: 'personal-cipher' } }]) + queueTableRows(workspaceEnvironment, [{ variables: { WORKSPACE_KEY: 'workspace-cipher' } }]) + + const snapshot = await getExecutionEnvironment('departed-owner', 'actor-1', 'workspace-1') + + expect(snapshot.workspaceDecrypted).toEqual({ WORKSPACE_KEY: 'plain:workspace-cipher' }) + expect(snapshot.personalDecrypted).toEqual({}) + expect(snapshot.personalEncrypted).toEqual({}) + expect(snapshot.personalOwners).toEqual({}) + expect(snapshot.conflicts).toEqual([]) + }) + + /** The departed identity's own variables must not reach the run that dropped it. */ + /** + * Degrading must not widen the credential-group filter. The workspace slice is + * still selected by the actor's own grants and the actor's own admin flag — + * dropping the personal slice removes secrets, it never adds any. + */ + it('does not read the departed personal identity when resolving workspace variables only', async () => { + mockCheckWorkspaceAccess.mockImplementation(async (_workspaceId: string, userId: string) => ({ + exists: true, + hasAccess: userId === 'actor-1', + canWrite: true, + canAdmin: false, + })) + queueTableRows(environment, [{ variables: { ACTOR_ONLY: 'actor-cipher' } }]) + queueTableRows(workspaceEnvironment, [{ variables: { WORKSPACE_KEY: 'workspace-cipher' } }]) + + const snapshot = await getExecutionEnvironment('departed-owner', 'actor-1', 'workspace-1') + + expect(mockGetAccessibleEnvCredentials).toHaveBeenCalledOnce() + expect(mockGetAccessibleEnvCredentials).toHaveBeenCalledWith('workspace-1', 'actor-1', { + isWorkspaceAdmin: false, + }) + // No credential grant, and the actor is not an admin, so the workspace + // secret stays filtered out rather than falling through unfiltered. + expect(snapshot.workspaceDecrypted).toEqual({}) + }) + + /** + * Admission deliberately stops blocking runs on the personal-variable + * identity, so that a suspended member does not take down their teammates' + * schedules and webhooks. That must not become a way for a suspended account's + * own credentials to keep running — the run continues, their namespace does not. + */ + it('resolves workspace variables only when the personal identity is suspended', async () => { + grantAdminTo('actor-1') + mockGetActivelyBannedUserIds.mockResolvedValue(['suspended-owner']) + queueTableRows(environment, [{ variables: { OWNER_KEY: 'owner-cipher' } }]) + queueTableRows(workspaceEnvironment, [{ variables: { WORKSPACE_KEY: 'workspace-cipher' } }]) + + const snapshot = await getExecutionEnvironment('suspended-owner', 'actor-1', 'workspace-1') + + expect(mockGetActivelyBannedUserIds).toHaveBeenCalledWith(['suspended-owner']) + expect(snapshot.personalDecrypted).toEqual({}) + expect(snapshot.workspaceDecrypted).toEqual({ WORKSPACE_KEY: 'plain:workspace-cipher' }) + }) + + /** + * The arrangement that slipped past a split-path-only check: a custom-block + * publisher who is also their workspace's billing account makes both + * identities equal, taking the single-identity shortcut. That path has no + * admission gate at all — `admitCustomBlockChildExecution` checks usage limits + * and nothing else — so the suspension has to be enforced here. + */ + it('withholds the personal namespace when both identities are the same suspended user', async () => { + grantAdminTo('publisher-1') + mockGetActivelyBannedUserIds.mockResolvedValue(['publisher-1']) + queueTableRows(environment, [{ variables: { PUBLISHER_KEY: 'publisher-cipher' } }]) + queueTableRows(workspaceEnvironment, [{ variables: { WORKSPACE_KEY: 'workspace-cipher' } }]) + + const snapshot = await getExecutionEnvironment('publisher-1', 'publisher-1', 'workspace-1') + + expect(snapshot.personalDecrypted).toEqual({}) + expect(snapshot.workspaceDecrypted).toEqual({ WORKSPACE_KEY: 'plain:workspace-cipher' }) + }) + + /** A workspaceless run has no workspace slice either, so a suspended identity lends nothing. */ + it('resolves nothing personal for a suspended identity with no workspace', async () => { + mockGetActivelyBannedUserIds.mockResolvedValue(['suspended-1']) + queueTableRows(environment, [{ variables: { PERSONAL_KEY: 'personal-cipher' } }]) + + const snapshot = await getExecutionEnvironment('suspended-1', 'suspended-1', undefined) + + expect(snapshot.personalDecrypted).toEqual({}) + }) + + /** The actor is cleared by admission, so only the personal identity is looked up. */ + it('does not re-check the execution actor for a ban', async () => { + grantAdminTo('actor-1') + queueTableRows(environment, [{ variables: {} }]) + queueTableRows(workspaceEnvironment, [{ variables: {} }]) + queueTableRows(environment, [{ variables: {} }]) + queueTableRows(workspaceEnvironment, [{ variables: {} }]) + + await getExecutionEnvironment('owner-1', 'actor-1', 'workspace-1') + + expect(mockGetActivelyBannedUserIds).toHaveBeenCalledOnce() + expect(mockGetActivelyBannedUserIds.mock.calls[0][0]).not.toContain('actor-1') + }) + + /** With no reachable identity there is nobody to authorize the workspace slice against. */ + it('raises when neither identity can reach the workspace', async () => { + mockCheckWorkspaceAccess.mockResolvedValue({ + exists: true, + hasAccess: false, + canWrite: false, + canAdmin: false, + }) + + await expect( + getExecutionEnvironment('departed-owner', 'departed-payer', 'workspace-1') + ).rejects.toThrow('Access denied to workspace workspace-1') + }) + + /** A workspace that is gone is a different fact from one an identity may not read. */ + it('raises when the workspace no longer exists', async () => { + mockCheckWorkspaceAccess.mockResolvedValue({ + exists: false, + hasAccess: false, + canWrite: false, + canAdmin: false, + }) + + await expect(getExecutionEnvironment('owner-1', 'actor-1', 'workspace-1')).rejects.toThrow( + 'Workspace workspace-1 does not exist' + ) + }) }) describe('upsertWorkspaceEnvVars', () => { diff --git a/apps/sim/lib/environment/utils.ts b/apps/sim/lib/environment/utils.ts index 065d610c0b0..0c72b260fb2 100644 --- a/apps/sim/lib/environment/utils.ts +++ b/apps/sim/lib/environment/utils.ts @@ -2,10 +2,10 @@ import { AuditAction, AuditResourceType, recordAudit } from '@sim/audit' import { db } from '@sim/db' import { environment, workspaceEnvironment } from '@sim/db/schema' import { createLogger } from '@sim/logger' -import { getErrorMessage } from '@sim/utils/errors' import { generateId } from '@sim/utils/id' import { eq, inArray } from 'drizzle-orm' import { LRUCache } from 'lru-cache' +import { getActivelyBannedUserIds } from '@/lib/auth/ban' import { decryptSecret, encryptSecret } from '@/lib/core/security/encryption' import { lockPersonalEnvMap, lockWorkspaceEnvMap } from '@/lib/credentials/env-locks' import { @@ -143,11 +143,25 @@ export async function getEnvironmentVariableKeys(userId: string): Promise<{ } } -export async function getPersonalAndWorkspaceEnv( +interface AccessibleEncryptedEnvironment { + personalEncrypted: Record + workspaceEncrypted: Record + personalOwners: Record + workspaceUnredactedKeys: string[] +} + +/** + * Loads only the encrypted environment slices the caller may use. + * + * Keeping this before decryption gives name-only consumers the exact same workspace, + * credential, shared-personal precedence, and stored-value checks as runtime resolution + * without exposing plaintext or touching the decrypted snapshot cache. + */ +async function loadAccessibleEncryptedEnvironment( userId: string, workspaceId?: string, options?: { workspaceAccess?: WorkspaceAccess } -): Promise { +): Promise { let workspaceCanAdmin = false if (workspaceId) { const access = options?.workspaceAccess ?? (await checkWorkspaceAccess(workspaceId, userId)) @@ -247,6 +261,103 @@ export async function getPersonalAndWorkspaceEnv( ) } + return { + personalEncrypted, + workspaceEncrypted, + personalOwners, + workspaceUnredactedKeys: accessibleEnvCredentials + .filter((row) => row.type === 'env_workspace' && row.unredacted) + .map((row) => row.envKey), + } +} + +/** + * Lists the effective environment names visible to a caller without decrypting values. + * This deliberately performs a fresh ACL-aware encrypted lookup instead of populating or + * reading the short-lived decrypted environment snapshot cache. + */ +export async function getEffectiveEnvironmentVariableNames( + userId: string, + workspaceId?: string +): Promise { + const { personalEncrypted, workspaceEncrypted } = await loadAccessibleEncryptedEnvironment( + userId, + workspaceId + ) + return [ + ...new Set([...Object.keys(personalEncrypted), ...Object.keys(workspaceEncrypted)]), + ].sort() +} + +export interface ResolvedEnvironmentVariable { + value: string + scope: 'personal' | 'workspace' + visible: boolean +} + +/** + * Resolves only the requested environment variables through a fresh ACL-aware lookup. + * + * This deliberately neither reads nor populates the runtime environment snapshot cache. + * Workspace values take precedence over personal values, matching normal resolution. Missing, + * inaccessible, and undecryptable values are all omitted so callers cannot distinguish them. + */ +export async function resolveEffectiveEnvironmentVariables( + userId: string, + workspaceId: string | undefined, + requestedNames: readonly string[] +): Promise> { + const names = [...new Set(requestedNames)] + if (names.length === 0) return {} + + const { personalEncrypted, workspaceEncrypted, personalOwners, workspaceUnredactedKeys } = + await loadAccessibleEncryptedEnvironment(userId, workspaceId) + const visibleWorkspaceNames = new Set(workspaceUnredactedKeys) + + const resolvedEntries = await Promise.all( + names.map(async (name) => { + const fromWorkspace = Object.hasOwn(workspaceEncrypted, name) + const fromPersonal = Object.hasOwn(personalEncrypted, name) + const encrypted = fromWorkspace + ? workspaceEncrypted[name] + : fromPersonal + ? personalEncrypted[name] + : undefined + if (encrypted === undefined) return null + + try { + const { decrypted } = await decryptSecret(encrypted) + return [ + name, + { + value: decrypted, + scope: fromWorkspace ? 'workspace' : 'personal', + visible: fromWorkspace + ? visibleWorkspaceNames.has(name) + : personalOwners[name] === userId, + }, + ] as const + } catch { + return null + } + }) + ) + + return Object.fromEntries( + resolvedEntries.filter( + (entry): entry is readonly [string, ResolvedEnvironmentVariable] => entry !== null + ) + ) +} + +export async function getPersonalAndWorkspaceEnv( + userId: string, + workspaceId?: string, + options?: { workspaceAccess?: WorkspaceAccess } +): Promise { + const { personalEncrypted, workspaceEncrypted, personalOwners, workspaceUnredactedKeys } = + await loadAccessibleEncryptedEnvironment(userId, workspaceId, options) + const decryptionFailures: string[] = [] const decryptAll = async (src: Record, source: 'personal' | 'workspace') => { @@ -256,12 +367,11 @@ export async function getPersonalAndWorkspaceEnv( try { const { decrypted } = await decryptSecret(v) return [k, decrypted] as const - } catch (error) { - logger.error(`Failed to decrypt ${source} environment variable "${k}"`, { + } catch { + logger.error('Failed to decrypt environment variable', { userId, workspaceId, source, - error: getErrorMessage(error, 'Unknown error'), }) decryptionFailures.push(k) return [k, ''] as const @@ -282,7 +392,6 @@ export async function getPersonalAndWorkspaceEnv( logger.warn('Some environment variables failed to decrypt', { userId, workspaceId, - failedKeys: decryptionFailures, failedCount: decryptionFailures.length, }) } @@ -295,9 +404,31 @@ export async function getPersonalAndWorkspaceEnv( personalOwners, conflicts, decryptionFailures, - workspaceUnredactedKeys: accessibleEnvCredentials - .filter((row) => row.type === 'env_workspace' && row.unredacted) - .map((row) => row.envKey), + workspaceUnredactedKeys, + } +} + +/** + * Keeps only the workspace slice of a snapshot resolved for a single identity. + * + * Used wherever a run has no personal namespace to lend, so the identity that + * authorized the workspace variables cannot leak its own personal ones in + * alongside them. `conflicts` is empty by construction once the personal slice + * is, and a decryption failure is only carried over when it belongs to the slice + * being kept. + */ +function toWorkspaceOnlySnapshot( + snapshot: EnvironmentResolutionSnapshot +): EnvironmentResolutionSnapshot { + return { + ...snapshot, + personalEncrypted: {}, + personalDecrypted: {}, + personalOwners: {}, + conflicts: [], + decryptionFailures: snapshot.decryptionFailures.filter( + (key) => key in snapshot.workspaceEncrypted + ), } } @@ -318,15 +449,34 @@ export async function getPersonalAndWorkspaceEnv( * A run whose two identities coincide, which is every interactive run, resolves * exactly as before through a single query. * - * When the actor has no access to the workspace at all, the personal identity is - * reused for both slices and the fault is reported rather than raised. - * `workspace.billedAccountUserId` is a stored column rather than a derivation, - * so an organization ownership transfer can leave it pointing at a user with no - * remaining access; failing here would take down every background execution in - * that workspace for a misconfiguration the run itself did not cause. The error - * line is what makes that state visible while it is repaired. + * Neither identity is a permission the run holds — both are stored pointers that + * outlive the access that made them valid, so a stale one is reported rather than + * raised. `workspace.billedAccountUserId` is a stored column rather than a + * derivation, so an ownership transfer can leave the actor pointing at a user with + * no remaining access; `workflow.userId` is likewise a stored pointer that + * member-removal repairs on the paths it knows about. Failing on either would take + * down every background execution in the workspace for a misconfiguration the run + * itself did not cause. The error lines are what make that state visible while it + * is repaired. + * + * The two stale cases degrade differently because the identities mean different + * things. An actor that cannot reach the workspace leaves the owner as the only + * identity to authorize the workspace slice against, so the run falls back to + * resolving both slices as the owner. A personal identity that cannot reach the + * workspace is no longer someone whose private namespace it is reasonable to lend + * — the same judgment already applied to an anonymous public-API call — so the run + * keeps the actor's workspace slice and resolves no personal variables at all. + * Continuing to lend a removed member's personal secrets to their former + * organization's background runs is the outcome to avoid, not the one to preserve. + * A reference to a variable that is no longer resolvable survives as its literal + * `{{NAME}}` and fails at the block that needs it, which names the missing + * variable instead of failing the run before any block has started. + * + * With no reachable identity on either side there is nobody to authorize the + * workspace slice against, and a filtered selection cannot be computed, so that + * case still raises. * - * That fallback is gated on the access decision alone, never on a failed query. + * These fallbacks are gated on the access decision alone, never on a failed query. * Widening to a `catch` would let a transient database fault silently promote the * run to the owner's broader secret selection, which is the opposite of what an * infrastructure error should do — those propagate and fail the run. @@ -337,35 +487,87 @@ export async function getExecutionEnvironment( workspaceId?: string ): Promise { if (personalUserId === undefined) { - const workspaceOnly = await getPersonalAndWorkspaceEnv(workspaceUserId, workspaceId) - return { - ...workspaceOnly, - personalEncrypted: {}, - personalDecrypted: {}, - personalOwners: {}, - conflicts: [], - decryptionFailures: workspaceOnly.decryptionFailures.filter( - (key) => key in workspaceOnly.workspaceEncrypted - ), - } + return toWorkspaceOnlySnapshot(await getPersonalAndWorkspaceEnv(workspaceUserId, workspaceId)) + } + + /** + * A suspended account lends nothing, from any path. + * + * Checked before the single-identity shortcut below rather than alongside the + * access lookups, because "the caller already cleared this identity" does not + * hold everywhere: a custom-block child is admitted by + * `admitCustomBlockChildExecution`, which checks usage limits and nothing + * else, and a provider URL-validation challenge resolves with no admission at + * all. Behind the shortcut, a publisher who is also their workspace's billing + * account made both identities equal and skipped the gate entirely — the one + * arrangement where suspension was silently ignored. + * + * Only the personal namespace is withheld. Workspace variables belong to the + * workspace rather than to a person, so they keep resolving and the runs a + * suspended member's teammates depend on keep working — which is the whole + * reason admission stopped blocking on this identity in the first place. + */ + if ((await getActivelyBannedUserIds([personalUserId])).length > 0) { + logger.error('Personal-environment identity is suspended; resolving workspace variables only', { + personalUserId, + workspaceUserId, + workspaceId, + }) + return toWorkspaceOnlySnapshot(await getPersonalAndWorkspaceEnv(workspaceUserId, workspaceId)) } if (!workspaceId || workspaceUserId === personalUserId) { return getPersonalAndWorkspaceEnv(personalUserId, workspaceId) } - const actorAccess = await checkWorkspaceAccess(workspaceId, workspaceUserId) + const [actorAccess, personalAccess] = await Promise.all([ + checkWorkspaceAccess(workspaceId, workspaceUserId), + checkWorkspaceAccess(workspaceId, personalUserId), + ]) + + /** + * A workspace that no longer exists and one an identity may not read are + * different facts, exactly as in {@link getPersonalAndWorkspaceEnv}. Only the + * second is a stale pointer worth degrading for. + */ + if (!personalAccess.exists) { + throw new Error(`Workspace ${workspaceId} does not exist`) + } + + if (!personalAccess.hasAccess) { + if (!actorAccess.hasAccess) { + logger.error('Neither execution identity can reach the workspace', { + personalUserId, + workspaceUserId, + workspaceId, + }) + throw new Error(`Access denied to workspace ${workspaceId}`) + } + + logger.error( + 'Personal-environment identity cannot reach the workspace; resolving workspace variables only', + { personalUserId, workspaceUserId, workspaceId } + ) + return toWorkspaceOnlySnapshot( + await getPersonalAndWorkspaceEnv(workspaceUserId, workspaceId, { + workspaceAccess: actorAccess, + }) + ) + } + if (!actorAccess.hasAccess) { logger.error('Execution actor cannot reach the workspace; falling back to the owner', { personalUserId, workspaceUserId, workspaceId, }) - return getPersonalAndWorkspaceEnv(personalUserId, workspaceId) + return getPersonalAndWorkspaceEnv(personalUserId, workspaceId, { + workspaceAccess: personalAccess, + }) } const [personal, actor] = await Promise.all([ - getPersonalAndWorkspaceEnv(personalUserId, workspaceId), + getPersonalAndWorkspaceEnv(personalUserId, workspaceId, { workspaceAccess: personalAccess }), getPersonalAndWorkspaceEnv(workspaceUserId, workspaceId, { workspaceAccess: actorAccess }), ]) diff --git a/apps/sim/lib/execution/cancel-workflow-execution.test.ts b/apps/sim/lib/execution/cancel-workflow-execution.test.ts index 1f51da65463..ee028cc23fc 100644 --- a/apps/sim/lib/execution/cancel-workflow-execution.test.ts +++ b/apps/sim/lib/execution/cancel-workflow-execution.test.ts @@ -1,637 +1,1840 @@ /** * @vitest-environment node */ + +import { databaseMock, dbChainMockFns, resetDbChainMock } from '@sim/testing' import { beforeEach, describe, expect, it, vi } from 'vitest' const { + mockMarkExecutionCancelled, + mockClearExecutionCancellation, mockAbortManualExecution, mockBeginPausedCancellation, - mockBlockQueuedResumes, - mockCancelWorkflowGroupExecution, - mockCaptureServerEvent, + mockStagePausedCancellation, + mockBlockQueuedResumesForCancellation, mockClearPausedCancellationIntent, mockCompletePausedCancellation, - mockGetJobQueue, + mockFinalizePausedCancellationForTerminalRun, mockGetPausedCancellationStatus, - mockMarkExecutionCancelled, - mockPublishWorkflowGroupCancellationEvent, + mockGetActiveResumeCancellationTarget, + mockGetActiveResumeCancellationTargets, + mockRollbackActiveResumeCancellation, + mockFinalizeExecutionStream, + mockReadExecutionMetaState, + mockWriteEvent, + mockWriteTerminalEvent, + mockCancelByExecution, + mockGetJobQueue, mockReleaseExecutionSlot, - mockUpdateSet, - mockUpdateReturning, - mockResolveWorkflowExecutionOwnership, + mockCancelWorkflowGroupExecution, + mockPublishWorkflowGroupCancellationEvent, } = vi.hoisted(() => ({ + mockMarkExecutionCancelled: vi.fn(), + mockClearExecutionCancellation: vi.fn(), mockAbortManualExecution: vi.fn(), mockBeginPausedCancellation: vi.fn(), - mockBlockQueuedResumes: vi.fn(), - mockCancelWorkflowGroupExecution: vi.fn(), - mockCaptureServerEvent: vi.fn(), + mockStagePausedCancellation: vi.fn(), + mockBlockQueuedResumesForCancellation: vi.fn(), mockClearPausedCancellationIntent: vi.fn(), mockCompletePausedCancellation: vi.fn(), - mockGetJobQueue: vi.fn(), + mockFinalizePausedCancellationForTerminalRun: vi.fn(), mockGetPausedCancellationStatus: vi.fn(), - mockMarkExecutionCancelled: vi.fn(), - mockPublishWorkflowGroupCancellationEvent: vi.fn(), + mockGetActiveResumeCancellationTarget: vi.fn(), + mockGetActiveResumeCancellationTargets: vi.fn(), + mockRollbackActiveResumeCancellation: vi.fn(), + mockFinalizeExecutionStream: vi.fn(), + mockReadExecutionMetaState: vi.fn(), + mockWriteEvent: vi.fn(), + mockWriteTerminalEvent: vi.fn(), + mockCancelByExecution: vi.fn(), + mockGetJobQueue: vi.fn(), mockReleaseExecutionSlot: vi.fn(), - mockUpdateSet: vi.fn(), - mockUpdateReturning: vi.fn(), - mockResolveWorkflowExecutionOwnership: vi.fn(), + mockCancelWorkflowGroupExecution: vi.fn(), + mockPublishWorkflowGroupCancellationEvent: vi.fn(), })) -vi.mock('@sim/db', () => ({ - db: { - update: () => ({ - set: (values: unknown) => { - mockUpdateSet(values) - return { where: () => ({ returning: () => Promise.resolve(mockUpdateReturning()) }) } - }, - }), - }, +vi.mock('@/lib/core/async-jobs', () => ({ + getJobQueue: mockGetJobQueue, })) vi.mock('@/lib/billing/calculations/usage-reservation', () => ({ releaseExecutionSlot: mockReleaseExecutionSlot, })) -vi.mock('@/lib/table/workflow-group-cancellation', () => ({ - cancelWorkflowGroupExecution: mockCancelWorkflowGroupExecution, - publishWorkflowGroupCancellationEvent: mockPublishWorkflowGroupCancellationEvent, -})) - vi.mock('@/lib/execution/cancellation', () => ({ - markExecutionCancelled: mockMarkExecutionCancelled, + markExecutionCancelled: (...args: unknown[]) => mockMarkExecutionCancelled(...args), + clearExecutionCancellation: (...args: unknown[]) => mockClearExecutionCancellation(...args), })) vi.mock('@/lib/execution/manual-cancellation', () => ({ - abortManualExecution: mockAbortManualExecution, + abortManualExecution: (...args: unknown[]) => mockAbortManualExecution(...args), })) -vi.mock('@/lib/execution/event-buffer', () => ({ - createExecutionEventWriter: () => ({ - writeTerminal: vi.fn().mockResolvedValue(undefined), - close: vi.fn().mockResolvedValue(undefined), - }), - readExecutionMetaState: vi.fn().mockResolvedValue({ status: 'missing' }), +vi.mock('@/lib/workflows/executor/human-in-the-loop-manager', () => ({ + PauseResumeManager: { + beginPausedCancellation: (...args: unknown[]) => mockBeginPausedCancellation(...args), + stagePausedCancellation: (...args: unknown[]) => mockStagePausedCancellation(...args), + blockQueuedResumesForCancellation: (...args: unknown[]) => + mockBlockQueuedResumesForCancellation(...args), + clearPausedCancellationIntent: (...args: unknown[]) => + mockClearPausedCancellationIntent(...args), + completePausedCancellation: (...args: unknown[]) => mockCompletePausedCancellation(...args), + finalizePausedCancellationForTerminalRun: (...args: unknown[]) => + mockFinalizePausedCancellationForTerminalRun(...args), + getPausedCancellationStatus: (...args: unknown[]) => mockGetPausedCancellationStatus(...args), + getActiveResumeCancellationTarget: (...args: unknown[]) => + mockGetActiveResumeCancellationTarget(...args), + getActiveResumeCancellationTargets: (...args: unknown[]) => + mockGetActiveResumeCancellationTargets(...args), + rollbackActiveResumeCancellation: (...args: unknown[]) => + mockRollbackActiveResumeCancellation(...args), + }, })) -vi.mock('@/lib/core/async-jobs', () => ({ - getJobQueue: mockGetJobQueue, +vi.mock('@/lib/table/workflow-group-cancellation', () => ({ + cancelWorkflowGroupExecution: (...args: unknown[]) => mockCancelWorkflowGroupExecution(...args), + publishWorkflowGroupCancellationEvent: (...args: unknown[]) => + mockPublishWorkflowGroupCancellationEvent(...args), })) -vi.mock('@/lib/posthog/server', () => ({ - captureServerEvent: mockCaptureServerEvent, +vi.mock('@/lib/execution/event-buffer', () => ({ + finalizeExecutionStream: (...args: unknown[]) => mockFinalizeExecutionStream(...args), + readExecutionMetaState: (...args: unknown[]) => mockReadExecutionMetaState(...args), + createExecutionEventWriter: () => ({ + write: (...args: unknown[]) => mockWriteEvent(...args), + writeTerminal: (...args: unknown[]) => mockWriteTerminalEvent(...args), + close: vi.fn().mockResolvedValue(undefined), + }), })) -vi.mock('@/lib/workflows/executor/execution-job-ids', () => ({ - WORKFLOW_EXECUTION_JOB_ID_PREFIX: 'workflow-execution:', -})) +import { cancelWorkflowExecutionContract } from '@/lib/api/contracts/workflows' +import { OrchestrationError, statusForOrchestrationError } from '@/lib/core/orchestration/types' +import { + type CancelWorkflowExecutionInput, + cancelWorkflowExecution, + WorkflowExecutionNotFoundError, +} from '@/lib/execution/cancel-workflow-execution' +import { WorkflowRunAlreadyTerminalError } from '@/lib/execution/workflow-run-already-terminal-error' + +const INPUT: CancelWorkflowExecutionInput = { + workflowId: 'wf-1', + executionId: 'ex-1', + workspaceId: 'workspace-1', + attributedUserId: 'user-1', +} -vi.mock('@/lib/workflows/executor/execution-queries', () => ({ - resolveWorkflowExecutionOwnership: mockResolveWorkflowExecutionOwnership, -})) +async function cancelAsResponse( + overrides: Partial = {} +): Promise { + try { + const result = await cancelWorkflowExecution({ ...INPUT, ...overrides }) + const body = cancelWorkflowExecutionContract.response.schema.parse(result) + return Response.json(body) + } catch (error) { + if (error instanceof WorkflowExecutionNotFoundError) { + return Response.json({ error: error.message }, { status: 404 }) + } + if (error instanceof OrchestrationError) { + return Response.json( + { error: error.message }, + { status: statusForOrchestrationError(error.code) } + ) + } + if (error instanceof Error) { + return Response.json({ error: error.message }, { status: 500 }) + } + throw error + } +} -vi.mock('@/lib/workflows/executor/human-in-the-loop-manager', () => ({ - PauseResumeManager: { - beginPausedCancellation: mockBeginPausedCancellation, - getPausedCancellationStatus: mockGetPausedCancellationStatus, - blockQueuedResumesForCancellation: mockBlockQueuedResumes, - clearPausedCancellationIntent: mockClearPausedCancellationIntent, - completePausedCancellation: mockCompletePausedCancellation, - }, -})) +const POST = async (..._args: unknown[]) => cancelAsResponse() +const makeRequest = () => undefined +const makeParams = () => undefined -import { cancelWorkflowExecution } from '@/lib/execution/cancel-workflow-execution' +const ACTIVE_RESUME_TARGET = { + resumeEntryId: 'resume-entry-1', + pausedExecutionId: 'paused-1', + parentExecutionId: 'ex-1', + resumeExecutionId: 'resume-ex-1', +} -/** - * The durable writes a workflow-group transition reports back. The transaction - * updates the workflow log only, the cell sidecar only, or both, so a single - * `kind` cannot answer whether this request wrote anything. - */ -const NO_WRITES = { workflowLogTerminalized: false, sidecarCancelled: false } as const -const LOG_WRITE = { workflowLogTerminalized: true, sidecarCancelled: false } as const -const SIDECAR_WRITE = { workflowLogTerminalized: false, sidecarCancelled: true } as const -const BOTH_WRITES = { workflowLogTerminalized: true, sidecarCancelled: true } as const - -const INPUT = { - executionId: 'execution-1', - workflowId: 'workflow-1', - userId: 'user-1', - workspaceId: 'workspace-1', +const REPLACEMENT_ACTIVE_RESUME_TARGET = { + ...ACTIVE_RESUME_TARGET, + resumeEntryId: 'resume-entry-2', + resumeExecutionId: 'resume-ex-2', } describe('cancelWorkflowExecution', () => { beforeEach(() => { vi.clearAllMocks() - mockResolveWorkflowExecutionOwnership.mockResolvedValue({ - belongsToWorkflow: true, - workflowGroupWorkspaceId: null, - priorStatus: 'running', - }) - mockUpdateReturning.mockReturnValue([{ id: 'log-1' }]) - mockBeginPausedCancellation.mockResolvedValue(false) - mockGetPausedCancellationStatus.mockResolvedValue(null) + resetDbChainMock() + dbChainMockFns.limit.mockResolvedValue([ + { + executionDeadlineAt: null, + executionOrigin: null, + status: 'running', + workspaceId: 'workspace-1', + }, + ]) + dbChainMockFns.returning.mockResolvedValue([{ status: 'cancelled' }]) + mockCancelByExecution.mockReset().mockResolvedValue(0) + mockGetJobQueue.mockReset().mockResolvedValue({ cancelByExecution: mockCancelByExecution }) + mockReleaseExecutionSlot.mockReset().mockResolvedValue(undefined) + mockCancelWorkflowGroupExecution.mockReset().mockResolvedValue({ kind: 'not_workflow_group' }) + mockPublishWorkflowGroupCancellationEvent.mockReset().mockResolvedValue(undefined) + mockClearExecutionCancellation.mockReset().mockResolvedValue(undefined) + mockMarkExecutionCancelled + .mockReset() + .mockResolvedValue({ durablyRecorded: false, reason: 'redis_unavailable' }) + mockAbortManualExecution.mockReset().mockReturnValue(false) + mockBeginPausedCancellation.mockReset().mockResolvedValue(false) + mockStagePausedCancellation.mockReset().mockResolvedValue({ kind: 'not_paused' }) + mockBlockQueuedResumesForCancellation.mockReset().mockResolvedValue(false) + mockClearPausedCancellationIntent.mockReset().mockResolvedValue(undefined) + mockCompletePausedCancellation.mockReset().mockResolvedValue(false) + mockFinalizePausedCancellationForTerminalRun.mockReset().mockResolvedValue(true) + mockGetPausedCancellationStatus.mockReset().mockResolvedValue(null) + mockGetActiveResumeCancellationTarget.mockReset().mockResolvedValue(null) + mockGetActiveResumeCancellationTargets.mockReset().mockResolvedValue([]) + mockRollbackActiveResumeCancellation.mockReset().mockResolvedValue(true) + mockFinalizeExecutionStream.mockReset().mockResolvedValue(true) + mockReadExecutionMetaState.mockReset().mockResolvedValue({ status: 'missing' }) + mockWriteEvent.mockReset().mockResolvedValue({ eventId: 1 }) + mockWriteTerminalEvent.mockReset().mockResolvedValue({ eventId: 1 }) + }) + + it('returns success when cancellation was durably recorded', async () => { + mockMarkExecutionCancelled.mockResolvedValue({ + durablyRecorded: true, + reason: 'recorded', + }) + + const response = await POST(makeRequest(), makeParams()) + + expect(response.status).toBe(200) + await expect(response.json()).resolves.toEqual({ + success: true, + executionId: 'ex-1', + redisAvailable: true, + durablyRecorded: true, + locallyAborted: false, + pausedCancelled: false, + reason: 'recorded', + }) + expect(mockCancelByExecution).toHaveBeenCalledWith( + { + workflowId: 'wf-1', + executionId: 'ex-1', + }, + 'standalone' + ) + expect(mockMarkExecutionCancelled).toHaveBeenCalledWith('ex-1', { + executionDeadlineAt: null, + }) + expect(mockClearExecutionCancellation).not.toHaveBeenCalled() + }) + + it('atomically claims one workflow-group attempt before signalling it', async () => { + dbChainMockFns.limit.mockResolvedValueOnce([ + { + executionDeadlineAt: null, + executionOrigin: 'workflow_group', + status: 'running', + workspaceId: 'workspace-1', + }, + ]) + mockCancelWorkflowGroupExecution.mockResolvedValue({ + kind: 'cancelled', + tableId: 'table-1', + rowId: 'row-1', + groupId: 'group-1', + }) + mockMarkExecutionCancelled.mockResolvedValue({ + durablyRecorded: true, + reason: 'recorded', + }) + + const response = await POST(makeRequest(), makeParams()) + + expect(response.status).toBe(200) + await expect(response.json()).resolves.toMatchObject({ + success: true, + executionId: 'ex-1', + redisAvailable: true, + durablyRecorded: true, + reason: 'recorded', + }) + expect(mockCancelWorkflowGroupExecution).toHaveBeenCalledWith({ + workspaceId: 'workspace-1', + workflowId: 'wf-1', + executionId: 'ex-1', + }) + expect(mockCancelWorkflowGroupExecution.mock.invocationCallOrder[0]).toBeLessThan( + mockMarkExecutionCancelled.mock.invocationCallOrder[0] + ) + expect(mockMarkExecutionCancelled.mock.invocationCallOrder[0]).toBeLessThan( + mockPublishWorkflowGroupCancellationEvent.mock.invocationCallOrder[0] + ) + expect(mockCancelByExecution).not.toHaveBeenCalled() + expect(mockStagePausedCancellation).toHaveBeenCalledWith('ex-1', 'wf-1') + expect(mockClearPausedCancellationIntent).not.toHaveBeenCalled() + }) + + it('keeps a claimed group cancellation retryable when signalling fails', async () => { + dbChainMockFns.limit.mockResolvedValueOnce([ + { + executionDeadlineAt: null, + executionOrigin: 'workflow_group', + status: 'running', + workspaceId: 'workspace-1', + }, + ]) + mockCancelWorkflowGroupExecution.mockResolvedValue({ + kind: 'cancelled', + tableId: 'table-1', + rowId: 'row-1', + groupId: 'group-1', + }) + mockMarkExecutionCancelled.mockResolvedValue({ + durablyRecorded: false, + reason: 'redis_unavailable', + }) + + const response = await POST(makeRequest(), makeParams()) + + expect(response.status).toBe(200) + await expect(response.json()).resolves.toMatchObject({ + success: false, + durablyRecorded: false, + reason: 'redis_unavailable', + }) + expect(mockMarkExecutionCancelled).toHaveBeenCalledOnce() + expect(mockCancelWorkflowGroupExecution).toHaveBeenCalledOnce() + expect(mockPublishWorkflowGroupCancellationEvent).not.toHaveBeenCalled() + expect(mockReleaseExecutionSlot).not.toHaveBeenCalled() + expect(mockCancelByExecution).not.toHaveBeenCalled() + }) + + it('clears a staged pause when a vanished group target prevents active-resume rollback', async () => { + dbChainMockFns.limit.mockResolvedValueOnce([ + { + executionDeadlineAt: null, + executionOrigin: 'workflow_group', + status: 'running', + workspaceId: 'workspace-1', + }, + ]) + mockStagePausedCancellation.mockResolvedValue({ + kind: 'active_resume', + target: ACTIVE_RESUME_TARGET, + }) mockMarkExecutionCancelled.mockResolvedValue({ durablyRecorded: true, reason: 'recorded' }) - mockAbortManualExecution.mockReturnValue(false) - mockBlockQueuedResumes.mockResolvedValue(undefined) - mockClearPausedCancellationIntent.mockResolvedValue(undefined) + mockRollbackActiveResumeCancellation.mockResolvedValue(false) + + const response = await POST(makeRequest(), makeParams()) + + expect(response.status).toBe(409) + await expect(response.json()).resolves.toEqual({ + error: 'Workflow group execution is no longer the active table execution', + }) + expect(mockRollbackActiveResumeCancellation).toHaveBeenCalledWith( + 'ex-1', + 'wf-1', + 'resume-entry-1' + ) + expect(mockClearPausedCancellationIntent).toHaveBeenCalledWith('ex-1', 'wf-1') + }) + + it('accepts an exact in-process group abort without cancelling its carrier', async () => { + dbChainMockFns.limit.mockResolvedValueOnce([ + { + executionDeadlineAt: null, + executionOrigin: 'workflow_group', + status: 'running', + workspaceId: 'workspace-1', + }, + ]) + mockMarkExecutionCancelled.mockResolvedValue({ + durablyRecorded: false, + reason: 'redis_unavailable', + }) + mockAbortManualExecution.mockReturnValue(true) + mockCancelWorkflowGroupExecution.mockResolvedValue({ + kind: 'cancelled', + tableId: 'table-1', + rowId: 'row-1', + groupId: 'group-1', + }) mockCompletePausedCancellation.mockResolvedValue(true) - mockReleaseExecutionSlot.mockResolvedValue(undefined) - mockPublishWorkflowGroupCancellationEvent.mockResolvedValue(undefined) - mockGetJobQueue.mockResolvedValue({ - getJob: vi.fn().mockResolvedValue(null), - cancelJob: vi.fn(), + + const response = await POST(makeRequest(), makeParams()) + + expect(response.status).toBe(200) + await expect(response.json()).resolves.toMatchObject({ + success: true, + durablyRecorded: false, + locallyAborted: true, + reason: 'redis_unavailable', }) + expect(mockCancelWorkflowGroupExecution).toHaveBeenCalledOnce() + expect(mockCancelByExecution).not.toHaveBeenCalled() }) - /** - * The row reads `cancelled` after a successful cancel just as it does after - * someone else's, so a status re-read alone would report this run's own work - * as `already_cancelled`. Nothing is re-read once the claim moved a row. - */ - it('reports a durable write when an active run is cancelled', async () => { - mockResolveWorkflowExecutionOwnership - .mockResolvedValueOnce({ - belongsToWorkflow: true, - workflowGroupWorkspaceId: null, - priorStatus: 'running', - }) - .mockResolvedValue({ - belongsToWorkflow: true, - workflowGroupWorkspaceId: null, - priorStatus: 'cancelled', - }) + it('does not cancel a shared carrier when an idle workflow-group pause appears', async () => { + dbChainMockFns.limit.mockResolvedValueOnce([ + { + executionDeadlineAt: null, + executionOrigin: 'workflow_group', + status: 'running', + workspaceId: 'workspace-1', + }, + ]) + mockMarkExecutionCancelled.mockResolvedValue({ durablyRecorded: true, reason: 'recorded' }) + mockCancelWorkflowGroupExecution.mockResolvedValue({ + kind: 'cancelled', + tableId: 'table-1', + rowId: 'row-1', + groupId: 'group-1', + }) + mockStagePausedCancellation + .mockResolvedValueOnce({ kind: 'not_paused' }) + .mockResolvedValue({ kind: 'idle' }) + mockCompletePausedCancellation.mockResolvedValue(true) - const result = await cancelWorkflowExecution(INPUT) - - expect(result).toMatchObject({ success: true, durablyRecorded: true, reason: 'recorded' }) - expect(mockResolveWorkflowExecutionOwnership).toHaveBeenCalledTimes(1) - }) - - /** - * A cancel against a run that already reached a terminal state changes - * nothing: the log claim's `status = 'running'` predicate matches no row and - * no terminal metadata moves. Reporting `recorded`/`durablyRecorded: true` - * there tells a caller a durable write happened when none did, so the outcome - * names the state that was actually observed instead. - */ - it.each([ - ['cancelled', 'already_cancelled'], - ['completed', 'already_completed'], - ['failed', 'already_failed'], - ])('reports a run already %s as a no-op rather than a durable write', async (status, reason) => { - mockResolveWorkflowExecutionOwnership.mockResolvedValue({ - belongsToWorkflow: true, - workflowGroupWorkspaceId: null, - priorStatus: status, - }) - mockUpdateReturning.mockReturnValue([]) - - const result = await cancelWorkflowExecution(INPUT) - - expect(result).toMatchObject({ success: true, durablyRecorded: false, reason }) - }) - - /** - * Reclassification only ever applies to an otherwise-clean outcome. A run that - * reached any terminal status can still carry paused-HITL state — a - * force-failed run keeps whatever pause rows it had — and when reconciling - * that genuinely fails, the caller is owed the step that failed rather than a - * no-op that also flips `success` to `true`. - */ - it.each([['cancelled'], ['completed'], ['failed']])( - 'still reports the failing step when a %s run has paused work left over', - async (status) => { - mockResolveWorkflowExecutionOwnership.mockResolvedValue({ - belongsToWorkflow: true, - workflowGroupWorkspaceId: null, - priorStatus: status, - }) - mockBeginPausedCancellation.mockResolvedValue(true) - mockCompletePausedCancellation.mockResolvedValue(false) + const response = await POST(makeRequest(), makeParams()) - const result = await cancelWorkflowExecution(INPUT) + expect(response.status).toBe(200) + await expect(response.json()).resolves.toMatchObject({ + success: true, + pausedCancelled: true, + reason: 'recorded', + }) + expect(mockCancelByExecution).not.toHaveBeenCalled() + expect(mockStagePausedCancellation.mock.invocationCallOrder[1]).toBeLessThan( + mockWriteTerminalEvent.mock.invocationCallOrder[0] + ) + expect(mockCompletePausedCancellation).toHaveBeenCalledWith('ex-1', 'wf-1') + }) - expect(result).toMatchObject({ - success: false, - durablyRecorded: true, - reason: 'paused_database_cancel_failed', - }) - } - ) + it('leaves a late workflow-group pause retryable when event publication fails', async () => { + dbChainMockFns.limit.mockResolvedValueOnce([ + { + executionDeadlineAt: null, + executionOrigin: 'workflow_group', + status: 'running', + workspaceId: 'workspace-1', + }, + ]) + mockMarkExecutionCancelled.mockResolvedValue({ durablyRecorded: true, reason: 'recorded' }) + mockCancelWorkflowGroupExecution.mockResolvedValue({ + kind: 'cancelled', + tableId: 'table-1', + rowId: 'row-1', + groupId: 'group-1', + }) + mockStagePausedCancellation + .mockResolvedValueOnce({ kind: 'not_paused' }) + .mockResolvedValue({ kind: 'idle' }) + mockWriteTerminalEvent.mockRejectedValue(new Error('Redis unavailable')) + + const response = await POST(makeRequest(), makeParams()) + + expect(response.status).toBe(200) + await expect(response.json()).resolves.toMatchObject({ + success: false, + pausedCancelled: false, + reason: 'paused_event_publish_failed', + }) + expect(mockCancelWorkflowGroupExecution).toHaveBeenCalledOnce() + expect(mockCancelWorkflowGroupExecution.mock.invocationCallOrder[0]).toBeLessThan( + mockWriteTerminalEvent.mock.invocationCallOrder[0] + ) + expect(mockCompletePausedCancellation).not.toHaveBeenCalled() + expect(mockClearPausedCancellationIntent).not.toHaveBeenCalled() + }) - /** - * The status read at entry can be stale: a run that finishes after it and - * before the claim leaves a `running` snapshot on a cancel whose claim matched - * no row. The claim's own row count is what separates that from a cancel this - * request really performed. - */ - it.each([ - ['completed', 'already_completed'], - ['failed', 'already_failed'], - ])('reports a run that reached %s after the entry read as a no-op', async (status, reason) => { - mockResolveWorkflowExecutionOwnership - .mockResolvedValueOnce({ - belongsToWorkflow: true, - workflowGroupWorkspaceId: null, - priorStatus: 'running', - }) - .mockResolvedValueOnce({ - belongsToWorkflow: true, - workflowGroupWorkspaceId: null, - priorStatus: status, - }) - mockUpdateReturning.mockReturnValue([]) + it('cancels an active workflow-group resume without cancelling its shared carrier', async () => { + dbChainMockFns.limit.mockResolvedValueOnce([ + { + executionDeadlineAt: null, + executionOrigin: 'workflow_group', + status: 'running', + workspaceId: 'workspace-1', + }, + ]) + mockStagePausedCancellation.mockResolvedValue({ + kind: 'active_resume', + target: ACTIVE_RESUME_TARGET, + }) + mockGetActiveResumeCancellationTarget.mockResolvedValue(ACTIVE_RESUME_TARGET) + mockMarkExecutionCancelled.mockResolvedValue({ + durablyRecorded: false, + reason: 'redis_unavailable', + }) + mockCancelByExecution.mockResolvedValue(1) + mockCancelWorkflowGroupExecution.mockResolvedValue({ + kind: 'cancelled', + tableId: 'table-1', + rowId: 'row-1', + groupId: 'group-1', + }) + mockCompletePausedCancellation.mockResolvedValue(true) - const result = await cancelWorkflowExecution(INPUT) + const response = await POST(makeRequest(), makeParams()) - expect(result).toMatchObject({ success: true, durablyRecorded: false, reason }) + expect(response.status).toBe(200) + await expect(response.json()).resolves.toMatchObject({ + success: true, + pausedCancelled: true, + reason: 'recorded', + }) + expect(mockCancelByExecution).toHaveBeenCalledOnce() + expect(mockCancelByExecution).toHaveBeenCalledWith( + { workflowId: 'wf-1', executionId: 'ex-1' }, + 'resume' + ) + expect(mockMarkExecutionCancelled).toHaveBeenCalledWith('resume-ex-1', { + executionDeadlineAt: null, + }) + expect(mockCancelWorkflowGroupExecution).toHaveBeenCalledOnce() + expect(mockCancelWorkflowGroupExecution.mock.invocationCallOrder[0]).toBeLessThan( + mockMarkExecutionCancelled.mock.invocationCallOrder[0] + ) + expect(mockCompletePausedCancellation).toHaveBeenCalledWith('ex-1', 'wf-1') + expect(mockWriteTerminalEvent.mock.invocationCallOrder[0]).toBeLessThan( + mockCompletePausedCancellation.mock.invocationCallOrder[0] + ) }) - it('reports an undifferentiated outcome when the claim finds no durable log row', async () => { - mockResolveWorkflowExecutionOwnership.mockResolvedValue({ - belongsToWorkflow: true, - workflowGroupWorkspaceId: null, - priorStatus: null, + it('keeps a claimed active group resume retryable when no stop backend accepts it', async () => { + dbChainMockFns.limit.mockResolvedValueOnce([ + { + executionDeadlineAt: null, + executionOrigin: 'workflow_group', + status: 'running', + workspaceId: 'workspace-1', + }, + ]) + mockStagePausedCancellation.mockResolvedValue({ + kind: 'active_resume', + target: ACTIVE_RESUME_TARGET, + }) + mockGetActiveResumeCancellationTarget.mockResolvedValue(ACTIVE_RESUME_TARGET) + mockCancelWorkflowGroupExecution.mockResolvedValue({ + kind: 'cancelled', + tableId: 'table-1', + rowId: 'row-1', + groupId: 'group-1', }) - mockUpdateReturning.mockReturnValue([]) - const result = await cancelWorkflowExecution(INPUT) + const response = await POST(makeRequest(), makeParams()) - expect(result).toMatchObject({ success: true, durablyRecorded: true, reason: 'recorded' }) + expect(response.status).toBe(200) + await expect(response.json()).resolves.toMatchObject({ + success: false, + durablyRecorded: true, + locallyAborted: false, + pausedCancelled: false, + reason: 'active_resume_signal_failed', + }) + expect(mockMarkExecutionCancelled).toHaveBeenCalledWith('resume-ex-1', { + executionDeadlineAt: null, + }) + expect(mockCancelByExecution).toHaveBeenCalledWith( + { workflowId: 'wf-1', executionId: 'ex-1' }, + 'resume' + ) + expect(mockCancelWorkflowGroupExecution.mock.invocationCallOrder[0]).toBeLessThan( + mockMarkExecutionCancelled.mock.invocationCallOrder[0] + ) + expect(mockRollbackActiveResumeCancellation).not.toHaveBeenCalled() + expect(mockPublishWorkflowGroupCancellationEvent).not.toHaveBeenCalled() }) - /** - * The re-read is purely observational — it only refines *which* no-op the - * caller is told about. A database that cannot answer it must not take the - * cancel down with it: the run has already been cancelled in Redis and its - * reservation still has to be released, so the failure degrades to the - * undifferentiated outcome rather than propagating. - */ - it('degrades to the undifferentiated outcome when the status re-read fails', async () => { - mockResolveWorkflowExecutionOwnership - .mockResolvedValueOnce({ - belongsToWorkflow: true, - workflowGroupWorkspaceId: null, - priorStatus: 'running', - }) - .mockRejectedValueOnce(new Error('connection terminated')) - mockUpdateReturning.mockReturnValue([]) + it('rechecks for a pause after the group terminal claim waits on persistence', async () => { + dbChainMockFns.limit.mockResolvedValueOnce([ + { + executionDeadlineAt: null, + executionOrigin: 'workflow_group', + status: 'running', + workspaceId: 'workspace-1', + }, + ]) + mockMarkExecutionCancelled.mockResolvedValue({ durablyRecorded: true, reason: 'recorded' }) + mockCancelWorkflowGroupExecution.mockResolvedValue({ + kind: 'cancelled', + tableId: 'table-1', + rowId: 'row-1', + groupId: 'group-1', + }) + mockStagePausedCancellation + .mockResolvedValueOnce({ kind: 'not_paused' }) + .mockResolvedValue({ kind: 'idle' }) + mockCompletePausedCancellation.mockResolvedValue(true) - const result = await cancelWorkflowExecution(INPUT) + const response = await POST(makeRequest(), makeParams()) - expect(result).toMatchObject({ success: true, durablyRecorded: true, reason: 'recorded' }) - expect(mockResolveWorkflowExecutionOwnership).toHaveBeenCalledTimes(2) - expect(mockReleaseExecutionSlot).toHaveBeenCalledWith('execution-1') + expect(response.status).toBe(200) + await expect(response.json()).resolves.toMatchObject({ + success: true, + pausedCancelled: true, + reason: 'recorded', + }) + expect(mockCancelWorkflowGroupExecution.mock.invocationCallOrder[0]).toBeLessThan( + mockStagePausedCancellation.mock.invocationCallOrder[1] + ) + expect(mockStagePausedCancellation.mock.invocationCallOrder[1]).toBeLessThan( + mockWriteTerminalEvent.mock.invocationCallOrder[0] + ) + expect(mockWriteTerminalEvent.mock.invocationCallOrder[0]).toBeLessThan( + mockCompletePausedCancellation.mock.invocationCallOrder[0] + ) + expect(mockWriteTerminalEvent).toHaveBeenCalledOnce() }) - it('releases the plan concurrency reservation after a successful cancellation', async () => { - const result = await cancelWorkflowExecution(INPUT) + it('rechecks for a regular pause after the terminal claim waits on persistence', async () => { + mockMarkExecutionCancelled.mockResolvedValue({ durablyRecorded: true, reason: 'recorded' }) + mockStagePausedCancellation + .mockResolvedValueOnce({ kind: 'not_paused' }) + .mockResolvedValue({ kind: 'idle' }) + mockCompletePausedCancellation.mockResolvedValue(true) + + const response = await POST(makeRequest(), makeParams()) - expect(result.success).toBe(true) - expect(mockReleaseExecutionSlot).toHaveBeenCalledWith('execution-1') + expect(response.status).toBe(200) + await expect(response.json()).resolves.toMatchObject({ + success: true, + pausedCancelled: true, + reason: 'recorded', + }) + expect(databaseMock.db.update.mock.invocationCallOrder[0]).toBeLessThan( + mockStagePausedCancellation.mock.invocationCallOrder[1] + ) + expect(mockStagePausedCancellation.mock.invocationCallOrder[1]).toBeLessThan( + mockWriteTerminalEvent.mock.invocationCallOrder[0] + ) + expect(mockWriteTerminalEvent.mock.invocationCallOrder[0]).toBeLessThan( + mockCompletePausedCancellation.mock.invocationCallOrder[0] + ) }) - it('keeps the reservation held when nothing could be cancelled', async () => { + it('uses generic cancellation for a regular Table-trigger-block execution', async () => { + dbChainMockFns.limit.mockResolvedValueOnce([ + { + executionDeadlineAt: null, + executionOrigin: null, + status: 'running', + trigger: 'table', + workspaceId: 'workspace-1', + }, + ]) mockMarkExecutionCancelled.mockResolvedValue({ + durablyRecorded: true, + reason: 'recorded', + }) + + const response = await POST(makeRequest(), makeParams()) + + expect(response.status).toBe(200) + expect(mockCancelWorkflowGroupExecution).not.toHaveBeenCalled() + expect(mockMarkExecutionCancelled).toHaveBeenCalledWith('ex-1', { + executionDeadlineAt: null, + }) + expect(mockCancelByExecution).toHaveBeenCalledOnce() + }) + + it('returns 409 without generic cancellation when the exact group attempt is stale', async () => { + dbChainMockFns.limit.mockResolvedValueOnce([ + { + executionDeadlineAt: null, + executionOrigin: 'workflow_group', + status: 'running', + workspaceId: 'workspace-1', + }, + ]) + mockMarkExecutionCancelled.mockResolvedValue({ durablyRecorded: true, reason: 'recorded' }) + mockCancelWorkflowGroupExecution.mockResolvedValue({ + kind: 'conflict', + status: 'no_longer_active', + }) + + const response = await POST(makeRequest(), makeParams()) + + expect(response.status).toBe(409) + await expect(response.json()).resolves.toEqual({ + error: 'Workflow group execution cannot be cancelled while no_longer_active', + }) + expect(mockMarkExecutionCancelled).not.toHaveBeenCalled() + expect(mockPublishWorkflowGroupCancellationEvent).not.toHaveBeenCalled() + expect(mockCancelByExecution).not.toHaveBeenCalled() + expect(mockStagePausedCancellation).toHaveBeenCalledOnce() + expect(mockClearExecutionCancellation).not.toHaveBeenCalled() + expect(mockClearPausedCancellationIntent).not.toHaveBeenCalled() + }) + + it('terminalizes a durable workflow-group log after its table sidecar was deleted', async () => { + dbChainMockFns.limit.mockResolvedValueOnce([ + { + executionDeadlineAt: null, + executionOrigin: 'workflow_group', + status: 'running', + workspaceId: 'workspace-1', + }, + ]) + mockMarkExecutionCancelled.mockResolvedValue({ durablyRecorded: true, reason: 'recorded' }) + mockCancelWorkflowGroupExecution.mockResolvedValue({ kind: 'cancelled_without_sidecar' }) + + const response = await POST(makeRequest(), makeParams()) + + expect(response.status).toBe(200) + await expect(response.json()).resolves.toMatchObject({ + success: true, + executionId: 'ex-1', + reason: 'recorded', + }) + expect(mockMarkExecutionCancelled).toHaveBeenCalledOnce() + expect(mockCancelByExecution).not.toHaveBeenCalled() + expect(mockClearExecutionCancellation).not.toHaveBeenCalled() + expect(mockClearPausedCancellationIntent).not.toHaveBeenCalled() + }) + + it('returns unsuccessful response when Redis is unavailable', async () => { + mockMarkExecutionCancelled.mockResolvedValue({ + durablyRecorded: false, + reason: 'redis_unavailable', + }) + + const response = await POST(makeRequest(), makeParams()) + + expect(response.status).toBe(200) + await expect(response.json()).resolves.toEqual({ + success: false, + executionId: 'ex-1', + redisAvailable: false, durablyRecorded: false, + locallyAborted: false, + pausedCancelled: false, reason: 'redis_unavailable', }) + }) + + it('returns unsuccessful response when Redis persistence fails', async () => { + mockMarkExecutionCancelled.mockResolvedValue({ + durablyRecorded: false, + reason: 'redis_write_failed', + }) - const result = await cancelWorkflowExecution(INPUT) + const response = await POST(makeRequest(), makeParams()) + + expect(response.status).toBe(200) + await expect(response.json()).resolves.toEqual({ + success: false, + executionId: 'ex-1', + redisAvailable: true, + durablyRecorded: false, + locallyAborted: false, + pausedCancelled: false, + reason: 'redis_write_failed', + }) + }) - expect(result.success).toBe(false) + it('returns success when local fallback aborts execution without Redis durability', async () => { + mockMarkExecutionCancelled.mockResolvedValue({ + durablyRecorded: false, + reason: 'redis_unavailable', + }) + mockAbortManualExecution.mockReturnValue(true) + + const response = await POST(makeRequest(), makeParams()) + + expect(response.status).toBe(200) + await expect(response.json()).resolves.toEqual({ + success: true, + executionId: 'ex-1', + redisAvailable: false, + durablyRecorded: false, + locallyAborted: true, + pausedCancelled: false, + reason: 'redis_unavailable', + }) + }) + + it('returns success when the queue backend cancels the active job', async () => { + mockMarkExecutionCancelled.mockResolvedValue({ + durablyRecorded: false, + reason: 'redis_unavailable', + }) + mockCancelByExecution.mockResolvedValue(1) + + const response = await POST(makeRequest(), makeParams()) + + expect(response.status).toBe(200) + await expect(response.json()).resolves.toMatchObject({ + success: true, + executionId: 'ex-1', + durablyRecorded: false, + locallyAborted: false, + reason: 'queue_cancelled', + }) + expect(mockClearExecutionCancellation).not.toHaveBeenCalled() + }) + + it('cancels a queued execution before its workflow log exists', async () => { + dbChainMockFns.limit.mockResolvedValueOnce([]) + mockCancelByExecution.mockResolvedValueOnce(1) + mockMarkExecutionCancelled.mockResolvedValueOnce({ + durablyRecorded: true, + reason: 'recorded', + }) + + const response = await POST(makeRequest(), makeParams()) + + expect(response.status).toBe(200) + await expect(response.json()).resolves.toEqual({ + success: true, + executionId: 'ex-1', + redisAvailable: true, + durablyRecorded: true, + locallyAborted: false, + pausedCancelled: false, + reason: 'queue_cancelled', + }) + expect(mockCancelByExecution).toHaveBeenCalledWith( + { + workflowId: 'wf-1', + executionId: 'ex-1', + }, + 'standalone' + ) + expect(mockMarkExecutionCancelled).toHaveBeenCalledWith('ex-1') + expect(mockReleaseExecutionSlot).toHaveBeenCalledWith('ex-1') + expect(mockClearExecutionCancellation).not.toHaveBeenCalled() + }) + + it('does not use an unscoped local abort before its workflow log exists', async () => { + dbChainMockFns.limit.mockResolvedValueOnce([]) + + const response = await POST(makeRequest(), makeParams()) + + expect(response.status).toBe(404) + await expect(response.json()).resolves.toEqual({ error: 'Execution not found' }) + expect(mockAbortManualExecution).not.toHaveBeenCalled() + expect(mockCancelByExecution).toHaveBeenCalledWith( + { + workflowId: 'wf-1', + executionId: 'ex-1', + }, + 'standalone' + ) + expect(mockMarkExecutionCancelled).not.toHaveBeenCalled() expect(mockReleaseExecutionSlot).not.toHaveBeenCalled() + expect(mockClearExecutionCancellation).not.toHaveBeenCalled() + }) + + it('finishes cancellation when a failed active-resume rollback detects a replacement', async () => { + mockStagePausedCancellation + .mockResolvedValueOnce({ kind: 'active_resume', target: ACTIVE_RESUME_TARGET }) + .mockResolvedValueOnce({ + kind: 'active_resume', + target: REPLACEMENT_ACTIVE_RESUME_TARGET, + }) + mockGetActiveResumeCancellationTarget.mockResolvedValueOnce(REPLACEMENT_ACTIVE_RESUME_TARGET) + mockRollbackActiveResumeCancellation.mockResolvedValueOnce(false) + mockMarkExecutionCancelled + .mockResolvedValueOnce({ durablyRecorded: false, reason: 'redis_unavailable' }) + .mockResolvedValueOnce({ durablyRecorded: true, reason: 'recorded' }) + .mockResolvedValueOnce({ durablyRecorded: true, reason: 'recorded' }) + mockCompletePausedCancellation.mockResolvedValueOnce(true) + + const response = await POST(makeRequest(), makeParams()) + + expect(response.status).toBe(200) + await expect(response.json()).resolves.toMatchObject({ + success: true, + durablyRecorded: true, + pausedCancelled: true, + reason: 'recorded', + }) + expect(mockRollbackActiveResumeCancellation).toHaveBeenCalledWith( + 'ex-1', + 'wf-1', + 'resume-entry-1' + ) + expect(mockMarkExecutionCancelled).toHaveBeenNthCalledWith(1, 'resume-ex-1', { + executionDeadlineAt: null, + }) + expect(mockMarkExecutionCancelled).toHaveBeenNthCalledWith(2, 'resume-ex-1', { + executionDeadlineAt: null, + }) + expect(mockMarkExecutionCancelled).toHaveBeenNthCalledWith(3, 'resume-ex-2', { + executionDeadlineAt: null, + }) + expect(mockCompletePausedCancellation).toHaveBeenCalledWith('ex-1', 'wf-1') }) - it('cancels and publishes the table cell sidecar of a workflow-group execution', async () => { - mockResolveWorkflowExecutionOwnership.mockResolvedValue({ - belongsToWorkflow: true, - workflowGroupWorkspaceId: 'workspace-1', - priorStatus: 'running', + it('finishes late pause cancellation when rollback detects a replacement resume', async () => { + mockStagePausedCancellation + .mockResolvedValueOnce({ kind: 'not_paused' }) + .mockResolvedValueOnce({ kind: 'active_resume', target: ACTIVE_RESUME_TARGET }) + .mockResolvedValueOnce({ + kind: 'active_resume', + target: REPLACEMENT_ACTIVE_RESUME_TARGET, + }) + mockGetActiveResumeCancellationTarget.mockResolvedValueOnce(REPLACEMENT_ACTIVE_RESUME_TARGET) + mockRollbackActiveResumeCancellation.mockResolvedValueOnce(false) + mockMarkExecutionCancelled + .mockResolvedValueOnce({ durablyRecorded: false, reason: 'redis_unavailable' }) + .mockResolvedValueOnce({ durablyRecorded: false, reason: 'redis_unavailable' }) + .mockResolvedValueOnce({ durablyRecorded: true, reason: 'recorded' }) + .mockResolvedValueOnce({ durablyRecorded: true, reason: 'recorded' }) + mockCompletePausedCancellation.mockResolvedValueOnce(true) + + const response = await POST(makeRequest(), makeParams()) + + expect(response.status).toBe(200) + await expect(response.json()).resolves.toMatchObject({ + success: true, + durablyRecorded: true, + pausedCancelled: true, + reason: 'recorded', + }) + expect(mockRollbackActiveResumeCancellation).toHaveBeenCalledWith( + 'ex-1', + 'wf-1', + 'resume-entry-1' + ) + expect(mockMarkExecutionCancelled).toHaveBeenNthCalledWith(1, 'ex-1', { + executionDeadlineAt: null, }) - const cancelled = { - kind: 'cancelled' as const, + expect(mockMarkExecutionCancelled).toHaveBeenNthCalledWith(2, 'resume-ex-1', { + executionDeadlineAt: null, + }) + expect(mockMarkExecutionCancelled).toHaveBeenNthCalledWith(3, 'resume-ex-1', { + executionDeadlineAt: null, + }) + expect(mockMarkExecutionCancelled).toHaveBeenNthCalledWith(4, 'resume-ex-2', { + executionDeadlineAt: null, + }) + expect(mockCompletePausedCancellation).toHaveBeenCalledWith('ex-1', 'wf-1') + }) + + it('does not treat replacement queue cancellation as confirmation of the original stop', async () => { + mockStagePausedCancellation + .mockResolvedValueOnce({ kind: 'active_resume', target: ACTIVE_RESUME_TARGET }) + .mockResolvedValueOnce({ + kind: 'active_resume', + target: REPLACEMENT_ACTIVE_RESUME_TARGET, + }) + mockGetActiveResumeCancellationTarget + .mockResolvedValueOnce(REPLACEMENT_ACTIVE_RESUME_TARGET) + .mockResolvedValueOnce(REPLACEMENT_ACTIVE_RESUME_TARGET) + mockRollbackActiveResumeCancellation.mockResolvedValueOnce(false) + mockCancelByExecution.mockResolvedValue(1) + mockMarkExecutionCancelled + .mockResolvedValueOnce({ durablyRecorded: false, reason: 'redis_unavailable' }) + .mockResolvedValueOnce({ durablyRecorded: false, reason: 'redis_unavailable' }) + .mockResolvedValueOnce({ durablyRecorded: true, reason: 'recorded' }) + + const response = await POST(makeRequest(), makeParams()) + + expect(response.status).toBe(200) + await expect(response.json()).resolves.toMatchObject({ + success: false, + pausedCancelled: false, + reason: 'active_resume_signal_failed', + }) + expect(mockMarkExecutionCancelled).toHaveBeenNthCalledWith(1, 'resume-ex-1', { + executionDeadlineAt: null, + }) + expect(mockMarkExecutionCancelled).toHaveBeenNthCalledWith(2, 'resume-ex-1', { + executionDeadlineAt: null, + }) + expect(mockMarkExecutionCancelled).toHaveBeenNthCalledWith(3, 'resume-ex-2', { + executionDeadlineAt: null, + }) + expect(mockWriteTerminalEvent).not.toHaveBeenCalled() + expect(mockCompletePausedCancellation).not.toHaveBeenCalled() + }) + + it('returns success when a paused HITL execution is cancelled directly in the database', async () => { + mockStagePausedCancellation.mockResolvedValue({ kind: 'idle' }) + mockCompletePausedCancellation.mockResolvedValue(true) + + const response = await POST(makeRequest(), makeParams()) + + expect(response.status).toBe(200) + await expect(response.json()).resolves.toEqual({ + success: true, + executionId: 'ex-1', + redisAvailable: true, + durablyRecorded: true, + locallyAborted: false, + pausedCancelled: true, + reason: 'recorded', + }) + expect(mockMarkExecutionCancelled).not.toHaveBeenCalled() + expect(mockWriteTerminalEvent).toHaveBeenCalledWith( + expect.objectContaining({ + type: 'execution:cancelled', + executionId: 'ex-1', + workflowId: 'wf-1', + }), + 'cancelled' + ) + expect(mockFinalizeExecutionStream).not.toHaveBeenCalled() + }) + + it('claims the paused workflow-group sidecar before publishing and finalizing', async () => { + dbChainMockFns.limit.mockResolvedValueOnce([ + { + executionDeadlineAt: null, + executionOrigin: 'workflow_group', + status: 'running', + workspaceId: 'workspace-1', + }, + ]) + mockCancelWorkflowGroupExecution.mockResolvedValue({ + kind: 'cancelled', tableId: 'table-1', rowId: 'row-1', groupId: 'group-1', - writes: BOTH_WRITES, - } - mockCancelWorkflowGroupExecution.mockResolvedValue(cancelled) + }) + mockStagePausedCancellation.mockResolvedValue({ kind: 'idle' }) + mockCompletePausedCancellation.mockResolvedValue(true) - const result = await cancelWorkflowExecution(INPUT) + const response = await POST(makeRequest(), makeParams()) - expect(result.success).toBe(true) - expect(mockCancelWorkflowGroupExecution).toHaveBeenCalledWith({ - workspaceId: 'workspace-1', - workflowId: 'workflow-1', - executionId: 'execution-1', - }) - expect(mockPublishWorkflowGroupCancellationEvent).toHaveBeenCalledWith(cancelled, 'execution-1') - expect(mockUpdateSet).not.toHaveBeenCalled() - expect(mockReleaseExecutionSlot).toHaveBeenCalledWith('execution-1') - }) - - /** - * A workflow-group log that is already `cancelled` can still own a cell - * sidecar left in `error`, and reconciling it to `cancelled` is a durable - * write this request performed. The terminal entry snapshot cannot see that - * work, so it must not reinterpret the outcome as a no-op — the API would - * otherwise tell the caller nothing changed and drop the cancellation event. - */ - it('reports a durable write when a cancelled group run still had its sidecar reconciled', async () => { - mockResolveWorkflowExecutionOwnership.mockResolvedValue({ - belongsToWorkflow: true, - workflowGroupWorkspaceId: 'workspace-1', - priorStatus: 'cancelled', - }) - const cancelled = { - kind: 'cancelled' as const, + expect(response.status).toBe(200) + await expect(response.json()).resolves.toMatchObject({ + success: true, + durablyRecorded: true, + pausedCancelled: true, + reason: 'recorded', + }) + expect(mockStagePausedCancellation.mock.invocationCallOrder[0]).toBeLessThan( + mockWriteTerminalEvent.mock.invocationCallOrder[0] + ) + expect(mockCancelWorkflowGroupExecution.mock.invocationCallOrder[0]).toBeLessThan( + mockWriteTerminalEvent.mock.invocationCallOrder[0] + ) + expect(mockWriteTerminalEvent.mock.invocationCallOrder[0]).toBeLessThan( + mockCompletePausedCancellation.mock.invocationCallOrder[0] + ) + expect(mockMarkExecutionCancelled).not.toHaveBeenCalled() + expect(mockCancelByExecution).not.toHaveBeenCalled() + expect(mockWriteTerminalEvent).toHaveBeenCalledOnce() + expect(mockCompletePausedCancellation).toHaveBeenCalledWith('ex-1', 'wf-1') + }) + + it('keeps a paused workflow-group cancellation reserved when event publication fails', async () => { + dbChainMockFns.limit.mockResolvedValueOnce([ + { + executionDeadlineAt: null, + executionOrigin: 'workflow_group', + status: 'running', + workspaceId: 'workspace-1', + }, + ]) + mockStagePausedCancellation.mockResolvedValue({ kind: 'idle' }) + mockCancelWorkflowGroupExecution.mockResolvedValue({ + kind: 'cancelled', tableId: 'table-1', rowId: 'row-1', groupId: 'group-1', - writes: SIDECAR_WRITE, - } - mockCancelWorkflowGroupExecution.mockResolvedValue(cancelled) + }) + mockWriteTerminalEvent.mockRejectedValue(new Error('Redis unavailable')) + + const response = await POST(makeRequest(), makeParams()) + + expect(response.status).toBe(200) + await expect(response.json()).resolves.toMatchObject({ + success: false, + pausedCancelled: false, + reason: 'paused_event_publish_failed', + }) + expect(mockCompletePausedCancellation).not.toHaveBeenCalled() + expect(mockCancelWorkflowGroupExecution).toHaveBeenCalledOnce() + expect(mockClearPausedCancellationIntent).not.toHaveBeenCalled() + expect(mockCancelByExecution).not.toHaveBeenCalled() + }) - const result = await cancelWorkflowExecution(INPUT) + it('publishes paused cancellation event even when Redis cancellation is recorded', async () => { + mockStagePausedCancellation.mockResolvedValue({ kind: 'idle' }) + mockCompletePausedCancellation.mockResolvedValue(true) + + const response = await POST(makeRequest(), makeParams()) - expect(result).toMatchObject({ success: true, durablyRecorded: true, reason: 'recorded' }) - expect(mockPublishWorkflowGroupCancellationEvent).toHaveBeenCalledWith(cancelled, 'execution-1') + expect(response.status).toBe(200) + await expect(response.json()).resolves.toMatchObject({ + success: true, + executionId: 'ex-1', + durablyRecorded: true, + pausedCancelled: true, + }) + expect(mockMarkExecutionCancelled).not.toHaveBeenCalled() + expect(mockWriteTerminalEvent).toHaveBeenCalledWith( + expect.objectContaining({ + type: 'execution:cancelled', + executionId: 'ex-1', + workflowId: 'wf-1', + }), + 'cancelled' + ) + expect(mockFinalizeExecutionStream).not.toHaveBeenCalled() }) - /** - * The group path terminalizes the workflow log itself when the cell sidecar - * is already gone, so that outcome is a durable write too. - */ - it('reports a durable write when the group path cancels a run whose sidecar is gone', async () => { - mockResolveWorkflowExecutionOwnership.mockResolvedValue({ - belongsToWorkflow: true, - workflowGroupWorkspaceId: 'workspace-1', - priorStatus: 'running', + it('does not confirm paused cancellation when terminal event publication fails', async () => { + mockStagePausedCancellation.mockResolvedValue({ kind: 'idle' }) + mockCompletePausedCancellation.mockResolvedValue(true) + mockWriteTerminalEvent.mockRejectedValue(new Error('Redis unavailable')) + + const response = await POST(makeRequest(), makeParams()) + + expect(response.status).toBe(200) + await expect(response.json()).resolves.toEqual({ + success: false, + executionId: 'ex-1', + redisAvailable: false, + durablyRecorded: true, + locallyAborted: false, + pausedCancelled: false, + reason: 'paused_event_publish_failed', }) - mockCancelWorkflowGroupExecution.mockResolvedValue({ - kind: 'cancelled_without_sidecar', - writes: LOG_WRITE, + expect(mockMarkExecutionCancelled).not.toHaveBeenCalled() + expect(mockCompletePausedCancellation).not.toHaveBeenCalled() + expect(mockWriteTerminalEvent).toHaveBeenCalledWith( + expect.objectContaining({ + type: 'execution:cancelled', + executionId: 'ex-1', + workflowId: 'wf-1', + }), + 'cancelled' + ) + expect(mockFinalizeExecutionStream).not.toHaveBeenCalled() + }) + + it('finishes reconciliation when the pause row is already cancelled', async () => { + mockStagePausedCancellation.mockResolvedValue({ kind: 'idle' }) + mockCompletePausedCancellation.mockResolvedValue(true) + + const response = await POST(makeRequest(), makeParams()) + + expect(response.status).toBe(200) + await expect(response.json()).resolves.toMatchObject({ + success: true, + pausedCancelled: true, + reason: 'recorded', + }) + expect(mockMarkExecutionCancelled).not.toHaveBeenCalled() + expect(mockWriteTerminalEvent.mock.invocationCallOrder[0]).toBeLessThan( + mockCompletePausedCancellation.mock.invocationCallOrder[0] + ) + }) + + it('stops before lookup when cancellation is already aborted', async () => { + const controller = new AbortController() + controller.abort() + + const response = await cancelAsResponse({ + abortSignal: controller.signal, + }) + + expect(response.status).toBe(409) + await expect(response.json()).resolves.toEqual({ + error: 'Request aborted before workflow run cancellation could be applied.', + }) + expect(databaseMock.db.select).not.toHaveBeenCalled() + expect(mockMarkExecutionCancelled).not.toHaveBeenCalled() + }) + + it('stops before mutation when cancellation is aborted during execution lookup', async () => { + const controller = new AbortController() + dbChainMockFns.limit.mockImplementationOnce(async () => { + controller.abort() + return [ + { + executionDeadlineAt: null, + executionOrigin: null, + status: 'running', + workspaceId: 'workspace-1', + }, + ] + }) + + const response = await cancelAsResponse({ + abortSignal: controller.signal, + }) + + expect(response.status).toBe(409) + expect(mockMarkExecutionCancelled).not.toHaveBeenCalled() + expect(mockAbortManualExecution).not.toHaveBeenCalled() + expect(mockCancelByExecution).not.toHaveBeenCalled() + }) + + it('rolls back pause staging when cancellation is aborted during staging', async () => { + const controller = new AbortController() + mockStagePausedCancellation.mockImplementationOnce(async () => { + controller.abort() + return { kind: 'idle' } + }) + + const response = await cancelAsResponse({ + abortSignal: controller.signal, + }) + + expect(response.status).toBe(409) + expect(mockClearPausedCancellationIntent).toHaveBeenCalledWith('ex-1', 'wf-1') + expect(mockMarkExecutionCancelled).not.toHaveBeenCalled() + expect(mockAbortManualExecution).not.toHaveBeenCalled() + expect(mockCancelByExecution).not.toHaveBeenCalled() + }) + + it('finishes cancellation when an aborted idle-pause rollback fails', async () => { + const controller = new AbortController() + mockStagePausedCancellation.mockImplementationOnce(async () => { + controller.abort() + return { kind: 'idle' } + }) + mockClearPausedCancellationIntent.mockRejectedValueOnce(new Error('database unavailable')) + mockCompletePausedCancellation.mockResolvedValue(true) + + const response = await cancelAsResponse({ + abortSignal: controller.signal, + }) + + expect(response.status).toBe(200) + await expect(response.json()).resolves.toMatchObject({ + success: true, + pausedCancelled: true, + reason: 'recorded', + }) + expect(mockClearPausedCancellationIntent).toHaveBeenCalledWith('ex-1', 'wf-1') + expect(mockWriteTerminalEvent).toHaveBeenCalledOnce() + expect(mockCompletePausedCancellation).toHaveBeenCalledWith('ex-1', 'wf-1') + }) + + it('rolls back an active resume staged while cancellation is aborted', async () => { + const controller = new AbortController() + mockStagePausedCancellation.mockImplementationOnce(async () => { + controller.abort() + return { kind: 'active_resume', target: ACTIVE_RESUME_TARGET } + }) + + const response = await cancelAsResponse({ + abortSignal: controller.signal, + }) + + expect(response.status).toBe(409) + expect(mockRollbackActiveResumeCancellation).toHaveBeenCalledWith( + 'ex-1', + 'wf-1', + 'resume-entry-1' + ) + expect(mockMarkExecutionCancelled).not.toHaveBeenCalled() + }) + + it('rolls back pause staging for an already-cancelled execution when cancellation is aborted', async () => { + const controller = new AbortController() + dbChainMockFns.limit.mockResolvedValueOnce([ + { + executionDeadlineAt: null, + executionOrigin: null, + status: 'cancelled', + workspaceId: 'workspace-1', + }, + ]) + mockStagePausedCancellation.mockImplementationOnce(async () => { + controller.abort() + return { kind: 'idle' } + }) + + const response = await cancelAsResponse({ + abortSignal: controller.signal, + }) + + expect(response.status).toBe(409) + expect(mockClearPausedCancellationIntent).toHaveBeenCalledWith('ex-1', 'wf-1') + expect(mockWriteTerminalEvent).not.toHaveBeenCalled() + expect(mockCompletePausedCancellation).not.toHaveBeenCalled() + expect(mockReleaseExecutionSlot).not.toHaveBeenCalled() + }) + + it('finishes cancellation when an aborted active-resume stage cannot be rolled back', async () => { + const controller = new AbortController() + mockStagePausedCancellation.mockImplementationOnce(async () => { + controller.abort() + return { kind: 'active_resume', target: ACTIVE_RESUME_TARGET } + }) + mockRollbackActiveResumeCancellation.mockResolvedValueOnce(false) + mockMarkExecutionCancelled.mockResolvedValue({ durablyRecorded: true, reason: 'recorded' }) + mockCompletePausedCancellation.mockResolvedValue(true) + + const response = await cancelAsResponse({ + abortSignal: controller.signal, + }) + + expect(response.status).toBe(200) + await expect(response.json()).resolves.toMatchObject({ success: true, pausedCancelled: true }) + expect(mockRollbackActiveResumeCancellation).toHaveBeenCalledWith( + 'ex-1', + 'wf-1', + 'resume-entry-1' + ) + expect(mockMarkExecutionCancelled).toHaveBeenCalledWith('resume-ex-1', { + executionDeadlineAt: null, }) + }) - const result = await cancelWorkflowExecution(INPUT) + it('returns 404 when the execution does not belong to the workflow', async () => { + dbChainMockFns.limit.mockResolvedValueOnce([]) - expect(result).toMatchObject({ success: true, durablyRecorded: true, reason: 'recorded' }) - expect(mockUpdateSet).not.toHaveBeenCalled() + const response = await POST(makeRequest(), makeParams()) + + expect(response.status).toBe(404) + expect(mockMarkExecutionCancelled).not.toHaveBeenCalled() + expect(mockCancelByExecution).toHaveBeenCalledWith( + { + workflowId: 'wf-1', + executionId: 'ex-1', + }, + 'standalone' + ) }) - /** - * The mirror case: a group run that was already terminal and whose sidecar was - * already `cancelled` leaves both records untouched, so it still reports the - * state it observed rather than a durable write. - */ - it('reports a group run that changed nothing as a no-op', async () => { - mockResolveWorkflowExecutionOwnership.mockResolvedValue({ - belongsToWorkflow: true, - workflowGroupWorkspaceId: 'workspace-1', - priorStatus: 'cancelled', + it('treats an already-cancelled execution as an idempotent success', async () => { + dbChainMockFns.limit.mockResolvedValueOnce([ + { executionDeadlineAt: null, status: 'cancelled', workspaceId: 'workspace-1' }, + ]) + + const response = await POST(makeRequest(), makeParams()) + + expect(response.status).toBe(200) + await expect(response.json()).resolves.toMatchObject({ + success: true, + durablyRecorded: false, + reason: 'already_cancelled', }) + expect(mockCancelByExecution).not.toHaveBeenCalled() + expect(mockReleaseExecutionSlot).toHaveBeenCalledWith('ex-1') + }) + + it('reconciles the exact sidecar when a workflow-group log is already cancelled', async () => { + dbChainMockFns.limit.mockResolvedValueOnce([ + { + executionDeadlineAt: null, + executionOrigin: 'workflow_group', + status: 'cancelled', + workspaceId: 'workspace-1', + }, + ]) mockCancelWorkflowGroupExecution.mockResolvedValue({ kind: 'already_cancelled', tableId: 'table-1', rowId: 'row-1', groupId: 'group-1', - writes: NO_WRITES, }) + mockMarkExecutionCancelled.mockResolvedValue({ durablyRecorded: true, reason: 'recorded' }) - const result = await cancelWorkflowExecution(INPUT) + const response = await POST(makeRequest(), makeParams()) - expect(result).toMatchObject({ + expect(response.status).toBe(200) + await expect(response.json()).resolves.toMatchObject({ success: true, - durablyRecorded: false, reason: 'already_cancelled', }) + expect(mockCancelWorkflowGroupExecution).toHaveBeenCalledWith({ + workspaceId: 'workspace-1', + workflowId: 'wf-1', + executionId: 'ex-1', + }) + expect(mockMarkExecutionCancelled).toHaveBeenCalledWith('ex-1', { + executionDeadlineAt: null, + }) + expect(mockPublishWorkflowGroupCancellationEvent).toHaveBeenCalledOnce() + expect(mockCancelByExecution).not.toHaveBeenCalled() }) - /** - * The same `already_cancelled` kind covers a transition that left the sidecar - * alone but still terminalized an active workflow log. That log write is - * durable, so the outcome must stay `recorded` and must not re-read a state - * this request itself wrote. - */ - it('reports a durable write when a group run only repaired its workflow log', async () => { - mockResolveWorkflowExecutionOwnership.mockResolvedValue({ - belongsToWorkflow: true, - workflowGroupWorkspaceId: 'workspace-1', - priorStatus: 'running', + it('finishes workflow-group reconciliation when abort arrives during its durable commit', async () => { + const controller = new AbortController() + dbChainMockFns.limit.mockResolvedValueOnce([ + { + executionDeadlineAt: null, + executionOrigin: 'workflow_group', + status: 'cancelled', + workspaceId: 'workspace-1', + }, + ]) + mockCancelWorkflowGroupExecution.mockImplementationOnce(async () => { + controller.abort() + return { + kind: 'already_cancelled', + tableId: 'table-1', + rowId: 'row-1', + groupId: 'group-1', + } + }) + mockStagePausedCancellation.mockResolvedValue({ kind: 'idle' }) + mockCompletePausedCancellation.mockResolvedValue(true) + + const response = await cancelAsResponse({ + abortSignal: controller.signal, }) + + expect(response.status).toBe(200) + await expect(response.json()).resolves.toMatchObject({ success: true, pausedCancelled: true }) + expect(mockClearPausedCancellationIntent).not.toHaveBeenCalled() + expect(mockPublishWorkflowGroupCancellationEvent).toHaveBeenCalledOnce() + expect(mockCompletePausedCancellation).toHaveBeenCalledWith('ex-1', 'wf-1') + }) + + it('does not finalize an already-cancelled group retry until exact stop is accepted', async () => { + dbChainMockFns.limit.mockResolvedValueOnce([ + { + executionDeadlineAt: null, + executionOrigin: 'workflow_group', + status: 'cancelled', + workspaceId: 'workspace-1', + }, + ]) mockCancelWorkflowGroupExecution.mockResolvedValue({ kind: 'already_cancelled', tableId: 'table-1', rowId: 'row-1', groupId: 'group-1', - writes: LOG_WRITE, }) - const result = await cancelWorkflowExecution(INPUT) + const response = await POST(makeRequest(), makeParams()) - expect(result).toMatchObject({ success: true, durablyRecorded: true, reason: 'recorded' }) - expect(mockResolveWorkflowExecutionOwnership).toHaveBeenCalledOnce() + expect(response.status).toBe(200) + await expect(response.json()).resolves.toMatchObject({ + success: false, + redisAvailable: false, + reason: 'redis_unavailable', + }) + expect(mockPublishWorkflowGroupCancellationEvent).not.toHaveBeenCalled() + expect(mockWriteTerminalEvent).not.toHaveBeenCalled() + expect(mockReleaseExecutionSlot).not.toHaveBeenCalled() }) - /** - * The lost race the sidecar-bearing kind used to hide: a concurrent cancel - * terminalized both records between the entry snapshot and this transaction, - * which then found the sidecar already `cancelled` and the log already - * `cancelled` and wrote nothing. A non-terminal entry snapshot cannot catch - * that, so the transition's own report of having written nothing is what - * forces the re-read — otherwise the request would claim a durable write and - * fire the v2 cancel analytics gate on a no-op. - */ - it('reports a group run that lost the race with its sidecar already cancelled as a no-op', async () => { - mockResolveWorkflowExecutionOwnership - .mockResolvedValueOnce({ - belongsToWorkflow: true, - workflowGroupWorkspaceId: 'workspace-1', - priorStatus: 'running', - }) - .mockResolvedValueOnce({ - belongsToWorkflow: true, - workflowGroupWorkspaceId: 'workspace-1', - priorStatus: 'cancelled', - }) + it('repairs a stranded active resume when the group log is already cancelled', async () => { + dbChainMockFns.limit.mockResolvedValueOnce([ + { + executionDeadlineAt: null, + executionOrigin: 'workflow_group', + status: 'cancelled', + workspaceId: 'workspace-1', + }, + ]) + mockStagePausedCancellation.mockResolvedValue({ + kind: 'active_resume', + target: ACTIVE_RESUME_TARGET, + }) + mockGetActiveResumeCancellationTarget.mockResolvedValue(ACTIVE_RESUME_TARGET) + mockAbortManualExecution.mockReturnValue(true) + mockCancelByExecution.mockResolvedValue(1) + mockCompletePausedCancellation.mockResolvedValue(true) mockCancelWorkflowGroupExecution.mockResolvedValue({ kind: 'already_cancelled', tableId: 'table-1', rowId: 'row-1', groupId: 'group-1', - writes: NO_WRITES, }) - const result = await cancelWorkflowExecution(INPUT) + const response = await POST(makeRequest(), makeParams()) - expect(result).toMatchObject({ + expect(response.status).toBe(200) + await expect(response.json()).resolves.toMatchObject({ success: true, - durablyRecorded: false, + locallyAborted: true, + pausedCancelled: true, reason: 'already_cancelled', }) - expect(mockResolveWorkflowExecutionOwnership).toHaveBeenCalledTimes(2) + expect(mockCancelByExecution).toHaveBeenCalledWith( + { workflowId: 'wf-1', executionId: 'ex-1' }, + 'resume' + ) + expect(mockAbortManualExecution).toHaveBeenCalledWith('resume-ex-1') + expect(mockCompletePausedCancellation).toHaveBeenCalledWith('ex-1', 'wf-1') + expect(mockWriteTerminalEvent).toHaveBeenCalledOnce() + expect(mockWriteTerminalEvent.mock.invocationCallOrder[0]).toBeLessThan( + mockCompletePausedCancellation.mock.invocationCallOrder[0] + ) }) - /** - * The lost-race re-read applies to the group path as well: an entry snapshot - * can still read `running` when the sidecar-less transition finds the log - * already `cancelled` and writes nothing. - */ - it('reports a group run that lost the race to another cancel as a no-op', async () => { - mockResolveWorkflowExecutionOwnership - .mockResolvedValueOnce({ - belongsToWorkflow: true, - workflowGroupWorkspaceId: 'workspace-1', - priorStatus: 'running', - }) - .mockResolvedValueOnce({ - belongsToWorkflow: true, - workflowGroupWorkspaceId: 'workspace-1', - priorStatus: 'cancelled', - }) - mockCancelWorkflowGroupExecution.mockResolvedValue({ - kind: 'already_cancelled_without_sidecar', - writes: NO_WRITES, + it('repairs a stranded active resume when a regular workflow log is already cancelled', async () => { + dbChainMockFns.limit.mockResolvedValueOnce([ + { + executionDeadlineAt: null, + executionOrigin: null, + status: 'cancelled', + workspaceId: 'workspace-1', + }, + ]) + mockStagePausedCancellation.mockResolvedValue({ + kind: 'active_resume', + target: ACTIVE_RESUME_TARGET, }) + mockGetActiveResumeCancellationTarget.mockResolvedValue(ACTIVE_RESUME_TARGET) + mockAbortManualExecution.mockReturnValue(true) + mockCancelByExecution.mockResolvedValue(1) + mockCompletePausedCancellation.mockResolvedValue(true) - const result = await cancelWorkflowExecution(INPUT) + const response = await POST(makeRequest(), makeParams()) - expect(result).toMatchObject({ + expect(response.status).toBe(200) + await expect(response.json()).resolves.toMatchObject({ success: true, - durablyRecorded: false, + locallyAborted: true, + pausedCancelled: true, reason: 'already_cancelled', }) - expect(mockResolveWorkflowExecutionOwnership).toHaveBeenCalledTimes(2) - }) - - it.each([ - [ - { kind: 'conflict' as const, status: 'completed', writes: NO_WRITES }, - 'cannot be cancelled while completed', - ], - [ - { kind: 'not_workflow_group' as const, writes: NO_WRITES }, - 'no longer the active table execution', - ], - ])( - 'releases the reservation before reporting a refused workflow-group cell claim as a conflict', - async (outcome, message) => { - mockResolveWorkflowExecutionOwnership.mockResolvedValue({ - belongsToWorkflow: true, - workflowGroupWorkspaceId: 'workspace-1', - }) - mockCancelWorkflowGroupExecution.mockResolvedValue(outcome) + expect(mockCancelByExecution).toHaveBeenCalledWith( + { workflowId: 'wf-1', executionId: 'ex-1' }, + 'resume' + ) + expect(mockWriteTerminalEvent.mock.invocationCallOrder[0]).toBeLessThan( + mockCompletePausedCancellation.mock.invocationCallOrder[0] + ) + expect(mockCancelWorkflowGroupExecution).not.toHaveBeenCalled() + }) - await expect(cancelWorkflowExecution(INPUT)).rejects.toMatchObject({ - code: 'conflict', - message: expect.stringContaining(message), - }) - expect(mockPublishWorkflowGroupCancellationEvent).not.toHaveBeenCalled() - expect(mockReleaseExecutionSlot).toHaveBeenCalledWith('execution-1') - } - ) + it('keeps an already-cancelled active resume retryable when publication fails', async () => { + dbChainMockFns.limit.mockResolvedValueOnce([ + { + executionDeadlineAt: null, + executionOrigin: null, + status: 'cancelled', + workspaceId: 'workspace-1', + }, + ]) + mockStagePausedCancellation.mockResolvedValue({ + kind: 'active_resume', + target: ACTIVE_RESUME_TARGET, + }) + mockGetActiveResumeCancellationTarget.mockResolvedValue(ACTIVE_RESUME_TARGET) + mockMarkExecutionCancelled.mockResolvedValue({ durablyRecorded: true, reason: 'recorded' }) + mockWriteTerminalEvent.mockRejectedValue(new Error('Redis unavailable')) - it.each([ - [{ kind: 'conflict' as const, status: 'completed', writes: NO_WRITES }], - [{ kind: 'not_workflow_group' as const, writes: NO_WRITES }], - ])( - 'keeps the reservation held when a refused claim follows a paused cancellation', - async (outcome) => { - mockResolveWorkflowExecutionOwnership.mockResolvedValue({ - belongsToWorkflow: true, - workflowGroupWorkspaceId: 'workspace-1', - }) - mockBeginPausedCancellation.mockResolvedValue(true) - mockCancelWorkflowGroupExecution.mockResolvedValue(outcome) + const response = await POST(makeRequest(), makeParams()) - await expect(cancelWorkflowExecution(INPUT)).rejects.toMatchObject({ code: 'conflict' }) - expect(mockReleaseExecutionSlot).not.toHaveBeenCalled() - } - ) + expect(response.status).toBe(200) + await expect(response.json()).resolves.toMatchObject({ + success: false, + pausedCancelled: false, + reason: 'paused_event_publish_failed', + }) + expect(mockCompletePausedCancellation).not.toHaveBeenCalled() + }) - it.each([ - [{ kind: 'conflict' as const, status: 'completed', writes: NO_WRITES }], - [{ kind: 'not_workflow_group' as const, writes: NO_WRITES }], - ])( - 'keeps the reservation held when a refused claim follows a failed cancellation', - async (outcome) => { - mockResolveWorkflowExecutionOwnership.mockResolvedValue({ - belongsToWorkflow: true, - workflowGroupWorkspaceId: 'workspace-1', - }) - mockMarkExecutionCancelled.mockResolvedValue({ - durablyRecorded: false, - reason: 'redis_unavailable', - }) - mockCancelWorkflowGroupExecution.mockResolvedValue(outcome) + it('fails closed when an already-cancelled group sidecar cannot be reconciled', async () => { + dbChainMockFns.limit.mockResolvedValueOnce([ + { + executionDeadlineAt: null, + executionOrigin: 'workflow_group', + status: 'cancelled', + workspaceId: 'workspace-1', + }, + ]) + mockCancelWorkflowGroupExecution.mockResolvedValue({ + kind: 'conflict', + status: 'completed', + }) + + const response = await POST(makeRequest(), makeParams()) + + expect(response.status).toBe(409) + await expect(response.json()).resolves.toEqual({ + error: 'Workflow group execution cannot be reconciled while completed', + }) + }) + + it('reports a reconciliation failure for an already-cancelled group sidecar', async () => { + dbChainMockFns.limit.mockResolvedValueOnce([ + { + executionDeadlineAt: null, + executionOrigin: 'workflow_group', + status: 'cancelled', + workspaceId: 'workspace-1', + }, + ]) + mockCancelWorkflowGroupExecution.mockRejectedValue(new Error('database unavailable')) + + const response = await POST(makeRequest(), makeParams()) - await expect(cancelWorkflowExecution(INPUT)).rejects.toMatchObject({ code: 'conflict' }) - expect(mockReleaseExecutionSlot).not.toHaveBeenCalled() + expect(response.status).toBe(500) + await expect(response.json()).resolves.toEqual({ error: 'database unavailable' }) + }) + + it.each(['completed', 'failed'] as const)( + 'raises a typed conflict when a standalone execution is already %s', + async (executionStatus) => { + dbChainMockFns.limit.mockResolvedValueOnce([ + { status: executionStatus, workspaceId: 'workspace-1' }, + ]) + + const error = await cancelWorkflowExecution(INPUT).catch((caught: unknown) => caught) + + expect(error).toBeInstanceOf(WorkflowRunAlreadyTerminalError) + expect(error).toEqual( + expect.objectContaining({ + code: 'conflict', + executionId: 'ex-1', + executionStatus, + redisAvailable: true, + locallyAborted: false, + }) + ) + expect(mockMarkExecutionCancelled).not.toHaveBeenCalled() + expect(mockCancelByExecution).not.toHaveBeenCalled() } ) - it('releases the reservation and rethrows when the workflow-group cancel fails unexpectedly', async () => { - mockResolveWorkflowExecutionOwnership.mockResolvedValue({ - belongsToWorkflow: true, - workflowGroupWorkspaceId: 'workspace-1', - priorStatus: 'running', + it('keeps workflow-group terminal conflicts strict', async () => { + dbChainMockFns.limit.mockResolvedValueOnce([ + { + executionOrigin: 'workflow_group', + status: 'completed', + workspaceId: 'workspace-1', + }, + ]) + + await expect(cancelWorkflowExecution(INPUT)).rejects.toMatchObject({ + name: 'OrchestrationError', + code: 'conflict', + message: 'Execution cannot be cancelled while completed', }) - const failure = new Error('Workflow-group cancellation lost its locked workflow-log claim') - mockCancelWorkflowGroupExecution.mockRejectedValue(failure) + expect(mockMarkExecutionCancelled).not.toHaveBeenCalled() + expect(mockCancelByExecution).not.toHaveBeenCalled() + }) - await expect(cancelWorkflowExecution(INPUT)).rejects.toBe(failure) - expect(mockReleaseExecutionSlot).toHaveBeenCalledWith('execution-1') - expect(mockPublishWorkflowGroupCancellationEvent).not.toHaveBeenCalled() - expect(mockUpdateSet).not.toHaveBeenCalled() + it('returns 409 when completion wins the terminal database race', async () => { + mockMarkExecutionCancelled.mockResolvedValue({ durablyRecorded: true, reason: 'recorded' }) + const returning = vi.fn().mockResolvedValue([]) + const where = vi.fn(() => ({ returning })) + databaseMock.db.update.mockReturnValueOnce({ set: vi.fn(() => ({ where })) }) + dbChainMockFns.limit + .mockResolvedValueOnce([ + { executionDeadlineAt: null, status: 'running', workspaceId: 'workspace-1' }, + ]) + .mockResolvedValueOnce([{ status: 'completed' }]) + + const error = await cancelWorkflowExecution(INPUT).catch((caught: unknown) => caught) + + expect(error).toBeInstanceOf(WorkflowRunAlreadyTerminalError) + expect(error).toEqual( + expect.objectContaining({ + code: 'conflict', + executionId: 'ex-1', + executionStatus: 'completed', + redisAvailable: true, + locallyAborted: false, + }) + ) + expect(returning).toHaveBeenCalledOnce() + expect(mockClearExecutionCancellation).toHaveBeenCalledWith('ex-1') + expect(mockWriteTerminalEvent).not.toHaveBeenCalled() + expect(mockReleaseExecutionSlot).not.toHaveBeenCalled() + }) + + it('finalizes paused cancellation state when resume completion wins the log claim', async () => { + mockStagePausedCancellation.mockResolvedValue({ kind: 'idle' }) + const returning = vi.fn().mockResolvedValue([]) + const where = vi.fn(() => ({ returning })) + databaseMock.db.update.mockReturnValueOnce({ set: vi.fn(() => ({ where })) }) + dbChainMockFns.limit + .mockResolvedValueOnce([ + { executionDeadlineAt: null, status: 'running', workspaceId: 'workspace-1' }, + ]) + .mockResolvedValueOnce([{ status: 'completed' }]) + + const response = await POST(makeRequest(), makeParams()) + + expect(response.status).toBe(409) + await expect(response.json()).resolves.toEqual({ + error: 'Execution cannot be cancelled while completed', + }) + expect(mockWriteTerminalEvent).not.toHaveBeenCalled() + expect(mockCompletePausedCancellation).not.toHaveBeenCalled() + expect(mockFinalizePausedCancellationForTerminalRun).toHaveBeenCalledWith('ex-1', 'wf-1', []) }) - it('keeps the reservation held when an unexpected workflow-group failure follows a paused cancellation', async () => { - mockResolveWorkflowExecutionOwnership.mockResolvedValue({ - belongsToWorkflow: true, - workflowGroupWorkspaceId: 'workspace-1', - priorStatus: 'running', + it('retries paused cancellation finalization before returning a terminal-race conflict', async () => { + mockStagePausedCancellation.mockResolvedValue({ kind: 'idle' }) + mockFinalizePausedCancellationForTerminalRun.mockRejectedValueOnce( + new Error('database unavailable') + ) + const returning = vi.fn().mockResolvedValue([]) + const where = vi.fn(() => ({ returning })) + databaseMock.db.update.mockReturnValueOnce({ set: vi.fn(() => ({ where })) }) + dbChainMockFns.limit + .mockResolvedValueOnce([ + { executionDeadlineAt: null, status: 'running', workspaceId: 'workspace-1' }, + ]) + .mockResolvedValueOnce([{ status: 'completed' }]) + + const response = await POST(makeRequest(), makeParams()) + + expect(response.status).toBe(409) + await expect(response.json()).resolves.toEqual({ + error: 'Execution cannot be cancelled while completed', }) - mockBeginPausedCancellation.mockResolvedValue(true) - mockCancelWorkflowGroupExecution.mockRejectedValue(new Error('serialization conflict')) + expect(mockFinalizePausedCancellationForTerminalRun).toHaveBeenCalledTimes(2) + expect(mockWriteTerminalEvent).not.toHaveBeenCalled() + }) - await expect(cancelWorkflowExecution(INPUT)).rejects.toThrow('serialization conflict') + it('keeps the active-resume stop marker when a terminal parent wins the log claim', async () => { + mockStagePausedCancellation.mockResolvedValue({ + kind: 'active_resume', + target: ACTIVE_RESUME_TARGET, + }) + mockGetActiveResumeCancellationTargets.mockResolvedValue([ACTIVE_RESUME_TARGET]) + mockMarkExecutionCancelled.mockResolvedValue({ durablyRecorded: true, reason: 'recorded' }) + const returning = vi.fn().mockResolvedValue([]) + const where = vi.fn(() => ({ returning })) + databaseMock.db.update.mockReturnValueOnce({ set: vi.fn(() => ({ where })) }) + dbChainMockFns.limit + .mockResolvedValueOnce([ + { executionDeadlineAt: null, status: 'running', workspaceId: 'workspace-1' }, + ]) + .mockResolvedValueOnce([{ status: 'completed' }]) + + const response = await POST(makeRequest(), makeParams()) + + expect(response.status).toBe(409) + await expect(response.json()).resolves.toEqual({ + error: 'Execution cannot be cancelled while completed', + }) + expect(mockFinalizePausedCancellationForTerminalRun).toHaveBeenCalledWith('ex-1', 'wf-1', [ + 'resume-entry-1', + ]) + expect(mockRollbackActiveResumeCancellation).not.toHaveBeenCalled() + expect(mockClearPausedCancellationIntent).not.toHaveBeenCalled() + expect(mockClearExecutionCancellation).not.toHaveBeenCalled() + }) + + it('does not finalize a claimed resume that cannot be stopped after its parent is terminal', async () => { + mockGetActiveResumeCancellationTargets.mockResolvedValue([ACTIVE_RESUME_TARGET]) + mockGetActiveResumeCancellationTarget.mockResolvedValue(ACTIVE_RESUME_TARGET) + dbChainMockFns.limit.mockResolvedValueOnce([ + { + executionDeadlineAt: null, + executionOrigin: null, + status: 'completed', + workspaceId: 'workspace-1', + }, + ]) + + const response = await POST(makeRequest(), makeParams()) + + expect(response.status).toBe(500) + await expect(response.json()).resolves.toEqual({ + error: 'Failed to reconcile paused execution after cancellation was rejected', + }) + expect(mockMarkExecutionCancelled).toHaveBeenCalledTimes(3) + expect(mockMarkExecutionCancelled).toHaveBeenCalledWith('resume-ex-1', { + executionDeadlineAt: null, + }) + expect(mockFinalizePausedCancellationForTerminalRun).not.toHaveBeenCalled() expect(mockReleaseExecutionSlot).not.toHaveBeenCalled() }) - it('keeps the reservation held when an unexpected workflow-group failure follows a failed cancellation', async () => { - mockResolveWorkflowExecutionOwnership.mockResolvedValue({ - belongsToWorkflow: true, - workflowGroupWorkspaceId: 'workspace-1', - priorStatus: 'running', + it('treats a concurrent cancellation as an idempotent success', async () => { + mockMarkExecutionCancelled.mockResolvedValue({ durablyRecorded: true, reason: 'recorded' }) + dbChainMockFns.limit + .mockResolvedValueOnce([ + { executionDeadlineAt: null, status: 'running', workspaceId: 'workspace-1' }, + ]) + .mockResolvedValueOnce([{ status: 'cancelled' }]) + dbChainMockFns.returning.mockResolvedValueOnce([]) + + const response = await POST(makeRequest(), makeParams()) + + expect(response.status).toBe(200) + await expect(response.json()).resolves.toMatchObject({ success: true, reason: 'recorded' }) + expect(mockClearExecutionCancellation).not.toHaveBeenCalled() + expect(mockWriteTerminalEvent).toHaveBeenCalledTimes(1) + expect(mockReleaseExecutionSlot).toHaveBeenCalledWith('ex-1') + }) + + it('updates execution log status in DB when durably recorded', async () => { + const mockReturning = vi.fn().mockResolvedValue([{ status: 'cancelled' }]) + const mockWhere = vi.fn(() => ({ returning: mockReturning })) + const mockSet = vi.fn(() => ({ where: mockWhere })) + databaseMock.db.update.mockReturnValueOnce({ set: mockSet }) + mockMarkExecutionCancelled.mockResolvedValue({ + durablyRecorded: true, + reason: 'recorded', }) + + const response = await POST(makeRequest(), makeParams()) + + expect(response.status).toBe(200) + await expect(response.json()).resolves.toMatchObject({ success: true, reason: 'recorded' }) + expect(databaseMock.db.update).toHaveBeenCalled() + expect(mockSet).toHaveBeenCalledWith({ + status: 'cancelled', + endedAt: expect.any(Date), + totalDurationMs: expect.anything(), + executionDeadlineAt: null, + }) + }) + + it('updates execution log status in DB when locally aborted', async () => { + const mockReturning = vi.fn().mockResolvedValue([{ status: 'cancelled' }]) + const mockWhere = vi.fn(() => ({ returning: mockReturning })) + const mockSet = vi.fn(() => ({ where: mockWhere })) + databaseMock.db.update.mockReturnValueOnce({ set: mockSet }) mockMarkExecutionCancelled.mockResolvedValue({ durablyRecorded: false, reason: 'redis_unavailable', }) - mockCancelWorkflowGroupExecution.mockRejectedValue(new Error('serialization conflict')) + mockAbortManualExecution.mockReturnValue(true) - await expect(cancelWorkflowExecution(INPUT)).rejects.toThrow('serialization conflict') - expect(mockReleaseExecutionSlot).not.toHaveBeenCalled() + const response = await POST(makeRequest(), makeParams()) + + expect(response.status).toBe(200) + await expect(response.json()).resolves.toMatchObject({ + success: true, + reason: 'redis_unavailable', + }) + expect(databaseMock.db.update).toHaveBeenCalled() + expect(mockSet).toHaveBeenCalledWith({ + status: 'cancelled', + endedAt: expect.any(Date), + totalDurationMs: expect.anything(), + executionDeadlineAt: null, + }) }) - it('leaves a standalone execution untouched by the workflow-group path', async () => { - await cancelWorkflowExecution(INPUT) + it('claims the execution log before finalizing a paused cancellation', async () => { + mockStagePausedCancellation.mockResolvedValue({ kind: 'idle' }) - expect(mockCancelWorkflowGroupExecution).not.toHaveBeenCalled() - expect(mockPublishWorkflowGroupCancellationEvent).not.toHaveBeenCalled() - expect(mockUpdateSet).toHaveBeenCalledWith(expect.objectContaining({ status: 'cancelled' })) + await POST(makeRequest(), makeParams()) + + expect(databaseMock.db.update).toHaveBeenCalled() }) - /** - * A cancelled run is terminal, so it owes the same two fields every other - * terminal write records. Without the duration it is invisible to the - * `minDurationMs`/`maxDurationMs` filters on `GET /api/v2/logs`. - */ - it('records how long the cancelled run had been going, not just when it stopped', async () => { - await cancelWorkflowExecution(INPUT) + it('does not confirm cancellation until the terminal database update succeeds', async () => { + mockMarkExecutionCancelled.mockResolvedValue({ + durablyRecorded: true, + reason: 'recorded', + }) + databaseMock.db.update.mockReturnValueOnce({ + set: vi.fn(() => ({ + where: vi.fn(() => { + throw new Error('DB connection failed') + }), + })), + }) + + const response = await POST(makeRequest(), makeParams()) - const [values] = mockUpdateSet.mock.calls.at(-1) as [Record] - expect(values.endedAt).toBeInstanceOf(Date) - expect(values.totalDurationMs).toBeDefined() + expect(response.status).toBe(200) + const data = await response.json() + expect(data).toMatchObject({ + success: false, + reason: 'cancellation_not_finalized', + }) + expect(mockClearExecutionCancellation).not.toHaveBeenCalled() }) }) diff --git a/apps/sim/lib/execution/cancel-workflow-execution.ts b/apps/sim/lib/execution/cancel-workflow-execution.ts index 10e92517854..89565495b5b 100644 --- a/apps/sim/lib/execution/cancel-workflow-execution.ts +++ b/apps/sim/lib/execution/cancel-workflow-execution.ts @@ -1,57 +1,39 @@ import { db } from '@sim/db' import { workflowExecutionLogs } from '@sim/db/schema' import { createLogger } from '@sim/logger' +import { toError } from '@sim/utils/errors' import { sleep } from '@sim/utils/helpers' -import { and, eq } from 'drizzle-orm' +import { and, eq, inArray } from 'drizzle-orm' import { releaseExecutionSlot } from '@/lib/billing/calculations/usage-reservation' import { getJobQueue } from '@/lib/core/async-jobs' +import type { ExecutionJobCancellationScope } from '@/lib/core/async-jobs/types' import { OrchestrationError } from '@/lib/core/orchestration/types' import { + clearExecutionCancellation, type ExecutionCancellationRecordResult, markExecutionCancelled, } from '@/lib/execution/cancellation' import { createExecutionEventWriter, readExecutionMetaState } from '@/lib/execution/event-buffer' import { abortManualExecution } from '@/lib/execution/manual-cancellation' +import { + isWorkflowRunAlreadyTerminalStatus, + WorkflowRunAlreadyTerminalError, +} from '@/lib/execution/workflow-run-already-terminal-error' import { cancelledExecutionLogFields } from '@/lib/logs/execution/cancellation' -import { captureServerEvent } from '@/lib/posthog/server' +import { workflowExecutionOriginSql } from '@/lib/logs/execution-origin' import { cancelWorkflowGroupExecution, type PublishableWorkflowGroupCancellation, publishWorkflowGroupCancellationEvent, - type WorkflowGroupCancellationWrites, } from '@/lib/table/workflow-group-cancellation' -import { WORKFLOW_EXECUTION_JOB_ID_PREFIX } from '@/lib/workflows/executor/execution-job-ids' -import { resolveWorkflowExecutionOwnership } from '@/lib/workflows/executor/execution-queries' import { PauseResumeManager } from '@/lib/workflows/executor/human-in-the-loop-manager' const logger = createLogger('CancelWorkflowExecution') const PAUSED_CANCELLATION_DB_ATTEMPTS = 3 const PAUSED_CANCELLATION_DB_RETRY_MS = 200 +const CANCELLATION_ABORTED_MESSAGE = + 'Request aborted before workflow run cancellation could be applied.' -async function cancelActiveWorkflowJob(executionId: string): Promise { - try { - const queue = await getJobQueue() - const job = await queue.getJob(`${WORKFLOW_EXECUTION_JOB_ID_PREFIX}${executionId}`) - if (!job || (job.status !== 'pending' && job.status !== 'processing')) return false - await queue.cancelJob(job.id) - logger.info('Cancelled active workflow queue job', { executionId, jobId: job.id }) - return true - } catch (error) { - logger.warn('Failed to cancel active workflow queue job', { executionId, error }) - return false - } -} - -/** - * Cancellation outcome vocabulary produced by this service, and so the whole - * vocabulary the public v2 endpoint can return. `recorded`/`redis_unavailable`/ - * `redis_write_failed` come from the Redis record step; the two `paused_*` - * values from the paused-HITL path; the three `already_*` values report a run - * that was already terminal when the request arrived, where the cancel claim - * matched no row and nothing durable was written. The internal cancel route - * resolves further outcomes on top of these — see - * `internalCancelWorkflowExecutionReasonSchema` in `lib/api/contracts/workflows`. - */ export type CancelWorkflowExecutionReason = | 'recorded' | 'already_cancelled' @@ -61,107 +43,226 @@ export type CancelWorkflowExecutionReason = | 'redis_write_failed' | 'paused_event_publish_failed' | 'paused_database_cancel_failed' + | 'queue_cancelled' + | 'active_resume_signal_failed' + | 'cancellation_not_finalized' -/** Maps each log status a cancel claim can never move to the outcome that reports it. */ -const TERMINAL_NO_OP_REASONS = { - cancelled: 'already_cancelled', - completed: 'already_completed', - failed: 'already_failed', -} as const satisfies Record - -type TerminalExecutionStatus = keyof typeof TERMINAL_NO_OP_REASONS - -function toTerminalExecutionStatus( - status: string | null | undefined -): TerminalExecutionStatus | null { - return typeof status === 'string' && status in TERMINAL_NO_OP_REASONS - ? (status as TerminalExecutionStatus) - : null +export interface CancelWorkflowExecutionResult { + success: boolean + executionId: string + redisAvailable: boolean + durablyRecorded: boolean + locallyAborted: boolean + pausedCancelled: boolean + reason?: CancelWorkflowExecutionReason } -/** - * What this request's own terminal claim did: moved the run to `cancelled` here - * and now, provably matched no row, or ran on a path that cannot tell. Every - * path that can terminalize the run — the direct log claim and the - * workflow-group transition — answers in this one vocabulary, so the report can - * ask a single question: did this request durably write? - * - * Only the direct claim ever answers `unknown`, and only when it could not run - * or its statement failed. The workflow-group transition always knows: it - * reports the writes it performed. - */ -type TerminalWriteOutcome = 'applied' | 'no_row' | 'unknown' +async function cancelQueuedExecutionJobs( + workflowId: string, + executionId: string, + scope: ExecutionJobCancellationScope +): Promise { + try { + const queue = await getJobQueue() + return await queue.cancelByExecution({ workflowId, executionId }, scope) + } catch (error) { + logger.warn('Failed to cancel queued execution jobs', { + workflowId, + executionId, + error: toError(error).message, + }) + return 0 + } +} -/** - * Reads a workflow-group transition's durability off the writes it reported - * rather than off its `kind`. Terminalizing the workflow log and cancelling the - * cell sidecar are each a durable write this request performed, and a single - * `kind` covers both a transition that did one of them and one that did - * neither: `already_cancelled` leaves a sidecar that was already `cancelled` - * alone, but may still have terminalized an active workflow log. - */ -function toTerminalWriteOutcome(writes: WorkflowGroupCancellationWrites): TerminalWriteOutcome { - return writes.workflowLogTerminalized || writes.sidecarCancelled ? 'applied' : 'no_row' +function abortLocalExecution(executionId: string): boolean { + try { + return abortManualExecution(executionId) + } catch (error) { + logger.warn('Failed to abort local execution', { + executionId, + error: toError(error).message, + }) + return false + } +} + +interface ExecutionStopSignalResult { + cancellation: ExecutionCancellationRecordResult + locallyAborted: boolean + queueJobsCancelled: number + accepted: boolean +} + +interface ExecutionStopSummary extends ExecutionStopSignalResult { + signalledExecutionIds: Set +} + +type ActiveResumeCancellationTarget = NonNullable< + Awaited> +> + +function createExecutionStopSummary(): ExecutionStopSummary { + return { + cancellation: { durablyRecorded: false, reason: 'redis_unavailable' }, + locallyAborted: false, + queueJobsCancelled: 0, + accepted: false, + signalledExecutionIds: new Set(), + } +} + +function mergeExecutionStopSignal( + summary: ExecutionStopSummary, + signalExecutionId: string, + result: ExecutionStopSignalResult +): void { + if (result.cancellation.durablyRecorded || !summary.cancellation.durablyRecorded) { + summary.cancellation = result.cancellation + } + summary.locallyAborted = summary.locallyAborted || result.locallyAborted + summary.queueJobsCancelled += result.queueJobsCancelled + summary.accepted = summary.accepted || result.accepted + summary.signalledExecutionIds.add(signalExecutionId) } /** - * Names the terminal state the cancel could not move, or `null` when it did - * real work or when this path cannot tell — in which case the caller keeps the - * undifferentiated report rather than guessing. - * - * A request that durably wrote is never a no-op, whatever the entry snapshot - * said. A run can be terminal at entry and still owe this request a real write: - * a workflow-group run whose log is already `cancelled` can carry a sidecar left - * in `error`, and reconciling it is a durable cancellation that the entry - * snapshot cannot see. - * - * The status read at entry is not enough on its own in the other direction - * either: a run that finishes between that read and the claim leaves a stale - * non-terminal snapshot behind a cancel that wrote nothing. The claim's own row - * count settles that, and a plain post-read cannot: after a successful cancel - * the row reads `cancelled` too, so the state has to be attributed to whoever - * wrote it. A claim that moved no row against a non-terminal snapshot re-reads - * the row it lost the race to, through the same ownership query the entry read - * came from. - * - * Purely observational — it gates no effect, and a read failure falls back to - * the undifferentiated report rather than failing the cancel. + * Commits cancellation after the caller's final abort check. Once signalling begins, the + * operation must finish reconciliation because workers may already have observed the durable, + * local, or queued signal; attempting to honor a later abort could revive only part of a run. */ -async function resolveTerminalNoOpReason( +async function signalExecutionStop(args: { + workflowId: string + signalExecutionId: string + queueBindingExecutionId?: string + executionDeadlineAt: Date | null + queueScope?: ExecutionJobCancellationScope +}): Promise { + const cancellation = await markExecutionCancelled(args.signalExecutionId, { + executionDeadlineAt: args.executionDeadlineAt, + }) + const locallyAborted = abortLocalExecution(args.signalExecutionId) + const queueJobsCancelled = args.queueScope + ? await cancelQueuedExecutionJobs( + args.workflowId, + args.queueBindingExecutionId ?? args.signalExecutionId, + args.queueScope + ) + : 0 + return { + cancellation, + locallyAborted, + queueJobsCancelled, + accepted: cancellation.durablyRecorded || locallyAborted || queueJobsCancelled > 0, + } +} + +async function signalAndRecordActiveResumeStop(args: { + workflowId: string + executionId: string + executionDeadlineAt: Date | null + target: ActiveResumeCancellationTarget + summary: ExecutionStopSummary +}): Promise { + const signal = await signalExecutionStop({ + workflowId: args.workflowId, + signalExecutionId: args.target.resumeExecutionId, + queueBindingExecutionId: args.executionId, + executionDeadlineAt: args.executionDeadlineAt, + queueScope: 'resume', + }) + mergeExecutionStopSignal(args.summary, args.target.resumeExecutionId, signal) + return didActiveResumeStop(args.executionId, args.workflowId, args.target, signal) +} + +async function didActiveResumeStop( executionId: string, workflowId: string, - priorTerminalStatus: TerminalExecutionStatus | null, - terminalWrite: TerminalWriteOutcome -): Promise { - if (terminalWrite === 'applied') return null - if (priorTerminalStatus !== null) return TERMINAL_NO_OP_REASONS[priorTerminalStatus] - if (terminalWrite !== 'no_row') return null - try { - const { priorStatus } = await resolveWorkflowExecutionOwnership(executionId, workflowId) - const terminalStatus = toTerminalExecutionStatus(priorStatus) - return terminalStatus !== null ? TERMINAL_NO_OP_REASONS[terminalStatus] : null - } catch (error) { - logger.warn('Failed to re-read execution status after an unmatched cancel claim', { + target: ActiveResumeCancellationTarget, + signal: ExecutionStopSignalResult +): Promise { + if (signal.cancellation.durablyRecorded || signal.locallyAborted) return true + const currentTarget = await PauseResumeManager.getActiveResumeCancellationTarget( + executionId, + workflowId + ) + if (currentTarget?.resumeEntryId === target.resumeEntryId && signal.queueJobsCancelled > 0) { + return true + } + if (currentTarget && currentTarget.resumeEntryId !== target.resumeEntryId) { + logger.warn('A replacement resume became active while cancellation was staged', { executionId, - error, + previousResumeEntryId: target.resumeEntryId, + currentResumeEntryId: currentTarget.resumeEntryId, }) - return null } + return currentTarget === null } -export interface CancelWorkflowExecutionResult { - success: boolean +type PausedCancellationStage = Awaited< + ReturnType +> + +function isPausedCancellationStage( + stage: PausedCancellationStage +): stage is Exclude { + return stage.kind !== 'not_paused' +} + +async function clearStopSignalMarkers(summary: ExecutionStopSummary): Promise { + await Promise.all( + [...summary.signalledExecutionIds].map((executionId) => clearExecutionCancellation(executionId)) + ) +} + +type ExecutionLogCancellationClaim = + | { kind: 'cancelled' } + | { kind: 'conflict'; status: string } + | { kind: 'not_found' } + +async function claimExecutionLogCancellation(args: { executionId: string - redisAvailable: boolean - durablyRecorded: boolean - locallyAborted: boolean - pausedCancelled: boolean - reason?: CancelWorkflowExecutionReason + workflowId: string + workspaceId: string +}): Promise { + const now = new Date() + const [cancelledExecution] = await db + .update(workflowExecutionLogs) + .set(cancelledExecutionLogFields(now)) + .where( + and( + eq(workflowExecutionLogs.executionId, args.executionId), + eq(workflowExecutionLogs.workflowId, args.workflowId), + eq(workflowExecutionLogs.workspaceId, args.workspaceId), + inArray(workflowExecutionLogs.status, ['running', 'pending']) + ) + ) + .returning({ status: workflowExecutionLogs.status }) + + if (cancelledExecution?.status === 'cancelled') return { kind: 'cancelled' } + + const currentExecution = await db + .select({ status: workflowExecutionLogs.status }) + .from(workflowExecutionLogs) + .where( + and( + eq(workflowExecutionLogs.executionId, args.executionId), + eq(workflowExecutionLogs.workflowId, args.workflowId), + eq(workflowExecutionLogs.workspaceId, args.workspaceId) + ) + ) + .limit(1) + .then((rows) => rows[0]) + + if (!currentExecution) return { kind: 'not_found' } + if (currentExecution.status === 'cancelled') return { kind: 'cancelled' } + return { kind: 'conflict', status: currentExecution.status } } async function completePausedCancellationWithRetry( executionId: string, - workflowId: string + workflowId: string, + options: { logMissing?: boolean } = {} ): Promise { for (let attempt = 1; attempt <= PAUSED_CANCELLATION_DB_ATTEMPTS; attempt++) { try { @@ -170,10 +271,12 @@ async function completePausedCancellationWithRetry( logger.info('Paused execution cancelled in database', { executionId, attempt }) return true } - logger.warn('Paused execution cancellation could not be completed in database', { - executionId, - attempt, - }) + if (options.logMissing !== false) { + logger.warn('Paused execution cancellation could not be completed in database', { + executionId, + attempt, + }) + } return false } catch (error) { logger.warn('Failed to complete paused execution cancellation in database', { @@ -189,14 +292,142 @@ async function completePausedCancellationWithRetry( return false } -async function ensurePausedCancellationEventPublished( +async function clearPausedCancellationIntentWithRetry( + executionId: string, + workflowId: string +): Promise { + for (let attempt = 1; attempt <= PAUSED_CANCELLATION_DB_ATTEMPTS; attempt++) { + try { + await PauseResumeManager.clearPausedCancellationIntent(executionId, workflowId) + return true + } catch (error) { + logger.warn('Failed to clear paused cancellation intent', { + executionId, + attempt, + error: toError(error).message, + }) + if (attempt < PAUSED_CANCELLATION_DB_ATTEMPTS) { + await sleep(PAUSED_CANCELLATION_DB_RETRY_MS) + } + } + } + return false +} + +async function finalizePausedCancellationForTerminalRunWithRetry( + executionId: string, + workflowId: string, + executionDeadlineAt: Date | null, + stopSummary: ExecutionStopSummary +): Promise { + for (let attempt = 1; attempt <= PAUSED_CANCELLATION_DB_ATTEMPTS; attempt++) { + try { + const activeResumeTargets = await PauseResumeManager.getActiveResumeCancellationTargets( + executionId, + workflowId + ) + const stoppedResumeEntryIds: string[] = [] + for (const target of activeResumeTargets) { + const stopped = await signalAndRecordActiveResumeStop({ + workflowId, + executionId, + executionDeadlineAt, + target, + summary: stopSummary, + }) + if (!stopped) break + stoppedResumeEntryIds.push(target.resumeEntryId) + } + + if (stoppedResumeEntryIds.length !== activeResumeTargets.length) { + logger.warn('Claimed resume could not be stopped during terminal cleanup', { + executionId, + attempt, + }) + if (attempt < PAUSED_CANCELLATION_DB_ATTEMPTS) { + await sleep(PAUSED_CANCELLATION_DB_RETRY_MS) + } + continue + } + + const finalized = await PauseResumeManager.finalizePausedCancellationForTerminalRun( + executionId, + workflowId, + stoppedResumeEntryIds + ) + if (finalized) return true + logger.warn('Paused cancellation terminal cleanup was rejected', { + executionId, + attempt, + }) + } catch (error) { + logger.warn('Failed to finalize paused cancellation after terminal race', { + executionId, + attempt, + error: toError(error).message, + }) + } + if (attempt < PAUSED_CANCELLATION_DB_ATTEMPTS) { + await sleep(PAUSED_CANCELLATION_DB_RETRY_MS) + } + } + return false +} + +async function restorePausedCancellationAfterRejectedCommit(args: { + executionId: string + workflowId: string + effectivePausedCancellationPath: boolean + activeResumeEntryId: string | null +}): Promise { + if (!args.effectivePausedCancellationPath) return true + + if (args.activeResumeEntryId) { + try { + const rolledBack = await PauseResumeManager.rollbackActiveResumeCancellation( + args.executionId, + args.workflowId, + args.activeResumeEntryId + ) + if (rolledBack) return true + logger.warn('Active resume rollback was rejected; clearing paused cancellation intent', { + executionId: args.executionId, + activeResumeEntryId: args.activeResumeEntryId, + }) + } catch (error) { + logger.warn('Active resume rollback failed; clearing paused cancellation intent', { + executionId: args.executionId, + activeResumeEntryId: args.activeResumeEntryId, + error: toError(error).message, + }) + } + } + + return clearPausedCancellationIntentWithRetry(args.executionId, args.workflowId) +} + +function throwPausedCancellationReconciliationFailed(): never { + throw new OrchestrationError( + 'internal', + 'Failed to reconcile paused execution after cancellation was rejected' + ) +} + +async function ensureCancellationEventPublished( executionId: string, workflowId: string, context: { workspaceId?: string; userId?: string } = {} ): Promise { - const metaState = await readExecutionMetaState(executionId) - if (metaState.status === 'found' && metaState.meta.status === 'cancelled') { - return true + try { + const metaState = await readExecutionMetaState(executionId) + if (metaState.status === 'found' && metaState.meta.status === 'cancelled') { + return true + } + } catch (error) { + logger.warn('Failed to read execution state before publishing cancellation', { + executionId, + error: toError(error).message, + }) } const writer = createExecutionEventWriter(executionId, { @@ -217,14 +448,14 @@ async function ensurePausedCancellationEventPublished( ) return true } catch (error) { - logger.warn('Failed to publish paused execution cancellation event', { + logger.warn('Failed to publish execution cancellation event', { executionId, error, }) return false } finally { await writer.close().catch((error) => { - logger.warn('Failed to close paused cancellation event writer', { + logger.warn('Failed to close cancellation event writer', { executionId, error, }) @@ -233,14 +464,13 @@ async function ensurePausedCancellationEventPublished( } export interface CancelWorkflowExecutionInput { - executionId: string workflowId: string - /** Actor for the analytics event. */ - userId: string - /** Workflow's workspace; feeds the event writer + analytics grouping. */ - workspaceId?: string - /** Legacy callers emit product analytics here; migrated adapters emit it after success. */ - captureAnalytics?: boolean + executionId: string + /** Human attribution resolved by the authorized application use case. */ + attributedUserId: string + /** Canonical workspace resolved with the workflow run. */ + workspaceId: string + abortSignal?: AbortSignal } export class WorkflowExecutionNotFoundError extends Error { @@ -250,312 +480,722 @@ export class WorkflowExecutionNotFoundError extends Error { } } -/** - * Cancels a workflow execution across the Redis abort record, the in-process - * aborter, the paused-HITL machinery, the workflow-group table cell sidecar, and - * the plan concurrency reservation. The interleaving is order-sensitive. Auth is - * the caller's responsibility; this throws on unexpected infrastructure errors. - * - * A workflow-group run whose cell sidecar refuses the claim throws - * `OrchestrationError('conflict')` rather than returning a success-shaped result: - * the cancel did not fully happen, and the internal route already answers 409 for - * exactly these two outcomes. - */ -export async function cancelWorkflowExecution( - input: CancelWorkflowExecutionInput -): Promise { - const { executionId, workflowId, userId, workspaceId } = input - - const { belongsToWorkflow, workflowGroupWorkspaceId, priorStatus } = - await resolveWorkflowExecutionOwnership(executionId, workflowId) - if (!belongsToWorkflow) throw new WorkflowExecutionNotFoundError() - const priorTerminalStatus = toTerminalExecutionStatus(priorStatus) - - let pausedCancellationStarted = false - let pausedCancelled = false +function throwCancellationAborted(): never { + throw new OrchestrationError('conflict', CANCELLATION_ABORTED_MESSAGE) +} + +function throwIfCancellationAborted(abortSignal?: AbortSignal): void { + if (abortSignal?.aborted) throwCancellationAborted() +} + +async function rollbackPausedCancellationAfterAbort(args: { + stage: PausedCancellationStage + workflowId: string + executionId: string + abortSignal?: AbortSignal +}): Promise { + if (!args.abortSignal?.aborted) return false + try { - pausedCancellationStarted = await PauseResumeManager.beginPausedCancellation( - executionId, - workflowId - ) + if (args.stage.kind === 'active_resume') { + const rolledBack = await PauseResumeManager.rollbackActiveResumeCancellation( + args.executionId, + args.workflowId, + args.stage.target.resumeEntryId + ) + if (!rolledBack) { + logger.warn('Aborted cancellation could not be rolled back; completing cancellation', { + executionId: args.executionId, + activeResumeEntryId: args.stage.target.resumeEntryId, + }) + return false + } + } else if (args.stage.kind === 'idle') { + await PauseResumeManager.clearPausedCancellationIntent(args.executionId, args.workflowId) + } } catch (error) { - logger.warn('Failed to begin paused execution cancellation in database', { - executionId, - error, + logger.warn('Failed to roll back aborted cancellation; completing cancellation', { + executionId: args.executionId, + stageKind: args.stage.kind, + error: toError(error).message, }) + return false } - const pendingPausedCancellation = pausedCancellationStarted - ? null - : await PauseResumeManager.getPausedCancellationStatus(executionId, workflowId) - const isPausedCancellationPath = pausedCancellationStarted || pendingPausedCancellation !== null - - const cancellation: ExecutionCancellationRecordResult = isPausedCancellationPath - ? { durablyRecorded: false, reason: 'redis_unavailable' } - : await markExecutionCancelled(executionId) - const locallyAborted = isPausedCancellationPath ? false : abortManualExecution(executionId) - const queuedJobCancelled = isPausedCancellationPath - ? false - : await cancelActiveWorkflowJob(executionId) - - if (pausedCancellationStarted) { - logger.info('Paused execution cancellation reserved in database', { executionId }) - } else if (cancellation.durablyRecorded) { - logger.info('Execution marked as cancelled in Redis', { executionId }) - } else if (queuedJobCancelled) { - logger.info('Execution cancelled in workflow queue', { executionId }) - } else if (locallyAborted) { - logger.info('Execution cancelled via local in-process fallback', { executionId }) - } else if (!pausedCancellationStarted) { - logger.warn('Execution cancellation was not durably recorded', { - executionId, - reason: cancellation.reason, + + return true +} + +async function rollbackActiveResumeAfterFailedSignal(args: { + executionId: string + workflowId: string + resumeEntryId: string +}): Promise { + try { + const rolledBack = await PauseResumeManager.rollbackActiveResumeCancellation( + args.executionId, + args.workflowId, + args.resumeEntryId + ) + if (!rolledBack) { + logger.warn('Active resume cancellation could not be rolled back; completing cancellation', { + executionId: args.executionId, + activeResumeEntryId: args.resumeEntryId, + }) + } + return rolledBack + } catch (error) { + logger.warn('Failed to roll back active resume cancellation; completing cancellation', { + executionId: args.executionId, + activeResumeEntryId: args.resumeEntryId, + error: toError(error).message, }) + return false } +} +function activeResumeSignalFailureResult( + executionId: string, + stopSummary: ExecutionStopSummary +): CancelWorkflowExecutionResult { + return { + success: false, + executionId, + redisAvailable: stopSummary.cancellation.reason !== 'redis_unavailable', + durablyRecorded: stopSummary.cancellation.durablyRecorded, + locallyAborted: stopSummary.locallyAborted, + pausedCancelled: false, + reason: 'active_resume_signal_failed', + } +} + +function resolveCancellationReason(args: { + activeResumeSignalFailed: boolean + pauseReconciliationFailed: boolean + effectivePausedCancellationPath: boolean + cancellationEventPublished: boolean + pausedCancelled: boolean + stopSummary: ExecutionStopSummary +}): CancelWorkflowExecutionReason { + if (args.activeResumeSignalFailed) return 'active_resume_signal_failed' + if (args.pauseReconciliationFailed) return 'paused_database_cancel_failed' + if (args.effectivePausedCancellationPath && !args.cancellationEventPublished) { + return 'paused_event_publish_failed' + } + if (args.effectivePausedCancellationPath && !args.pausedCancelled) { + return 'paused_database_cancel_failed' + } + if (args.effectivePausedCancellationPath) return 'recorded' if ( - !isPausedCancellationPath && - (cancellation.durablyRecorded || queuedJobCancelled || locallyAborted) + args.stopSummary.queueJobsCancelled > 0 && + !args.stopSummary.cancellation.durablyRecorded && + !args.stopSummary.locallyAborted ) { - await PauseResumeManager.blockQueuedResumesForCancellation(executionId, workflowId).catch( - (error) => { - logger.warn('Failed to block queued paused resumes after cancellation', { + return 'queue_cancelled' + } + return args.stopSummary.cancellation.reason +} + +/** + * Applies the full queued, active, paused, resumed, and workflow-group + * cancellation lifecycle to an already-authorized canonical workflow run. + * Authorization and principal handling belong to `cancelWorkflowRun`; this + * service accepts only canonical identifiers and returns transport-neutral + * results or orchestration errors. + */ +export async function cancelWorkflowExecution({ + workflowId, + executionId, + attributedUserId, + workspaceId, + abortSignal, +}: CancelWorkflowExecutionInput): Promise { + try { + throwIfCancellationAborted(abortSignal) + + const execution = await db + .select({ + executionDeadlineAt: workflowExecutionLogs.executionDeadlineAt, + executionOrigin: workflowExecutionOriginSql(), + status: workflowExecutionLogs.status, + workspaceId: workflowExecutionLogs.workspaceId, + }) + .from(workflowExecutionLogs) + .where( + and( + eq(workflowExecutionLogs.executionId, executionId), + eq(workflowExecutionLogs.workflowId, workflowId), + eq(workflowExecutionLogs.workspaceId, workspaceId) + ) + ) + .limit(1) + .then((rows) => rows[0]) + + throwIfCancellationAborted(abortSignal) + + if (!execution) { + const queueJobsCancelled = await cancelQueuedExecutionJobs( + workflowId, + executionId, + 'standalone' + ) + if (queueJobsCancelled > 0) { + const locallyAborted = abortLocalExecution(executionId) + const cancellation = await markExecutionCancelled(executionId) + await PauseResumeManager.blockQueuedResumesForCancellation(executionId, workflowId).catch( + (error) => { + logger.warn('Failed to block queued resumes after queued-run cancellation', { + executionId, + error, + }) + } + ) + await releaseExecutionSlot(executionId).catch((error) => { + logger.warn('Failed to release reservation after queued-run cancellation', { + executionId, + error, + }) + }) + + return { + success: true, + executionId, + redisAvailable: cancellation.reason !== 'redis_unavailable', + durablyRecorded: cancellation.durablyRecorded, + locallyAborted, + pausedCancelled: false, + reason: 'queue_cancelled', + } + } + + throw new WorkflowExecutionNotFoundError() + } + + const isWorkflowGroupExecution = execution.executionOrigin === 'workflow_group' + + if (execution.status === 'cancelled') { + let groupCancellationToPublish: PublishableWorkflowGroupCancellation | null = null + let groupCancellationCommitted = false + if (isWorkflowGroupExecution) { + throwIfCancellationAborted(abortSignal) + + const workflowGroupCancellation = await cancelWorkflowGroupExecution({ + workspaceId: execution.workspaceId, + workflowId, executionId, - error, }) + if (workflowGroupCancellation.kind === 'conflict') { + throw new OrchestrationError( + 'conflict', + `Workflow group execution cannot be reconciled while ${workflowGroupCancellation.status}` + ) + } + if (workflowGroupCancellation.kind === 'not_workflow_group') { + throw new OrchestrationError( + 'conflict', + 'Workflow group execution is no longer the active table execution' + ) + } + if ( + workflowGroupCancellation.kind === 'cancelled' || + workflowGroupCancellation.kind === 'already_cancelled' + ) { + groupCancellationToPublish = workflowGroupCancellation + groupCancellationCommitted = true + } } - ) - } else if (!isPausedCancellationPath) { - await PauseResumeManager.clearPausedCancellationIntent(executionId, workflowId).catch( - (error) => { - logger.warn('Failed to clear paused cancellation intent after unsuccessful cancellation', { + + const stopSummary = createExecutionStopSummary() + let pausedCancelled = false + const pausedCancellationStage = await PauseResumeManager.stagePausedCancellation( + executionId, + workflowId + ) + if ( + !groupCancellationCommitted && + (await rollbackPausedCancellationAfterAbort({ + stage: pausedCancellationStage, + workflowId, + executionId, + abortSignal, + })) + ) { + throwCancellationAborted() + } + + const hasPausedCancellation = isPausedCancellationStage(pausedCancellationStage) + const requiresCancellationEvent = hasPausedCancellation || isWorkflowGroupExecution + let cancellationEventPublished = !requiresCancellationEvent + let activeResumeSignalFailed = false + let exactStopSatisfied = true + if (pausedCancellationStage.kind === 'active_resume') { + exactStopSatisfied = await signalAndRecordActiveResumeStop({ + workflowId, executionId, - error, + executionDeadlineAt: execution.executionDeadlineAt, + target: pausedCancellationStage.target, + summary: stopSummary, + }) + activeResumeSignalFailed = !exactStopSatisfied + } else if (isWorkflowGroupExecution && !hasPausedCancellation) { + const retrySignal = await signalExecutionStop({ + workflowId, + signalExecutionId: executionId, + executionDeadlineAt: execution.executionDeadlineAt, }) + mergeExecutionStopSignal(stopSummary, executionId, retrySignal) + exactStopSatisfied = retrySignal.accepted } - ) - } - let pausedCancellationPublished = false - let pausedCancellationPublishFailed = false - if (pausedCancellationStarted) { - pausedCancellationPublished = await ensurePausedCancellationEventPublished( - executionId, - workflowId, - { workspaceId, userId } - ) - pausedCancellationPublishFailed = !pausedCancellationPublished - if (pausedCancellationPublished) { - pausedCancelled = await completePausedCancellationWithRetry(executionId, workflowId) + if (groupCancellationToPublish && exactStopSatisfied) { + await publishWorkflowGroupCancellationEvent(groupCancellationToPublish, executionId) + } + + if (requiresCancellationEvent && exactStopSatisfied) { + cancellationEventPublished = await ensureCancellationEventPublished( + executionId, + workflowId, + { + workspaceId: execution.workspaceId, + userId: attributedUserId, + } + ) + } + if (hasPausedCancellation && cancellationEventPublished && exactStopSatisfied) { + pausedCancelled = await completePausedCancellationWithRetry(executionId, workflowId, { + logMissing: false, + }) + } + + if (exactStopSatisfied) { + await releaseExecutionSlot(executionId).catch((error) => { + logger.warn('Failed to release reservation while reconciling cancelled execution', { + executionId, + error: toError(error).message, + }) + }) + } + + if (pausedCancelled) { + await clearStopSignalMarkers(stopSummary) + } + + const pausedReconciliationSucceeded = + exactStopSatisfied && + (!hasPausedCancellation || (cancellationEventPublished && pausedCancelled)) + return { + success: pausedReconciliationSucceeded, + executionId, + redisAvailable: requiresCancellationEvent ? cancellationEventPublished : true, + durablyRecorded: false, + locallyAborted: stopSummary.locallyAborted, + pausedCancelled, + reason: activeResumeSignalFailed + ? 'active_resume_signal_failed' + : !exactStopSatisfied + ? stopSummary.cancellation.reason + : hasPausedCancellation && !cancellationEventPublished + ? 'paused_event_publish_failed' + : hasPausedCancellation && !pausedCancelled + ? 'paused_database_cancel_failed' + : 'already_cancelled', + } } - } else { - if (pendingPausedCancellation === 'cancelled') { - pausedCancellationPublished = await ensurePausedCancellationEventPublished( + + if (execution.status !== 'running' && execution.status !== 'pending') { + const stopSummary = createExecutionStopSummary() + const pausedCancellationFinalized = await finalizePausedCancellationForTerminalRunWithRetry( executionId, workflowId, - { workspaceId, userId } + execution.executionDeadlineAt, + stopSummary + ) + if (!pausedCancellationFinalized) throwPausedCancellationReconciliationFailed() + + if (!isWorkflowGroupExecution && isWorkflowRunAlreadyTerminalStatus(execution.status)) { + throw new WorkflowRunAlreadyTerminalError({ + executionId, + executionStatus: execution.status, + redisAvailable: true, + locallyAborted: false, + }) + } + throw new OrchestrationError( + 'conflict', + `Execution cannot be cancelled while ${execution.status}` ) - pausedCancellationPublishFailed = !pausedCancellationPublished - pausedCancelled = pausedCancellationPublished - } else if (pendingPausedCancellation === 'cancelling') { - pausedCancellationPublished = await ensurePausedCancellationEventPublished( + } + + logger.info('Cancel execution requested', { workflowId, executionId, attributedUserId }) + + const stopSummary = createExecutionStopSummary() + let pausedCancelled = false + let pausedCancellationStage = await PauseResumeManager.stagePausedCancellation( + executionId, + workflowId + ) + if ( + await rollbackPausedCancellationAfterAbort({ + stage: pausedCancellationStage, + workflowId, executionId, + abortSignal, + }) + ) { + throwCancellationAborted() + } + + let effectivePausedCancellationPath = isPausedCancellationStage(pausedCancellationStage) + let activeResumeTarget = + pausedCancellationStage.kind === 'active_resume' ? pausedCancellationStage.target : null + let activeResumeEntryId = activeResumeTarget?.resumeEntryId ?? null + let activeResumeSignalAccepted = false + const activeResumeTargetsNeedingStopConfirmation: ActiveResumeCancellationTarget[] = [] + + if (activeResumeTarget && !isWorkflowGroupExecution) { + activeResumeSignalAccepted = await signalAndRecordActiveResumeStop({ workflowId, - { workspaceId, userId } - ) - pausedCancellationPublishFailed = !pausedCancellationPublished - if (pausedCancellationPublished) { - pausedCancelled = await completePausedCancellationWithRetry(executionId, workflowId) + executionId, + executionDeadlineAt: execution.executionDeadlineAt, + target: activeResumeTarget, + summary: stopSummary, + }) + + if (!activeResumeSignalAccepted) { + const failedResumeEntryId = activeResumeTarget.resumeEntryId + const rolledBack = await rollbackActiveResumeAfterFailedSignal({ + executionId, + workflowId, + resumeEntryId: failedResumeEntryId, + }) + if (rolledBack) { + await clearStopSignalMarkers(stopSummary) + return activeResumeSignalFailureResult(executionId, stopSummary) + } + activeResumeTargetsNeedingStopConfirmation.push(activeResumeTarget) + } + } else if (!effectivePausedCancellationPath && !isWorkflowGroupExecution) { + const signal = await signalExecutionStop({ + workflowId, + signalExecutionId: executionId, + executionDeadlineAt: execution.executionDeadlineAt, + queueScope: 'standalone', + }) + mergeExecutionStopSignal(stopSummary, executionId, signal) + + if (!signal.accepted) { + pausedCancellationStage = await PauseResumeManager.stagePausedCancellation( + executionId, + workflowId + ) + const postLateStageAbort = await rollbackPausedCancellationAfterAbort({ + stage: pausedCancellationStage, + workflowId, + executionId, + abortSignal, + }) + if (postLateStageAbort) { + await clearStopSignalMarkers(stopSummary) + throwCancellationAborted() + } + + effectivePausedCancellationPath = isPausedCancellationStage(pausedCancellationStage) + activeResumeTarget = + pausedCancellationStage.kind === 'active_resume' ? pausedCancellationStage.target : null + activeResumeEntryId = activeResumeTarget?.resumeEntryId ?? null + + if (activeResumeTarget) { + activeResumeSignalAccepted = await signalAndRecordActiveResumeStop({ + workflowId, + executionId, + executionDeadlineAt: execution.executionDeadlineAt, + target: activeResumeTarget, + summary: stopSummary, + }) + if (!activeResumeSignalAccepted) { + const failedResumeEntryId = activeResumeTarget.resumeEntryId + const rolledBack = await rollbackActiveResumeAfterFailedSignal({ + executionId, + workflowId, + resumeEntryId: failedResumeEntryId, + }) + if (rolledBack) { + await clearStopSignalMarkers(stopSummary) + return activeResumeSignalFailureResult(executionId, stopSummary) + } + activeResumeTargetsNeedingStopConfirmation.push(activeResumeTarget) + } + } else if (!effectivePausedCancellationPath) { + return { + success: false, + executionId, + redisAvailable: stopSummary.cancellation.reason !== 'redis_unavailable', + durablyRecorded: stopSummary.cancellation.durablyRecorded, + locallyAborted: stopSummary.locallyAborted, + pausedCancelled: false, + reason: stopSummary.cancellation.reason, + } + } } } - } - if ( - pausedCancellationPublishFailed && - (pausedCancellationStarted || pendingPausedCancellation === 'cancelling') - ) { - await PauseResumeManager.clearPausedCancellationIntent(executionId, workflowId).catch( - (error) => { - logger.warn('Failed to clear paused cancellation intent after publish failure', { + let terminalCancellationClaimed = false + let competingTerminalStatus: string | null = null + let workflowGroupNoLongerActive = false + let groupCancellationToPublish: PublishableWorkflowGroupCancellation | null = null + try { + if (isWorkflowGroupExecution) { + const workflowGroupCancellation = await cancelWorkflowGroupExecution({ + workspaceId: execution.workspaceId, + workflowId, executionId, - error, }) + if (workflowGroupCancellation.kind === 'conflict') { + competingTerminalStatus = workflowGroupCancellation.status + } else if (workflowGroupCancellation.kind === 'not_workflow_group') { + workflowGroupNoLongerActive = true + } else { + terminalCancellationClaimed = true + if ( + workflowGroupCancellation.kind === 'cancelled' || + workflowGroupCancellation.kind === 'already_cancelled' + ) { + groupCancellationToPublish = workflowGroupCancellation + } + } + } else { + const claim = await claimExecutionLogCancellation({ + executionId, + workflowId, + workspaceId: execution.workspaceId, + }) + if (claim.kind === 'cancelled') { + terminalCancellationClaimed = true + } else { + competingTerminalStatus = claim.kind === 'conflict' ? claim.status : 'no_longer_active' + } } - ) - } + } catch (dbError) { + logger.warn('Failed to finalize cancelled execution directly', { + executionId, + error: toError(dbError).message, + }) + } - const success = - (isPausedCancellationPath - ? pausedCancelled && pausedCancellationPublished - : cancellation.durablyRecorded || queuedJobCancelled) || locallyAborted - - /** - * Frees the plan concurrency reservation once the stop-the-work effects above - * have actually taken. The paused path keeps its reservation because a paused - * run never held an in-flight slot to give back, and an unsuccessful ordinary - * cancel keeps it because the run may still be executing. - */ - const releaseSlotForStoppedExecution = async (): Promise => { - if (!success || isPausedCancellationPath) return - await releaseExecutionSlot(executionId).catch((error) => { - logger.warn('Failed to release reservation after execution cancellation', { + if (workflowGroupNoLongerActive) { + await clearStopSignalMarkers(stopSummary) + const pausedCancellationRestored = await restorePausedCancellationAfterRejectedCommit({ executionId, - error, + workflowId, + effectivePausedCancellationPath, + activeResumeEntryId, }) - }) - } + if (!pausedCancellationRestored) throwPausedCancellationReconciliationFailed() + throw new OrchestrationError( + 'conflict', + 'Workflow group execution is no longer the active table execution' + ) + } - /** - * The sidecar transition can fail outright — a lost claim, a serialization - * conflict, a connection blip. The stop-the-work effects above have already - * fired, so the run is going down regardless and the reservation must not be - * stranded; but the cell is left in an unknown state, so the failure is - * re-thrown rather than swallowed into a success-shaped result. - */ - let groupCancellation: Awaited> | null = null - if (workflowGroupWorkspaceId) { - try { - groupCancellation = await cancelWorkflowGroupExecution({ - workspaceId: workflowGroupWorkspaceId, + if (competingTerminalStatus) { + if (isWorkflowGroupExecution) { + await clearStopSignalMarkers(stopSummary) + const pausedCancellationRestored = await restorePausedCancellationAfterRejectedCommit({ + executionId, + workflowId, + effectivePausedCancellationPath, + activeResumeEntryId, + }) + if (!pausedCancellationRestored) throwPausedCancellationReconciliationFailed() + } else if (effectivePausedCancellationPath) { + const pausedCancellationFinalized = await finalizePausedCancellationForTerminalRunWithRetry( + executionId, + workflowId, + execution.executionDeadlineAt, + stopSummary + ) + if (!pausedCancellationFinalized) throwPausedCancellationReconciliationFailed() + } else { + await clearStopSignalMarkers(stopSummary) + } + if ( + !isWorkflowGroupExecution && + isWorkflowRunAlreadyTerminalStatus(competingTerminalStatus) + ) { + throw new WorkflowRunAlreadyTerminalError({ + executionId, + executionStatus: competingTerminalStatus, + redisAvailable: stopSummary.cancellation.reason !== 'redis_unavailable', + locallyAborted: stopSummary.locallyAborted, + }) + } + throw new OrchestrationError( + 'conflict', + isWorkflowGroupExecution + ? `Workflow group execution cannot be cancelled while ${competingTerminalStatus}` + : `Execution cannot be cancelled while ${competingTerminalStatus}` + ) + } + + if (!terminalCancellationClaimed) { + if (effectivePausedCancellationPath && !stopSummary.accepted) { + if (activeResumeEntryId) { + await PauseResumeManager.rollbackActiveResumeCancellation( + executionId, + workflowId, + activeResumeEntryId + ) + } else { + await PauseResumeManager.clearPausedCancellationIntent(executionId, workflowId) + } + } + return { + success: false, + executionId, + redisAvailable: stopSummary.cancellation.reason !== 'redis_unavailable', + durablyRecorded: stopSummary.cancellation.durablyRecorded, + locallyAborted: stopSummary.locallyAborted, + pausedCancelled: false, + reason: 'cancellation_not_finalized', + } + } + + let pauseReconciliationFailed = false + let activeResumeSignalFailed = false + for (const target of activeResumeTargetsNeedingStopConfirmation) { + const stopConfirmed = await signalAndRecordActiveResumeStop({ workflowId, executionId, + executionDeadlineAt: execution.executionDeadlineAt, + target, + summary: stopSummary, }) + if (target.resumeEntryId === activeResumeEntryId) { + activeResumeSignalAccepted = stopConfirmed + } + activeResumeSignalFailed = activeResumeSignalFailed || !stopConfirmed + } + if (isWorkflowGroupExecution) { + if (activeResumeTarget) { + activeResumeSignalAccepted = await signalAndRecordActiveResumeStop({ + workflowId, + executionId, + executionDeadlineAt: execution.executionDeadlineAt, + target: activeResumeTarget, + summary: stopSummary, + }) + activeResumeSignalFailed = !activeResumeSignalAccepted + } else if (!effectivePausedCancellationPath) { + const groupSignal = await signalExecutionStop({ + workflowId, + signalExecutionId: executionId, + executionDeadlineAt: execution.executionDeadlineAt, + }) + mergeExecutionStopSignal(stopSummary, executionId, groupSignal) + } + } + + try { + const postClaimPausedCancellationStage = await PauseResumeManager.stagePausedCancellation( + executionId, + workflowId + ) + if (isPausedCancellationStage(postClaimPausedCancellationStage)) { + effectivePausedCancellationPath = true + if (postClaimPausedCancellationStage.kind === 'active_resume') { + const currentActiveResume = postClaimPausedCancellationStage.target + const alreadyAttemptedCurrentResume = + currentActiveResume.resumeEntryId === activeResumeEntryId && + stopSummary.signalledExecutionIds.has(currentActiveResume.resumeExecutionId) + if (!alreadyAttemptedCurrentResume) { + activeResumeSignalAccepted = await signalAndRecordActiveResumeStop({ + workflowId, + executionId, + executionDeadlineAt: execution.executionDeadlineAt, + target: currentActiveResume, + summary: stopSummary, + }) + } + activeResumeSignalFailed = activeResumeSignalFailed || !activeResumeSignalAccepted + } + } } catch (error) { - logger.error('Workflow group execution cancellation failed unexpectedly', { + pauseReconciliationFailed = true + effectivePausedCancellationPath = true + logger.warn('Failed to recheck paused execution after terminal cancellation claim', { executionId, - error, + error: toError(error).message, }) - await releaseSlotForStoppedExecution() - throw error } - } - - /** - * Both refusals mean the cell claim was lost, never that the run is still - * going: the sidecar conflicts only on a terminal workflow log or a terminal - * cell, and `not_workflow_group` means the log is not a group run at all. - * Every refusal is a terminal-or-absent state that carries no evidence of - * liveness, so nothing is left running to hold the reservation and it is - * released before the 409 rather than left to expire. (The Redis abort record - * is reversible — see `clearExecutionCancellation` — but a refusal gives no - * reason to reverse it.) - */ - if (groupCancellation?.kind === 'conflict') { - logger.warn('Workflow group execution could not be cancelled', { - executionId, - status: groupCancellation.status, - }) - await releaseSlotForStoppedExecution() - throw new OrchestrationError( - 'conflict', - `Workflow group execution cannot be cancelled while ${groupCancellation.status}` - ) - } - if (groupCancellation?.kind === 'not_workflow_group') { - logger.warn('Workflow group execution is no longer the active table execution', { executionId }) - await releaseSlotForStoppedExecution() - throw new OrchestrationError( - 'conflict', - 'Workflow group execution is no longer the active table execution' - ) - } - const groupCancellationToPublish: PublishableWorkflowGroupCancellation | null = - groupCancellation?.kind === 'cancelled' || groupCancellation?.kind === 'already_cancelled' - ? groupCancellation - : null - - /** - * The claim's row count is read back only to report it — `returning` changes - * what the statement returns, never the row it writes or the rows it matches. - */ - let terminalWrite: TerminalWriteOutcome = 'unknown' - if (groupCancellation !== null) { - terminalWrite = toTerminalWriteOutcome(groupCancellation.writes) - } else if ( - (cancellation.durablyRecorded || queuedJobCancelled || locallyAborted) && - !pausedCancelled - ) { - try { - const cancelledAt = new Date() - const claimedRows = await db - .update(workflowExecutionLogs) - .set(cancelledExecutionLogFields(cancelledAt)) - .where( - and( - eq(workflowExecutionLogs.executionId, executionId), - eq(workflowExecutionLogs.status, 'running') - ) - ) - .returning({ id: workflowExecutionLogs.id }) - terminalWrite = claimedRows.length > 0 ? 'applied' : 'no_row' - } catch (dbError) { - logger.warn('Failed to update execution log status directly', { - executionId, - error: dbError, + const executionStopSatisfied = effectivePausedCancellationPath + ? !activeResumeSignalFailed + : stopSummary.accepted + if (groupCancellationToPublish && executionStopSatisfied && !pauseReconciliationFailed) { + await publishWorkflowGroupCancellationEvent(groupCancellationToPublish, executionId) + } + let cancellationEventPublished = false + if (executionStopSatisfied && !pauseReconciliationFailed) { + cancellationEventPublished = await ensureCancellationEventPublished(executionId, workflowId, { + workspaceId: execution.workspaceId, + userId: attributedUserId, }) } - } - if (groupCancellationToPublish && success) { - await publishWorkflowGroupCancellationEvent(groupCancellationToPublish, executionId) - } + if (effectivePausedCancellationPath) { + if (cancellationEventPublished && !pauseReconciliationFailed && !activeResumeSignalFailed) { + pausedCancelled = await completePausedCancellationWithRetry(executionId, workflowId) + } + } else if (executionStopSatisfied) { + await releaseExecutionSlot(executionId).catch((error) => { + logger.warn('Failed to release reservation after execution cancellation', { + executionId, + error: toError(error).message, + }) + }) + } - await releaseSlotForStoppedExecution() + const success = effectivePausedCancellationPath + ? pausedCancelled && + cancellationEventPublished && + !pauseReconciliationFailed && + !activeResumeSignalFailed + : executionStopSatisfied - if (success && input.captureAnalytics !== false) { - captureServerEvent( - userId, - 'workflow_execution_cancelled', - { workflow_id: workflowId, workspace_id: workspaceId ?? '' }, - workspaceId ? { groups: { workspace: workspaceId } } : undefined - ) - } + if (effectivePausedCancellationPath && pausedCancelled && cancellationEventPublished) { + await clearStopSignalMarkers(stopSummary) + } - const durablyRecorded = isPausedCancellationPath - ? pausedCancellationPublished - : pausedCancelled || cancellation.durablyRecorded || queuedJobCancelled - const reason: CancelWorkflowExecutionReason = pausedCancellationPublishFailed - ? 'paused_event_publish_failed' - : !pausedCancelled && isPausedCancellationPath - ? 'paused_database_cancel_failed' - : pausedCancelled && !pausedCancellationPublished - ? 'paused_event_publish_failed' - : pausedCancelled || isPausedCancellationPath - ? 'recorded' - : queuedJobCancelled - ? 'recorded' - : cancellation.reason - - /** - * A run that was already terminal when the request arrived cannot be - * cancelled again: the claim's `status = 'running'` predicate matched no row - * and no terminal metadata moved, so `recorded`/`durablyRecorded: true` would - * claim a durable write that never happened. Every effect above still ran - * exactly as before — only the report changes. The request is still satisfied, - * because the run is not running, so `success` stays `true`. - * - * Reinterpreting is only ever right when nothing else went wrong. A terminal - * run — cancelled, or force-failed with paused state left behind — can still - * carry real paused-HITL reconciliation work, and a genuine failure there owes - * the caller the step that failed, not a no-op. So only an otherwise-clean - * `recorded` is a candidate, whatever the prior status was — and only when - * this request wrote nothing durable on any path. - */ - const terminalNoOpReason = - reason === 'recorded' && !pausedCancelled - ? await resolveTerminalNoOpReason(executionId, workflowId, priorTerminalStatus, terminalWrite) - : null + const durablyRecorded = effectivePausedCancellationPath + ? true + : stopSummary.cancellation.durablyRecorded + const reason = resolveCancellationReason({ + activeResumeSignalFailed, + pauseReconciliationFailed, + effectivePausedCancellationPath, + cancellationEventPublished, + pausedCancelled, + stopSummary, + }) - return { - success: terminalNoOpReason ? true : success, - executionId, - redisAvailable: - isPausedCancellationPath || pausedCancelled - ? pausedCancellationPublished - : cancellation.reason !== 'redis_unavailable', - durablyRecorded: terminalNoOpReason ? false : durablyRecorded, - locallyAborted, - pausedCancelled, - reason: terminalNoOpReason ?? reason, + return { + success, + executionId, + redisAvailable: + effectivePausedCancellationPath || pausedCancelled + ? cancellationEventPublished + : stopSummary.cancellation.reason !== 'redis_unavailable', + durablyRecorded, + locallyAborted: stopSummary.locallyAborted, + pausedCancelled, + reason, + } + } catch (error) { + const normalizedError = toError(error) + logger.error('Failed to cancel execution', { + workflowId, + executionId, + error: normalizedError.message, + }) + throw error } } diff --git a/apps/sim/lib/execution/event-buffer.test.ts b/apps/sim/lib/execution/event-buffer.test.ts index 1659c4e675f..75d568c30f7 100644 --- a/apps/sim/lib/execution/event-buffer.test.ts +++ b/apps/sim/lib/execution/event-buffer.test.ts @@ -2,8 +2,7 @@ * @vitest-environment node */ import { redisConfigMockFns, resetEnvMock, resetRedisConfigMock, setEnv } from '@sim/testing' -import { sleep } from '@sim/utils/helpers' -import { afterAll, beforeEach, describe, expect, it, vi } from 'vitest' +import { afterAll, afterEach, beforeEach, describe, expect, it, vi } from 'vitest' import type { ExecutionEventEntry } from '@/lib/execution/event-buffer' import { clearLargeValueCacheForTests } from '@/lib/execution/payloads/cache' import { LARGE_VALUE_REF_MARKER } from '@/lib/execution/payloads/large-value-ref' @@ -51,6 +50,10 @@ afterAll(() => { resetRedisConfigMock() }) +afterEach(() => { + vi.useRealTimers() +}) + import { createExecutionEventWriter, flushExecutionStreamReplayBuffer, @@ -654,15 +657,18 @@ describe('execution event buffer', () => { }) }) + vi.useFakeTimers() const writer = createExecutionEventWriter('exec-1') await writer.write(makeEvent('first')) + // The write only arms the flush timer; fire it so the flush is in flight. + await vi.runOnlyPendingTimersAsync() await firstFlushStarted const terminalWrite = writer.writeTerminal(makeEvent('terminal'), 'complete') // Let writeTerminal's queued body actually enqueue its entry before the // in-flight flush resolves — otherwise the scheduled loop finds nothing left // to drain and the race under test never forms. - await sleep(5) + await vi.advanceTimersByTimeAsync(5) releaseFirstFlush?.() await terminalWrite @@ -776,10 +782,12 @@ describe('execution event buffer', () => { return [1, 'ok', 0, 0] }) + vi.useFakeTimers() const writer = createExecutionEventWriter('exec-1') await writer.write(makeEvent('a')) - await sleep(60) + // Fire the scheduled flush (and any backoff it arms) before the caller's own. + await vi.runAllTimersAsync() await expect(writer.flush()).resolves.toBeUndefined() }) diff --git a/apps/sim/lib/execution/isolated-vm.ts b/apps/sim/lib/execution/isolated-vm.ts index ba9832fe41f..93bbf55fedb 100644 --- a/apps/sim/lib/execution/isolated-vm.ts +++ b/apps/sim/lib/execution/isolated-vm.ts @@ -251,9 +251,14 @@ function truncateString(value: string, maxChars: number): { value: string; trunc } function normalizeFetchOptions(options?: IsolatedFetchOptions): SecureFetchOptions { - if (!options) return { maxResponseBytes: MAX_FETCH_RESPONSE_BYTES } + // The Function block's `fetch()` reaches whatever the workflow author's script + // asks for, so it is governed as a request target rather than a configured one. + if (!options) { + return { profile: 'requestTarget', maxResponseBytes: MAX_FETCH_RESPONSE_BYTES } + } const normalized: SecureFetchOptions = { + profile: 'requestTarget', maxResponseBytes: MAX_FETCH_RESPONSE_BYTES, } diff --git a/apps/sim/lib/execution/payloads/sandbox-file-mount-ref.ts b/apps/sim/lib/execution/payloads/sandbox-file-mount-ref.ts new file mode 100644 index 00000000000..55555db2163 --- /dev/null +++ b/apps/sim/lib/execution/payloads/sandbox-file-mount-ref.ts @@ -0,0 +1,124 @@ +import { isUserFileWithMetadata } from '@/lib/core/utils/user-file' +import type { UserFile } from '@/executor/types' + +export const SANDBOX_FILE_MOUNT_REF_MARKER = '__simSandboxFileMount' +export const SANDBOX_FILE_MOUNT_REF_VERSION = 1 + +/** + * A request to place one file on the sandbox filesystem, standing in for the + * path until the sandbox exists. + * + * Emitted when code references ``. Reference resolution happens + * long before a sandbox is created, and mount paths are only known once the whole + * set is planned (they are sanitized and de-duplicated together), so the resolver + * leaves this marker and the function runtime swaps in the real path. + * + * Same shape as {@link LargeValueRef}: a marker a later layer materializes. It + * exists only where the caller wrote `.path`, which is what keeps a bare + * `` reference — the common case, and the one that runs fine in the + * isolated VM — from being dragged into a remote sandbox it never needed. + */ +export interface SandboxFileMountRef { + [SANDBOX_FILE_MOUNT_REF_MARKER]: true + version: typeof SANDBOX_FILE_MOUNT_REF_VERSION + file: UserFile +} + +export function createSandboxFileMountRef(file: UserFile): SandboxFileMountRef { + return { + [SANDBOX_FILE_MOUNT_REF_MARKER]: true, + version: SANDBOX_FILE_MOUNT_REF_VERSION, + file, + } +} + +export function isSandboxFileMountRef(value: unknown): value is SandboxFileMountRef { + if (!value || typeof value !== 'object') return false + + const candidate = value as Record + return ( + candidate[SANDBOX_FILE_MOUNT_REF_MARKER] === true && + candidate.version === SANDBOX_FILE_MOUNT_REF_VERSION && + isUserFileWithMetadata(candidate.file) + ) +} + +/** + * Replaces every mount marker in a value with whatever `resolvePath` returns for + * its file, leaving the rest of the structure untouched. + * + * Rebuilds containers rather than mutating them: the same resolved block output + * can be shared with other consumers, and a marker can sit anywhere inside a + * referenced object, not only at the top level. + */ +export function replaceSandboxFileMountRefs( + value: unknown, + resolvePath: (file: UserFile) => string, + seen = new WeakMap() +): unknown { + if (!value || typeof value !== 'object') return value + if (isSandboxFileMountRef(value)) return resolvePath(value.file) + + const existing = seen.get(value) + if (existing !== undefined) return existing + + if (Array.isArray(value)) { + const next: unknown[] = [] + seen.set(value, next) + for (const item of value) next.push(replaceSandboxFileMountRefs(item, resolvePath, seen)) + return next + } + + // Only plain containers are rebuilt. A Date, Buffer, Map, or class instance + // has no own enumerable entries worth walking, and reconstructing one from + // Object.entries would quietly replace it with a stripped plain object — a + // Date becoming `{}` on its way to the sandbox. Such a value cannot hold a + // mount marker anyway, so passing it through is both safer and complete. + const prototype = Object.getPrototypeOf(value) + if (prototype !== Object.prototype && prototype !== null) return value + + const next: Record = {} + seen.set(value, next) + for (const [key, item] of Object.entries(value)) { + // defineProperty, not assignment: a own `__proto__` key would otherwise hit + // Object.prototype's setter and vanish before the value reaches the sandbox. + Object.defineProperty(next, key, { + value: replaceSandboxFileMountRefs(item, resolvePath, seen), + enumerable: true, + writable: true, + configurable: true, + }) + } + return next +} + +/** Every file a value asks to have mounted, in first-seen order. */ +export function collectSandboxFileMountRefs( + value: unknown, + found: UserFile[] = [], + seen = new WeakSet() +): UserFile[] { + if (!value || typeof value !== 'object') return found + if (isSandboxFileMountRef(value)) { + found.push(value.file) + return found + } + if (seen.has(value)) return found + seen.add(value) + + if (Array.isArray(value)) { + for (const item of value) collectSandboxFileMountRefs(item, found, seen) + return found + } + + // Same plain-container rule the replacement pass applies. The two walks have to + // agree on the tree: a marker counted here but skipped there would mount a file + // whose reference never became a path. + const prototype = Object.getPrototypeOf(value) + if (prototype !== Object.prototype && prototype !== null) return found + + for (const item of Object.values(value)) { + collectSandboxFileMountRefs(item, found, seen) + } + return found +} diff --git a/apps/sim/lib/execution/preprocessing.test.ts b/apps/sim/lib/execution/preprocessing.test.ts index 03f8a83c1f8..e2168f21d58 100644 --- a/apps/sim/lib/execution/preprocessing.test.ts +++ b/apps/sim/lib/execution/preprocessing.test.ts @@ -500,23 +500,70 @@ describe('preprocessExecution ban gate', () => { expect(mockCheckRateLimit).toHaveBeenCalledTimes(1) }) - it('checks the actor, caller-provided userId, and workflow owner in one call', async () => { + /** The default is the blocking one: an undeclared `userId` stays a candidate. */ + it('checks the actor and the caller-provided userId by default', async () => { const result = await preprocessExecution(baseOptions) expect(result.success).toBe(true) expect(mockGetActivelyBannedUserIds).toHaveBeenCalledTimes(1) + expect(mockGetActivelyBannedUserIds).toHaveBeenCalledWith(['billed-account-1', 'owner-1']) + }) + + /** + * Resume is the shape that must keep blocking: it passes the live + * authenticated resumer as `userId` while attribution stays pinned to the + * original actor across the pause, and deliberately leaves + * `useAuthenticatedUserAsActor` false. Keying the gate on that flag excluded + * exactly the person who just acted. + */ + it('checks a live resumer whose captured attribution names a different actor', async () => { + mockGetActivelyBannedUserIds.mockImplementation(async (ids: string[]) => + ids.filter((id) => id === 'suspended-resumer') + ) + + const result = await preprocessExecution({ + ...baseOptions, + userId: 'suspended-resumer', + billingAttribution: { ...ORGANIZATION_ATTRIBUTION, actorUserId: 'original-actor-1' } as any, + }) + expect(mockGetActivelyBannedUserIds).toHaveBeenCalledWith([ - 'billed-account-1', - 'owner-1', - 'creator-1', + 'original-actor-1', + 'suspended-resumer', ]) + expect(result).toMatchObject({ + success: false, + error: { statusCode: 403, message: 'Account suspended' }, + }) }) - it('excludes the "unknown" sentinel userId but still checks the workflow owner', async () => { + it('excludes the "unknown" sentinel userId', async () => { const result = await preprocessExecution({ ...baseOptions, userId: 'unknown' }) expect(result.success).toBe(true) - expect(mockGetActivelyBannedUserIds).toHaveBeenCalledWith(['billed-account-1', 'creator-1']) + expect(mockGetActivelyBannedUserIds).toHaveBeenCalledWith(['billed-account-1']) + }) + + /** + * The webhook and deployed-chat shape: `userId` names the workflow owner or + * the chat's creator, so a ban on them must not take down automation their + * teammates depend on. Those call sites declare it explicitly rather than the + * gate inferring it. + */ + it('skips a userId the caller declares a stored reference', async () => { + mockGetActivelyBannedUserIds.mockImplementation(async (ids: string[]) => + ids.filter((id) => id === 'creator-1') + ) + + const result = await preprocessExecution({ + ...baseOptions, + userId: 'creator-1', + userIdIsStoredReference: true, + }) + + expect(result.success).toBe(true) + expect(mockGetActivelyBannedUserIds).toHaveBeenCalledWith(['billed-account-1']) + expect(mockGetActivelyBannedUserIds.mock.calls[0][0]).not.toContain('creator-1') }) it('fails closed with 500 when the ban check errors', async () => { diff --git a/apps/sim/lib/execution/preprocessing.ts b/apps/sim/lib/execution/preprocessing.ts index c68dda80679..07f648a378e 100644 --- a/apps/sim/lib/execution/preprocessing.ts +++ b/apps/sim/lib/execution/preprocessing.ts @@ -97,6 +97,18 @@ export interface PreprocessExecutionOptions { triggerData?: SessionStartParams['triggerData'] /** Use the authenticated user as actor for client executions and personal API keys. */ useAuthenticatedUserAsActor?: boolean + /** + * Declares that `userId` names a stored reference — a workflow owner, a chat's + * creator — rather than someone who just acted, so the suspension gate skips + * it. Suspending one member must not take down the schedules, webhooks, and + * deployed chats their teammates depend on merely because that person's name + * sits on the row. + * + * Defaults to false so an unset call site keeps blocking. Withholding the + * suspended account's personal variables is handled separately, in + * {@link getExecutionEnvironment}. + */ + userIdIsStoredReference?: boolean /** Pre-fetched workflow row for caller context; preprocessing still re-checks active state. */ workflowRecord?: WorkflowRecord /** @@ -189,6 +201,7 @@ export async function preprocessExecution( loggingSession: providedLoggingSession, triggerData, useAuthenticatedUserAsActor = false, + userIdIsStoredReference = false, workflowRecord: prefetchedWorkflowRecord, billingAttribution: providedBillingAttribution, executionType = 'sync', @@ -449,17 +462,30 @@ export async function preprocessExecution( const banCheck = (async (): Promise => { /** - * Blocks when the resolved actor, workflow owner, or caller-provided user - * has an active ban or blocked email domain. Including the workflow owner - * covers system-triggered executions. + * Blocks when an identity this run actually acts as has an active ban or + * blocked email domain. + * + * `userId` is a candidate unless the caller declares it a stored reference. + * The default is deliberately the blocking one: callers overload the + * parameter, and only the caller knows which kind it passed, so a call site + * that forgets to say must fail closed rather than silently admit a + * suspended account. + * + * `useAuthenticatedUserAsActor` cannot stand in for that declaration, which + * an earlier revision of this gate assumed. Resume passes the live + * authenticated resumer as `userId` and leaves that flag false on purpose — + * attribution is captured before the pause and must not move — so keying on + * it excluded exactly the person who just acted. + * + * A stored reference being banned must not take down work their teammates + * still depend on — but it must not lend that person's credentials either, + * which is why {@link getExecutionEnvironment} drops a suspended identity's + * personal namespace rather than this gate blocking the whole run. */ const banCandidateIds = [actorUserId] - if (userId && userId !== 'unknown' && userId !== actorUserId) { + if (!userIdIsStoredReference && userId && userId !== 'unknown' && userId !== actorUserId) { banCandidateIds.push(userId) } - if (workflowRecord.userId && !banCandidateIds.includes(workflowRecord.userId)) { - banCandidateIds.push(workflowRecord.userId) - } try { const bannedUserIds = await getActivelyBannedUserIds(banCandidateIds) if (bannedUserIds.length > 0) { diff --git a/apps/sim/lib/execution/remote-sandbox/conformance.test.ts b/apps/sim/lib/execution/remote-sandbox/conformance.test.ts index 0f4271033ef..e6f915f7dc3 100644 --- a/apps/sim/lib/execution/remote-sandbox/conformance.test.ts +++ b/apps/sim/lib/execution/remote-sandbox/conformance.test.ts @@ -9,6 +9,7 @@ import { Readable } from 'node:stream' import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' import { CodeLanguage } from '@/lib/execution/languages' +import { SANDBOX_OUTPUT_DIR_SENTINEL } from '@/lib/execution/remote-sandbox/sandbox-paths' const { mockResolveSandbox, @@ -22,12 +23,14 @@ const { mockE2BFilesRead, mockE2BFilesRemove, mockE2BFilesWrite, + mockE2BFilesList, mockE2BKill, mockDaytonaCreate, mockInterpreterRunCode, mockProcessCodeRun, mockExecuteCommand, mockGetFileDetails, + mockListFiles, mockUploadFile, mockDownloadFile, mockDownloadFileStream, @@ -71,12 +74,14 @@ const { mockE2BFilesRead: vi.fn(), mockE2BFilesRemove: vi.fn(), mockE2BFilesWrite: vi.fn(), + mockE2BFilesList: vi.fn(), mockE2BKill: vi.fn(), mockDaytonaCreate: vi.fn(), mockInterpreterRunCode: vi.fn(), mockProcessCodeRun: vi.fn(), mockExecuteCommand: vi.fn(), mockGetFileDetails: vi.fn(), + mockListFiles: vi.fn(), mockUploadFile: vi.fn(), mockDownloadFile: vi.fn(), mockDownloadFileStream: vi.fn(), @@ -118,12 +123,21 @@ import { SIM_RESULT_PREFIX, withPiSandbox, } from '@/lib/execution/remote-sandbox' -import { daytonaProvider } from '@/lib/execution/remote-sandbox/daytona' -import { E2B_MAX_SANDBOX_LIFETIME_MS, e2bProvider } from '@/lib/execution/remote-sandbox/e2b' +import { + daytonaProvider, + resolveDaytonaSandboxLifetimeMs, +} from '@/lib/execution/remote-sandbox/daytona' +import { + E2B_MAX_SANDBOX_LIFETIME_MS, + e2bProvider, + resolveE2BSandboxLifetimeMs, +} from '@/lib/execution/remote-sandbox/e2b' import { MAX_SANDBOX_OUTPUT_BYTES, + MAX_SANDBOX_OUTPUT_FILES, MAX_SANDBOX_PROCESS_OUTPUT_BYTES, MAX_SANDBOX_STREAMED_OUTPUT_TAIL_BYTES, + readTrustedSandboxOutputCost, } from '@/lib/execution/remote-sandbox/output-limits' import { PI_SANDBOX_MIN_LIFETIME_MS, @@ -134,6 +148,27 @@ import { resolveProvider } from '@/lib/execution/remote-sandbox/provider' type Provider = 'e2b' | 'daytona' const PROVIDERS: Provider[] = ['e2b', 'daytona'] +describe('provider-effective sandbox lifetimes', () => { + it('matches E2B second and Daytona minute rounding', () => { + expect(resolveE2BSandboxLifetimeMs(1001)).toBe(2000) + expect(resolveDaytonaSandboxLifetimeMs(1001)).toBe(60_000) + }) + + it.each(PROVIDERS)('reports the %s SDK create dispatch time', async (provider) => { + useProvider(provider) + const onProviderRequestStarted = vi.fn() + const createMock = provider === 'e2b' ? mockE2BCreate : mockDaytonaCreate + + await resolveProvider().create('code', { lifetimeMs: 1000, onProviderRequestStarted }) + + expect(onProviderRequestStarted).toHaveBeenCalledOnce() + expect(onProviderRequestStarted).toHaveBeenCalledWith(expect.any(Number)) + expect(onProviderRequestStarted.mock.invocationCallOrder[0]).toBeLessThan( + createMock.mock.invocationCallOrder[0] + ) + }) +}) + /** Points the shared layer at one provider via the SANDBOX_PROVIDER env var. */ function useProvider(provider: Provider) { mockEnv.SANDBOX_PROVIDER = provider @@ -266,6 +301,7 @@ beforeEach(() => { read: mockE2BFilesRead, remove: mockE2BFilesRemove, write: mockE2BFilesWrite, + list: mockE2BFilesList, }, kill: mockE2BKill, }) @@ -296,6 +332,7 @@ beforeEach(() => { downloadFile: mockDownloadFile, downloadFileStream: mockDownloadFileStream, getFileDetails: mockGetFileDetails, + listFiles: mockListFiles, }, delete: mockDelete, }) @@ -330,6 +367,47 @@ describe.each(PROVIDERS)('sandbox conformance [%s]', (provider) => { expect(res.result).toEqual({ ok: true }) expect(res.stdout).toBe('hello') expect(res.error).toBeUndefined() + expect(res.cost).toBeUndefined() + }) + + it('adds provider cost to a metered successful code result', async () => { + stubCodeRun(provider, `${SIM_RESULT_PREFIX}{"ok":true}`) + let now = 1_800_000_000_000 + const nowSpy = vi.spyOn(Date, 'now').mockImplementation(() => ++now) + + try { + const res = await executeInSandbox({ + code: 'x', + language: CodeLanguage.Python, + timeoutMs: 1000, + meterUsage: true, + }) + + expect(res.cost).toEqual({ input: 0, output: 0, total: expect.any(Number) }) + expect(res.cost?.total).toBeGreaterThan(0) + } finally { + nowSpy.mockRestore() + } + }) + + it('adds provider cost to a metered successful shell result', async () => { + stubShellCommand(provider, 'ok', '', 0) + let now = 1_800_000_000_000 + const nowSpy = vi.spyOn(Date, 'now').mockImplementation(() => ++now) + + try { + const res = await executeShellInSandbox({ + code: 'echo ok', + envs: {}, + timeoutMs: 1000, + meterUsage: true, + }) + + expect(res.cost).toEqual({ input: 0, output: 0, total: expect.any(Number) }) + expect(res.cost?.total).toBeGreaterThan(0) + } finally { + nowSpy.mockRestore() + } }) it('takes the LAST marker so user output cannot shadow the real result', async () => { @@ -354,10 +432,12 @@ describe.each(PROVIDERS)('sandbox conformance [%s]', (provider) => { code: 'x', language: CodeLanguage.Python, timeoutMs: 1000, + meterUsage: true, }) expect(res.result).toBeNull() expect(res.error).toContain('corrupted in transport') + expect(res.cost).toBeUndefined() }) it('survives a large single-line payload without chunk corruption', async () => { @@ -386,13 +466,19 @@ describe.each(PROVIDERS)('sandbox conformance [%s]', (provider) => { }) } - await expect( - executeInSandbox({ code: 'x', language: CodeLanguage.Python, timeoutMs: 1000 }) - ).rejects.toMatchObject({ + const error = await executeInSandbox({ + code: 'x', + language: CodeLanguage.Python, + timeoutMs: 1000, + meterUsage: true, + }).catch((error: unknown) => error) + + expect(error).toMatchObject({ code: 'sandbox_output_limit_exceeded', outputKind: 'process', limitBytes: MAX_SANDBOX_PROCESS_OUTPUT_BYTES, }) + expect(readTrustedSandboxOutputCost(error)).toBeUndefined() expect(provider === 'e2b' ? mockE2BKill : mockDelete).toHaveBeenCalledTimes(1) }) @@ -495,11 +581,13 @@ describe.each(PROVIDERS)('sandbox conformance [%s]', (provider) => { code: 'raise ValueError("boom")', language: CodeLanguage.Python, timeoutMs: 1000, + meterUsage: true, }) expect(res.error).toBe('ValueError: boom') expect(res.stdout).toContain('ValueError: boom') expect(res.result).toBeNull() + expect(res.cost).toEqual({ input: 0, output: 0, total: expect.any(Number) }) }) it('normalizes Python code budget expiry to a typed timeout abort', async () => { @@ -600,6 +688,252 @@ describe.each(PROVIDERS)('sandbox conformance [%s]', (provider) => { ).rejects.toThrow(/Failed to fetch mounted file/) }) + /** Stubs one directory listing in whichever shape the provider returns. */ + function stubOutputDirListing( + entries: Array<{ path: string; size: number; kind?: 'file' | 'dir' }> + ) { + if (provider === 'e2b') { + mockE2BFilesList.mockResolvedValueOnce( + entries.map((entry) => ({ + name: entry.path.split('/').pop(), + path: entry.path, + size: entry.size, + type: entry.kind === 'dir' ? 'dir' : 'file', + })) + ) + } else { + mockListFiles.mockResolvedValueOnce( + entries.map((entry) => ({ + name: entry.path.split('/').pop(), + path: entry.path, + size: entry.size, + isDir: entry.kind === 'dir', + mode: entry.kind === 'dir' ? 'drwxr-xr-x' : '-rw-r--r--', + })) + ) + } + } + + it('bills a completed run whose harvest produced more files than it can export', async () => { + // The sandbox executed and was paid for; the refusal is about what the code + // wrote, so it belongs with the post-completion export failures rather than + // the provider failures the policy absorbs. + stubCodeRun(provider, `${SIM_RESULT_PREFIX}null`) + stubOutputDirListing( + Array.from({ length: MAX_SANDBOX_OUTPUT_FILES + 1 }, (_, index) => ({ + path: `/tmp/sim/outputs/file-${index}.txt`, + size: 1, + })) + ) + + const error = await executeInSandbox({ + code: 'x', + language: CodeLanguage.Python, + timeoutMs: 1000, + outputSandboxDir: '/tmp/sim/outputs', + meterUsage: true, + }).catch((error: unknown) => error) + + expect(error).toMatchObject({ code: 'sandbox_output_not_exportable' }) + expect(readTrustedSandboxOutputCost(error)).toEqual({ + input: 0, + output: 0, + total: expect.any(Number), + }) + }) + + it('creates the output directory before user code runs', async () => { + stubCodeRun(provider, `__SIM_RESULT__=${JSON.stringify('done')}`) + stubOutputDirListing([]) + + await executeInSandbox({ + code: 'x', + language: CodeLanguage.Python, + timeoutMs: 1000, + outputSandboxDir: '/tmp/sim/outputs', + }) + + // Regression guard. `outputSandboxDir` is this layer's contract, so this + // layer has to create the directory: when creation lived in the caller's + // runtime prologue instead, calling executeInSandbox directly left user + // code writing into a directory that did not exist, and every write was + // ENOENT. The sentinel must be written before the code file that runs. + const writeMock = provider === 'e2b' ? mockE2BFilesWrite : mockUploadFile + const writtenPaths = writeMock.mock.calls.map((call) => + provider === 'e2b' ? call[0] : call[1] + ) + const sentinelIndex = writtenPaths.findIndex((path: string) => + path?.includes('/tmp/sim/outputs/.sim-keep') + ) + const codeIndex = writtenPaths.findIndex((path: string) => path?.includes('.sim-function-')) + expect(sentinelIndex).toBeGreaterThanOrEqual(0) + expect(codeIndex).toBeGreaterThanOrEqual(0) + expect(sentinelIndex).toBeLessThan(codeIndex) + }) + + it('keeps the directory sentinel out of the harvest', async () => { + stubCodeRun(provider, `__SIM_RESULT__=${JSON.stringify('done')}`) + stubOutputDirListing([ + { path: `/tmp/sim/outputs/${SANDBOX_OUTPUT_DIR_SENTINEL}`, size: 0 }, + { path: '/tmp/sim/outputs/real.txt', size: 4 }, + ]) + stubOutputFileSizes(provider, 4) + stubOutputFileRead(provider, 'real') + + const result = await executeInSandbox({ + code: 'x', + language: CodeLanguage.Python, + timeoutMs: 1000, + outputSandboxDir: '/tmp/sim/outputs', + }) + + expect(result.collectedFiles?.map((file) => file.relativePath)).toEqual(['real.txt']) + }) + + it('harvests files written to the output directory', async () => { + stubCodeRun(provider, `__SIM_RESULT__=${JSON.stringify('done')}`) + stubOutputDirListing([{ path: '/tmp/sim/outputs/report.csv', size: 5 }]) + stubOutputFileSizes(provider, 5) + stubOutputFileRead(provider, 'a,b\n1') + + const result = await executeInSandbox({ + code: 'x', + language: CodeLanguage.Python, + timeoutMs: 1000, + outputSandboxDir: '/tmp/sim/outputs', + }) + + expect(result.collectedFiles).toEqual([ + { + path: '/tmp/sim/outputs/report.csv', + relativePath: 'report.csv', + // Always base64, so an arbitrary harvested filename can never be + // decoded as utf8 and silently corrupted. + contentBase64: Buffer.from('a,b\n1').toString('base64'), + byteLength: 5, + }, + ]) + }) + + it('excludes directories from the harvest', async () => { + stubCodeRun(provider, `__SIM_RESULT__=${JSON.stringify('done')}`) + stubOutputDirListing([{ path: '/tmp/sim/outputs/nested', size: 0, kind: 'dir' }]) + + const result = await executeInSandbox({ + code: 'x', + language: CodeLanguage.Python, + timeoutMs: 1000, + outputSandboxDir: '/tmp/sim/outputs', + }) + + expect(result.collectedFiles).toBeUndefined() + }) + + it('refuses a harvest whose nesting outran the listing depth', async () => { + stubCodeRun(provider, `__SIM_RESULT__=${JSON.stringify('done')}`) + // A directory reported at the traversal limit still holds unlisted files. + // Returning the shallow ones would drop the rest without a word. + const deep = Array.from({ length: 12 }, (_, index) => `l${index + 1}`).join('/') + stubOutputDirListing([ + { path: '/tmp/sim/outputs/shallow.txt', size: 4 }, + { path: `/tmp/sim/outputs/${deep}`, size: 0, kind: 'dir' }, + ]) + + await expect( + executeInSandbox({ + code: 'x', + language: CodeLanguage.Python, + timeoutMs: 1000, + outputSandboxDir: '/tmp/sim/outputs', + }) + ).rejects.toThrow(/nested deeper than 12 levels/) + }) + + it('refuses a harvest over the output file count rather than truncating it', async () => { + stubCodeRun(provider, `__SIM_RESULT__=${JSON.stringify('done')}`) + stubOutputDirListing( + Array.from({ length: 21 }, (_, index) => ({ + path: `/tmp/sim/outputs/file-${index}.txt`, + size: 1, + })) + ) + + await expect( + executeInSandbox({ + code: 'x', + language: CodeLanguage.Python, + timeoutMs: 1000, + outputSandboxDir: '/tmp/sim/outputs', + }) + ).rejects.toThrow(/over the 20-file export limit/) + }) + + it('spends one file ceiling across declared and harvested outputs', async () => { + // The limit is what an execution exports, not what one directory holds, so a + // request that both declares and harvests cannot take 20 of each. + stubCodeRun(provider, `__SIM_RESULT__=${JSON.stringify('done')}`) + stubOutputFileSizes(provider, 1, 1) + stubOutputDirListing( + Array.from({ length: MAX_SANDBOX_OUTPUT_FILES - 1 }, (_, index) => ({ + path: `/tmp/sim/outputs/file-${index}.txt`, + size: 1, + })) + ) + + await expect( + executeInSandbox({ + code: 'x', + language: CodeLanguage.Python, + timeoutMs: 1000, + outputSandboxPaths: ['/out/first.txt', '/out/second.txt'], + outputSandboxDir: '/tmp/sim/outputs', + }) + ).rejects.toThrow(/produced 21 files .* over the 20-file export limit/) + }) + + it('does not charge a declared path inside the harvest directory to the ceiling twice', async () => { + // The directory holds exactly the limit and the request names one of those + // files. Charging it on both sides would refuse a run exporting 20 files. + stubCodeRun(provider, `__SIM_RESULT__=${JSON.stringify('done')}`) + // One inspection for the declared path, then one per file actually read. + stubOutputFileSizes(provider, ...Array.from({ length: MAX_SANDBOX_OUTPUT_FILES + 1 }, () => 1)) + stubOutputDirListing( + Array.from({ length: MAX_SANDBOX_OUTPUT_FILES }, (_, index) => ({ + path: `/tmp/sim/outputs/file-${index}.txt`, + size: 1, + })) + ) + for (let index = 0; index < MAX_SANDBOX_OUTPUT_FILES; index += 1) { + stubOutputFileRead(provider, 'x') + } + + const result = await executeInSandbox({ + code: 'x', + language: CodeLanguage.Python, + timeoutMs: 1000, + outputSandboxPath: '/tmp/sim/outputs/file-0.txt', + outputSandboxDir: '/tmp/sim/outputs', + }) + + // Exported once as a declared path, rather than a second time as a harvest. + expect(Object.keys(result.exportedFiles ?? {})).toEqual(['/tmp/sim/outputs/file-0.txt']) + expect(result.collectedFiles).toHaveLength(MAX_SANDBOX_OUTPUT_FILES - 1) + expect(result.collectedFiles?.map((file) => file.relativePath)).not.toContain('file-0.txt') + }) + + it('does not list the output directory when no harvest was requested', async () => { + stubCodeRun(provider, `__SIM_RESULT__=${JSON.stringify('done')}`) + + const result = await executeInSandbox({ + code: 'x', + language: CodeLanguage.Python, + timeoutMs: 1000, + }) + + expect(result.collectedFiles).toBeUndefined() + expect(provider === 'e2b' ? mockE2BFilesList : mockListFiles).not.toHaveBeenCalled() + }) + it('materializes private code inputs after dependencies and user files', async () => { const privateText = 'line one\n"quoted"\\slash\0tail' const privateBytes = Uint8Array.from([0, 10, 34, 92, 255]).buffer @@ -816,11 +1150,17 @@ describe.each(PROVIDERS)('sandbox conformance [%s]', (provider) => { */ stubShellCommand(provider, provider === 'daytona' ? 'boom detail' : '', 'boom detail', 3) - const res = await executeShellInSandbox({ code: 'false', envs: {}, timeoutMs: 1000 }) + const res = await executeShellInSandbox({ + code: 'false', + envs: {}, + timeoutMs: 1000, + meterUsage: true, + }) expect(res.result).toBeNull() expect(res.error).toContain('boom detail') expect(res.stdout).toContain('boom detail') + expect(res.cost).toEqual({ input: 0, output: 0, total: expect.any(Number) }) }) it('terminates shell execution when streamed process output exceeds the byte budget', async () => { @@ -922,18 +1262,24 @@ describe.each(PROVIDERS)('sandbox conformance [%s]', (provider) => { stubCodeRun(provider, `${SIM_RESULT_PREFIX}null`) stubOutputFileSizes(provider, MAX_SANDBOX_OUTPUT_BYTES + 1) - await expect( - executeInSandbox({ - code: 'x', - language: CodeLanguage.Python, - timeoutMs: 1000, - outputSandboxPath: '/out/report.txt', - }) - ).rejects.toMatchObject({ + const error = await executeInSandbox({ + code: 'x', + language: CodeLanguage.Python, + timeoutMs: 1000, + outputSandboxPath: '/out/report.txt', + meterUsage: true, + }).catch((error: unknown) => error) + + expect(error).toMatchObject({ code: 'sandbox_output_limit_exceeded', attemptedBytes: MAX_SANDBOX_OUTPUT_BYTES + 1, limitBytes: MAX_SANDBOX_OUTPUT_BYTES, }) + expect(readTrustedSandboxOutputCost(error)).toEqual({ + input: 0, + output: 0, + total: expect.any(Number), + }) expect(provider === 'e2b' ? mockE2BFilesRead : mockDownloadFileStream).not.toHaveBeenCalled() }) @@ -986,15 +1332,96 @@ describe.each(PROVIDERS)('sandbox conformance [%s]', (provider) => { mockGetFileDetails.mockResolvedValueOnce({ size: 1, isDir: false, mode: 'prw-r--r--' }) } - await expect( - executeInSandbox({ - code: 'x', - language: CodeLanguage.Python, + const error = await executeInSandbox({ + code: 'x', + language: CodeLanguage.Python, + timeoutMs: 1000, + outputSandboxPath: '/out/link.txt', + meterUsage: true, + }).catch((error: unknown) => error) + + expect(error).toMatchObject({ code: 'sandbox_output_file_invalid' }) + expect(readTrustedSandboxOutputCost(error)).toEqual({ + input: 0, + output: 0, + total: expect.any(Number), + }) + expect(provider === 'e2b' ? mockE2BFilesRead : mockDownloadFileStream).not.toHaveBeenCalled() + }) + + it.each(['oversized', 'non-regular'] as const)( + 'retains metered shell cost for a completed execution with %s output', + async (failure) => { + stubShellCommand(provider, '', '', 0) + if (failure === 'oversized') { + stubOutputFileSizes(provider, MAX_SANDBOX_OUTPUT_BYTES + 1) + } else if (provider === 'e2b') { + mockE2BFilesGetInfo.mockResolvedValueOnce({ size: 1, type: 'symlink' }) + } else { + mockGetFileDetails.mockResolvedValueOnce({ size: 1, isDir: false, mode: 'prw-r--r--' }) + } + + const error = await executeShellInSandbox({ + code: 'echo done', + envs: {}, timeoutMs: 1000, - outputSandboxPath: '/out/link.txt', + outputSandboxPath: '/out/result.txt', + meterUsage: true, + }).catch((error: unknown) => error) + + expect(error).toMatchObject({ + code: + failure === 'oversized' ? 'sandbox_output_limit_exceeded' : 'sandbox_output_file_invalid', }) - ).rejects.toMatchObject({ code: 'sandbox_output_file_invalid' }) - expect(provider === 'e2b' ? mockE2BFilesRead : mockDownloadFileStream).not.toHaveBeenCalled() + expect(readTrustedSandboxOutputCost(error)).toEqual({ + input: 0, + output: 0, + total: expect.any(Number), + }) + } + ) + + it('does not attach cost to a generic provider failure during output collection', async () => { + stubCodeRun(provider, `${SIM_RESULT_PREFIX}null`) + stubOutputFileSizes(provider, 1, 1) + const failure = new Error('provider file read failed') + if (provider === 'e2b') { + mockE2BFilesRead.mockRejectedValueOnce(failure) + } else { + mockDownloadFileStream.mockRejectedValueOnce(failure) + } + + const error = await executeInSandbox({ + code: 'x', + language: CodeLanguage.Python, + timeoutMs: 1000, + outputSandboxPath: '/out/result.txt', + meterUsage: true, + }).catch((error: unknown) => error) + + expect(error).toBe(failure) + expect(readTrustedSandboxOutputCost(error)).toBeUndefined() + }) + + it('does not attach cost to a generic provider failure during output inspection', async () => { + stubCodeRun(provider, `${SIM_RESULT_PREFIX}null`) + const failure = new Error('provider file metadata failed') + if (provider === 'e2b') { + mockE2BFilesGetInfo.mockRejectedValueOnce(failure) + } else { + mockGetFileDetails.mockRejectedValueOnce(failure) + } + + const error = await executeInSandbox({ + code: 'x', + language: CodeLanguage.Python, + timeoutMs: 1000, + outputSandboxPath: '/out/result.txt', + meterUsage: true, + }).catch((error: unknown) => error) + + expect(error).toBe(failure) + expect(readTrustedSandboxOutputCost(error)).toBeUndefined() }) it('does not return code results when cancellation arrives during output collection', async () => { @@ -1313,6 +1740,58 @@ describe('provider stream recovery', () => { expect(mockGetSessionCommand).toHaveBeenCalledWith(expect.any(String), 'cmd_1') }) + it('fails an at-most-once Daytona run closed when final status has no exit code', async () => { + mockGetSessionCommand.mockResolvedValueOnce({}) + const sandbox = await daytonaProvider.create('code') + + await expect( + sandbox.runCommand('node function.js', { timeoutMs: 1000, atMostOnce: true }) + ).rejects.toMatchObject({ + name: 'SandboxLaunchIndeterminateError', + retryable: false, + code: 'sandbox_launch_indeterminate', + }) + }) + + it('fails an at-most-once Daytona run closed when its readiness handshake never completes', async () => { + mockGetSessionCommandLogs.mockResolvedValueOnce(undefined) + mockGetSessionCommand.mockResolvedValueOnce({ exitCode: 78 }) + const sandbox = await daytonaProvider.create('code') + + await expect( + sandbox.runCommand('node function.js', { timeoutMs: 1000, atMostOnce: true }) + ).rejects.toMatchObject({ + name: 'SandboxLaunchIndeterminateError', + retryable: false, + code: 'sandbox_launch_indeterminate', + }) + expect(mockSendSessionCommandInput).not.toHaveBeenCalled() + }) + + it('fails an at-most-once Daytona run closed when final status lookup fails', async () => { + mockGetSessionCommand.mockRejectedValueOnce(new Error('control plane unavailable')) + const sandbox = await daytonaProvider.create('code') + + await expect( + sandbox.runCommand('node function.js', { timeoutMs: 1000, atMostOnce: true }) + ).rejects.toMatchObject({ + name: 'SandboxLaunchIndeterminateError', + retryable: false, + code: 'sandbox_launch_indeterminate', + }) + }) + + it('preserves a pre-dispatch Daytona failure for at-most-once runs', async () => { + const failure = new Error('session unavailable') + mockCreateSession.mockRejectedValueOnce(failure) + const sandbox = await daytonaProvider.create('code') + + await expect( + sandbox.runCommand('node function.js', { timeoutMs: 1000, atMostOnce: true }) + ).rejects.toBe(failure) + expect(mockExecuteSessionCommand).not.toHaveBeenCalled() + }) + it('keeps the original Daytona deadline while recovering a disconnected stream', async () => { mockGetSessionCommandLogs .mockRejectedValueOnce(new Error('stream disconnected')) @@ -2253,10 +2732,12 @@ describe('Pi sandbox lifetime', () => { code: 'x', language: CodeLanguage.Python, timeoutMs: 7 * 24 * 60 * 60 * 1000, + meterUsage: true, }) expect(result.error).toContain('E2B reached its 24-hour limit') expect(result.error).toContain('workflow timeout may be longer') + expect(result.cost).toBeUndefined() expect(mockRecordSandboxProviderLimit).toHaveBeenCalledWith({ provider: 'e2b', operation: 'code', @@ -2280,9 +2761,11 @@ describe('Pi sandbox lifetime', () => { const result = await executeShellInSandbox({ code: 'sleep infinity', timeoutMs: 7 * 24 * 60 * 60 * 1000, + meterUsage: true, }) expect(result.error).toContain('E2B reached its 24-hour limit') + expect(result.cost).toBeUndefined() expect(mockRecordSandboxProviderLimit).toHaveBeenCalledWith({ provider: 'e2b', operation: 'command', diff --git a/apps/sim/lib/execution/remote-sandbox/daytona.ts b/apps/sim/lib/execution/remote-sandbox/daytona.ts index 0df2ae61443..11df31c48f5 100644 --- a/apps/sim/lib/execution/remote-sandbox/daytona.ts +++ b/apps/sim/lib/execution/remote-sandbox/daytona.ts @@ -27,11 +27,13 @@ import { SandboxProcessOutputBudget, tailStreamedSandboxOutput, } from '@/lib/execution/remote-sandbox/output-limits' +import { resolveSandboxDirectoryEntryPath } from '@/lib/execution/remote-sandbox/sandbox-paths' import type { CreateSandboxOptions, RunCommandOptions, SandboxCodeResult, SandboxCommandResult, + SandboxDirectoryEntry, SandboxHandle, SandboxKind, SandboxProvider, @@ -41,6 +43,11 @@ const logger = createLogger('DaytonaSandboxProvider') const DAYTONA_DEFAULT_SANDBOX_TTL_MS = 24 * 60 * 60 * 1000 const DAYTONA_STREAM_READY_MARKER = '__SIM_DAYTONA_STREAM_READY__' +/** Daytona expresses sandbox TTLs as whole minutes. */ +export function resolveDaytonaSandboxLifetimeMs(lifetimeMs: number): number { + return Math.max(1, Math.ceil(lifetimeMs / 60_000)) * 60_000 +} + /** Daytona expresses every timeout in seconds; the rest of Sim works in milliseconds. */ function toSeconds(timeoutMs: number): number { return Math.max(1, Math.ceil(timeoutMs / 1000)) @@ -294,6 +301,7 @@ class DaytonaSandboxHandle implements SandboxHandle { // must never have. const finalStdout = () => (retainStdout ? stdout : tailStreamedSandboxOutput(stdout)) const finalStderr = () => (retainStderr ? stderr : tailStreamedSandboxOutput(stderr)) + let commandDispatched = false try { await this.sandbox.process.createSession(sessionId) sessionCreated = true @@ -332,6 +340,7 @@ class DaytonaSandboxHandle implements SandboxHandle { if (typeof commandId !== 'string' || commandId.length === 0) { throw new SandboxLaunchIndeterminateError('Daytona') } + commandDispatched = true // Accumulate the streamed chunks as well as forwarding them: callers read // markers out of stdout (the Pi cloud flow parses __BASE_SHA__/__CHANGED__) // and format failures from stderr, so returning empty strings here would @@ -653,7 +662,14 @@ class DaytonaSandboxHandle implements SandboxHandle { } const finished = await this.sandbox.process.getSessionCommand(sessionId, commandId) - const exitCode = finished.exitCode ?? 0 + if (options.atMostOnce && !releaseRequested) { + throw new SandboxLaunchIndeterminateError('Daytona') + } + const exitCode = finished.exitCode + if (typeof exitCode !== 'number' || !Number.isFinite(exitCode)) { + if (options.atMostOnce) throw new SandboxLaunchIndeterminateError('Daytona') + return { stdout: finalStdout(), stderr: finalStderr(), exitCode: 0 } + } return { stdout: finalStdout(), stderr: finalStderr(), exitCode } } catch (error) { if (isSandboxOutputLimitError(error)) { @@ -674,6 +690,12 @@ class DaytonaSandboxHandle implements SandboxHandle { timedOut: true, } } + if (options.atMostOnce) { + if (commandDispatched) { + throw new SandboxLaunchIndeterminateError('Daytona', { cause: error }) + } + throw error + } if (operation === 'code') throw error return { stdout: finalStdout(), stderr: finalStderr() || getErrorMessage(error), exitCode: 1 } } finally { @@ -742,6 +764,24 @@ class DaytonaSandboxHandle implements SandboxHandle { await this.sandbox.fs.uploadFile(buffer, path) } + async listFiles(path: string, options?: { depth?: number }): Promise { + const entries = await this.sandbox.fs.listFiles(path, { + ...(options?.depth !== undefined ? { depth: options.depth } : {}), + }) + + const files: SandboxDirectoryEntry[] = [] + for (const entry of entries) { + const resolved = resolveSandboxDirectoryEntryPath(path, entry.path ?? entry.name) + if (!resolved) continue + files.push({ + ...resolved, + kind: entry.isDir ? 'directory' : 'file', + size: entry.size, + }) + } + return files + } + async kill(): Promise { if (this.killed) return if (!this.killPromise) { @@ -775,6 +815,7 @@ function shellQuote(value: string): string { export const daytonaProvider: SandboxProvider = { id: 'daytona', dependencyStrategy: 'runtime', + resolveLifetimeMs: resolveDaytonaSandboxLifetimeMs, async create(kind: SandboxKind, options?: CreateSandboxOptions): Promise { const apiKey = env.DAYTONA_API_KEY if (!apiKey) { @@ -790,11 +831,11 @@ export const daytonaProvider: SandboxProvider = { snapshot, language: toDaytonaLanguage(language), ephemeral: true, - ttlMinutes: Math.max( - 1, - Math.ceil((options?.lifetimeMs ?? DAYTONA_DEFAULT_SANDBOX_TTL_MS) / 60_000) - ), + ttlMinutes: + resolveDaytonaSandboxLifetimeMs(options?.lifetimeMs ?? DAYTONA_DEFAULT_SANDBOX_TTL_MS) / + 60_000, } + options?.onProviderRequestStarted?.(Date.now()) const sandbox = await daytona.create(createOptions) return new DaytonaSandboxHandle(sandbox, language) diff --git a/apps/sim/lib/execution/remote-sandbox/e2b.ts b/apps/sim/lib/execution/remote-sandbox/e2b.ts index db387fd93e7..c8922b2f3d6 100644 --- a/apps/sim/lib/execution/remote-sandbox/e2b.ts +++ b/apps/sim/lib/execution/remote-sandbox/e2b.ts @@ -43,6 +43,7 @@ import { SandboxProcessOutputBudget, tailStreamedSandboxOutput, } from '@/lib/execution/remote-sandbox/output-limits' +import { resolveSandboxDirectoryEntryPath } from '@/lib/execution/remote-sandbox/sandbox-paths' import { quoteDependency, type SandboxSpec, @@ -55,6 +56,7 @@ import type { RunCommandOptions, SandboxCodeResult, SandboxCommandResult, + SandboxDirectoryEntry, SandboxHandle, SandboxImageBuild, SandboxImageBuilder, @@ -95,6 +97,11 @@ export const E2B_SANDBOX_MATERIALIZER_REVISION = FUNCTION_SANDBOX_MATERIALIZER_R /** Maximum continuous sandbox lifetime supported by E2B. */ export const E2B_MAX_SANDBOX_LIFETIME_MS = 24 * 60 * 60 * 1000 +/** E2B sends sandbox lifetimes as whole seconds. */ +export function resolveE2BSandboxLifetimeMs(lifetimeMs: number): number { + return Math.min(Math.ceil(lifetimeMs / 1000) * 1000, E2B_MAX_SANDBOX_LIFETIME_MS) +} + const E2B_PROVIDER_LIMIT_ERROR = 'E2B reached its 24-hour limit for a single sandbox execution. The workflow timeout may be longer, but this Function call must finish within 24 hours.' const E2B_TIMEOUT_MESSAGE_PATTERN = @@ -365,7 +372,7 @@ class E2BSandboxHandle implements SandboxHandle { return { text: '', stdout: result.stdout, stderr: result.stderr, timedOut: true } } if (result.exitCode !== 0) { - if (result.stderr === E2B_PROVIDER_LIMIT_ERROR) { + if (result.providerFailure === 'provider_limit') { return { text: '', stdout: result.stdout, @@ -375,6 +382,7 @@ class E2BSandboxHandle implements SandboxHandle { value: E2B_PROVIDER_LIMIT_ERROR, traceback: E2B_PROVIDER_LIMIT_ERROR, }, + providerFailure: result.providerFailure, } } return processCodeFailure(result) @@ -534,7 +542,12 @@ class E2BSandboxHandle implements SandboxHandle { if (isNonRetryableExecutionError(error)) throw error if (reachedE2BProviderLimit(error, this.providerLimitAtMs, options.signal)) { recordSandboxProviderLimit({ provider: 'e2b', operation }) - return { stdout: '', stderr: E2B_PROVIDER_LIMIT_ERROR, exitCode: 1 } + return { + stdout: '', + stderr: E2B_PROVIDER_LIMIT_ERROR, + exitCode: 1, + providerFailure: 'provider_limit', + } } // The SDK throws on non-zero exit; callers want the streams, not a throw. const failure = error as { @@ -628,6 +641,25 @@ class E2BSandboxHandle implements SandboxHandle { await this.sandbox.files.write(path, content as string) } + async listFiles(path: string, options?: { depth?: number }): Promise { + const entries = await this.sandbox.files.list(path, { + ...(options?.depth !== undefined ? { depth: options.depth } : {}), + }) + + const files: SandboxDirectoryEntry[] = [] + for (const entry of entries) { + if (entry.type !== 'file' && entry.type !== 'dir') continue + const resolved = resolveSandboxDirectoryEntryPath(path, entry.path) + if (!resolved) continue + files.push({ + ...resolved, + kind: entry.type === 'dir' ? 'directory' : 'file', + size: entry.size, + }) + } + return files + } + async kill(): Promise { if (this.killed) return if (!this.killPromise) { @@ -842,6 +874,7 @@ export const e2bProvider: SandboxProvider = { id: 'e2b', dependencyStrategy: 'prebuilt', images: e2bImages, + resolveLifetimeMs: resolveE2BSandboxLifetimeMs, async create(kind: SandboxKind, options?: CreateSandboxOptions): Promise { const apiKey = env.E2B_API_KEY if (!apiKey) { @@ -860,7 +893,9 @@ export const e2bProvider: SandboxProvider = { // default — longer than the lifetime it asked for, which is the opposite of // what it requested. const effectiveLifetimeMs = - options?.lifetimeMs !== undefined ? e2bTimeoutMs(options.lifetimeMs) : undefined + options?.lifetimeMs !== undefined + ? resolveE2BSandboxLifetimeMs(options.lifetimeMs) + : undefined const createOptions = { apiKey, ...(effectiveLifetimeMs !== undefined ? { timeoutMs: effectiveLifetimeMs } : {}), @@ -868,6 +903,7 @@ export const e2bProvider: SandboxProvider = { const { Sandbox } = await import('@e2b/code-interpreter') const lifetimeStartedAtMs = Date.now() + options?.onProviderRequestStarted?.(lifetimeStartedAtMs) const sandbox = await Sandbox.create(templateName, createOptions) return new E2BSandboxHandle( diff --git a/apps/sim/lib/execution/remote-sandbox/function-resources.ts b/apps/sim/lib/execution/remote-sandbox/function-resources.ts index afbe013c32f..9061d043da9 100644 --- a/apps/sim/lib/execution/remote-sandbox/function-resources.ts +++ b/apps/sim/lib/execution/remote-sandbox/function-resources.ts @@ -2,6 +2,7 @@ export const FUNCTION_SANDBOX_CPU_COUNT = 2 export const FUNCTION_SANDBOX_MEMORY_GB = 4 export const FUNCTION_SANDBOX_MEMORY_MB = FUNCTION_SANDBOX_MEMORY_GB * 1024 +export const FUNCTION_DAYTONA_DISK_GB = 10 /** Bump when custom dependency-layer rendering changes without a semantic spec change. */ export const FUNCTION_SANDBOX_MATERIALIZER_REVISION = 2 diff --git a/apps/sim/lib/execution/remote-sandbox/image-registry.test.ts b/apps/sim/lib/execution/remote-sandbox/image-registry.test.ts index 250daf087ef..dd799a39c54 100644 --- a/apps/sim/lib/execution/remote-sandbox/image-registry.test.ts +++ b/apps/sim/lib/execution/remote-sandbox/image-registry.test.ts @@ -119,13 +119,10 @@ import { cleanupSandboxImages, ensureSandboxImage, FAILED_BUILD_RETRY_COOLDOWN_MS, - LEGACY_SANDBOX_IMAGE_BUILD_TASK_ID, - PREVIOUS_SANDBOX_IMAGE_BUILD_TASK_ID, releaseSandboxImage, runSandboxImageBuild, SANDBOX_IMAGE_BUILD_TASK_ID, sandboxBuildIdempotencyKey, - sandboxImageBuildTaskIds, } from '@/lib/execution/remote-sandbox/image-registry' const READY_IMAGE = { @@ -519,15 +516,8 @@ describe('runSandboxImageBuild attempt ownership', () => { systemPackages: [], } - it('routes each renderer revision to a distinct Trigger.dev task ID', () => { - expect(SANDBOX_IMAGE_BUILD_TASK_ID).toBe('sandbox-image-build-v2') - expect(PREVIOUS_SANDBOX_IMAGE_BUILD_TASK_ID).toBe('sandbox-image-build-v1') - expect(LEGACY_SANDBOX_IMAGE_BUILD_TASK_ID).toBe('sandbox-image-build') - expect(sandboxImageBuildTaskIds(2)).toEqual({ - current: 'sandbox-image-build-v2', - previous: 'sandbox-image-build-v1', - legacy: 'sandbox-image-build', - }) + it('uses one stable Trigger.dev task ID', () => { + expect(SANDBOX_IMAGE_BUILD_TASK_ID).toBe('sandbox-image-build') }) it('refuses an app-new/task-old renderer mismatch before claiming the row', async () => { diff --git a/apps/sim/lib/execution/remote-sandbox/image-registry.ts b/apps/sim/lib/execution/remote-sandbox/image-registry.ts index 9fabd8f0150..3f80df3786f 100644 --- a/apps/sim/lib/execution/remote-sandbox/image-registry.ts +++ b/apps/sim/lib/execution/remote-sandbox/image-registry.ts @@ -17,7 +17,6 @@ import { providerBuildError, type SandboxBuildError, } from '@/lib/execution/remote-sandbox/build-errors' -import { FUNCTION_SANDBOX_MATERIALIZER_REVISION } from '@/lib/execution/remote-sandbox/function-resources' import { resolveProvider } from '@/lib/execution/remote-sandbox/provider' import { invalidateSandboxResolution } from '@/lib/execution/remote-sandbox/resolve' import type { SandboxSpec } from '@/lib/execution/remote-sandbox/sandbox-spec' @@ -44,23 +43,7 @@ const STALE_BUILD_MS = BUILD_POLL_CAP_MS * 2 const POLL_BASE_MS = 3_000 const POLL_MAX_MS = 20_000 -/** Task IDs kept live together so either web-first or worker-first rollouts drain safely. */ -export function sandboxImageBuildTaskIds(rendererRevision: number): { - current: string - previous?: string - legacy: string -} { - return { - current: `sandbox-image-build-v${rendererRevision}`, - ...(rendererRevision > 1 ? { previous: `sandbox-image-build-v${rendererRevision - 1}` } : {}), - legacy: 'sandbox-image-build', - } -} - -const SANDBOX_IMAGE_TASK_IDS = sandboxImageBuildTaskIds(FUNCTION_SANDBOX_MATERIALIZER_REVISION) -export const SANDBOX_IMAGE_BUILD_TASK_ID = SANDBOX_IMAGE_TASK_IDS.current -export const PREVIOUS_SANDBOX_IMAGE_BUILD_TASK_ID = SANDBOX_IMAGE_TASK_IDS.previous -export const LEGACY_SANDBOX_IMAGE_BUILD_TASK_ID = SANDBOX_IMAGE_TASK_IDS.legacy +export const SANDBOX_IMAGE_BUILD_TASK_ID = 'sandbox-image-build' export interface SandboxImageBuildPayload { provider: SandboxProviderId diff --git a/apps/sim/lib/execution/remote-sandbox/index.ts b/apps/sim/lib/execution/remote-sandbox/index.ts index 7055a88ec2e..3b4e92015b8 100644 --- a/apps/sim/lib/execution/remote-sandbox/index.ts +++ b/apps/sim/lib/execution/remote-sandbox/index.ts @@ -1,6 +1,11 @@ import { createLogger } from '@sim/logger' import { getErrorMessage } from '@sim/utils/errors' import { generateShortId } from '@sim/utils/id' +import { + createSandboxPricing, + priceSandboxUsage, + type SandboxPricing, +} from '@/lib/billing/sandbox-pricing' import { createTimeoutAbortController, getRemainingExecutionMs, @@ -10,10 +15,17 @@ import { recordSandboxTeardownFailure } from '@/lib/core/execution-limits/metric import { buildJavaScriptRuntimeBindingsSource } from '@/lib/execution/code-placeholders/javascript-runtime' import { SANDBOX_SYSTEM_PATH } from '@/lib/execution/remote-sandbox/cli-tools.server' import { + attachTrustedSandboxOutputCost, isSandboxOutputFileError, isSandboxOutputLimitError, + isSandboxOutputNotExportableError, MAX_SANDBOX_OUTPUT_BYTES, + MAX_SANDBOX_OUTPUT_FILES, MAX_SANDBOX_PROCESS_OUTPUT_BYTES, + MAX_SANDBOX_URL_MOUNT_BYTES, + SandboxOutputDepthError, + SandboxOutputDirectoryMissingError, + SandboxOutputFileCountError, SandboxOutputLimitError, } from '@/lib/execution/remote-sandbox/output-limits' import { resolvePiSandboxLifetimeMs } from '@/lib/execution/remote-sandbox/pi-lifetime' @@ -25,20 +37,31 @@ import { repairMissingSandboxImage, resolveWorkspaceSandbox, } from '@/lib/execution/remote-sandbox/resolve' +import { + SANDBOX_OUTPUT_DIR_MAX_DEPTH, + SANDBOX_OUTPUT_DIR_SENTINEL, +} from '@/lib/execution/remote-sandbox/sandbox-paths' import type { CreateSandboxOptions, SandboxCodeResult, + SandboxCollectedFile, SandboxCommandResult, + SandboxCostSink, + SandboxDirectoryEntry, + SandboxExecutionCost, SandboxExecutionRequest, SandboxExecutionResult, SandboxFile, SandboxHandle, SandboxKind, SandboxPrivateInput, + SandboxProvider, + SandboxProviderId, SandboxShellExecutionRequest, } from '@/lib/execution/remote-sandbox/types' export type { + SandboxCostSink, SandboxExecutionRequest, SandboxExecutionResult, SandboxFile, @@ -48,14 +71,41 @@ export type { const logger = createLogger('RemoteSandbox') +interface CreatedSandbox { + sandbox: SandboxHandle + providerId: SandboxProviderId + startedAtMs: number + effectiveLifetimeMs?: number + pricing?: SandboxPricing +} + async function createSandbox( kind: SandboxKind, - options?: CreateSandboxOptions -): Promise { - const provider = resolveProvider() - const sandbox = await provider.create(kind, options) + options?: CreateSandboxOptions, + meterUsage = false, + provider: SandboxProvider = resolveProvider() +): Promise { + const effectiveLifetimeMs = + options?.lifetimeMs !== undefined ? provider.resolveLifetimeMs(options.lifetimeMs) : undefined + if (meterUsage && effectiveLifetimeMs === undefined) { + throw new Error('Metered sandbox execution requires a provider lifetime') + } + const pricing = meterUsage ? createSandboxPricing(provider.id) : undefined + let startedAtMs = Date.now() + const providerOptions = { + ...options, + ...(effectiveLifetimeMs !== undefined ? { lifetimeMs: effectiveLifetimeMs } : {}), + ...(meterUsage ? { onProviderRequestStarted: (value: number) => (startedAtMs = value) } : {}), + } + const sandbox = await provider.create(kind, providerOptions) logger.info('Created sandbox', { provider: provider.id, kind, sandboxId: sandbox.sandboxId }) - return sandbox + return { + sandbox, + providerId: provider.id, + startedAtMs, + ...(effectiveLifetimeMs !== undefined ? { effectiveLifetimeMs } : {}), + ...(pricing ? { pricing } : {}), + } } /** @@ -72,10 +122,11 @@ async function createSelectedSandbox( kind: SandboxKind, options: CreateSandboxOptions, selected: ResolvedSandbox | null, - signal: AbortSignal -): Promise { + signal: AbortSignal, + meterUsage = false +): Promise { try { - return await createSandbox(kind, options) + return await createSandbox(kind, options, meterUsage) } catch (error) { throwIfAborted(signal) if (!selected) throw error @@ -155,10 +206,13 @@ function throwIfSandboxTimedOut(result: { timedOut?: boolean }): void { if (result.timedOut) throw new DOMException('timeout', 'AbortError') } -function bindSandboxAbort(sandbox: SandboxHandle, signal?: AbortSignal) { +function bindSandboxAbort( + sandbox: SandboxHandle, + provider: SandboxProviderId, + signal?: AbortSignal +) { let killed = false let killPromise: Promise | null = null - const provider = resolveProvider().id const kill = (reason: 'cleanup' | 'cancellation' | 'timeout'): Promise => { if (killed) return Promise.resolve() if (!killPromise) { @@ -200,6 +254,52 @@ function bindSandboxAbort(sandbox: SandboxHandle, signal?: AbortSignal) { } } +function calculateSandboxCost( + created: CreatedSandbox, + cleanupStartedAtMs: number +): SandboxExecutionCost | undefined { + if (!created.pricing || created.effectiveLifetimeMs === undefined) return undefined + const usage = priceSandboxUsage( + created.pricing, + cleanupStartedAtMs - created.startedAtMs, + created.effectiveLifetimeMs + ) + return { input: 0, output: 0, total: usage.billedCost } +} + +/** + * Fetches one URL mount inside the sandbox, bounded by MAX_BYTES. + * + * Three mechanisms, because no one of them is sufficient on its own. + * `--max-filesize` refuses an oversized object before a byte moves, but only when + * the response declares a Content-Length — a chunked or length-less reply walks + * straight past it. `head -c` therefore caps what can ever reach the disk at one + * byte over the limit, so a mis-declared object cannot fill the sandbox while we + * wait to notice. The final size check is what turns that truncated file into a + * refusal rather than a silently corrupted mount. + * + * curl's exit status travels through a file because its status is lost in a + * pipeline, and losing it would let a 403 on an expired URL look like a + * successful empty download. The size check is consulted first: when `head` + * closes the pipe early curl dies of EPIPE, and "over the limit" is the useful + * message there, not the write error it provokes. + * + * MAX_BYTES, URL, DST, and DIR all arrive as environment variables, never + * interpolated, so a presigned query string cannot break out of the command. + */ +const FETCH_URL_MOUNT_COMMAND = [ + 'set -e', + '[ -n "$DIR" ] && mkdir -p "$DIR"', + 'STATUS_FILE=$(mktemp)', + 'STATUS=0', + '{ curl -fsS --retry 3 --retry-connrefused --max-time 300 --max-filesize "$MAX_BYTES" "$URL" || STATUS=$?; echo "$STATUS" > "$STATUS_FILE"; } | head -c "$(( MAX_BYTES + 1 ))" > "$DST"', + 'STATUS=$(cat "$STATUS_FILE")', + 'rm -f "$STATUS_FILE"', + 'SIZE=$(wc -c < "$DST")', + 'if [ "$SIZE" -gt "$MAX_BYTES" ]; then rm -f "$DST"; echo "mounted file exceeds the $MAX_BYTES byte limit" >&2; exit 1; fi', + 'if [ "$STATUS" -ne 0 ]; then rm -f "$DST"; echo "curl exited $STATUS" >&2; exit 1; fi', +].join('\n') + /** * Materializes sandbox input files before user code runs. `content` entries are written inline; * `url` entries are fetched from inside the sandbox via `curl` — their bytes never pass through the @@ -220,16 +320,24 @@ async function writeSandboxInputs( const dir = file.path.slice(0, file.path.lastIndexOf('/')) let result: SandboxCommandResult try { - result = await sandbox.runCommand( - 'set -e; [ -n "$DIR" ] && mkdir -p "$DIR"; curl -fsS --retry 3 --retry-connrefused --max-time 300 "$URL" -o "$DST"', - { - envs: { URL: file.url, DST: file.path, DIR: dir }, - timeoutMs: Math.min(300_000, remainingSandboxBudgetMs(opts.signal)), - maxOutputBytes: MAX_SANDBOX_PROCESS_OUTPUT_BYTES, - signal: opts.signal, - rootUser: opts.rootUser, - } - ) + result = await sandbox.runCommand(FETCH_URL_MOUNT_COMMAND, { + envs: { + URL: file.url, + DST: file.path, + DIR: dir, + // Clamped, not just defaulted: `sandboxFiles` reaches this layer from + // the request body, so a declared ceiling is a caller's number. It may + // lower the limit for its own mount but never raise it past the one + // this layer guarantees. + MAX_BYTES: String( + Math.min(file.maxBytes ?? MAX_SANDBOX_URL_MOUNT_BYTES, MAX_SANDBOX_URL_MOUNT_BYTES) + ), + }, + timeoutMs: Math.min(300_000, remainingSandboxBudgetMs(opts.signal)), + maxOutputBytes: MAX_SANDBOX_PROCESS_OUTPUT_BYTES, + signal: opts.signal, + rootUser: opts.rootUser, + }) } catch (error) { throwIfAborted(opts.signal) throw new Error( @@ -406,14 +514,14 @@ async function readSandboxOutputFile( logger.warn('Failed to read requested sandbox output file', { sandboxId: sandbox.sandboxId, }) - return undefined + throw error } } async function inspectSandboxOutputFileSize( sandbox: SandboxHandle, outputSandboxPath: string -): Promise { +): Promise { try { const size = await sandbox.getFileSize(outputSandboxPath) if (!Number.isSafeInteger(size) || size < 0) { @@ -425,7 +533,7 @@ async function inspectSandboxOutputFileSize( logger.warn('Failed to inspect requested sandbox output file', { sandboxId: sandbox.sandboxId, }) - return undefined + throw error } } @@ -441,17 +549,96 @@ function requestedOutputSandboxPaths(req: { ] } +/** + * Enumerates the harvest directory, refusing anything it cannot return in full — + * too many files, or nesting past what the listing reaches — before a single + * byte is read. Sorted so a multi-file result is stable run to run rather than + * inheriting whatever order the provider happened to return. + * + * `declaredPaths` are the files the request already named. One sitting inside the + * directory is dropped rather than harvested a second time, and the rest count + * toward the ceiling: the limit is what one execution exports, not what one + * directory holds, so declaring and harvesting cannot spend it twice. + */ +async function listOutputDirectoryFiles( + sandbox: SandboxHandle, + outputSandboxDir: string, + declaredPaths: ReadonlySet, + signal: AbortSignal +): Promise { + let listed: SandboxDirectoryEntry[] + try { + listed = await sandbox.listFiles(outputSandboxDir, { depth: SANDBOX_OUTPUT_DIR_MAX_DEPTH }) + } catch (error) { + // The directory is created before user code runs, so the only way it can be + // missing now is that the code removed it. Providers report that as a raw + // `lstat ... no such file or directory`, which reads like a Sim fault; say + // what actually happened instead. Anything else propagates untouched rather + // than being flattened into "produced nothing". + if (/not_?found|no such file|ENOENT/i.test(getErrorMessage(error))) { + throw new SandboxOutputDirectoryMissingError(outputSandboxDir) + } + throw error + } + const entries = listed.filter((entry) => entry.relativePath !== SANDBOX_OUTPUT_DIR_SENTINEL) + remainingSandboxBudgetMs(signal) + + // A directory sitting exactly at the traversal limit still has unlisted + // contents, and the providers report no truncation of their own. Refuse + // rather than return a partial harvest: a file the code wrote and the caller + // never receives is worse than an error naming the reason. + const truncatedAt = entries.find( + (entry) => + entry.kind === 'directory' && + entry.relativePath.split('/').length >= SANDBOX_OUTPUT_DIR_MAX_DEPTH + ) + if (truncatedAt) { + throw new SandboxOutputDepthError( + `${outputSandboxDir}/${truncatedAt.relativePath}`, + SANDBOX_OUTPUT_DIR_MAX_DEPTH + ) + } + + const files = entries.filter((entry) => entry.kind === 'file' && !declaredPaths.has(entry.path)) + const exported = declaredPaths.size + files.length + if (exported > MAX_SANDBOX_OUTPUT_FILES) { + throw new SandboxOutputFileCountError(exported, outputSandboxDir) + } + return files.sort((a, b) => a.path.localeCompare(b.path)) +} + +/** + * Brings the harvest directory into existence before user code runs. + * + * Owned here rather than by the caller's runtime prologue because + * `outputSandboxDir` is this layer's contract: a caller that asks for a harvest + * must not also have to know it is responsible for creating the directory, or + * the first write in their code is ENOENT. + */ +async function ensureSandboxOutputDir( + sandbox: SandboxHandle, + outputSandboxDir: string | undefined, + signal: AbortSignal +): Promise { + if (!outputSandboxDir) return + await sandbox.writeFile(`${outputSandboxDir}/${SANDBOX_OUTPUT_DIR_SENTINEL}`, '') + remainingSandboxBudgetMs(signal) +} + async function collectExportedFiles( sandbox: SandboxHandle, - req: { outputSandboxPath?: string; outputSandboxPaths?: string[] }, + req: { outputSandboxPath?: string; outputSandboxPaths?: string[]; outputSandboxDir?: string }, options: { signal: AbortSignal } -): Promise<{ exportedFiles?: Record; exportedFileContent?: string }> { +): Promise<{ + exportedFiles?: Record + exportedFileContent?: string + collectedFiles?: SandboxCollectedFile[] +}> { const readablePaths: string[] = [] let totalOutputBytes = 0 for (const outputSandboxPath of requestedOutputSandboxPaths(req)) { const size = await inspectSandboxOutputFileSize(sandbox, outputSandboxPath) remainingSandboxBudgetMs(options.signal) - if (size === undefined) continue totalOutputBytes += size if (totalOutputBytes > MAX_SANDBOX_OUTPUT_BYTES) { throw new SandboxOutputLimitError(totalOutputBytes) @@ -459,6 +646,22 @@ async function collectExportedFiles( readablePaths.push(outputSandboxPath) } + // Sized into the same running total as the declared paths, so an execution + // cannot spend the byte ceiling twice by both declaring and harvesting. The + // listing applies the same rule to the file-count ceiling and drops a declared + // path that happens to sit inside the harvest directory — double-billing it + // would reject a single output larger than half the ceiling as oversized. + const declaredPaths = new Set(readablePaths) + const discovered = req.outputSandboxDir + ? await listOutputDirectoryFiles(sandbox, req.outputSandboxDir, declaredPaths, options.signal) + : [] + for (const entry of discovered) { + totalOutputBytes += entry.size + if (totalOutputBytes > MAX_SANDBOX_OUTPUT_BYTES) { + throw new SandboxOutputLimitError(totalOutputBytes) + } + } + const exportedFiles: Record = {} let readOutputBytes = 0 for (const outputSandboxPath of readablePaths) { @@ -484,9 +687,45 @@ async function collectExportedFiles( throw error } } + + const collectedFiles: SandboxCollectedFile[] = [] + for (const entry of discovered) { + try { + // Always base64: a harvested filename is arbitrary, and the extension + // allowlist that picks an encoding for a declared path would decode a + // `.parquet` or an extensionless binary as utf8 — substituting U+FFFD and + // delivering corruption that still looks like a valid file. + const file = await sandbox.readFileWithLimit(entry.path, { + maxBytes: MAX_SANDBOX_OUTPUT_BYTES - readOutputBytes, + encoding: 'base64', + signal: options.signal, + }) + remainingSandboxBudgetMs(options.signal) + readOutputBytes += file.byteLength + collectedFiles.push({ + path: entry.path, + relativePath: entry.relativePath, + contentBase64: file.content, + byteLength: file.byteLength, + }) + } catch (error) { + if (isSandboxOutputLimitError(error)) { + throw new SandboxOutputLimitError( + readOutputBytes + error.attemptedBytes, + MAX_SANDBOX_OUTPUT_BYTES + ) + } + // Unlike a declared path, a harvested file was just observed to exist, so + // a failed read is an anomaly rather than a caller mistake. Dropping it + // would silently lose output the code successfully produced. + throw error + } + } + return { exportedFileContent: req.outputSandboxPath ? exportedFiles[req.outputSandboxPath] : undefined, exportedFiles: Object.keys(exportedFiles).length ? exportedFiles : undefined, + collectedFiles: collectedFiles.length ? collectedFiles : undefined, } } @@ -506,6 +745,33 @@ function installBudgetMs(timeoutMs: number): number { return Math.max(0, Math.min(RUNTIME_INSTALL_TIMEOUT_MS, timeoutMs - MIN_CODE_BUDGET_MS)) } +/** + * Held back from the code's own budget when an execution will export files. + * + * The export runs after the code succeeds and draws on the same wall clock, so + * without a reserve a long install plus long-running code can time out during + * the read — destroying work the code already finished, under an error that + * only says "timeout". + */ +const MIN_EXPORT_BUDGET_MS = 10_000 + +/** + * The budget handed to user code, less an export reserve when this request will + * read files back. Short budgets are left alone: taking the reserve out of one + * would starve the code to buy time for an export it never reaches. + */ +function codeBudgetMs( + req: { outputSandboxPath?: string; outputSandboxPaths?: string[]; outputSandboxDir?: string }, + signal: AbortSignal +): number { + const remainingMs = remainingSandboxBudgetMs(signal) + const exportsFiles = Boolean( + req.outputSandboxDir || req.outputSandboxPath || req.outputSandboxPaths?.length + ) + if (!exportsFiles || remainingMs <= MIN_EXPORT_BUDGET_MS * 2) return remainingMs + return remainingMs - MIN_EXPORT_BUDGET_MS +} + /** * Installs a runtime sandbox's dependencies out of the caller's budget and * uses the shared wall-clock budget, so creation and every later phase consume @@ -547,7 +813,7 @@ async function executeInSandboxWithinBudget( }) throwIfAborted(signal) - const sandbox = await createSelectedSandbox( + const created = await createSelectedSandbox( kind, { language, @@ -555,10 +821,14 @@ async function executeInSandboxWithinBudget( lifetimeMs: remainingSandboxBudgetMs(signal), }, selected, - signal + signal, + req.meterUsage ) + const sandbox = created.sandbox const sandboxId = sandbox.sandboxId - const abortBinding = bindSandboxAbort(sandbox, signal) + const abortBinding = bindSandboxAbort(sandbox, created.providerId, signal) + let billableResult: SandboxExecutionResult | undefined + let billableOutputError: unknown try { throwIfAborted(signal) @@ -568,6 +838,7 @@ async function executeInSandboxWithinBudget( // await provisionWithinBudget(sandbox, selected, signal) await writeSandboxInputs(sandbox, req.sandboxFiles, { signal }) + await ensureSandboxOutputDir(sandbox, req.outputSandboxDir, signal) const privateInputEnvironment = await writeSandboxPrivateInputs( sandbox, req.privateInputs, @@ -583,7 +854,7 @@ async function executeInSandboxWithinBudget( let execution: SandboxCodeResult try { execution = await sandbox.runCode(code, { - timeoutMs: remainingSandboxBudgetMs(signal), + timeoutMs: codeBudgetMs(req, signal), javascriptPreload: buildJavaScriptRuntimeBindingsSource(req.runtimeBindings ?? []), maxOutputBytes: MAX_SANDBOX_PROCESS_OUTPUT_BYTES, signal, @@ -602,12 +873,14 @@ async function executeInSandboxWithinBudget( sandboxId, hasTraceback: Boolean(execution.error.traceback), }) - return { + const executionResult = { result: null, stdout: execution.error.traceback || errorMessage, error: errorMessage, sandboxId, } + if (execution.providerFailure !== 'provider_limit') billableResult = executionResult + return executionResult } // Distinct sources (final-expression text, stdout, stderr) join with '\n' so @@ -636,19 +909,47 @@ async function executeInSandboxWithinBudget( } } - const { exportedFiles, exportedFileContent } = await collectExportedFiles(sandbox, req, { - signal, - }) - throwIfAborted(signal) - - return { + billableResult = { result: extraction.result, stdout: cleanedStdout, sandboxId, - exportedFileContent, - exportedFiles, } + try { + const { exportedFiles, exportedFileContent, collectedFiles } = await collectExportedFiles( + sandbox, + req, + { signal } + ) + throwIfAborted(signal) + billableResult.exportedFileContent = exportedFileContent + billableResult.exportedFiles = exportedFiles + billableResult.collectedFiles = collectedFiles + } catch (error) { + /* + * A harvest that cannot return what the run produced — too many files, too + * deep, or an output directory the code deleted — is the caller's to fix + * and arrives only after the sandbox has already executed. It belongs with + * the other post-completion export failures the policy bills, not with the + * provider failures it absorbs; leaving it out let a completed run whose + * code wrote one file too many go free. + */ + if ( + isSandboxOutputLimitError(error) || + isSandboxOutputFileError(error) || + isSandboxOutputNotExportableError(error) + ) { + billableOutputError = error + } + throw error + } + return billableResult } finally { + const cleanupStartedAtMs = Date.now() + const cost = calculateSandboxCost(created, cleanupStartedAtMs) + if (cost && billableResult) billableResult.cost = cost + if (cost && billableOutputError) { + attachTrustedSandboxOutputCost(billableOutputError, cost) + } abortBinding.detach() await abortBinding.cleanup() } @@ -677,14 +978,18 @@ async function executeShellInSandboxWithinBudget( }) throwIfAborted(signal) - const sandbox = await createSelectedSandbox( + const created = await createSelectedSandbox( kind, { imageRef: selected?.imageRef, lifetimeMs: remainingSandboxBudgetMs(signal) }, selected, - signal + signal, + req.meterUsage ) + const sandbox = created.sandbox const sandboxId = sandbox.sandboxId - const abortBinding = bindSandboxAbort(sandbox, signal) + const abortBinding = bindSandboxAbort(sandbox, created.providerId, signal) + let billableResult: SandboxExecutionResult | undefined + let billableOutputError: unknown try { throwIfAborted(signal) @@ -696,6 +1001,7 @@ async function executeShellInSandboxWithinBudget( rootUser: true, signal, }) + await ensureSandboxOutputDir(sandbox, req.outputSandboxDir, signal) const privateInputEnvironment = await writeSandboxPrivateInputs( sandbox, req.privateInputs, @@ -711,7 +1017,7 @@ async function executeShellInSandboxWithinBudget( PATH: selected?.envs?.PATH ?? SANDBOX_SYSTEM_PATH, ...privateInputEnvironment, }, - timeoutMs: remainingSandboxBudgetMs(signal), + timeoutMs: codeBudgetMs(req, signal), maxOutputBytes: MAX_SANDBOX_PROCESS_OUTPUT_BYTES, signal, rootUser: true, @@ -735,7 +1041,9 @@ async function executeShellInSandboxWithinBudget( sandboxId, exitCode: result.exitCode, }) - return { result: null, stdout, error: errorMessage, sandboxId } + const executionResult = { result: null, stdout, error: errorMessage, sandboxId } + if (result.providerFailure !== 'provider_limit') billableResult = executionResult + return executionResult } // Shell scripts have no wrapper: any __SIM_RESULT__ line is user-authored @@ -744,19 +1052,47 @@ async function executeShellInSandboxWithinBudget( const extraction = extractSimResult(stdout) const parsed = extraction.parseFailed ? extraction.rawPayload : extraction.result - const { exportedFiles, exportedFileContent } = await collectExportedFiles(sandbox, req, { - signal, - }) - throwIfAborted(signal) - - return { + billableResult = { result: parsed, stdout: extraction.cleanedStdout, sandboxId, - exportedFileContent, - exportedFiles, } + try { + const { exportedFiles, exportedFileContent, collectedFiles } = await collectExportedFiles( + sandbox, + req, + { signal } + ) + throwIfAborted(signal) + billableResult.exportedFileContent = exportedFileContent + billableResult.exportedFiles = exportedFiles + billableResult.collectedFiles = collectedFiles + } catch (error) { + /* + * A harvest that cannot return what the run produced — too many files, too + * deep, or an output directory the code deleted — is the caller's to fix + * and arrives only after the sandbox has already executed. It belongs with + * the other post-completion export failures the policy bills, not with the + * provider failures it absorbs; leaving it out let a completed run whose + * code wrote one file too many go free. + */ + if ( + isSandboxOutputLimitError(error) || + isSandboxOutputFileError(error) || + isSandboxOutputNotExportableError(error) + ) { + billableOutputError = error + } + throw error + } + return billableResult } finally { + const cleanupStartedAtMs = Date.now() + const cost = calculateSandboxCost(created, cleanupStartedAtMs) + if (cost && billableResult) billableResult.cost = cost + if (cost && billableOutputError) { + attachTrustedSandboxOutputCost(billableOutputError, cost) + } abortBinding.detach() await abortBinding.cleanup() } @@ -813,12 +1149,13 @@ export interface PiSandboxRunner { * caller's sandbox body, which would have buried the change in whitespace. */ export async function withPiSandbox( - options: { lifetimeMs?: number }, + options: { lifetimeMs?: number; cost?: SandboxCostSink }, fn: (runner: PiSandboxRunner) => Promise ): Promise { const lifetimeMs = options.lifetimeMs !== undefined ? options.lifetimeMs : resolvePiSandboxLifetimeMs() - const sandbox = await createSandbox('pi', { lifetimeMs }) + const created = await createSandbox('pi', { lifetimeMs }, Boolean(options.cost)) + const { sandbox } = created logger.info('Started Pi sandbox', { sandboxId: sandbox.sandboxId, lifetimeMs }) const runner: PiSandboxRunner = { @@ -835,9 +1172,31 @@ export async function withPiSandbox( writeFile: (path, content) => sandbox.writeFile(path, content), } + let sessionCompleted = false try { - return await fn(runner) + const result = await fn(runner) + sessionCompleted = true + return result } finally { + /* + * Charged only for a session that ran to completion, which is the same rule + * the Function path applies to its own outcomes: a run whose sandbox never + * delivered is not billed, because a charge nobody can tie to delivered work + * is not one worth defending. A session that ends by throwing — a provider + * crash, a lifetime limit, a cancellation — is absorbed, and a create that + * throws never reaches here at all. + * + * A command exiting non-zero is not a failure by this rule. `fn` returns + * normally there, the agent produced its answer, and the Function path bills + * its own non-zero exits for the same reason. + * + * Measured up to teardown rather than to the last command, so the window + * covers the whole time the provider held the sandbox. + */ + if (sessionCompleted) { + const cost = calculateSandboxCost(created, Date.now()) + if (cost && options.cost) options.cost.total += cost.total + } try { await sandbox.kill() } catch { diff --git a/apps/sim/lib/execution/remote-sandbox/output-limits.ts b/apps/sim/lib/execution/remote-sandbox/output-limits.ts index 91fc5cb3616..e260660aa19 100644 --- a/apps/sim/lib/execution/remote-sandbox/output-limits.ts +++ b/apps/sim/lib/execution/remote-sandbox/output-limits.ts @@ -1,5 +1,27 @@ +import type { SandboxExecutionCost } from '@/lib/execution/remote-sandbox/types' + export const MAX_SANDBOX_OUTPUT_BYTES = 50 * 1024 * 1024 +/** + * Hard ceiling on a single URL-mounted input, enforced inside the sandbox by + * `curl --max-filesize` against the bytes actually served. + * + * The planner checks a recorded size first for a fast, well-worded failure; this + * is the backstop for when that size understates the stored object, and it is + * what a URL mount falls back to when the caller declares no ceiling of its own. + * URL bytes never enter the web process, so the resource being bounded is + * sandbox disk. + */ +export const MAX_SANDBOX_URL_MOUNT_BYTES = 500 * 1024 * 1024 + +/** + * How many files one execution may export, whether declared by path or + * discovered by harvesting the output directory. Exceeding it is an error rather + * than a truncation: silently returning the first 20 of 100 files reads as + * success while losing the rest. + */ +export const MAX_SANDBOX_OUTPUT_FILES = 20 + /** * Maximum combined stdout, stderr, result text, and structured error text kept * for one sandbox operation. Function results larger than this should be @@ -52,6 +74,51 @@ export function appendStreamedSandboxOutput(current: string, chunk: string): str export const SANDBOX_OUTPUT_LIMIT_CODE = 'sandbox_output_limit_exceeded' as const export const SANDBOX_OUTPUT_FILE_INVALID_CODE = 'sandbox_output_file_invalid' as const +/** + * The harvest cannot return what the run produced — too many files, or nested + * past what the listing reaches. Both are the caller's to fix and neither is + * retryable, so they share a code and are reported as one 400. + */ +export const SANDBOX_OUTPUT_NOT_EXPORTABLE_CODE = 'sandbox_output_not_exportable' as const + +/** More files in the harvest directory than one execution may export. */ +export class SandboxOutputFileCountError extends Error { + readonly code = SANDBOX_OUTPUT_NOT_EXPORTABLE_CODE + + constructor(observedFiles: number, directory: string, limit = MAX_SANDBOX_OUTPUT_FILES) { + super( + `Sandbox produced ${observedFiles} files in ${directory}, over the ${limit}-file export limit. Write fewer files, or archive them into a single .zip.` + ) + this.name = 'SandboxOutputFileCountError' + } +} + +/** Harvest directory nested deeper than the listing can reach. */ +export class SandboxOutputDepthError extends Error { + readonly code = SANDBOX_OUTPUT_NOT_EXPORTABLE_CODE + + constructor(directoryPath: string, maxDepth: number) { + super( + `Sandbox output "${directoryPath}" is nested deeper than ${maxDepth} levels, so its contents cannot be returned. Write results closer to the top of the output directory, or archive the tree into a single file.` + ) + this.name = 'SandboxOutputDepthError' + } +} + +const trustedSandboxOutputCosts = new WeakMap() + +/** Associates Sim-calculated cost with a trusted post-execution output error. */ +export function attachTrustedSandboxOutputCost(error: unknown, cost: SandboxExecutionCost): void { + if (typeof error !== 'object' || error === null) return + trustedSandboxOutputCosts.set(error, cost) +} + +/** Reads cost only when the sandbox lifecycle attached it after a completed execution. */ +export function readTrustedSandboxOutputCost(error: unknown): SandboxExecutionCost | undefined { + return typeof error === 'object' && error !== null + ? trustedSandboxOutputCosts.get(error) + : undefined +} export class SandboxOutputFileError extends Error { readonly code = SANDBOX_OUTPUT_FILE_INVALID_CODE @@ -136,3 +203,28 @@ export function isSandboxOutputFileError(error: unknown): error is SandboxOutput (error as { code?: unknown }).code === SANDBOX_OUTPUT_FILE_INVALID_CODE) ) } + +/** The harvest directory was removed by the code that was supposed to fill it. */ +export class SandboxOutputDirectoryMissingError extends Error { + readonly code = SANDBOX_OUTPUT_NOT_EXPORTABLE_CODE + + constructor(directoryPath: string) { + super( + `The sandbox output directory ${directoryPath} no longer exists — the code deleted it. Write files into it rather than replacing it; no files could be returned from this run.` + ) + this.name = 'SandboxOutputDirectoryMissingError' + } +} + +export function isSandboxOutputNotExportableError( + error: unknown +): error is + | SandboxOutputFileCountError + | SandboxOutputDepthError + | SandboxOutputDirectoryMissingError { + return ( + typeof error === 'object' && + error !== null && + (error as { code?: unknown }).code === SANDBOX_OUTPUT_NOT_EXPORTABLE_CODE + ) +} diff --git a/apps/sim/lib/execution/remote-sandbox/pi-sandbox-billing.smoke.test.ts b/apps/sim/lib/execution/remote-sandbox/pi-sandbox-billing.smoke.test.ts new file mode 100644 index 00000000000..648eb64e491 --- /dev/null +++ b/apps/sim/lib/execution/remote-sandbox/pi-sandbox-billing.smoke.test.ts @@ -0,0 +1,96 @@ +/** + * @vitest-environment node + * + * Checks that a Pi session's sandbox is actually metered against a real provider. + * + * The handler-level test mocks the backend and writes into the sink by hand, so + * it proves the wiring from a backend to the block's cost and nothing else. It + * would still pass if `withPiSandbox` never metered at all — which is exactly + * the bug this path had. Only a real Pi sandbox shows that creation is metered, + * that teardown reports, and that the amount tracks the session's real lifetime. + * + * Enable with `SANDBOX_BILLING_SMOKE=1`, against whichever provider + * `SANDBOX_PROVIDER` selects. Needs that provider's Pi image configured + * (`E2B_PI_TEMPLATE_ID` / `DAYTONA_PI_SNAPSHOT_ID`). + */ +import { describe, expect, it } from 'vitest' +import { createSandboxPricing } from '@/lib/billing/sandbox-pricing' +import { withPiSandbox } from '@/lib/execution/remote-sandbox' +import { resolveProvider } from '@/lib/execution/remote-sandbox/provider' +import type { SandboxCostSink } from '@/lib/execution/remote-sandbox/types' + +const smokeEnabled = process.env.SANDBOX_BILLING_SMOKE === '1' +const CASE_TIMEOUT_MS = 5 * 60_000 + +/** Long enough that provisioning jitter cannot dominate the measured session. */ +const SLEEP_SECONDS = 5 +/** Well under any provider ceiling, so the lifetime cap never clamps the charge. */ +const LIFETIME_MS = 10 * 60_000 + +describe.skipIf(!smokeEnabled)('pi sandbox billing smoke', () => { + it( + 'bills the session a Pi sandbox was held for', + async () => { + const pricing = createSandboxPricing(resolveProvider().id) + const usdPerBilledSecond = + (pricing.resources.vcpu * pricing.rates.cpuUsdPerVcpuSecond + + pricing.resources.memoryGiB * pricing.rates.memoryUsdPerGiBSecond + + pricing.resources.diskGiB * pricing.rates.diskUsdPerGiBSecond) * + pricing.multiplier + + const sandboxCost: SandboxCostSink = { total: 0 } + const wallClockStartedAtMs = Date.now() + const exitCode = await withPiSandbox( + { lifetimeMs: LIFETIME_MS, cost: sandboxCost }, + async (runner) => { + const result = await runner.run(`sleep ${SLEEP_SECONDS}; echo held`, { + envs: {}, + timeoutMs: CASE_TIMEOUT_MS, + }) + return result.exitCode + } + ) + const wallClockMs = Date.now() - wallClockStartedAtMs + + expect(exitCode).toBe(0) + expect(sandboxCost.total).toBeGreaterThanOrEqual(SLEEP_SECONDS * usdPerBilledSecond) + expect(sandboxCost.total).toBeLessThanOrEqual((wallClockMs / 1000) * usdPerBilledSecond) + }, + CASE_TIMEOUT_MS + ) + + it( + 'bills nothing for a session that ended by throwing', + async () => { + // Mirrors the Function path: a sandbox that never delivered is absorbed + // rather than charged. Covers a provider crash, a lifetime limit, and a + // cancellation alike, since all three reach here the same way. + const sandboxCost: SandboxCostSink = { total: 0 } + + await expect( + withPiSandbox({ lifetimeMs: LIFETIME_MS, cost: sandboxCost }, async (runner) => { + await runner.run('echo started', { envs: {}, timeoutMs: CASE_TIMEOUT_MS }) + throw new Error('session failed after the sandbox was provisioned') + }) + ).rejects.toThrow('session failed after the sandbox was provisioned') + + expect(sandboxCost.total).toBe(0) + }, + CASE_TIMEOUT_MS + ) + + it( + 'bills nothing when no sink is supplied', + async () => { + // The mothership and any other internal caller must stay free, and the + // absence of a sink is the whole mechanism keeping them that way. + const held = await withPiSandbox({ lifetimeMs: LIFETIME_MS }, async (runner) => { + const result = await runner.run('echo held', { envs: {}, timeoutMs: CASE_TIMEOUT_MS }) + return result.exitCode + }) + + expect(held).toBe(0) + }, + CASE_TIMEOUT_MS + ) +}) diff --git a/apps/sim/lib/execution/remote-sandbox/sandbox-billing.smoke.test.ts b/apps/sim/lib/execution/remote-sandbox/sandbox-billing.smoke.test.ts new file mode 100644 index 00000000000..6b5ebfa63ee --- /dev/null +++ b/apps/sim/lib/execution/remote-sandbox/sandbox-billing.smoke.test.ts @@ -0,0 +1,78 @@ +/** + * @vitest-environment node + * + * Checks the metered amount against a real provider run. + * + * `sandbox-pricing.test.ts` pins the arithmetic and the conformance suite proves + * a cost is produced, attached, and routed — but that suite stubs the provider + * and mocks `Date.now()`, so its clock advances one millisecond per call. Under + * those conditions `total > 0` is the strongest claim available, and it would + * hold just as well if the metered window measured the wrong instants. Only a + * real run can show that the window tracks the sandbox's actual lifetime. + * + * Enable with `SANDBOX_BILLING_SMOKE=1`. Runs against whichever provider + * `SANDBOX_PROVIDER` selects, so point it at each in turn to cover both. + */ +import { describe, expect, it } from 'vitest' +import { createSandboxPricing } from '@/lib/billing/sandbox-pricing' +import { CodeLanguage } from '@/lib/execution/languages' +import { executeInSandbox } from '@/lib/execution/remote-sandbox' +import { resolveProvider } from '@/lib/execution/remote-sandbox/provider' + +const smokeEnabled = process.env.SANDBOX_BILLING_SMOKE === '1' +const CASE_TIMEOUT_MS = 5 * 60_000 +const RUN_TIMEOUT_MS = 4 * 60_000 + +/** Long enough that provisioning jitter cannot dominate the measured runtime. */ +const SLEEP_SECONDS = 5 + +describe.skipIf(!smokeEnabled)('sandbox billing smoke', () => { + it( + 'bills the sandbox lifetime at the provider rate', + async () => { + const pricing = createSandboxPricing(resolveProvider().id) + const usdPerSecond = + pricing.resources.vcpu * pricing.rates.cpuUsdPerVcpuSecond + + pricing.resources.memoryGiB * pricing.rates.memoryUsdPerGiBSecond + + pricing.resources.diskGiB * pricing.rates.diskUsdPerGiBSecond + const usdPerBilledSecond = usdPerSecond * pricing.multiplier + + const wallClockStartedAtMs = Date.now() + const result = await executeInSandbox({ + code: `import time\ntime.sleep(${SLEEP_SECONDS})\nprint("slept")`, + language: CodeLanguage.Python, + timeoutMs: RUN_TIMEOUT_MS, + meterUsage: true, + }) + const wallClockMs = Date.now() - wallClockStartedAtMs + + expect(result.cost).toEqual({ input: 0, output: 0, total: expect.any(Number) }) + const billed = result.cost?.total ?? 0 + + /** + * The window opens immediately before the provider create call and closes + * before teardown, so it has to cover the sleep and cannot exceed the whole + * call measured from out here. A rate error, a wrong resource constant, or a + * window anchored to the wrong instant all land outside these bounds — which + * an `expect.any(Number)` assertion cannot see. + */ + expect(billed).toBeGreaterThanOrEqual(SLEEP_SECONDS * usdPerBilledSecond) + expect(billed).toBeLessThanOrEqual((wallClockMs / 1000) * usdPerBilledSecond) + }, + CASE_TIMEOUT_MS + ) + + it( + 'bills nothing when the caller did not ask for metering', + async () => { + const result = await executeInSandbox({ + code: 'print("unmetered")', + language: CodeLanguage.Python, + timeoutMs: RUN_TIMEOUT_MS, + }) + + expect(result.cost).toBeUndefined() + }, + CASE_TIMEOUT_MS + ) +}) diff --git a/apps/sim/lib/execution/remote-sandbox/sandbox-files.smoke.test.ts b/apps/sim/lib/execution/remote-sandbox/sandbox-files.smoke.test.ts new file mode 100644 index 00000000000..72f60ec4087 --- /dev/null +++ b/apps/sim/lib/execution/remote-sandbox/sandbox-files.smoke.test.ts @@ -0,0 +1,348 @@ +/** + * @vitest-environment node + * + * End-to-end file I/O against a real sandbox provider. + * + * The conformance suite proves both adapters agree on a mocked SDK; this proves + * the contract survives the actual provider — that a mount really lands where + * the code expects, that the output directory really exists before user code + * runs, and that harvested bytes really come back unchanged. + * + * Enable with `SANDBOX_FILES_SMOKE=1`. Requires `E2B_API_KEY` and + * `E2B_FUNCTION_TEMPLATE_ID`; set `SANDBOX_PROVIDER=daytona` (with + * `DAYTONA_API_KEY` and `DAYTONA_SHELL_SNAPSHOT_ID`) to run the same table + * against Daytona instead. Each case creates and destroys one sandbox. + */ +import { createHash } from 'node:crypto' +import { describe, expect, it } from 'vitest' +import { CodeLanguage } from '@/lib/execution/languages' +import { + executeInSandbox, + executeShellInSandbox, + SIM_RESULT_PREFIX, +} from '@/lib/execution/remote-sandbox' +import { SANDBOX_INPUT_DIR, SANDBOX_OUTPUT_DIR } from '@/lib/execution/remote-sandbox/sandbox-paths' + +const smokeEnabled = process.env.SANDBOX_FILES_SMOKE === '1' +const CASE_TIMEOUT_MS = 5 * 60_000 +const RUN_TIMEOUT_MS = 4 * 60_000 + +/** Bytes that a UTF-8 round trip would destroy — the corruption we must not see. */ +const BINARY_FIXTURE = Buffer.from([ + 0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a, 0x00, 0xff, 0xfe, 0x80, 0x7f, 0xc3, 0x28, +]) + +function sha256(buffer: Buffer): string { + return createHash('sha256').update(buffer).digest('hex') +} + +function decode(contentBase64: string): Buffer { + return Buffer.from(contentBase64, 'base64') +} + +/** + * Emits the result marker by hand. These cases drive the sandbox layer directly, + * below the wrapper `execute-request` builds, so `__sim_result__` and a bare + * `return` are not available here — the marker is what proves the code ran to + * completion rather than dying partway. + */ +function pythonResult(expression: string): string { + return `import json; print('${SIM_RESULT_PREFIX}' + json.dumps(${expression}))` +} + +function javascriptResult(expression: string): string { + return `console.log('\\n${SIM_RESULT_PREFIX}' + JSON.stringify(${expression}))` +} + +describe.skipIf(!smokeEnabled)('sandbox file I/O smoke', () => { + it( + 'mounts inputs, harvests outputs, and preserves binary bytes exactly', + async () => { + const result = await executeInSandbox({ + code: [ + 'import os, shutil', + `text = open(os.path.join(${JSON.stringify(SANDBOX_INPUT_DIR)}, 'notes.txt')).read()`, + `blob = open(os.path.join(${JSON.stringify(SANDBOX_INPUT_DIR)}, 'fixture.bin'), 'rb').read()`, + `out = ${JSON.stringify(SANDBOX_OUTPUT_DIR)}`, + "open(os.path.join(out, 'echo.txt'), 'w').write(text.upper())", + // Copied byte-for-byte so any encoding mistake anywhere in the round + // trip shows up as a hash mismatch rather than a plausible-looking file. + "open(os.path.join(out, 'copy.bin'), 'wb').write(blob)", + "os.makedirs(os.path.join(out, 'nested'), exist_ok=True)", + "open(os.path.join(out, 'nested', 'deep.txt'), 'w').write('nested')", + "open(os.path.join(out, 'empty.txt'), 'w').write('')", + pythonResult('{"len": len(blob)}'), + ].join('\n'), + language: CodeLanguage.Python, + timeoutMs: RUN_TIMEOUT_MS, + sandboxFiles: [ + { path: `${SANDBOX_INPUT_DIR}/notes.txt`, content: 'hello sandbox' }, + { + path: `${SANDBOX_INPUT_DIR}/fixture.bin`, + content: BINARY_FIXTURE.toString('base64'), + encoding: 'base64', + }, + ], + outputSandboxDir: SANDBOX_OUTPUT_DIR, + }) + + expect(result.error).toBeUndefined() + expect(result.result).toEqual({ len: BINARY_FIXTURE.length }) + + const byPath = new Map((result.collectedFiles ?? []).map((file) => [file.relativePath, file])) + expect([...byPath.keys()].sort()).toEqual([ + 'copy.bin', + 'echo.txt', + 'empty.txt', + 'nested/deep.txt', + ]) + + expect(decode(byPath.get('echo.txt')!.contentBase64).toString('utf8')).toBe('HELLO SANDBOX') + expect(sha256(decode(byPath.get('copy.bin')!.contentBase64))).toBe(sha256(BINARY_FIXTURE)) + expect(byPath.get('copy.bin')!.byteLength).toBe(BINARY_FIXTURE.length) + expect(decode(byPath.get('nested/deep.txt')!.contentBase64).toString('utf8')).toBe('nested') + expect(byPath.get('empty.txt')!.byteLength).toBe(0) + }, + CASE_TIMEOUT_MS + ) + + it( + 'reads a mounted file and creates the output directory before JavaScript user code runs', + async () => { + const result = await executeInSandbox({ + code: [ + "import { readFileSync, writeFileSync, existsSync } from 'node:fs'", + `const out = ${JSON.stringify(SANDBOX_OUTPUT_DIR)}`, + // Asserted from inside the sandbox: if the directory were not created + // before user code, the very first write is ENOENT. + 'if (!existsSync(out)) throw new Error("output dir missing before user code")', + // The point of resolving `` to a path rather than + // inlining bytes is that every language can just open it. Python and + // Shell prove that in the cases either side of this one. + `const seed = readFileSync(${JSON.stringify(`${SANDBOX_INPUT_DIR}/seed.txt`)}, 'utf8')`, + 'writeFileSync(out + "/from-js.json", JSON.stringify({ seed }))', + javascriptResult('{ wrote: true }'), + ].join('\n'), + language: CodeLanguage.JavaScript, + timeoutMs: RUN_TIMEOUT_MS, + sandboxFiles: [{ path: `${SANDBOX_INPUT_DIR}/seed.txt`, content: 'js seed' }], + outputSandboxDir: SANDBOX_OUTPUT_DIR, + }) + + expect(result.error).toBeUndefined() + expect(result.collectedFiles).toHaveLength(1) + expect(result.collectedFiles?.[0].relativePath).toBe('from-js.json') + expect(JSON.parse(decode(result.collectedFiles![0].contentBase64).toString('utf8'))).toEqual({ + seed: 'js seed', + }) + }, + CASE_TIMEOUT_MS + ) + + it( + 'creates the output directory before shell user code runs', + async () => { + const result = await executeShellInSandbox({ + code: [ + `test -d ${SANDBOX_OUTPUT_DIR} || { echo "output dir missing" >&2; exit 1; }`, + `cp ${SANDBOX_INPUT_DIR}/seed.txt ${SANDBOX_OUTPUT_DIR}/from-shell.txt`, + `echo "${SIM_RESULT_PREFIX}\\"done\\""`, + ].join('\n'), + envs: {}, + timeoutMs: RUN_TIMEOUT_MS, + sandboxFiles: [{ path: `${SANDBOX_INPUT_DIR}/seed.txt`, content: 'shell seed' }], + outputSandboxDir: SANDBOX_OUTPUT_DIR, + }) + + expect(result.error).toBeUndefined() + expect(result.collectedFiles).toHaveLength(1) + expect(decode(result.collectedFiles![0].contentBase64).toString('utf8')).toBe('shell seed') + }, + CASE_TIMEOUT_MS + ) + + it( + 'returns nothing rather than failing when the code writes no files', + async () => { + const result = await executeInSandbox({ + code: pythonResult('"no files"'), + language: CodeLanguage.Python, + timeoutMs: RUN_TIMEOUT_MS, + outputSandboxDir: SANDBOX_OUTPUT_DIR, + }) + + expect(result.error).toBeUndefined() + expect(result.result).toBe('no files') + // "Produced nothing" is an ordinary outcome; the directory exists because + // the prologue made it, so listing it must succeed and come back empty. + expect(result.collectedFiles).toBeUndefined() + }, + CASE_TIMEOUT_MS + ) + + it( + 'skips directories and follows symlinks identically on either provider', + async () => { + const result = await executeInSandbox({ + code: [ + 'import os', + `out = ${JSON.stringify(SANDBOX_OUTPUT_DIR)}`, + "open(os.path.join(out, 'real.txt'), 'w').write('real')", + "os.symlink('/etc/passwd', os.path.join(out, 'linked.txt'))", + "os.makedirs(os.path.join(out, 'adir'), exist_ok=True)", + pythonResult('"planted"'), + ].join('\n'), + language: CodeLanguage.Python, + timeoutMs: RUN_TIMEOUT_MS, + outputSandboxDir: SANDBOX_OUTPUT_DIR, + }) + + expect(result.error).toBeUndefined() + // Followed rather than excluded, and the same on both providers — Daytona + // resolves links in its listing with no field that would reveal one, and + // the code could copy the target's bytes into the directory itself + // anyway. The empty directory is skipped on both. + expect((result.collectedFiles ?? []).map((file) => file.relativePath).sort()).toEqual([ + 'linked.txt', + 'real.txt', + ]) + }, + CASE_TIMEOUT_MS + ) + + it( + 'refuses a harvest over the file-count limit instead of truncating it', + async () => { + await expect( + executeInSandbox({ + code: [ + 'import os', + `out = ${JSON.stringify(SANDBOX_OUTPUT_DIR)}`, + 'for i in range(21):', + " open(os.path.join(out, f'file-{i}.txt'), 'w').write(str(i))", + pythonResult('"wrote 21"'), + ].join('\n'), + language: CodeLanguage.Python, + timeoutMs: RUN_TIMEOUT_MS, + outputSandboxDir: SANDBOX_OUTPUT_DIR, + }) + ).rejects.toThrow(/over the 20-file export limit/) + }, + CASE_TIMEOUT_MS + ) + + it( + 'probe: how deep a nested output is still harvested', + async () => { + const result = await executeInSandbox({ + code: [ + 'import os', + `out = ${JSON.stringify(SANDBOX_OUTPUT_DIR)}`, + 'for depth in range(1, 6):', + " d = os.path.join(out, *[f'l{i}' for i in range(1, depth + 1)])", + ' os.makedirs(d, exist_ok=True)', + " open(os.path.join(d, 'leaf.txt'), 'w').write(str(depth))", + pythonResult('"nested"'), + ].join('\n'), + language: CodeLanguage.Python, + timeoutMs: RUN_TIMEOUT_MS, + outputSandboxDir: SANDBOX_OUTPUT_DIR, + }) + + expect(result.error).toBeUndefined() + // Nesting deeper than the listing depth must not vanish silently — losing + // a file the code successfully wrote is worse than refusing the harvest. + expect((result.collectedFiles ?? []).map((file) => file.relativePath).sort()).toEqual([ + 'l1/l2/l3/l4/l5/leaf.txt', + 'l1/l2/l3/l4/leaf.txt', + 'l1/l2/l3/leaf.txt', + 'l1/l2/leaf.txt', + 'l1/leaf.txt', + ]) + }, + CASE_TIMEOUT_MS + ) + + it( + 'probe: a file name containing a newline survives the listing', + async () => { + const result = await executeInSandbox({ + code: [ + 'import os', + `out = ${JSON.stringify(SANDBOX_OUTPUT_DIR)}`, + `open(os.path.join(out, 'we\\nird.txt'), 'w').write('newline name')`, + pythonResult('"newline"'), + ].join('\n'), + language: CodeLanguage.Python, + timeoutMs: RUN_TIMEOUT_MS, + outputSandboxDir: SANDBOX_OUTPUT_DIR, + }) + + expect(result.error).toBeUndefined() + // A structured listing has no delimiter to corrupt, unlike the `find` + // manifest this deliberately avoids. + expect(result.collectedFiles).toHaveLength(1) + expect(decode(result.collectedFiles![0].contentBase64).toString('utf8')).toBe('newline name') + }, + CASE_TIMEOUT_MS + ) + + it( + 'names the cause when user code deletes the output directory', + async () => { + await expect( + executeInSandbox({ + code: [ + 'import shutil', + `shutil.rmtree(${JSON.stringify(SANDBOX_OUTPUT_DIR)})`, + pythonResult('"deleted"'), + ].join('\n'), + language: CodeLanguage.Python, + timeoutMs: RUN_TIMEOUT_MS, + outputSandboxDir: SANDBOX_OUTPUT_DIR, + }) + // Without this the caller sees a raw `lstat ... no such file or + // directory`, which reads like a platform fault rather than their own + // `rmtree`. + ).rejects.toThrow(/no longer exists — the code deleted it/) + }, + CASE_TIMEOUT_MS + ) + + it( + 'round-trips awkward file names', + async () => { + const result = await executeInSandbox({ + code: [ + 'import os', + `out = ${JSON.stringify(SANDBOX_OUTPUT_DIR)}`, + "open(os.path.join(out, 'Q4 Sales (Final).csv'), 'w').write('a,b')", + "open(os.path.join(out, 'rapport-café.txt'), 'w', encoding='utf-8').write('café')", + "open(os.path.join(out, 'archive.tar.gz'), 'wb').write(b'\\x1f\\x8b\\x08')", + "open(os.path.join(out, 'noext'), 'wb').write(b'\\x00\\x01\\x02')", + pythonResult('"named"'), + ].join('\n'), + language: CodeLanguage.Python, + timeoutMs: RUN_TIMEOUT_MS, + outputSandboxDir: SANDBOX_OUTPUT_DIR, + }) + + expect(result.error).toBeUndefined() + const byPath = new Map((result.collectedFiles ?? []).map((file) => [file.relativePath, file])) + expect([...byPath.keys()].sort()).toEqual([ + 'Q4 Sales (Final).csv', + 'archive.tar.gz', + 'noext', + 'rapport-café.txt', + ]) + // Extension-less and gzip content must survive: neither is in the + // allowlist that decides encoding for a declared output path. + expect(decode(byPath.get('noext')!.contentBase64)).toEqual(Buffer.from([0, 1, 2])) + expect(decode(byPath.get('archive.tar.gz')!.contentBase64)).toEqual( + Buffer.from([0x1f, 0x8b, 0x08]) + ) + expect(decode(byPath.get('rapport-café.txt')!.contentBase64).toString('utf8')).toBe('café') + }, + CASE_TIMEOUT_MS + ) +}) diff --git a/apps/sim/lib/execution/remote-sandbox/sandbox-paths.ts b/apps/sim/lib/execution/remote-sandbox/sandbox-paths.ts new file mode 100644 index 00000000000..d6bddf3eeb1 --- /dev/null +++ b/apps/sim/lib/execution/remote-sandbox/sandbox-paths.ts @@ -0,0 +1,106 @@ +/** + * Filesystem contract shared by every layer that touches a sandbox mount: the + * resolver that plans mount paths, the sandbox layer that creates the output + * directory and enumerates it, and the tool description that teaches a model + * where to write. + * + * Deliberately under `/tmp` rather than a home directory. E2B's default user is + * `user` with workdir `/home/user`, but the Daytona image is built from + * `python:3.13-slim-trixie` with no `useradd`, `USER`, or `WORKDIR`, so + * `/home/user` does not exist there and Daytona resolves relative paths against + * its own working directory. `/tmp` is present and writable by any user on any + * Linux image, which keeps one literal correct on both providers — and lets the + * tool description name that literal to the model instead of a path that has to + * be resolved per provider before it can be quoted. + */ + +/** + * Where mounted input files are materialized before user code runs. + * + * Both directories sit under `/tmp/sim/`, while the runtime's own scratch files + * (`/tmp/.sim-private-input-*`, `/tmp/.sim-env-*`, `/tmp/.sim-command-*`) are + * dotfiles at the `/tmp` root — so enumerating the output directory cannot reach + * them. + */ +export const SANDBOX_INPUT_DIR = '/tmp/sim/inputs' + +/** Files user code writes here are harvested back as platform file objects. */ +export const SANDBOX_OUTPUT_DIR = '/tmp/sim/outputs' + +/** + * Sentinel written to bring the output directory into existence before user code + * runs, and skipped when the directory is harvested. + * + * A directory cannot be created through the providers' filesystem APIs directly, + * but writing a file creates its parents — the same trick the Copilot directory + * mount already uses to materialize an empty folder. Doing it this way keeps the + * cost at one filesystem write; a `mkdir -p` command would instead cost a whole + * session on Daytona, which creates and tears one down per command. + * + * The suffix is not decoration: the harvest filters this name out, so a plainer + * one like `.sim-keep` would silently swallow a user file that happened to share + * it. + */ +export const SANDBOX_OUTPUT_DIR_SENTINEL = '.sim-keep-97f2c1a4' + +/** + * How deep the output directory is enumerated, counted in path segments — a file + * at `a/b/leaf.txt` is depth 3. + * + * Set far above any plausible layout rather than close to it, because the + * providers' listings take a depth and give no signal that they stopped. A file + * below the limit is one the code successfully wrote and the caller never + * receives, so the harvest also refuses outright when it sees a directory + * sitting at the limit — that entry is the evidence the listing was cut short. + */ +export const SANDBOX_OUTPUT_DIR_MAX_DEPTH = 12 + +/** + * How many files one Function block invocation may mount. Far below the Copilot + * ceiling: a block names its inputs one at a time, so a large count is a mistake + * rather than a legitimate bulk mount. + * + * Lives here, with the other mount bounds, because the boundary contract needs it + * too — and this module imports nothing, so a contract can read it without + * pulling the server-only mount resolver into a client-reachable graph. + */ +export const MAX_BLOCK_MOUNTED_FILES = 20 + +/** Trailing-slash-insensitive directory prefix, for joining and stripping. */ +function withTrailingSlash(dir: string): string { + return dir.endsWith('/') ? dir : `${dir}/` +} + +/** + * Resolves one provider directory entry to an absolute path plus its path + * relative to the listed directory. + * + * Providers disagree on whether a listing reports absolute or directory-relative + * paths, and Daytona resolves relative paths against its own working directory + * rather than the listed one — so a relative entry is joined to the directory we + * asked for instead of being trusted as-is. Returns null when the result escapes + * that directory, which is what keeps a `..` component in a provider-reported + * name from reaching a reader. + */ +export function resolveSandboxDirectoryEntryPath( + dir: string, + reportedPath: string +): { path: string; relativePath: string } | null { + const prefix = withTrailingSlash(dir) + const absolute = reportedPath.startsWith('/') ? reportedPath : `${prefix}${reportedPath}` + + const segments: string[] = [] + for (const segment of absolute.split('/')) { + if (segment === '' || segment === '.') continue + if (segment === '..') { + if (segments.length === 0) return null + segments.pop() + continue + } + segments.push(segment) + } + const normalized = `/${segments.join('/')}` + + if (!normalized.startsWith(prefix)) return null + return { path: normalized, relativePath: normalized.slice(prefix.length) } +} diff --git a/apps/sim/lib/execution/remote-sandbox/types.ts b/apps/sim/lib/execution/remote-sandbox/types.ts index c2f4e7b2b28..10732ad35ae 100644 --- a/apps/sim/lib/execution/remote-sandbox/types.ts +++ b/apps/sim/lib/execution/remote-sandbox/types.ts @@ -18,7 +18,20 @@ export type SandboxProviderId = 'e2b' | 'daytona' */ export type SandboxFile = | { type?: 'content'; path: string; content: string; encoding?: 'base64' } - | { type: 'url'; path: string; url: string } + | { + type: 'url' + path: string + url: string + /** + * Ceiling enforced on the bytes actually transferred, rather than on a size + * the caller reported. A caller's pre-read check is a fast, well-worded + * failure; this is what makes it true when the recorded size understates + * the stored object. Optional only because it crosses the wire; a mount + * that omits it still gets `MAX_SANDBOX_URL_MOUNT_BYTES`, so the cap + * cannot be skipped by omission. + */ + maxBytes?: number + } /** * An internal runtime payload materialized at an opaque sandbox path. @@ -47,12 +60,20 @@ export interface SandboxExecutionRequest { * (mothership-docs) that has python-pptx/docx/openpyxl/reportlab installed. */ sandboxKind?: 'code' | 'mothership' | 'doc' + /** + * Harvest every regular file under this directory after the code succeeds. + * Unlike {@link outputSandboxPaths}, the paths are discovered rather than + * declared, so a model that only authors `code` can still return files. + */ + outputSandboxDir?: string /** Scope for {@link sandboxId}; a sandbox from another workspace is rejected. */ workspaceId?: string /** Workspace sandbox whose dependency set this execution runs against. */ sandboxId?: string /** Cancels the provider sandbox when the caller's execution budget expires. */ signal?: AbortSignal + /** Adds the remote provider cost to a completed, billable Function outcome. */ + meterUsage?: boolean } export interface SandboxShellExecutionRequest { @@ -70,12 +91,34 @@ export interface SandboxShellExecutionRequest { * they run in the doc image (mothership-docs). */ sandboxKind?: 'shell' | 'mothership' | 'doc' + /** See {@link SandboxExecutionRequest.outputSandboxDir}. */ + outputSandboxDir?: string /** Scope for {@link sandboxId}; a sandbox from another workspace is rejected. */ workspaceId?: string /** Workspace sandbox whose dependency set this execution runs against. */ sandboxId?: string /** Cancels the provider sandbox when the caller's execution budget expires. */ signal?: AbortSignal + /** Adds the remote provider cost to a completed, billable Function outcome. */ + meterUsage?: boolean +} + +export interface SandboxExecutionCost { + input: number + output: number + total: number +} + +/** + * Running total a caller accumulates sandbox charges into. + * + * A long-lived sandbox reports its cost when it is torn down, which is after the + * value its caller cares about has already been returned. Handing the layer a + * sink lets the charge land without reshaping every return type between here and + * the block that owns the bill. + */ +export interface SandboxCostSink { + total: number } export interface SandboxExecutionResult { @@ -85,6 +128,25 @@ export interface SandboxExecutionResult { error?: string exportedFileContent?: string exportedFiles?: Record + /** + * Files discovered under {@link SandboxExecutionRequest.outputSandboxDir}. + * + * Always base64, never utf8: the extension allowlist that decides encoding for + * a declared path cannot classify an arbitrary harvested filename, and + * decoding real binary as utf8 substitutes U+FFFD silently — corruption that + * arrives looking like a valid file. Base64 is lossless for any byte + * sequence, and the byte budget is enforced on the decoded length. + */ + collectedFiles?: SandboxCollectedFile[] + cost?: SandboxExecutionCost +} + +/** One harvested output file, carried as base64 with its decoded length. */ +export interface SandboxCollectedFile { + path: string + relativePath: string + contentBase64: string + byteLength: number } /** Result of one command run inside a sandbox. */ @@ -94,6 +156,8 @@ export interface SandboxCommandResult { exitCode: number /** The provider stopped the command because its supplied execution budget elapsed. */ timedOut?: boolean + /** The provider ended execution for an infrastructure reason, not a user-process outcome. */ + providerFailure?: 'provider_limit' } /** @@ -117,6 +181,8 @@ export interface SandboxCodeResult { error?: SandboxCodeError /** The provider stopped the code runner because its supplied execution budget elapsed. */ timedOut?: boolean + /** The provider ended execution for an infrastructure reason, not a user-program outcome. */ + providerFailure?: 'provider_limit' } export interface RunCommandOptions { @@ -180,9 +246,50 @@ export interface SandboxHandle { * delivered without any shell parsing. */ writeFile(path: string, content: string | ArrayBuffer): Promise + /** + * Lists regular files under a directory, recursively to `depth`. + * + * Uses each provider's filesystem API rather than shelling out to `find`. + * A shell listing would cost a session per call on Daytona (its + * `runCommand` creates one, writes an env file, executes, then deletes it), + * depend on GNU coreutils that a future base image need not carry, and be + * corrupted by a filename containing a newline — which user code controls. + * + * Symlinks are followed, not excluded. Daytona's listing resolves them and + * reports no field distinguishing one from a regular file, so excluding them + * is only possible on E2B — and doing it there alone would be a cross-provider + * divergence that reads as a security property while providing none. It + * provides none because the harvest is not a privilege boundary: it runs as + * the same identity as the code, which can already read any file the sandbox + * can and copy the bytes into the output directory itself. + * + * Directories are returned alongside files rather than filtered out, because + * a directory sitting at the traversal limit is the only evidence that the + * listing was cut short — see the truncation check in the harvest. + * + * Errors propagate rather than degrading to an empty list. The output + * directory is created before user code runs, so a listing failure is a real + * fault, and reporting it as "produced nothing" would turn a transient + * provider error into silent loss of the caller's files. + */ + listFiles(path: string, options?: { depth?: number }): Promise kill(): Promise } +/** One entry discovered by {@link SandboxHandle.listFiles}. */ +export interface SandboxDirectoryEntry { + /** Absolute path inside the sandbox. */ + path: string + /** Path relative to the listed directory, retaining any subdirectories. */ + relativePath: string + kind: 'file' | 'directory' + /** + * Provider-reported size. Advisory only — the read re-enforces its own limit, + * since the file can change between listing and read. + */ + size: number +} + export interface CreateSandboxOptions { /** Bound at creation — see {@link SandboxHandle.runCode}. */ language?: CodeLanguage @@ -199,6 +306,8 @@ export interface CreateSandboxOptions { * and creates the sandbox as ephemeral. */ lifetimeMs?: number + /** Reports the instant immediately before the provider SDK create request is dispatched. */ + onProviderRequestStarted?: (startedAtMs: number) => void } /** @@ -286,5 +395,7 @@ export interface SandboxProvider { readonly dependencyStrategy: SandboxDependencyStrategy /** Present exactly when {@link dependencyStrategy} is `prebuilt`. */ readonly images?: SandboxImageBuilder + /** Resolves the provider's rounded lifetime for both creation and metering. */ + resolveLifetimeMs(lifetimeMs: number): number create(kind: SandboxKind, options?: CreateSandboxOptions): Promise } diff --git a/apps/sim/lib/execution/sim-helpers.smoke.test.ts b/apps/sim/lib/execution/sim-helpers.smoke.test.ts new file mode 100644 index 00000000000..9e66b0b4a55 --- /dev/null +++ b/apps/sim/lib/execution/sim-helpers.smoke.test.ts @@ -0,0 +1,233 @@ +/** + * @vitest-environment node + * + * The `sim.*` helper namespace, exercised in a real isolate. + * + * `isolated-vm.test.ts` mocks the spawn, so it never proves the namespace is + * reachable from user code — only that the process plumbing is called. These + * cases run the actual worker and assert a value crosses the boundary in both + * directions, which is the only way the frozen `global.sim` shim and the + * broker's JSON marshalling are covered at all. + * + * Enable with `SIM_HELPERS_SMOKE=1`. Needs `isolated-vm` installed for the + * running Node (prebuilds exist for 22/24 only; other versions source-build). + */ +import { describe, expect, it } from 'vitest' +import { executeInIsolatedVM, type IsolatedVMBrokerHandler } from '@/lib/execution/isolated-vm' + +const smokeEnabled = process.env.SIM_HELPERS_SMOKE === '1' +const CASE_TIMEOUT_MS = 60_000 + +const FILE = { + id: 'file_1', + name: 'notes.txt', + url: 'https://storage.example/notes.txt', + size: 11, + type: 'text/plain', + key: 'execution/ws/wf/exec/abc/notes.txt', + context: 'execution', +} + +/** Records what user code asked for, and answers the way the runtime does. */ +function recordingBrokers(): { + brokers: Record + calls: Array<{ name: string; args: unknown }> +} { + const calls: Array<{ name: string; args: unknown }> = [] + const record = + (name: string, reply: (args: any) => unknown): IsolatedVMBrokerHandler => + async (args: any) => { + calls.push({ name, args }) + return reply(args) + } + + return { + calls, + brokers: { + 'sim.files.readText': record('sim.files.readText', () => 'hello world'), + 'sim.files.readBase64': record('sim.files.readBase64', () => + Buffer.from('hello world').toString('base64') + ), + 'sim.files.readTextChunk': record('sim.files.readTextChunk', (args) => ({ + content: 'hello'.slice(0, args?.options?.length ?? 5), + offset: args?.options?.offset ?? 0, + })), + 'sim.values.read': record('sim.values.read', () => ({ rows: [1, 2, 3] })), + 'sim.values.readArray': record('sim.values.readArray', () => [{ a: 1 }, { a: 2 }]), + }, + } +} + +function run(code: string, brokers: Record) { + return executeInIsolatedVM( + { + code, + params: {}, + envVars: {}, + contextVariables: { simFile: FILE }, + timeoutMs: 20_000, + requestId: 'sim-helpers-smoke', + }, + { brokers } + ) +} + +describe.skipIf(!smokeEnabled)('sim.* helpers in a real isolate', () => { + it( + 'exposes sim.files reads to user code and returns their values', + async () => { + const { brokers, calls } = recordingBrokers() + + const result = await run( + [ + 'const text = await sim.files.readText(simFile)', + 'const b64 = await sim.files.readBase64(simFile)', + 'return { text, b64 }', + ].join('\n'), + brokers + ) + + expect(result.error).toBeUndefined() + expect(result.result).toEqual({ + text: 'hello world', + b64: Buffer.from('hello world').toString('base64'), + }) + // The file object must cross intact — the broker authorizes on its `key`, + // so a shim that dropped fields would fail open at the wrong layer. + expect(calls.map((call) => call.name)).toEqual(['sim.files.readText', 'sim.files.readBase64']) + expect((calls[0].args as { file: typeof FILE }).file).toEqual(FILE) + }, + CASE_TIMEOUT_MS + ) + + it( + 'passes options through and returns structured chunk results', + async () => { + const { brokers, calls } = recordingBrokers() + + const result = await run( + 'return await sim.files.readTextChunk(simFile, { offset: 0, length: 5 })', + brokers + ) + + expect(result.error).toBeUndefined() + expect(result.result).toEqual({ content: 'hello', offset: 0 }) + expect((calls[0].args as { options: unknown }).options).toEqual({ offset: 0, length: 5 }) + }, + CASE_TIMEOUT_MS + ) + + it( + 'exposes sim.values reads for offloaded large values', + async () => { + const { brokers } = recordingBrokers() + + const result = await run( + [ + 'const value = await sim.values.read({ __simLargeValueRef: true })', + 'const rows = await sim.values.readArray({ __simLargeValueRef: true })', + 'return { value, rowCount: rows.length }', + ].join('\n'), + brokers + ) + + expect(result.error).toBeUndefined() + expect(result.result).toEqual({ value: { rows: [1, 2, 3] }, rowCount: 2 }) + }, + CASE_TIMEOUT_MS + ) + + it( + 'surfaces a broker rejection as an ordinary error the code can catch', + async () => { + const brokers: Record = { + 'sim.files.readText': async () => { + throw new Error('File is not available in this execution.') + }, + } + + const result = await run( + [ + 'try {', + ' await sim.files.readText(simFile)', + ' return { caught: false }', + '} catch (error) {', + ' return { caught: true, message: String(error.message) }', + '}', + ].join('\n'), + brokers + ) + + // A denied read has to reach user code as a catchable error, not kill the + // isolate — the same file may be optional to the script. + expect(result.error).toBeUndefined() + expect(result.result).toMatchObject({ caught: true }) + expect((result.result as { message: string }).message).toContain('not available') + }, + CASE_TIMEOUT_MS + ) + + it( + 'pins which globals the fast local runtime actually provides', + async () => { + const { brokers } = recordingBrokers() + + const result = await run( + [ + 'const names = ["sim","fetch","console","JSON","Uint8Array",', + ' "Buffer","require","process","atob","TextDecoder","crypto","setTimeout"]', + 'const out = {}', + 'for (const name of names) out[name] = typeof globalThis[name] !== "undefined"', + 'return out', + ].join('\n'), + brokers + ) + + expect(result.error).toBeUndefined() + // The isolate/sandbox split made concrete. The fast runtime is plain + // ECMAScript plus `fetch` and `sim.*` — no Node built-ins, and notably no + // `crypto`, `TextDecoder`, or even `setTimeout`. Reaching for any of them + // is what makes a block need an import, which is what moves it to the + // slower remote sandbox. The block tip documents exactly this list, so + // pin it here rather than letting it drift. + expect(result.result).toEqual({ + sim: true, + fetch: true, + console: true, + JSON: true, + Uint8Array: true, + Buffer: false, + require: false, + process: false, + atob: false, + TextDecoder: false, + crypto: false, + setTimeout: false, + }) + }, + CASE_TIMEOUT_MS + ) + + it( + 'freezes the namespace so user code cannot replace a helper', + async () => { + const { brokers } = recordingBrokers() + + const result = await run( + [ + 'let replaced = true', + 'try { sim.files.readText = () => "spoofed" } catch { replaced = false }', + 'const text = await sim.files.readText(simFile)', + 'return { replaced, text }', + ].join('\n'), + brokers + ) + + expect(result.error).toBeUndefined() + // Whether the assignment throws or is silently ignored, the real helper + // must still be the one that runs. + expect((result.result as { text: string }).text).toBe('hello world') + }, + CASE_TIMEOUT_MS + ) +}) diff --git a/apps/sim/lib/execution/workflow-run-already-terminal-error.ts b/apps/sim/lib/execution/workflow-run-already-terminal-error.ts new file mode 100644 index 00000000000..0ff4966aec4 --- /dev/null +++ b/apps/sim/lib/execution/workflow-run-already-terminal-error.ts @@ -0,0 +1,33 @@ +import { OrchestrationError } from '@/lib/core/orchestration/types' + +export type WorkflowRunAlreadyTerminalStatus = 'completed' | 'failed' + +interface WorkflowRunAlreadyTerminalErrorOptions { + executionId: string + executionStatus: WorkflowRunAlreadyTerminalStatus + redisAvailable: boolean + locallyAborted: boolean +} + +/** A standalone run reached a non-cancellable terminal state before cancellation won. */ +export class WorkflowRunAlreadyTerminalError extends OrchestrationError { + readonly executionId: string + readonly executionStatus: WorkflowRunAlreadyTerminalStatus + readonly redisAvailable: boolean + readonly locallyAborted: boolean + + constructor(options: WorkflowRunAlreadyTerminalErrorOptions) { + super('conflict', `Execution cannot be cancelled while ${options.executionStatus}`) + this.name = 'WorkflowRunAlreadyTerminalError' + this.executionId = options.executionId + this.executionStatus = options.executionStatus + this.redisAvailable = options.redisAvailable + this.locallyAborted = options.locallyAborted + } +} + +export function isWorkflowRunAlreadyTerminalStatus( + status: string +): status is WorkflowRunAlreadyTerminalStatus { + return status === 'completed' || status === 'failed' +} diff --git a/apps/sim/lib/file-parsers/doc-parser.test.ts b/apps/sim/lib/file-parsers/doc-parser.test.ts index e4865fcbea0..c7ed3cfe557 100644 --- a/apps/sim/lib/file-parsers/doc-parser.test.ts +++ b/apps/sim/lib/file-parsers/doc-parser.test.ts @@ -5,12 +5,14 @@ import JSZip from 'jszip' import { beforeEach, describe, expect, it, vi } from 'vitest' import { ZipBombError } from '@/lib/file-parsers/ooxml-limits' -const { mockParseOfficeAsync, mockExtractRawText } = vi.hoisted(() => ({ - mockParseOfficeAsync: vi.fn(), +const { mockParseOfficeText, mockExtractRawText } = vi.hoisted(() => ({ + mockParseOfficeText: vi.fn(), mockExtractRawText: vi.fn(), })) -vi.mock('officeparser', () => ({ parseOfficeAsync: mockParseOfficeAsync })) +vi.mock('@/lib/file-parsers/officeparser-module', () => ({ + parseOfficeText: mockParseOfficeText, +})) vi.mock('mammoth', () => ({ default: { extractRawText: mockExtractRawText }, extractRawText: mockExtractRawText, @@ -61,7 +63,7 @@ describe('DocParser.parseBuffer', () => { const bomb = await buildDeclaredOversizeArchive(2 * 1024 * 1024 * 1024) await expect(new DocParser().parseBuffer(bomb)).rejects.toThrow() - expect(mockParseOfficeAsync).not.toHaveBeenCalled() + expect(mockParseOfficeText).not.toHaveBeenCalled() expect(mockExtractRawText).not.toHaveBeenCalled() }) @@ -87,7 +89,7 @@ describe('DocParser.parseBuffer', () => { } await expect(new DocParser().parseBuffer(lying)).rejects.toThrow(/do not match declared sizes/) - expect(mockParseOfficeAsync).not.toHaveBeenCalled() + expect(mockParseOfficeText).not.toHaveBeenCalled() expect(mockExtractRawText).not.toHaveBeenCalled() }) @@ -98,14 +100,14 @@ describe('DocParser.parseBuffer', () => { await expect(new DocParser().parseBuffer(buffer)).rejects.toThrow( /refusing to parse an unverifiable ZIP-shaped archive/ ) - expect(mockParseOfficeAsync).not.toHaveBeenCalled() + expect(mockParseOfficeText).not.toHaveBeenCalled() }) it('still parses a well-formed OOXML archive renamed to .doc', async () => { const zip = new JSZip() zip.file('word/document.xml', 'hello') const buffer = await zip.generateAsync({ type: 'nodebuffer', compression: 'DEFLATE' }) - mockParseOfficeAsync.mockResolvedValue('hello') + mockParseOfficeText.mockResolvedValue('hello') const result = await new DocParser().parseBuffer(buffer) @@ -114,11 +116,11 @@ describe('DocParser.parseBuffer', () => { }) it('no-ops the guard for a legacy OLE .doc and parses it', async () => { - mockParseOfficeAsync.mockResolvedValue('legacy doc text') + mockParseOfficeText.mockResolvedValue('legacy doc text') const result = await new DocParser().parseBuffer(buildLegacyOleDoc()) - expect(mockParseOfficeAsync).toHaveBeenCalledOnce() + expect(mockParseOfficeText).toHaveBeenCalledOnce() expect(result.content).toBe('legacy doc text') }) diff --git a/apps/sim/lib/file-parsers/doc-parser.ts b/apps/sim/lib/file-parsers/doc-parser.ts index c15424b30ee..4c684acdf2a 100644 --- a/apps/sim/lib/file-parsers/doc-parser.ts +++ b/apps/sim/lib/file-parsers/doc-parser.ts @@ -2,15 +2,15 @@ import { existsSync } from 'fs' import { readFile } from 'fs/promises' import { createLogger } from '@sim/logger' import { FileParserError } from '@/lib/file-parsers/errors' -import { loadParseOfficeAsync } from '@/lib/file-parsers/officeparser-module' -import type { FileParseResult, FileParser } from '@/lib/file-parsers/types' +import { parseOfficeText } from '@/lib/file-parsers/officeparser-module' +import type { FileParseOptions, FileParseResult, FileParser } from '@/lib/file-parsers/types' import { sanitizeTextForUTF8 } from '@/lib/file-parsers/utils' import { assertOoxmlArchiveWithinLimits } from '@/lib/file-parsers/zip-guard' const logger = createLogger('DocParser') export class DocParser implements FileParser { - async parseFile(filePath: string): Promise { + async parseFile(filePath: string, options: FileParseOptions = {}): Promise { if (!filePath) { throw new Error('No file path provided') } @@ -19,8 +19,8 @@ export class DocParser implements FileParser { throw new Error(`File not found: ${filePath}`) } - const buffer = await readFile(filePath) - return this.parseBuffer(buffer) + const buffer = await readFile(filePath, { signal: options.signal }) + return this.parseBuffer(buffer, options) } /** @@ -29,18 +29,17 @@ export class DocParser implements FileParser { * zip-bomb guard must run here exactly as it does in the docx/pptx/xlsx * parsers. It no-ops for genuine legacy OLE `.doc` buffers. */ - async parseBuffer(buffer: Buffer): Promise { + async parseBuffer(buffer: Buffer, options: FileParseOptions = {}): Promise { try { + options.signal?.throwIfAborted() if (!buffer || buffer.length === 0) { throw new FileParserError('empty_input', 'Empty buffer provided') } assertOoxmlArchiveWithinLimits(buffer) - const parseOfficeAsync = await loadParseOfficeAsync() - try { - const result = await parseOfficeAsync(buffer) + const result = await parseOfficeText(buffer, options) if (result) { const resultString = typeof result === 'string' ? result : String(result) @@ -57,12 +56,14 @@ export class DocParser implements FileParser { } } } catch (officeError) { + options.signal?.throwIfAborted() logger.warn('officeparser failed, trying mammoth:', officeError) } try { const mammoth = await import('mammoth') const result = await mammoth.extractRawText({ buffer }) + options.signal?.throwIfAborted() if (result.value && result.value.trim().length > 0) { const content = sanitizeTextForUTF8(result.value.trim()) @@ -76,9 +77,11 @@ export class DocParser implements FileParser { } } } catch (mammothError) { + options.signal?.throwIfAborted() logger.warn('mammoth failed:', mammothError) } + options.signal?.throwIfAborted() return this.fallbackExtraction(buffer) } catch (error) { logger.error('DOC parsing error:', error) diff --git a/apps/sim/lib/file-parsers/docx-parser.ts b/apps/sim/lib/file-parsers/docx-parser.ts index 55a3a869c54..7a6c6038849 100644 --- a/apps/sim/lib/file-parsers/docx-parser.ts +++ b/apps/sim/lib/file-parsers/docx-parser.ts @@ -6,8 +6,8 @@ import { isEncryptedOfficeParserError, toFileParserError, } from '@/lib/file-parsers/errors' -import { loadParseOfficeAsync } from '@/lib/file-parsers/officeparser-module' -import type { FileParseResult, FileParser } from '@/lib/file-parsers/types' +import { parseOfficeText } from '@/lib/file-parsers/officeparser-module' +import type { FileParseOptions, FileParseResult, FileParser } from '@/lib/file-parsers/types' import { sanitizeTextForUTF8 } from '@/lib/file-parsers/utils' import { assertOoxmlArchiveWithinLimits } from '@/lib/file-parsers/zip-guard' @@ -24,17 +24,18 @@ interface MammothResult { } export class DocxParser implements FileParser { - async parseFile(filePath: string): Promise { + async parseFile(filePath: string, options: FileParseOptions = {}): Promise { if (!filePath) { throw new Error('No file path provided') } - const buffer = await readFile(filePath) - return this.parseBuffer(buffer) + const buffer = await readFile(filePath, { signal: options.signal }) + return this.parseBuffer(buffer, options) } - async parseBuffer(buffer: Buffer): Promise { + async parseBuffer(buffer: Buffer, options: FileParseOptions = {}): Promise { try { + options.signal?.throwIfAborted() if (!buffer || buffer.length === 0) { throw new FileParserError('empty_input', 'Empty buffer provided') } @@ -46,6 +47,7 @@ export class DocxParser implements FileParser { try { const result = await mammoth.extractRawText({ buffer }) + options.signal?.throwIfAborted() if (result.value && result.value.trim().length > 0) { let htmlResult: MammothResult = { value: '', messages: [] } @@ -54,6 +56,7 @@ export class DocxParser implements FileParser { } catch { // HTML conversion is optional } + options.signal?.throwIfAborted() return { content: sanitizeTextForUTF8(result.value), @@ -66,14 +69,13 @@ export class DocxParser implements FileParser { } parserReturnedEmpty = true } catch (mammothError) { + options.signal?.throwIfAborted() logger.warn('mammoth failed, trying officeparser:', mammothError) extractionErrors.push(mammothError) } - const parseOfficeAsync = await loadParseOfficeAsync() - try { - const result = await parseOfficeAsync(buffer) + const result = await parseOfficeText(buffer, options) if (result) { const resultString = typeof result === 'string' ? result : String(result) @@ -91,6 +93,7 @@ export class DocxParser implements FileParser { } parserReturnedEmpty = true } catch (officeError) { + options.signal?.throwIfAborted() logger.warn('officeparser failed:', officeError) extractionErrors.push(officeError) } @@ -132,6 +135,7 @@ export class DocxParser implements FileParser { new AggregateError(extractionErrors) ) } catch (error) { + options.signal?.throwIfAborted() logger.error('DOCX parsing error:', error) throw toFileParserError(error, 'invalid_format', 'Failed to parse DOCX buffer') } diff --git a/apps/sim/lib/file-parsers/index.ts b/apps/sim/lib/file-parsers/index.ts index 9ae9e855fab..fa5f36888a1 100644 --- a/apps/sim/lib/file-parsers/index.ts +++ b/apps/sim/lib/file-parsers/index.ts @@ -17,7 +17,12 @@ import { OpenDocumentParser } from '@/lib/file-parsers/opendocument-parser' import { PdfParser } from '@/lib/file-parsers/pdf-parser' import { PptxParser } from '@/lib/file-parsers/pptx-parser' import { TxtParser } from '@/lib/file-parsers/txt-parser' -import type { FileParseResult, FileParser, SupportedFileType } from '@/lib/file-parsers/types' +import type { + FileParseOptions, + FileParseResult, + FileParser, + SupportedFileType, +} from '@/lib/file-parsers/types' import { XlsxParser } from '@/lib/file-parsers/xlsx-parser' import { parseYAML, parseYAMLBuffer } from '@/lib/file-parsers/yaml-parser' import { assertOoxmlArchiveWithinLimits } from '@/lib/file-parsers/zip-guard' @@ -84,9 +89,13 @@ const SUPPORTED_EXTENSIONS_TEXT = [...PARSERS.keys()].join(', ') /** * Parse a file based on its extension * @param filePath Path to the file + * @param options Cancellation options for parsers that support them * @returns Parsed content and metadata */ -export async function parseFile(filePath: string): Promise { +export async function parseFile( + filePath: string, + options: FileParseOptions = {} +): Promise { try { if (!filePath) { throw new Error('No file path provided') @@ -105,7 +114,7 @@ export async function parseFile(filePath: string): Promise { ) } - return await parser.parseFile(filePath) + return await parser.parseFile(filePath, options) } catch (error) { logger.error('File parsing error:', error) throw error @@ -116,6 +125,7 @@ export async function parseFile(filePath: string): Promise { * Parse a buffer based on file extension * @param buffer Buffer containing the file data * @param extension File extension without the dot (e.g., 'pdf', 'csv') + * @param options Cancellation options for parsers that support them * @returns Parsed content and metadata * * The zip-bomb guard runs here for every extension, not just the OOXML ones: @@ -123,7 +133,11 @@ export async function parseFile(filePath: string): Promise { * for buffers that are not ZIP archives. Individual parsers still call it so a * direct `parser.parseBuffer` caller is covered too. */ -export async function parseBuffer(buffer: Buffer, extension: string): Promise { +export async function parseBuffer( + buffer: Buffer, + extension: string, + options: FileParseOptions = {} +): Promise { try { if (!buffer || buffer.length === 0) { throw new FileParserError('empty_input', 'Empty buffer provided') @@ -152,7 +166,7 @@ export async function parseBuffer(buffer: Buffer, extension: string): Promise 'slide text' +const { mockParseOfficeAsync } = vi.hoisted(() => ({ mockParseOfficeAsync: vi.fn() })) + +vi.mock('officeparser', () => ({ parseOfficeAsync: mockParseOfficeAsync })) + +import { parseOfficeText, resolveParseOfficeAsync } from '@/lib/file-parsers/officeparser-module' describe('resolveParseOfficeAsync', () => { - /** - * Node and webpack synthesize named exports from `officeparser`'s CommonJS - * `module.exports`, so this is the shape the app server sees — and the only - * one the code used to handle. - */ - it('resolves the named export when the bundler synthesizes one', () => { + const parse = vi.fn() + + it('resolves the named export', () => { expect(resolveParseOfficeAsync({ parseOfficeAsync: parse })).toBe(parse) }) - /** - * esbuild — which builds the Trigger.dev worker — puts `module.exports` on - * `default` and leaves the named export undefined. Reading the named export - * directly yielded `undefined` there, and calling it threw - * `TypeError: parseOfficeAsync is not a function`, which every parser treats - * as a library failure and answers with a `degraded` scrape that the document - * pipeline then rejects. This is the shape that broke production: every - * `.pptx` and legacy `.doc` from a connector reported "No text could be - * extracted" while the same files parsed fine through the app. - */ - it('resolves through default when the bundler namespaces the CommonJS exports', () => { + it('resolves a bundled default export', () => { expect(resolveParseOfficeAsync({ default: { parseOfficeAsync: parse } })).toBe(parse) }) - /** A CommonJS module whose `module.exports` IS the function. */ - it('resolves a default export that is itself callable', () => { + it('resolves a callable default export', () => { expect(resolveParseOfficeAsync({ default: parse })).toBe(parse) }) - /** - * Fails loudly rather than handing back `undefined` for a caller to invoke — - * the undefined call is what produced a misleading "no text could be - * extracted" report instead of naming the real fault. - */ - it('throws when no shape exposes the entry point', () => { + it('throws when the entry point is absent', () => { expect(() => resolveParseOfficeAsync({})).toThrow('did not expose parseOfficeAsync') }) }) + +describe('parseOfficeText', () => { + beforeEach(() => { + vi.clearAllMocks() + mockParseOfficeAsync.mockResolvedValue('slide text') + }) + + it('preserves the legacy plain-text shape', async () => { + const input = Buffer.from('office archive') + + await expect(parseOfficeText(input)).resolves.toBe('slide text') + expect(mockParseOfficeAsync).toHaveBeenCalledWith(input) + }) + + it('rejects cancellation before loading the parser', async () => { + const controller = new AbortController() + controller.abort() + + await expect( + parseOfficeText(Buffer.from('office archive'), { signal: controller.signal }) + ).rejects.toMatchObject({ name: 'AbortError' }) + expect(mockParseOfficeAsync).not.toHaveBeenCalled() + }) + + it('rejects cancellation after parsing', async () => { + const controller = new AbortController() + mockParseOfficeAsync.mockImplementationOnce(async () => { + controller.abort() + return 'slide text' + }) + + await expect( + parseOfficeText(Buffer.from('office archive'), { signal: controller.signal }) + ).rejects.toMatchObject({ name: 'AbortError' }) + }) +}) diff --git a/apps/sim/lib/file-parsers/officeparser-module.ts b/apps/sim/lib/file-parsers/officeparser-module.ts index cf1ceb078f7..724822302f9 100644 --- a/apps/sim/lib/file-parsers/officeparser-module.ts +++ b/apps/sim/lib/file-parsers/officeparser-module.ts @@ -1,6 +1,6 @@ import { FileParserError } from '@/lib/file-parsers/errors' +import type { FileParseOptions } from '@/lib/file-parsers/types' -/** `officeparser`'s single entry point, as every parser here calls it. */ type ParseOfficeAsync = (input: Buffer) => Promise interface OfficeParserModule { @@ -9,28 +9,10 @@ interface OfficeParserModule { } /** - * Resolves `officeparser`'s entry point across module systems. - * - * `officeparser` is CommonJS — `main: officeParser.js`, no `type` and no - * `exports` map — so what `await import('officeparser')` yields depends on who - * built the code. Node and webpack synthesize named exports from the CJS - * `module.exports`, so `.parseOfficeAsync` is there. esbuild, which builds the - * Trigger.dev worker bundle, puts `module.exports` on `.default` and leaves the - * named export undefined. - * - * Reading the named export directly therefore worked everywhere except the - * worker, where `parseOfficeAsync` was `undefined` and calling it threw - * `TypeError: parseOfficeAsync is not a function`. Every parser here treats that - * as "the library failed" and falls back to scraping the archive, which returns - * `degraded: true` — and the document pipeline rejects a degraded parse outright. - * The visible result was every `.pptx` and legacy `.doc` from a connector - * failing as "No text could be extracted", while the same files parsed fine - * through the app. - * - * Reading both shapes fixes it at the source rather than per bundler: the - * alternative is externalizing the package in each build config, which has to be - * repeated for every bundler this code runs under and silently regresses the day - * one is missed. + * Resolve the parser entry point across ESM and bundled CommonJS namespace + * shapes, including the bundler shape where `module.exports` itself becomes + * the function on `default` — a mismatch here has silently degraded every + * `.pptx`/`.doc` parse in production before. */ export function resolveParseOfficeAsync(mod: OfficeParserModule): ParseOfficeAsync { if (typeof mod.parseOfficeAsync === 'function') return mod.parseOfficeAsync @@ -40,14 +22,7 @@ export function resolveParseOfficeAsync(mod: OfficeParserModule): ParseOfficeAsy throw new Error('officeparser did not expose parseOfficeAsync') } -/** - * Split from {@link resolveParseOfficeAsync} so the shape handling is testable. - * The failing shape cannot be reproduced by mocking the specifier — Vitest's - * module-namespace proxy throws on a missing export rather than yielding the - * `undefined` the real bundle produces — so a test that goes through `import` - * can only assert the shape that already worked. - */ -export async function loadParseOfficeAsync(): Promise { +async function loadParseOfficeAsync(): Promise { try { return resolveParseOfficeAsync((await import('officeparser')) as OfficeParserModule) } catch (error) { @@ -58,3 +33,15 @@ export async function loadParseOfficeAsync(): Promise { ) } } + +/** Parse an Office archive as plain text after the caller enforces app-level archive limits. */ +export async function parseOfficeText( + input: Buffer, + options: FileParseOptions = {} +): Promise { + options.signal?.throwIfAborted() + const parseOfficeAsync = await loadParseOfficeAsync() + const result = await parseOfficeAsync(input) + options.signal?.throwIfAborted() + return result +} diff --git a/apps/sim/lib/file-parsers/opendocument-parser.ts b/apps/sim/lib/file-parsers/opendocument-parser.ts index a97321ca9a0..1a33e8a72d7 100644 --- a/apps/sim/lib/file-parsers/opendocument-parser.ts +++ b/apps/sim/lib/file-parsers/opendocument-parser.ts @@ -2,8 +2,8 @@ import { existsSync } from 'fs' import { readFile } from 'fs/promises' import { createLogger } from '@sim/logger' import { FileParserError, isEncryptedOfficeParserError } from '@/lib/file-parsers/errors' -import { loadParseOfficeAsync } from '@/lib/file-parsers/officeparser-module' -import type { FileParseResult, FileParser } from '@/lib/file-parsers/types' +import { parseOfficeText } from '@/lib/file-parsers/officeparser-module' +import type { FileParseOptions, FileParseResult, FileParser } from '@/lib/file-parsers/types' import { sanitizeTextForUTF8 } from '@/lib/file-parsers/utils' import { assertOoxmlArchiveWithinLimits } from '@/lib/file-parsers/zip-guard' @@ -24,7 +24,7 @@ const logger = createLogger('OpenDocumentParser') * and renders with per-sheet structure rather than one flat text run. */ export class OpenDocumentParser implements FileParser { - async parseFile(filePath: string): Promise { + async parseFile(filePath: string, options: FileParseOptions = {}): Promise { if (!filePath) { throw new Error('No file path provided') } @@ -33,11 +33,12 @@ export class OpenDocumentParser implements FileParser { throw new Error(`File not found: ${filePath}`) } - const buffer = await readFile(filePath) - return this.parseBuffer(buffer) + const buffer = await readFile(filePath, { signal: options.signal }) + return this.parseBuffer(buffer, options) } - async parseBuffer(buffer: Buffer): Promise { + async parseBuffer(buffer: Buffer, options: FileParseOptions = {}): Promise { + options.signal?.throwIfAborted() if (!buffer || buffer.length === 0) { throw new FileParserError('empty_input', 'Empty buffer provided') } @@ -48,13 +49,12 @@ export class OpenDocumentParser implements FileParser { */ assertOoxmlArchiveWithinLimits(buffer) - const parseOfficeAsync = await loadParseOfficeAsync() - let extracted: string try { - const result = await parseOfficeAsync(buffer) + const result = await parseOfficeText(buffer, options) extracted = typeof result === 'string' ? result : '' } catch (error) { + options.signal?.throwIfAborted() logger.error('OpenDocument parsing failed', { error: (error as Error).message }) if (isEncryptedOfficeParserError(error)) { throw new FileParserError( diff --git a/apps/sim/lib/file-parsers/parser-formats.test.ts b/apps/sim/lib/file-parsers/parser-formats.test.ts index fedaec26a1a..0864cbc736d 100644 --- a/apps/sim/lib/file-parsers/parser-formats.test.ts +++ b/apps/sim/lib/file-parsers/parser-formats.test.ts @@ -260,7 +260,7 @@ describe('OpenDocumentParser', () => { zip.file('META-INF/manifest.xml', '') zip.file( 'content.xml', - `${bodyXml}` + `${bodyXml}` ) return zip.generateAsync({ type: 'nodebuffer', compression: 'DEFLATE' }) as Promise } @@ -280,7 +280,7 @@ describe('OpenDocumentParser', () => { it('extracts slide text from an odp', async () => { const buffer = await buildOdf( 'application/vnd.oasis.opendocument.presentation', - 'OpenDocument slide' + 'OpenDocument slide' ) const result = await new OpenDocumentParser().parseBuffer(buffer) diff --git a/apps/sim/lib/file-parsers/pdf-parser-cancellation.test.ts b/apps/sim/lib/file-parsers/pdf-parser-cancellation.test.ts new file mode 100644 index 00000000000..cef7ef6550f --- /dev/null +++ b/apps/sim/lib/file-parsers/pdf-parser-cancellation.test.ts @@ -0,0 +1,138 @@ +/** + * @vitest-environment node + */ +import type { PDFDocumentProxy, PDFPageProxy } from 'pdfjs-dist/types/src/pdf' +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' + +const { mockOpenPdfDocument } = vi.hoisted(() => ({ + mockOpenPdfDocument: vi.fn(), +})) + +vi.mock('@/lib/file-parsers/pdfjs-server', () => ({ + openPdfDocument: mockOpenPdfDocument, +})) + +import { PdfParser } from '@/lib/file-parsers/pdf-parser' + +describe('PdfParser cancellation', () => { + beforeEach(() => { + vi.clearAllMocks() + }) + + afterEach(() => { + vi.useRealTimers() + }) + + it('rejects an already-cancelled parse before opening pdf.js', async () => { + const controller = new AbortController() + controller.abort() + + await expect( + new PdfParser().parseBuffer(Buffer.from('%PDF-1.4'), { signal: controller.signal }) + ).rejects.toMatchObject({ name: 'AbortError' }) + expect(mockOpenPdfDocument).not.toHaveBeenCalled() + }) + + it('forwards cancellation while pdf.js is opening', async () => { + const controller = new AbortController() + mockOpenPdfDocument.mockImplementationOnce( + (_data: Uint8Array, signal?: AbortSignal) => + new Promise((_resolve, reject) => { + signal?.addEventListener('abort', () => reject(signal.reason), { once: true }) + }) + ) + + const parsing = new PdfParser().parseBuffer(Buffer.from('%PDF-1.4'), { + signal: controller.signal, + }) + await vi.waitFor(() => expect(mockOpenPdfDocument).toHaveBeenCalledOnce()) + controller.abort() + + await expect(parsing).rejects.toMatchObject({ name: 'AbortError' }) + expect(mockOpenPdfDocument).toHaveBeenCalledWith(expect.any(Uint8Array), controller.signal) + }) + + it('cancels a pending text reader and releases page and document state', async () => { + const reader = { + cancel: vi.fn().mockResolvedValue(undefined), + read: vi.fn(() => new Promise(() => {})), + } + const page = { + cleanup: vi.fn(), + streamTextContent: vi.fn(() => ({ getReader: () => reader })), + } as PDFPageProxy + const pdf = { + destroy: vi.fn().mockResolvedValue(undefined), + getPage: vi.fn().mockResolvedValue(page), + numPages: 1, + } as PDFDocumentProxy + mockOpenPdfDocument.mockResolvedValueOnce(pdf) + const controller = new AbortController() + + const parsing = new PdfParser().parseBuffer(Buffer.from('%PDF-1.4'), { + signal: controller.signal, + }) + await vi.waitFor(() => expect(reader.read).toHaveBeenCalledOnce()) + controller.abort() + + await expect(parsing).rejects.toMatchObject({ name: 'AbortError' }) + expect(reader.cancel).toHaveBeenCalledOnce() + expect(page.cleanup).toHaveBeenCalledOnce() + expect(pdf.destroy).toHaveBeenCalledOnce() + }) + + it('cancels a stalled text reader at the extraction deadline and returns a partial result', async () => { + vi.useFakeTimers() + const reader = { + cancel: vi.fn().mockResolvedValue(undefined), + read: vi + .fn() + .mockResolvedValueOnce({ value: { items: [{ str: 'partial page text' }] }, done: false }) + .mockImplementation(() => new Promise(() => {})), + } + const page = { + cleanup: vi.fn(), + streamTextContent: vi.fn(() => ({ getReader: () => reader })), + } as PDFPageProxy + const pdf = { + destroy: vi.fn().mockResolvedValue(undefined), + getPage: vi.fn().mockResolvedValue(page), + numPages: 1, + } as PDFDocumentProxy + mockOpenPdfDocument.mockResolvedValueOnce(pdf) + + const parsing = new PdfParser().parseBuffer(Buffer.from('%PDF-1.4')) + await vi.advanceTimersByTimeAsync(0) + expect(reader.read).toHaveBeenCalledTimes(2) + + await vi.advanceTimersByTimeAsync(60_000) + const result = await parsing + + expect(result.metadata).toMatchObject({ pageCount: 1, truncated: true }) + expect(result.content).toContain('partial page text') + expect(result.content).toMatch(/PDF text truncated at parser limits/) + expect(reader.cancel).toHaveBeenCalledOnce() + expect(page.cleanup).toHaveBeenCalledOnce() + expect(pdf.destroy).toHaveBeenCalledOnce() + }) + + it('stops at the extraction deadline when loading a page stalls', async () => { + vi.useFakeTimers() + const pdf = { + destroy: vi.fn().mockResolvedValue(undefined), + getPage: vi.fn(() => new Promise(() => {})), + numPages: 1, + } as PDFDocumentProxy + mockOpenPdfDocument.mockResolvedValueOnce(pdf) + + const parsing = new PdfParser().parseBuffer(Buffer.from('%PDF-1.4')) + await vi.advanceTimersByTimeAsync(0) + expect(pdf.getPage).toHaveBeenCalledOnce() + + await vi.advanceTimersByTimeAsync(60_000) + const result = await parsing + + expect(result.metadata).toMatchObject({ pageCount: 1, truncated: true }) + expect(pdf.destroy).toHaveBeenCalledOnce() + }) +}) diff --git a/apps/sim/lib/file-parsers/pdf-parser.test.ts b/apps/sim/lib/file-parsers/pdf-parser.test.ts index 19c1606871e..b1f352223c5 100644 --- a/apps/sim/lib/file-parsers/pdf-parser.test.ts +++ b/apps/sim/lib/file-parsers/pdf-parser.test.ts @@ -4,6 +4,7 @@ import { deflateSync } from 'zlib' import { describe, expect, it } from 'vitest' import { MAX_PDF_TEXT_CHARS, PdfParser } from '@/lib/file-parsers/pdf-parser' +import { openPdfDocument } from '@/lib/file-parsers/pdfjs-server' /** * Builds a single-page PDF that draws 64 characters per repeat from a @@ -49,8 +50,26 @@ function buildTextFreePdf(pageCount: number): Buffer { ]) } +/** Builds a structurally valid PDF that requires a password before opening. */ +function buildEncryptedPdf(): Buffer { + const ownerAndUserKey = '00'.repeat(32) + const documentId = '11'.repeat(16) + + return assemblePdf( + [ + Buffer.from('<< /Type /Catalog /Pages 2 0 R >>'), + Buffer.from('<< /Type /Pages /Kids [3 0 R] /Count 1 >>'), + Buffer.from('<< /Type /Page /Parent 2 0 R /MediaBox [0 0 612 792] >>'), + Buffer.from( + `<< /Filter /Standard /V 1 /R 2 /O <${ownerAndUserKey}> /U <${ownerAndUserKey}> /P -4 >>` + ), + ], + `/Encrypt 4 0 R /ID [<${documentId}> <${documentId}>]` + ) +} + /** Serializes numbered objects into a PDF with a matching xref table and trailer. */ -function assemblePdf(objects: Buffer[]): Buffer { +function assemblePdf(objects: Buffer[], trailerEntries = ''): Buffer { const chunks: Buffer[] = [Buffer.from('%PDF-1.4\n')] const offsets: number[] = [] let offset = chunks[0].length @@ -72,7 +91,7 @@ function assemblePdf(objects: Buffer[]): Buffer { chunks.push( Buffer.from( `xref\n0 ${objects.length + 1}\n0000000000 65535 f \n${xrefRows}` + - `trailer\n<< /Size ${objects.length + 1} /Root 1 0 R >>\nstartxref\n${offset}\n%%EOF\n` + `trailer\n<< /Size ${objects.length + 1} /Root 1 0 R ${trailerEntries} >>\nstartxref\n${offset}\n%%EOF\n` ) ) @@ -80,6 +99,27 @@ function assemblePdf(objects: Buffer[]): Buffer { } describe('PdfParser', () => { + it('preloads the server worker instead of relying on a runtime-relative worker path', async () => { + const previousWorker: unknown = Reflect.get(globalThis, 'pdfjsWorker') + Reflect.deleteProperty(globalThis, 'pdfjsWorker') + + const pdf = await openPdfDocument(new Uint8Array(buildTextFreePdf(1))) + + try { + expect(Reflect.get(globalThis, 'pdfjsWorker')).toEqual({ + WorkerMessageHandler: expect.anything(), + }) + expect(pdf.numPages).toBe(1) + } finally { + await pdf.destroy() + if (previousWorker === undefined) { + Reflect.deleteProperty(globalThis, 'pdfjsWorker') + } else { + Reflect.set(globalThis, 'pdfjsWorker', previousWorker) + } + } + }) + it('bounds extracted text from a compression-bomb PDF instead of exhausting the heap', async () => { const bomb = buildTextBombPdf(200_000) expect(bomb.length).toBeLessThan(200 * 1024) @@ -103,6 +143,7 @@ describe('PdfParser', () => { expect(result.metadata?.truncated).toBe(false) expect(result.metadata?.warning).toBeUndefined() expect(result.metadata?.pageCount).toBe(1) + expect(result.metadata?.source).toBe('unpdf') expect(result.content).toContain('AAAA') expect(result.content).not.toContain('truncated') }, 30_000) @@ -113,4 +154,16 @@ describe('PdfParser', () => { expect(result.content.trim()).toBe('') expect(result.content).not.toContain('[...') }, 30_000) + + it('rejects malformed PDF input', async () => { + await expect(new PdfParser().parseBuffer(Buffer.from('%PDF-1.4\nnot a PDF'))).rejects.toThrow( + /Invalid PDF|PDF structure|document/i + ) + }) + + it('preserves the password-required error for encrypted PDFs', async () => { + await expect(new PdfParser().parseBuffer(buildEncryptedPdf())).rejects.toMatchObject({ + name: 'PasswordException', + }) + }) }) diff --git a/apps/sim/lib/file-parsers/pdf-parser.ts b/apps/sim/lib/file-parsers/pdf-parser.ts index 81b10847d27..e4106b19f9e 100644 --- a/apps/sim/lib/file-parsers/pdf-parser.ts +++ b/apps/sim/lib/file-parsers/pdf-parser.ts @@ -1,6 +1,8 @@ import { readFile } from 'fs/promises' import { createLogger } from '@sim/logger' -import type { FileParseResult, FileParser } from '@/lib/file-parsers/types' +import type { PDFDocumentProxy, PDFPageProxy } from 'pdfjs-dist/types/src/pdf' +import { openPdfDocument } from '@/lib/file-parsers/pdfjs-server' +import type { FileParseOptions, FileParseResult, FileParser } from '@/lib/file-parsers/types' import { sanitizeTextForUTF8, truncationNotice } from '@/lib/file-parsers/utils' const logger = createLogger('PdfParser') @@ -19,9 +21,10 @@ export const MAX_PDF_TEXT_CHARS = 10_000_000 const PDF_EXTRACTION_TIMEOUT_MS = 60_000 const PDF_TRUNCATION_WARNING = 'PDF text extraction stopped at a parser limit and is incomplete' +const PDF_READ_DEADLINE_REACHED = Symbol('PDF_READ_DEADLINE_REACHED') -type PdfDocumentProxy = Awaited> -type PdfPageProxy = Awaited> +/** Stable metadata identifier retained for documents indexed before the parser swap. */ +const PDF_PARSER_SOURCE = 'unpdf' interface TextContentChunk { items?: Array<{ str?: unknown; hasEOL?: unknown }> @@ -45,6 +48,69 @@ interface BoundedExtraction { truncated: boolean } +function waitForAbort( + operation: Promise, + signal?: AbortSignal, + onAbort?: () => void +): Promise { + if (!signal) return operation + + try { + signal.throwIfAborted() + } catch (error) { + onAbort?.() + return Promise.reject(error) + } + + return new Promise((resolve, reject) => { + const cleanup = () => signal.removeEventListener('abort', handleAbort) + const handleAbort = () => { + cleanup() + onAbort?.() + reject(signal.reason) + } + + signal.addEventListener('abort', handleAbort, { once: true }) + if (signal.aborted) handleAbort() + operation.then( + (value) => { + cleanup() + resolve(value) + }, + (error: unknown) => { + cleanup() + reject(error) + } + ) + }) +} + +function waitForDeadline( + operation: Promise, + deadline: number, + signal: AbortSignal | undefined, + onDeadline: () => void, + onAbort: () => void +): Promise { + const remainingMs = deadline - Date.now() + if (remainingMs <= 0) { + onDeadline() + return Promise.resolve(PDF_READ_DEADLINE_REACHED) + } + + let timeoutId: ReturnType | undefined + const deadlineReached = new Promise((resolve) => { + timeoutId = setTimeout(() => { + resolve(PDF_READ_DEADLINE_REACHED) + onDeadline() + }, remainingMs) + }) + + return waitForAbort(Promise.race([operation, deadlineReached]), signal, onAbort).finally(() => { + if (timeoutId !== undefined) clearTimeout(timeoutId) + }) +} + /** * Reads one page's text through pdf.js's streaming API, stopping once the * character budget or the deadline is spent. @@ -57,10 +123,12 @@ interface BoundedExtraction { * evaluator rather than letting it run the expansion to completion. */ async function readPageWithinBudget( - page: PdfPageProxy, + page: PDFPageProxy, budget: number, - deadline: number + deadline: number, + signal?: AbortSignal ): Promise { + signal?.throwIfAborted() const reader = page .streamTextContent() .getReader() as ReadableStreamDefaultReader @@ -69,14 +137,33 @@ async function readPageWithinBudget( let remaining = budget let completed = false let dropped = false + let deadlineReached = false + let cancellation: Promise | undefined + + const cancelReader = (reason: unknown): Promise => { + cancellation ??= reader.cancel(reason).catch(() => {}) + return cancellation + } try { /** * Loops until content is actually dropped rather than until the budget hits * zero: text that ends exactly on the budget is complete, not truncated. */ - while (!dropped && Date.now() <= deadline) { - const { value, done } = await reader.read() + while (!dropped) { + const result = await waitForDeadline( + reader.read(), + deadline, + signal, + () => void cancelReader(new Error('PDF text extraction deadline exceeded')), + () => void cancelReader(signal?.reason) + ) + if (result === PDF_READ_DEADLINE_REACHED) { + deadlineReached = true + break + } + + const { value, done } = result if (done) { completed = true break @@ -99,19 +186,18 @@ async function readPageWithinBudget( } } finally { if (!completed) { - try { - await reader.cancel(new Error('PDF text extraction budget exceeded')) - } catch { - // Cancelling a stream that already failed is not itself an error, and - // throwing here would mask whatever ended the read loop. - } + const pendingCancellation = cancelReader(new Error('PDF text extraction budget exceeded')) + if (!signal?.aborted && !deadlineReached) await pendingCancellation } } return { text: parts.join(''), used: budget - remaining, completed } } -async function extractTextWithinBudget(pdf: PdfDocumentProxy): Promise { +async function extractTextWithinBudget( + pdf: PDFDocumentProxy, + signal?: AbortSignal +): Promise { const deadline = Date.now() + PDF_EXTRACTION_TIMEOUT_MS const totalPages = pdf.numPages const pageLimit = Math.min(totalPages, MAX_PDF_PAGES) @@ -121,8 +207,32 @@ async function extractTextWithinBudget(pdf: PdfDocumentProxy): Promise pageLimit for (let pageNumber = 1; pageNumber <= pageLimit; pageNumber++) { - const page = await pdf.getPage(pageNumber) - const { text, used, completed } = await readPageWithinBudget(page, remainingChars, deadline) + signal?.throwIfAborted() + const pagePromise = pdf.getPage(pageNumber) + const cleanupLatePage = () => { + void pagePromise.then((latePage) => latePage.cleanup()).catch(() => {}) + } + const pageResult = await waitForDeadline( + pagePromise, + deadline, + signal, + cleanupLatePage, + cleanupLatePage + ) + if (pageResult === PDF_READ_DEADLINE_REACHED) { + truncated = true + break + } + + const page = pageResult + let extraction: PageExtraction + try { + extraction = await readPageWithinBudget(page, remainingChars, deadline, signal) + } finally { + page.cleanup() + } + + const { text, used, completed } = extraction remainingChars -= used @@ -131,7 +241,6 @@ async function extractTextWithinBudget(pdf: PdfDocumentProxy): Promise 0) { pageTexts.push(text) } - page.cleanup() if (!completed) { truncated = true @@ -148,7 +257,7 @@ async function extractTextWithinBudget(pdf: PdfDocumentProxy): Promise { + async parseFile(filePath: string, options: FileParseOptions = {}): Promise { try { logger.info('Starting to parse file:', filePath) @@ -157,28 +266,29 @@ export class PdfParser implements FileParser { } logger.info('Reading file...') - const dataBuffer = await readFile(filePath) + const dataBuffer = await readFile(filePath, { signal: options.signal }) logger.info('File read successfully, size:', dataBuffer.length) - return this.parseBuffer(dataBuffer) + return this.parseBuffer(dataBuffer, options) } catch (error) { logger.error('Error reading file:', error) throw error } } - async parseBuffer(dataBuffer: Buffer): Promise { + async parseBuffer(dataBuffer: Buffer, options: FileParseOptions = {}): Promise { try { + options.signal?.throwIfAborted() logger.info('Starting to parse buffer, size:', dataBuffer.length) - const { getDocumentProxy } = await import('unpdf') - const uint8Array = new Uint8Array(dataBuffer) - - const pdf = await getDocumentProxy(uint8Array) + const pdf = await openPdfDocument(uint8Array, options.signal) try { - const { text, totalPages, pagesRead, truncated } = await extractTextWithinBudget(pdf) + const { text, totalPages, pagesRead, truncated } = await extractTextWithinBudget( + pdf, + options.signal + ) logger.info('PDF parsed successfully, pages:', totalPages, 'text length:', text.length) @@ -204,7 +314,7 @@ export class PdfParser implements FileParser { content: body + notice, metadata: { pageCount: totalPages, - source: 'unpdf', + source: PDF_PARSER_SOURCE, truncated, warning: truncated ? PDF_TRUNCATION_WARNING : undefined, }, diff --git a/apps/sim/lib/file-parsers/pdfjs-server.test.ts b/apps/sim/lib/file-parsers/pdfjs-server.test.ts new file mode 100644 index 00000000000..768493d79c1 --- /dev/null +++ b/apps/sim/lib/file-parsers/pdfjs-server.test.ts @@ -0,0 +1,47 @@ +/** + * @vitest-environment node + */ +import type { PDFDocumentLoadingTask } from 'pdfjs-dist/types/src/pdf' +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const { mockGetDocument, workerMessageHandler } = vi.hoisted(() => ({ + mockGetDocument: vi.fn(), + workerMessageHandler: {}, +})) + +vi.mock('pdfjs-dist/legacy/build/pdf.mjs', () => ({ getDocument: mockGetDocument })) +vi.mock('pdfjs-dist/legacy/build/pdf.worker.mjs', () => ({ + WorkerMessageHandler: workerMessageHandler, +})) + +import { openPdfDocument } from '@/lib/file-parsers/pdfjs-server' + +describe('openPdfDocument', () => { + beforeEach(() => { + vi.clearAllMocks() + }) + + it('destroys a pending loading task immediately when parsing is cancelled', async () => { + let resolveLoading: ((pdf: { destroy: () => Promise }) => void) | undefined + const lateDocumentDestroy = vi.fn().mockResolvedValue(undefined) + const destroy = vi.fn().mockResolvedValue(undefined) + const loadingTask = { + destroy, + promise: new Promise((resolve) => { + resolveLoading = resolve + }), + } as PDFDocumentLoadingTask + mockGetDocument.mockReturnValueOnce(loadingTask) + const controller = new AbortController() + + const opening = openPdfDocument(new Uint8Array([1, 2, 3]), controller.signal) + await vi.waitFor(() => expect(mockGetDocument).toHaveBeenCalledOnce()) + controller.abort() + + await expect(opening).rejects.toMatchObject({ name: 'AbortError' }) + expect(destroy).toHaveBeenCalledOnce() + + resolveLoading?.({ destroy: lateDocumentDestroy }) + await vi.waitFor(() => expect(lateDocumentDestroy).toHaveBeenCalledOnce()) + }) +}) diff --git a/apps/sim/lib/file-parsers/pdfjs-server.ts b/apps/sim/lib/file-parsers/pdfjs-server.ts new file mode 100644 index 00000000000..88184fef7f5 --- /dev/null +++ b/apps/sim/lib/file-parsers/pdfjs-server.ts @@ -0,0 +1,71 @@ +import type { PDFDocumentLoadingTask, PDFDocumentProxy } from 'pdfjs-dist/types/src/pdf' + +function waitForLoadingTask( + loadingTask: PDFDocumentLoadingTask, + signal?: AbortSignal +): Promise { + if (!signal) return loadingTask.promise + + const destroy = () => { + try { + void loadingTask.destroy().catch(() => {}) + } catch {} + } + + if (signal.aborted) { + destroy() + signal.throwIfAborted() + } + + let aborted = false + return new Promise((resolve, reject) => { + const cleanup = () => signal.removeEventListener('abort', handleAbort) + const handleAbort = () => { + aborted = true + cleanup() + destroy() + reject(signal.reason) + } + + signal.addEventListener('abort', handleAbort, { once: true }) + loadingTask.promise.then( + (pdf) => { + cleanup() + if (aborted) { + void pdf.destroy().catch(() => {}) + return + } + resolve(pdf) + }, + (error: unknown) => { + cleanup() + reject(error) + } + ) + }) +} + +/** Open a PDF with the server-compatible pdf.js build and hardened defaults. */ +export async function openPdfDocument( + data: Uint8Array, + signal?: AbortSignal +): Promise { + signal?.throwIfAborted() + const [{ getDocument }, { WorkerMessageHandler }] = await Promise.all([ + import('pdfjs-dist/legacy/build/pdf.mjs'), + import('pdfjs-dist/legacy/build/pdf.worker.mjs'), + ]) + signal?.throwIfAborted() + + Object.assign(globalThis, { + pdfjsWorker: { WorkerMessageHandler }, + }) + + const loadingTask = getDocument({ + data, + isEvalSupported: false, + useSystemFonts: true, + }) + + return waitForLoadingTask(loadingTask, signal) +} diff --git a/apps/sim/lib/file-parsers/pptx-parser.test.ts b/apps/sim/lib/file-parsers/pptx-parser.test.ts index c7cda6e2ba5..93cca1433be 100644 --- a/apps/sim/lib/file-parsers/pptx-parser.test.ts +++ b/apps/sim/lib/file-parsers/pptx-parser.test.ts @@ -3,12 +3,12 @@ */ import { describe, expect, it, vi } from 'vitest' -const { mockParseOfficeAsync } = vi.hoisted(() => ({ - mockParseOfficeAsync: vi.fn(), +const { mockParseOfficeText } = vi.hoisted(() => ({ + mockParseOfficeText: vi.fn(), })) vi.mock('@/lib/file-parsers/officeparser-module', () => ({ - loadParseOfficeAsync: vi.fn(async () => mockParseOfficeAsync), + parseOfficeText: mockParseOfficeText, })) import type { FileParserError } from '@/lib/file-parsers/errors' @@ -17,7 +17,7 @@ import { PptxParser } from '@/lib/file-parsers/pptx-parser' describe('PptxParser', () => { it('classifies encrypted legacy presentations before degraded extraction', async () => { const libraryError = new Error('File is password-protected') - mockParseOfficeAsync.mockRejectedValueOnce(libraryError) + mockParseOfficeText.mockRejectedValueOnce(libraryError) const legacyOleBuffer = Buffer.from([0xd0, 0xcf, 0x11, 0xe0, 0xa1, 0xb1, 0x1a, 0xe1]) const result = new PptxParser().parseBuffer(legacyOleBuffer) @@ -27,4 +27,19 @@ describe('PptxParser', () => { cause: libraryError, }) }) + + it('preserves cancellation instead of degrading to scraped bytes', async () => { + const controller = new AbortController() + const abortError = new DOMException('The operation was aborted', 'AbortError') + mockParseOfficeText.mockImplementationOnce(async () => { + controller.abort(abortError) + throw abortError + }) + + await expect( + new PptxParser().parseBuffer(Buffer.from('legacy presentation'), { + signal: controller.signal, + }) + ).rejects.toBe(abortError) + }) }) diff --git a/apps/sim/lib/file-parsers/pptx-parser.ts b/apps/sim/lib/file-parsers/pptx-parser.ts index 10db14bd6b1..db7c50aa37d 100644 --- a/apps/sim/lib/file-parsers/pptx-parser.ts +++ b/apps/sim/lib/file-parsers/pptx-parser.ts @@ -2,15 +2,15 @@ import { existsSync } from 'fs' import { readFile } from 'fs/promises' import { createLogger } from '@sim/logger' import { FileParserError, isEncryptedOfficeParserError } from '@/lib/file-parsers/errors' -import { loadParseOfficeAsync } from '@/lib/file-parsers/officeparser-module' -import type { FileParseResult, FileParser } from '@/lib/file-parsers/types' +import { parseOfficeText } from '@/lib/file-parsers/officeparser-module' +import type { FileParseOptions, FileParseResult, FileParser } from '@/lib/file-parsers/types' import { sanitizeTextForUTF8 } from '@/lib/file-parsers/utils' import { assertOoxmlArchiveWithinLimits } from '@/lib/file-parsers/zip-guard' const logger = createLogger('PptxParser') export class PptxParser implements FileParser { - async parseFile(filePath: string): Promise { + async parseFile(filePath: string, options: FileParseOptions = {}): Promise { if (!filePath) { throw new Error('No file path provided') } @@ -21,23 +21,22 @@ export class PptxParser implements FileParser { logger.info(`Parsing PowerPoint file: ${filePath}`) - const buffer = await readFile(filePath) - return this.parseBuffer(buffer) + const buffer = await readFile(filePath, { signal: options.signal }) + return this.parseBuffer(buffer, options) } - async parseBuffer(buffer: Buffer): Promise { + async parseBuffer(buffer: Buffer, options: FileParseOptions = {}): Promise { logger.info('Parsing PowerPoint buffer, size:', buffer.length) + options.signal?.throwIfAborted() if (!buffer || buffer.length === 0) { throw new FileParserError('empty_input', 'Empty buffer provided') } assertOoxmlArchiveWithinLimits(buffer) - const parseOfficeAsync = await loadParseOfficeAsync() - try { - const result = await parseOfficeAsync(buffer) + const result = await parseOfficeText(buffer, options) if (!result || typeof result !== 'string') { return this.fallbackExtraction(buffer) @@ -55,6 +54,7 @@ export class PptxParser implements FileParser { }, } } catch (extractError) { + options.signal?.throwIfAborted() if (isEncryptedOfficeParserError(extractError)) { throw new FileParserError( 'encrypted_file', diff --git a/apps/sim/lib/file-parsers/types.ts b/apps/sim/lib/file-parsers/types.ts index 71f9d9764d0..a054657f5c9 100644 --- a/apps/sim/lib/file-parsers/types.ts +++ b/apps/sim/lib/file-parsers/types.ts @@ -31,9 +31,13 @@ export interface FileParseResult { metadata?: FileParseMetadata } +export interface FileParseOptions { + signal?: AbortSignal +} + export interface FileParser { - parseFile(filePath: string): Promise - parseBuffer?(buffer: Buffer): Promise + parseFile(filePath: string, options?: FileParseOptions): Promise + parseBuffer?(buffer: Buffer, options?: FileParseOptions): Promise } export type SupportedFileType = diff --git a/apps/sim/lib/file-parsers/yaml-limits.test.ts b/apps/sim/lib/file-parsers/yaml-limits.test.ts new file mode 100644 index 00000000000..b03dc85edd0 --- /dev/null +++ b/apps/sim/lib/file-parsers/yaml-limits.test.ts @@ -0,0 +1,139 @@ +/** + * @vitest-environment node + */ +import { describe, expect, it } from 'vitest' +import { + createYamlExpansionBudget, + isYamlExpansionBudgetExhausted, + measureYamlExpansion, + type YamlExpansionLimits, +} from '@/lib/file-parsers/yaml-limits' + +const LIMITS: YamlExpansionLimits = { + maxNodes: 1000, + maxSerializedBytes: 64 * 1024, + maxDepth: 10, +} + +const limits = (overrides: Partial = {}): YamlExpansionLimits => ({ + ...LIMITS, + ...overrides, +}) + +describe('measureYamlExpansion', () => { + it('reports the depth of the expanded tree', () => { + expect(measureYamlExpansion('scalar', LIMITS)).toEqual({ within: true, depth: 0 }) + expect(measureYamlExpansion([1, 2, 3], LIMITS)).toEqual({ within: true, depth: 1 }) + expect(measureYamlExpansion({ a: { b: { c: 1 } } }, LIMITS)).toEqual({ within: true, depth: 3 }) + }) + + it('counts an empty container as a level', () => { + expect(measureYamlExpansion([], LIMITS)).toEqual({ within: true, depth: 1 }) + expect(measureYamlExpansion({ a: {} }, LIMITS)).toEqual({ within: true, depth: 2 }) + }) + + it('charges an aliased subtree once per path that reaches it', () => { + const shared = [1, 2, 3, 4, 5] + const aliased = { a: shared, b: shared, c: shared } + + // 19 nodes when every reach is charged (root + 3 refs + 3x5 elements); 9 if the + // shared array were counted once, which is what makes an alias bomb invisible. + expect(measureYamlExpansion(aliased, limits({ maxNodes: 19 }))).toEqual({ + within: true, + depth: 2, + }) + expect(measureYamlExpansion(aliased, limits({ maxNodes: 18 })).within).toBe(false) + }) + + it('terminates on a self-referential anchor instead of recursing forever', () => { + const cyclic: Record = {} + cyclic.self = cyclic + + const measured = measureYamlExpansion(cyclic, LIMITS) + + expect(measured.within).toBe(false) + if (!measured.within) expect(measured.reason).toContain('nesting depth') + }) + + it('rejects a wide fan-out of containers without enumerating it first', () => { + // The traversal holds one frame per level, not one per pending node, so a node + // whose fan-out dwarfs the budget trips the cap part way through rather than + // after building a frame for every sibling. + const wide = Array.from({ length: 100_000 }, () => ({ a: 1 })) + + const measured = measureYamlExpansion(wide, limits({ maxNodes: 50 })) + + expect(measured.within).toBe(false) + if (!measured.within) expect(measured.reason).toContain('expanded nodes') + }) + + it('charges a long number its serialized length, not the flat allowance', () => { + // Each serializes to 24 characters; the flat non-string allowance is 16. + const longNumbers = Array.from({ length: 200 }, () => -1.2345678901234567e-308) + const shortNumbers = Array.from({ length: 200 }, () => 1) + + // Sits between what 200 short numbers cost (~4.4 KB) and what 200 long ones do + // (~6 KB); under the flat allowance both would land on the same side of it. + const byteCap = limits({ maxSerializedBytes: 5000 }) + + expect(measureYamlExpansion(shortNumbers, byteCap).within).toBe(true) + expect(measureYamlExpansion(longNumbers, byteCap).within).toBe(false) + }) + + it('charges a Date its quoted ISO length, not the flat allowance', () => { + // The default js-yaml schema turns `!!timestamp` into a Date, and JSON.stringify + // emits it as a 26-character quoted string — an aliased list of them would + // otherwise be charged 16 apiece and slip past the byte cap. + const dates = Array.from({ length: 200 }, () => new Date('2026-08-31T00:00:00.000Z')) + const booleans = Array.from({ length: 200 }, () => true) + + const byteCap = limits({ maxSerializedBytes: 5000 }) + + expect(measureYamlExpansion(booleans, byteCap).within).toBe(true) + expect(measureYamlExpansion(dates, byteCap).within).toBe(false) + }) + + it('charges object keys, so an aliased object with long keys cannot bypass the cap', () => { + const key = 'k'.repeat(500) + const shared = { [key]: 1 } + const aliased = Array.from({ length: 50 }, () => shared) + + const measured = measureYamlExpansion(aliased, limits({ maxSerializedBytes: 10_000 })) + + expect(measured.within).toBe(false) + if (!measured.within) expect(measured.reason).toContain('serialized size') + }) + + it('draws several documents down one shared budget', () => { + const budget = createYamlExpansionBudget(limits({ maxNodes: 30 })) + const doc = Array.from({ length: 10 }, (_, i) => i) + + expect(measureYamlExpansion(doc, limits({ maxNodes: 30 }), budget).within).toBe(true) + expect(isYamlExpansionBudgetExhausted(budget)).toBe(false) + expect(measureYamlExpansion(doc, limits({ maxNodes: 30 }), budget).within).toBe(true) + // The third pass runs out: 3 x 11 nodes exceeds the 30 the budget was created with. + expect(measureYamlExpansion(doc, limits({ maxNodes: 30 }), budget).within).toBe(false) + expect(isYamlExpansionBudgetExhausted(budget)).toBe(true) + }) + + it('leaves a shared budget usable after a depth rejection', () => { + // Depth costs only its own nesting, so one over-deep document must not bankrupt + // the documents that share its budget. + const budget = createYamlExpansionBudget(limits({ maxDepth: 2 })) + const deep = { a: { b: { c: { d: 1 } } } } + + expect(measureYamlExpansion(deep, limits({ maxDepth: 2 }), budget).within).toBe(false) + expect(isYamlExpansionBudgetExhausted(budget)).toBe(false) + expect(measureYamlExpansion({ ok: 1 }, limits({ maxDepth: 2 }), budget).within).toBe(true) + }) + + it('ignores inherited properties when walking an object', () => { + const parent = { inherited: 'x'.repeat(5000) } + const child = Object.create(parent) as Record + child.own = 1 + + const measured = measureYamlExpansion(child, limits({ maxSerializedBytes: 200 })) + + expect(measured).toEqual({ within: true, depth: 1 }) + }) +}) diff --git a/apps/sim/lib/file-parsers/yaml-limits.ts b/apps/sim/lib/file-parsers/yaml-limits.ts new file mode 100644 index 00000000000..a7822949640 --- /dev/null +++ b/apps/sim/lib/file-parsers/yaml-limits.ts @@ -0,0 +1,232 @@ +/** + * Bounded traversal of a parsed YAML value, shared by every consumer that walks + * one as a tree. + * + * `yaml.load` resolves aliases into shared references, so the parsed value is a + * compact DAG that costs whatever the source cost. The amplification happens + * afterwards, in whatever expands that DAG back into a tree — `JSON.stringify` + * in the file parser, the fence renderers in the page compiler. A sub-kilobyte + * source can carry millions of expanded nodes, so the expansion has to be + * measured and rejected before anything materializes it. + * + * Repeated (aliased) references are intentionally charged on every reach, which + * is what makes the amplification visible here rather than at materialization + * time. Charging on reach also terminates on self-referential anchors. + */ + +/** Ceilings for one traversal. Callers pick values matched to what they render. */ +export interface YamlExpansionLimits { + /** Expanded nodes — every value reached, aliases counted once per path. */ + maxNodes: number + /** Estimated pretty-printed JSON size of the expanded tree. */ + maxSerializedBytes: number + /** Nesting depth, which also bounds the traversal's own working set. */ + maxDepth: number +} + +/** + * Allowance remaining across every traversal that shares one unit of work — a + * page compile parses its frontmatter and each `sim:` fence separately, and it + * is their SUM that a request pays for, so they draw down one budget rather than + * each getting the full limits. + */ +export interface YamlExpansionBudget { + nodes: number + bytes: number +} + +export function createYamlExpansionBudget(limits: YamlExpansionLimits): YamlExpansionBudget { + return { nodes: limits.maxNodes, bytes: limits.maxSerializedBytes } +} + +/** True once a budget has nothing left, so callers can skip parsing entirely. */ +export function isYamlExpansionBudgetExhausted(budget: YamlExpansionBudget): boolean { + return budget.nodes <= 0 || budget.bytes <= 0 +} + +export type YamlExpansionResult = + | { within: true; depth: number } + | { within: false; reason: string } + +/** + * Exact serialized length (in UTF-16 code units — the unit V8 allocates for the + * resulting string) that `JSON.stringify` produces for a string, accounting for + * the escape expansion of quotes, backslashes, control characters, and lone + * surrogates. Computed precisely rather than with a flat multiplier so plain + * text is charged its true size (no false rejection of large legitimate + * documents) while escape-heavy strings are charged their real, larger cost + * (no cap bypass). + */ +function serializedStringLength(value: string): number { + let length = 2 // surrounding quotes + for (let i = 0; i < value.length; i++) { + const code = value.charCodeAt(i) + if (code === 0x22 /* " */ || code === 0x5c /* \ */) { + length += 2 + } else if (code < 0x20) { + // \b \t \n \f \r use two-char escapes; other control chars use \uXXXX (six) + length += + code === 0x08 || code === 0x09 || code === 0x0a || code === 0x0c || code === 0x0d ? 2 : 6 + } else if (code >= 0xd800 && code <= 0xdfff) { + // Well-formed JSON.stringify emits a valid high+low surrogate pair as-is + // (two code units) but escapes a lone surrogate to \uXXXX (six). + const next = i + 1 < value.length ? value.charCodeAt(i + 1) : 0 + if (code <= 0xdbff && next >= 0xdc00 && next <= 0xdfff) { + length += 2 + i++ + } else { + length += 6 + } + } else { + length += 1 + } + } + return length +} + +/** + * Flat allowance for a value whose serialized form is bounded by its own kind: + * `true`, `false`, `null`, and the punctuation a container contributes on its own + * line all fit well inside it. + */ +const NON_STRING_NODE_BYTES = 16 + +/** `"2026-08-31T00:00:00.000Z"` — 24 characters of ISO 8601 plus its two quotes. */ +const SERIALIZED_DATE_BYTES = 26 + +/** + * Estimate the pretty-printed (`JSON.stringify(value, null, 2)`) size a single + * value node contributes, including the indentation/newline overhead that + * dominates deeply nested alias bombs and the exact escape expansion of strings. + */ +function estimateNodeBytes(value: unknown, depth: number): number { + const indentOverhead = depth * 2 + 4 + if (typeof value === 'string') return indentOverhead + serializedStringLength(value) + // Two non-string values outgrow the flat allowance, and charging them the allowance + // would let a document of them exceed the byte cap by half again: a double serializes + // to as many as 24 characters (`-1.2345678901234567e-308`), and a `Date` — which the + // default js-yaml schema produces for a `!!timestamp`, so the file parser sees them — + // serializes to a 26-character quoted ISO string. Taking the larger of the two never + // charges less than the flat allowance did. + if (typeof value === 'number') { + return indentOverhead + Math.max(NON_STRING_NODE_BYTES, String(value).length) + } + if (value instanceof Date) { + return indentOverhead + Math.max(NON_STRING_NODE_BYTES, SERIALIZED_DATE_BYTES) + } + return indentOverhead + NON_STRING_NODE_BYTES +} + +/** + * Estimate the serialized size of an object key (`"key": `). Keys are re-emitted + * on every alias expansion of their parent object, so an aliased object with a + * long key amplifies just like an aliased value — this must be charged or the + * size cap is trivially bypassed. + */ +function estimateKeyBytes(key: string): number { + return serializedStringLength(key) + 2 // ": " +} + +function isContainer(value: unknown): value is object { + return value !== null && typeof value === 'object' +} + +/** One child of a container, with the serialized cost of naming it. */ +interface YamlChild { + keyBytes: number + value: unknown +} + +/** + * Yields a container's children one at a time. + * + * Lazily, and via `for...in` rather than `Object.entries`, because this runs on + * untrusted input: eagerly building the child list would let a single wide node + * allocate an array proportional to its fan-out *before* the first byte of it is + * charged, which is the allocation the guard exists to prevent. + */ +function* childrenOf(container: object): Generator { + if (Array.isArray(container)) { + for (const value of container) yield { keyBytes: 0, value } + return + } + for (const key in container) { + if (Object.hasOwn(container, key)) { + yield { keyBytes: estimateKeyBytes(key), value: (container as Record)[key] } + } + } +} + +/** + * Iteratively walk the parsed value, charging every reached node against + * `budget`, and return the document depth. + * + * Each node is charged as it is reached, before any of its own children are, so a + * pathologically wide fan-out (an array of millions of aliases) trips a limit part + * way through that node rather than after enumerating it. The traversal holds one + * frame per level of nesting rather than one per pending node, so its own working + * set is bounded by `maxDepth` and not by the document's width — a guard that + * allocated in proportion to the fan-out it is meant to reject would be its own + * exhaustion path. + * + * A size or node rejection leaves the budget spent, because reaching it means the + * allowance ran out mid-walk — a shared budget therefore short-circuits every + * later document instead of paying for a full walk each time. A depth rejection + * costs only its own nesting, so it does not draw the budget down further and + * later documents sharing it still get measured. + */ +export function measureYamlExpansion( + root: unknown, + limits: YamlExpansionLimits, + budget: YamlExpansionBudget = createYamlExpansionBudget(limits) +): YamlExpansionResult { + let maxDepth = 0 + + /** Draws the node down the budget and returns a rejection reason, or null when it fits. */ + const charge = (bytes: number): string | null => { + if (--budget.nodes < 0) { + return `YAML document exceeds the maximum of ${limits.maxNodes} expanded nodes (possible alias-expansion bomb)` + } + budget.bytes -= bytes + if (budget.bytes < 0) { + return `YAML document expands beyond the maximum serialized size of ${limits.maxSerializedBytes} bytes (possible alias-expansion bomb)` + } + return null + } + + const tooDeep: YamlExpansionResult = { + within: false, + reason: `YAML document exceeds the maximum nesting depth of ${limits.maxDepth}`, + } + + const rootOverflow = charge(estimateNodeBytes(root, 0)) + if (rootOverflow) return { within: false, reason: rootOverflow } + + /** One frame per level of nesting; `depth` is the depth of the children it yields. */ + const stack: Array<{ children: Generator; depth: number }> = [] + + const descend = (container: object, depth: number): boolean => { + if (depth > maxDepth) maxDepth = depth + if (depth > limits.maxDepth) return false + stack.push({ children: childrenOf(container), depth }) + return true + } + + if (isContainer(root) && !descend(root, 1)) return tooDeep + + while (stack.length > 0) { + const frame = stack[stack.length - 1] + const next = frame.children.next() + if (next.done) { + stack.pop() + continue + } + + const { keyBytes, value } = next.value + const overflow = charge(keyBytes + estimateNodeBytes(value, frame.depth)) + if (overflow) return { within: false, reason: overflow } + if (isContainer(value) && !descend(value, frame.depth + 1)) return tooDeep + } + + return { within: true, depth: maxDepth } +} diff --git a/apps/sim/lib/file-parsers/yaml-parser.ts b/apps/sim/lib/file-parsers/yaml-parser.ts index 339f5a86853..c8ed21517cd 100644 --- a/apps/sim/lib/file-parsers/yaml-parser.ts +++ b/apps/sim/lib/file-parsers/yaml-parser.ts @@ -2,33 +2,19 @@ import { getErrorMessage } from '@sim/utils/errors' import * as yaml from 'js-yaml' import { FileParserError } from '@/lib/file-parsers/errors' import type { FileParseResult } from '@/lib/file-parsers/types' +import { measureYamlExpansion, type YamlExpansionLimits } from '@/lib/file-parsers/yaml-limits' /** - * Hard cap on the number of expanded nodes visited while validating a parsed - * YAML document. `yaml.load` resolves aliases into shared references, so the - * in-memory value is a compact DAG, but `JSON.stringify` expands that DAG into - * a full tree — duplicating every shared node. A tiny "billion laughs" alias - * bomb therefore expands to millions/billions of nodes at serialize time. This - * cap (and the byte cap below) bound the traversal so the amplification is - * detected and rejected before it ever reaches `JSON.stringify`. It also stops - * traversal of self-referential (cyclic) YAML anchors. + * What a parsed YAML file may expand to once `JSON.stringify` walks its alias + * DAG as a tree. The node cap also stops traversal of self-referential anchors; + * the byte cap bounds output a sub-1 KB input can inflate to hundreds of MB; + * the depth cap bounds the traversal's own working set. */ -const MAX_YAML_EXPANDED_NODES = 5_000_000 - -/** - * Cap on the estimated serialized (pretty-printed JSON) size of the document. - * Alias expansion inflates output far beyond the input size — a sub-1 KB input - * can serialize to hundreds of MB — so we estimate output bytes during the - * bounded traversal and abort past this limit rather than allocating them. - */ -const MAX_YAML_SERIALIZED_BYTES = 64 * 1024 * 1024 - -/** - * Cap on nesting depth. Guards the depth computation (previously an unbounded - * recursion that also spread large arrays into `Math.max(...array)`, risking a - * stack overflow) and rejects pathologically deep documents. - */ -const MAX_YAML_DEPTH = 500 +const FILE_PARSER_YAML_LIMITS: YamlExpansionLimits = { + maxNodes: 5_000_000, + maxSerializedBytes: 64 * 1024 * 1024, + maxDepth: 500, +} /** * Raised when a parsed YAML document exceeds the complexity limits above. @@ -51,128 +37,15 @@ export function isYamlComplexityError(error: unknown): error is YamlComplexityEr } /** - * Exact serialized length (in UTF-16 code units — the unit V8 allocates for the - * resulting string) that `JSON.stringify` produces for a string, accounting for - * the escape expansion of quotes, backslashes, control characters, and lone - * surrogates. Computed precisely rather than with a flat multiplier so plain - * text is charged its true size (no false rejection of large legitimate - * documents) while escape-heavy strings are charged their real, larger cost - * (no cap bypass). - */ -function serializedStringLength(value: string): number { - let length = 2 // surrounding quotes - for (let i = 0; i < value.length; i++) { - const code = value.charCodeAt(i) - if (code === 0x22 /* " */ || code === 0x5c /* \ */) { - length += 2 - } else if (code < 0x20) { - // \b \t \n \f \r use two-char escapes; other control chars use \uXXXX (six) - length += - code === 0x08 || code === 0x09 || code === 0x0a || code === 0x0c || code === 0x0d ? 2 : 6 - } else if (code >= 0xd800 && code <= 0xdfff) { - // Well-formed JSON.stringify emits a valid high+low surrogate pair as-is - // (two code units) but escapes a lone surrogate to \uXXXX (six). - const next = i + 1 < value.length ? value.charCodeAt(i + 1) : 0 - if (code <= 0xdbff && next >= 0xdc00 && next <= 0xdfff) { - length += 2 - i++ - } else { - length += 6 - } - } else { - length += 1 - } - } - return length -} - -/** - * Estimate the pretty-printed (`JSON.stringify(value, null, 2)`) size a single - * value node contributes, including the indentation/newline overhead that - * dominates deeply nested alias bombs and the exact escape expansion of strings. - */ -function estimateNodeBytes(value: unknown, depth: number): number { - const indentOverhead = depth * 2 + 4 - if (typeof value === 'string') return indentOverhead + serializedStringLength(value) - return indentOverhead + 16 -} - -/** - * Estimate the serialized size of an object key (`"key": `). Keys are re-emitted - * on every alias expansion of their parent object, so an aliased object with a - * long key amplifies just like an aliased value — this must be charged or the - * size cap is trivially bypassed. - */ -function estimateKeyBytes(key: string): number { - return serializedStringLength(key) + 2 // ": " -} - -/** - * Iteratively walk the parsed YAML value with strict node-count, output-size, - * and depth limits, returning the document depth. Repeated (aliased) references - * are intentionally counted each time they are reached, mirroring the way - * `JSON.stringify` expands them — this is what makes the alias-expansion bomb - * detectable before serialization. - * - * Each node is charged against the caps as it is *enqueued*, before its own - * children are pushed, and only container nodes are pushed onto the traversal - * stack. A pathologically wide fan-out (e.g. an array of millions of aliases) - * therefore trips a cap during the enqueue loop instead of first materializing - * millions of stack entries and exhausting memory inside the guard itself. + * Validate that a parsed YAML value stays within the file parser's expansion + * limits, returning the document depth. * * @throws {YamlComplexityError} when any limit is exceeded */ export function assertYamlWithinLimits(root: unknown): number { - let visited = 0 - let estimatedBytes = 0 - let maxDepth = 0 - - const charge = (bytes: number): void => { - if (++visited > MAX_YAML_EXPANDED_NODES) { - throw new YamlComplexityError( - `YAML document exceeds the maximum of ${MAX_YAML_EXPANDED_NODES} expanded nodes (possible alias-expansion bomb)` - ) - } - estimatedBytes += bytes - if (estimatedBytes > MAX_YAML_SERIALIZED_BYTES) { - throw new YamlComplexityError( - `YAML document expands beyond the maximum serialized size of ${MAX_YAML_SERIALIZED_BYTES} bytes (possible alias-expansion bomb)` - ) - } - } - - const isContainer = (value: unknown): value is object => - value !== null && typeof value === 'object' - - charge(estimateNodeBytes(root, 0)) - const stack: Array<{ value: object; depth: number }> = [] - if (isContainer(root)) stack.push({ value: root, depth: 0 }) - - while (stack.length > 0) { - const { value, depth } = stack.pop()! - const childDepth = depth + 1 - - if (childDepth > maxDepth) maxDepth = childDepth - if (childDepth > MAX_YAML_DEPTH) { - throw new YamlComplexityError( - `YAML document exceeds the maximum nesting depth of ${MAX_YAML_DEPTH}` - ) - } - - if (Array.isArray(value)) { - for (const child of value) { - charge(estimateNodeBytes(child, childDepth)) - if (isContainer(child)) stack.push({ value: child, depth: childDepth }) - } - } else { - for (const [key, child] of Object.entries(value as Record)) { - charge(estimateKeyBytes(key) + estimateNodeBytes(child, childDepth)) - if (isContainer(child)) stack.push({ value: child, depth: childDepth }) - } - } - } - - return maxDepth + const measured = measureYamlExpansion(root, FILE_PARSER_YAML_LIMITS) + if (!measured.within) throw new YamlComplexityError(measured.reason) + return measured.depth } /** diff --git a/apps/sim/lib/function-execution/application/execute-function.ts b/apps/sim/lib/function-execution/application/execute-function.ts index da18f303abe..66b73058abe 100644 --- a/apps/sim/lib/function-execution/application/execute-function.ts +++ b/apps/sim/lib/function-execution/application/execute-function.ts @@ -1,5 +1,5 @@ import { resolvePrincipalAttribution, resolvePrincipalSubject } from '@sim/auth/principal' -import { type FunctionExecuteBody, functionExecuteBodySchema } from '@/lib/api/contracts' +import { type FunctionExecuteBody, functionExecuteBodySchema } from '@/lib/api/contracts/hotspots' import { defineAuthorizedWorkspaceUseCase } from '@/lib/core/application' import { OrchestrationError } from '@/lib/core/orchestration/types' import { functionExecutionDelegationPolicy } from '@/lib/function-execution/application/authorization' diff --git a/apps/sim/lib/function-execution/application/operations.ts b/apps/sim/lib/function-execution/application/operations.ts index ba37ad117e5..66231142760 100644 --- a/apps/sim/lib/function-execution/application/operations.ts +++ b/apps/sim/lib/function-execution/application/operations.ts @@ -1,10 +1,12 @@ import { defineWorkspaceOperation } from '@/lib/core/application' export const functionExecutionOperations = { + // permission-group-exempt: running a Function block is the workflow executing its own code; no group key names code execution, and a gate here would fail runs the group permits execute: defineWorkspaceOperation({ id: 'function-executions.execute', minimumRole: 'read', workspaceApiKey: 'deny', + capability: 'none', principalKinds: ['delegated'], delegatedServices: ['executor', 'copilot'], }), diff --git a/apps/sim/lib/function-execution/execute-request.test.ts b/apps/sim/lib/function-execution/execute-request.test.ts index 2fd1b35eaa5..4b821c084cb 100644 --- a/apps/sim/lib/function-execution/execute-request.test.ts +++ b/apps/sim/lib/function-execution/execute-request.test.ts @@ -23,6 +23,7 @@ import { PRIVATE_SECRET_PROVENANCE_HEADER, } from '@/lib/execution/private-tool-metadata' import { + attachTrustedSandboxOutputCost, MAX_SANDBOX_OUTPUT_BYTES, SandboxOutputFileError, SandboxOutputLimitError, @@ -84,7 +85,11 @@ vi.mock('@/lib/copilot/request/tools/files', () => ({ md: 'text/markdown', html: 'text/html', }, - normalizeOutputWorkspaceFileName: vi.fn((p: string) => p.replace(/^files\//, '')), + normalizeOutputWorkspaceFileName: vi.fn((p: string) => { + const normalized = p.trim().replace(/^\/+|\/+$/g, '') + if (!normalized) throw new Error('Output path must include a file name') + return normalized.replace(/^files\//, '') + }), resolveOutputFormat: vi.fn(() => 'json'), getOutputFileDeclarations: vi.fn((params: Record) => { if (Array.isArray(params.outputs?.files)) { @@ -143,7 +148,35 @@ vi.mock('@/lib/uploads', () => ({ vi.mock('@/lib/workflows/utils', () => workflowsUtilsMock) -import { validateProxyUrl } from '@/lib/core/security/input-validation' +/** + * Only the I/O half is stubbed. Path naming, transports, ceilings and + * authorization are covered against the real implementation in + * `sandbox-mounts.test.ts`; what matters here is the wiring — that a marker + * becomes a mount and that the context variable ends up holding the path. + */ +vi.mock('@/lib/function-execution/sandbox-mounts', () => ({ + planUserFileMounts: (files: Array<{ key: string; name: string }>) => + files.map((userFile) => ({ userFile, mountPath: `/tmp/sim/inputs/${userFile.name}` })), + resolveUserFileMounts: async ({ + planned, + }: { + planned: Array<{ userFile: { name: string }; mountPath: string }> + }) => ({ + sandboxFiles: planned.map(({ mountPath }) => ({ + type: 'url' as const, + path: mountPath, + url: 'https://presigned.example/object', + })), + manifest: planned.map(({ userFile, mountPath }) => ({ + name: userFile.name, + path: mountPath, + size: 1, + type: 'application/pdf', + })), + }), +})) + +import { validateExternalUrl } from '@/lib/core/security/input-validation' import { clearLargeValueCacheForTests } from '@/lib/execution/payloads/cache' import { isLargeArrayManifest } from '@/lib/execution/payloads/large-array-manifest-metadata' import { isLargeValueRef } from '@/lib/execution/payloads/large-value-ref' @@ -192,6 +225,25 @@ async function POST(request: NextRequest): Promise { afterAll(resetEnvFlagsMock) +/** + * A `` reference as it reaches the function runtime: the resolver + * leaves a mount marker in the context variables, which is what asks this run for a + * sandbox filesystem. + */ +const MOUNT_REF = { + __simSandboxFileMount: true, + version: 1, + file: { + id: 'file_1', + name: 'doc.pdf', + url: 'https://storage.example/doc.pdf', + size: 12, + type: 'application/pdf', + key: 'execution/workspace-1/wf-1/exec-1/abc/doc.pdf', + context: 'execution', + }, +} + describe('Function execution request', () => { beforeEach(() => { vi.clearAllMocks() @@ -281,6 +333,7 @@ describe('Function execution request', () => { result: 'done', stdout: 'ok', sandboxId: 'sandbox-123', + cost: { input: 0, output: 0, total: 0.00012345 }, exportedFiles: { '/tmp/out.txt': 'owned by attacker' }, }) mockWriteWorkspaceFileByPath.mockRejectedValueOnce( @@ -291,6 +344,8 @@ describe('Function execution request', () => { code: 'print("done")', language: 'python', workspaceId: 'workspace-victim', + workflowId: 'workflow-1', + executionId: 'execution-1', outputs: { files: [{ path: 'files/README.md', mode: 'overwrite', sandboxPath: '/tmp/out.txt' }], }, @@ -301,6 +356,7 @@ describe('Function execution request', () => { expect(response.status).toBe(403) expect(data).toHaveProperty('error', 'Insufficient workspace permissions') + expect(data.output.cost).toEqual({ input: 0, output: 0, total: 0.00012345 }) expect(mockWriteWorkspaceFileByPath).toHaveBeenCalledTimes(1) }) @@ -320,6 +376,113 @@ describe('Function execution request', () => { expect(mockExecuteShellInSandbox).not.toHaveBeenCalled() }) + it.each([ + { language: 'python', code: 'return 42', kind: 'code' }, + { language: 'shell', code: 'echo ready', kind: 'shell' }, + ])( + 'meters a standard workflow Function $kind sandbox and preserves its cost', + async ({ language, code, kind }) => { + envFlagsMock.isRemoteSandboxEnabled = true + const cost = { input: 0, output: 0, total: 0.00012345 } + const executeSandbox = kind === 'shell' ? mockExecuteShellInSandbox : mockExecuteInSandbox + executeSandbox.mockResolvedValueOnce({ + result: 42, + stdout: 'ready', + sandboxId: `sandbox-${kind}`, + cost, + }) + + const response = await POST( + createMockRequest('POST', { + code, + language, + workflowId: 'workflow-1', + workspaceId: 'workspace-1', + executionId: 'execution-1', + }) + ) + const data = await response.json() + + expect(response.status).toBe(200) + expect(executeSandbox).toHaveBeenCalledWith(expect.objectContaining({ meterUsage: true })) + expect(data.output.cost).toEqual(cost) + } + ) + + it.each([ + { + language: 'javascript', + code: 'import "node:path"\nthrow new Error("boom")', + kind: 'code', + }, + { language: 'python', code: 'raise ValueError("boom")', kind: 'code' }, + { language: 'shell', code: 'exit 1', kind: 'shell' }, + ])( + 'preserves sandbox cost in a failed remote $language Function response', + async ({ language, code, kind }) => { + envFlagsMock.isRemoteSandboxEnabled = true + const cost = { input: 0, output: 0, total: 0.00012345 } + const executeSandbox = kind === 'shell' ? mockExecuteShellInSandbox : mockExecuteInSandbox + executeSandbox.mockResolvedValueOnce({ + result: null, + stdout: 'boom', + error: 'boom', + sandboxId: `sandbox-${language}`, + cost, + }) + + const response = await POST( + createMockRequest('POST', { + code, + language, + workflowId: 'workflow-1', + workspaceId: 'workspace-1', + executionId: 'execution-1', + }) + ) + const data = await response.json() + + expect(response.status).toBe(422) + expect(executeSandbox).toHaveBeenCalledWith(expect.objectContaining({ meterUsage: true })) + expect(data.output.cost).toEqual(cost) + } + ) + + it('does not meter a non-workflow remote Function call', async () => { + envFlagsMock.isRemoteSandboxEnabled = true + + const response = await POST( + createMockRequest('POST', { + code: 'import path from "node:path"\nreturn path.sep', + language: 'javascript', + }) + ) + + expect(response.status).toBe(200) + expect(mockExecuteInSandbox).toHaveBeenCalledWith( + expect.objectContaining({ meterUsage: false }) + ) + }) + + it('keeps a custom Function tool local even when workflow context is present', async () => { + envFlagsMock.isRemoteSandboxEnabled = true + + const response = await POST( + createMockRequest('POST', { + code: 'return 42', + language: 'python', + workflowId: 'workflow-1', + workspaceId: 'workspace-1', + executionId: 'execution-1', + isCustomTool: true, + }) + ) + + expect(response.status).toBe(200) + expect(mockExecuteInIsolatedVM).toHaveBeenCalledOnce() + expect(mockExecuteInSandbox).not.toHaveBeenCalled() + }) + it('does not accept a Mothership sandbox profile from the request body', async () => { const req = createMockRequest('POST', { code: 'return "test"', @@ -375,6 +538,7 @@ describe('Function execution request', () => { expect.objectContaining({ language, sandboxKind: 'mothership', + meterUsage: false, }) ) expect(mockExecuteInIsolatedVM).not.toHaveBeenCalled() @@ -396,7 +560,7 @@ describe('Function execution request', () => { expect(response.status).toBe(200) expect(mockExecuteShellInSandbox).toHaveBeenCalledWith( - expect.objectContaining({ sandboxKind: 'mothership' }) + expect.objectContaining({ sandboxKind: 'mothership', meterUsage: false }) ) }) @@ -576,23 +740,25 @@ describe('Function execution request', () => { expect(data.output.result).toBe('undefined') }) + const proxyTarget = (url: string) => validateExternalUrl(url, 'url', 'proxy') + it.concurrent('should block SSRF attacks through secure fetch wrapper', async () => { - expect(validateProxyUrl('http://169.254.169.254/latest/meta-data/').isValid).toBe(false) - expect(validateProxyUrl('http://127.0.0.1:8080/admin').isValid).toBe(true) - expect(validateProxyUrl('http://192.168.1.1/config').isValid).toBe(false) - expect(validateProxyUrl('http://10.0.0.1/internal').isValid).toBe(false) + expect(proxyTarget('http://169.254.169.254/latest/meta-data/').isValid).toBe(false) + expect(proxyTarget('http://127.0.0.1:8080/admin').isValid).toBe(false) + expect(proxyTarget('http://192.168.1.1/config').isValid).toBe(false) + expect(proxyTarget('http://10.0.0.1/internal').isValid).toBe(false) }) it.concurrent('should allow legitimate external URLs', async () => { - expect(validateProxyUrl('https://api.github.com/user').isValid).toBe(true) - expect(validateProxyUrl('https://httpbin.org/get').isValid).toBe(true) - expect(validateProxyUrl('https://example.com/api').isValid).toBe(true) + expect(proxyTarget('https://api.github.com/user').isValid).toBe(true) + expect(proxyTarget('https://httpbin.org/get').isValid).toBe(true) + expect(proxyTarget('https://example.com/api').isValid).toBe(true) }) it.concurrent('should block dangerous protocols', async () => { - expect(validateProxyUrl('file:///etc/passwd').isValid).toBe(false) - expect(validateProxyUrl('ftp://internal.server/files').isValid).toBe(false) - expect(validateProxyUrl('gopher://old.server/menu').isValid).toBe(false) + expect(proxyTarget('file:///etc/passwd').isValid).toBe(false) + expect(proxyTarget('ftp://internal.server/files').isValid).toBe(false) + expect(proxyTarget('gopher://old.server/menu').isValid).toBe(false) }) }) @@ -697,6 +863,7 @@ describe('Function execution request', () => { result: 'done', stdout: 'ok', sandboxId: 'sandbox-123', + cost: { input: 0, output: 0, total: 0.00023456 }, exportedFiles: { '/home/user/chart.png': 'iVBORw0KGgo=', '/home/user/summary.json': '{"ok":true}', @@ -707,6 +874,8 @@ describe('Function execution request', () => { code: 'print("done")', language: 'python', workspaceId: 'workspace-1', + workflowId: 'workflow-1', + executionId: 'execution-1', outputs: { files: [ { @@ -753,6 +922,7 @@ describe('Function execution request', () => { }) ) expect(data.output.result.files).toHaveLength(2) + expect(data.output.cost).toEqual({ input: 0, output: 0, total: 0.00023456 }) expect(data.resources).toEqual([ expect.objectContaining({ path: 'files/reports/chart.png' }), expect.objectContaining({ path: 'files/reports/summary.json' }), @@ -1331,9 +1501,10 @@ describe('Function execution request', () => { it('preserves output-limit classification from provider-side size inspection', async () => { envFlagsMock.isRemoteSandboxEnabled = true - mockExecuteInSandbox.mockRejectedValueOnce( - new SandboxOutputLimitError(MAX_SANDBOX_OUTPUT_BYTES + 1) - ) + const error = new SandboxOutputLimitError(MAX_SANDBOX_OUTPUT_BYTES + 1) + const cost = { input: 0, output: 0, total: 0.00023456 } + attachTrustedSandboxOutputCost(error, cost) + mockExecuteInSandbox.mockRejectedValueOnce(error) const req = createMockRequest('POST', { code: 'print("done")', @@ -1354,6 +1525,7 @@ describe('Function execution request', () => { expect(response.status).toBe(400) expect(data.error).toBe(`Sandbox output files exceed ${MAX_SANDBOX_OUTPUT_BYTES} bytes total`) + expect(data.output.cost).toEqual(cost) expect(mockWriteWorkspaceFileByPath).not.toHaveBeenCalled() }) @@ -1375,15 +1547,41 @@ describe('Function execution request', () => { expect(response.status).toBe(400) expect(data.error).toContain('must reference a regular file') + expect(data.output.cost).toBeUndefined() expect(mockWriteWorkspaceFileByPath).not.toHaveBeenCalled() }) + it.each(['/', '///', ' / '])( + 'rejects malformed workspace output destination %j before sandbox execution', + async (path) => { + envFlagsMock.isRemoteSandboxEnabled = true + + const response = await POST( + createMockRequest('POST', { + code: 'print("done")', + language: 'python', + workspaceId: 'workspace-1', + outputs: { + files: [{ path, sandboxPath: '/out/report.json' }], + }, + }) + ) + const data = await response.json() + + expect(response.status).toBe(400) + expect(data.error).toBe('Output path must include a file name') + expect(mockExecuteInSandbox).not.toHaveBeenCalled() + expect(mockExecuteShellInSandbox).not.toHaveBeenCalled() + } + ) + it('prevalidates all sandbox output destinations before writing any files', async () => { envFlagsMock.isRemoteSandboxEnabled = true mockExecuteInSandbox.mockResolvedValueOnce({ result: 'done', stdout: 'ok', sandboxId: 'sandbox-123', + cost: { input: 0, output: 0, total: 0.00023456 }, exportedFiles: { '/home/user/first.json': '{"first":true}', '/home/user/second.json': '{"second":true}', @@ -1397,6 +1595,8 @@ describe('Function execution request', () => { code: 'print("done")', language: 'python', workspaceId: 'workspace-1', + workflowId: 'workflow-1', + executionId: 'execution-1', outputs: { files: [ { @@ -1419,6 +1619,7 @@ describe('Function execution request', () => { expect(response.status).toBe(400) expect(data.success).toBe(false) expect(data.error).toContain('Directory not yet created') + expect(data.output.cost).toEqual({ input: 0, output: 0, total: 0.00023456 }) expect(mockWriteWorkspaceFileByPath).not.toHaveBeenCalled() }) @@ -1500,7 +1701,7 @@ describe('Function execution request', () => { expect(mockWriteWorkspaceFileByPath).not.toHaveBeenCalled() }) - it('rejects sandboxPath outputs when the call would run in isolated-vm (E2B enabled, JS without imports)', async () => { + it('routes plain JavaScript to the remote sandbox when it declares a sandboxPath output', async () => { envFlagsMock.isRemoteSandboxEnabled = true const req = createMockRequest('POST', { @@ -1518,6 +1719,25 @@ describe('Function execution request', () => { }, }) + await POST(req) + + // Needing a sandbox filesystem selects the remote runtime the same way a + // selected sandbox image does. Refusing here instead would dead-end the + // caller: "add an import" is not a fix anyone should have to discover. + expect(mockExecuteInSandbox).toHaveBeenCalled() + expect(mockExecuteInIsolatedVM).not.toHaveBeenCalled() + }) + + it('refuses sandbox file inputs/outputs when no remote sandbox is configured', async () => { + envFlagsMock.isRemoteSandboxEnabled = false + + const req = createMockRequest('POST', { + code: 'return "content"', + language: 'javascript', + workspaceId: 'workspace-1', + contextVariables: { doc: MOUNT_REF }, + }) + const response = await POST(req) const data = await response.json() @@ -1529,6 +1749,201 @@ describe('Function execution request', () => { expect(mockWriteWorkspaceFileByPath).not.toHaveBeenCalled() }) + it('refuses sandbox file inputs/outputs for a custom tool, which always runs in isolated-vm', async () => { + envFlagsMock.isRemoteSandboxEnabled = true + + const req = createMockRequest('POST', { + code: 'return "content"', + language: 'javascript', + workspaceId: 'workspace-1', + isCustomTool: true, + contextVariables: { doc: MOUNT_REF }, + }) + + const response = await POST(req) + const data = await response.json() + + expect(response.status).toBe(422) + expect(data.success).toBe(false) + expect(data.error).toContain('custom tools always run in the isolated JavaScript VM') + expect(mockExecuteInSandbox).not.toHaveBeenCalled() + }) + + it('reports a harvest the sandbox refused as a 400 carrying its reason', async () => { + envFlagsMock.isRemoteSandboxEnabled = true + mockExecuteInSandbox.mockRejectedValueOnce( + Object.assign(new Error('Sandbox produced 21 files in /tmp/sim/outputs'), { + code: 'sandbox_output_not_exportable', + }) + ) + + const req = createMockRequest('POST', { + code: 'x', + language: 'python', + workspaceId: 'workspace-1', + }) + + const response = await POST(req) + const data = await response.json() + + // Writing too many files is the caller's to fix, so it must not surface + // as an opaque 500 that hides the count and the remedy. + expect(response.status).toBe(400) + expect(data.error).toContain('21 files') + }) + + it('scans a harvested plaintext secret even under a binary file name', async () => { + envFlagsMock.isRemoteSandboxEnabled = true + mockExecuteInSandbox.mockResolvedValueOnce({ + result: null, + stdout: '', + sandboxId: 'sbx', + collectedFiles: [ + { + path: '/tmp/sim/outputs/leak.png', + relativePath: 'leak.png', + // Valid UTF-8 carrying the resolved secret, named as an image. + contentBase64: Buffer.from('token=super-secret-value').toString('base64'), + byteLength: 24, + }, + ], + }) + + const req = createMockRequest('POST', { + // The placeholder has to be in the code: compiling it is what puts the + // resolved value in scope for the output scan. + code: 'token = {{MY_SECRET}}', + language: 'python', + workspaceId: 'workspace-1', + workflowId: 'workflow-1', + executionId: 'execution-1', + envVars: { MY_SECRET: 'super-secret-value' }, + }) + + const response = await POST(req) + const data = await response.json() + + // Classifying by file name let a secret written as plaintext under a + // binary extension skip the only provenance guard and be returned with a + // downloadable URL. Content decides now, so the name cannot dodge it. + expect(response.status).toBe(400) + expect(data.error).toContain('leak.png') + expect(data.error).toContain('resolved secret') + }) + + it('scans a harvested secret even when one invalid byte makes it non-UTF-8', async () => { + envFlagsMock.isRemoteSandboxEnabled = true + mockExecuteInSandbox.mockResolvedValueOnce({ + result: null, + stdout: '', + sandboxId: 'sbx', + collectedFiles: [ + { + path: '/tmp/sim/outputs/mixed.bin', + relativePath: 'mixed.bin', + // Literal secret plus one invalid byte, so the buffer is not valid + // UTF-8 — which used to be enough to skip the scan entirely. + contentBase64: Buffer.concat([ + Buffer.from('token=super-secret-value'), + Buffer.from([0xff]), + ]).toString('base64'), + byteLength: 25, + }, + ], + }) + + const req = createMockRequest('POST', { + code: 'token = {{MY_SECRET}}', + language: 'python', + workspaceId: 'workspace-1', + workflowId: 'workflow-1', + executionId: 'execution-1', + envVars: { MY_SECRET: 'super-secret-value' }, + }) + + const response = await POST(req) + const data = await response.json() + + // A lossy UTF-8 decode keeps ASCII runs intact, so the literal is still + // there to find — appending a byte must not buy an exemption. + expect(response.status).toBe(400) + expect(data.error).toContain('mixed.bin') + expect(data.error).toContain('resolved secret') + }) + + it('mounts a reference and hands the code its path', async () => { + envFlagsMock.isRemoteSandboxEnabled = true + + const req = createMockRequest('POST', { + code: 'x', + language: 'python', + workspaceId: 'workspace-1', + workflowId: 'workflow-1', + executionId: 'execution-1', + contextVariables: { doc: MOUNT_REF }, + }) + + await POST(req) + + const call = mockExecuteInSandbox.mock.calls[0]?.[0] + expect(call.sandboxFiles).toEqual([ + { type: 'url', path: '/tmp/sim/inputs/doc.pdf', url: 'https://presigned.example/object' }, + ]) + // The marker must not survive into the code's view of the variable — the + // whole point is that every language sees a plain path string. + const runtimePayload = call.privateInputs + .map((input: { content: string }) => input.content) + .find((content: string) => content.includes('contextVariables')) + expect(runtimePayload).toContain('/tmp/sim/inputs/doc.pdf') + expect(runtimePayload).not.toContain('__simSandboxFileMount') + }) + + it('harvests the output directory on every remote run, with no toggle', async () => { + envFlagsMock.isRemoteSandboxEnabled = true + + const req = createMockRequest('POST', { + code: 'x', + language: 'python', + workspaceId: 'workspace-1', + }) + + await POST(req) + + expect(mockExecuteInSandbox.mock.calls[0]?.[0].outputSandboxDir).toBe('/tmp/sim/outputs') + }) + + it('does not ask for an output directory on an isolate run', async () => { + envFlagsMock.isRemoteSandboxEnabled = true + + const req = createMockRequest('POST', { + code: 'return 1', + language: 'javascript', + workspaceId: 'workspace-1', + }) + + await POST(req) + + // Harvesting is free only because it rides an existing sandbox; an + // isolate run must not gain one just to look for files. + expect(mockExecuteInSandbox).not.toHaveBeenCalled() + expect(mockExecuteInIsolatedVM).toHaveBeenCalled() + }) + + it('leaves a plain JavaScript call with no file inputs or outputs in isolated-vm', async () => { + envFlagsMock.isRemoteSandboxEnabled = true + + const req = createMockRequest('POST', { + code: 'return "content"', + language: 'javascript', + workspaceId: 'workspace-1', + }) + + await POST(req) + + expect(mockExecuteInIsolatedVM).toHaveBeenCalled() + expect(mockExecuteInSandbox).not.toHaveBeenCalled() + }) + it('rejects sandbox file mounts when the call would run in isolated-vm', async () => { const req = createMockRequest('POST', { code: 'return 1', @@ -1725,6 +2140,7 @@ describe('Function execution request', () => { result: null, stdout: 'generated 1 preview', sandboxId: 'sandbox-123', + cost: { input: 0, output: 0, total: 0.00034567 }, exportedFiles: { '/tmp/fellows-previews.zip': archiveBase64 }, }) @@ -1733,6 +2149,8 @@ describe('Function execution request', () => { code: source, language: 'python', workspaceId: 'workspace-1', + workflowId: 'workflow-1', + executionId: 'execution-1', sandboxId: 'fellows-sandbox', envVars: { AIRTABLE_PAT: 'stub-airtable-token', @@ -1754,6 +2172,9 @@ describe('Function execution request', () => { ) expect(response.status).toBe(200) + await expect(response.clone().json()).resolves.toMatchObject({ + output: { cost: { input: 0, output: 0, total: 0.00034567 } }, + }) const sandboxRequest = mockExecuteInSandbox.mock.calls[0][0] expect(sandboxRequest.code).toContain("['bq', 'query'") expect(sandboxRequest.code).toContain('__sim_exec_globals__["__name__"] = "__main__"') diff --git a/apps/sim/lib/function-execution/execute-request.ts b/apps/sim/lib/function-execution/execute-request.ts index 443a3fdc0e1..1392221fbd0 100644 --- a/apps/sim/lib/function-execution/execute-request.ts +++ b/apps/sim/lib/function-execution/execute-request.ts @@ -58,6 +58,10 @@ import { readUserFileContent, unavailableLargeValueError, } from '@/lib/execution/payloads/materialization.server' +import { + collectSandboxFileMountRefs, + replaceSandboxFileMountRefs, +} from '@/lib/execution/payloads/sandbox-file-mount-ref' import { compactExecutionPayload } from '@/lib/execution/payloads/serializer' import { materializeLargeValueRef } from '@/lib/execution/payloads/store' import { @@ -76,20 +80,32 @@ import { import { isSandboxOutputFileError, isSandboxOutputLimitError, + isSandboxOutputNotExportableError, MAX_SANDBOX_OUTPUT_BYTES, + readTrustedSandboxOutputCost, } from '@/lib/execution/remote-sandbox/output-limits' +import { + MAX_BLOCK_MOUNTED_FILES, + SANDBOX_OUTPUT_DIR, +} from '@/lib/execution/remote-sandbox/sandbox-paths' +import type { SandboxCollectedFile, SandboxFile } from '@/lib/execution/remote-sandbox/types' import { isExecutionResourceLimitError } from '@/lib/execution/resource-errors' +import { planUserFileMounts, resolveUserFileMounts } from '@/lib/function-execution/sandbox-mounts' +import { uploadExecutionFile } from '@/lib/uploads/contexts/execution/execution-file-manager' import { EXACT_EMPTY_WORKSPACE_FILE_SECRET_PROVENANCE, mergeWorkspaceFileSecretProvenance, type WorkspaceFileSecretProvenance, } from '@/lib/uploads/contexts/workspace/workspace-file-secret-provenance' +import { deleteFiles } from '@/lib/uploads/core/storage-service' +import { getFileExtension, getMimeTypeFromExtension } from '@/lib/uploads/utils/file-utils' import { getWorkflowById } from '@/lib/workflows/utils' import { rebindWorkspaceFileDelegatedPrincipal } from '@/lib/workspace-files/application/delegated-principal' import { fileOperations } from '@/lib/workspace-files/application/operations' import { readWorkspaceFileContent } from '@/lib/workspace-files/application/read-workspace-file-content' import { resolveWorkspaceFileReference } from '@/lib/workspace-files/application/resolve-workspace-file-reference' -import { escapeRegExp, normalizeName, REFERENCE } from '@/executor/constants' +import { escapeRegExp, normalizeName, REFERENCE, sanitizeFileName } from '@/executor/constants' +import type { UserFile } from '@/executor/types' import { type OutputSchema, resolveBlockReference } from '@/executor/utils/block-reference' import { createReferencePattern, @@ -100,6 +116,7 @@ import { type ResolvedSecretMatcher, scanResolvedSecretString, } from '@/executor/utils/resolved-secret-content-projection' +import { isNonIdentifyingSecretLiteral } from '@/executor/utils/resolved-secret-match-policy' import type { ResolvedSecretTraceProvenanceV1 } from '@/executor/utils/resolved-secret-trace-registry' const logger = createLogger('FunctionExecuteAPI') @@ -112,6 +129,12 @@ const MAX_SANDBOX_OUTPUT_FILES = 20 const MAX_PRIVATE_FILE_SECRET_MATCH_EVENTS = 1_000_000 const SANDBOX_RUNTIME_PAYLOAD_PATH_ENV = '__SIM_RUNTIME_PAYLOAD_PATH' +interface FunctionExecutionCost { + input: number + output: number + total: number +} + interface SandboxRuntimePayload { params: Record environmentVariables: Record @@ -1202,11 +1225,25 @@ function activateReferencedSecretProvenance(context: FunctionRouteExecutionConte } } -/** Compiled secret names that still demand redaction — the exempt ones don't count. */ +/** + * Compiled secret names that still demand redaction, and whose value a scan could + * actually find. Exempt names don't count. + * + * Non-identifying literals are excluded on the same predicate + * {@link createResolvedSecretMatcher} uses to drop them, because the two decisions + * have to agree. When every in-scope value is shorter than the substitutable-literal + * minimum, the matcher builds nothing and returns `undefined`; a counter that still + * reported those names would send + * {@link getOutputFileSecretProvenance} down its no-matcher branch and classify + * every output as `unknown` — failing an export while claiming it contains a + * secret that, by that very policy, is too short to be attributed to anything. + */ function countProtectedOutputSecretNames(context: FunctionRouteExecutionContext): number { let count = 0 - for (const name of context.outputSecretPlaintextsByName.keys()) { - if (!context.unredactedSecretNames.has(name)) count += 1 + for (const [name, plaintext] of context.outputSecretPlaintextsByName) { + if (context.unredactedSecretNames.has(name)) continue + if (isNonIdentifyingSecretLiteral(plaintext)) continue + count += 1 } return count } @@ -1402,10 +1439,20 @@ function exportFailure( error: string, status: number, stdout: string, - executionTime: number + executionTime: number, + cost: FunctionExecutionCost | undefined ): NextResponse { return NextResponse.json( - { success: false, error, output: { result: null, stdout: cleanStdout(stdout), executionTime } }, + { + success: false, + error, + output: { + result: null, + stdout: cleanStdout(stdout), + executionTime, + ...(cost ? { cost } : {}), + }, + }, { status } ) } @@ -1428,6 +1475,7 @@ async function maybeExportSandboxFileToWorkspace(args: { exportedFileContent?: string stdout: string executionTime: number + cost?: FunctionExecutionCost }) { const { routeContext, @@ -1443,6 +1491,7 @@ async function maybeExportSandboxFileToWorkspace(args: { exportedFileContent, stdout, executionTime, + cost, } = args if (!outputSandboxPath) return null @@ -1452,7 +1501,8 @@ async function maybeExportSandboxFileToWorkspace(args: { 'outputSandboxPath requires outputPath. Set outputPath to the destination workspace file, e.g. "files/result.csv".', 400, stdout, - executionTime + executionTime, + cost ) } @@ -1464,7 +1514,8 @@ async function maybeExportSandboxFileToWorkspace(args: { 'Workspace context required to save sandbox file to workspace', 400, stdout, - executionTime + executionTime, + cost ) } @@ -1473,7 +1524,8 @@ async function maybeExportSandboxFileToWorkspace(args: { `Sandbox file "${outputSandboxPath}" was not found or could not be read`, 500, stdout, - executionTime + executionTime, + cost ) } @@ -1491,7 +1543,8 @@ async function maybeExportSandboxFileToWorkspace(args: { `Sandbox output files exceed ${MAX_SANDBOX_OUTPUT_BYTES} bytes total`, 400, stdout, - executionTime + executionTime, + cost ) } const fileBuffer = isBinary @@ -1565,6 +1618,7 @@ async function maybeExportSandboxFileToWorkspace(args: { }, stdout: cleanStdout(stdout), executionTime, + ...(cost ? { cost } : {}), }, resources: [{ type: 'file', id: written.id, title: written.name, path: written.vfsPath }], }) @@ -1573,7 +1627,8 @@ async function maybeExportSandboxFileToWorkspace(args: { getErrorMessage(error, 'Failed to export sandbox file'), workspaceFileExportErrorStatus(error), stdout, - executionTime + executionTime, + cost ) } } @@ -1588,6 +1643,7 @@ async function maybeExportSandboxFilesToWorkspace(args: { exportedFileContent?: string stdout: string executionTime: number + cost?: FunctionExecutionCost }) { const sandboxFiles = args.outputFiles.filter((file) => file.sandboxPath) if (sandboxFiles.length === 0) return null @@ -1596,7 +1652,8 @@ async function maybeExportSandboxFilesToWorkspace(args: { `Too many sandbox output files requested (${sandboxFiles.length}). Maximum is ${MAX_SANDBOX_OUTPUT_FILES}.`, 400, args.stdout, - args.executionTime + args.executionTime, + args.cost ) } @@ -1617,6 +1674,7 @@ async function maybeExportSandboxFilesToWorkspace(args: { args.exportedFileContent, stdout: args.stdout, executionTime: args.executionTime, + cost: args.cost, }) } @@ -1628,7 +1686,8 @@ async function maybeExportSandboxFilesToWorkspace(args: { 'Workspace context required to save sandbox files to workspace', 400, args.stdout, - args.executionTime + args.executionTime, + args.cost ) } @@ -1642,7 +1701,8 @@ async function maybeExportSandboxFilesToWorkspace(args: { `Sandbox file "${sandboxPath}" was not found or could not be read`, 500, args.stdout, - args.executionTime + args.executionTime, + args.cost ) } const outputPath = file.formatPath ?? file.path @@ -1659,7 +1719,8 @@ async function maybeExportSandboxFilesToWorkspace(args: { `Sandbox output files exceed ${MAX_SANDBOX_OUTPUT_BYTES} bytes total`, 400, args.stdout, - args.executionTime + args.executionTime, + args.cost ) } const scanBuffer = isBinary ? Buffer.from(content, 'base64') : Buffer.from(content, 'utf-8') @@ -1708,7 +1769,8 @@ async function maybeExportSandboxFilesToWorkspace(args: { getErrorMessage(error, 'Invalid sandbox output destination'), workspaceFileExportErrorStatus(error), args.stdout, - args.executionTime + args.executionTime, + args.cost ) } const duplicateDestination = validationPaths.find( @@ -1719,7 +1781,8 @@ async function maybeExportSandboxFilesToWorkspace(args: { `Duplicate sandbox output destination: ${duplicateDestination}`, 400, args.stdout, - args.executionTime + args.executionTime, + args.cost ) } @@ -1775,7 +1838,8 @@ async function maybeExportSandboxFilesToWorkspace(args: { getErrorMessage(error, 'Failed to export sandbox files'), workspaceFileExportErrorStatus(error), args.stdout, - args.executionTime + args.executionTime, + args.cost ) } @@ -1814,6 +1878,7 @@ async function maybeExportSandboxFilesToWorkspace(args: { }, stdout: cleanStdout(args.stdout), executionTime: args.executionTime, + ...(args.cost ? { cost: args.cost } : {}), }, resources: writtenFiles.map((file) => ({ type: 'file', @@ -1824,6 +1889,182 @@ async function maybeExportSandboxFilesToWorkspace(args: { }) } +/** + * Combines caller-supplied mounts — Copilot resolves its own workspace paths — + * with those resolved from platform file objects. + * + * A duplicate destination is rejected rather than settled by order: + * `writeSandboxInputs` materializes in sequence, so the later entry would + * silently overwrite the earlier one and the code would find something other + * than what it asked for at that path. + */ +function mergeSandboxFileMounts( + callerFiles: SandboxFile[] | undefined, + resolvedFiles: SandboxFile[] +): SandboxFile[] | undefined { + if (!callerFiles?.length) return resolvedFiles.length > 0 ? resolvedFiles : undefined + if (resolvedFiles.length === 0) return callerFiles + + const merged = [...callerFiles, ...resolvedFiles] + const seen = new Set() + for (const file of merged) { + if (seen.has(file.path)) { + throw new Error(`Duplicate sandbox mount path: ${file.path}`) + } + seen.add(file.path) + } + return merged +} + +/** + * A harvested file's name, derived from its path relative to the output + * directory. Subdirectories are folded into the name rather than dropped, so + * `reports/q4.csv` and `q4.csv` stay distinguishable — and a `/` never survives + * into a name that later reaches an email attachment or an upload filename. + */ +function collectedFileName(relativePath: string): string { + return sanitizeFileName(relativePath.split('/').filter(Boolean).join('-')) || 'file' +} + +/** + * Persists files harvested from the sandbox output directory as platform file + * objects, so any downstream tool that accepts a file can consume them. + * + * Uploaded here, one at a time, rather than handed to the declarative + * file-output pipeline as bytes: that path would carry the whole export budget + * as base64 through `JSON.stringify`, a response buffer, and a re-parse, so + * several multiples of the payload would be live at once for a value that is a + * couple of hundred bytes per file once stored. + */ +/** + * Removes files already uploaded when a later one in the same harvest is refused. + * + * The route answers with a failure and hands back no references, so anything + * uploaded before the refusal is unreachable — but it still occupies storage, + * and the harvest is all-or-nothing by design. Best-effort on purpose: the + * caller needs to hear why its export was refused, not that the tidy-up failed. + */ +async function discardUploadedExecutionFiles(files: readonly UserFile[]): Promise { + if (files.length === 0) return + try { + await deleteFiles( + files.map((file) => file.key), + 'execution' + ) + } catch (error) { + logger.warn('Could not remove partially uploaded sandbox output files', { + fileCount: files.length, + error: getErrorMessage(error), + }) + } +} + +async function collectExecutionOutputFiles(args: { + routeContext: FunctionRouteExecutionContext + authUserId: string + workflowId?: string + workspaceId?: string + executionId?: string + collectedFiles: SandboxCollectedFile[] + stdout: string + executionTime: number + cost?: FunctionExecutionCost +}): Promise<{ files: UserFile[] } | { response: NextResponse }> { + const { routeContext, collectedFiles } = args + if (collectedFiles.length === 0) return { files: [] } + + const resolvedWorkspaceId = + args.workspaceId || + (args.workflowId ? (await getWorkflowById(args.workflowId))?.workspaceId : undefined) + + // Fails rather than returning an empty list: the code did produce files, and + // reporting success without them would read as "your script wrote nothing". + if (!resolvedWorkspaceId || !args.workflowId || !args.executionId) { + return { + response: exportFailure( + 'Workspace, workflow, and execution context are required to return files from the sandbox.', + 400, + args.stdout, + args.executionTime, + args.cost + ), + } + } + + const files: UserFile[] = [] + // The harvest is all-or-nothing, so a throw partway through has to take the + // uploads that already succeeded with it. Without this they linger in storage + // with nothing referencing them, since the failure response carries no keys. + try { + for (const collected of args.collectedFiles) { + const buffer = Buffer.from(collected.contentBase64, 'base64') + const name = collectedFileName(collected.relativePath) + const mimeType = getMimeTypeFromExtension(getFileExtension(name)) + + // Scanned unconditionally — never gated on whether the bytes look textual. + // Both a filename check and a UTF-8 round-trip were trivially defeated: name + // the file `.png`, or append one invalid byte, and a plaintext secret sailed + // past. A lossy UTF-8 decode preserves ASCII runs, so a literal secret is + // findable in any buffer, textual or not. + // + // What stays out of reach is a secret carried in transformed form — deflated + // inside a PDF, re-encoded — which no substring scan can see. That is an + // inherent limit of scanning, not a hole in the gate, and it is why these + // files are execution-scoped rather than durable workspace files. + { + const provenance = await getOutputFileSecretProvenance(buffer, false, routeContext, { + userId: args.authUserId, + workspaceId: resolvedWorkspaceId, + }) + // An execution-scoped file has nowhere to record a provenance envelope, so + // one carrying a resolved secret cannot ship under a lock the way a + // workspace file can — it is refused instead. + if (provenance.status !== 'exact' || provenance.entries.length > 0) { + await discardUploadedExecutionFiles(files) + return { + response: exportFailure( + `Sandbox output file "${name}" contains a resolved secret value and was not returned. Write the file without embedding secret values, or export it to a workspace file where its provenance can be recorded.`, + 400, + args.stdout, + args.executionTime, + args.cost + ), + } + } + } + + const userFile = await uploadExecutionFile( + { + workspaceId: resolvedWorkspaceId, + workflowId: args.workflowId, + executionId: args.executionId, + }, + buffer, + name, + mimeType, + args.authUserId + ) + files.push(userFile) + } + } catch (error) { + await discardUploadedExecutionFiles(files) + throw error + } + + // Registers the new keys on the execution so downstream blocks are authorized + // to read them back. + routeContext.fileKeys = [ + ...new Set([...(routeContext.fileKeys ?? []), ...files.map((file) => file.key)]), + ] + + logger.info('Returned sandbox output files', { + fileCount: files.length, + totalBytes: files.reduce((total, file) => total + file.size, 0), + }) + + return { files } +} + export interface TrustedFunctionExecutionAuth { attributedUserId: string fileAccessUserId?: string @@ -1924,9 +2165,12 @@ export async function executeFunctionRequest( allowLargeValueWorkflowScope = false, workspaceId, isCustomTool = false, + files: mountedUserFiles, _sandboxFiles, } = body + const meterRemoteSandboxUsage = Boolean(workflowId && !isCustomTool && !usesMothershipSandbox) + if (selectedSandboxId && !isRemoteSandboxEnabled) { return NextResponse.json( { success: false, error: 'The Function code sandbox is not configured' }, @@ -1985,6 +2229,27 @@ export async function executeFunctionRequest( privateResolvedSecretNamesMetadataType ) } + try { + for (const file of outputFiles) { + normalizeOutputWorkspaceFileName(file.formatPath ?? file.path) + } + } catch (error) { + return appendPrivateResolvedSecretNames( + NextResponse.json( + { + success: false, + error: getErrorMessage(error, 'Invalid sandbox output destination'), + }, + { status: 400 } + ), + includePrivateResolvedSecretNames ? [] : null, + privateResolvedSecretNamesMetadataType + ) + } + + // Planned before the runtime is chosen because it is pure: it decides whether + // this execution needs a sandbox filesystem at all, without spending a presign + // or a byte of transfer on a request the guard below may still refuse. const executionParams = { ...params } executionParams._context = undefined @@ -2040,6 +2305,34 @@ export async function executeFunctionRequest( ...codeResolution.contextVariables, ...preResolvedContextVariables, } + + /** + * Files this run must place on the sandbox filesystem: those a caller passed + * explicitly — how an agent supplies one, since a model cannot write a block + * reference — plus every file the code asked for with ``, + * which arrives as a marker inside the resolved context variables. + */ + const plannedFileMounts = planUserFileMounts([ + ...((mountedUserFiles ?? []) as UserFile[]), + ...collectSandboxFileMountRefs(contextVariables), + ]) + if (plannedFileMounts.length > MAX_BLOCK_MOUNTED_FILES) { + return functionJsonResponse( + { + success: false, + error: `Too many files mounted into the sandbox (${plannedFileMounts.length}). Maximum is ${MAX_BLOCK_MOUNTED_FILES}.`, + output: { result: null, stdout: '', executionTime: Date.now() - startTime }, + }, + routeContext, + { status: 400 } + ) + } + const requestsSandboxFilesystem = + plannedFileMounts.length > 0 || + Boolean(_sandboxFiles?.length) || + outputSandboxPaths.length > 0 || + Boolean(outputSandboxPath) + const compilation = await compileCodePlaceholders({ code: codeResolution.resolvedCode, language: lang, @@ -2104,13 +2397,143 @@ export async function executeFunctionRequest( hasImports = jsImports.trim().length > 0 || extractionResult.hasRequireCalls } - if (lang === CodeLanguage.Shell) { - if (!remoteSandboxEnabled) { - throw new Error( - 'Shell execution requires a remote code sandbox to be enabled. Please contact your administrator to enable it.' - ) - } + if (lang === CodeLanguage.Shell && !remoteSandboxEnabled) { + throw new Error( + 'Shell execution requires a remote code sandbox to be enabled. Please contact your administrator to enable it.' + ) + } + + if (lang === CodeLanguage.Python && !remoteSandboxEnabled) { + throw new Error( + 'Python execution requires a remote code sandbox to be enabled. Please contact your administrator to enable it, or use JavaScript instead.' + ) + } + if (lang === CodeLanguage.JavaScript && hasImports && !remoteSandboxEnabled) { + throw new Error( + 'JavaScript code with import statements requires a remote code sandbox to be enabled. Please remove the import statements, or contact your administrator to enable it.' + ) + } + + /** + * Mounting files or harvesting outputs needs a real filesystem, so it selects + * the remote sandbox the same way a selected sandbox image does. Without this + * a plain-JavaScript block that merely attaches a file would land in + * isolated-vm and be refused by the guard below — a dead end, since "add an + * import" is not a fix a caller should have to discover. + */ + const useRemoteSandbox = + usesMothershipSandbox || + (remoteSandboxEnabled && + !isCustomTool && + (lang === CodeLanguage.Shell || + lang === CodeLanguage.Python || + (lang === CodeLanguage.JavaScript && + (hasImports || Boolean(selectedSandboxId) || requestsSandboxFilesystem)))) + + if (useRemoteSandbox && containsLargeValueRef(contextVariables)) { + throw new Error( + 'Large execution values require the JavaScript isolated-vm runtime. Remove imports, select a nested field, or read the value in a JavaScript function without a remote sandbox.' + ) + } + + // Sandbox file mounts and file exports only exist in the remote sandbox + // runtime; isolated-vm has no filesystem. Silently dropping a declared + // sandbox input/output here produced "export succeeded" responses with zero + // bytes written, so refuse the call instead. Widening `useRemoteSandbox` + // above means the only ways to arrive here are a deployment with no remote + // sandbox at all, or a custom tool — which is why neither remediation + // suggests switching language. + if (!useRemoteSandbox && requestsSandboxFilesystem) { + const remediation = !remoteSandboxEnabled + ? "No remote code sandbox is enabled on this deployment, so there is no sandbox filesystem for any language. Pass input data via params and return output as the code's return value with outputs.files[].path (no sandboxPath)." + : "custom tools always run in the isolated JavaScript VM, which has no sandbox filesystem. Pass input data via params and return output as the code's return value." + return functionJsonResponse( + { + success: false, + error: `Sandbox file inputs/outputs are unavailable for this call: ${remediation}`, + output: { result: null, stdout: '', executionTime: Date.now() - startTime }, + }, + routeContext, + { status: 422 } + ) + } + + // Resolved only after the guard: a request about to be refused must not mint + // presigned URLs or buffer bytes on its way out. + let resolvedMounts: Awaited> + try { + resolvedMounts = await resolveUserFileMounts({ + planned: plannedFileMounts, + context: { + principal: auth.principal, + workflowId, + workspaceId, + executionId, + largeValueExecutionIds, + largeValueKeys, + fileKeys, + allowLargeValueWorkflowScope, + userId: auth.fileAccessUserId, + requestId, + logger, + }, + }) + } catch (error) { + // Everything this can raise is about the files the caller named — a mount + // it may not read, one over a size ceiling, a set over the aggregate. The + // messages already say which file and what to do, so they are the response + // rather than a 500 that reads like the platform broke. Matches the + // too-many-files refusal above. + logger.warn(`[${requestId}] Could not resolve sandbox file mounts`, { + error: getErrorMessage(error), + }) + return functionJsonResponse( + { + success: false, + error: getErrorMessage(error, 'Could not mount the requested files into the sandbox.'), + output: { result: null, stdout: '', executionTime: Date.now() - startTime }, + }, + routeContext, + { status: 400 } + ) + } + const { sandboxFiles: userFileMounts, manifest: mountManifest } = resolvedMounts + const sandboxFiles = mergeSandboxFileMounts(_sandboxFiles, userFileMounts) + + // Every `` marker becomes the path its file was mounted at, + // so the code reads a plain string in whichever language it is written in. + const mountPathsByKey = new Map( + plannedFileMounts.map(({ userFile, mountPath }) => [userFile.key, mountPath]) + ) + for (const [name, value] of Object.entries(contextVariables)) { + contextVariables[name] = replaceSandboxFileMountRefs( + value, + (file) => mountPathsByKey.get(file.key) ?? file.name + ) + } + + // Harvested on every remote run rather than behind a switch: the directory is + // Sim's own, so nothing lands there unless the code put it there, and the cost + // is one listing on a run that already paid for a sandbox. Isolate runs never + // reach here, so they stay as fast as they were. + // + // Declared sandbox outputs opt out. That request names exactly which paths to + // export and answers with that export's own result, so harvesting alongside it + // would collect files the response has no shape to carry — they would be read, + // scanned, uploaded, and then dropped. Making the exclusion explicit here keeps + // it from resting on which branch happens to return first. + const declaresSandboxOutputs = outputFiles.some((file) => file.sandboxPath) + const outputSandboxDir = + useRemoteSandbox && !declaresSandboxOutputs ? SANDBOX_OUTPUT_DIR : undefined + + if (mountManifest.length > 0) { + logger.info(`[${requestId}] Mounted files into sandbox`, { + mountCount: mountManifest.length, + }) + } + + if (lang === CodeLanguage.Shell) { const shellEnvs: Record = {} for (const [k, v] of Object.entries(envVars)) { shellEnvs[k] = serializeForShellEnv(v) @@ -2133,20 +2556,24 @@ export async function executeFunctionRequest( error: shellError, exportedFileContent, exportedFiles, + collectedFiles: shellCollectedFiles, + cost: shellCost, } = await executeShellInSandbox({ code: resolvedCode, envs: shellEnvs, timeoutMs: timeout, - sandboxFiles: _sandboxFiles, + sandboxFiles, privateInputs: compilerPrivateInputs, outputSandboxPath, outputSandboxPaths, + outputSandboxDir, workspaceId, sandboxId: selectedSandboxId, ...(usesMothershipSandbox && !selectedSandboxId ? { sandboxKind: 'mothership' as const } : {}), signal: executionSignal, + meterUsage: meterRemoteSandboxUsage, }) const executionTime = Date.now() - execStart @@ -2161,7 +2588,12 @@ export async function executeFunctionRequest( { success: false, error: scrubInternalIdentifiers(shellError, compilerInternalIdentifiers), - output: { result: null, stdout: cleanStdout(shellStdout), executionTime }, + output: { + result: null, + stdout: cleanStdout(shellStdout), + executionTime, + ...(shellCost ? { cost: shellCost } : {}), + }, }, routeContext, { status: 422 } @@ -2179,72 +2611,43 @@ export async function executeFunctionRequest( exportedFileContent, stdout: shellStdout, executionTime, + cost: shellCost, }) if (fileExportResponse) { return appendResolvedSecretNames(fileExportResponse, routeContext) } } + const shellOutputFiles = await collectExecutionOutputFiles({ + routeContext, + authUserId: auth.attributedUserId, + workflowId, + workspaceId, + executionId, + collectedFiles: shellCollectedFiles ?? [], + stdout: shellStdout, + executionTime, + cost: shellCost, + }) + if ('response' in shellOutputFiles) { + return appendResolvedSecretNames(shellOutputFiles.response, routeContext) + } + return functionJsonResponse( { success: true, - output: { result: shellResult ?? null, stdout: cleanStdout(shellStdout), executionTime }, + output: { + result: shellResult ?? null, + stdout: cleanStdout(shellStdout), + executionTime, + files: shellOutputFiles.files, + ...(shellCost ? { cost: shellCost } : {}), + }, }, routeContext ) } - if (lang === CodeLanguage.Python && !remoteSandboxEnabled) { - throw new Error( - 'Python execution requires a remote code sandbox to be enabled. Please contact your administrator to enable it, or use JavaScript instead.' - ) - } - - if (lang === CodeLanguage.JavaScript && hasImports && !remoteSandboxEnabled) { - throw new Error( - 'JavaScript code with import statements requires a remote code sandbox to be enabled. Please remove the import statements, or contact your administrator to enable it.' - ) - } - - const useRemoteSandbox = - usesMothershipSandbox || - (remoteSandboxEnabled && - !isCustomTool && - (lang === CodeLanguage.Python || - (lang === CodeLanguage.JavaScript && (hasImports || Boolean(selectedSandboxId))))) - - if (useRemoteSandbox && containsLargeValueRef(contextVariables)) { - throw new Error( - 'Large execution values require the JavaScript isolated-vm runtime. Remove imports, select a nested field, or read the value in a JavaScript function without a remote sandbox.' - ) - } - - // Sandbox file mounts and sandboxPath exports only exist in the remote - // sandbox runtime; isolated-vm has no filesystem. Silently dropping a declared - // sandbox input/output here produced "export succeeded" responses with - // zero bytes written, so refuse the call instead. The remediation depends - // on WHY this call runs in isolated-vm — "switch to python" is a dead end - // when no remote sandbox is enabled or the call is a custom tool. - if ( - !useRemoteSandbox && - (outputSandboxPaths.length > 0 || outputSandboxPath || _sandboxFiles?.length) - ) { - const remediation = !remoteSandboxEnabled - ? "No remote code sandbox is enabled on this deployment, so there is no sandbox filesystem for any language. Pass input data via params and return output as the code's return value with outputs.files[].path (no sandboxPath)." - : isCustomTool - ? "custom tools always run in the isolated JavaScript VM, which has no sandbox filesystem. Pass input data via params and return output as the code's return value." - : 'plain JavaScript runs in the isolated VM, which has no sandbox filesystem. Use language "python" so the code runs in the remote sandbox, or drop sandboxPath and return the file content as the code\'s return value with outputs.files[].path.' - return functionJsonResponse( - { - success: false, - error: `Sandbox file inputs/outputs are unavailable for this call: ${remediation}`, - output: { result: null, stdout: '', executionTime: Date.now() - startTime }, - }, - routeContext, - { status: 422 } - ) - } - if (useRemoteSandbox) { logger.info(`[${requestId}] E2B status`, { enabled: remoteSandboxEnabled, @@ -2300,21 +2703,25 @@ export async function executeFunctionRequest( error: e2bError, exportedFileContent, exportedFiles, + collectedFiles: jsCollectedFiles, + cost: sandboxCost, } = await executeInSandbox({ code: codeForE2B, language: CodeLanguage.JavaScript, timeoutMs: timeout, - sandboxFiles: _sandboxFiles, + sandboxFiles, privateInputs: [...compilerPrivateInputs, runtimePrivateInput], runtimeBindings: compilerRuntimeBindings, outputSandboxPath, outputSandboxPaths, + outputSandboxDir, workspaceId, sandboxId: selectedSandboxId, ...(usesMothershipSandbox && !selectedSandboxId ? { sandboxKind: 'mothership' as const } : {}), signal: executionSignal, + meterUsage: meterRemoteSandboxUsage, }) const executionTime = Date.now() - execStart stdout += e2bStdout @@ -2340,7 +2747,12 @@ export async function executeFunctionRequest( { success: false, error: formattedError, - output: { result: null, stdout: cleanedOutput, executionTime }, + output: { + result: null, + stdout: cleanedOutput, + executionTime, + ...(sandboxCost ? { cost: sandboxCost } : {}), + }, }, routeContext, { status: 422 } @@ -2358,16 +2770,38 @@ export async function executeFunctionRequest( exportedFileContent, stdout, executionTime, + cost: sandboxCost, }) if (fileExportResponse) { return appendResolvedSecretNames(fileExportResponse, routeContext) } } + const jsOutputFiles = await collectExecutionOutputFiles({ + routeContext, + authUserId: auth.attributedUserId, + workflowId, + workspaceId, + executionId, + collectedFiles: jsCollectedFiles ?? [], + stdout, + executionTime, + cost: sandboxCost, + }) + if ('response' in jsOutputFiles) { + return appendResolvedSecretNames(jsOutputFiles.response, routeContext) + } + return functionJsonResponse( { success: true, - output: { result: e2bResult ?? null, stdout: cleanStdout(stdout), executionTime }, + output: { + result: e2bResult ?? null, + stdout: cleanStdout(stdout), + executionTime, + files: jsOutputFiles.files, + ...(sandboxCost ? { cost: sandboxCost } : {}), + }, }, routeContext ) @@ -2391,20 +2825,24 @@ export async function executeFunctionRequest( error: e2bError, exportedFileContent, exportedFiles, + collectedFiles: pythonCollectedFiles, + cost: sandboxCost, } = await executeInSandbox({ code: codeForE2B, language: CodeLanguage.Python, timeoutMs: timeout, - sandboxFiles: _sandboxFiles, + sandboxFiles, privateInputs: [...compilerPrivateInputs, runtimePrivateInput], outputSandboxPath, outputSandboxPaths, + outputSandboxDir, workspaceId, sandboxId: selectedSandboxId, ...(usesMothershipSandbox && !selectedSandboxId ? { sandboxKind: 'mothership' as const } : {}), signal: executionSignal, + meterUsage: meterRemoteSandboxUsage, }) const executionTime = Date.now() - execStart stdout += e2bStdout @@ -2430,7 +2868,12 @@ export async function executeFunctionRequest( { success: false, error: formattedError, - output: { result: null, stdout: cleanedOutput, executionTime }, + output: { + result: null, + stdout: cleanedOutput, + executionTime, + ...(sandboxCost ? { cost: sandboxCost } : {}), + }, }, routeContext, { status: 422 } @@ -2448,16 +2891,38 @@ export async function executeFunctionRequest( exportedFileContent, stdout, executionTime, + cost: sandboxCost, }) if (fileExportResponse) { return appendResolvedSecretNames(fileExportResponse, routeContext) } } + const pythonOutputFiles = await collectExecutionOutputFiles({ + routeContext, + authUserId: auth.attributedUserId, + workflowId, + workspaceId, + executionId, + collectedFiles: pythonCollectedFiles ?? [], + stdout, + executionTime, + cost: sandboxCost, + }) + if ('response' in pythonOutputFiles) { + return appendResolvedSecretNames(pythonOutputFiles.response, routeContext) + } + return functionJsonResponse( { success: true, - output: { result: e2bResult ?? null, stdout: cleanStdout(stdout), executionTime }, + output: { + result: e2bResult ?? null, + stdout: cleanStdout(stdout), + executionTime, + files: pythonOutputFiles.files, + ...(sandboxCost ? { cost: sandboxCost } : {}), + }, }, routeContext ) @@ -2636,11 +3101,21 @@ export async function executeFunctionRequest( privateResolvedSecretNamesMetadataType ) } - if (isSandboxOutputLimitError(error) || isSandboxOutputFileError(error)) { + if ( + isSandboxOutputLimitError(error) || + isSandboxOutputFileError(error) || + isSandboxOutputNotExportableError(error) + ) { + const cost = readTrustedSandboxOutputCost(error) const outputLimitResponse = { success: false, error: error.message, - output: { result: null, stdout: cleanStdout(stdout), executionTime }, + output: { + result: null, + stdout: cleanStdout(stdout), + executionTime, + ...(cost ? { cost } : {}), + }, } return routeContext ? functionJsonResponse(outputLimitResponse, routeContext, { status: 400 }) diff --git a/apps/sim/lib/function-execution/sandbox-mounts.test.ts b/apps/sim/lib/function-execution/sandbox-mounts.test.ts new file mode 100644 index 00000000000..2f49c989885 --- /dev/null +++ b/apps/sim/lib/function-execution/sandbox-mounts.test.ts @@ -0,0 +1,272 @@ +/** + * @vitest-environment node + * + * Mount resolution for platform file objects. The authorization assertions run + * against the real `assertUserFileContentAccess` rather than a stub: which files + * a Function block may mount is the security-relevant part of this module, and + * mocking it away would leave exactly that untested. + */ +import { beforeEach, describe, expect, it, vi } from 'vitest' +import type { UserFile } from '@/executor/types' + +const { + mockHasCloudStorage, + mockGeneratePresignedDownloadUrl, + mockDownloadServableFileFromStorage, + mockReadWorkspaceFileRecordByKey, +} = vi.hoisted(() => ({ + mockHasCloudStorage: vi.fn(), + mockGeneratePresignedDownloadUrl: vi.fn(), + mockDownloadServableFileFromStorage: vi.fn(), + mockReadWorkspaceFileRecordByKey: vi.fn(), +})) + +vi.mock('@/lib/uploads/core/storage-service', () => ({ + hasCloudStorage: mockHasCloudStorage, + generatePresignedDownloadUrl: mockGeneratePresignedDownloadUrl, +})) + +vi.mock('@/lib/uploads/utils/file-utils.server', () => ({ + downloadServableFileFromStorage: mockDownloadServableFileFromStorage, +})) + +vi.mock('@/lib/workspace-files/application/read-workspace-file-content-by-key', () => ({ + readWorkspaceFileRecordByKey: { execute: mockReadWorkspaceFileRecordByKey }, +})) + +import { + MOUNT_URL_TTL_SECONDS, + planUserFileMounts, + resolveUserFileMounts, +} from '@/lib/function-execution/sandbox-mounts' + +const WORKSPACE_ID = 'ws-1' +const WORKFLOW_ID = 'wf-1' +const EXECUTION_ID = 'exec-1' + +function executionFile(overrides: Partial = {}): UserFile { + return { + id: 'file_1', + name: 'report.csv', + url: 'https://storage.example/report.csv', + size: 32, + type: 'text/csv', + key: `execution/${WORKSPACE_ID}/${WORKFLOW_ID}/${EXECUTION_ID}/abc/report.csv`, + context: 'execution', + ...overrides, + } +} + +function workspaceFile(overrides: Partial = {}): UserFile { + return { + id: 'wf_1', + name: 'brief.pdf', + url: 'https://storage.example/brief.pdf', + size: 64, + type: 'application/pdf', + key: `workspace/${WORKSPACE_ID}/brief.pdf`, + context: 'workspace', + ...overrides, + } +} + +const executionContext = { + workspaceId: WORKSPACE_ID, + workflowId: WORKFLOW_ID, + executionId: EXECUTION_ID, + userId: 'user-1', + requestId: 'req-1', +} + +describe('planUserFileMounts', () => { + it('sanitizes names into a single safe path segment', () => { + const planned = planUserFileMounts([executionFile({ name: 'Q4 Sales (Final).csv' })]) + + expect(planned[0].mountPath).toBe('/tmp/sim/inputs/Q4-Sales-_Final_.csv') + }) + + it('cannot be escaped by a traversal in the file name', () => { + const planned = planUserFileMounts([ + executionFile({ name: '../../etc/passwd' }), + executionFile({ id: 'file_2', key: 'execution/other', name: '..' }), + ]) + + for (const { mountPath } of planned) { + expect(mountPath.startsWith('/tmp/sim/inputs/')).toBe(true) + expect(mountPath).not.toContain('/../') + expect(mountPath.endsWith('/..')).toBe(false) + } + }) + + it('suffixes colliding names so neither file is silently overwritten', () => { + const planned = planUserFileMounts([ + executionFile({ id: 'file_1', key: 'execution/a/report.csv', name: 'report.csv' }), + executionFile({ id: 'file_2', key: 'execution/b/report.csv', name: 'report.csv' }), + executionFile({ id: 'file_3', key: 'execution/c/report.csv', name: 'report.csv' }), + ]) + + expect(planned.map((entry) => entry.mountPath)).toEqual([ + '/tmp/sim/inputs/report.csv', + '/tmp/sim/inputs/report-2.csv', + '/tmp/sim/inputs/report-3.csv', + ]) + }) + + it('mounts one storage key once however many sources named it', () => { + // A caller listing the same file twice, and a `` marker for + // a file the caller also passed explicitly, both land in one list here. A + // second copy of identical bytes costs a presign and a duplicate transfer, + // and charges the byte budget and the 20-file ceiling twice over. + const planned = planUserFileMounts([ + executionFile({ id: 'file_1', name: 'report.csv' }), + executionFile({ id: 'file_1_again', name: 'report.csv' }), + executionFile({ id: 'file_2', name: 'renamed.csv' }), + workspaceFile(), + ]) + + expect(planned.map((entry) => entry.mountPath)).toEqual([ + '/tmp/sim/inputs/report.csv', + '/tmp/sim/inputs/brief.pdf', + ]) + }) +}) + +describe('resolveUserFileMounts', () => { + beforeEach(() => { + vi.clearAllMocks() + mockHasCloudStorage.mockReturnValue(true) + mockGeneratePresignedDownloadUrl.mockResolvedValue('https://presigned.example/object') + mockReadWorkspaceFileRecordByKey.mockResolvedValue({ file: { id: 'wf_1' } }) + // Sized from the file being read: the aggregate budget counts bytes actually + // buffered, so a fixed-size stub would never let the total ceiling trip. + mockDownloadServableFileFromStorage.mockImplementation(async (file: UserFile) => ({ + buffer: file.size > 16 ? Buffer.alloc(file.size) : Buffer.from('a,b\n1,2'), + contentType: file.type, + })) + }) + + it('mounts by presigned URL when cloud storage is configured', async () => { + const planned = planUserFileMounts([executionFile()]) + + const { sandboxFiles, manifest } = await resolveUserFileMounts({ + planned, + context: executionContext, + }) + + // The sandbox fetches the bytes itself, so nothing transits the web process. + expect(sandboxFiles).toEqual([ + { + type: 'url', + path: '/tmp/sim/inputs/report.csv', + url: 'https://presigned.example/object', + // Granted exactly what the mount was charged against the aggregate, so + // an understated size is refused rather than silently overrunning it. + maxBytes: 32, + }, + ]) + expect(mockGeneratePresignedDownloadUrl).toHaveBeenCalledWith( + planned[0].userFile.key, + 'execution', + MOUNT_URL_TTL_SECONDS + ) + expect(mockDownloadServableFileFromStorage).not.toHaveBeenCalled() + expect(manifest).toEqual([ + { name: 'report.csv', path: '/tmp/sim/inputs/report.csv', size: 32, type: 'text/csv' }, + ]) + }) + + it('buffers bytes inline when there is no cloud storage to presign from', async () => { + mockHasCloudStorage.mockReturnValue(false) + + const { sandboxFiles } = await resolveUserFileMounts({ + planned: planUserFileMounts([executionFile()]), + context: executionContext, + }) + + // A presigned URL under local storage is an app-internal serve path the + // remote sandbox cannot reach. + expect(mockGeneratePresignedDownloadUrl).not.toHaveBeenCalled() + expect(sandboxFiles).toEqual([ + { + path: '/tmp/sim/inputs/report.csv', + content: Buffer.alloc(32).toString('base64'), + encoding: 'base64', + }, + ]) + }) + + it('rejects a file over the per-file mount ceiling before presigning it', async () => { + await expect( + resolveUserFileMounts({ + planned: planUserFileMounts([executionFile({ size: 600 * 1024 * 1024 })]), + context: executionContext, + }) + ).rejects.toThrow(/per-file mount limit/) + + expect(mockGeneratePresignedDownloadUrl).not.toHaveBeenCalled() + }) + + it('rejects a batch over the total inline budget', async () => { + mockHasCloudStorage.mockReturnValue(false) + + await expect( + resolveUserFileMounts({ + planned: planUserFileMounts( + ['a', 'b', 'c', 'd', 'e', 'f'].map((id) => + executionFile({ + id, + key: `execution/${WORKSPACE_ID}/${WORKFLOW_ID}/${EXECUTION_ID}/${id}/${id}.bin`, + name: `${id}.bin`, + size: 9 * 1024 * 1024, + }) + ) + ), + context: executionContext, + }) + ).rejects.toThrow(/total mount limit/) + }) + + it('authorizes a design-time workspace upload through its workspace record', async () => { + const { sandboxFiles } = await resolveUserFileMounts({ + planned: planUserFileMounts([workspaceFile()]), + context: { ...executionContext, principal: { kind: 'sim_user' } as never }, + }) + + // The common case for a Function block: a file pinned in the block config is + // a workspace key, which never touches the execution-scope check at all. + expect(mockReadWorkspaceFileRecordByKey).toHaveBeenCalledWith( + expect.objectContaining({ + input: expect.objectContaining({ key: `workspace/${WORKSPACE_ID}/brief.pdf` }), + }) + ) + expect(sandboxFiles).toHaveLength(1) + }) + + it('refuses an execution file belonging to a different workflow', async () => { + const foreign = executionFile({ + key: `execution/${WORKSPACE_ID}/other-workflow/other-exec/xyz/secrets.csv`, + }) + + await expect( + resolveUserFileMounts({ + planned: planUserFileMounts([foreign]), + context: executionContext, + }) + ).rejects.toThrow(/not available in this execution/) + + expect(mockGeneratePresignedDownloadUrl).not.toHaveBeenCalled() + }) + + it('admits an execution file from another run when its key is in the allowlist', async () => { + const priorRun = executionFile({ + key: `execution/${WORKSPACE_ID}/${WORKFLOW_ID}/earlier-exec/xyz/prior.csv`, + }) + + const { sandboxFiles } = await resolveUserFileMounts({ + planned: planUserFileMounts([priorRun]), + context: { ...executionContext, fileKeys: [priorRun.key] }, + }) + + expect(sandboxFiles).toHaveLength(1) + }) +}) diff --git a/apps/sim/lib/function-execution/sandbox-mounts.ts b/apps/sim/lib/function-execution/sandbox-mounts.ts new file mode 100644 index 00000000000..cbad1fa2859 --- /dev/null +++ b/apps/sim/lib/function-execution/sandbox-mounts.ts @@ -0,0 +1,325 @@ +import { createLogger } from '@sim/logger' +import { + assertUserFileContentAccess, + type ExecutionMaterializationContext, + readUserFileContentWithContributors, +} from '@/lib/execution/payloads/materialization.server' +import { MAX_SANDBOX_URL_MOUNT_BYTES } from '@/lib/execution/remote-sandbox/output-limits' +import { SANDBOX_INPUT_DIR } from '@/lib/execution/remote-sandbox/sandbox-paths' +import type { SandboxFile } from '@/lib/execution/remote-sandbox/types' +import { buildStorageKeySegment } from '@/lib/uploads/core/storage-key' +import { generatePresignedDownloadUrl, hasCloudStorage } from '@/lib/uploads/core/storage-service' +import type { StorageContext } from '@/lib/uploads/shared/types' +import { + isGeneratedDocumentSourceType, + resolveTrustedFileContext, +} from '@/lib/uploads/utils/file-utils' +import type { UserFile } from '@/executor/types' + +const logger = createLogger('SandboxMounts') + +/** + * Lifetime of a presigned URL handed to the sandbox to fetch a mounted object. + * The URL grants read to exactly that one object and dies with the sandbox. + * + * Sized well past the worst provisioning path rather than the typical one: a + * runtime-strategy sandbox can spend up to RUNTIME_INSTALL_TIMEOUT_MS installing + * dependencies, and only then does the in-sandbox `curl` start its own 300s + * window. At the previous 600s the URL could expire mid-download and surface as + * an opaque "failed to fetch mounted file". + */ +export const MOUNT_URL_TTL_SECONDS = 1800 + +/** + * Per-file ceiling for URL-mounted files, shared with the sandbox layer that + * enforces it on the transferred bytes so the pre-check and the backstop can + * never drift apart. + */ +export const MOUNT_URL_MAX_BYTES = MAX_SANDBOX_URL_MOUNT_BYTES + +/** + * Aggregate ceiling across all URL mounts in one request. Rejects an oversized + * request up front instead of filling the sandbox disk one slow fetch at a time. + */ +export const MAX_TOTAL_URL_BYTES = 2 * 1024 * 1024 * 1024 + +/** Per-file ceiling when bytes must pass through the web process. */ +export const MAX_INLINE_MOUNT_FILE_BYTES = 10 * 1024 * 1024 + +/** Aggregate ceiling for buffered mounts, bounding web heap rather than disk. */ +export const MAX_INLINE_MOUNT_TOTAL_BYTES = 50 * 1024 * 1024 + +/** + * Running byte totals for one resolve pass. `buffered` bytes pass through the web + * process; `url` bytes are fetched straight into the sandbox. Tracked separately + * because the two ceilings protect different resources — web heap vs sandbox disk. + */ +export interface SandboxMountBudget { + buffered: number + url: number +} + +export function createSandboxMountBudget(): SandboxMountBudget { + return { buffered: 0, url: 0 } +} + +/** One object to mount, independent of how the caller located it. */ +export interface SandboxMountSource { + mountPath: string + key: string + storageContext: StorageContext + /** Size recorded for the stored object, used for the pre-read ceilings. */ + declaredSize: number + /** + * True when `key` holds generator source rather than the servable bytes. Such + * an object must never be presigned: the sandbox would receive source text + * under a `.docx` name and the caller's script would fail on a file that looks + * fine. It also means {@link declaredSize} describes the generator, not the + * document, so the pre-read ceilings say nothing and the read is capped instead. + */ + rendersFromSource: boolean + /** + * Bounded read producing the inline payload. Only called on the buffered + * branch, so a URL mount never reads bytes into the web process. + */ + readInline(maxBytes: number): Promise +} + +export interface SandboxInlineMountPayload { + content: string + encoding?: 'base64' + /** Decoded length, which is what the buffered budget counts. */ + byteLength: number +} + +/** + * Mounts one stored object into the sandbox and records its bytes against the + * running totals. + * + * With cloud storage the sandbox fetches the bytes itself from a presigned URL; + * with local storage a presigned URL is an app-internal serve path a remote + * sandbox cannot reach, so the bytes are buffered through the web process under + * the tighter inline ceilings. + */ +export async function pushSandboxFileMount( + sandboxFiles: SandboxFile[], + source: SandboxMountSource, + budget: SandboxMountBudget +): Promise { + if (hasCloudStorage() && !source.rendersFromSource) { + /** + * The number this mount is both admitted on and later held to. + * + * Resolved once, before any comparison, because a non-finite size makes every + * `>` test false — an aggregate check reading `budget.url + NaN` would pass + * silently while the mount still consumed real budget. A missing or + * nonsensical size therefore costs the per-file maximum rather than nothing, + * and a zero takes a one-byte floor, since zero reads as "unlimited" to curl. + */ + const grantedBytes = + Number.isFinite(source.declaredSize) && source.declaredSize >= 0 + ? Math.max(1, source.declaredSize) + : MOUNT_URL_MAX_BYTES + + if (grantedBytes > MOUNT_URL_MAX_BYTES) { + throw new Error( + `Input file "${source.mountPath}" is ${Math.round(grantedBytes / 1024 / 1024)}MB, over the ${MOUNT_URL_MAX_BYTES / 1024 / 1024}MB per-file mount limit.` + ) + } + if (budget.url + grantedBytes > MAX_TOTAL_URL_BYTES) { + throw new Error( + `Mounting "${source.mountPath}" would exceed the ${MAX_TOTAL_URL_BYTES / 1024 / 1024 / 1024}GB total mount limit. Mount fewer or smaller files.` + ) + } + const url = await generatePresignedDownloadUrl( + source.key, + source.storageContext, + MOUNT_URL_TTL_SECONDS + ) + /** + * Granted exactly what it was charged, so the aggregate stays honest without a + * stat round-trip per file. Charging the recorded size while permitting the + * global per-file maximum would let understated sizes accumulate far past the + * ceiling — twenty mounts each claiming a byte and each allowed 500MB. + */ + sandboxFiles.push({ + type: 'url', + path: source.mountPath, + url, + maxBytes: grantedBytes, + }) + budget.url += grantedBytes + return + } + + const remainingBudget = Math.max(0, MAX_INLINE_MOUNT_TOTAL_BYTES - budget.buffered) + + if (!source.rendersFromSource) { + if (source.declaredSize > MAX_INLINE_MOUNT_FILE_BYTES) { + throw new Error( + `Input file "${source.mountPath}" is ${Math.round(source.declaredSize / 1024 / 1024)}MB, over the ${MAX_INLINE_MOUNT_FILE_BYTES / 1024 / 1024}MB per-file mount limit.` + ) + } + if (source.declaredSize > remainingBudget) { + throw new Error( + `Mounting "${source.mountPath}" would exceed the ${MAX_INLINE_MOUNT_TOTAL_BYTES / 1024 / 1024}MB total mount limit. Mount fewer or smaller files.` + ) + } + } + + const inline = await source.readInline(Math.min(MAX_INLINE_MOUNT_FILE_BYTES, remainingBudget)) + sandboxFiles.push({ + path: source.mountPath, + content: inline.content, + ...(inline.encoding ? { encoding: inline.encoding } : {}), + }) + budget.buffered += inline.byteLength +} + +export interface PlannedUserFileMount { + userFile: UserFile + mountPath: string +} + +/** What the running code is told about its mounts, so it never guesses a path. */ +export interface SandboxMountManifestEntry { + name: string + path: string + size: number + type: string +} + +/** + * Derives a mount file name that is safe as a path segment. + * + * `sanitizeFileName` (via {@link buildStorageKeySegment}) already maps `/` and + * `\` to `_`, so no traversal survives it; the explicit guards cover the + * degenerate remainders it does leave intact, since `.` and `-` are permitted + * characters and `..` would otherwise pass through unchanged. + */ +function safeMountFileName(name: string): string { + const segment = buildStorageKeySegment('', name) + if (!segment || segment === '.' || segment === '..') return 'file' + return segment +} + +function uniqueMountFileName(name: string, used: Set): string { + const safe = safeMountFileName(name) + if (!used.has(safe)) { + used.add(safe) + return safe + } + // Two upstream blocks each producing `report.csv` must both survive: without a + // suffix the second write silently overwrites the first and the code sees one file. + const dot = safe.lastIndexOf('.') + const stem = dot > 0 ? safe.slice(0, dot) : safe + const extension = dot > 0 ? safe.slice(dot) : '' + for (let attempt = 2; ; attempt += 1) { + const candidate = `${stem}-${attempt}${extension}` + if (!used.has(candidate)) { + used.add(candidate) + return candidate + } + } +} + +/** + * Assigns each file a deterministic mount path. Pure and I/O-free, so a caller + * can decide whether an execution needs a sandbox filesystem before spending a + * presign or a byte of transfer on a request that may still be refused. + * + * A storage key mounts once. The same object arrives from independent sources — + * a caller listing it twice, or listing one the code also asked for with + * `` — and a second copy of identical bytes costs a presign, a + * duplicate transfer, and a second charge against both the byte budget and the + * per-request file ceiling. First occurrence wins, so the name listed first is + * the one the code sees. + */ +export function planUserFileMounts( + files: readonly UserFile[], + mountDir: string = SANDBOX_INPUT_DIR +): PlannedUserFileMount[] { + const used = new Set() + const mountedKeys = new Set() + const planned: PlannedUserFileMount[] = [] + + for (const userFile of files) { + if (mountedKeys.has(userFile.key)) continue + mountedKeys.add(userFile.key) + planned.push({ + userFile, + mountPath: `${mountDir}/${uniqueMountFileName(userFile.name, used)}`, + }) + } + + return planned +} + +/** + * Resolves planned platform file objects into sandbox mounts. + * + * Authorization runs through {@link assertUserFileContentAccess} rather than the + * tool-file check used by ordinary integrations. For an `execution/` key the + * latter grants on workspace membership alone, which would let a Function block + * mount any execution file from any past run of any workflow in the workspace; + * this one additionally requires the workflow to match and the key to be in the + * execution's allowlist. It is asserted before the transport branches, because + * the URL path never reads the bytes and so never reaches the check embedded in + * the reader. + */ +export async function resolveUserFileMounts(args: { + planned: readonly PlannedUserFileMount[] + context: ExecutionMaterializationContext +}): Promise<{ sandboxFiles: SandboxFile[]; manifest: SandboxMountManifestEntry[] }> { + const sandboxFiles: SandboxFile[] = [] + const manifest: SandboxMountManifestEntry[] = [] + const budget = createSandboxMountBudget() + + for (const { userFile, mountPath } of args.planned) { + const storageContext = resolveTrustedFileContext(userFile.key, userFile.context) + await assertUserFileContentAccess(userFile, args.context) + + await pushSandboxFileMount( + sandboxFiles, + { + mountPath, + key: userFile.key, + storageContext, + declaredSize: userFile.size, + rendersFromSource: isGeneratedDocumentSourceType(userFile.type), + readInline: async (maxBytes) => { + // Base64 regardless of content type: the payload is reproduced exactly + // for any byte sequence, and picking utf8 for a mistyped binary would + // substitute U+FFFD and hand the code a corrupted file. + const { content } = await readUserFileContentWithContributors(userFile, { + ...args.context, + encoding: 'base64', + maxBytes, + maxSourceBytes: maxBytes, + }) + return { + content, + encoding: 'base64' as const, + byteLength: Buffer.byteLength(content, 'base64'), + } + }, + }, + budget + ) + + manifest.push({ + name: userFile.name, + path: mountPath, + size: userFile.size, + type: userFile.type, + }) + } + + logger.info('Resolved sandbox file mounts', { + mountCount: sandboxFiles.length, + bufferedBytes: budget.buffered, + urlBytes: budget.url, + }) + + return { sandboxFiles, manifest } +} diff --git a/apps/sim/lib/guardrails/validation-client.ts b/apps/sim/lib/guardrails/validation-client.ts index 51e2a8c0f3b..2061040f7ee 100644 --- a/apps/sim/lib/guardrails/validation-client.ts +++ b/apps/sim/lib/guardrails/validation-client.ts @@ -1,9 +1,10 @@ -import type { GuardrailsPiiValidateBody, GuardrailsPiiValidateResult } from '@/lib/api/contracts' import { + type GuardrailsPiiValidateBody, + type GuardrailsPiiValidateResult, guardrailsPiiValidateBodySchema, guardrailsPiiValidateContract, guardrailsPiiValidateResponseSchema, -} from '@/lib/api/contracts' +} from '@/lib/api/contracts/hotspots' import { generateInternalToken } from '@/lib/auth/internal' import { DEFAULT_MAX_ERROR_BODY_BYTES, diff --git a/apps/sim/lib/imap/connection.server.test.ts b/apps/sim/lib/imap/connection.server.test.ts new file mode 100644 index 00000000000..1411057ee52 --- /dev/null +++ b/apps/sim/lib/imap/connection.server.test.ts @@ -0,0 +1,251 @@ +/** + * @vitest-environment node + */ +import { environmentUtilsMockFns, resetEnvironmentUtilsMock } from '@sim/testing' +import { afterAll, beforeEach, describe, expect, it, vi } from 'vitest' + +const { mockImapFlow, mockValidateDatabaseHost } = vi.hoisted(() => ({ + mockImapFlow: vi.fn(), + mockValidateDatabaseHost: vi.fn(), +})) + +vi.mock('imapflow', () => ({ + ImapFlow: function MockImapFlow(options: unknown) { + mockImapFlow(options) + }, +})) + +vi.mock('@/lib/core/security/input-validation.server', () => ({ + validateDatabaseHost: mockValidateDatabaseHost, +})) + +import { + createSecureImapClient, + type ImapConnectionPolicyError, + normalizeLiteralImapConnection, + normalizeResolvedImapConnection, + resolveImapConnectionForActor, +} from '@/lib/imap/connection.server' + +describe('IMAP connection policy', () => { + beforeEach(() => { + vi.clearAllMocks() + resetEnvironmentUtilsMock() + mockValidateDatabaseHost.mockResolvedValue({ + isValid: true, + sanitized: 'imap.example.com', + resolvedIP: '203.0.113.10', + }) + }) + + afterAll(resetEnvironmentUtilsMock) + + it('accepts literal configuration while requiring TLS or STARTTLS on the pinned host', async () => { + const secureConnection = normalizeLiteralImapConnection({ + host: ' imap.example.com ', + username: 'mailbox-user', + password: 'literal-password', + }) + const startTlsConnection = normalizeLiteralImapConnection({ + host: 'imap.example.com', + port: '143', + secure: 'false', + username: 'mailbox-user', + password: 'literal-password', + }) + + await createSecureImapClient(secureConnection) + await createSecureImapClient(startTlsConnection) + + expect(secureConnection).toEqual({ + host: 'imap.example.com', + port: 993, + secure: true, + username: 'mailbox-user', + password: 'literal-password', + }) + expect(mockValidateDatabaseHost).toHaveBeenCalledTimes(2) + expect(mockValidateDatabaseHost).toHaveBeenNthCalledWith(1, 'imap.example.com', 'host', { + logDetails: false, + }) + expect(mockValidateDatabaseHost).toHaveBeenNthCalledWith(2, 'imap.example.com', 'host', { + logDetails: false, + }) + expect(mockImapFlow).toHaveBeenNthCalledWith( + 1, + expect.objectContaining({ + host: '203.0.113.10', + servername: 'imap.example.com', + port: 993, + secure: true, + auth: { user: 'mailbox-user', pass: 'literal-password' }, + tls: { rejectUnauthorized: true }, + logger: false, + }) + ) + expect(mockImapFlow.mock.calls[0]?.[0]).not.toHaveProperty('doSTARTTLS') + expect(mockImapFlow).toHaveBeenNthCalledWith( + 2, + expect.objectContaining({ secure: false, port: 143, doSTARTTLS: true }) + ) + }) + + it('preserves the legacy TLS defaults for nullable connection values', () => { + expect( + normalizeLiteralImapConnection({ + host: 'imap.example.com', + port: null, + secure: null, + username: 'mailbox-user', + password: 'literal-password', + }) + ).toEqual({ + host: 'imap.example.com', + port: 993, + secure: true, + username: 'mailbox-user', + password: 'literal-password', + }) + }) + + it('resolves exact personal and visible shared references for the deployment actor', async () => { + environmentUtilsMockFns.mockResolveEffectiveEnvironmentVariables.mockResolvedValue({ + PERSONAL_PASSWORD: { + value: 'personal-password', + scope: 'personal', + visible: true, + }, + SHARED_HOST: { value: 'imap.shared.example', scope: 'workspace', visible: false }, + SHARED_PORT: { value: '143', scope: 'workspace', visible: false }, + SHARED_SECURE: { value: 'false', scope: 'workspace', visible: false }, + SHARED_USERNAME: { value: 'shared-user', scope: 'workspace', visible: true }, + }) + + await expect( + resolveImapConnectionForActor({ + connection: { + host: '{{SHARED_HOST}}', + port: '{{SHARED_PORT}}', + secure: '{{SHARED_SECURE}}', + username: '{{SHARED_USERNAME}}', + password: '{{PERSONAL_PASSWORD}}', + }, + actorUserId: 'actor-1', + workspaceId: 'workspace-1', + }) + ).resolves.toEqual({ + host: 'imap.shared.example', + port: 143, + secure: false, + username: 'shared-user', + password: 'personal-password', + }) + expect(environmentUtilsMockFns.mockResolveEffectiveEnvironmentVariables).toHaveBeenCalledWith( + 'actor-1', + 'workspace-1', + ['SHARED_HOST', 'SHARED_PORT', 'SHARED_SECURE', 'SHARED_USERNAME', 'PERSONAL_PASSWORD'] + ) + expect(environmentUtilsMockFns.mockGetEffectiveEnvironmentSnapshot).not.toHaveBeenCalled() + }) + + it('rejects hidden shared username and password references before DNS or ImapFlow', async () => { + environmentUtilsMockFns.mockResolveEffectiveEnvironmentVariables.mockResolvedValue({ + HIDDEN_AUTH: { value: 'use-only-secret', scope: 'workspace', visible: false }, + }) + + for (const field of ['username', 'password'] as const) { + const connection = { + host: 'imap.example.com', + username: 'literal-user', + password: 'literal-password', + [field]: '{{HIDDEN_AUTH}}', + } + await expect( + resolveImapConnectionForActor({ + connection, + actorUserId: 'actor-1', + workspaceId: 'workspace-1', + }) + ).rejects.toMatchObject>({ + name: 'ImapConnectionPolicyError', + code: 'hidden_auth', + message: 'IMAP connection is unavailable', + }) + } + + expect(mockValidateDatabaseHost).not.toHaveBeenCalled() + expect(mockImapFlow).not.toHaveBeenCalled() + }) + + it('permits braces introduced by authorized resolution while keeping raw literals strict', () => { + expect( + normalizeResolvedImapConnection({ + host: 'imap.example.com', + username: 'mailbox-user', + password: 'literal{{brace}}secret', + }) + ).toMatchObject({ password: 'literal{{brace}}secret' }) + + expect(() => + normalizeLiteralImapConnection({ + host: 'imap.example.com', + username: 'mailbox-user', + password: 'literal{{brace}}secret', + }) + ).toThrowError( + expect.objectContaining>({ + name: 'ImapConnectionPolicyError', + code: 'context', + }) + ) + }) + + it('rejects unresolved workflow references in literal IMAP connection fields', () => { + expect(() => + normalizeLiteralImapConnection({ + host: 'imap.example.com', + username: '', + password: 'literal-password', + }) + ).toThrowError( + expect.objectContaining>({ + name: 'ImapConnectionPolicyError', + code: 'context', + }) + ) + }) + + it('reauthorizes requested references on every resolution and fails closed after revocation', async () => { + environmentUtilsMockFns.mockResolveEffectiveEnvironmentVariables + .mockResolvedValueOnce({ + PASSWORD: { value: 'visible-password', scope: 'workspace', visible: true }, + }) + .mockResolvedValueOnce({ + PASSWORD: { value: 'hidden-password', scope: 'workspace', visible: false }, + }) + const input = { + connection: { + host: 'imap.example.com', + username: 'mailbox-user', + password: '{{PASSWORD}}', + }, + actorUserId: 'actor-1', + workspaceId: 'workspace-1', + } + + await expect(resolveImapConnectionForActor(input)).resolves.toMatchObject({ + password: 'visible-password', + }) + await expect(resolveImapConnectionForActor(input)).rejects.toMatchObject< + Partial + >({ + name: 'ImapConnectionPolicyError', + code: 'hidden_auth', + }) + + expect(environmentUtilsMockFns.mockResolveEffectiveEnvironmentVariables).toHaveBeenCalledTimes( + 2 + ) + expect(environmentUtilsMockFns.mockGetEffectiveEnvironmentSnapshot).not.toHaveBeenCalled() + }) +}) diff --git a/apps/sim/lib/imap/connection.server.ts b/apps/sim/lib/imap/connection.server.ts new file mode 100644 index 00000000000..98a922b76dd --- /dev/null +++ b/apps/sim/lib/imap/connection.server.ts @@ -0,0 +1,184 @@ +import { ImapFlow } from 'imapflow' +import { validateDatabaseHost } from '@/lib/core/security/input-validation.server' +import { resolveEffectiveEnvironmentVariables } from '@/lib/environment/utils' +import { containsReference } from '@/lib/workflows/sanitization/references' + +const EXACT_ENVIRONMENT_REFERENCE = /^\{\{\s*([A-Za-z_][A-Za-z0-9_]*)\s*\}\}$/ + +export class ImapConnectionPolicyError extends Error { + constructor(readonly code: 'context' | 'hidden_auth' | 'destination' | 'transport') { + super('IMAP connection is unavailable') + this.name = 'ImapConnectionPolicyError' + } +} + +export interface ImapConnectionInput { + host: unknown + port?: unknown + secure?: unknown + username: unknown + password: unknown +} + +export interface ResolvedImapConnection { + host: string + port: number + secure: boolean + username: string + password: string +} + +function containsUnresolvedReference(value: string): boolean { + return value.includes('{{') || value.includes('}}') || containsReference(value) +} + +export function hasImapEnvironmentReferences(input: ImapConnectionInput): boolean { + return [input.host, input.port, input.secure, input.username, input.password].some( + (value) => typeof value === 'string' && EXACT_ENVIRONMENT_REFERENCE.test(value) + ) +} + +function normalizeConnection(input: ImapConnectionInput): ResolvedImapConnection { + const host = typeof input.host === 'string' ? input.host.trim() : '' + const username = typeof input.username === 'string' ? input.username : '' + const password = typeof input.password === 'string' ? input.password : '' + const port = + input.port === null || input.port === undefined || input.port === '' ? 993 : Number(input.port) + const secure = + input.secure === null || input.secure === undefined || input.secure === '' + ? true + : typeof input.secure === 'string' + ? input.secure.toLowerCase() === 'true' + : input.secure === true + + if (!host || !username || !password || !Number.isInteger(port) || port < 1 || port > 65_535) { + throw new ImapConnectionPolicyError('context') + } + return { host, port, secure, username, password } +} + +/** Resolves only exact references and rejects hidden shared authentication material. */ +export async function resolveImapConnectionForActor(input: { + connection: ImapConnectionInput + actorUserId: string + workspaceId?: string | null +}): Promise { + if (!hasImapEnvironmentReferences(input.connection)) { + return normalizeLiteralImapConnection(input.connection) + } + + const referenceNames = [ + ...new Set( + [ + input.connection.host, + input.connection.port, + input.connection.secure, + input.connection.username, + input.connection.password, + ].flatMap((value) => { + if (typeof value !== 'string') return [] + const match = EXACT_ENVIRONMENT_REFERENCE.exec(value) + if (match) return [match[1]] + if (containsUnresolvedReference(value)) throw new ImapConnectionPolicyError('context') + return [] + }) + ), + ] + const resolvedVariables = await resolveEffectiveEnvironmentVariables( + input.actorUserId, + input.workspaceId ?? undefined, + referenceNames + ) + + const resolve = (field: 'host' | 'port' | 'secure' | 'username' | 'password', value: unknown) => { + if (typeof value !== 'string') return value + const match = EXACT_ENVIRONMENT_REFERENCE.exec(value) + if (!match) return value + const name = match[1] + const variable = Object.hasOwn(resolvedVariables, name) ? resolvedVariables[name] : undefined + if (!variable) throw new ImapConnectionPolicyError('context') + if ((field === 'username' || field === 'password') && !variable.visible) { + throw new ImapConnectionPolicyError('hidden_auth') + } + return variable.value + } + + return normalizeResolvedImapConnection({ + host: resolve('host', input.connection.host), + port: resolve('port', input.connection.port), + secure: resolve('secure', input.connection.secure), + username: resolve('username', input.connection.username), + password: resolve('password', input.connection.password), + }) +} + +/** Normalizes values only after exact environment references have been resolved and authorized. */ +export function normalizeResolvedImapConnection( + input: ImapConnectionInput +): ResolvedImapConnection { + return normalizeConnection(input) +} + +export function normalizeLiteralImapConnection(input: ImapConnectionInput): ResolvedImapConnection { + for (const value of [input.host, input.port, input.secure, input.username, input.password]) { + if (typeof value === 'string' && containsUnresolvedReference(value)) { + throw new ImapConnectionPolicyError('context') + } + } + return normalizeConnection(input) +} + +/** Validates and pins the user-controlled destination before any IMAP authentication occurs. */ +export async function createSecureImapClient( + connection: ResolvedImapConnection, + signal?: AbortSignal +): Promise { + signal?.throwIfAborted() + const validation = await validateDatabaseHost(connection.host, 'host', { logDetails: false }) + signal?.throwIfAborted() + if (!validation.isValid || !validation.resolvedIP) { + throw new ImapConnectionPolicyError('destination') + } + return new ImapFlow({ + host: validation.resolvedIP, + servername: connection.host, + port: connection.port, + secure: connection.secure, + ...(connection.secure ? {} : { doSTARTTLS: true }), + auth: { user: connection.username, pass: connection.password }, + tls: { rejectUnauthorized: true }, + logger: false, + }) +} + +export async function listImapMailboxes( + connection: ResolvedImapConnection, + signal?: AbortSignal +): Promise> { + const client = await createSecureImapClient(connection, signal) + const abort = () => client.close() + signal?.addEventListener('abort', abort, { once: true }) + try { + signal?.throwIfAborted() + await client.connect() + signal?.throwIfAborted() + const mailboxes = (await client.list()).map((mailbox) => ({ + path: mailbox.path, + name: mailbox.name, + delimiter: mailbox.delimiter, + })) + mailboxes.sort((left, right) => { + if (left.path === 'INBOX') return -1 + if (right.path === 'INBOX') return 1 + return left.path.localeCompare(right.path) + }) + return mailboxes + } finally { + signal?.removeEventListener('abort', abort) + try { + await client.logout() + } catch { + client.close() + } + } +} diff --git a/apps/sim/lib/integrations/icon-mapping.ts b/apps/sim/lib/integrations/icon-mapping.ts index aa5a326a0b9..1927f934f5f 100644 --- a/apps/sim/lib/integrations/icon-mapping.ts +++ b/apps/sim/lib/integrations/icon-mapping.ts @@ -116,6 +116,7 @@ import { HexIcon, HubspotIcon, HuggingFaceIcon, + HumanInTheLoopIcon, HunterIOIcon, IAMIcon, IcypeasIcon, @@ -205,6 +206,7 @@ import { RootlyIcon, RssIcon, S3Icon, + SailPointIcon, SalesforceIcon, SapConcurIcon, SapS4HanaIcon, @@ -393,6 +395,7 @@ export const blockTypeToIconMap: Record = { hex: HexIcon, hubspot: HubspotIcon, huggingface: HuggingFaceIcon, + human_in_the_loop_v2: HumanInTheLoopIcon, hunter: HunterIOIcon, iam: IAMIcon, icypeas: IcypeasIcon, @@ -491,6 +494,7 @@ export const blockTypeToIconMap: Record = { rootly: RootlyIcon, rss: RssIcon, s3: S3Icon, + sailpoint: SailPointIcon, salesforce: SalesforceIcon, sap_concur: SapConcurIcon, sap_s4hana: SapS4HanaIcon, @@ -525,6 +529,7 @@ export const blockTypeToIconMap: Record = { stt_v2: STTIcon, supabase: SupabaseIcon, table: Table, + table_v2: Table, tailscale: TailscaleIcon, tavily: TavilyIcon, telegram: TelegramIcon, diff --git a/apps/sim/lib/integrations/principal-scope.server.ts b/apps/sim/lib/integrations/principal-scope.server.ts index f7d0a6b993d..3e239092b15 100644 --- a/apps/sim/lib/integrations/principal-scope.server.ts +++ b/apps/sim/lib/integrations/principal-scope.server.ts @@ -1,7 +1,7 @@ import type { Principal } from '@sim/auth/principal' import { getAllowedIntegrationsFromEnv } from '@/lib/core/config/env-flags' -import { intersectIntegrationAllowlists } from '@/lib/permission-groups/integration-allowlist' -import { getUserPermissionConfig } from '@/ee/access-control/utils/permission-check' +import { resolvePermissionGroupConfig } from '@/lib/permission-groups/config-scope.server' +import { intersectAccessControlAllowlists } from '@/lib/permission-groups/integration-allowlist' /** * The workspace integration gate, shared by every catalog that projects @@ -15,7 +15,7 @@ import { getUserPermissionConfig } from '@/ee/access-control/utils/permission-ch * first time either changed, and the two endpoints describe the same * integrations. * - * Server-only: `getUserPermissionConfig` reads the database. + * Server-only: `resolvePermissionGroupConfig` reads the database. */ /** @@ -41,16 +41,21 @@ export function principalUserId(principal: Principal): string | undefined { * The intersection of the caller's permission-group allowlist with the * deployment's `ALLOWED_INTEGRATIONS`. A principal with no user contributes no * permission-group half, leaving the deployment allowlist alone. + * + * Each half is successor-resolved *before* the intersection, so a group naming + * `slack_v2` and a deployment naming `slack` still meet. Callers resolve the + * type they test the same way. */ export async function allowedIntegrationTypes( principal: Principal, workspaceId: string ): Promise | null> { const userId = principalUserId(principal) - const permissionConfig = userId ? await getUserPermissionConfig(userId, workspaceId) : null - const integrations = intersectIntegrationAllowlists( + const permissionConfig = userId + ? await resolvePermissionGroupConfig(userId, workspaceId, undefined) + : null + return intersectAccessControlAllowlists( permissionConfig?.allowedIntegrations ?? null, getAllowedIntegrationsFromEnv() ) - return integrations ? new Set(integrations.map((type) => type.toLowerCase())) : null } diff --git a/apps/sim/lib/internal/agiloft/client.test.ts b/apps/sim/lib/internal/agiloft/client.test.ts index b20c68312ef..a6ca3f50b86 100644 --- a/apps/sim/lib/internal/agiloft/client.test.ts +++ b/apps/sim/lib/internal/agiloft/client.test.ts @@ -79,7 +79,8 @@ describe('executeAgiloftRequest', () => { expect(mockValidateUrlWithDNS).toHaveBeenCalledWith( 'https://example.agiloft.com', - 'instanceUrl' + 'instanceUrl', + 'configuredEndpoint' ) const calls = mockSecureFetch.mock.calls diff --git a/apps/sim/lib/internal/agiloft/client.ts b/apps/sim/lib/internal/agiloft/client.ts index a837c890f38..d016c7e136f 100644 --- a/apps/sim/lib/internal/agiloft/client.ts +++ b/apps/sim/lib/internal/agiloft/client.ts @@ -31,9 +31,9 @@ export async function resolveAgiloftInstance( signal?: AbortSignal ): Promise { signal?.throwIfAborted() - const validation = await validateUrlWithDNS(instanceUrl, 'instanceUrl') + const validation = await validateUrlWithDNS(instanceUrl, 'instanceUrl', 'configuredEndpoint') signal?.throwIfAborted() - if (!validation.isValid || !validation.resolvedIP) { + if (!validation.isValid) { throw new Error(validation.error || 'Invalid Agiloft instance URL') } return validation.resolvedIP @@ -81,6 +81,7 @@ export async function agiloftLoginPinned( const base = params.instanceUrl.replace(/\/$/, '') const response = await secureFetchWithPinnedIP(`${base}/ewws/EWLogin`, resolvedIP, { + profile: 'configuredEndpoint', method: 'POST', headers: { 'Content-Type': 'application/x-www-form-urlencoded' }, body: formEncode( @@ -139,6 +140,7 @@ export async function agiloftLogoutPinned( `${base}/ewws/EWLogout?$KB=${kb}&$lang=${AGILOFT_LANG}`, resolvedIP, { + profile: 'configuredEndpoint', method: 'POST', headers: { Authorization: authorization }, maxResponseBytes: MAX_JSON_API_RESPONSE_BYTES, @@ -182,6 +184,7 @@ export async function executeAgiloftRequest( try { const req = buildRequest(base) const response = await secureFetchWithPinnedIP(req.url, resolvedIP, { + profile: 'configuredEndpoint', method: req.method, headers: { ...req.headers, @@ -280,6 +283,7 @@ export async function executeAlrestRequest( try { const req = buildRequest(agiloftAlrestBase(params.instanceUrl, params.knowledgeBase)) const response = await secureFetchWithPinnedIP(req.url, resolvedIP, { + profile: 'configuredEndpoint', method: req.method, headers: { ...req.headers, Authorization: session.authorization }, body: req.body, @@ -313,6 +317,7 @@ export async function executeEwRequest( const resolvedIP = await resolveAgiloftInstance(params.instanceUrl, signal) const req = buildRequest(params.instanceUrl.replace(/\/$/, '')) const response = await secureFetchWithPinnedIP(req.url, resolvedIP, { + profile: 'configuredEndpoint', method: req.method, headers: req.headers, body: req.body, diff --git a/apps/sim/lib/internal/agiloft/operations.test.ts b/apps/sim/lib/internal/agiloft/operations.test.ts index 922e1b33a57..a9e373f3244 100644 --- a/apps/sim/lib/internal/agiloft/operations.test.ts +++ b/apps/sim/lib/internal/agiloft/operations.test.ts @@ -178,6 +178,7 @@ describe('Agiloft operations', () => { expect.stringContaining('/ewws/EWRetrieve'), '203.0.113.10', { + profile: 'configuredEndpoint', method: 'GET', maxResponseBytes: 25 * 1024 * 1024, signal: controller.signal, diff --git a/apps/sim/lib/internal/agiloft/operations.ts b/apps/sim/lib/internal/agiloft/operations.ts index 91c983b44b5..a346fd60ef8 100644 --- a/apps/sim/lib/internal/agiloft/operations.ts +++ b/apps/sim/lib/internal/agiloft/operations.ts @@ -853,6 +853,7 @@ export async function executeAgiloftAttachFile( buildAttachFileUrl(input.instanceUrl.replace(/\/$/, ''), input, fileName), resolvedIP, { + profile: 'configuredEndpoint', method: 'PUT', headers: { 'Content-Type': 'application/octet-stream' }, body: buffer, @@ -903,7 +904,12 @@ export async function executeAgiloftRetrieveAttachment( const response = await secureFetchWithPinnedIP( buildRetrieveAttachmentUrl(input.instanceUrl.replace(/\/$/, ''), input), resolvedIP, - { method: 'GET', maxResponseBytes: AGILOFT_MAX_ATTACHMENT_BYTES, signal: context.signal } + { + profile: 'configuredEndpoint', + method: 'GET', + maxResponseBytes: AGILOFT_MAX_ATTACHMENT_BYTES, + signal: context.signal, + } ) if (!response.ok) { const text = await response.text() diff --git a/apps/sim/lib/internal/ashby/execute-tool.ts b/apps/sim/lib/internal/ashby/execute-tool.ts new file mode 100644 index 00000000000..4e485576575 --- /dev/null +++ b/apps/sim/lib/internal/ashby/execute-tool.ts @@ -0,0 +1,26 @@ +import { getValidationErrorMessage } from '@/lib/api/server' +import { executeAshbyUpload } from '@/lib/internal/ashby/operations' +import { ashbyUploadInputSchema } from '@/lib/internal/ashby/schema' +import type { InternalToolOperationHandler } from '@/lib/internal/tool-operations/types' + +export const executeAshbyTool: InternalToolOperationHandler = async (request) => { + request.signal?.throwIfAborted() + if (!request.context.userId) + return Response.json({ success: false, error: 'Authentication required' }, { status: 401 }) + if (request.toolId !== 'ashby_upload_resume' && request.toolId !== 'ashby_upload_candidate_file') + return Response.json( + { success: false, error: `Unsupported Ashby tool: ${request.toolId}` }, + { status: 500 } + ) + const parsed = ashbyUploadInputSchema.safeParse(request.input) + if (!parsed.success) + return Response.json( + { success: false, error: getValidationErrorMessage(parsed.error, 'Invalid request data') }, + { status: 400 } + ) + return executeAshbyUpload( + parsed.data, + request.toolId === 'ashby_upload_resume' ? 'resume' : 'file', + { userId: request.context.userId, requestId: request.requestId, signal: request.signal } + ) +} diff --git a/apps/sim/lib/internal/ashby/operations.test.ts b/apps/sim/lib/internal/ashby/operations.test.ts new file mode 100644 index 00000000000..54b3ec0ee5b --- /dev/null +++ b/apps/sim/lib/internal/ashby/operations.test.ts @@ -0,0 +1,178 @@ +/** + * @vitest-environment node + */ +import { beforeEach, describe, expect, it, vi } from 'vitest' +import { PayloadSizeLimitError } from '@/lib/core/utils/stream-limits' + +const mocks = vi.hoisted(() => ({ + assertToolFileAccess: vi.fn(), + downloadServableFileFromStorage: vi.fn(), + secureFetchWithPinnedIP: vi.fn(), + validateUrlWithDNS: vi.fn(), +})) + +vi.mock('@/app/api/files/authorization', () => ({ + assertToolFileAccess: mocks.assertToolFileAccess, +})) +vi.mock('@/lib/uploads/utils/file-utils.server', () => ({ + downloadServableFileFromStorage: mocks.downloadServableFileFromStorage, +})) +vi.mock('@/lib/core/security/input-validation.server', () => ({ + secureFetchWithPinnedIP: mocks.secureFetchWithPinnedIP, + validateUrlWithDNS: mocks.validateUrlWithDNS, +})) + +import { executeAshbyUpload } from '@/lib/internal/ashby/operations' +import { ashbyUploadInputSchema } from '@/lib/internal/ashby/schema' + +const FILE = { + id: 'file-1', + key: 'workspace/workspace-1/resume.pdf', + name: 'resume.pdf', + size: 4, + type: 'application/pdf', + url: '/api/files/serve?key=resume.pdf', +} + +describe('executeAshbyUpload', () => { + beforeEach(() => { + vi.clearAllMocks() + mocks.assertToolFileAccess.mockResolvedValue(null) + mocks.downloadServableFileFromStorage.mockResolvedValue({ + buffer: Buffer.from('resume'), + contentType: 'application/pdf', + }) + mocks.validateUrlWithDNS.mockResolvedValue({ isValid: true, resolvedIP: '203.0.113.10' }) + mocks.secureFetchWithPinnedIP.mockResolvedValue({ ok: true, status: 204 }) + vi.stubGlobal( + 'fetch', + vi + .fn() + .mockResolvedValueOnce( + Response.json({ + success: true, + results: { + handle: 'handle-1', + url: 'https://uploads.example.com/form', + fields: { key: 'candidate/file' }, + }, + }) + ) + .mockResolvedValueOnce( + Response.json({ success: true, results: { id: 'candidate-1', name: 'Ada' } }) + ) + ) + }) + + it('authorizes storage, pins the presigned URL, uploads bytes, and attaches the handle', async () => { + const response = await executeAshbyUpload( + { + apiKey: 'key', + candidateId: 'candidate-1', + file: FILE, + fileName: null, + onBehalfOfUserId: 'user-1', + }, + 'resume', + { userId: 'sim-user', requestId: 'request-1' } + ) + expect(response.status).toBe(200) + await expect(response.json()).resolves.toMatchObject({ + success: true, + output: { id: 'candidate-1' }, + }) + expect(mocks.assertToolFileAccess).toHaveBeenCalledOnce() + expect(mocks.downloadServableFileFromStorage).toHaveBeenCalledOnce() + expect(mocks.validateUrlWithDNS).toHaveBeenCalledWith( + 'https://uploads.example.com/form', + 'uploadUrl', + 'contentFetch' + ) + expect(mocks.secureFetchWithPinnedIP).toHaveBeenCalledOnce() + const uploadOptions = mocks.secureFetchWithPinnedIP.mock.calls[0][2] + const multipartBody = new TextDecoder().decode(uploadOptions.body as Uint8Array) + expect(multipartBody).toContain('name="Content-Type"') + expect(multipartBody).toContain('application/pdf') + expect(uploadOptions.headers['Content-Length']).toBe(String(uploadOptions.body.byteLength)) + const attachBody = JSON.parse(String(vi.mocked(fetch).mock.calls[1][1]?.body)) + expect(attachBody).toEqual({ candidateId: 'candidate-1', resumeHandle: 'handle-1' }) + }) + + it('returns the storage authorization denial without downloading or uploading', async () => { + mocks.assertToolFileAccess.mockResolvedValue( + Response.json({ success: false, error: 'File not found' }, { status: 404 }) + ) + const response = await executeAshbyUpload( + { + apiKey: 'key', + candidateId: 'candidate-1', + file: FILE, + fileName: null, + onBehalfOfUserId: null, + }, + 'file', + { userId: 'sim-user', requestId: 'request-1' } + ) + expect(response.status).toBe(404) + expect(mocks.downloadServableFileFromStorage).not.toHaveBeenCalled() + expect(fetch).not.toHaveBeenCalled() + }) + + it('parses advanced-mode JSON file inputs and trims the candidate ID at the boundary', () => { + const parsed = ashbyUploadInputSchema.parse({ + apiKey: 'key', + candidateId: ' candidate-1 ', + file: JSON.stringify(FILE), + }) + expect(parsed.candidateId).toBe('candidate-1') + expect(parsed.file).toEqual(FILE) + }) + + it('uploads the servable artifact bytes with their resolved content type', async () => { + mocks.downloadServableFileFromStorage.mockResolvedValueOnce({ + buffer: Buffer.from('compiled-docx'), + contentType: 'application/vnd.openxmlformats-officedocument.wordprocessingml.document', + }) + const response = await executeAshbyUpload( + { + apiKey: 'key', + candidateId: 'candidate-1', + file: FILE, + fileName: 'resume.docx', + onBehalfOfUserId: null, + }, + 'resume', + { userId: 'sim-user', requestId: 'request-1' } + ) + expect(response.status).toBe(200) + const registrationBody = JSON.parse(String(vi.mocked(fetch).mock.calls[0][1]?.body)) + expect(registrationBody).toMatchObject({ + filename: 'resume.docx', + contentType: 'application/vnd.openxmlformats-officedocument.wordprocessingml.document', + contentLength: Buffer.byteLength('compiled-docx'), + }) + }) + + it('returns 413 when the servable file exceeds Ashby upload limits', async () => { + mocks.downloadServableFileFromStorage.mockRejectedValueOnce( + new PayloadSizeLimitError({ + label: 'servable file download', + maxBytes: 25 * 1024 * 1024, + observedBytes: 25 * 1024 * 1024 + 1, + }) + ) + const response = await executeAshbyUpload( + { + apiKey: 'key', + candidateId: 'candidate-1', + file: FILE, + fileName: null, + onBehalfOfUserId: null, + }, + 'file', + { userId: 'sim-user', requestId: 'request-1' } + ) + expect(response.status).toBe(413) + expect(fetch).not.toHaveBeenCalled() + }) +}) diff --git a/apps/sim/lib/internal/ashby/operations.ts b/apps/sim/lib/internal/ashby/operations.ts new file mode 100644 index 00000000000..ee327909f80 --- /dev/null +++ b/apps/sim/lib/internal/ashby/operations.ts @@ -0,0 +1,172 @@ +import { createLogger } from '@sim/logger' +import { getErrorMessage } from '@sim/utils/errors' +import { isRecordLike } from '@sim/utils/object' +import { + secureFetchWithPinnedIP, + validateUrlWithDNS, +} from '@/lib/core/security/input-validation.server' +import { + DEFAULT_MAX_ERROR_BODY_BYTES, + isPayloadSizeLimitError, + readResponseJsonWithLimit, + readResponseTextWithLimit, +} from '@/lib/core/utils/stream-limits' +import type { AshbyUploadInput } from '@/lib/internal/ashby/schema' +import { MAX_BUFFERED_TRANSFER_BYTES } from '@/lib/uploads/shared/types' +import { processFilesToUserFiles } from '@/lib/uploads/utils/file-utils' +import { downloadServableFileFromStorage } from '@/lib/uploads/utils/file-utils.server' +import { docNotReadyResponse } from '@/lib/uploads/utils/servable-file-response' +import { assertToolFileAccess } from '@/app/api/files/authorization' +import { ashbyAuthHeaders, ashbyErrorMessage, mapCandidate } from '@/tools/ashby/utils' + +const logger = createLogger('AshbyFileUpload') +const MAX_ASHBY_JSON_BYTES = 2 * 1024 * 1024 +const MAX_ASHBY_UPLOAD_BYTES = Math.min(MAX_BUFFERED_TRANSFER_BYTES, 25 * 1024 * 1024) + +interface Context { + userId: string + requestId: string + signal?: AbortSignal +} + +function failure(error: string, status: number): Response { + return Response.json({ success: false, error }, { status }) +} + +async function ashbyPost( + apiKey: string, + path: string, + body: Record, + onBehalfOfUserId: string | null | undefined, + signal?: AbortSignal +): Promise { + const response = await fetch(`https://api.ashbyhq.com/${path}`, { + method: 'POST', + headers: ashbyAuthHeaders(apiKey, onBehalfOfUserId ?? undefined), + body: JSON.stringify(body), + signal, + }) + const data = await readResponseJsonWithLimit(response, { + maxBytes: MAX_ASHBY_JSON_BYTES, + label: `Ashby ${path} response`, + signal, + }) + if (!response.ok || !isRecordLike(data) || data.success !== true) { + throw new Error(ashbyErrorMessage(data, `Ashby ${path} failed (HTTP ${response.status})`)) + } + return data.results +} + +export async function executeAshbyUpload( + input: AshbyUploadInput, + kind: 'resume' | 'file', + context: Context +): Promise { + try { + context.signal?.throwIfAborted() + const userFile = processFilesToUserFiles([input.file], context.requestId, logger)[0] + if (!userFile) return failure('Invalid file input', 400) + const denied = await assertToolFileAccess( + userFile.key, + context.userId, + context.requestId, + logger + ) + context.signal?.throwIfAborted() + if (denied) return denied + + const servableFile = await downloadServableFileFromStorage( + userFile, + context.requestId, + logger, + { + maxBytes: MAX_ASHBY_UPLOAD_BYTES, + signal: context.signal, + } + ) + const { buffer } = servableFile + context.signal?.throwIfAborted() + if (buffer.length === 0) return failure('File is empty', 400) + + const filename = input.fileName?.trim() || userFile.name + const contentType = servableFile.contentType || userFile.type || 'application/octet-stream' + const registration = await ashbyPost( + input.apiKey, + 'file.createFileUploadHandle', + { + fileUploadContext: kind === 'resume' ? 'CandidateResume' : 'CandidateFiles', + filename, + contentType, + contentLength: buffer.length, + }, + input.onBehalfOfUserId, + context.signal + ) + if ( + !isRecordLike(registration) || + typeof registration.handle !== 'string' || + typeof registration.url !== 'string' || + !isRecordLike(registration.fields) + ) { + return failure('Ashby returned an invalid file upload handle', 502) + } + + const form = new FormData() + form.append('Content-Type', contentType) + for (const [name, value] of Object.entries(registration.fields)) { + if (typeof value !== 'string') + return failure(`Ashby returned an invalid upload form field: ${name}`, 502) + form.append(name, value) + } + form.append('file', new Blob([new Uint8Array(buffer)], { type: contentType }), filename) + const encoded = new Response(form) + const contentTypeHeader = encoded.headers.get('content-type') + if (!contentTypeHeader) return failure('Failed to encode Ashby upload form', 500) + const multipartBody = new Uint8Array(await encoded.arrayBuffer()) + const validation = await validateUrlWithDNS(registration.url, 'uploadUrl', 'contentFetch') + context.signal?.throwIfAborted() + if (!validation.isValid) return failure(validation.error || 'Invalid Ashby upload URL', 400) + const uploaded = await secureFetchWithPinnedIP(registration.url, validation.resolvedIP, { + profile: 'contentFetch', + method: 'POST', + headers: { + 'Content-Type': contentTypeHeader, + 'Content-Length': String(multipartBody.byteLength), + }, + body: multipartBody, + maxResponseBytes: DEFAULT_MAX_ERROR_BODY_BYTES, + signal: context.signal, + }) + if (!uploaded.ok) { + await readResponseTextWithLimit(uploaded, { + maxBytes: DEFAULT_MAX_ERROR_BODY_BYTES, + label: 'Ashby presigned upload error response', + signal: context.signal, + }).catch(() => '') + return failure(`Failed to upload file bytes to Ashby (HTTP ${uploaded.status})`, 502) + } + + const candidate = await ashbyPost( + input.apiKey, + kind === 'resume' ? 'candidate.uploadResume' : 'candidate.uploadFile', + { + candidateId: input.candidateId.trim(), + [kind === 'resume' ? 'resumeHandle' : 'fileHandle']: registration.handle, + }, + input.onBehalfOfUserId, + context.signal + ) + return Response.json({ success: true, output: mapCandidate(candidate) }) + } catch (error) { + context.signal?.throwIfAborted() + const notReady = docNotReadyResponse(error) + if (notReady) return notReady + logger.error(`[${context.requestId}] Ashby candidate file upload failed`, { + error: getErrorMessage(error), + }) + return failure( + getErrorMessage(error, 'Unknown Ashby upload error'), + isPayloadSizeLimitError(error) ? 413 : 500 + ) + } +} diff --git a/apps/sim/lib/internal/ashby/schema.ts b/apps/sim/lib/internal/ashby/schema.ts new file mode 100644 index 00000000000..4b127098276 --- /dev/null +++ b/apps/sim/lib/internal/ashby/schema.ts @@ -0,0 +1,17 @@ +import { z } from 'zod' +import { FileInputSchema, parseRawFileInput } from '@/lib/uploads/utils/file-schemas' + +export const ashbyUploadInputSchema = z.object({ + apiKey: z.string().min(1, 'Ashby API key is required'), + candidateId: z.string().trim().min(1, 'Candidate ID is required'), + file: FileInputSchema.transform((value, context) => { + const parsed = parseRawFileInput(value) + if (parsed) return parsed + context.addIssue({ code: 'custom', message: 'File must reference an uploaded file' }) + return z.NEVER + }), + fileName: z.string().optional().nullable(), + onBehalfOfUserId: z.string().optional().nullable(), +}) + +export type AshbyUploadInput = z.output diff --git a/apps/sim/lib/internal/azure-data-explorer/client.ts b/apps/sim/lib/internal/azure-data-explorer/client.ts index eaca58026b7..c5c71b957b7 100644 --- a/apps/sim/lib/internal/azure-data-explorer/client.ts +++ b/apps/sim/lib/internal/azure-data-explorer/client.ts @@ -84,6 +84,7 @@ async function fetchAccessToken( const response = await secureFetchWithValidation( `${authority}/${encodeURIComponent(input.tenantId)}/oauth2/token`, { + profile: 'configuredEndpoint', method: 'POST', headers: { 'Content-Type': 'application/x-www-form-urlencoded', @@ -269,6 +270,7 @@ export async function requestAzureDataExplorer( const response = await secureFetchWithValidation( `${clusterUrl.origin}/v1/rest/${input.endpoint}`, { + profile: 'configuredEndpoint', method: 'POST', headers, body: JSON.stringify({ diff --git a/apps/sim/lib/internal/brex/client.test.ts b/apps/sim/lib/internal/brex/client.test.ts index 1c07cfa4b7d..cad850caf7b 100644 --- a/apps/sim/lib/internal/brex/client.test.ts +++ b/apps/sim/lib/internal/brex/client.test.ts @@ -73,6 +73,7 @@ describe('BrexReceiptClient', () => { ) expect(mocks.pinnedFetch).toHaveBeenCalledWith('https://upload.example/file', '52.216.0.1', { + profile: 'contentFetch', method: 'PUT', headers: { 'Content-Length': String(buffer.byteLength) }, body: new Uint8Array(buffer), diff --git a/apps/sim/lib/internal/brex/client.ts b/apps/sim/lib/internal/brex/client.ts index eab3a5b0edf..82f1b0fdca9 100644 --- a/apps/sim/lib/internal/brex/client.ts +++ b/apps/sim/lib/internal/brex/client.ts @@ -71,12 +71,13 @@ export class BrexReceiptClient { async uploadReceipt(uri: string, buffer: Buffer): Promise { this.signal?.throwIfAborted() - const validation = await validateUrlWithDNS(uri, 'uri') + const validation = await validateUrlWithDNS(uri, 'uri', 'contentFetch') this.signal?.throwIfAborted() - if (!validation.isValid || !validation.resolvedIP) { + if (!validation.isValid) { throw new BrexReceiptError('Brex returned an invalid upload URL', 502) } const response = await secureFetchWithPinnedIP(uri, validation.resolvedIP, { + profile: 'contentFetch', method: 'PUT', headers: { 'Content-Length': String(buffer.byteLength) }, body: new Uint8Array(buffer), diff --git a/apps/sim/lib/internal/buffer/operations.ts b/apps/sim/lib/internal/buffer/operations.ts index 71c7f6c0262..644846f7179 100644 --- a/apps/sim/lib/internal/buffer/operations.ts +++ b/apps/sim/lib/internal/buffer/operations.ts @@ -1,11 +1,13 @@ import { createLogger } from '@sim/logger' import { getErrorMessage } from '@sim/utils/errors' +import type { EgressProfile } from '@/lib/core/security/egress/profiles' import { secureFetchWithPinnedIP, validateUrlWithDNS, } from '@/lib/core/security/input-validation.server' import { BufferOperationError } from '@/lib/internal/buffer/errors' import type { BufferCreatePostInput, BufferEditPostInput } from '@/lib/internal/buffer/input' +import { isInternalFileUrl } from '@/lib/uploads/utils/file-utils' import { resolveFileInputToUrl } from '@/lib/uploads/utils/file-utils.server' import { BUFFER_API_URL, @@ -63,19 +65,25 @@ async function resolveMediaKind(args: { mimeType?: string pathOrName: string fileUrl: string + profile: EgressProfile context: BufferOperationContext }): Promise<'image' | 'video' | null> { - const { mimeType, pathOrName, fileUrl, context } = args + const { mimeType, pathOrName, fileUrl, profile, context } = args if (mimeType?.startsWith('video/')) return 'video' if (mimeType?.startsWith('image/')) return 'image' const extensionKind = mediaKindFromExtension(pathOrName) if (extensionKind) return extensionKind + // An uploaded file resolves to a presigned URL against Sim's own storage, + // which on a self-hosted deployment legitimately sits on a private address + // (`configuredEndpoint`); a caller-supplied URL stays content (`contentFetch`) + // so a rebinding host cannot steer this probe onto a private address. try { - const validation = await validateUrlWithDNS(fileUrl, 'media') + const validation = await validateUrlWithDNS(fileUrl, 'media', profile) context.signal?.throwIfAborted() - if (validation.isValid && validation.resolvedIP) { + if (validation.isValid) { const probe = await secureFetchWithPinnedIP(fileUrl, validation.resolvedIP, { + profile, method: 'HEAD', timeout: MEDIA_PROBE_TIMEOUT_MS, signal: context.signal, @@ -101,6 +109,10 @@ async function resolveMediaAsset( context.signal?.throwIfAborted() const media = input.media const isFileInput = typeof media === 'object' + // An uploaded file, or a path that names Sim's own storage, is internal; any + // other string is a caller-supplied URL and stays content. + const mediaProfile: EgressProfile = + isFileInput || isInternalFileUrl(media) ? 'configuredEndpoint' : 'contentFetch' const resolution = await resolveFileInputToUrl({ file: isFileInput ? media : undefined, filePath: isFileInput ? undefined : media, @@ -124,6 +136,7 @@ async function resolveMediaAsset( mimeType: isFileInput ? media.type : undefined, pathOrName: isFileInput ? media.name || '' : media, fileUrl: resolution.fileUrl, + profile: mediaProfile, context, }) if (!kind) { diff --git a/apps/sim/lib/internal/clickhouse/client.test.ts b/apps/sim/lib/internal/clickhouse/client.test.ts index 2c77749162e..7a89177687c 100644 --- a/apps/sim/lib/internal/clickhouse/client.test.ts +++ b/apps/sim/lib/internal/clickhouse/client.test.ts @@ -95,7 +95,7 @@ describe('clickhouseRequest DNS pinning', () => { const [url, , options] = mockSecureFetchWithPinnedIP.mock.calls[0] expect(url).toMatch(/^https:\/\//) - expect(options.allowHttp).toBe(false) + expect(options.profile).toBe('selfHostedService') }) it('allows http for the initial request when secure is false', async () => { @@ -103,7 +103,7 @@ describe('clickhouseRequest DNS pinning', () => { const [url, , options] = mockSecureFetchWithPinnedIP.mock.calls[0] expect(url).toMatch(/^http:\/\//) - expect(options.allowHttp).toBe(true) + expect(options.profile).toBe('selfHostedService') }) it('brackets an unbracketed IPv6 literal when constructing the request URL', async () => { diff --git a/apps/sim/lib/internal/clickhouse/client.ts b/apps/sim/lib/internal/clickhouse/client.ts index cde12f9ff3f..4027509ad1a 100644 --- a/apps/sim/lib/internal/clickhouse/client.ts +++ b/apps/sim/lib/internal/clickhouse/client.ts @@ -64,7 +64,7 @@ export async function requestClickHouse( url.searchParams.set('database', config.database) if (options.readOnly) url.searchParams.set('readonly', '1') - const response = await secureFetchWithPinnedIP(url.toString(), hostValidation.resolvedIP!, { + const response = await secureFetchWithPinnedIP(url.toString(), hostValidation.resolvedIP, { method: 'POST', headers: { 'X-ClickHouse-User': config.username, @@ -74,7 +74,7 @@ export async function requestClickHouse( }, body: statement, timeout: REQUEST_TIMEOUT_MS, - allowHttp: !config.secure, + profile: 'selfHostedService', maxResponseBytes: MAX_JSON_API_RESPONSE_BYTES, redirectPolicy: { mode: 'standard', diff --git a/apps/sim/lib/internal/cloudwatch/client.ts b/apps/sim/lib/internal/cloudwatch/client.ts index 7d6182e709a..f198a8a25c2 100644 --- a/apps/sim/lib/internal/cloudwatch/client.ts +++ b/apps/sim/lib/internal/cloudwatch/client.ts @@ -125,7 +125,7 @@ const MAX_LOG_STREAMS_PAGES = 20 const logger = createLogger('CloudWatchUtils') -interface DescribedLogStream { +export interface DescribedLogStream { logStreamName: string lastEventTimestamp: number | undefined firstEventTimestamp: number | undefined @@ -145,13 +145,25 @@ interface DescribedLogStream { export async function describeLogStreams( client: CloudWatchLogsClient, logGroupName: string, - options?: { prefix?: string; limit?: number }, + options?: { + prefix?: string + limit?: number + nextToken?: string + suppressTruncationLog?: boolean + }, signal?: AbortSignal -): Promise<{ logStreams: DescribedLogStream[] }> { +): Promise<{ + logStreams: DescribedLogStream[] + truncated: boolean + pages: number + nextToken?: string +}> { const hasPrefix = Boolean(options?.prefix) const totalLimit = options?.limit const logStreams: DescribedLogStream[] = [] - let nextToken: string | undefined + let nextToken = options?.nextToken + let pages = 0 + let truncated = false for (let page = 0; page < MAX_LOG_STREAMS_PAGES; page++) { const pageLimit = @@ -165,10 +177,11 @@ export async function describeLogStreams( ? { orderBy: 'LogStreamName', logStreamNamePrefix: options!.prefix } : { orderBy: 'LastEventTime', descending: true }), limit: pageLimit, - ...(nextToken && { nextToken }), + ...(nextToken !== undefined ? { nextToken } : {}), }) const response = await client.send(command, { abortSignal: signal }) + pages = page + 1 for (const ls of response.logStreams ?? []) { logStreams.push({ @@ -185,15 +198,21 @@ export async function describeLogStreams( if (totalLimit !== undefined && logStreams.length >= totalLimit) break if (page === MAX_LOG_STREAMS_PAGES - 1) { - logger.warn( - `DescribeLogStreams hit pagination cap of ${MAX_LOG_STREAMS_PAGES} pages; log stream list may be incomplete`, - { logGroupName } - ) + truncated = true + if (!options?.suppressTruncationLog) { + logger.warn( + `DescribeLogStreams hit pagination cap of ${MAX_LOG_STREAMS_PAGES} pages; log stream list may be incomplete`, + { logGroupName } + ) + } } } return { logStreams: totalLimit !== undefined ? logStreams.slice(0, totalLimit) : logStreams, + truncated, + pages, + ...(nextToken !== undefined ? { nextToken } : {}), } } diff --git a/apps/sim/lib/internal/cloudwatch/execute-tool.ts b/apps/sim/lib/internal/cloudwatch/execute-tool.ts index 8a26ef1cef7..209d76e3e88 100644 --- a/apps/sim/lib/internal/cloudwatch/execute-tool.ts +++ b/apps/sim/lib/internal/cloudwatch/execute-tool.ts @@ -1,9 +1,5 @@ import { toError } from '@sim/utils/errors' import type { AnyApiRouteContract, ContractBody } from '@/lib/api/contracts' -import { - cloudwatchLogGroupsSelectorContract, - cloudwatchLogStreamsSelectorContract, -} from '@/lib/api/contracts/selectors/cloudwatch' import { awsCloudwatchDescribeAlarmHistoryContract } from '@/lib/api/contracts/tools/aws/cloudwatch-describe-alarm-history' import { awsCloudwatchDescribeAlarmsContract } from '@/lib/api/contracts/tools/aws/cloudwatch-describe-alarms' import { awsCloudwatchFilterLogEventsContract } from '@/lib/api/contracts/tools/aws/cloudwatch-filter-log-events' @@ -15,6 +11,10 @@ import { awsCloudwatchPutLogGroupRetentionContract } from '@/lib/api/contracts/t import { awsCloudwatchPutMetricDataContract } from '@/lib/api/contracts/tools/aws/cloudwatch-put-metric-data' import { awsCloudwatchQueryLogsContract } from '@/lib/api/contracts/tools/aws/cloudwatch-query-logs' import { awsCloudwatchUnmuteAlarmContract } from '@/lib/api/contracts/tools/aws/cloudwatch-unmute-alarm' +import { + cloudwatchLogGroupsContract, + cloudwatchLogStreamsContract, +} from '@/lib/api/contracts/tools/cloudwatch' import { CloudWatchInputError, executeCloudwatchDescribeAlarmHistory, @@ -81,7 +81,7 @@ export const executeCloudwatchTool: InternalToolOperationHandler = async ({ ) case 'cloudwatch_describe_log_groups': return executeOperation( - cloudwatchLogGroupsSelectorContract, + cloudwatchLogGroupsContract, input, executeCloudwatchDescribeLogGroups, 'Failed to describe CloudWatch log groups', @@ -89,7 +89,7 @@ export const executeCloudwatchTool: InternalToolOperationHandler = async ({ ) case 'cloudwatch_describe_log_streams': return executeOperation( - cloudwatchLogStreamsSelectorContract, + cloudwatchLogStreamsContract, input, executeCloudwatchDescribeLogStreams, 'Failed to describe CloudWatch log streams', diff --git a/apps/sim/lib/internal/cloudwatch/operations.test.ts b/apps/sim/lib/internal/cloudwatch/operations.test.ts index a4ddbe0229c..c34a7ef6231 100644 --- a/apps/sim/lib/internal/cloudwatch/operations.test.ts +++ b/apps/sim/lib/internal/cloudwatch/operations.test.ts @@ -6,6 +6,7 @@ import { beforeEach, describe, expect, it, vi } from 'vitest' const mocks = vi.hoisted(() => ({ createCloudWatchClient: vi.fn(), createCloudWatchLogsClient: vi.fn(), + describeLogStreams: vi.fn(), destroy: vi.fn(), send: vi.fn(), })) @@ -13,7 +14,7 @@ const mocks = vi.hoisted(() => ({ vi.mock('@/lib/internal/cloudwatch/client', () => ({ createCloudWatchClient: mocks.createCloudWatchClient, createCloudWatchLogsClient: mocks.createCloudWatchLogsClient, - describeLogStreams: vi.fn(), + describeLogStreams: mocks.describeLogStreams, filterLogEvents: vi.fn(), getLogEvents: vi.fn(), pollQueryResults: vi.fn(), @@ -21,6 +22,8 @@ vi.mock('@/lib/internal/cloudwatch/client', () => ({ import { CloudWatchInputError, + executeCloudwatchDescribeLogGroups, + executeCloudwatchDescribeLogStreams, executeCloudwatchGetMetricStatistics, executeCloudwatchListMetrics, } from '@/lib/internal/cloudwatch/operations' @@ -35,6 +38,86 @@ describe('CloudWatch operations', () => { beforeEach(() => { vi.clearAllMocks() mocks.createCloudWatchClient.mockReturnValue({ send: mocks.send, destroy: mocks.destroy }) + mocks.createCloudWatchLogsClient.mockReturnValue({ + send: mocks.send, + destroy: mocks.destroy, + }) + }) + + it('uses the shared bounded listing primitive for log groups', async () => { + const controller = new AbortController() + mocks.send.mockResolvedValue({ + logGroups: [ + { + logGroupName: '/aws/lambda/example', + arn: 'arn:aws:logs:us-east-1:123:log-group:/aws/lambda/example', + storedBytes: 1, + }, + ], + }) + + await expect( + executeCloudwatchDescribeLogGroups({ ...CONNECTION, prefix: '/aws' }, controller.signal) + ).resolves.toEqual({ + success: true, + output: { + logGroups: [ + { + logGroupName: '/aws/lambda/example', + arn: 'arn:aws:logs:us-east-1:123:log-group:/aws/lambda/example', + storedBytes: 1, + retentionInDays: undefined, + creationTime: undefined, + }, + ], + }, + }) + + expect(mocks.send.mock.calls[0]?.[1]).toEqual({ abortSignal: controller.signal }) + expect(mocks.destroy).toHaveBeenCalledOnce() + }) + + it('uses the shared bounded listing primitive for log streams', async () => { + const controller = new AbortController() + mocks.describeLogStreams.mockResolvedValue({ + logStreams: [ + { + logStreamName: 'stream', + lastEventTimestamp: undefined, + firstEventTimestamp: undefined, + creationTime: undefined, + storedBytes: 0, + }, + ], + }) + + await expect( + executeCloudwatchDescribeLogStreams( + { ...CONNECTION, logGroupName: '/aws/lambda/example' }, + controller.signal + ) + ).resolves.toEqual({ + success: true, + output: { + logStreams: [ + { + logStreamName: 'stream', + lastEventTimestamp: undefined, + firstEventTimestamp: undefined, + creationTime: undefined, + storedBytes: 0, + }, + ], + }, + }) + + expect(mocks.describeLogStreams).toHaveBeenCalledWith( + expect.anything(), + '/aws/lambda/example', + { prefix: undefined, limit: undefined }, + controller.signal + ) + expect(mocks.destroy).toHaveBeenCalledOnce() }) it('forwards cancellation across paginated metric requests and destroys the client', async () => { diff --git a/apps/sim/lib/internal/cloudwatch/operations.ts b/apps/sim/lib/internal/cloudwatch/operations.ts index 51082581223..ac578069def 100644 --- a/apps/sim/lib/internal/cloudwatch/operations.ts +++ b/apps/sim/lib/internal/cloudwatch/operations.ts @@ -12,15 +12,10 @@ import { } from '@aws-sdk/client-cloudwatch' import { DeleteRetentionPolicyCommand, - DescribeLogGroupsCommand, PutRetentionPolicyCommand, StartQueryCommand, } from '@aws-sdk/client-cloudwatch-logs' import { createLogger } from '@sim/logger' -import type { - CloudwatchLogGroupsSelectorBody, - CloudwatchLogStreamsSelectorBody, -} from '@/lib/api/contracts/selectors/cloudwatch' import type { AwsCloudwatchDescribeAlarmHistoryBody } from '@/lib/api/contracts/tools/aws/cloudwatch-describe-alarm-history' import type { AwsCloudwatchDescribeAlarmsBody } from '@/lib/api/contracts/tools/aws/cloudwatch-describe-alarms' import type { AwsCloudwatchFilterLogEventsBody } from '@/lib/api/contracts/tools/aws/cloudwatch-filter-log-events' @@ -32,20 +27,22 @@ import type { AwsCloudwatchPutLogGroupRetentionBody } from '@/lib/api/contracts/ import type { AwsCloudwatchPutMetricDataBody } from '@/lib/api/contracts/tools/aws/cloudwatch-put-metric-data' import type { AwsCloudwatchQueryLogsBody } from '@/lib/api/contracts/tools/aws/cloudwatch-query-logs' import type { AwsCloudwatchUnmuteAlarmBody } from '@/lib/api/contracts/tools/aws/cloudwatch-unmute-alarm' +import type { + CloudwatchLogGroupsBody, + CloudwatchLogStreamsBody, +} from '@/lib/api/contracts/tools/cloudwatch' import { createCloudWatchClient, createCloudWatchLogsClient, - describeLogStreams, filterLogEvents, getLogEvents, pollQueryResults, } from '@/lib/internal/cloudwatch/client' +import { listCloudWatchLogGroups, listCloudWatchLogStreams } from '@/tools/cloudwatch/listing' const logger = createLogger('CloudWatchOperations') const ALARM_HISTORY_PAGE_SIZE = 100 const MAX_ALARM_HISTORY_PAGES = 20 -const LOG_GROUPS_PAGE_SIZE = 50 -const MAX_LOG_GROUPS_PAGES = 20 const METRICS_PAGE_SIZE = 500 const MAX_METRICS_PAGES = 20 const NON_IDEMPOTENT_MAX_ATTEMPTS = 1 @@ -165,76 +162,30 @@ export async function executeCloudwatchDescribeAlarms( } export async function executeCloudwatchDescribeLogGroups( - input: CloudwatchLogGroupsSelectorBody, + input: CloudwatchLogGroupsBody, signal?: AbortSignal ) { - const client = createCloudWatchLogsClient(input) - try { - const logGroups: { - logGroupName: string - arn: string - storedBytes: number - retentionInDays: number | undefined - creationTime: number | undefined - }[] = [] - let nextToken: string | undefined - for (let page = 0; page < MAX_LOG_GROUPS_PAGES; page++) { - const pageLimit = - input.limit !== undefined - ? Math.min(LOG_GROUPS_PAGE_SIZE, input.limit - logGroups.length) - : LOG_GROUPS_PAGE_SIZE - const response = await client.send( - new DescribeLogGroupsCommand({ - ...(input.prefix && { logGroupNamePrefix: input.prefix }), - limit: pageLimit, - ...(nextToken && { nextToken }), - }), - { abortSignal: signal } - ) - for (const group of response.logGroups ?? []) { - logGroups.push({ - logGroupName: group.logGroupName ?? '', - arn: group.arn ?? '', - storedBytes: group.storedBytes ?? 0, - retentionInDays: group.retentionInDays, - creationTime: group.creationTime, - }) - } - nextToken = response.nextToken - if (!nextToken || (input.limit !== undefined && logGroups.length >= input.limit)) break - if (page === MAX_LOG_GROUPS_PAGES - 1) { - logger.warn( - `DescribeLogGroups hit pagination cap of ${MAX_LOG_GROUPS_PAGES} pages; log group list may be incomplete` - ) - } - } - return { - success: true, - output: { - logGroups: input.limit !== undefined ? logGroups.slice(0, input.limit) : logGroups, - }, - } - } finally { - client.destroy() - } + const { items: logGroups } = await listCloudWatchLogGroups({ + credentials: input, + prefix: input.prefix, + limit: input.limit, + signal, + }) + return { success: true, output: { logGroups } } } export async function executeCloudwatchDescribeLogStreams( - input: CloudwatchLogStreamsSelectorBody, + input: CloudwatchLogStreamsBody, signal?: AbortSignal ) { - const client = createCloudWatchLogsClient(input) - try { - const result = await describeLogStreams( - client, - input.logGroupName, - { prefix: input.prefix, limit: input.limit }, - signal - ) - return { success: true, output: { logStreams: result.logStreams } } - } finally { - client.destroy() - } + const { items: logStreams } = await listCloudWatchLogStreams({ + credentials: input, + logGroupName: input.logGroupName, + prefix: input.prefix, + limit: input.limit, + signal, + }) + return { success: true, output: { logStreams } } } export async function executeCloudwatchFilterLogEvents( diff --git a/apps/sim/lib/internal/confluence/execute-tool.ts b/apps/sim/lib/internal/confluence/execute-tool.ts index acc85ad3225..64c257aff4b 100644 --- a/apps/sim/lib/internal/confluence/execute-tool.ts +++ b/apps/sim/lib/internal/confluence/execute-tool.ts @@ -28,8 +28,8 @@ import { confluenceListSpacesContract, confluencePageAncestorsContract, confluencePageChildrenContract, + confluencePageContract, confluencePageDescendantsContract, - confluencePageSelectorContract, confluencePagesByLabelContract, confluencePageVersionsContract, confluenceSearchContract, @@ -46,7 +46,7 @@ import { confluenceUpdateSpaceContract, confluenceUploadAttachmentContract, confluenceUserContract, -} from '@/lib/api/contracts/selectors/confluence' +} from '@/lib/api/contracts/tools/confluence' import { ConfluenceOperationError } from '@/lib/internal/confluence/errors' import { type ConfluenceOperationContext, @@ -335,11 +335,7 @@ export const executeConfluenceTool: InternalToolOperationHandler = async (reques case 'confluence_list_spaces': return executeOperation(confluenceListSpacesContract, request, executeConfluenceListSpaces) case 'confluence_retrieve': - return executeOperation( - confluencePageSelectorContract, - request, - executeConfluenceRetrievePage - ) + return executeOperation(confluencePageContract, request, executeConfluenceRetrievePage) case 'confluence_search': return executeOperation(confluenceSearchContract, request, executeConfluenceSearch) case 'confluence_search_in_space': diff --git a/apps/sim/lib/internal/confluence/operations.test.ts b/apps/sim/lib/internal/confluence/operations.test.ts index 460cf35682c..8a1c1d70b31 100644 --- a/apps/sim/lib/internal/confluence/operations.test.ts +++ b/apps/sim/lib/internal/confluence/operations.test.ts @@ -24,6 +24,8 @@ vi.mock('@/lib/uploads/utils/file-utils.server', () => ({ import { ConfluenceOperationError } from '@/lib/internal/confluence/errors' import { executeConfluenceListLabels, + executeConfluenceListPagesInSpace, + executeConfluenceSearchInSpace, executeConfluenceUploadAttachment, } from '@/lib/internal/confluence/operations' @@ -83,6 +85,63 @@ describe('Confluence operations', () => { expect(response.bodyUsed).toBe(true) }) + it.each([ + { selectedValue: 'ENG', expectedCalls: 2 }, + { selectedValue: '12345', expectedCalls: 1 }, + ])( + 'uses numeric space IDs for V2 requests when the selected value is $selectedValue', + async ({ selectedValue, expectedCalls }) => { + const fetchMock = vi.fn(async (request: string | URL | Request) => { + const url = String(request) + if (url.includes('/spaces?')) { + return Response.json({ + results: [{ id: '12345', key: 'ENG', name: 'Engineering', status: 'current' }], + }) + } + return Response.json({ results: [] }) + }) + vi.stubGlobal('fetch', fetchMock) + + await expect( + executeConfluenceListPagesInSpace( + { ...CONNECTION, spaceId: selectedValue, limit: 25 }, + { headers: new Headers(), requestId: 'request-1' } + ) + ).resolves.toEqual({ pages: [], nextCursor: null }) + + expect(fetchMock).toHaveBeenCalledTimes(expectedCalls) + const urls = fetchMock.mock.calls.map(([request]) => String(request)) + expect(urls.at(-1)).toContain('/spaces/12345/pages?limit=25') + if (selectedValue === 'ENG') { + expect(urls[0]).toContain('/spaces?keys=ENG&limit=1&status=current') + } + } + ) + + it('resolves a legacy numeric space value before constructing key-based CQL', async () => { + const fetchMock = vi.fn(async (request: string | URL | Request) => { + const url = String(request) + if (url.includes('/api/v2/spaces/12345')) { + return Response.json({ id: '12345', key: 'ENG', name: 'Engineering' }) + } + return Response.json({ results: [], totalSize: 0 }) + }) + vi.stubGlobal('fetch', fetchMock) + + await expect( + executeConfluenceSearchInSpace( + { ...CONNECTION, spaceKey: '12345', query: 'release notes', limit: 25 }, + { headers: new Headers(), requestId: 'request-1' } + ) + ).resolves.toEqual({ results: [], spaceKey: 'ENG', totalSize: 0 }) + + expect(fetchMock).toHaveBeenCalledTimes(2) + const searchUrl = String(fetchMock.mock.calls[1][0]) + expect(new URL(searchUrl).searchParams.get('cql')).toBe( + 'space = "ENG" AND text ~ "release notes"' + ) + }) + it('fails closed before downloading a stored file without an acting user', async () => { let caught: unknown try { diff --git a/apps/sim/lib/internal/confluence/operations.ts b/apps/sim/lib/internal/confluence/operations.ts index e78940adccd..824f529b244 100644 --- a/apps/sim/lib/internal/confluence/operations.ts +++ b/apps/sim/lib/internal/confluence/operations.ts @@ -41,7 +41,7 @@ import type { ConfluenceUpdateSpaceBody, ConfluenceUploadAttachmentBody, ConfluenceUserBody, -} from '@/lib/api/contracts/selectors/confluence' +} from '@/lib/api/contracts/tools/confluence' import { validateAlphanumericId, validateNumericId, @@ -52,6 +52,7 @@ import { isPayloadSizeLimitError } from '@/lib/core/utils/stream-limits' import { asArray, asObject, + type ConfluenceClient, createConfluenceClient, type JsonObject, nested, @@ -122,6 +123,71 @@ function mappedPage(value: unknown): JsonObject { } } +const NUMERIC_SPACE_ID_PATTERN = /^[1-9][0-9]{0,19}$/ +const SPACE_STATUSES = ['current', 'archived'] as const + +function normalizedConfluenceSpaceKey(value: string): string { + const spaceKey = value.trim() + if (!spaceKey || spaceKey.length > 255 || spaceKey.includes('\0')) { + throw new ConfluenceOperationError('Invalid Confluence space key', 400) + } + return spaceKey +} + +async function findConfluenceSpaceByKey( + client: ConfluenceClient, + spaceKey: string, + signal?: AbortSignal +): Promise { + for (const status of SPACE_STATUSES) { + const query = new URLSearchParams({ keys: spaceKey, limit: '1', status }) + const data = await client.json(client.apiV2(`/spaces?${query}`), {}, signal) + const match = asArray(data.results) + .map(asObject) + .find((space) => space.key === spaceKey) + if (match) return match + } + return null +} + +async function resolveConfluenceSpaceId( + client: ConfluenceClient, + value: string, + signal?: AbortSignal +): Promise { + const spaceIdentifier = value.trim() + if (NUMERIC_SPACE_ID_PATTERN.test(spaceIdentifier)) return spaceIdentifier + + const spaceKey = normalizedConfluenceSpaceKey(spaceIdentifier) + const space = await findConfluenceSpaceByKey(client, spaceKey, signal) + const resolvedId = space?.id + if ( + (typeof resolvedId !== 'string' && typeof resolvedId !== 'number') || + !NUMERIC_SPACE_ID_PATTERN.test(String(resolvedId)) + ) { + throw new ConfluenceOperationError(`Confluence space key "${spaceKey}" was not found`, 404) + } + return String(resolvedId) +} + +async function resolveConfluenceSpaceKey( + client: ConfluenceClient, + value: string, + signal?: AbortSignal +): Promise { + const spaceIdentifier = normalizedConfluenceSpaceKey(value) + if (!NUMERIC_SPACE_ID_PATTERN.test(spaceIdentifier)) return spaceIdentifier + + const space = await client.json(client.apiV2(`/spaces/${spaceIdentifier}`), {}, signal) + if (typeof space.key !== 'string' || !space.key) { + throw new ConfluenceOperationError( + `Confluence space ID "${spaceIdentifier}" did not return a space key`, + 422 + ) + } + return space.key +} + export async function executeConfluenceRetrievePage( input: ConfluencePageBody, context: ConfluenceOperationContext @@ -208,17 +274,11 @@ export async function executeConfluenceCreatePage( input: ConfluenceCreatePageBody, context: ConfluenceOperationContext ) { - if (!/^\d+$/.test(String(input.spaceId))) { - throw new ConfluenceOperationError( - 'Invalid Space ID. The Space ID must be a numeric value, not the space key from the URL. Use the "list" operation to get all spaces with their numeric IDs.', - 400 - ) - } - assertId(input.spaceId, 'spaceId') - if (input.parentId) assertId(input.parentId, 'parentId') const client = await createConfluenceClient(input, context.signal) + const spaceId = await resolveConfluenceSpaceId(client, input.spaceId, context.signal) + if (input.parentId) assertId(input.parentId, 'parentId') const body: JsonObject = { - spaceId: input.spaceId, + spaceId, status: 'current', title: input.title, body: { representation: 'storage', value: input.content }, @@ -853,9 +913,9 @@ export async function executeConfluenceSearchInSpace( input: ConfluenceSearchInSpaceBody, context: ConfluenceOperationContext ) { - assertId(input.spaceKey, 'spaceKey') const client = await createConfluenceClient(input, context.signal) - let cql = `space = "${escapeCql(input.spaceKey)}"` + const spaceKey = await resolveConfluenceSpaceKey(client, input.spaceKey, context.signal) + let cql = `space = "${escapeCql(spaceKey)}"` if (input.query) cql += ` AND text ~ "${escapeCql(input.query)}"` if (input.contentType) cql += ` AND type = "${escapeCql(input.contentType)}"` const query = new URLSearchParams({ cql, limit: cappedLimit(input.limit) }) @@ -873,21 +933,21 @@ export async function executeConfluenceSearchInSpace( lastModified: result.lastModified ?? null, } }) - return { results, spaceKey: input.spaceKey, totalSize: data.totalSize ?? results.length } + return { results, spaceKey, totalSize: data.totalSize ?? results.length } } export async function executeConfluenceListBlogPostsInSpace( input: ConfluenceSpaceBlogPostsBody, context: ConfluenceOperationContext ) { - assertId(input.spaceId, 'spaceId') const client = await createConfluenceClient(input, context.signal) + const spaceId = await resolveConfluenceSpaceId(client, input.spaceId, context.signal) const query = new URLSearchParams({ limit: cappedLimit(input.limit) }) if (input.status) query.set('status', input.status) if (input.bodyFormat) query.set('body-format', input.bodyFormat) if (input.cursor) query.set('cursor', input.cursor) const data = await client.json( - client.apiV2(`/spaces/${input.spaceId}/blogposts?${query}`), + client.apiV2(`/spaces/${spaceId}/blogposts?${query}`), {}, context.signal ) @@ -914,14 +974,14 @@ export async function executeConfluenceListPagesInSpace( input: ConfluenceSpacePagesBody, context: ConfluenceOperationContext ) { - assertId(input.spaceId, 'spaceId') const client = await createConfluenceClient(input, context.signal) + const spaceId = await resolveConfluenceSpaceId(client, input.spaceId, context.signal) const query = new URLSearchParams({ limit: cappedLimit(input.limit) }) if (input.status) query.set('status', input.status) if (input.bodyFormat) query.set('body-format', input.bodyFormat) if (input.cursor) query.set('cursor', input.cursor) const data = await client.json( - client.apiV2(`/spaces/${input.spaceId}/pages?${query}`), + client.apiV2(`/spaces/${spaceId}/pages?${query}`), {}, context.signal ) @@ -938,12 +998,12 @@ export async function executeConfluenceListSpaceLabels( input: ConfluenceSpaceLabelsQuery, context: ConfluenceOperationContext ) { - assertId(input.spaceId, 'spaceId') const client = await createConfluenceClient(input, context.signal) + const spaceId = await resolveConfluenceSpaceId(client, input.spaceId, context.signal) const query = new URLSearchParams({ limit: cappedLimit(input.limit) }) if (input.cursor) query.set('cursor', input.cursor) const data = await client.json( - client.apiV2(`/spaces/${input.spaceId}/labels?${query}`), + client.apiV2(`/spaces/${spaceId}/labels?${query}`), {}, context.signal ) @@ -952,7 +1012,7 @@ export async function executeConfluenceListSpaceLabels( const label = asObject(value) return { id: label.id, name: label.name, prefix: label.prefix || 'global' } }), - spaceId: input.spaceId, + spaceId, nextCursor: nextCursor(data), } } @@ -961,13 +1021,13 @@ export async function executeConfluenceListSpacePermissions( input: ConfluenceSpacePermissionsBody, context: ConfluenceOperationContext ) { - assertId(input.spaceId, 'spaceId') assertCursor(input.cursor) const client = await createConfluenceClient(input, context.signal) + const spaceId = await resolveConfluenceSpaceId(client, input.spaceId, context.signal) const query = new URLSearchParams({ limit: cappedLimit(input.limit) }) if (input.cursor) query.set('cursor', input.cursor) const data = await client.json( - client.apiV2(`/spaces/${input.spaceId}/permissions?${query}`), + client.apiV2(`/spaces/${spaceId}/permissions?${query}`), {}, context.signal ) @@ -984,7 +1044,7 @@ export async function executeConfluenceListSpacePermissions( unlicensedAccess: permission.unlicensedAccess ?? false, } }), - spaceId: input.spaceId, + spaceId, nextCursor: nextCursor(data), } } @@ -1025,10 +1085,11 @@ export async function executeConfluenceCreateBlogPost( throw new ConfluenceOperationError('Invalid create blog post request', 400) } const client = await createConfluenceClient(input, context.signal) + const spaceId = await resolveConfluenceSpaceId(client, input.spaceId, context.signal) const data = await client.json( client.apiV2('/blogposts'), jsonInit('POST', { - spaceId: input.spaceId, + spaceId, status: input.status || 'current', title: input.title, body: { representation: 'storage', value: input.content }, @@ -1123,9 +1184,9 @@ export async function executeConfluenceGetSpace( input: ConfluenceGetSpaceQuery, context: ConfluenceOperationContext ) { - assertId(input.spaceId, 'spaceId') const client = await createConfluenceClient(input, context.signal) - return client.json(client.apiV2(`/spaces/${input.spaceId}`), {}, context.signal) + const spaceId = await resolveConfluenceSpaceId(client, input.spaceId, context.signal) + return client.json(client.apiV2(`/spaces/${spaceId}`), {}, context.signal) } export async function executeConfluenceCreateSpace( @@ -1144,7 +1205,6 @@ export async function executeConfluenceUpdateSpace( input: ConfluenceUpdateSpaceBody, context: ConfluenceOperationContext ) { - assertId(input.spaceId, 'spaceId') if (!input.name && input.description === undefined) { throw new ConfluenceOperationError( 'At least one of name or description is required for update', @@ -1152,7 +1212,8 @@ export async function executeConfluenceUpdateSpace( ) } const client = await createConfluenceClient(input, context.signal) - const current = await client.json(client.apiV2(`/spaces/${input.spaceId}`), {}, context.signal) + const spaceId = await resolveConfluenceSpaceId(client, input.spaceId, context.signal) + const current = await client.json(client.apiV2(`/spaces/${spaceId}`), {}, context.signal) const body: JsonObject = { name: input.name || current.name } if (input.description !== undefined) { body.description = { plain: { value: input.description, representation: 'plain' } } @@ -1168,9 +1229,9 @@ export async function executeConfluenceDeleteSpace( input: ConfluenceDeleteSpaceBody, context: ConfluenceOperationContext ) { - assertId(input.spaceId, 'spaceId') const client = await createConfluenceClient(input, context.signal) - const current = await client.json(client.apiV2(`/spaces/${input.spaceId}`), {}, context.signal) + const spaceId = await resolveConfluenceSpaceId(client, input.spaceId, context.signal) + const current = await client.json(client.apiV2(`/spaces/${spaceId}`), {}, context.signal) const response = await client.fetch( client.rest(`/space/${encodeURIComponent(String(current.key))}`), { method: 'DELETE' }, @@ -1190,7 +1251,7 @@ export async function executeConfluenceDeleteSpace( context.signal?.throwIfAborted() } return { - spaceId: input.spaceId, + spaceId, deleted: true, longTaskId: longTask.id, longTaskStatusLink: nested(longTask, 'links', 'status'), @@ -1228,16 +1289,16 @@ export async function executeConfluenceSpaceProperties( input: ConfluenceSpacePropertiesBody, context: ConfluenceOperationContext ) { - assertId(input.spaceId, 'spaceId') const client = await createConfluenceClient(input, context.signal) - const base = client.apiV2(`/spaces/${input.spaceId}/properties`) + const spaceId = await resolveConfluenceSpaceId(client, input.spaceId, context.signal) + const base = client.apiV2(`/spaces/${spaceId}/properties`) if (input.action === 'delete') { if (!input.propertyId) { throw new ConfluenceOperationError('Property ID is required for delete action', 400) } assertId(input.propertyId, 'propertyId') await client.delete(`${base}/${encodeURIComponent(input.propertyId)}`, context.signal) - return { spaceId: input.spaceId, propertyId: input.propertyId, deleted: true } + return { spaceId, propertyId: input.propertyId, deleted: true } } if (input.action === 'create') { if (!input.key) { @@ -1248,7 +1309,7 @@ export async function executeConfluenceSpaceProperties( jsonInit('POST', { key: input.key, value: input.value ?? {} }), context.signal ) - return { propertyId: data.id, key: data.key, value: data.value ?? null, spaceId: input.spaceId } + return { propertyId: data.id, key: data.key, value: data.value ?? null, spaceId } } assertCursor(input.cursor) const query = new URLSearchParams({ limit: cappedLimit(input.limit) }) @@ -1259,7 +1320,7 @@ export async function executeConfluenceSpaceProperties( const property = asObject(value) return { id: property.id, key: property.key, value: property.value ?? null } }), - spaceId: input.spaceId, + spaceId, nextCursor: nextCursor(data), } } diff --git a/apps/sim/lib/internal/cursor/operations.test.ts b/apps/sim/lib/internal/cursor/operations.test.ts index 1105d9df52c..1eff865fe46 100644 --- a/apps/sim/lib/internal/cursor/operations.test.ts +++ b/apps/sim/lib/internal/cursor/operations.test.ts @@ -45,7 +45,7 @@ describe('downloadCursorArtifact', () => { expect(mocks.secureFetchWithPinnedIP).toHaveBeenCalledWith( 'https://download.example/artifact', '203.0.113.1', - { signal: controller.signal } + { profile: 'contentFetch', signal: controller.signal } ) expect(result.output.file).toEqual({ name: 'index.ts', diff --git a/apps/sim/lib/internal/cursor/operations.ts b/apps/sim/lib/internal/cursor/operations.ts index 2c1ce81b477..3365cd7969d 100644 --- a/apps/sim/lib/internal/cursor/operations.ts +++ b/apps/sim/lib/internal/cursor/operations.ts @@ -65,12 +65,13 @@ export async function downloadCursorArtifact( throw new CursorOperationError('No download URL returned for artifact', 400) } - const validation = await validateUrlWithDNS(downloadUrl, 'downloadUrl') + const validation = await validateUrlWithDNS(downloadUrl, 'downloadUrl', 'contentFetch') context.signal?.throwIfAborted() - if (!validation.isValid || !validation.resolvedIP) { + if (!validation.isValid) { throw new CursorOperationError(validation.error || 'Invalid download URL', 400) } const downloadResponse = await secureFetchWithPinnedIP(downloadUrl, validation.resolvedIP, { + profile: 'contentFetch', signal: context.signal, }) if (!downloadResponse.ok) { diff --git a/apps/sim/lib/internal/enrichment/execute-tool.ts b/apps/sim/lib/internal/enrichment/execute-tool.ts index 0d353357566..232dab84bd7 100644 --- a/apps/sim/lib/internal/enrichment/execute-tool.ts +++ b/apps/sim/lib/internal/enrichment/execute-tool.ts @@ -41,6 +41,7 @@ export const executeEnrichmentTool: InternalToolOperationHandler = async (reques return executeEnrichment(parsed.data, { workspaceId: request.context.workspaceId, + userId: request.context.userId, signal: request.signal, resolvedSecretTraceRegistry: request.context.resolvedSecretTraceRegistry, }) diff --git a/apps/sim/lib/internal/enrichment/operations.ts b/apps/sim/lib/internal/enrichment/operations.ts index ee9f9a1f97d..173d497e744 100644 --- a/apps/sim/lib/internal/enrichment/operations.ts +++ b/apps/sim/lib/internal/enrichment/operations.ts @@ -8,6 +8,8 @@ const logger = createLogger('EnrichmentOperations') export interface EnrichmentOperationContext { workspaceId: string + /** The acting user, so the per-tool permission gate applies to the provider call. */ + userId: string signal?: AbortSignal resolvedSecretTraceRegistry?: ResolvedSecretTraceRegistry } @@ -24,6 +26,7 @@ export async function executeEnrichment( const { result, cost, error, provider } = await runEnrichment(enrichment, input.inputs, { workspaceId: context.workspaceId, + userId: context.userId, signal: context.signal, resolvedSecretTraceRegistry: context.resolvedSecretTraceRegistry, }) diff --git a/apps/sim/lib/internal/extend/client.ts b/apps/sim/lib/internal/extend/client.ts index df11e20eca3..c1ee29b1499 100644 --- a/apps/sim/lib/internal/extend/client.ts +++ b/apps/sim/lib/internal/extend/client.ts @@ -21,15 +21,20 @@ export async function submitExtendParse( signal?: AbortSignal ): Promise> { signal?.throwIfAborted() - const validation = await validateUrlWithDNS(EXTEND_ENDPOINT, 'Extend API URL') + const validation = await validateUrlWithDNS( + EXTEND_ENDPOINT, + 'Extend API URL', + 'configuredEndpoint' + ) signal?.throwIfAborted() - if (!validation.isValid || !validation.resolvedIP) { + if (!validation.isValid) { throw new ExtendOperationError(502, { success: false, error: 'Failed to reach Extend API' }) } let response: Awaited> try { response = await secureFetchWithPinnedIP(EXTEND_ENDPOINT, validation.resolvedIP, { + profile: 'configuredEndpoint', method: 'POST', headers: { 'Content-Type': 'application/json', diff --git a/apps/sim/lib/internal/file/execute-tool.test.ts b/apps/sim/lib/internal/file/execute-tool.test.ts index cadc082eacf..08c8bd0c679 100644 --- a/apps/sim/lib/internal/file/execute-tool.test.ts +++ b/apps/sim/lib/internal/file/execute-tool.test.ts @@ -9,6 +9,8 @@ const mocks = vi.hoisted(() => ({ createPrincipal: vi.fn(), executeManage: vi.fn(), executeParser: vi.fn(), + searchContent: vi.fn(), + getProvenance: vi.fn(), })) vi.mock('@/lib/internal/principals/executor', () => ({ @@ -17,12 +19,27 @@ vi.mock('@/lib/internal/principals/executor', () => ({ vi.mock('@/lib/internal/file/operations', () => ({ executeFileManageOperation: mocks.executeManage, + getFileContentProvenance: mocks.getProvenance, + fileContentJsonResponse: ( + body: Record, + includePrivateProvenance: boolean, + init?: ResponseInit, + provenance?: Record + ) => + Response.json( + includePrivateProvenance ? { ...body, __resolvedSecretTraceProvenance: provenance } : body, + init + ), })) vi.mock('@/lib/internal/file/parser', () => ({ executeFileParserOperation: mocks.executeParser, })) +vi.mock('@/lib/workspace-files/application/search-workspace-file-content', () => ({ + searchWorkspaceFileContent: { execute: mocks.searchContent }, +})) + import { executeFileTool } from '@/lib/internal/file/execute-tool' import type { InternalToolOperationCall } from '@/lib/internal/tool-operations/types' import { WORKSPACE_FILES_DELEGATION_AUDIENCE } from '@/lib/workspace-files/application/authorization' @@ -53,6 +70,26 @@ const BILLING_ATTRIBUTION = { payerSubscription: null, } satisfies BillingAttributionSnapshot +const SEARCH_RESULT = { + results: [{ fileId: 'file-1', lineNumber: 2, text: 'needle' }], + count: 1, + truncated: false, + complete: true, + indexStatus: { + readyFiles: 1, + pendingFiles: 0, + failedFiles: 0, + skippedFiles: 0, + partialFiles: 0, + }, + sources: [ + { + identity: { fileId: 'file-1', key: 'workspace/workspace-1/file.txt' }, + ownerUserId: 'user-1', + }, + ], +} + function request( toolId: string, input: unknown, @@ -92,6 +129,144 @@ describe('executeFileTool', () => { }) mocks.executeManage.mockResolvedValue(Response.json({ success: true })) mocks.executeParser.mockResolvedValue(Response.json({ success: true })) + mocks.searchContent.mockResolvedValue(SEARCH_RESULT) + mocks.getProvenance.mockResolvedValue({ version: 1, complete: true, entries: [] }) + }) + + it('searches with the trusted workspace and delegated executor principal', async () => { + const response = await executeFileTool( + request('file_search', { query: 'needle', maxResults: 25 }) + ) + + expect(response.status).toBe(200) + await expect(response.json()).resolves.toMatchObject({ + success: true, + data: { + results: [{ fileId: 'file-1', lineNumber: 2, text: 'needle' }], + count: 1, + }, + }) + expect(mocks.searchContent).toHaveBeenCalledWith({ + principal: expect.objectContaining({ serviceId: 'executor' }), + input: { + workspaceId: 'workspace-1', + query: 'needle', + mode: 'regex', + maxResults: 25, + signal: undefined, + }, + }) + expect(mocks.executeManage).not.toHaveBeenCalled() + }) + + it('uses the default search cap and aggregates provenance for every matched file', async () => { + const response = await executeFileTool( + request( + 'file_search', + { query: 'needle' }, + { + headers: new Headers({ + 'x-sim-request-private-tool-metadata': 'resolved-secret-provenance-v1', + }), + } + ) + ) + + expect(mocks.searchContent).toHaveBeenCalledWith( + expect.objectContaining({ + input: { workspaceId: 'workspace-1', query: 'needle', mode: 'regex', maxResults: 50 }, + }) + ) + expect(mocks.getProvenance).toHaveBeenCalledWith( + expect.objectContaining({ serviceId: 'executor' }), + 'workspace-1', + expect.arrayContaining([ + expect.objectContaining({ + identity: expect.objectContaining({ fileId: 'file-1' }), + }), + ]), + undefined + ) + const body = await response.json() + expect(body.data.sources).toBeUndefined() + expect(body.__resolvedSecretTraceProvenance).toMatchObject({ complete: true }) + }) + + it.each([ + [{ query: 'ab', maxResults: 50 }, 400], + [{ query: 'abc\0def', maxResults: 50 }, 400], + [{ query: 'needle', maxResults: 201 }, 400], + [{ query: 'needle', maxResults: 0 }, 400], + [{ query: 'needle', mode: 'glob' }, 400], + ])('rejects invalid search input before authorization', async (input, status) => { + const response = await executeFileTool(request('file_search', input)) + + expect(response.status).toBe(status) + expect(mocks.createPrincipal).not.toHaveBeenCalled() + expect(mocks.searchContent).not.toHaveBeenCalled() + }) + + it('forwards an explicitly configured exact-match mode', async () => { + await executeFileTool(request('file_search', { query: 'needle', mode: 'exact' })) + + expect(mocks.searchContent).toHaveBeenCalledWith( + expect.objectContaining({ + input: expect.objectContaining({ query: 'needle', mode: 'exact' }), + }) + ) + }) + + it('does not expose unexpected search infrastructure errors', async () => { + mocks.searchContent.mockRejectedValueOnce(new Error('database host and query details')) + + const response = await executeFileTool(request('file_search', { query: 'needle' })) + + expect(response.status).toBe(500) + await expect(response.json()).resolves.toEqual({ + success: false, + error: 'Failed to search workspace files', + }) + }) + + it('propagates cancellation that arrives while search work is running', async () => { + const controller = new AbortController() + mocks.searchContent.mockImplementationOnce(async () => { + controller.abort(new DOMException('cancelled', 'AbortError')) + return SEARCH_RESULT + }) + + await expect( + executeFileTool(request('file_search', { query: 'needle' }, { signal: controller.signal })) + ).rejects.toMatchObject({ name: 'AbortError' }) + expect(mocks.searchContent).toHaveBeenCalledWith( + expect.objectContaining({ + input: expect.objectContaining({ signal: controller.signal }), + }) + ) + expect(mocks.getProvenance).not.toHaveBeenCalled() + }) + + it('propagates cancellation that arrives while search provenance is loading', async () => { + const controller = new AbortController() + mocks.getProvenance.mockImplementationOnce(async () => { + controller.abort(new DOMException('cancelled', 'AbortError')) + return { version: 1, complete: true, entries: [] } + }) + + await expect( + executeFileTool( + request( + 'file_search', + { query: 'needle' }, + { + signal: controller.signal, + headers: new Headers({ + 'x-sim-request-private-tool-metadata': 'resolved-secret-provenance-v1', + }), + } + ) + ) + ).rejects.toMatchObject({ name: 'AbortError' }) }) it.each(Object.entries(MANAGE_INPUTS))('validates and dispatches %s', async (toolId, input) => { diff --git a/apps/sim/lib/internal/file/execute-tool.ts b/apps/sim/lib/internal/file/execute-tool.ts index 1d70d1f5a0f..3ad2939254f 100644 --- a/apps/sim/lib/internal/file/execute-tool.ts +++ b/apps/sim/lib/internal/file/execute-tool.ts @@ -1,9 +1,19 @@ import { resolvePrincipalAttribution, resolvePrincipalSubject } from '@sim/auth/principal' import { createLogger } from '@sim/logger' -import { getErrorMessage } from '@sim/utils/errors' +import { getErrorMessage, toError } from '@sim/utils/errors' +import { z } from 'zod' import { fileParseContract } from '@/lib/api/contracts/storage-transfer' import { fileManageContract } from '@/lib/api/contracts/tools/file' -import { executeFileManageOperation } from '@/lib/internal/file/operations' +import { asOrchestrationError, statusForOrchestrationError } from '@/lib/core/orchestration/types' +import { + RESOLVED_SECRET_PROVENANCE_METADATA_V1, + requestsPrivateToolMetadata, +} from '@/lib/execution/private-tool-metadata' +import { + executeFileManageOperation, + fileContentJsonResponse, + getFileContentProvenance, +} from '@/lib/internal/file/operations' import { executeFileParserOperation } from '@/lib/internal/file/parser' import { createExecutorPrincipalFromExecutionContext } from '@/lib/internal/principals/executor' import { @@ -14,6 +24,14 @@ import { import { parseInternalToolInput } from '@/lib/internal/tool-operations/parse-input' import type { InternalToolOperationHandler } from '@/lib/internal/tool-operations/types' import { WORKSPACE_FILES_DELEGATION_AUDIENCE } from '@/lib/workspace-files/application/authorization' +import { searchWorkspaceFileContent } from '@/lib/workspace-files/application/search-workspace-file-content' +import { + FILE_SEARCH_DEFAULT_MAX_RESULTS, + FILE_SEARCH_MAX_QUERY_LENGTH, + FILE_SEARCH_MAX_RESULTS, + FILE_SEARCH_MIN_QUERY_LENGTH, +} from '@/lib/workspace-files/search/constants' +import { FILE_SEARCH_MODES } from '@/lib/workspace-files/search/pattern' const logger = createLogger('FileToolExecution') @@ -29,9 +47,27 @@ const FILE_MANAGE_TOOL_IDS = new Set([ 'file_parser_v2', 'file_parser_v3', 'file_read', + 'file_search', 'file_write', ]) +const fileSearchInputSchema = z + .object({ + query: z + .string() + .min(FILE_SEARCH_MIN_QUERY_LENGTH) + .max(FILE_SEARCH_MAX_QUERY_LENGTH) + .refine((query) => !query.includes('\0'), 'Search query cannot contain NUL characters'), + mode: z.enum(FILE_SEARCH_MODES).default('regex'), + maxResults: z + .number() + .int() + .min(1) + .max(FILE_SEARCH_MAX_RESULTS) + .default(FILE_SEARCH_DEFAULT_MAX_RESULTS), + }) + .strict() + export const executeFileTool: InternalToolOperationHandler = async (request) => { request.signal?.throwIfAborted() if (!FILE_MANAGE_TOOL_IDS.has(request.toolId)) { @@ -46,6 +82,14 @@ export const executeFileTool: InternalToolOperationHandler = async (request) => return Response.json({ success: false, error: 'Authentication required' }, { status: 401 }) } + const isSearchTool = request.toolId === 'file_search' + const searchInput = isSearchTool ? fileSearchInputSchema.safeParse(request.input) : null + if (searchInput && !searchInput.success) { + return Response.json( + { success: false, error: searchInput.error.issues[0]?.message ?? 'Invalid search input' }, + { status: 400 } + ) + } const isParserTool = request.toolId === 'file_fetch' || request.toolId === 'file_parser' || @@ -53,15 +97,43 @@ export const executeFileTool: InternalToolOperationHandler = async (request) => request.toolId === 'file_parser_v3' const parserInput = isParserTool ? parseInternalToolInput(fileParseContract, request.input) : null if (parserInput && !parserInput.success) return parserInput.response - const manageInput = isParserTool - ? null - : parseInternalToolInput(fileManageContract, request.input) + const manageInput = + isParserTool || isSearchTool ? null : parseInternalToolInput(fileManageContract, request.input) if (manageInput && !manageInput.success) return manageInput.response try { const principal = await createExecutorPrincipalFromExecutionContext({ context: request.context, audience: WORKSPACE_FILES_DELEGATION_AUDIENCE, }) + if (searchInput) { + request.signal?.throwIfAborted() + const result = await searchWorkspaceFileContent.execute({ + principal, + input: { + workspaceId, + query: searchInput.data.query, + mode: searchInput.data.mode, + maxResults: searchInput.data.maxResults, + signal: request.signal, + }, + }) + request.signal?.throwIfAborted() + const { sources, ...data } = result + const includePrivateProvenance = requestsPrivateToolMetadata( + request.headers, + RESOLVED_SECRET_PROVENANCE_METADATA_V1 + ) + const provenance = includePrivateProvenance + ? await getFileContentProvenance(principal, workspaceId, sources, request.signal) + : undefined + request.signal?.throwIfAborted() + return fileContentJsonResponse( + { success: true, data }, + includePrivateProvenance, + undefined, + provenance + ) + } const { attributedUserId } = resolvePrincipalAttribution(principal, { workspaceBillingOwnerUserId: request.context.billingAttribution?.billedAccountUserId, }) @@ -111,12 +183,24 @@ export const executeFileTool: InternalToolOperationHandler = async (request) => { status: internalToolIdentityFaultStatus(identityFault) } ) } + const orchestrationError = request.toolId === 'file_search' ? asOrchestrationError(error) : null + if (orchestrationError) { + return Response.json( + { success: false, error: orchestrationError.message }, + { status: statusForOrchestrationError(orchestrationError.code) } + ) + } + const isSearchFailure = request.toolId === 'file_search' const message = getErrorMessage(error, 'Unknown error') logger.error('File operation dispatch failed', { - error: message, + error: isSearchFailure ? 'Workspace file search failed' : message, + errorType: isSearchFailure ? toError(error).name : undefined, requestId: request.requestId, toolId: request.toolId, }) - return Response.json({ success: false, error: message }, { status: 500 }) + return Response.json( + { success: false, error: isSearchFailure ? 'Failed to search workspace files' : message }, + { status: 500 } + ) } } diff --git a/apps/sim/lib/internal/file/operations.test.ts b/apps/sim/lib/internal/file/operations.test.ts index 57413837212..deb2f78593f 100644 --- a/apps/sim/lib/internal/file/operations.test.ts +++ b/apps/sim/lib/internal/file/operations.test.ts @@ -164,6 +164,7 @@ vi.mock('@/app/api/files/authorization', () => ({ import { fileManageBodySchema } from '@/lib/api/contracts/tools/file' import { executeFileManageOperation } from '@/lib/internal/file/operations' +import { FileConflictError } from '@/lib/uploads/contexts/workspace' import { createWorkspaceFileDelegatedPrincipal } from '@/lib/workspace-files/application/delegated-principal' async function POST(request: Request): Promise { @@ -333,6 +334,47 @@ describe('file manage operations', () => { scope: { userId: 'user-1', workspaceId: 'workspace-1' }, }, }) + expect(mockGetBoundWorkspaceFileSecretProvenance).toHaveBeenNthCalledWith( + 1, + 'workspace-1', + expect.objectContaining({ fileId: 'file-1', contentUpdatedAt: CONTENT_UPDATED_AT }) + ) + expect(mockGetBoundWorkspaceFileSecretProvenance).toHaveBeenNthCalledWith( + 2, + 'workspace-1', + expect.objectContaining({ fileId: 'file-2', contentUpdatedAt: CONTENT_UPDATED_AT }) + ) + }) + + it('pins resolved file-input provenance to the captured content revision', async () => { + mockResolveWorkspaceFileReference.mockResolvedValue(workspaceFile('file-1')) + mockGetBoundWorkspaceFileSecretProvenance.mockResolvedValue({ + status: 'exact', + entries: [], + }) + + const response = await POST( + createMockRequest( + 'POST', + { + operation: 'content', + workspaceId: 'workspace-1', + fileInput: { + key: 'workspace/workspace-1/file-1.txt', + name: 'file-1.txt', + type: 'text/plain', + size: 6, + }, + }, + PRIVATE_REQUEST_HEADER + ) + ) + + expect(response.status).toBe(200) + expect(mockGetBoundWorkspaceFileSecretProvenance).toHaveBeenCalledWith( + 'workspace-1', + expect.objectContaining({ fileId: 'file-1', contentUpdatedAt: CONTENT_UPDATED_AT }) + ) }) it('stores exact causal provenance from a different user in the actor workspace', async () => { @@ -580,6 +622,210 @@ describe('file manage operations', () => { ) }) + it('replaces the existing file at the target path when overwrite is on', async () => { + const existing = workspaceFile('report') + mockResolveWorkspaceFileReference.mockResolvedValue(existing) + mockUpdateWorkspaceFileContent.mockResolvedValue(existing) + + const response = await POST( + createMockRequest('POST', { + operation: 'write', + workspaceId: 'workspace-1', + fileName: 'report.txt', + content: 'fresh', + overwrite: true, + }) + ) + + expect(response.status).toBe(200) + expect(mockUploadWorkspaceFile).not.toHaveBeenCalled() + expect(mockUpdateWorkspaceFileContent).toHaveBeenCalledWith( + 'workspace-1', + 'report', + 'user-1', + Buffer.from('fresh'), + 'text/plain', + { + expectedUpdatedAt: CONTENT_UPDATED_AT, + secretProvenancePolicy: { mode: 'replace', provenance: { status: 'exact', entries: [] } }, + } + ) + await expect(response.json()).resolves.toMatchObject({ + success: true, + data: { id: 'report', name: 'report.txt' }, + }) + }) + + it('creates the file when overwrite finds nothing at the target path', async () => { + mockResolveWorkspaceFileReference.mockResolvedValue(null) + + const response = await POST( + createMockRequest('POST', { + operation: 'write', + workspaceId: 'workspace-1', + fileName: 'report.txt', + content: 'fresh', + overwrite: true, + }) + ) + + expect(response.status).toBe(200) + expect(mockUpdateWorkspaceFileContent).not.toHaveBeenCalled() + expect(mockUploadWorkspaceFile).toHaveBeenCalledWith( + 'workspace-1', + 'user-1', + Buffer.from('fresh'), + 'report.txt', + 'text/plain', + // Exact, so a path created by a concurrent write conflicts instead of being suffixed. + expect.objectContaining({ exactName: true, folderId: null }) + ) + }) + + it('surfaces a conflict when a concurrent write claims the overwrite path', async () => { + mockResolveWorkspaceFileReference.mockResolvedValue(null) + mockUploadWorkspaceFile.mockRejectedValue(new FileConflictError('report.txt')) + + const response = await POST( + createMockRequest('POST', { + operation: 'write', + workspaceId: 'workspace-1', + fileName: 'report.txt', + content: 'fresh', + overwrite: true, + }) + ) + + expect(response.status).toBe(409) + await expect(response.json()).resolves.toMatchObject({ success: false }) + }) + + it('never overwrites a same-named file resolved outside the target folder', async () => { + mockResolveWorkspaceFileReference.mockResolvedValue({ + ...workspaceFile('report'), + folderId: 'folder-9', + }) + + const response = await POST( + createMockRequest('POST', { + operation: 'write', + workspaceId: 'workspace-1', + fileName: 'report.txt', + content: 'fresh', + overwrite: true, + }) + ) + + expect(response.status).toBe(200) + expect(mockUpdateWorkspaceFileContent).not.toHaveBeenCalled() + expect(mockUploadWorkspaceFile).toHaveBeenCalled() + }) + + it('keeps the suffixing create path when overwrite is off', async () => { + mockResolveWorkspaceFileReference.mockResolvedValue(workspaceFile('report')) + + const response = await POST( + createMockRequest('POST', { + operation: 'write', + workspaceId: 'workspace-1', + fileName: 'report.txt', + content: 'fresh', + }) + ) + + expect(response.status).toBe(200) + expect(mockUpdateWorkspaceFileContent).not.toHaveBeenCalled() + expect(mockUploadWorkspaceFile).toHaveBeenCalledWith( + 'workspace-1', + 'user-1', + Buffer.from('fresh'), + 'report.txt', + 'text/plain', + expect.objectContaining({ exactName: false }) + ) + }) + + it('overwrites an existing file with the bytes of a stored file input', async () => { + const existing = workspaceFile('report') + mockResolveWorkspaceFileReference.mockResolvedValue(existing) + mockUpdateWorkspaceFileContent.mockResolvedValue(existing) + mockGetBoundWorkspaceFileSecretProvenance.mockResolvedValue({ status: 'exact', entries: [] }) + + const response = await POST( + createMockRequest('POST', { + operation: 'write', + workspaceId: 'workspace-1', + fileName: 'report.txt', + fileInput: { + key: 'workspace/workspace-1/source.txt', + name: 'source.txt', + type: 'text/plain', + size: 6, + }, + overwrite: true, + }) + ) + + expect(response.status).toBe(200) + expect(mockUploadWorkspaceFile).not.toHaveBeenCalled() + expect(mockUpdateWorkspaceFileContent).toHaveBeenCalledWith( + 'workspace-1', + 'report', + 'user-1', + Buffer.from('content:source.txt'), + 'text/plain', + expect.objectContaining({ expectedUpdatedAt: CONTENT_UPDATED_AT }) + ) + }) + + it('downgrades provenance when overwriting a file owned by another user', async () => { + const existing = workspaceFile('report', 'other-user') + mockResolveWorkspaceFileReference.mockResolvedValue(existing) + mockUpdateWorkspaceFileContent.mockResolvedValue(existing) + + const response = await POST( + createMockRequest( + 'POST', + { + operation: 'write', + workspaceId: 'workspace-1', + fileName: 'report.txt', + content: 'secret-value', + overwrite: true, + __privateSecretProvenance: { + version: 1, + complete: true, + selections: [ + { + key: 'content', + provenance: { + version: 1, + complete: true, + entries: [{ name: 'TOKEN', encryptedValue: 'encrypted-token' }], + scope: { userId: 'user-1', workspaceId: 'workspace-1' }, + }, + }, + ], + }, + }, + PRIVATE_SECRET_PROVENANCE_HEADER + ) + ) + + expect(response.status).toBe(200) + expect(mockUpdateWorkspaceFileContent).toHaveBeenCalledWith( + 'workspace-1', + 'report', + 'user-1', + Buffer.from('secret-value'), + 'text/plain', + { + expectedUpdatedAt: CONTENT_UPDATED_AT, + secretProvenancePolicy: { mode: 'replace', provenance: { status: 'unknown' } }, + } + ) + }) + it('atomically binds append provenance to the exact predecessor version', async () => { const existing = workspaceFile('file-1') mockResolveWorkspaceFileReference.mockResolvedValue(existing) diff --git a/apps/sim/lib/internal/file/operations.ts b/apps/sim/lib/internal/file/operations.ts index 190592076f8..1fa6e79ff26 100644 --- a/apps/sim/lib/internal/file/operations.ts +++ b/apps/sim/lib/internal/file/operations.ts @@ -12,6 +12,7 @@ import { acquireLock, releaseLock } from '@/lib/core/config/redis' import { OrchestrationError } from '@/lib/core/orchestration/types' import { isPayloadSizeLimitError } from '@/lib/core/utils/stream-limits' import { ensureAbsoluteUrl } from '@/lib/core/utils/urls' +import { isUserFile } from '@/lib/core/utils/user-file' import { durableSecretProvenanceFromPrivateBundle } from '@/lib/execution/durable-secret-provenance' import { inspectPrivateSecretProvenanceRequest, @@ -43,7 +44,7 @@ import { import { getFileExtension, getMimeTypeFromExtension, - inferContextFromKey, + tryInferContextFromKey, } from '@/lib/uploads/utils/file-utils' import { downloadFileFromStorage, @@ -158,6 +159,12 @@ const fileInputToUserFile = (fileInput: unknown) => { if (!fileUrl && !key) return null + // A key this normalizer cannot classify is request input we cannot use, which + // is what `null` already means here — the throwing form would turn a malformed + // client value into a 500 from every operation that normalizes a file input. + const context = key ? tryInferContextFromKey(key) : null + if (key && !context) return null + return { id: key || fileUrl, name: @@ -169,7 +176,9 @@ const fileInputToUserFile = (fileInput: unknown) => { ? record.type.trim() : 'application/octet-stream', key, - context: inferContextFromKey(key), + // Only absent when there is no key at all — an unclassifiable one returned + // above rather than reaching here. + context: context ?? undefined, } } @@ -220,6 +229,14 @@ const MAX_GET_CONTENT_FILE_BYTES = 64 * 1024 * 1024 /** Combined extracted-text cap so the content array stays within the large-value-ref ceiling. */ const MAX_GET_CONTENT_TOTAL_BYTES = 64 * 1024 * 1024 +/** + * Cap on a file stored through `write`'s `fileInput`, pinned to the destination's + * own ceiling. A larger cap here would let a 50–100MB file be downloaded and + * base64-encoded in this process only for `createWorkspaceFile` to reject it, so + * the expensive transfer is refused up front instead. + */ +const MAX_WRITE_FILE_INPUT_BYTES = MAX_WORKSPACE_FILE_CONTENT_BYTES + /** Per-file download cap for the compress operation. */ const MAX_COMPRESS_FILE_BYTES = 100 * 1024 * 1024 /** Combined input cap for the compress operation to bound in-memory archiving. */ @@ -288,12 +305,15 @@ const extractUserFileTextContent = async ( return `[Binary file: ${userFile.name} (${userFile.type || 'application/octet-stream'}, ${buffer.length} bytes). Cannot extract text content.]` } -interface FileContentSource { - file: UserFile +export interface FileContentProvenanceSource { identity?: WorkspaceFileSecretProvenanceIdentity ownerUserId?: string } +interface FileContentSource extends FileContentProvenanceSource { + file: UserFile +} + async function bindSelectedContentFile( principal: Principal, workspaceId: string, @@ -317,16 +337,23 @@ async function bindSelectedContentFile( return { file, - identity: { fileId: metadata.id, key: metadata.key, context: 'workspace' }, + identity: { + fileId: metadata.id, + key: metadata.key, + context: 'workspace', + contentUpdatedAt: metadata.contentUpdatedAt ?? undefined, + }, ownerUserId: metadata.uploadedBy, } } -async function getFileContentProvenance( +export async function getFileContentProvenance( principal: Principal, workspaceId: string, - sources: readonly FileContentSource[] + sources: readonly FileContentProvenanceSource[], + signal?: AbortSignal ): Promise { + signal?.throwIfAborted() const ownerIds = new Set( sources .map((source) => source.ownerUserId) @@ -339,14 +366,20 @@ async function getFileContentProvenance( const accumulator = new ResolvedSecretTraceProvenanceAccumulator(scope) for (const source of sources) { + signal?.throwIfAborted() if (!source.identity || !source.ownerUserId) { accumulator.markIncomplete('file-source-unidentified') continue } const { provenance } = await readWorkspaceFileSecretProvenance.execute({ principal, - input: { fileId: source.identity.fileId, assertedWorkspaceId: workspaceId }, + input: { + fileId: source.identity.fileId, + assertedWorkspaceId: workspaceId, + expectedContentUpdatedAt: source.identity.contentUpdatedAt, + }, }) + signal?.throwIfAborted() /** * `unrecorded` is a more specific `unknown`, and this accumulator has not opted into the * workspace file surface's policy, so it latches exactly as it did before. @@ -451,6 +484,35 @@ function resolveFileWriteSecretProvenance(options: { return { success: true, contentProvenance: content } } +/** + * Resolves the file an overwriting write should replace, or null when nothing exists at the + * target path. The shared reference resolver falls back to a workspace-wide name match, so the + * result is accepted only when it sits at exactly the folder and leaf name being written. + */ +async function resolveWriteOverwriteTarget(options: { + principal: Principal + workspaceId: string + folderId: string | null + folderSegments: string[] + leafName: string +}) { + const { principal, workspaceId, folderId, folderSegments, leafName } = options + let existing: Awaited> + try { + existing = await resolveWorkspaceFileReference({ + principal, + operation: fileOperations.updateContent, + workspaceId, + reference: [...folderSegments, leafName].join('/'), + }) + } catch (error) { + if (error instanceof OrchestrationError && error.code === 'not_found') return null + throw error + } + if ((existing.folderId ?? null) !== folderId || existing.name !== leafName) return null + return existing +} + async function deriveWorkspaceFileSecretProvenance(options: { principal: Principal workspaceId: string @@ -476,7 +538,7 @@ async function deriveWorkspaceFileSecretProvenance(options: { return mergeWorkspaceFileSecretProvenance(...provenances) } -function fileContentJsonResponse( +export function fileContentJsonResponse( body: Record, includePrivateProvenance: boolean, init?: ResponseInit, @@ -698,7 +760,12 @@ export async function executeFileManageOperation( return [ { file: userFile, - identity: { fileId: file.id, key: file.key, context: 'workspace' }, + identity: { + fileId: file.id, + key: file.key, + context: 'workspace', + contentUpdatedAt: file.contentUpdatedAt ?? undefined, + }, ownerUserId: file.uploadedBy, }, ] @@ -742,14 +809,14 @@ export async function executeFileManageOperation( logger.info('File content extracted', { count: contents.length }) const provenance = includePrivateContentProvenance - ? await getFileContentProvenance(principal, workspaceId, sources) + ? await getFileContentProvenance(principal, workspaceId, sources, signal) : undefined return contentResponse({ success: true, data: { contents } }, undefined, provenance) } case 'write': { - const { fileName, content, contentType } = body + const { fileName, content, fileInput, contentType, overwrite } = body signal?.throwIfAborted() const provenanceResolution = resolveFileWriteSecretProvenance({ headers, @@ -763,33 +830,162 @@ export async function executeFileManageOperation( { status: 400 } ) } - const { folderSegments, leafName } = splitWorkspaceFilePath(fileName) + + // Storing an existing file object rather than text: read its bytes under + // the caller's own authorization, then write them unchanged. Base64 so a + // binary payload survives — decoding it as UTF-8 would corrupt it. + let sourceEncoding: 'utf-8' | 'base64' = 'utf-8' + let sourceContent = content ?? '' + let sourceName = fileName + let sourceContentType = contentType + /** + * Copying bytes carries the source's secret lineage, exactly as archiving + * does. Without this the copy would land with no provenance row — the + * "safe" state — and a file the platform had locked as secret-derived + * would be readable again under its new id. + * + * A source with no workspace row resolves to `unknown` rather than empty, + * because nothing durable records what went into it. + */ + let inputProvenance: WorkspaceFileSecretProvenance | undefined + if (fileInput !== undefined && fileInput !== null) { + /** + * Two shapes reach here and only one already identifies a file. A block + * reference, or an id the tool layer resolved through the execution + * index or workspace metadata, arrives carrying `id`/`key`/`url`/`name`. + * The file picker instead stores `{name, path, key, size, type}` with no + * `id` or `url`, which the shared normalizer turns into one — the same + * conversion every other operation in this file applies to its input. + * + * Identity is all that is demanded, deliberately. `size` is never read + * before the download and the download reports the real content type, so + * requiring them would reject an otherwise usable reference over two + * fields nothing depends on. + */ + const sourceFile: UserFile | null = isUserFile(fileInput) + ? { + ...fileInput, + size: fileInput.size ?? 0, + type: fileInput.type ?? 'application/octet-stream', + } + : fileInputToUserFile(fileInput) + if (!sourceFile) { + return Response.json( + { success: false, error: 'fileInput must be a file object' }, + { status: 400 } + ) + } + const denied = await assertOperationFileAccess(sourceFile, context) + if (denied) return denied + + inputProvenance = await deriveWorkspaceFileSecretProvenance({ + principal, + workspaceId, + targetOwnerUserId: userId, + sources: [await bindSelectedContentFile(principal, workspaceId, sourceFile)], + }) + + const downloaded = await downloadServableFileFromStorage(sourceFile, requestId, logger, { + maxBytes: MAX_WRITE_FILE_INPUT_BYTES, + signal, + // A generated document that references other files needs a principal + // to resolve them; without one the resolver can only serve an + // already-published artifact and throws when there is none. + filePrincipal: principal, + }) + sourceEncoding = 'base64' + sourceContent = downloaded.buffer.toString('base64') + sourceName = fileName?.trim() || sourceFile.name + sourceContentType = contentType || downloaded.contentType || sourceFile.type + } + const writeProvenanceSources = [ + provenanceResolution.contentProvenance, + inputProvenance, + ].filter((entry): entry is WorkspaceFileSecretProvenance => entry !== undefined) + // Left undefined when neither side recorded anything, so a plain text + // write still stores no provenance row rather than an empty one. + const writeProvenance = writeProvenanceSources.length + ? mergeWorkspaceFileSecretProvenance(...writeProvenanceSources) + : undefined + + const { folderSegments, leafName } = splitWorkspaceFilePath(sourceName ?? '') await admitCreateWorkspaceFile(principal, workspaceId) const { folderId } = await ensureWorkspaceFileFolderPathOperation.execute({ principal, input: { workspaceId, pathSegments: folderSegments }, }) - const mimeType = contentType || getMimeTypeFromExtension(getFileExtension(leafName)) + const mimeType = sourceContentType || getMimeTypeFromExtension(getFileExtension(leafName)) + + if (overwrite) { + const existing = await resolveWriteOverwriteTarget({ + principal, + workspaceId, + folderId: folderId ?? null, + folderSegments, + leafName, + }) + if (existing) { + // Writing into a file someone else owns must not hand its owner an + // exact, re-resolvable secret lineage, exactly as appending does. + const overwriteProvenance = + writeProvenance?.status === 'exact' && + writeProvenance.entries.length > 0 && + existing.uploadedBy !== userId + ? { status: 'unknown' as const } + : writeProvenance + const { file: overwritten } = await updateWorkspaceFileContent.execute({ + principal, + input: { + fileId: existing.id, + assertedWorkspaceId: workspaceId, + content: sourceContent, + encoding: sourceEncoding, + contentType: mimeType, + provenanceMode: 'replace_empty', + expectedUpdatedAt: existing.contentUpdatedAt ?? undefined, + ...(overwriteProvenance ? { secretProvenance: overwriteProvenance } : {}), + }, + }) + + logger.info('File overwritten', { + fileId: overwritten.id, + name: overwritten.name, + size: overwritten.size, + }) + + return Response.json({ + success: true, + data: { + id: overwritten.id, + name: overwritten.name, + size: overwritten.size, + url: ensureAbsoluteUrl(overwritten.url ?? overwritten.path), + }, + }) + } + } + const result = await createWorkspaceFile.execute({ principal, input: { workspaceId, name: leafName, contentType: mimeType, - content: content ?? '', - encoding: 'utf-8', + content: sourceContent, + encoding: sourceEncoding, folderId, - exactName: false, - ...(provenanceResolution.contentProvenance - ? { secretProvenance: provenanceResolution.contentProvenance } - : {}), + // An overwrite that found no target must land on the exact path or fail. Suffixing + // would silently satisfy the request at the wrong name when a concurrent write + // created that path in between; exactName surfaces the race as a conflict instead. + exactName: Boolean(overwrite), + ...(writeProvenance ? { secretProvenance: writeProvenance } : {}), }, }) - const fileBuffer = Buffer.from(content ?? '', 'utf-8') + const fileBuffer = Buffer.from(sourceContent, sourceEncoding) logger.info('File created', { fileId: result.file.id, - name: fileName, + name: sourceName, size: fileBuffer.length, }) diff --git a/apps/sim/lib/internal/file/parser.test.ts b/apps/sim/lib/internal/file/parser.test.ts index 0b69bfe7332..dc091099550 100644 --- a/apps/sim/lib/internal/file/parser.test.ts +++ b/apps/sim/lib/internal/file/parser.test.ts @@ -3,6 +3,8 @@ * * @vitest-environment node */ + +import { Readable } from 'node:stream' import { authMockFns, createMockRequest, @@ -24,14 +26,15 @@ const { mockGetStorageProvider, mockIsUsingCloudStorage, mockIsSupportedFileType, - mockParseFile, mockParseBuffer, + mockPdfParseBuffer, + mockCreateReadStream, mockFsAccess, mockFsStat, - mockFsReadFile, mockFsWriteFile, mockJoin, actualPath, + mockUploadExecutionFile, mockUploadWorkspaceFile, mockReadWorkspaceFileNameByKey, } = vi.hoisted(() => { @@ -43,17 +46,17 @@ const { mockGetStorageProvider: vi.fn().mockReturnValue('s3'), mockIsUsingCloudStorage: vi.fn().mockReturnValue(true), mockIsSupportedFileType: vi.fn().mockReturnValue(true), - mockParseFile: vi.fn().mockResolvedValue({ - content: 'parsed content', - metadata: { pageCount: 1 }, - }), mockParseBuffer: vi.fn().mockResolvedValue({ content: 'parsed buffer content', metadata: { pageCount: 1 }, }), + mockPdfParseBuffer: vi.fn().mockResolvedValue({ + content: 'parsed PDF content', + metadata: { pageCount: 1 }, + }), + mockCreateReadStream: vi.fn(), mockFsAccess: vi.fn().mockResolvedValue(undefined), mockFsStat: vi.fn().mockImplementation(() => ({ isFile: () => true, size: 17 })), - mockFsReadFile: vi.fn().mockResolvedValue(Buffer.from('test file content')), mockFsWriteFile: vi.fn().mockResolvedValue(undefined), mockJoin: vi.fn((...args: string[]): string => { if (args[0] === '/test/uploads') { @@ -62,6 +65,7 @@ const { return actualPath.join(...args) }), actualPath, + mockUploadExecutionFile: vi.fn(), mockUploadWorkspaceFile: vi .fn() .mockImplementation( @@ -93,10 +97,21 @@ vi.mock('@/lib/uploads', () => ({ vi.mock('@/lib/file-parsers', () => ({ isSupportedFileType: mockIsSupportedFileType, - parseFile: mockParseFile, parseBuffer: mockParseBuffer, })) +vi.mock('node:fs', () => ({ + createReadStream: mockCreateReadStream, +})) + +vi.mock('@/lib/file-parsers/pdf-parser', () => ({ + PdfParser: class { + parseBuffer(...args: Parameters) { + return mockPdfParseBuffer(...args) + } + }, +})) + vi.mock('@/lib/uploads/core/storage-service', () => storageServiceMock) vi.mock('path', () => ({ @@ -118,7 +133,7 @@ vi.mock('@/lib/core/utils/logging', () => ({ })) vi.mock('@/lib/uploads/contexts/execution', () => ({ - uploadExecutionFile: vi.fn(), + uploadExecutionFile: mockUploadExecutionFile, })) vi.mock('@/lib/uploads/contexts/workspace/workspace-file-manager', () => ({ @@ -139,12 +154,10 @@ vi.mock('fs/promises', () => ({ default: { access: mockFsAccess, stat: mockFsStat, - readFile: mockFsReadFile, writeFile: mockFsWriteFile, }, access: mockFsAccess, stat: mockFsStat, - readFile: mockFsReadFile, writeFile: mockFsWriteFile, })) @@ -228,18 +241,27 @@ describe('file parser operation', () => { storageServiceMockFns.mockHasCloudStorage.mockReturnValue(true) storageServiceMockFns.mockDownloadFile.mockResolvedValue(Buffer.from('test file content')) mockFsStat.mockResolvedValue({ isFile: () => true, size: 17 }) - mockFsReadFile.mockResolvedValue(Buffer.from('test file content')) + mockCreateReadStream.mockImplementation(() => Readable.from([Buffer.from('test file content')])) mockIsSupportedFileType.mockReturnValue(true) + mockUploadExecutionFile.mockResolvedValue({ + id: 'file_test', + name: 'report.pdf', + url: '/api/files/serve/execution/report.pdf', + size: 17, + type: 'application/pdf', + key: 'execution/report.pdf', + context: 'execution', + }) mockUploadWorkspaceFile.mockClear() mockReadWorkspaceFileNameByKey.mockResolvedValue({ name: null }) - mockParseFile.mockResolvedValue({ - content: 'parsed content', - metadata: { pageCount: 1 }, - }) mockParseBuffer.mockResolvedValue({ content: 'parsed buffer content', metadata: { pageCount: 1 }, }) + mockPdfParseBuffer.mockResolvedValue({ + content: 'parsed PDF content', + metadata: { pageCount: 1 }, + }) }) afterEach(() => { @@ -346,6 +368,100 @@ describe('file parser operation', () => { expect(data.output.content).toBe('plain text content') }) + it('forwards request cancellation to specialized buffer parsers', async () => { + inputValidationMockFns.mockValidateUrlWithDNS.mockResolvedValue({ + isValid: true, + resolvedIP: '203.0.113.10', + }) + inputValidationMockFns.mockSecureFetchWithPinnedIP.mockResolvedValue( + new Response('office bytes', { + status: 200, + headers: { 'content-type': 'application/octet-stream' }, + }) + ) + const req = createMockRequest('POST', { + filePath: 'https://example.com/report.docx', + }) + + const response = await POST(req) + + expect(response.status).toBe(200) + expect(mockParseBuffer).toHaveBeenCalledWith(expect.any(Buffer), 'docx', { + signal: req.signal, + }) + }) + + it('forwards request cancellation to external PDF parsing', async () => { + inputValidationMockFns.mockValidateUrlWithDNS.mockResolvedValue({ + isValid: true, + resolvedIP: '203.0.113.10', + }) + inputValidationMockFns.mockSecureFetchWithPinnedIP.mockResolvedValue( + new Response('pdf bytes', { + status: 200, + headers: { 'content-type': 'application/pdf' }, + }) + ) + const req = createMockRequest('POST', { + filePath: 'https://example.com/report.pdf', + }) + + const response = await POST(req) + + expect(response.status).toBe(200) + expect(mockPdfParseBuffer).toHaveBeenCalledWith(expect.any(Buffer), { + signal: req.signal, + }) + }) + + it('forwards request cancellation to cloud PDF parsing', async () => { + const req = createMockRequest('POST', { + filePath: '/api/files/serve/execution/workspace-1/workflow-1/execution-1/report.pdf', + }) + + const response = await POST(req) + + expect(response.status).toBe(200) + expect(mockPdfParseBuffer).toHaveBeenCalledWith(expect.any(Buffer), { + signal: req.signal, + }) + }) + + it('parses and uploads one bounded local-file snapshot', async () => { + setupFileApiMocks({ + cloudEnabled: false, + storageProvider: 'local', + authenticated: true, + }) + mockFsStat.mockResolvedValue({ isFile: () => true, size: 3 }) + const req = createMockRequest('POST', { + filePath: 'workspace/report.pdf', + }) + + const response = await POST(req) + + const data = await response.json() + const parsedBuffer = mockParseBuffer.mock.calls[0][0] + + expect(response.status).toBe(200) + expect(data.output).toMatchObject({ + content: 'parsed buffer content', + fileType: 'application/pdf', + size: 17, + }) + expect(mockCreateReadStream).toHaveBeenCalledWith('/test/uploads/workspace/report.pdf') + expect(mockCreateReadStream).toHaveBeenCalledOnce() + expect(mockParseBuffer).toHaveBeenCalledWith(parsedBuffer, 'pdf', { signal: req.signal }) + expect(mockParseBuffer).toHaveBeenCalledOnce() + expect(mockUploadExecutionFile).toHaveBeenCalledWith( + expect.any(Object), + parsedBuffer, + 'report.pdf', + 'application/pdf', + 'test-user-id' + ) + }) + it('should reject parser complexity limits instead of returning raw text', async () => { setupFileApiMocks({ cloudEnabled: true, @@ -647,7 +763,9 @@ describe('file parser operation', () => { expect(response.status).toBe(200) expect(data.success).toBe(false) expect(data.error).toContain('too large') - expect(mockFsReadFile).not.toHaveBeenCalled() + expect(mockCreateReadStream).not.toHaveBeenCalled() + expect(mockParseBuffer).not.toHaveBeenCalled() + expect(mockUploadExecutionFile).not.toHaveBeenCalled() }) it('should process execution file URLs with context query param', async () => { diff --git a/apps/sim/lib/internal/file/parser.ts b/apps/sim/lib/internal/file/parser.ts index 20be1fd610d..f47dc5e749f 100644 --- a/apps/sim/lib/internal/file/parser.ts +++ b/apps/sim/lib/internal/file/parser.ts @@ -1,3 +1,4 @@ +import { createReadStream } from 'node:fs' import { Buffer, isUtf8 } from 'buffer' import { createHash } from 'crypto' import fsPromises from 'fs/promises' @@ -10,12 +11,16 @@ import binaryExtensionsList from 'binary-extensions' import type { ContractBody } from '@/lib/api/contracts' import type { fileParseContract } from '@/lib/api/contracts/storage-transfer' import { sanitizeUrlForLog } from '@/lib/core/utils/logging' -import { assertKnownSizeWithinLimit, isPayloadSizeLimitError } from '@/lib/core/utils/stream-limits' +import { + assertKnownSizeWithinLimit, + isPayloadSizeLimitError, + readNodeStreamToBufferWithLimit, +} from '@/lib/core/utils/stream-limits' import { assertUserFileContentAccess, type ExecutionMaterializationContext, } from '@/lib/execution/payloads/materialization.server' -import { isSupportedFileType, parseFile } from '@/lib/file-parsers' +import { isSupportedFileType, parseBuffer } from '@/lib/file-parsers' import { isFileParserError } from '@/lib/file-parsers/errors' import { isUsingCloudStorage, StorageService } from '@/lib/uploads' import { uploadExecutionFile } from '@/lib/uploads/contexts/execution' @@ -562,7 +567,14 @@ async function handleExternalUrl( let parseResult: ParseResult if (extension === 'pdf') { - parseResult = await handlePdfBuffer(buffer, filename, fileType, url, maxParsedOutputBytes) + parseResult = await handlePdfBuffer( + buffer, + filename, + fileType, + url, + maxParsedOutputBytes, + signal + ) } else if (extension === 'csv') { parseResult = await handleCsvBuffer(buffer, filename, fileType, url, maxParsedOutputBytes) } else if (isSupportedFileType(extension)) { @@ -572,7 +584,8 @@ async function handleExternalUrl( extension, fileType, url, - maxParsedOutputBytes + maxParsedOutputBytes, + signal ) } else { parseResult = handleGenericBuffer(buffer, filename, extension, fileType, maxParsedOutputBytes) @@ -585,6 +598,7 @@ async function handleExternalUrl( return parseResult } catch (error) { + signal?.throwIfAborted() logger.error(`Error handling external URL ${sanitizeUrlForLog(url)}:`, error) if (isPayloadSizeLimitError(error)) { logger.warn('Rejected oversized external file parse payload', { @@ -747,7 +761,8 @@ async function handleCloudFile( filename, fileType, normalizedFilePath, - maxParsedOutputBytes + maxParsedOutputBytes, + signal ) } else if (extension === 'csv') { parseResult = await handleCsvBuffer( @@ -764,7 +779,8 @@ async function handleCloudFile( extension, fileType, normalizedFilePath, - maxParsedOutputBytes + maxParsedOutputBytes, + signal ) } else { parseResult = handleGenericBuffer( @@ -792,6 +808,7 @@ async function handleCloudFile( return parseResult } catch (error) { + signal?.throwIfAborted() logger.error(`Error handling cloud file ${filePath}:`, error) const errorMessage = (error as Error).message @@ -869,13 +886,17 @@ async function handleLocalFile( const stats = await fsPromises.stat(fullPath) assertKnownSizeWithinLimit(stats.size, maxDownloadBytes, 'local file') - const result = await parseFile(fullPath) + const fileBuffer = await readNodeStreamToBufferWithLimit(createReadStream(fullPath), { + maxBytes: maxDownloadBytes, + label: 'local file', + signal, + }) + const extension = path.extname(filename).toLowerCase().substring(1) + const result = await parseBuffer(fileBuffer, extension, { signal }) const content = assertParsedContentWithinLimit(result.content, maxParsedOutputBytes) - const fileBuffer = await fsPromises.readFile(fullPath) signal?.throwIfAborted() const hash = createHash('md5').update(fileBuffer).digest('hex') - const extension = path.extname(filename).toLowerCase().substring(1) const mimeType = fileType || getMimeTypeFromExtension(extension) // Store file in execution storage if executionContext is provided @@ -904,12 +925,13 @@ async function handleLocalFile( userFile, metadata: { fileType: mimeType, - size: stats.size, + size: fileBuffer.length, hash, processingTime: 0, }, } } catch (error) { + signal?.throwIfAborted() logger.error(`Error handling local file ${filePath}:`, error) if (isPayloadSizeLimitError(error)) { logger.warn('Rejected oversized local file parse payload', { @@ -946,12 +968,14 @@ async function handlePdfBuffer( filename: string, fileType?: string, originalPath?: string, - maxParsedOutputBytes?: number + maxParsedOutputBytes?: number, + signal?: AbortSignal ): Promise { try { + signal?.throwIfAborted() logger.info(`Parsing PDF in memory: ${filename}`) - const result = await parseBufferAsPdf(fileBuffer) + const result = await parseBufferAsPdf(fileBuffer, signal) const content = result.content || @@ -970,6 +994,7 @@ async function handlePdfBuffer( }, } } catch (error) { + signal?.throwIfAborted() if (isPayloadSizeLimitError(error)) throw error logger.error('Failed to parse PDF in memory:', error) @@ -1049,7 +1074,8 @@ async function handleGenericTextBuffer( extension: string, fileType?: string, originalPath?: string, - maxParsedOutputBytes?: number + maxParsedOutputBytes?: number, + signal?: AbortSignal ): Promise { try { logger.info(`Parsing text file in memory: ${filename}`) @@ -1058,7 +1084,7 @@ async function handleGenericTextBuffer( const { parseBuffer, isSupportedFileType } = await import('@/lib/file-parsers') if (isSupportedFileType(extension)) { - const result = await parseBuffer(fileBuffer, extension) + const result = await parseBuffer(fileBuffer, extension, { signal }) return { success: true, @@ -1073,6 +1099,7 @@ async function handleGenericTextBuffer( } } } catch (parserError) { + signal?.throwIfAborted() if (isPayloadSizeLimitError(parserError)) throw parserError if (isFileParserError(parserError) && parserError.code === 'complexity_limit') { throw parserError @@ -1145,14 +1172,16 @@ function handleGenericBuffer( /** * Parse a PDF buffer */ -async function parseBufferAsPdf(buffer: Buffer) { +async function parseBufferAsPdf(buffer: Buffer, signal?: AbortSignal) { try { + signal?.throwIfAborted() const { PdfParser } = await import('@/lib/file-parsers/pdf-parser') const parser = new PdfParser() logger.info('Using main PDF parser for buffer') - return await parser.parseBuffer(buffer) + return await parser.parseBuffer(buffer, { signal }) } catch (error) { + signal?.throwIfAborted() throw new Error(`PDF parsing failed: ${(error as Error).message}`) } } diff --git a/apps/sim/lib/internal/github/operations.ts b/apps/sim/lib/internal/github/operations.ts index 49522552564..b30e3941b9a 100644 --- a/apps/sim/lib/internal/github/operations.ts +++ b/apps/sim/lib/internal/github/operations.ts @@ -319,10 +319,11 @@ async function fetchChangedFileContent( ): Promise { if (file.status === 'removed' || !file.raw_url || remainingBytes <= 0) return undefined try { - const validation = await validateUrlWithDNS(file.raw_url, 'rawUrl') + const validation = await validateUrlWithDNS(file.raw_url, 'rawUrl', 'contentFetch') context.signal?.throwIfAborted() - if (!validation.isValid || !validation.resolvedIP) return undefined + if (!validation.isValid) return undefined const response = await secureFetchWithPinnedIP(file.raw_url, validation.resolvedIP, { + profile: 'contentFetch', headers: { Authorization: `Bearer ${apiKey}`, 'X-GitHub-Api-Version': '2022-11-28', @@ -356,13 +357,14 @@ export async function getGitHubLatestCommit( const repo = encodeURIComponent(input.repo) const revision = encodeURIComponent(input.branch || 'HEAD') const commitUrl = `https://api.github.com/repos/${owner}/${repo}/commits/${revision}` - const validation = await validateUrlWithDNS(commitUrl, 'commitUrl') + const validation = await validateUrlWithDNS(commitUrl, 'commitUrl', 'configuredEndpoint') context.signal?.throwIfAborted() - if (!validation.isValid || !validation.resolvedIP) { + if (!validation.isValid) { throw new GitHubOperationError(validation.error || 'Invalid GitHub commit URL', 400) } const response = await secureFetchWithPinnedIP(commitUrl, validation.resolvedIP, { + profile: 'configuredEndpoint', method: 'GET', headers: { Accept: 'application/vnd.github.v3+json', diff --git a/apps/sim/lib/internal/google-drive/client.test.ts b/apps/sim/lib/internal/google-drive/client.test.ts index c2f1f965248..a66d48d2d8a 100644 --- a/apps/sim/lib/internal/google-drive/client.test.ts +++ b/apps/sim/lib/internal/google-drive/client.test.ts @@ -36,7 +36,8 @@ describe('requestGoogleDrive', () => { expect(mocks.validateUrl).toHaveBeenCalledWith( 'https://www.googleapis.com/drive/v3/files/file-1', - 'metadataUrl' + 'metadataUrl', + 'configuredEndpoint' ) expect(mocks.secureFetch).toHaveBeenCalledWith( 'https://www.googleapis.com/drive/v3/files/file-1', diff --git a/apps/sim/lib/internal/google-drive/client.ts b/apps/sim/lib/internal/google-drive/client.ts index 3397fcf1fb6..b7c51619d97 100644 --- a/apps/sim/lib/internal/google-drive/client.ts +++ b/apps/sim/lib/internal/google-drive/client.ts @@ -25,7 +25,7 @@ export async function requestGoogleDrive( options: GoogleDriveRequestOptions ): Promise { options.signal?.throwIfAborted() - const validation = await validateUrlWithDNS(options.url, options.label) + const validation = await validateUrlWithDNS(options.url, options.label, 'configuredEndpoint') options.signal?.throwIfAborted() if (!validation.isValid) { throw new GoogleDriveOperationError(400, { @@ -34,7 +34,8 @@ export async function requestGoogleDrive( }) } - return secureFetchWithPinnedIP(options.url, validation.resolvedIP!, { + return secureFetchWithPinnedIP(options.url, validation.resolvedIP, { + profile: 'configuredEndpoint', method: options.method, headers: { Authorization: `Bearer ${options.accessToken}`, diff --git a/apps/sim/lib/internal/google-slides/operations.ts b/apps/sim/lib/internal/google-slides/operations.ts index 9aa5158533e..4f8041bd9ad 100644 --- a/apps/sim/lib/internal/google-slides/operations.ts +++ b/apps/sim/lib/internal/google-slides/operations.ts @@ -42,9 +42,13 @@ export async function exportGoogleSlidesPresentation( const exportFormat = input.exportFormat ?? 'PDF' const mimeType = FORMAT_TO_MIME[exportFormat] const exportUrl = `https://www.googleapis.com/drive/v3/files/${encodeURIComponent(input.presentationId)}/export?mimeType=${encodeURIComponent(mimeType)}` - const validation = await validateUrlWithDNS(exportUrl, 'googleSlidesExportUrl') + const validation = await validateUrlWithDNS( + exportUrl, + 'googleSlidesExportUrl', + 'configuredEndpoint' + ) context.signal?.throwIfAborted() - if (!validation.isValid || !validation.resolvedIP) { + if (!validation.isValid) { throw new GoogleSlidesOperationError( validation.error || 'Invalid Google Slides export URL', 400 @@ -52,6 +56,7 @@ export async function exportGoogleSlidesPresentation( } const response = await secureFetchWithPinnedIP(exportUrl, validation.resolvedIP, { + profile: 'configuredEndpoint', headers: { Authorization: `Bearer ${input.accessToken}` }, maxResponseBytes: MAX_GOOGLE_SLIDES_EXPORT_BYTES, signal: context.signal, diff --git a/apps/sim/lib/internal/google-vault/operations.test.ts b/apps/sim/lib/internal/google-vault/operations.test.ts index 69f16be89ef..fb7d34b9045 100644 --- a/apps/sim/lib/internal/google-vault/operations.test.ts +++ b/apps/sim/lib/internal/google-vault/operations.test.ts @@ -46,6 +46,7 @@ describe('downloadGoogleVaultExportFile', () => { expect.stringContaining('/storage/v1/b/bucket-1/o/exports%2Fresult.zip?alt=media'), '203.0.113.1', { + profile: 'configuredEndpoint', method: 'GET', headers: { Authorization: 'Bearer token' }, maxResponseBytes: MAX_BUFFERED_TRANSFER_BYTES, diff --git a/apps/sim/lib/internal/google-vault/operations.ts b/apps/sim/lib/internal/google-vault/operations.ts index 14762da9e66..87683ae4b2b 100644 --- a/apps/sim/lib/internal/google-vault/operations.ts +++ b/apps/sim/lib/internal/google-vault/operations.ts @@ -42,9 +42,9 @@ export async function downloadGoogleVaultExportFile( const bucket = encodeURIComponent(input.bucketName) const object = encodeURIComponent(input.objectName) const downloadUrl = `https://storage.googleapis.com/storage/v1/b/${bucket}/o/${object}?alt=media` - const validation = await validateUrlWithDNS(downloadUrl, 'downloadUrl') + const validation = await validateUrlWithDNS(downloadUrl, 'downloadUrl', 'configuredEndpoint') context.signal?.throwIfAborted() - if (!validation.isValid || !validation.resolvedIP) { + if (!validation.isValid) { throw new GoogleVaultOperationError( enhanceGoogleVaultError(validation.error || 'Invalid URL'), 400 @@ -52,6 +52,7 @@ export async function downloadGoogleVaultExportFile( } const response = await secureFetchWithPinnedIP(downloadUrl, validation.resolvedIP, { + profile: 'configuredEndpoint', method: 'GET', headers: { Authorization: `Bearer ${input.accessToken}` }, maxResponseBytes: MAX_BUFFERED_TRANSFER_BYTES, diff --git a/apps/sim/lib/internal/grafana/client.test.ts b/apps/sim/lib/internal/grafana/client.test.ts index e041a9ecb21..a1d098c757a 100644 --- a/apps/sim/lib/internal/grafana/client.test.ts +++ b/apps/sim/lib/internal/grafana/client.test.ts @@ -40,7 +40,8 @@ describe('GrafanaClient', () => { expect(mocks.validateUrlWithDNS).toHaveBeenCalledWith( 'https://grafana.example.com/api/folders/folder-1', - 'baseUrl' + 'baseUrl', + 'configuredEndpoint' ) expect(mocks.secureFetchWithPinnedIP).toHaveBeenCalledWith( 'https://grafana.example.com/api/folders/folder-1', diff --git a/apps/sim/lib/internal/grafana/client.ts b/apps/sim/lib/internal/grafana/client.ts index bccf83c8f95..33a52364dab 100644 --- a/apps/sim/lib/internal/grafana/client.ts +++ b/apps/sim/lib/internal/grafana/client.ts @@ -28,9 +28,9 @@ export class GrafanaClient { ): Promise { this.signal?.throwIfAborted() const url = `${this.baseUrl}${path}` - const validation = await validateUrlWithDNS(url, 'baseUrl') + const validation = await validateUrlWithDNS(url, 'baseUrl', 'configuredEndpoint') this.signal?.throwIfAborted() - if (!validation.isValid || !validation.resolvedIP) { + if (!validation.isValid) { return { success: false, error: `Invalid Grafana baseUrl: ${validation.error}` } } @@ -43,6 +43,7 @@ export class GrafanaClient { if (this.organizationId) headers['X-Grafana-Org-Id'] = this.organizationId const response = await secureFetchWithPinnedIP(url, validation.resolvedIP, { + profile: 'configuredEndpoint', method: options.method, headers, ...(options.body === undefined ? {} : { body: JSON.stringify(options.body) }), diff --git a/apps/sim/lib/internal/image/fetch.ts b/apps/sim/lib/internal/image/fetch.ts index 23d8760d647..40d1786a315 100644 --- a/apps/sim/lib/internal/image/fetch.ts +++ b/apps/sim/lib/internal/image/fetch.ts @@ -32,13 +32,14 @@ export async function fetchRemoteImage( signal?: AbortSignal ): Promise { signal?.throwIfAborted() - const validation = await validateUrlWithDNS(imageUrl, 'imageUrl') - if (!validation.isValid || !validation.resolvedIP) { + const validation = await validateUrlWithDNS(imageUrl, 'imageUrl', 'contentFetch') + if (!validation.isValid) { throw new RemoteImageFetchError(validation.error || 'Invalid image URL', 403) } try { const response = await secureFetchWithPinnedIP(imageUrl, validation.resolvedIP, { + profile: 'contentFetch', method: 'GET', maxResponseBytes: MAX_REMOTE_IMAGE_BYTES, signal, diff --git a/apps/sim/lib/internal/image/operations.test.ts b/apps/sim/lib/internal/image/operations.test.ts index 1ee4c7bef53..bf0d46ee125 100644 --- a/apps/sim/lib/internal/image/operations.test.ts +++ b/apps/sim/lib/internal/image/operations.test.ts @@ -9,10 +9,17 @@ const mocks = vi.hoisted(() => ({ uploadCopilotFile: vi.fn(), uploadExecutionFile: vi.fn(), getFalAICostMetadata: vi.fn(), + validateUrlWithDNS: vi.fn(), + secureFetchWithPinnedIP: vi.fn(), })) vi.stubGlobal('fetch', mocks.fetch) +vi.mock('@/lib/core/security/input-validation.server', () => ({ + validateUrlWithDNS: mocks.validateUrlWithDNS, + secureFetchWithPinnedIP: mocks.secureFetchWithPinnedIP, +})) + vi.mock('@sim/utils/helpers', () => ({ interruptibleSleep: mocks.interruptibleSleep, })) @@ -48,18 +55,26 @@ describe('image operations', () => { vi.stubGlobal('fetch', mocks.fetch) mocks.interruptibleSleep.mockResolvedValue(undefined) mocks.uploadCopilotFile.mockResolvedValue({ url: 'https://sim.test/generated.png' }) + // Content-derived queue URLs are validated and pinned; default to allowed. + mocks.validateUrlWithDNS.mockResolvedValue({ + isValid: true, + resolvedIP: '203.0.113.1', + originalHostname: 'queue.fal.run', + }) }) it('submits a Fal.ai job once and only polls the created job', async () => { const inlineImage = `data:image/png;base64,${Buffer.from('png').toString('base64')}` - mocks.fetch - .mockResolvedValueOnce( - Response.json({ - request_id: 'job-1', - status_url: 'https://queue.fal.run/status/job-1', - response_url: 'https://queue.fal.run/result/job-1', - }) - ) + // The job is created against the fixed public queue host over plain fetch. + mocks.fetch.mockResolvedValueOnce( + Response.json({ + request_id: 'job-1', + status_url: 'https://queue.fal.run/status/job-1', + response_url: 'https://queue.fal.run/result/job-1', + }) + ) + // The response-derived status/result URLs are polled over the guarded path. + mocks.secureFetchWithPinnedIP .mockResolvedValueOnce(Response.json({ status: 'IN_QUEUE' })) .mockResolvedValueOnce(Response.json({ status: 'COMPLETED' })) .mockResolvedValueOnce(Response.json({ images: [{ url: inlineImage }] })) @@ -71,12 +86,10 @@ describe('image operations', () => { expect(response.status).toBe(200) expect((await response.json()).imageUrl).toBe('https://sim.test/generated.png') - const urls = mocks.fetch.mock.calls.map(([url]) => String(url)) - expect(urls.filter((url) => url === 'https://queue.fal.run/fal-ai/nano-banana-2')).toHaveLength( - 1 - ) - expect(urls).toEqual([ + expect(mocks.fetch.mock.calls.map(([url]) => String(url))).toEqual([ 'https://queue.fal.run/fal-ai/nano-banana-2', + ]) + expect(mocks.secureFetchWithPinnedIP.mock.calls.map(([url]) => String(url))).toEqual([ 'https://queue.fal.run/status/job-1', 'https://queue.fal.run/status/job-1', 'https://queue.fal.run/result/job-1', @@ -103,6 +116,7 @@ describe('image operations', () => { ).rejects.toMatchObject({ name: 'AbortError' }) expect(mocks.fetch).toHaveBeenCalledTimes(1) + expect(mocks.secureFetchWithPinnedIP).not.toHaveBeenCalled() expect(mocks.uploadCopilotFile).not.toHaveBeenCalled() }) }) diff --git a/apps/sim/lib/internal/image/operations.ts b/apps/sim/lib/internal/image/operations.ts index 591b4979168..a1af8fd4b66 100644 --- a/apps/sim/lib/internal/image/operations.ts +++ b/apps/sim/lib/internal/image/operations.ts @@ -376,12 +376,13 @@ async function bufferFromImageUrl( } } - const urlValidation = await validateUrlWithDNS(url, 'imageUrl') - if (!urlValidation.isValid || !urlValidation.resolvedIP) { + const urlValidation = await validateUrlWithDNS(url, 'imageUrl', 'contentFetch') + if (!urlValidation.isValid) { throw new Error(urlValidation.error || 'Generated image URL failed validation') } const imageResponse = await secureFetchWithPinnedIP(url, urlValidation.resolvedIP, { + profile: 'contentFetch', method: 'GET', maxResponseBytes: MAX_IMAGE_BYTES, signal, @@ -747,6 +748,15 @@ async function generateWithFalAI( logger.info(`[${requestId}] Fal.ai image request created: ${falRequestId}`) + // `status_url`/`response_url` are read out of the queue response, so they are + // content: guard and pin them so a spoofed response cannot point an + // authenticated poll at an internal address. The status URL is constant across + // the loop, so it is validated once and the pinned address reused. + const statusValidation = await validateUrlWithDNS(statusUrl, 'statusUrl', 'contentFetch') + if (!statusValidation.isValid) { + throw new Error(statusValidation.error) + } + const pollIntervalMs = 3000 const maxAttempts = Math.ceil(getMaxExecutionTimeout() / pollIntervalMs) let attempts = 0 @@ -755,10 +765,12 @@ async function generateWithFalAI( await interruptibleSleep(pollIntervalMs, signal) signal?.throwIfAborted() - const statusResponse = await fetch(statusUrl, { + const statusResponse = await secureFetchWithPinnedIP(statusUrl, statusValidation.resolvedIP, { + profile: 'contentFetch', headers: { Authorization: `Key ${apiKey}`, }, + maxResponseBytes: MAX_IMAGE_JSON_BYTES, signal, }) @@ -787,15 +799,19 @@ async function generateWithFalAI( throw new Error(`Fal.ai generation failed: ${getFalAIErrorMessage(statusError)}`) } - const resultResponse = await fetch( - getStringProperty(statusData, 'response_url') || responseUrl, - { - headers: { - Authorization: `Key ${apiKey}`, - }, - signal, - } - ) + const resultUrl = getStringProperty(statusData, 'response_url') || responseUrl + const resultValidation = await validateUrlWithDNS(resultUrl, 'resultUrl', 'contentFetch') + if (!resultValidation.isValid) { + throw new Error(resultValidation.error) + } + const resultResponse = await secureFetchWithPinnedIP(resultUrl, resultValidation.resolvedIP, { + profile: 'contentFetch', + headers: { + Authorization: `Key ${apiKey}`, + }, + maxResponseBytes: MAX_IMAGE_JSON_BYTES, + signal, + }) if (!resultResponse.ok) { await readResponseTextWithLimit(resultResponse, { diff --git a/apps/sim/lib/internal/jsm/assets.ts b/apps/sim/lib/internal/jsm/assets.ts index 283cced6f08..d31fb4fb456 100644 --- a/apps/sim/lib/internal/jsm/assets.ts +++ b/apps/sim/lib/internal/jsm/assets.ts @@ -9,7 +9,7 @@ import type { jsmObjectTypeAttributesContract, jsmSearchObjectsAqlContract, jsmUpdateObjectContract, -} from '@/lib/api/contracts/selectors/jsm' +} from '@/lib/api/contracts/tools/jsm' import { asArray, createJsmAssetsClient } from '@/lib/internal/jsm/client' import { mapAssetObject } from '@/tools/jsm/utils' diff --git a/apps/sim/lib/internal/jsm/client.ts b/apps/sim/lib/internal/jsm/client.ts index 6a2ada6c6aa..1dfb8df2665 100644 --- a/apps/sim/lib/internal/jsm/client.ts +++ b/apps/sim/lib/internal/jsm/client.ts @@ -1,7 +1,4 @@ -import { - validateAssetsWorkspaceId, - validateJiraCloudId, -} from '@/lib/core/security/input-validation' +import { validateJiraCloudId } from '@/lib/core/security/input-validation' import { JsmOperationError } from '@/lib/internal/jsm/errors' import { getJiraCloudId, parseAtlassianErrorMessage } from '@/tools/jira/utils' import { resolveAssetsContext } from '@/tools/jsm/utils' @@ -175,7 +172,7 @@ export async function createJsmAssetsClient( signal?.throwIfAborted() const cloudId = validateJiraCloudId(context.cloudId, 'cloudId') if (!cloudId.isValid) throw new JsmOperationError(cloudId.error || 'Invalid cloudId', 400) - const workspaceId = validateAssetsWorkspaceId(context.workspaceId, 'workspaceId') + const workspaceId = validateJiraCloudId(context.workspaceId, 'workspaceId') if (!workspaceId.isValid) { throw new JsmOperationError(workspaceId.error || 'Invalid workspaceId', 400) } diff --git a/apps/sim/lib/internal/jsm/execute-tool.ts b/apps/sim/lib/internal/jsm/execute-tool.ts index 340bc738d61..41d1f957c60 100644 --- a/apps/sim/lib/internal/jsm/execute-tool.ts +++ b/apps/sim/lib/internal/jsm/execute-tool.ts @@ -39,7 +39,7 @@ import { jsmTransitionContract, jsmTransitionsContract, jsmUpdateObjectContract, -} from '@/lib/api/contracts/selectors/jsm' +} from '@/lib/api/contracts/tools/jsm' import { executeJsmCreateObject, executeJsmDeleteObject, diff --git a/apps/sim/lib/internal/jsm/forms.ts b/apps/sim/lib/internal/jsm/forms.ts index 9b7efd15d5c..5ef4c6324ee 100644 --- a/apps/sim/lib/internal/jsm/forms.ts +++ b/apps/sim/lib/internal/jsm/forms.ts @@ -12,7 +12,7 @@ import type { JsmReopenFormBody, JsmSaveFormAnswersBody, JsmSubmitFormBody, -} from '@/lib/api/contracts/selectors/jsm' +} from '@/lib/api/contracts/tools/jsm' import { validateJiraCloudId, validateJiraIssueKey } from '@/lib/core/security/input-validation' import { asArray, asObject, createJsmClient, nested } from '@/lib/internal/jsm/client' import { JsmOperationError } from '@/lib/internal/jsm/errors' diff --git a/apps/sim/lib/internal/jsm/operations.test.ts b/apps/sim/lib/internal/jsm/operations.test.ts index f7b89934037..de58cfab7b5 100644 --- a/apps/sim/lib/internal/jsm/operations.test.ts +++ b/apps/sim/lib/internal/jsm/operations.test.ts @@ -45,7 +45,7 @@ vi.mock('@/tools/jsm/utils', () => ({ mapAssetObject: mocks.mapAssetObject })) import { executeJsmSearchObjectsAql } from '@/lib/internal/jsm/assets' import { executeJsmSubmitForm } from '@/lib/internal/jsm/forms' -import { executeJsmCreateRequest, listJsmServiceDeskOptions } from '@/lib/internal/jsm/service-desk' +import { executeJsmCreateRequest } from '@/lib/internal/jsm/service-desk' const BASE = { domain: 'example.atlassian.net', @@ -58,32 +58,6 @@ describe('JSM operations', () => { vi.clearAllMocks() }) - it('drains selector pages using the number of returned rows as the next offset', async () => { - mocks.client.json - .mockResolvedValueOnce({ - values: [{ id: '1', projectName: 'One' }], - _links: { next: 'next' }, - }) - .mockResolvedValueOnce({ values: [{ id: '2', projectName: 'Two' }], isLastPage: true }) - - await expect(listJsmServiceDeskOptions(BASE)).resolves.toEqual([ - { id: '1', name: 'One' }, - { id: '2', name: 'Two' }, - ]) - expect(mocks.client.json).toHaveBeenNthCalledWith( - 1, - 'service:/servicedesk?start=0&limit=100', - {}, - undefined - ) - expect(mocks.client.json).toHaveBeenNthCalledWith( - 2, - 'service:/servicedesk?start=1&limit=100', - {}, - undefined - ) - }) - it('keeps form answers separate from explicitly supplied request field values', async () => { mocks.client.json.mockResolvedValueOnce({ issueKey: 'HELP-1' }) await executeJsmCreateRequest({ diff --git a/apps/sim/lib/internal/jsm/service-desk.ts b/apps/sim/lib/internal/jsm/service-desk.ts index 711bc6b7996..1218e4436a4 100644 --- a/apps/sim/lib/internal/jsm/service-desk.ts +++ b/apps/sim/lib/internal/jsm/service-desk.ts @@ -1,4 +1,3 @@ -import { createLogger } from '@sim/logger' import type { JsmApprovalsBody, JsmCommentBody, @@ -16,7 +15,7 @@ import type { JsmSlaBody, JsmTransitionBody, JsmTransitionsBody, -} from '@/lib/api/contracts/selectors/jsm' +} from '@/lib/api/contracts/tools/jsm' import { validateAlphanumericId, validateEnum, @@ -25,10 +24,6 @@ import { import { asArray, asObject, createJsmClient } from '@/lib/internal/jsm/client' import { JsmOperationError } from '@/lib/internal/jsm/errors' -const logger = createLogger('JsmServiceDeskOperations') -const SELECTOR_PAGE_SIZE = 100 -const SELECTOR_MAX_PAGES = 50 - function validateId(value: string, field: string): void { const validation = validateAlphanumericId(value, field) if (!validation.isValid) throw new JsmOperationError(validation.error || `Invalid ${field}`, 400) @@ -660,58 +655,3 @@ export async function executeJsmAddOrganization(input: JsmOrganizationBody, sign }, } } - -interface SelectorConnectionInput { - domain: string - accessToken: string -} - -async function collectSelectorValues( - input: SelectorConnectionInput, - path: string, - signal?: AbortSignal -): Promise[]> { - const client = await createJsmClient(input, signal) - const values: Record[] = [] - let start = 0 - for (let page = 0; page < SELECTOR_MAX_PAGES; page++) { - signal?.throwIfAborted() - const data = await client.json( - client.service(`${path}?start=${start}&limit=${SELECTOR_PAGE_SIZE}`), - {}, - signal - ) - const pageValues = asArray(data.values).map(asObject) - values.push(...pageValues) - const links = asObject(data._links) - if (data.isLastPage === true || !links.next || pageValues.length === 0) return values - start += pageValues.length - } - logger.warn('JSM selector hit pagination cap; list may be incomplete', { - pages: SELECTOR_MAX_PAGES, - collected: values.length, - path, - }) - return values -} - -export async function listJsmServiceDeskOptions( - input: SelectorConnectionInput, - signal?: AbortSignal -) { - const values = await collectSelectorValues(input, '/servicedesk', signal) - return values.map((value) => ({ id: String(value.id), name: String(value.projectName) })) -} - -export async function listJsmRequestTypeOptions( - input: SelectorConnectionInput & { serviceDeskId: string }, - signal?: AbortSignal -) { - validateId(input.serviceDeskId, 'serviceDeskId') - const values = await collectSelectorValues( - input, - serviceDeskPath(input.serviceDeskId, '/requesttype'), - signal - ) - return values.map((value) => ({ id: String(value.id), name: String(value.name) })) -} diff --git a/apps/sim/lib/internal/jupyter/client.test.ts b/apps/sim/lib/internal/jupyter/client.test.ts index c88f3694337..9f2256bf54e 100644 --- a/apps/sim/lib/internal/jupyter/client.test.ts +++ b/apps/sim/lib/internal/jupyter/client.test.ts @@ -43,7 +43,7 @@ describe('Jupyter client', () => { expect(securityMocks.validateUrlWithDNS).toHaveBeenCalledWith( 'http://jupyter.example.com:8888/base/api/kernels', 'serverUrl', - { allowHttp: true } + 'selfHostedService' ) expect(securityMocks.secureFetchWithPinnedIP).toHaveBeenCalledWith( 'http://jupyter.example.com:8888/base/api/kernels', @@ -55,7 +55,7 @@ describe('Jupyter client', () => { 'Content-Type': 'application/json', }, body: JSON.stringify({ name: 'python3' }), - allowHttp: true, + profile: 'selfHostedService', maxRedirects: 0, maxResponseBytes: 10 * 1024 * 1024, signal: controller.signal, diff --git a/apps/sim/lib/internal/jupyter/client.ts b/apps/sim/lib/internal/jupyter/client.ts index b6f88b02358..3e1b3acc3cb 100644 --- a/apps/sim/lib/internal/jupyter/client.ts +++ b/apps/sim/lib/internal/jupyter/client.ts @@ -43,9 +43,9 @@ export async function requestJupyterApi( } const url = `${base}/api/${input.path}` - const urlValidation = await validateUrlWithDNS(url, 'serverUrl', { allowHttp: true }) + const urlValidation = await validateUrlWithDNS(url, 'serverUrl', 'selfHostedService') signal?.throwIfAborted() - if (!urlValidation.isValid || !urlValidation.resolvedIP) { + if (!urlValidation.isValid) { throw new InvalidJupyterTargetError(`Invalid Jupyter serverUrl: ${urlValidation.error}`) } @@ -57,7 +57,7 @@ export async function requestJupyterApi( ...(hasBody ? { 'Content-Type': 'application/json' } : {}), }, body: hasBody ? JSON.stringify(input.body) : undefined, - allowHttp: true, + profile: 'selfHostedService', maxRedirects: 0, maxResponseBytes: MAX_JSON_API_RESPONSE_BYTES, signal, diff --git a/apps/sim/lib/internal/linq/client.ts b/apps/sim/lib/internal/linq/client.ts index 52dc3359407..7ab0fd5e45f 100644 --- a/apps/sim/lib/internal/linq/client.ts +++ b/apps/sim/lib/internal/linq/client.ts @@ -86,12 +86,13 @@ export async function uploadLinqAttachmentBytes( signal?: AbortSignal ): Promise { signal?.throwIfAborted() - const validation = await validateUrlWithDNS(registration.uploadUrl, 'uploadUrl') + const validation = await validateUrlWithDNS(registration.uploadUrl, 'uploadUrl', 'contentFetch') signal?.throwIfAborted() - if (!validation.isValid || !validation.resolvedIP) { + if (!validation.isValid) { throw new LinqOperationError(validation.error || 'Invalid Linq upload URL', 400) } const response = await secureFetchWithPinnedIP(registration.uploadUrl, validation.resolvedIP, { + profile: 'contentFetch', method: registration.httpMethod, headers: registration.requiredHeaders, body: new Uint8Array(buffer), diff --git a/apps/sim/lib/internal/microsoft-dataverse/client.ts b/apps/sim/lib/internal/microsoft-dataverse/client.ts index 7fc2c067d1d..316c6ce2788 100644 --- a/apps/sim/lib/internal/microsoft-dataverse/client.ts +++ b/apps/sim/lib/internal/microsoft-dataverse/client.ts @@ -19,6 +19,8 @@ export async function uploadDataverseFile( const response = await secureFetchWithValidation( input.uploadUrl, { + // Built in process from the configured `environmentUrl`, not response-derived. + profile: 'configuredEndpoint', method: 'PATCH', headers: { Authorization: `Bearer ${input.accessToken}`, diff --git a/apps/sim/lib/internal/microsoft-word/client.ts b/apps/sim/lib/internal/microsoft-word/client.ts index db123397537..b3948e9f26e 100644 --- a/apps/sim/lib/internal/microsoft-word/client.ts +++ b/apps/sim/lib/internal/microsoft-word/client.ts @@ -1,3 +1,4 @@ +import type { EgressProfile } from '@/lib/core/security/egress/profiles' import { secureFetchWithPinnedIP, validateUrlWithDNS, @@ -55,15 +56,16 @@ export class GraphRequestError extends Error { async function graphFetch( url: string, paramName: string, - options: NonNullable[2]> + options: Omit[2]>, 'profile'>, + profile: EgressProfile = 'configuredEndpoint' ) { options.signal?.throwIfAborted() - const validation = await validateUrlWithDNS(url, paramName) + const validation = await validateUrlWithDNS(url, paramName, profile) options.signal?.throwIfAborted() if (!validation.isValid) { throw new GraphRequestError(validation.error || `Invalid ${paramName}`, 400) } - return secureFetchWithPinnedIP(url, validation.resolvedIP as string, options) + return secureFetchWithPinnedIP(url, validation.resolvedIP, { ...options, profile }) } /** Reads a Graph error body and raises it as a {@link GraphRequestError}. */ @@ -302,7 +304,8 @@ const UPLOAD_FRAGMENT_BYTES = 10 * 1024 * 1024 * * The URL is preauthenticated and on another host; Graph documents that sending * `Authorization` here can itself fail the request with a 401, so no bearer - * token is attached. + * token is attached. It comes out of a Graph response rather than from + * configuration, so it is judged under the `contentFetch` provenance. * * @see https://learn.microsoft.com/en-us/graph/api/driveitem-createuploadsession */ @@ -317,15 +320,22 @@ async function uploadSessionBytes( const end = Math.min(start + UPLOAD_FRAGMENT_BYTES, total) - 1 const fragment = content.subarray(start, end + 1) - const response = await graphFetch(uploadUrl, 'documentUploadUrl', { - method: 'PUT', - headers: { - 'Content-Length': String(fragment.length), - 'Content-Range': `bytes ${start}-${end}/${total}`, + const response = await graphFetch( + uploadUrl, + 'documentUploadSessionUrl', + { + method: 'PUT', + headers: { + 'Content-Length': String(fragment.length), + 'Content-Range': `bytes ${start}-${end}/${total}`, + }, + body: fragment, + signal, }, - body: fragment, - signal, - }) + // The upload URL is named by a Graph response rather than configured, so + // it is judged as content: preauthenticated, public, and https-only. + 'contentFetch' + ) if (response.status === 412 || response.status === 409) { throw documentChangedError() diff --git a/apps/sim/lib/internal/mistral/client.ts b/apps/sim/lib/internal/mistral/client.ts index 593445a037b..8ba528144ad 100644 --- a/apps/sim/lib/internal/mistral/client.ts +++ b/apps/sim/lib/internal/mistral/client.ts @@ -17,9 +17,13 @@ export async function submitMistralOcr( maxResponseBytes = DEFAULT_MAX_RESPONSE_BYTES ): Promise { signal?.throwIfAborted() - const validation = await validateUrlWithDNS(MISTRAL_ENDPOINT, 'Mistral API URL') + const validation = await validateUrlWithDNS( + MISTRAL_ENDPOINT, + 'Mistral API URL', + 'configuredEndpoint' + ) signal?.throwIfAborted() - if (!validation.isValid || !validation.resolvedIP) { + if (!validation.isValid) { throw new MistralOperationError(502, { success: false, error: 'Failed to reach Mistral API', @@ -27,6 +31,7 @@ export async function submitMistralOcr( } const response = await secureFetchWithPinnedIP(MISTRAL_ENDPOINT, validation.resolvedIP, { + profile: 'configuredEndpoint', method: 'POST', headers: { 'Content-Type': 'application/json', diff --git a/apps/sim/lib/internal/mistral/operations.ts b/apps/sim/lib/internal/mistral/operations.ts index 4ced4dab583..910cdaf4a30 100644 --- a/apps/sim/lib/internal/mistral/operations.ts +++ b/apps/sim/lib/internal/mistral/operations.ts @@ -162,7 +162,7 @@ async function buildUrlDocument( }) } else { const { validateUrlWithDNS } = await import('@/lib/core/security/input-validation.server') - const validation = await validateUrlWithDNS(fileUrl, 'filePath') + const validation = await validateUrlWithDNS(fileUrl, 'filePath', 'contentFetch') context.signal?.throwIfAborted() if (!validation.isValid) { throw new MistralOperationError(400, { success: false, error: validation.error }) diff --git a/apps/sim/lib/internal/mysql/client.test.ts b/apps/sim/lib/internal/mysql/client.test.ts index 3ae9faff36e..6894a504469 100644 --- a/apps/sim/lib/internal/mysql/client.test.ts +++ b/apps/sim/lib/internal/mysql/client.test.ts @@ -3,18 +3,26 @@ */ import { beforeEach, describe, expect, it, vi } from 'vitest' -const { mockCreateConnection, mockNetConnect, mockValidateDatabaseHost } = vi.hoisted(() => ({ - mockCreateConnection: vi.fn(), - mockNetConnect: vi.fn(), - mockValidateDatabaseHost: vi.fn(), -})) +const { mockCreateConnection, mockNetConnect, mockTypedParameterNull, mockValidateDatabaseHost } = + vi.hoisted(() => { + class MockTypedParameter {} + return { + mockCreateConnection: vi.fn(), + mockNetConnect: vi.fn(), + mockTypedParameterNull: vi.fn(() => new MockTypedParameter()), + mockValidateDatabaseHost: vi.fn(), + } + }) vi.mock('node:net', () => ({ default: { connect: mockNetConnect }, })) vi.mock('mysql2/promise', () => ({ - default: { createConnection: mockCreateConnection }, + default: { + createConnection: mockCreateConnection, + TypedParameter: { NULL: mockTypedParameterNull }, + }, })) vi.mock('@/lib/core/security/input-validation.server', () => ({ @@ -114,4 +122,53 @@ describe('MySQL client', () => { await expect(execution).rejects.toMatchObject({ name: 'AbortError' }) expect(connection.destroy).toHaveBeenCalledOnce() }) + + it('rejects unsupported bind values before executing the query', async () => { + const connection = { execute: vi.fn(), destroy: vi.fn() } + + await expect(executeMysqlCommand(connection as never, 'SELECT ?', [undefined])).rejects.toThrow( + 'MySQL bind values must contain only supported scalar or structured values' + ) + expect(connection.execute).not.toHaveBeenCalled() + }) + + it.each([ + new Map([['key', 'value']]), + new Set(['value']), + /value/, + Object.assign(Object.create(null) as Record, { key: 'value' }), + ])('rejects non-plain structured bind values: %s', async (value) => { + const connection = { execute: vi.fn(), destroy: vi.fn() } + + await expect(executeMysqlCommand(connection as never, 'SELECT ?', [value])).rejects.toThrow( + 'MySQL bind values must contain only supported scalar or structured values' + ) + expect(connection.execute).not.toHaveBeenCalled() + }) + + it('accepts nested plain structured bind values', async () => { + const result = { affectedRows: 1 } + const values = [{ nested: ['value', 1, true, null] }] + const connection = { + execute: vi.fn().mockResolvedValue([result]), + destroy: vi.fn(), + } + + await expect(executeMysqlCommand(connection as never, 'SELECT ?', values)).resolves.toBe(result) + expect(connection.execute).toHaveBeenCalledWith('SELECT ?', values) + }) + + it('accepts mysql2 typed parameters', async () => { + const result = { affectedRows: 1 } + const typedParameter = mockTypedParameterNull() + const connection = { + execute: vi.fn().mockResolvedValue([result]), + destroy: vi.fn(), + } + + await expect( + executeMysqlCommand(connection as never, 'SELECT ?', [typedParameter]) + ).resolves.toBe(result) + expect(connection.execute).toHaveBeenCalledWith('SELECT ?', [typedParameter]) + }) }) diff --git a/apps/sim/lib/internal/mysql/client.ts b/apps/sim/lib/internal/mysql/client.ts index 6c72f55816e..3fd425bba5c 100644 --- a/apps/sim/lib/internal/mysql/client.ts +++ b/apps/sim/lib/internal/mysql/client.ts @@ -11,6 +11,35 @@ export interface MysqlConnectionConfig { ssl: 'disabled' | 'required' | 'preferred' } +const MYSQL_TYPED_PARAMETER_PROTOTYPE = Object.getPrototypeOf(mysql.TypedParameter.NULL()) + +function isMysqlExecuteValue(value: unknown): value is mysql.ExecuteValues { + if ( + value === null || + ['string', 'number', 'bigint', 'boolean'].includes(typeof value) || + value instanceof Date || + value instanceof Blob || + value instanceof Uint8Array + ) { + return true + } + + if (Array.isArray(value)) return value.every(isMysqlExecuteValue) + if (typeof value !== 'object') return false + const prototype = Object.getPrototypeOf(value) + if (prototype === MYSQL_TYPED_PARAMETER_PROTOTYPE) return true + if (prototype !== Object.prototype) return false + return Object.values(value).every(isMysqlExecuteValue) +} + +function assertMysqlExecuteValues( + values: unknown[] | undefined +): asserts values is mysql.ExecuteValues[] | undefined { + if (values?.some((value) => !isMysqlExecuteValue(value))) { + throw new TypeError('MySQL bind values must contain only supported scalar or structured values') + } +} + export async function createMysqlConnection( config: MysqlConnectionConfig, signal?: AbortSignal @@ -70,6 +99,7 @@ export async function executeMysqlCommand( signal?.addEventListener('abort', destroyConnection, { once: true }) try { + assertMysqlExecuteValues(values) const [result] = await connection.execute(query, values) signal?.throwIfAborted() return result diff --git a/apps/sim/lib/internal/onedrive/operations.ts b/apps/sim/lib/internal/onedrive/operations.ts index 0eaab6fb5f7..c28f6002ab6 100644 --- a/apps/sim/lib/internal/onedrive/operations.ts +++ b/apps/sim/lib/internal/onedrive/operations.ts @@ -104,14 +104,14 @@ async function readGraphJson( async function graphRequest( url: string, - init: Parameters[1], + init: Omit[1], 'profile'>, label: string, signal?: AbortSignal ): Promise { signal?.throwIfAborted() return secureFetchWithValidation( url, - { ...init, maxResponseBytes: MAX_GRAPH_JSON_BYTES, signal }, + { ...init, profile: 'configuredEndpoint', maxResponseBytes: MAX_GRAPH_JSON_BYTES, signal }, label ) } @@ -427,12 +427,13 @@ async function fetchGraph( maxResponseBytes: number, signal?: AbortSignal ) { - const validation = await validateUrlWithDNS(url, label) + const validation = await validateUrlWithDNS(url, label, 'contentFetch') signal?.throwIfAborted() - if (!validation.isValid || !validation.resolvedIP) { + if (!validation.isValid) { throw new OneDriveOperationError(validation.error || `Invalid ${label}`, 400) } return secureFetchWithPinnedIP(url, validation.resolvedIP, { + profile: 'contentFetch', headers: { Authorization: `Bearer ${accessToken}` }, maxResponseBytes, signal, diff --git a/apps/sim/lib/internal/onepassword/client.test.ts b/apps/sim/lib/internal/onepassword/client.test.ts index 1dda28e2e38..ef9f7805ee2 100644 --- a/apps/sim/lib/internal/onepassword/client.test.ts +++ b/apps/sim/lib/internal/onepassword/client.test.ts @@ -4,9 +4,10 @@ import { resetEnvFlagsMock, setEnvFlags } from '@sim/testing' import { afterAll, beforeEach, describe, expect, it, vi } from 'vitest' -const { mockDnsLookup, mockSecureFetch } = vi.hoisted(() => ({ +const { mockDnsLookup, mockSecureFetch, mockValidateUrlWithDNS } = vi.hoisted(() => ({ mockDnsLookup: vi.fn(), mockSecureFetch: vi.fn(), + mockValidateUrlWithDNS: vi.fn(), })) vi.mock('dns/promises', () => ({ @@ -15,6 +16,7 @@ vi.mock('dns/promises', () => ({ vi.mock('@/lib/core/security/input-validation.server', () => ({ MAX_JSON_API_RESPONSE_BYTES: 10 * 1024 * 1024, secureFetchWithPinnedIP: mockSecureFetch, + validateUrlWithDNS: mockValidateUrlWithDNS, })) import { connectRequest, validateConnectServerUrl } from '@/lib/internal/onepassword/client' @@ -27,100 +29,31 @@ describe('validateConnectServerUrl', () => { setEnvFlags({ isHosted: false }) }) - it('rejects a non-URL string', async () => { - await expect(validateConnectServerUrl('not a url')).rejects.toThrow('is not a valid URL') - }) - - describe('hosted deployment', () => { - beforeEach(() => { - setEnvFlags({ isHosted: true }) - }) - - it.each([ - ['loopback', 'http://127.0.0.1:8080'], - ['RFC1918 10.x', 'http://10.0.0.5'], - ['RFC1918 192.168.x', 'http://192.168.1.1:8443'], - ['RFC1918 172.16.x', 'http://172.16.0.9'], - ['link-local metadata', 'http://169.254.169.254'], - ['IPv4-mapped IPv6 private', 'http://[::ffff:10.0.0.1]'], - ['IPv6 loopback', 'http://[::1]'], - ])('blocks %s', async (_label, url) => { - await expect(validateConnectServerUrl(url)).rejects.toThrow( - 'cannot point to a private or reserved IP address' - ) - }) - - it('allows a public IP literal', async () => { - await expect(validateConnectServerUrl('https://8.8.8.8')).resolves.toBe('8.8.8.8') - }) - - it('blocks a hostname that resolves to a private IP', async () => { - mockDnsLookup.mockResolvedValue([{ address: '10.1.2.3', family: 4 }]) - await expect(validateConnectServerUrl('https://connect.internal')).rejects.toThrow( - 'cannot point to a private or reserved IP address' - ) - }) + it('delegates to the egress guard as a self-hosted service', async () => { + mockValidateUrlWithDNS.mockResolvedValue({ isValid: true, resolvedIP: '10.0.0.9' }) - it('allows a hostname that resolves to a public IP', async () => { - mockDnsLookup.mockResolvedValue([{ address: '93.184.216.34', family: 4 }]) - await expect(validateConnectServerUrl('https://connect.example.com')).resolves.toBe( - '93.184.216.34' - ) - }) - - it('prefers the IPv4 address for a dual-stack host (avoids unreachable IPv6 pin)', async () => { - mockDnsLookup.mockResolvedValue([ - { address: '2606:4700::6810:85e5', family: 6 }, - { address: '93.184.216.34', family: 4 }, - ]) - await expect(validateConnectServerUrl('https://connect.example.com')).resolves.toBe( - '93.184.216.34' - ) - }) + await expect(validateConnectServerUrl('http://connect.internal:8080')).resolves.toBe('10.0.0.9') - it('pins the sole IPv6 address for an IPv6-only host', async () => { - mockDnsLookup.mockResolvedValue([{ address: '2606:4700::6810:85e5', family: 6 }]) - await expect(validateConnectServerUrl('https://connect.example.com')).resolves.toBe( - '2606:4700::6810:85e5' - ) - }) + // The profile is the whole policy decision: Connect is ordinarily deployed + // inside a network, on plain HTTP, on an arbitrary port. Which addresses that + // permits is the guard's contract, covered by its own tests rather than + // restated here — this file used to carry a copy of them alongside a copy of + // the implementation. + expect(mockValidateUrlWithDNS).toHaveBeenCalledWith( + 'http://connect.internal:8080', + '1Password server URL', + 'selfHostedService' + ) }) - describe('self-hosted deployment', () => { - beforeEach(() => { - setEnvFlags({ isHosted: false }) - }) - - it.each([ - ['loopback', 'http://127.0.0.1:8080', '127.0.0.1'], - ['RFC1918 10.x', 'http://10.0.0.5', '10.0.0.5'], - ['RFC1918 192.168.x', 'http://192.168.1.1:8443', '192.168.1.1'], - ])('allows %s (private Connect server)', async (_label, url, expected) => { - await expect(validateConnectServerUrl(url)).resolves.toBe(expected) - }) - - it('still blocks link-local metadata', async () => { - await expect(validateConnectServerUrl('http://169.254.169.254')).rejects.toThrow( - 'cannot point to a link-local address' - ) + it('surfaces the guard refusal verbatim', async () => { + mockValidateUrlWithDNS.mockResolvedValue({ + isValid: false, + error: '1Password server URL resolves to a private or reserved address (10.0.0.9).', }) - it('still blocks IPv6 link-local', async () => { - await expect(validateConnectServerUrl('http://[fe80::1]')).rejects.toThrow( - 'cannot point to a link-local address' - ) - }) - - it('allows a hostname that resolves to a private IP', async () => { - mockDnsLookup.mockResolvedValue([{ address: '10.1.2.3', family: 4 }]) - await expect(validateConnectServerUrl('https://connect.internal')).resolves.toBe('10.1.2.3') - }) - }) - - it('rejects when DNS resolution fails', async () => { - mockDnsLookup.mockRejectedValue(new Error('ENOTFOUND')) - await expect(validateConnectServerUrl('https://nope.invalid')).rejects.toThrow( - 'could not be resolved' + await expect(validateConnectServerUrl('http://connect.internal')).rejects.toThrow( + 'resolves to a private or reserved address (10.0.0.9)' ) }) }) @@ -129,6 +62,7 @@ describe('connectRequest', () => { beforeEach(() => { vi.clearAllMocks() setEnvFlags({ isHosted: false }) + mockValidateUrlWithDNS.mockResolvedValue({ isValid: true, resolvedIP: '8.8.8.8' }) mockSecureFetch.mockResolvedValue({ ok: true, status: 200 }) }) @@ -151,7 +85,7 @@ describe('connectRequest', () => { 'Content-Type': 'application/json', }, body: '{"title":"Example"}', - allowHttp: true, + profile: 'selfHostedService', maxResponseBytes: 10 * 1024 * 1024, signal: controller.signal, }) diff --git a/apps/sim/lib/internal/onepassword/client.ts b/apps/sim/lib/internal/onepassword/client.ts index 8ac448c09a8..1903bc34eec 100644 --- a/apps/sim/lib/internal/onepassword/client.ts +++ b/apps/sim/lib/internal/onepassword/client.ts @@ -10,17 +10,12 @@ import type { VaultOverview, Website, } from '@1password/sdk' -import { createLogger } from '@sim/logger' -import { resolveHostAddresses } from '@sim/security/dns' -import { isPrivateIp, unwrapIpv6Brackets } from '@sim/security/ssrf' -import { toError } from '@sim/utils/errors' import { generateId } from '@sim/utils/id' -import * as ipaddr from 'ipaddr.js' -import { isHosted } from '@/lib/core/config/env-flags' import { MAX_JSON_API_RESPONSE_BYTES, type SecureFetchResponse, secureFetchWithPinnedIP, + validateUrlWithDNS, } from '@/lib/core/security/input-validation.server' /** Connect-format field type strings returned by normalization. */ @@ -262,85 +257,32 @@ export async function createOnePasswordClient(serviceAccountToken: string, signa return client } -const connectLogger = createLogger('OnePasswordConnect') - /** - * Enforces the SSRF policy for a resolved Connect server IP. + * Validates a Connect server URL against the deployment's egress policy and + * returns the resolved IP for DNS pinning. * - * On the hosted service, all private and reserved IPs are blocked — a tenant has - * no legitimate reason to point Connect at the platform's internal network. On - * self-hosted deployments only link-local (cloud metadata) is blocked, since the - * operator controls both the workflows and the network and Connect servers - * legitimately live on private (RFC1918) addresses. + * The `selfHostedService` profile matches how Connect is deployed: plain HTTP on + * an arbitrary port is ordinary, loopback is reachable off the hosted platform, + * and a Connect server on the rest of a private network is reachable once the + * operator names it in the egress allowlist. * - * @throws Error if the IP is not permitted under the active policy. - */ -function assertConnectIpAllowed(ip: string, hostname: string): void { - if (isHosted) { - if (isPrivateIp(ip)) { - connectLogger.warn('1Password Connect server URL resolves to a private or reserved IP', { - hostname, - resolvedIP: ip, - }) - throw new Error('1Password server URL cannot point to a private or reserved IP address') - } - return - } - - if (ipaddr.isValid(ip) && ipaddr.process(ip).range() === 'linkLocal') { - connectLogger.warn('1Password Connect server URL resolves to a link-local IP', { - hostname, - resolvedIP: ip, - }) - throw new Error('1Password server URL cannot point to a link-local address') - } -} - -/** - * Validates a Connect server URL against the SSRF policy and returns the resolved - * IP for DNS pinning to prevent TOCTOU rebinding. See {@link assertConnectIpAllowed} - * for the hosted vs. self-hosted policy. - * @throws Error if the URL is invalid, fails the IP policy, or DNS fails. + * @throws Error if the URL is invalid, refused by the policy, or unresolvable. */ export async function validateConnectServerUrl( serverUrl: string, signal?: AbortSignal ): Promise { signal?.throwIfAborted() - let hostname: string - try { - hostname = new URL(serverUrl).hostname - } catch { - throw new Error('1Password server URL is not a valid URL') - } - - const clean = unwrapIpv6Brackets(hostname) - - if (ipaddr.isValid(clean)) { - assertConnectIpAllowed(clean, clean) - return clean - } - - let addresses: string[] - let address: string - try { - const resolved = await resolveHostAddresses(clean) - signal?.throwIfAborted() - addresses = resolved.addresses - address = resolved.preferred - } catch (error) { - signal?.throwIfAborted() - connectLogger.warn('DNS lookup failed for 1Password Connect server URL', { - hostname: clean, - error: toError(error).message, - }) - throw new Error('1Password server URL hostname could not be resolved') - } - - for (const candidate of addresses) { - assertConnectIpAllowed(candidate, clean) + const validation = await validateUrlWithDNS( + serverUrl, + '1Password server URL', + 'selfHostedService' + ) + signal?.throwIfAborted() + if (!validation.isValid) { + throw new Error(validation.error) } - return address + return validation.resolvedIP } /** @@ -379,7 +321,7 @@ export async function connectRequest(options: { method: options.method, headers, body: options.body ? JSON.stringify(options.body) : undefined, - allowHttp: true, + profile: 'selfHostedService', maxResponseBytes: options.maxResponseBytes ?? MAX_JSON_API_RESPONSE_BYTES, signal: options.signal, }) diff --git a/apps/sim/lib/internal/pipedrive/client.ts b/apps/sim/lib/internal/pipedrive/client.ts index c4427e3f26c..0fb90aaf4f4 100644 --- a/apps/sim/lib/internal/pipedrive/client.ts +++ b/apps/sim/lib/internal/pipedrive/client.ts @@ -48,12 +48,13 @@ export async function listPipedriveFiles( if (input.sort) url.searchParams.set('sort', input.sort) if (input.limit) url.searchParams.set('limit', input.limit) if (input.start) url.searchParams.set('start', input.start) - const validation = await validateUrlWithDNS(url.toString(), 'apiUrl') + const validation = await validateUrlWithDNS(url.toString(), 'apiUrl', 'configuredEndpoint') signal?.throwIfAborted() - if (!validation.isValid || !validation.resolvedIP) { + if (!validation.isValid) { throw new PipedriveOperationError(validation.error || 'Invalid Pipedrive API URL', 400) } const response = await secureFetchWithPinnedIP(url.toString(), validation.resolvedIP, { + profile: 'configuredEndpoint', method: 'GET', headers: getPipedriveAuthHeaders(input), maxResponseBytes: MAX_JSON_API_RESPONSE_BYTES, @@ -92,14 +93,15 @@ export async function downloadPipedriveFile( signal?: AbortSignal ): Promise<{ buffer: Buffer; contentType: string | null } | null> { signal?.throwIfAborted() - const validation = await validateUrlWithDNS(fileUrl, 'fileUrl') + const validation = await validateUrlWithDNS(fileUrl, 'fileUrl', 'contentFetch') signal?.throwIfAborted() - if (!validation.isValid || !validation.resolvedIP) return null + if (!validation.isValid) return null const authHeaders: Record = input.authStyle === 'x-api-token' ? { 'x-api-token': input.accessToken } : { Authorization: `Bearer ${input.accessToken}` } const response = await secureFetchWithPinnedIP(fileUrl, validation.resolvedIP, { + profile: 'contentFetch', method: 'GET', headers: isPipedriveHost(fileUrl) ? authHeaders : {}, maxResponseBytes: maxBytes, diff --git a/apps/sim/lib/internal/principals/executor.ts b/apps/sim/lib/internal/principals/executor.ts index 4aeeb57e5a8..1c40c232116 100644 --- a/apps/sim/lib/internal/principals/executor.ts +++ b/apps/sim/lib/internal/principals/executor.ts @@ -29,7 +29,13 @@ export function resolveExecutorOriginSubject(origin: ExecutorDelegationOrigin): return subjectUserId } -async function bindExecutorPrincipal( +/** + * Binds an executor delegation origin to a delegated principal in-process, + * without minting and re-verifying a delegation JWT. The underlying binding + * still re-validates the workflow and deployment context, so trust matches the + * wire path minus the signature check, which proves nothing in-process. + */ +export async function createExecutorPrincipalFromDelegationOrigin( origin: ExecutorDelegationOrigin, audience: string, resourceScope?: DelegatedPrincipal['resourceScope'], @@ -74,5 +80,11 @@ export async function createExecutorPrincipalFromExecutionContext({ }: CreateExecutorPrincipalFromExecutionContextInput) { const origin = context.executorDelegationOrigin if (!origin) throw new ExecutorDelegationOriginRequiredError() - return bindExecutorPrincipal(origin, audience, resourceScope, expiresAt, context.userId) + return createExecutorPrincipalFromDelegationOrigin( + origin, + audience, + resourceScope, + expiresAt, + context.userId + ) } diff --git a/apps/sim/lib/internal/pulse/client.ts b/apps/sim/lib/internal/pulse/client.ts index c2deac2b4a2..5518b945a2e 100644 --- a/apps/sim/lib/internal/pulse/client.ts +++ b/apps/sim/lib/internal/pulse/client.ts @@ -19,9 +19,9 @@ export async function submitPulseParse( signal?: AbortSignal ): Promise { signal?.throwIfAborted() - const validation = await validateUrlWithDNS(PULSE_ENDPOINT, 'Pulse API URL') + const validation = await validateUrlWithDNS(PULSE_ENDPOINT, 'Pulse API URL', 'configuredEndpoint') signal?.throwIfAborted() - if (!validation.isValid || !validation.resolvedIP) { + if (!validation.isValid) { throw new PulseOperationError(502, { success: false, error: 'Failed to reach Pulse API' }) } @@ -30,6 +30,7 @@ export async function submitPulseParse( const body = Buffer.from(await payload.arrayBuffer()) signal?.throwIfAborted() const response = await secureFetchWithPinnedIP(PULSE_ENDPOINT, validation.resolvedIP, { + profile: 'configuredEndpoint', method: 'POST', headers: { 'x-api-key': apiKey, 'Content-Type': contentType }, body, diff --git a/apps/sim/lib/internal/rds/client.test.ts b/apps/sim/lib/internal/rds/client.test.ts new file mode 100644 index 00000000000..d3dcd667f5e --- /dev/null +++ b/apps/sim/lib/internal/rds/client.test.ts @@ -0,0 +1,31 @@ +/** + * @vitest-environment node + */ +import type { RDSDataClient } from '@aws-sdk/client-rds-data' +import { describe, expect, it, vi } from 'vitest' +import { executeStatement } from '@/lib/internal/rds/client' + +describe('executeStatement', () => { + it('preserves null elements in nested array values', async () => { + const send = vi.fn().mockResolvedValue({ + columnMetadata: [{ name: 'values' }], + records: [ + [ + { + arrayValue: { + arrayValues: [{ stringValues: ['first', null] }, null, { longValues: [1, null] }], + }, + }, + ], + ], + }) + const client = { send } as unknown as RDSDataClient + + await expect( + executeStatement(client, 'resource-arn', 'secret-arn', 'database', 'SELECT values') + ).resolves.toEqual({ + rows: [{ values: [['first', null], null, [1, null]] }], + rowCount: 1, + }) + }) +}) diff --git a/apps/sim/lib/internal/rds/client.ts b/apps/sim/lib/internal/rds/client.ts index 458266a9ee3..683e9a1aee4 100644 --- a/apps/sim/lib/internal/rds/client.ts +++ b/apps/sim/lib/internal/rds/client.ts @@ -74,7 +74,11 @@ function parseFieldValue(field: Field): unknown { if (arr.longValues) return arr.longValues if (arr.doubleValues) return arr.doubleValues if (arr.booleanValues) return arr.booleanValues - if (arr.arrayValues) return arr.arrayValues.map((f) => parseFieldValue({ arrayValue: f })) + if (arr.arrayValues) { + return arr.arrayValues.map((value) => + value === null ? null : parseFieldValue({ arrayValue: value }) + ) + } return [] } return null diff --git a/apps/sim/lib/internal/reducto/client.ts b/apps/sim/lib/internal/reducto/client.ts index b5a79fb7383..8fd209b862f 100644 --- a/apps/sim/lib/internal/reducto/client.ts +++ b/apps/sim/lib/internal/reducto/client.ts @@ -19,9 +19,13 @@ export async function submitReductoParse( signal?: AbortSignal ): Promise { signal?.throwIfAborted() - const validation = await validateUrlWithDNS(REDUCTO_ENDPOINT, 'Reducto API URL') + const validation = await validateUrlWithDNS( + REDUCTO_ENDPOINT, + 'Reducto API URL', + 'configuredEndpoint' + ) signal?.throwIfAborted() - if (!validation.isValid || !validation.resolvedIP) { + if (!validation.isValid) { throw new ReductoOperationError(502, { success: false, error: 'Failed to reach Reducto API', @@ -29,6 +33,7 @@ export async function submitReductoParse( } const response = await secureFetchWithPinnedIP(REDUCTO_ENDPOINT, validation.resolvedIP, { + profile: 'configuredEndpoint', method: 'POST', headers: { 'Content-Type': 'application/json', diff --git a/apps/sim/lib/internal/sailpoint/client.test.ts b/apps/sim/lib/internal/sailpoint/client.test.ts new file mode 100644 index 00000000000..082c4068d4a --- /dev/null +++ b/apps/sim/lib/internal/sailpoint/client.test.ts @@ -0,0 +1,173 @@ +/** + * @vitest-environment node + */ +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import { + clearSailPointTokenStateForTests, + getSailPointAccessToken, + getSailPointTokenStateForTests, + readTotalCount, + resolveSailPointHosts, + sailpointFetch, +} from '@/lib/internal/sailpoint/client' + +const mockFetch = vi.fn() + +function tokenResponse(token: string, expiresIn = 3600): Response { + return Response.json({ access_token: token, expires_in: expiresIn }) +} + +describe('SailPoint client', () => { + beforeEach(() => { + clearSailPointTokenStateForTests() + mockFetch.mockReset() + vi.stubGlobal('fetch', mockFetch) + }) + + afterEach(() => { + vi.useRealTimers() + vi.unstubAllGlobals() + }) + + it('accepts only commercial and government tenant hosts', () => { + expect(resolveSailPointHosts('acme').host).toBe('acme.api.identitynow.com') + expect(resolveSailPointHosts('https://agency.api.identitynowgov.com').host).toBe( + 'agency.api.identitynowgov.com' + ) + expect(() => resolveSailPointHosts('acme.api.identitynow.com.evil.test')).toThrow( + 'not an allowed' + ) + }) + + it('isolates cache entries by the exact credential secret', async () => { + mockFetch + .mockResolvedValueOnce(tokenResponse('first')) + .mockResolvedValueOnce(tokenResponse('second')) + + const common = { tenant: 'acme', clientId: 'client' } + expect(await getSailPointAccessToken({ ...common, clientSecret: 'one' })).toBe('first') + expect(await getSailPointAccessToken({ ...common, clientSecret: 'two' })).toBe('second') + expect(mockFetch).toHaveBeenCalledTimes(2) + }) + + it('single-flights concurrent exchanges for the same credentials', async () => { + let release: ((response: Response) => void) | undefined + mockFetch.mockImplementationOnce( + () => + new Promise((resolve) => { + release = resolve + }) + ) + const credentials = { tenant: 'acme', clientId: 'client', clientSecret: 'secret' } + const first = getSailPointAccessToken(credentials) + const second = getSailPointAccessToken(credentials) + expect(mockFetch).toHaveBeenCalledTimes(1) + release?.(tokenResponse('shared')) + await expect(Promise.all([first, second])).resolves.toEqual(['shared', 'shared']) + }) + + it('lets one token waiter abort without cancelling the shared exchange', async () => { + let release: ((response: Response) => void) | undefined + mockFetch.mockImplementationOnce( + () => + new Promise((resolve) => { + release = resolve + }) + ) + const credentials = { tenant: 'acme', clientId: 'client', clientSecret: 'secret' } + const controller = new AbortController() + const first = getSailPointAccessToken(credentials, controller.signal) + const second = getSailPointAccessToken(credentials) + + controller.abort(new Error('caller stopped')) + await expect(first).rejects.toThrow('caller stopped') + release?.(tokenResponse('shared')) + await expect(second).resolves.toBe('shared') + expect(mockFetch).toHaveBeenCalledTimes(1) + expect(getSailPointTokenStateForTests().exchangeSize).toBe(0) + }) + + it('expires cached tokens before their provider expiry', async () => { + vi.useFakeTimers() + vi.setSystemTime(new Date('2026-01-01T00:00:00.000Z')) + mockFetch + .mockResolvedValueOnce(tokenResponse('old', 100)) + .mockResolvedValueOnce(tokenResponse('new', 100)) + const credentials = { tenant: 'acme', clientId: 'client', clientSecret: 'secret' } + + expect(await getSailPointAccessToken(credentials)).toBe('old') + vi.setSystemTime(new Date('2026-01-01T00:01:31.000Z')) + expect(await getSailPointAccessToken(credentials)).toBe('new') + }) + + it('evicts the oldest token when the bounded cache is full', async () => { + mockFetch.mockImplementation(async () => tokenResponse('token')) + for (let index = 0; index < 101; index += 1) { + await getSailPointAccessToken({ + tenant: 'acme', + clientId: `client-${index}`, + clientSecret: 'secret', + }) + } + expect(getSailPointTokenStateForTests()).toEqual({ cacheSize: 100, exchangeSize: 0 }) + }) + + it('rejects provider responses larger than the shared JSON cap', async () => { + mockFetch.mockResolvedValueOnce(tokenResponse('token')).mockResolvedValueOnce( + new Response('{}', { + status: 200, + headers: { 'content-length': String(10 * 1024 * 1024 + 1) }, + }) + ) + const credentials = { tenant: 'acme', clientId: 'client', clientSecret: 'secret' } + + await expect( + sailpointFetch(credentials, (hosts) => ({ + url: `${hosts.apiBaseUrl}/identities/v1`, + init: { method: 'GET' }, + })) + ).rejects.toThrow(/maximum|limit|exceeds/i) + }) + + it('aborts during rate-limit backoff without another provider call', async () => { + mockFetch + .mockResolvedValueOnce(tokenResponse('token')) + .mockResolvedValueOnce(new Response(null, { status: 429, headers: { 'retry-after': '30' } })) + const controller = new AbortController() + const credentials = { tenant: 'acme', clientId: 'client', clientSecret: 'secret' } + const pending = sailpointFetch( + credentials, + (hosts) => ({ + url: `${hosts.apiBaseUrl}/identities/v1`, + init: { method: 'GET' }, + }), + { signal: controller.signal } + ) + + await vi.waitFor(() => expect(mockFetch).toHaveBeenCalledTimes(2)) + controller.abort(new Error('stop retrying')) + await expect(pending).rejects.toThrow('stop retrying') + expect(mockFetch).toHaveBeenCalledTimes(2) + }) + + it('rejects redirects for token and authenticated provider requests', async () => { + mockFetch + .mockResolvedValueOnce(tokenResponse('token')) + .mockResolvedValueOnce(Response.json({ id: 'identity' })) + const credentials = { tenant: 'acme', clientId: 'client', clientSecret: 'secret' } + + await sailpointFetch(credentials, (hosts) => ({ + url: `${hosts.apiBaseUrl}/identities/v1/id`, + init: { method: 'GET' }, + })) + + expect(mockFetch.mock.calls[0][1]?.redirect).toBe('error') + expect(mockFetch.mock.calls[1][1]?.redirect).toBe('error') + }) + + it('accepts only non-negative integer total counts', () => { + expect(readTotalCount(new Headers({ 'x-total-count': '7' }))).toBe(7) + expect(readTotalCount(new Headers({ 'x-total-count': '1.5' }))).toBeNull() + expect(readTotalCount(new Headers({ 'x-total-count': '-1' }))).toBeNull() + }) +}) diff --git a/apps/sim/lib/internal/sailpoint/client.ts b/apps/sim/lib/internal/sailpoint/client.ts new file mode 100644 index 00000000000..395492cea38 --- /dev/null +++ b/apps/sim/lib/internal/sailpoint/client.ts @@ -0,0 +1,314 @@ +import { createHash } from 'node:crypto' +import { interruptibleSleep } from '@sim/utils/helpers' +import { isRecordLike } from '@sim/utils/object' +import { backoffWithJitter, parseRetryAfter } from '@sim/utils/retry' +import { MAX_JSON_API_RESPONSE_BYTES } from '@/lib/core/security/input-validation.server' +import { + consumeOrCancelBody, + DEFAULT_MAX_ERROR_BODY_BYTES, + readResponseTextWithLimit, +} from '@/lib/core/utils/stream-limits' + +export interface SailPointCredentials { + clientId: string + clientSecret: string + tenant: string +} + +export interface SailPointHosts { + apiBaseUrl: string + host: string + tokenUrl: string +} + +export interface SailPointFetchResult { + data: unknown + headers: Headers + ok: boolean + status: number +} + +interface CachedToken { + expiresAt: number + token: string +} + +const MAX_FETCH_RETRIES = 4 +const MAX_TOKEN_CACHE_ENTRIES = 100 +const MAX_TOKEN_EXCHANGES = 100 +const MAX_TOKEN_RESPONSE_BYTES = 1024 * 1024 +const TOKEN_EXCHANGE_TIMEOUT_MS = 30_000 +const TOKEN_EXPIRY_BUFFER_MS = 60_000 +const tokenCache = new Map() +const tokenExchanges = new Map>() + +async function waitForPromiseWithSignal(promise: Promise, signal?: AbortSignal): Promise { + if (!signal) return promise + signal.throwIfAborted() + + return new Promise((resolve, reject) => { + const onAbort = () => + reject(signal.reason ?? new DOMException('The operation was aborted', 'AbortError')) + signal.addEventListener('abort', onAbort, { once: true }) + promise.then(resolve, reject).finally(() => signal.removeEventListener('abort', onAbort)) + }) +} + +export function resolveSailPointHosts(tenant: string): SailPointHosts { + let host = tenant.trim().replace(/^https?:\/\//i, '') + host = host + .replace(/[/?#].*$/, '') + .replace(/\.+$/, '') + .toLowerCase() + + if (!host) throw new Error('SailPoint tenant is required') + if (!host.includes('.')) { + if (!/^[a-z0-9][a-z0-9-]*$/.test(host)) { + throw new Error(`Invalid SailPoint tenant "${tenant}"`) + } + host = `${host}.api.identitynow.com` + } + + const suffix = ['.api.identitynow.com', '.api.identitynowgov.com'].find((candidate) => + host.endsWith(candidate) + ) + const tenantPrefix = suffix ? host.slice(0, -suffix.length) : '' + if (!suffix || !tenantPrefix || !/^[a-z0-9](?:[a-z0-9.-]*[a-z0-9])?$/.test(tenantPrefix)) { + throw new Error( + `SailPoint host "${host}" is not an allowed Identity Security Cloud tenant host` + ) + } + + return { + apiBaseUrl: `https://${host}`, + host, + tokenUrl: `https://${host}/oauth/token`, + } +} + +export function getSailPointErrorMessage(data: unknown, fallback: string): string { + if (typeof data === 'string') return data || fallback + if (!isRecordLike(data)) return fallback + + if (Array.isArray(data.messages) && data.messages.length > 0) { + const first = data.messages[0] + if (isRecordLike(first) && typeof first.text === 'string' && first.text) { + const trackingId = typeof data.trackingId === 'string' ? data.trackingId : null + return trackingId ? `${first.text} (trackingId: ${trackingId})` : first.text + } + } + if (typeof data.error_description === 'string' && data.error_description) { + return data.error_description + } + if (typeof data.message === 'string' && data.message) return data.message + if (typeof data.error === 'string' && data.error) return data.error + return fallback +} + +function credentialsCacheKey(credentials: SailPointCredentials): string { + const { host } = resolveSailPointHosts(credentials.tenant) + const secretHash = createHash('sha256').update(credentials.clientSecret).digest('hex') + return `${host}:${credentials.clientId}:${secretHash}` +} + +function pruneTokenCache(now: number): void { + for (const [key, value] of tokenCache) { + if (value.expiresAt <= now) tokenCache.delete(key) + } + while (tokenCache.size >= MAX_TOKEN_CACHE_ENTRIES) { + const oldest = tokenCache.keys().next().value + if (typeof oldest !== 'string') break + tokenCache.delete(oldest) + } +} + +function cacheToken(key: string, token: CachedToken): void { + pruneTokenCache(Date.now()) + tokenCache.delete(key) + tokenCache.set(key, token) +} + +async function readBoundedBody( + response: Response, + maxBytes: number, + signal?: AbortSignal +): Promise { + if (response.status === 204) return null + const text = await readResponseTextWithLimit(response, { + maxBytes, + label: 'SailPoint response body', + signal, + }) + if (!text) return null + try { + return JSON.parse(text) as unknown + } catch { + return text + } +} + +async function exchangeAccessToken( + credentials: SailPointCredentials, + signal?: AbortSignal +): Promise { + const { tokenUrl } = resolveSailPointHosts(credentials.tenant) + let attempt = 0 + + while (true) { + signal?.throwIfAborted() + const response = await fetch(tokenUrl, { + method: 'POST', + headers: { + Accept: 'application/json', + 'Content-Type': 'application/x-www-form-urlencoded', + }, + body: new URLSearchParams({ + grant_type: 'client_credentials', + client_id: credentials.clientId, + client_secret: credentials.clientSecret, + }).toString(), + cache: 'no-store', + redirect: 'error', + signal, + }) + + if (response.status === 429 && attempt < MAX_FETCH_RETRIES) { + const retryAfterMs = parseRetryAfter(response.headers.get('retry-after')) + await consumeOrCancelBody(response, DEFAULT_MAX_ERROR_BODY_BYTES) + attempt += 1 + await interruptibleSleep(backoffWithJitter(attempt, retryAfterMs), signal) + signal?.throwIfAborted() + continue + } + + const data = await readBoundedBody( + response, + response.ok ? MAX_TOKEN_RESPONSE_BYTES : DEFAULT_MAX_ERROR_BODY_BYTES, + signal + ) + if (!response.ok) { + throw new Error(getSailPointErrorMessage(data, 'Failed to authenticate with SailPoint')) + } + if (!isRecordLike(data) || typeof data.access_token !== 'string' || !data.access_token) { + throw new Error('SailPoint authentication did not return an access token') + } + + const parsedExpiry = Number(data.expires_in) + const expiresInSeconds = Number.isFinite(parsedExpiry) && parsedExpiry > 0 ? parsedExpiry : 3600 + const bufferMs = Math.min(TOKEN_EXPIRY_BUFFER_MS, expiresInSeconds * 100) + const key = credentialsCacheKey(credentials) + cacheToken(key, { + token: data.access_token, + expiresAt: Date.now() + Math.max(expiresInSeconds * 1000 - bufferMs, 0), + }) + return data.access_token + } +} + +export function invalidateSailPointToken(credentials: SailPointCredentials): void { + tokenCache.delete(credentialsCacheKey(credentials)) +} + +export async function getSailPointAccessToken( + credentials: SailPointCredentials, + signal?: AbortSignal +): Promise { + signal?.throwIfAborted() + const key = credentialsCacheKey(credentials) + const now = Date.now() + const cached = tokenCache.get(key) + if (cached && cached.expiresAt > now) { + tokenCache.delete(key) + tokenCache.set(key, cached) + return cached.token + } + if (cached) tokenCache.delete(key) + + const existing = tokenExchanges.get(key) + if (existing) return waitForPromiseWithSignal(existing, signal) + if (tokenExchanges.size >= MAX_TOKEN_EXCHANGES) { + throw new Error('Too many concurrent SailPoint token exchanges') + } + + const exchange = exchangeAccessToken( + credentials, + AbortSignal.timeout(TOKEN_EXCHANGE_TIMEOUT_MS) + ).finally(() => { + tokenExchanges.delete(key) + }) + tokenExchanges.set(key, exchange) + return waitForPromiseWithSignal(exchange, signal) +} + +export async function sailpointFetch( + credentials: SailPointCredentials, + buildRequest: (hosts: SailPointHosts) => { init: RequestInit; url: string }, + options: { maxRetries?: number; signal?: AbortSignal } = {} +): Promise { + const maxRetries = Math.min(Math.max(options.maxRetries ?? MAX_FETCH_RETRIES, 0), 10) + const hosts = resolveSailPointHosts(credentials.tenant) + let attempt = 0 + let refreshedOn401 = false + + while (true) { + options.signal?.throwIfAborted() + const token = await getSailPointAccessToken(credentials, options.signal) + const { init, url } = buildRequest(hosts) + const headers = new Headers(init.headers) + headers.set('Authorization', `Bearer ${token}`) + if (!headers.has('Accept')) headers.set('Accept', 'application/json') + + const response = await fetch(url, { + ...init, + cache: 'no-store', + headers, + redirect: 'error', + signal: options.signal, + }) + + if (response.status === 401 && !refreshedOn401) { + await consumeOrCancelBody(response, DEFAULT_MAX_ERROR_BODY_BYTES) + invalidateSailPointToken(credentials) + refreshedOn401 = true + continue + } + if (response.status === 429 && attempt < maxRetries) { + const retryAfterMs = parseRetryAfter(response.headers.get('retry-after')) + await consumeOrCancelBody(response, DEFAULT_MAX_ERROR_BODY_BYTES) + attempt += 1 + await interruptibleSleep(backoffWithJitter(attempt, retryAfterMs), options.signal) + options.signal?.throwIfAborted() + continue + } + + const data = await readBoundedBody( + response, + response.ok ? MAX_JSON_API_RESPONSE_BYTES : DEFAULT_MAX_ERROR_BODY_BYTES, + options.signal + ) + return { + data, + headers: response.headers, + ok: response.ok, + status: response.status, + } + } +} + +export function readTotalCount(headers: Headers): number | null { + const raw = headers.get('x-total-count') + if (!raw) return null + const parsed = Number(raw) + return Number.isInteger(parsed) && parsed >= 0 ? parsed : null +} + +/** Clears process-local authentication state for deterministic tests. */ +export function clearSailPointTokenStateForTests(): void { + tokenCache.clear() + tokenExchanges.clear() +} + +/** Returns cache sizes for deterministic boundary tests. */ +export function getSailPointTokenStateForTests(): { cacheSize: number; exchangeSize: number } { + return { cacheSize: tokenCache.size, exchangeSize: tokenExchanges.size } +} diff --git a/apps/sim/lib/internal/sailpoint/execute-tool.test.ts b/apps/sim/lib/internal/sailpoint/execute-tool.test.ts new file mode 100644 index 00000000000..5d3a8ac0478 --- /dev/null +++ b/apps/sim/lib/internal/sailpoint/execute-tool.test.ts @@ -0,0 +1,834 @@ +/** + * @vitest-environment node + */ +import { beforeEach, describe, expect, it, vi } from 'vitest' +import { PayloadSizeLimitError } from '@/lib/core/utils/stream-limits' + +const fileMocks = vi.hoisted(() => ({ + assertToolFileAccess: vi.fn(), + downloadServableFileFromStorage: vi.fn(), + processFilesToUserFiles: vi.fn(), +})) + +vi.mock('@/app/api/files/authorization', () => ({ + assertToolFileAccess: fileMocks.assertToolFileAccess, +})) +vi.mock('@/lib/uploads/utils/file-utils', () => ({ + processFilesToUserFiles: fileMocks.processFilesToUserFiles, +})) +vi.mock('@/lib/uploads/utils/file-utils.server', () => ({ + downloadServableFileFromStorage: fileMocks.downloadServableFileFromStorage, +})) + +import { clearSailPointTokenStateForTests } from '@/lib/internal/sailpoint/client' +import { executeSailPointTool } from '@/lib/internal/sailpoint/execute-tool' +import { MAX_SAILPOINT_CSV_BYTES } from '@/lib/internal/sailpoint/operations' + +const mockFetch = vi.fn() +const credentials = { clientId: 'client', clientSecret: 'secret', tenant: 'acme' } + +function tokenResponse(): Response { + return Response.json({ access_token: 'token', expires_in: 3600 }) +} + +function request(operation: string, input: Record, userId?: string) { + return executeSailPointTool({ + toolId: operation, + input: { ...credentials, operation, ...input }, + headers: new Headers(), + context: { workflowId: 'workflow', userId }, + requestId: 'request-id', + }) +} + +interface OperationCase { + body?: unknown + input: Record + method: 'GET' | 'POST' + operation: string + path: string + providerBody?: unknown + providerStatus?: number + total?: number +} + +const OPERATION_CASES: OperationCase[] = [ + { + operation: 'sailpoint_search', + method: 'POST', + path: '/search/v1', + input: { indices: ['identities'], query: { query: 'name:a*' } }, + body: { indices: ['identities'], query: { query: 'name:a*' } }, + providerBody: [], + }, + { + operation: 'sailpoint_search_count', + method: 'POST', + path: '/search/v1/count', + input: { queryType: 'DSL', queryDsl: { match_all: {} } }, + body: { queryType: 'DSL', queryDsl: { match_all: {} } }, + providerStatus: 204, + total: 7, + }, + { + operation: 'sailpoint_search_aggregate', + method: 'POST', + path: '/search/v1/aggregate?count=true', + input: { aggregationsDsl: { names: { terms: { field: 'name' } } }, count: true }, + body: { + aggregationType: 'DSL', + aggregationsDsl: { names: { terms: { field: 'name' } } }, + }, + providerBody: { aggregations: { names: { buckets: [] } }, hits: [] }, + total: 3, + }, + { + operation: 'sailpoint_list_identities', + method: 'GET', + path: '/identities/v1', + input: {}, + providerBody: [], + }, + { + operation: 'sailpoint_get_identity', + method: 'GET', + path: '/identities/v1/id', + input: { id: 'id' }, + }, + { + operation: 'sailpoint_list_identity_entitlements', + method: 'GET', + path: '/entitlements/v1/identities/id/entitlements', + input: { id: 'id' }, + providerBody: [], + }, + { + operation: 'sailpoint_list_accounts', + method: 'GET', + path: '/accounts/v1', + input: {}, + providerBody: [], + }, + { + operation: 'sailpoint_get_account', + method: 'GET', + path: '/accounts/v1/id', + input: { id: 'id' }, + }, + { + operation: 'sailpoint_get_account_entitlements', + method: 'GET', + path: '/accounts/v1/id/entitlements', + input: { id: 'id' }, + providerBody: [], + }, + { + operation: 'sailpoint_list_entitlements', + method: 'GET', + path: '/entitlements/v1', + input: {}, + providerBody: [], + }, + { + operation: 'sailpoint_get_entitlement', + method: 'GET', + path: '/entitlements/v1/id', + input: { id: 'id' }, + }, + { + operation: 'sailpoint_get_entitlement_request_config', + method: 'GET', + path: '/entitlements/v1/id/entitlement-request-config', + input: { id: 'id' }, + }, + { + operation: 'sailpoint_list_roles', + method: 'GET', + path: '/roles/v1', + input: {}, + providerBody: [], + }, + { operation: 'sailpoint_get_role', method: 'GET', path: '/roles/v1/id', input: { id: 'id' } }, + { + operation: 'sailpoint_get_role_entitlements', + method: 'GET', + path: '/roles/v1/id/entitlements', + input: { id: 'id' }, + providerBody: [], + }, + { + operation: 'sailpoint_list_access_profiles', + method: 'GET', + path: '/access-profiles/v1', + input: {}, + providerBody: [], + }, + { + operation: 'sailpoint_get_access_profile', + method: 'GET', + path: '/access-profiles/v1/id', + input: { id: 'id' }, + }, + { + operation: 'sailpoint_get_access_profile_entitlements', + method: 'GET', + path: '/access-profiles/v1/id/entitlements', + input: { id: 'id' }, + providerBody: [], + }, + { + operation: 'sailpoint_list_sources', + method: 'GET', + path: '/sources/v1', + input: {}, + providerBody: [], + }, + { operation: 'sailpoint_get_source', method: 'GET', path: '/sources/v1/id', input: { id: 'id' } }, + { + operation: 'sailpoint_list_account_activities', + method: 'GET', + path: '/account-activities/v1', + input: {}, + providerBody: [], + }, + { + operation: 'sailpoint_get_account_activity', + method: 'GET', + path: '/account-activities/v1/id', + input: { id: 'id' }, + }, + { + operation: 'sailpoint_list_campaigns', + method: 'GET', + path: '/campaigns/v1', + input: {}, + providerBody: [], + }, + { + operation: 'sailpoint_get_campaign', + method: 'GET', + path: '/campaigns/v1/id', + input: { id: 'id' }, + }, + { + operation: 'sailpoint_list_certifications', + method: 'GET', + path: '/certifications/v1', + input: {}, + providerBody: [], + }, + { + operation: 'sailpoint_get_certification', + method: 'GET', + path: '/certifications/v1/id', + input: { id: 'id' }, + }, + { + operation: 'sailpoint_list_certification_review_items', + method: 'GET', + path: '/certifications/v1/id/access-review-items', + input: { id: 'id' }, + providerBody: [], + }, + { + operation: 'sailpoint_decide_certification_review_items', + method: 'POST', + path: '/certifications/v1/id/decide', + input: { id: 'id', decisions: [{ id: 'review', decision: 'APPROVE', bulk: true }] }, + body: [{ id: 'review', decision: 'APPROVE', bulk: true }], + }, + { + operation: 'sailpoint_sign_off_certification', + method: 'POST', + path: '/certifications/v1/id/sign-off', + input: { id: 'id' }, + }, + { + operation: 'sailpoint_request_access', + method: 'POST', + path: '/access-requests/v1', + input: { requestedFor: ['identity'], requestedItems: [{ type: 'ROLE', id: 'role' }] }, + body: { requestedFor: ['identity'], requestedItems: [{ type: 'ROLE', id: 'role' }] }, + providerStatus: 202, + providerBody: { newRequests: [], existingRequests: [] }, + }, + { + operation: 'sailpoint_get_account_selections', + method: 'POST', + path: '/access-requests/v1/accounts-selection', + input: { requestedFor: ['identity'], requestedItems: [{ type: 'ROLE', id: 'role' }] }, + body: { requestedFor: ['identity'], requestedItems: [{ type: 'ROLE', id: 'role' }] }, + }, + { + operation: 'sailpoint_get_access_request_config', + method: 'GET', + path: '/access-request-config/v2', + input: {}, + }, + { + operation: 'sailpoint_cancel_access_request', + method: 'POST', + path: '/access-requests/v1/cancel', + input: { accountActivityId: 'activity', comment: 'cancel' }, + body: { accountActivityId: 'activity', comment: 'cancel' }, + providerStatus: 202, + }, + { + operation: 'sailpoint_get_access_request_status', + method: 'GET', + path: '/access-request-status/v1', + input: {}, + providerBody: [], + }, + { + operation: 'sailpoint_list_pending_access_request_approvals', + method: 'GET', + path: '/access-request-approvals/v1/pending', + input: {}, + providerBody: [], + }, + { + operation: 'sailpoint_approve_access_request', + method: 'POST', + path: '/access-request-approvals/v1/approval/approve', + input: { approvalId: 'approval' }, + providerStatus: 202, + }, + { + operation: 'sailpoint_reject_access_request', + method: 'POST', + path: '/access-request-approvals/v1/approval/reject', + input: { approvalId: 'approval', comment: 'reject' }, + body: { comment: 'reject' }, + providerStatus: 202, + }, + { + operation: 'sailpoint_get_task_status', + method: 'GET', + path: '/task-status/v1/id', + input: { id: 'id' }, + }, +] + +describe('SailPoint internal tool handler', () => { + beforeEach(() => { + clearSailPointTokenStateForTests() + mockFetch.mockReset() + vi.stubGlobal('fetch', mockFetch) + fileMocks.assertToolFileAccess.mockReset().mockResolvedValue(null) + fileMocks.processFilesToUserFiles.mockReset() + fileMocks.downloadServableFileFromStorage.mockReset() + }) + + it.each(OPERATION_CASES)( + 'uses the documented path and method for $operation', + async (testCase) => { + const headers = + testCase.total === undefined ? undefined : { 'x-total-count': String(testCase.total) } + const status = testCase.providerStatus ?? 200 + const providerResponse = + status === 204 + ? new Response(null, { status, headers }) + : Response.json(testCase.providerBody ?? { id: 'resource' }, { status, headers }) + mockFetch.mockResolvedValueOnce(tokenResponse()).mockResolvedValueOnce(providerResponse) + + const response = await request(testCase.operation, testCase.input) + expect(response.status).toBe(200) + const providerCall = mockFetch.mock.calls[1] + expect( + new URL(String(providerCall[0])).pathname + new URL(String(providerCall[0])).search + ).toBe(testCase.path) + const init = providerCall[1] + expect(init?.method).toBe(testCase.method) + if (testCase.body !== undefined) { + expect(JSON.parse(String(init?.body))).toEqual(testCase.body) + } + } + ) + + it('sends the experimental header for account-selection discovery', async () => { + mockFetch + .mockResolvedValueOnce(tokenResponse()) + .mockResolvedValueOnce(Response.json({ identities: [] })) + const response = await request('sailpoint_get_account_selections', { + requestedFor: ['identity'], + requestedItems: [{ type: 'ROLE', id: 'role' }], + }) + + expect(response.status).toBe(200) + const headers = new Headers(mockFetch.mock.calls[1][1]?.headers) + expect(headers.get('x-sailpoint-experimental')).toBe('true') + }) + + it.each([ + ['sailpoint_get_account_selections', 'accountSelections', { identities: [] }], + ['sailpoint_get_access_request_config', 'accessRequestConfig', { accessRequest: {} }], + [ + 'sailpoint_get_entitlement_request_config', + 'entitlementRequestConfig', + { accessRequestConfig: {} }, + ], + ])('maps %s to its resource-named output', async (operation, outputKey, providerBody) => { + clearSailPointTokenStateForTests() + mockFetch.mockReset() + mockFetch + .mockResolvedValueOnce(tokenResponse()) + .mockResolvedValueOnce(Response.json(providerBody)) + const input = + operation === 'sailpoint_get_account_selections' + ? { requestedFor: ['identity'], requestedItems: [{ type: 'ROLE', id: 'role' }] } + : operation === 'sailpoint_get_entitlement_request_config' + ? { id: 'entitlement' } + : {} + + const response = await request(operation, input) + await expect(response.json()).resolves.toEqual({ + success: true, + output: { [outputKey]: providerBody }, + }) + }) + + it('rejects a primitive resource response', async () => { + mockFetch + .mockResolvedValueOnce(tokenResponse()) + .mockResolvedValueOnce(Response.json('not-a-resource')) + const response = await request('sailpoint_get_access_request_config', {}) + + expect(response.status).toBe(502) + await expect(response.json()).resolves.toMatchObject({ + success: false, + error: 'SailPoint returned an invalid resource response', + }) + }) + + it('preserves access-request tracking and accepted status', async () => { + const tracking = { + newRequests: [{ requestedFor: 'identity', accessRequestIds: ['new'] }], + existingRequests: [{ requestedFor: 'identity', accessRequestIds: ['existing'] }], + } + mockFetch + .mockResolvedValueOnce(tokenResponse()) + .mockResolvedValueOnce(Response.json(tracking, { status: 202 })) + const response = await request('sailpoint_request_access', { + requestedFor: ['identity'], + requestedItems: [{ type: 'ROLE', id: 'role' }], + }) + const body = await response.json() + expect(body.output).toEqual({ accepted: true, status: 202, ...tracking }) + }) + + it.each([ + { newRequests: 'invalid', existingRequests: [] }, + { newRequests: [], existingRequests: { id: 'invalid' } }, + ])('rejects malformed access-request tracking arrays', async (tracking) => { + mockFetch + .mockResolvedValueOnce(tokenResponse()) + .mockResolvedValueOnce(Response.json(tracking, { status: 202 })) + const response = await request('sailpoint_request_access', { + requestedFor: ['identity'], + requestedItems: [{ type: 'ROLE', id: 'role' }], + }) + + expect(response.status).toBe(502) + }) + + it('forwards every advanced Search body field without inventing default indices', async () => { + const input = { + queryType: 'DSL', + queryVersion: '7.10', + query: { query: 'name:a*', fields: 'name', timeZone: 'UTC', innerHit: { type: 'access' } }, + queryDsl: { match_all: {} }, + textQuery: { terms: ['alice'], fields: ['name'], matchAny: true, contains: false }, + typeAheadQuery: { + query: 'Ali', + field: 'name', + nestedType: 'access', + maxExpansions: 20, + size: 5, + sort: 'asc', + sortByValue: true, + }, + includeNested: false, + queryResultFilter: { includes: ['name'], excludes: ['stacktrace'] }, + aggregationType: 'DSL', + aggregationsVersion: '7.10', + aggregationsDsl: { names: { terms: { field: 'name' } } }, + sort: ['name', '+id'], + searchAfter: ['Alice', 'id'], + filters: { status: { terms: ['ACTIVE'], exclude: false } }, + limit: 25, + offset: 5, + count: true, + } + mockFetch + .mockResolvedValueOnce(tokenResponse()) + .mockResolvedValueOnce(Response.json([], { headers: { 'x-total-count': '1' } })) + const response = await request('sailpoint_search', input) + expect(response.status).toBe(200) + const [url, init] = mockFetch.mock.calls[1] + expect(String(url)).toMatch(/\/search\/v1\?limit=25&offset=5&count=true$/) + expect(JSON.parse(String(init?.body))).toEqual({ + queryType: input.queryType, + queryVersion: input.queryVersion, + query: input.query, + queryDsl: input.queryDsl, + textQuery: input.textQuery, + typeAheadQuery: input.typeAheadQuery, + includeNested: input.includeNested, + queryResultFilter: input.queryResultFilter, + aggregationType: input.aggregationType, + aggregationsVersion: input.aggregationsVersion, + aggregationsDsl: input.aggregationsDsl, + sort: input.sort, + searchAfter: input.searchAfter, + filters: input.filters, + }) + expect(JSON.parse(String(init?.body))).not.toHaveProperty('indices') + }) + + it.each(['sailpoint_search', 'sailpoint_search_count'])( + 'requires a query for the default SAILPOINT mode in %s', + async (operation) => { + const response = await request(operation, {}) + expect(response.status).toBe(400) + expect(mockFetch).not.toHaveBeenCalled() + } + ) + + it('requires queryType=DSL when queryDsl is used', async () => { + const response = await request('sailpoint_search', { queryDsl: { match_all: {} } }) + expect(response.status).toBe(400) + expect(mockFetch).not.toHaveBeenCalled() + }) + + it.each([ + ['aggregationsDsl', {}], + ['aggregationsDsl', '{}'], + ['aggregations', {}], + ['aggregations', '{}'], + ])('rejects an empty %s aggregate definition', async (field, value) => { + const response = await request('sailpoint_search_aggregate', { [field]: value }) + expect(response.status).toBe(400) + expect(await response.json()).toMatchObject({ + error: expect.stringContaining('must be a non-empty object'), + }) + expect(mockFetch).not.toHaveBeenCalled() + }) + + it.each([ + [ + 'sailpoint_list_entitlements', + { + segmentedForIdentity: 'identity', + forSegmentIds: 'segment-a,segment-b', + includeUnsegmented: false, + searchAfter: 'cursor', + filters: 'name sw "A"', + sorters: 'name', + limit: 10, + offset: 2, + count: true, + }, + '/entitlements/v1?filters=name+sw+%22A%22&sorters=name&segmented-for-identity=identity&for-segment-ids=segment-a%2Csegment-b&include-unsegmented=false&searchAfter=cursor&limit=10&offset=2&count=true', + ], + [ + 'sailpoint_list_roles', + { forSubadmin: 'me', forSegmentIds: 'segment', includeUnsegmented: false }, + '/roles/v1?for-subadmin=me&for-segment-ids=segment&include-unsegmented=false', + ], + [ + 'sailpoint_list_access_profiles', + { forSubadmin: 'me', forSegmentIds: 'segment', includeUnsegmented: false }, + '/access-profiles/v1?for-subadmin=me&for-segment-ids=segment&include-unsegmented=false', + ], + ])('forwards current collection parameters for %s', async (operation, input, path) => { + mockFetch.mockResolvedValueOnce(tokenResponse()).mockResolvedValueOnce(Response.json([])) + const response = await request(operation, input) + expect(response.status).toBe(200) + expect( + new URL(String(mockFetch.mock.calls[1][0])).pathname + + new URL(String(mockFetch.mock.calls[1][0])).search + ).toBe(path) + }) + + it('rejects pairwise certification review selectors', async () => { + const response = await request('sailpoint_list_certification_review_items', { + id: 'certification', + entitlements: 'entitlement', + roles: 'role', + }) + expect(response.status).toBe(400) + expect(mockFetch).not.toHaveBeenCalled() + }) + + it.each(['sailpoint_list_account_activities', 'sailpoint_get_access_request_status'])( + 'rejects conflicting requested/regarding identity scopes for %s', + async (operation) => { + const response = await request(operation, { + requestedFor: 'identity', + regardingIdentity: 'identity', + }) + expect(response.status).toBe(400) + expect(mockFetch).not.toHaveBeenCalled() + } + ) + + it('forwards the nested requestedForWithRequestedItems shape', async () => { + const nested = [ + { + identityId: 'identity', + identityType: 'HUMAN', + requestedItems: [ + { + type: 'ENTITLEMENT', + id: 'entitlement', + accountSelection: [{ sourceId: 'source', accounts: [{ nativeIdentity: 'native-id' }] }], + }, + ], + }, + ] + mockFetch + .mockResolvedValueOnce(tokenResponse()) + .mockResolvedValueOnce( + Response.json({ newRequests: [], existingRequests: [] }, { status: 202 }) + ) + const response = await request('sailpoint_request_access', { + requestedForWithRequestedItems: nested, + }) + expect(response.status).toBe(200) + expect(JSON.parse(String(mockFetch.mock.calls[1][1]?.body))).toEqual({ + requestedForWithRequestedItems: nested, + }) + }) + + it('rejects an account selection without an account UUID or native identity', async () => { + const response = await request('sailpoint_get_account_selections', { + requestedForWithRequestedItems: [ + { + identityId: 'machine', + identityType: 'MACHINE', + requestedItems: [ + { + type: 'ENTITLEMENT', + id: 'entitlement', + accountSelection: [{ sourceId: 'source', accounts: [{}] }], + }, + ], + }, + ], + }) + + expect(response.status).toBe(400) + expect(mockFetch).not.toHaveBeenCalled() + }) + + it.each(['sailpoint_request_access', 'sailpoint_get_account_selections'])( + 'enforces the recipient cap for entitlement inputs at the %s handler boundary', + async (operation) => { + const response = await request(operation, { + requestType: 'MODIFY_ACCESS', + requestedFor: Array.from({ length: 11 }, (_, index) => `identity-${index}`), + requestedItems: [{ type: 'ENTITLEMENT', id: 'entitlement' }], + }) + + expect(response.status).toBe(400) + expect(mockFetch).not.toHaveBeenCalled() + } + ) + + it.each(['sailpoint_request_access', 'sailpoint_get_account_selections'])( + 'enforces the nested entitlement cap at the %s handler boundary', + async (operation) => { + const response = await request(operation, { + requestType: 'MODIFY_ACCESS', + requestedForWithRequestedItems: [ + { + identityId: 'identity', + identityType: 'HUMAN', + requestedItems: Array.from({ length: 26 }, (_, index) => ({ + type: 'ENTITLEMENT', + id: `entitlement-${index}`, + })), + }, + ], + }) + + expect(response.status).toBe(400) + expect(mockFetch).not.toHaveBeenCalled() + } + ) + + it('accepts multiple role revokes but rejects multiple entitlement revokes', async () => { + mockFetch + .mockResolvedValueOnce(tokenResponse()) + .mockResolvedValueOnce( + Response.json({ newRequests: [], existingRequests: [] }, { status: 202 }) + ) + const accepted = await request('sailpoint_request_access', { + requestType: 'REVOKE_ACCESS', + requestedFor: ['identity'], + requestedItems: [ + { type: 'ROLE', id: 'one', comment: 'remove' }, + { type: 'ROLE', id: 'two', comment: 'remove' }, + ], + }) + expect(accepted.status).toBe(200) + + clearSailPointTokenStateForTests() + mockFetch.mockReset() + const rejected = await request('sailpoint_request_access', { + requestType: 'REVOKE_ACCESS', + requestedFor: ['identity'], + requestedItems: [ + { type: 'ENTITLEMENT', id: 'one', comment: 'remove' }, + { type: 'ENTITLEMENT', id: 'two', comment: 'remove' }, + ], + }) + expect(rejected.status).toBe(400) + expect(mockFetch).not.toHaveBeenCalled() + }) + + it('maps resource reads to their resource-named output', async () => { + mockFetch + .mockResolvedValueOnce(tokenResponse()) + .mockResolvedValueOnce(Response.json({ id: 'identity' })) + const response = await request('sailpoint_get_identity', { id: 'identity' }) + await expect(response.json()).resolves.toEqual({ + success: true, + output: { identity: { id: 'identity' } }, + }) + }) + + it('rejects mismatched tool and input operation before making a provider call', async () => { + const response = await executeSailPointTool({ + toolId: 'sailpoint_get_identity', + input: { ...credentials, operation: 'sailpoint_get_account', id: 'id' }, + headers: new Headers(), + context: { workflowId: 'workflow' }, + requestId: 'request-id', + }) + expect(response.status).toBe(400) + expect(mockFetch).not.toHaveBeenCalled() + }) + + it.each([ + ['sailpoint_load_accounts', '/sources/v1/source/load-accounts'], + ['sailpoint_load_entitlements', '/sources/v1/source/load-entitlements'], + ])('authorizes and bounds the CSV for %s', async (operation, path) => { + fileMocks.processFilesToUserFiles.mockReturnValue([ + { key: 'workspace/file.csv', name: 'file.csv', type: 'text/csv' }, + ]) + fileMocks.downloadServableFileFromStorage.mockResolvedValue({ + buffer: Buffer.from('id,name'), + contentType: 'text/csv', + }) + const providerBody = + operation === 'sailpoint_load_accounts' + ? { success: true, task: { id: 'task' } } + : { id: 'task', uniqueName: 'aggregation' } + mockFetch + .mockResolvedValueOnce(tokenResponse()) + .mockResolvedValueOnce(Response.json(providerBody, { status: 202 })) + + const response = await request( + operation, + { sourceId: 'source', file: { key: 'file', name: 'file.csv', size: 7 } }, + 'user' + ) + expect(response.status).toBe(200) + const envelope = await response.clone().json() + if (operation === 'sailpoint_load_accounts') { + expect(envelope.output).toEqual({ success: true, task: { id: 'task' } }) + } else { + expect(envelope.output).toEqual({ task: providerBody }) + } + expect(fileMocks.assertToolFileAccess).toHaveBeenCalledWith( + 'workspace/file.csv', + 'user', + 'request-id', + expect.anything() + ) + expect(fileMocks.downloadServableFileFromStorage).toHaveBeenCalledWith( + expect.objectContaining({ key: 'workspace/file.csv' }), + 'request-id', + expect.anything(), + { maxBytes: MAX_SAILPOINT_CSV_BYTES, signal: undefined } + ) + expect(new URL(String(mockFetch.mock.calls[1][0])).pathname).toBe(path) + }) + + it('parses a serialized file descriptor and preserves an empty selected CSV', async () => { + fileMocks.processFilesToUserFiles.mockReturnValue([ + { key: 'workspace/empty.csv', name: 'empty.csv', type: 'text/csv' }, + ]) + fileMocks.downloadServableFileFromStorage.mockResolvedValue({ + buffer: Buffer.alloc(0), + contentType: 'text/csv', + }) + mockFetch + .mockResolvedValueOnce(tokenResponse()) + .mockResolvedValueOnce( + Response.json({ success: true, task: { id: 'task' } }, { status: 202 }) + ) + const serializedFile = JSON.stringify({ key: 'empty', name: 'empty.csv', size: 0 }) + + const response = await request( + 'sailpoint_load_accounts', + { sourceId: 'source', file: serializedFile }, + 'user' + ) + + expect(response.status).toBe(200) + expect(fileMocks.processFilesToUserFiles).toHaveBeenCalledWith( + [{ key: 'empty', name: 'empty.csv', size: 0 }], + 'request-id', + expect.anything() + ) + const form = mockFetch.mock.calls[1][1]?.body as FormData + const uploaded = form.get('file') + expect(uploaded).toBeInstanceOf(Blob) + expect((uploaded as Blob).size).toBe(0) + }) + + it('maps an oversized stored CSV to a bounded validation error', async () => { + fileMocks.processFilesToUserFiles.mockReturnValue([ + { key: 'workspace/file.csv', name: 'file.csv', type: 'text/csv' }, + ]) + fileMocks.downloadServableFileFromStorage.mockRejectedValue( + new PayloadSizeLimitError({ + label: 'SailPoint CSV', + maxBytes: MAX_SAILPOINT_CSV_BYTES, + observedBytes: MAX_SAILPOINT_CSV_BYTES + 1, + }) + ) + const response = await request( + 'sailpoint_load_accounts', + { sourceId: 'source', file: { key: 'file', name: 'file.csv', size: 7 } }, + 'user' + ) + expect(response.status).toBe(400) + await expect(response.json()).resolves.toMatchObject({ + success: false, + error: 'SailPoint CSV file exceeds the 25 MiB limit', + }) + expect(mockFetch).not.toHaveBeenCalled() + }) + + it('allows actorless provider calls but rejects stored-file loads without an actor', async () => { + mockFetch + .mockResolvedValueOnce(tokenResponse()) + .mockResolvedValueOnce(Response.json({ id: 'identity' })) + const providerResponse = await request('sailpoint_get_identity', { id: 'identity' }) + expect(providerResponse.status).toBe(200) + + clearSailPointTokenStateForTests() + mockFetch.mockReset() + const fileResponse = await request('sailpoint_load_accounts', { + sourceId: 'source', + file: { key: 'file', name: 'file.csv', size: 7 }, + }) + expect(fileResponse.status).toBe(401) + expect(fileMocks.processFilesToUserFiles).not.toHaveBeenCalled() + expect(mockFetch).not.toHaveBeenCalled() + }) +}) diff --git a/apps/sim/lib/internal/sailpoint/execute-tool.ts b/apps/sim/lib/internal/sailpoint/execute-tool.ts new file mode 100644 index 00000000000..1bd9ad3e0da --- /dev/null +++ b/apps/sim/lib/internal/sailpoint/execute-tool.ts @@ -0,0 +1,70 @@ +import { getErrorMessage } from '@sim/utils/errors' +import { isRecordLike } from '@sim/utils/object' +import { getValidationErrorMessage } from '@/lib/api/server' +import { DEFAULT_MAX_JSON_BODY_BYTES } from '@/lib/api/server/validation' +import { isPayloadSizeLimitError } from '@/lib/core/utils/stream-limits' +import { executeSailPointOperation } from '@/lib/internal/sailpoint/operations' +import { parseSailPointInput } from '@/lib/internal/sailpoint/schema' +import type { InternalToolOperationHandler } from '@/lib/internal/tool-operations/types' + +export const executeSailPointTool: InternalToolOperationHandler = async (request) => { + request.signal?.throwIfAborted() + let serializedInput: string + try { + serializedInput = JSON.stringify(request.input) ?? '' + } catch { + return Response.json({ success: false, error: 'Invalid request data' }, { status: 400 }) + } + if (Buffer.byteLength(serializedInput, 'utf8') > DEFAULT_MAX_JSON_BODY_BYTES) { + return Response.json( + { + success: false, + error: `Request body exceeds the maximum allowed size of ${DEFAULT_MAX_JSON_BODY_BYTES} bytes`, + }, + { status: 413 } + ) + } + + if (!isRecordLike(request.input) || request.input.operation !== request.toolId) { + return Response.json( + { success: false, error: 'SailPoint input operation must match the executing tool ID' }, + { status: 400 } + ) + } + + const parsed = parseSailPointInput(request.toolId, request.input) + if (!parsed) { + return Response.json( + { success: false, error: `Unsupported SailPoint tool: ${request.toolId}` }, + { status: 500 } + ) + } + if (!parsed.success) { + return Response.json( + { + success: false, + error: getValidationErrorMessage(parsed.error, 'Invalid SailPoint request'), + }, + { status: 400 } + ) + } + + try { + const response = await executeSailPointOperation(parsed.data, { + requestId: request.requestId, + signal: request.signal, + userId: request.context.userId, + }) + request.signal?.throwIfAborted() + return response + } catch (error) { + request.signal?.throwIfAborted() + return Response.json( + { + success: false, + error: getErrorMessage(error, 'SailPoint request failed'), + }, + { status: isPayloadSizeLimitError(error) ? 502 : 500 } + ) + } +} diff --git a/apps/sim/lib/internal/sailpoint/operations.ts b/apps/sim/lib/internal/sailpoint/operations.ts new file mode 100644 index 00000000000..1d3b5fa24de --- /dev/null +++ b/apps/sim/lib/internal/sailpoint/operations.ts @@ -0,0 +1,740 @@ +import { createLogger } from '@sim/logger' +import { getErrorMessage } from '@sim/utils/errors' +import { filterUndefined, isRecordLike } from '@sim/utils/object' +import { isPayloadSizeLimitError } from '@/lib/core/utils/stream-limits' +import { + getSailPointErrorMessage, + readTotalCount, + type SailPointCredentials, + type SailPointFetchResult, + type SailPointHosts, + sailpointFetch, +} from '@/lib/internal/sailpoint/client' +import type { SailPointInput } from '@/lib/internal/sailpoint/schema' +import { parseRawFileInput } from '@/lib/uploads/utils/file-schemas' +import { processFilesToUserFiles } from '@/lib/uploads/utils/file-utils' +import { downloadServableFileFromStorage } from '@/lib/uploads/utils/file-utils.server' +import { docNotReadyResponse } from '@/lib/uploads/utils/servable-file-response' +import { assertToolFileAccess } from '@/app/api/files/authorization' + +const logger = createLogger('SailPointOperations') + +export const MAX_SAILPOINT_CSV_BYTES = 25 * 1024 * 1024 + +export interface SailPointOperationContext { + requestId: string + signal?: AbortSignal + userId?: string +} + +type InputRecord = SailPointInput & Record +type ResourceKey = + | 'accessProfile' + | 'accessRequestConfig' + | 'account' + | 'accountActivity' + | 'accountSelections' + | 'campaign' + | 'certification' + | 'entitlement' + | 'entitlementRequestConfig' + | 'identity' + | 'role' + | 'source' + | 'task' +type ResultKind = + | 'aggregate' + | 'count' + | 'list' + | 'request-access' + | 'search' + | 'write' + | ResourceKey + +const RESOURCE_KEYS = new Set([ + 'accessProfile', + 'accessRequestConfig', + 'account', + 'accountActivity', + 'accountSelections', + 'campaign', + 'certification', + 'entitlement', + 'entitlementRequestConfig', + 'identity', + 'role', + 'source', + 'task', +]) + +function queryString(params: Record): string { + const query = new URLSearchParams() + for (const [key, value] of Object.entries(params)) { + if (value === undefined || value === null || value === '') continue + query.set(key, String(value)) + } + const serialized = query.toString() + return serialized ? `?${serialized}` : '' +} + +function encodeId(value: unknown): string { + return encodeURIComponent(String(value)) +} + +function toStringList(value: unknown): string[] | undefined { + if (value == null) return undefined + const normalize = (entry: unknown): string | null => { + if (typeof entry === 'string') return entry.trim() || null + if (typeof entry === 'number' || typeof entry === 'boolean') return String(entry) + return null + } + if (Array.isArray(value)) { + const values = value.map(normalize).filter((entry): entry is string => entry !== null) + return values.length ? values : undefined + } + if (typeof value === 'string') { + const values = value + .split(',') + .map((entry) => entry.trim()) + .filter(Boolean) + return values.length ? values : undefined + } + return undefined +} + +function jsonRequest(body: unknown): RequestInit { + return { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify(body), + } +} + +function searchBody(input: InputRecord): Record { + return filterUndefined({ + indices: toStringList(input.indices), + queryType: input.queryType, + queryVersion: input.queryVersion, + query: typeof input.query === 'string' ? { query: input.query } : input.query, + queryDsl: input.queryDsl, + textQuery: input.textQuery, + typeAheadQuery: input.typeAheadQuery, + includeNested: input.includeNested, + queryResultFilter: input.queryResultFilter, + aggregationType: + input.aggregationType ?? (input.aggregationsDsl !== undefined ? 'DSL' : undefined), + aggregationsVersion: input.aggregationsVersion, + aggregationsDsl: input.aggregationsDsl, + aggregations: input.aggregations, + sort: toStringList(input.sort), + searchAfter: toStringList(input.searchAfter), + filters: input.filters, + }) +} + +function accessRequestBody(input: InputRecord): Record { + return filterUndefined({ + requestedFor: input.requestedFor, + requestedItems: input.requestedItems, + requestedForWithRequestedItems: input.requestedForWithRequestedItems, + requestType: input.requestType, + clientMetadata: input.clientMetadata, + }) +} + +function failureResponse(error: string, status: number): Response { + return Response.json({ success: false, error }, { status }) +} + +function providerFailure(result: SailPointFetchResult): Response { + return failureResponse( + getSailPointErrorMessage(result.data, 'SailPoint request failed'), + result.status || 502 + ) +} + +function requireArray(result: SailPointFetchResult): unknown[] | Response { + if (Array.isArray(result.data)) return result.data + return failureResponse('SailPoint returned an invalid list response', 502) +} + +function outputForResult(result: SailPointFetchResult, kind: ResultKind): Response { + if (!result.ok) return providerFailure(result) + + if (kind === 'list' || kind === 'search') { + const items = requireArray(result) + if (items instanceof Response) return items + const output = { + count: items.length, + totalCount: readTotalCount(result.headers), + ...(kind === 'search' ? { results: items } : { items }), + } + return Response.json({ success: true, output }) + } + + if (kind === 'count') { + const total = + readTotalCount(result.headers) ?? (typeof result.data === 'number' ? result.data : null) + if (total === null) { + return failureResponse('SailPoint did not return X-Total-Count', 502) + } + return Response.json({ success: true, output: { total } }) + } + + if (kind === 'aggregate') { + if (!isRecordLike(result.data)) { + return failureResponse('SailPoint returned an invalid aggregate response', 502) + } + const aggregate = result.data + if (aggregate.aggregations !== undefined && !isRecordLike(aggregate.aggregations)) { + return failureResponse('SailPoint returned invalid aggregation results', 502) + } + if (aggregate.hits !== undefined && !Array.isArray(aggregate.hits)) { + return failureResponse('SailPoint returned invalid aggregation hits', 502) + } + return Response.json({ + success: true, + output: { + aggregations: aggregate.aggregations ?? null, + hits: Array.isArray(aggregate.hits) ? aggregate.hits : [], + totalCount: readTotalCount(result.headers), + }, + }) + } + + if (RESOURCE_KEYS.has(kind)) { + if (!isRecordLike(result.data)) { + return failureResponse('SailPoint returned an invalid resource response', 502) + } + return Response.json({ success: true, output: { [kind]: result.data } }) + } + + if (kind === 'request-access') { + if (!isRecordLike(result.data)) { + return failureResponse('SailPoint returned an invalid access-request response', 502) + } + const response = result.data + if (response.newRequests !== undefined && !Array.isArray(response.newRequests)) { + return failureResponse('SailPoint returned invalid new access-request records', 502) + } + if (response.existingRequests !== undefined && !Array.isArray(response.existingRequests)) { + return failureResponse('SailPoint returned invalid existing access-request records', 502) + } + return Response.json({ + success: true, + output: { + accepted: true, + status: result.status, + newRequests: Array.isArray(response.newRequests) ? response.newRequests : [], + existingRequests: Array.isArray(response.existingRequests) ? response.existingRequests : [], + }, + }) + } + + return Response.json({ + success: true, + output: { accepted: true, status: result.status }, + }) +} + +async function executeRequest( + credentials: SailPointCredentials, + context: SailPointOperationContext, + buildRequest: (hosts: SailPointHosts) => { init: RequestInit; url: string }, + kind: ResultKind +): Promise { + const result = await sailpointFetch(credentials, buildRequest, { signal: context.signal }) + return outputForResult(result, kind) +} + +async function executeLoad( + input: InputRecord, + credentials: SailPointCredentials, + context: SailPointOperationContext +): Promise { + context.signal?.throwIfAborted() + let fileBuffer: Buffer | null = null + let fileName = 'aggregation.csv' + let fileType = 'text/csv' + + if (input.file != null) { + if (!context.userId) return failureResponse('Authentication required for stored files', 401) + const parsedFile = parseRawFileInput(input.file) + if (!parsedFile) return failureResponse('Invalid file input', 400) + const userFiles = processFilesToUserFiles([parsedFile], context.requestId, logger) + const userFile = userFiles[0] + if (!userFile) return failureResponse('Invalid file input', 400) + + const denied = await assertToolFileAccess( + userFile.key, + context.userId, + context.requestId, + logger + ) + context.signal?.throwIfAborted() + if (denied) return denied + + try { + const downloaded = await downloadServableFileFromStorage( + userFile, + context.requestId, + logger, + { maxBytes: MAX_SAILPOINT_CSV_BYTES, signal: context.signal } + ) + fileBuffer = downloaded.buffer + fileName = userFile.name || fileName + fileType = userFile.type || fileType + } catch (error) { + context.signal?.throwIfAborted() + const notReady = docNotReadyResponse(error) + if (notReady) return notReady + if (isPayloadSizeLimitError(error)) { + return failureResponse('SailPoint CSV file exceeds the 25 MiB limit', 400) + } + logger.error('Failed to download SailPoint CSV file', { + error: getErrorMessage(error), + requestId: context.requestId, + }) + return failureResponse(getErrorMessage(error, 'Failed to download file'), 500) + } + } + + const isAccountLoad = input.operation === 'sailpoint_load_accounts' + const path = isAccountLoad + ? `/sources/v1/${encodeId(input.sourceId)}/load-accounts` + : `/sources/v1/${encodeId(input.sourceId)}/load-entitlements` + + const result = await sailpointFetch( + credentials, + (hosts) => { + const form = new FormData() + if (fileBuffer !== null) { + form.append('file', new Blob([new Uint8Array(fileBuffer)], { type: fileType }), fileName) + } + if (isAccountLoad && input.disableOptimization === true) { + form.append('disableOptimization', 'true') + } + return { url: `${hosts.apiBaseUrl}${path}`, init: { method: 'POST', body: form } } + }, + { signal: context.signal } + ) + if (!result.ok) return providerFailure(result) + if (isAccountLoad) { + const body = isRecordLike(result.data) ? result.data : null + if (!body || !isRecordLike(body.task)) { + return failureResponse('SailPoint returned an invalid account-load task response', 502) + } + return Response.json({ + success: true, + output: { success: body.success === true, task: body.task ?? null }, + }) + } + if (!isRecordLike(result.data)) { + return failureResponse('SailPoint returned an invalid entitlement-load task response', 502) + } + return Response.json({ success: true, output: { task: result.data } }) +} + +export async function executeSailPointOperation( + parsedInput: SailPointInput, + context: SailPointOperationContext +): Promise { + const input = parsedInput as InputRecord + const credentials: SailPointCredentials = { + clientId: String(input.clientId), + clientSecret: String(input.clientSecret), + tenant: String(input.tenant), + } + + switch (input.operation) { + case 'sailpoint_search': { + return executeRequest( + credentials, + context, + (hosts) => ({ + url: `${hosts.apiBaseUrl}/search/v1${queryString({ limit: input.limit, offset: input.offset, count: input.count })}`, + init: jsonRequest(searchBody(input)), + }), + 'search' + ) + } + case 'sailpoint_search_count': + return executeRequest( + credentials, + context, + (hosts) => ({ + url: `${hosts.apiBaseUrl}/search/v1/count`, + init: jsonRequest(searchBody(input)), + }), + 'count' + ) + case 'sailpoint_search_aggregate': + return executeRequest( + credentials, + context, + (hosts) => ({ + url: `${hosts.apiBaseUrl}/search/v1/aggregate${queryString({ limit: input.limit, offset: input.offset, count: input.count })}`, + init: jsonRequest(searchBody(input)), + }), + 'aggregate' + ) + case 'sailpoint_list_identities': + return executeRequest( + credentials, + context, + (hosts) => ({ + url: `${hosts.apiBaseUrl}/identities/v1${queryString({ filters: input.filters, sorters: input.sorters, defaultFilter: input.defaultFilter, limit: input.limit, offset: input.offset, count: input.count })}`, + init: { method: 'GET' }, + }), + 'list' + ) + case 'sailpoint_get_identity': + return executeRequest( + credentials, + context, + (hosts) => ({ + url: `${hosts.apiBaseUrl}/identities/v1/${encodeId(input.id)}`, + init: { method: 'GET' }, + }), + 'identity' + ) + case 'sailpoint_list_identity_entitlements': + return executeRequest( + credentials, + context, + (hosts) => ({ + url: `${hosts.apiBaseUrl}/entitlements/v1/identities/${encodeId(input.id)}/entitlements${queryString({ limit: input.limit, offset: input.offset, count: input.count })}`, + init: { method: 'GET' }, + }), + 'list' + ) + case 'sailpoint_list_accounts': + return executeRequest( + credentials, + context, + (hosts) => ({ + url: `${hosts.apiBaseUrl}/accounts/v1${queryString({ filters: input.filters, sorters: input.sorters, detailLevel: input.detailLevel, limit: input.limit, offset: input.offset, count: input.count })}`, + init: { method: 'GET' }, + }), + 'list' + ) + case 'sailpoint_get_account': + return executeRequest( + credentials, + context, + (hosts) => ({ + url: `${hosts.apiBaseUrl}/accounts/v1/${encodeId(input.id)}`, + init: { method: 'GET' }, + }), + 'account' + ) + case 'sailpoint_get_account_entitlements': + return executeRequest( + credentials, + context, + (hosts) => ({ + url: `${hosts.apiBaseUrl}/accounts/v1/${encodeId(input.id)}/entitlements${queryString({ limit: input.limit, offset: input.offset, count: input.count })}`, + init: { method: 'GET' }, + }), + 'list' + ) + case 'sailpoint_list_entitlements': + return executeRequest( + credentials, + context, + (hosts) => ({ + url: `${hosts.apiBaseUrl}/entitlements/v1${queryString({ filters: input.filters, sorters: input.sorters, 'segmented-for-identity': input.segmentedForIdentity, 'for-segment-ids': input.forSegmentIds, 'include-unsegmented': input.includeUnsegmented, searchAfter: input.searchAfter, limit: input.limit, offset: input.offset, count: input.count })}`, + init: { method: 'GET' }, + }), + 'list' + ) + case 'sailpoint_get_entitlement': + return executeRequest( + credentials, + context, + (hosts) => ({ + url: `${hosts.apiBaseUrl}/entitlements/v1/${encodeId(input.id)}`, + init: { method: 'GET' }, + }), + 'entitlement' + ) + case 'sailpoint_get_entitlement_request_config': + return executeRequest( + credentials, + context, + (hosts) => ({ + url: `${hosts.apiBaseUrl}/entitlements/v1/${encodeId(input.id)}/entitlement-request-config`, + init: { method: 'GET' }, + }), + 'entitlementRequestConfig' + ) + case 'sailpoint_list_roles': + return executeRequest( + credentials, + context, + (hosts) => ({ + url: `${hosts.apiBaseUrl}/roles/v1${queryString({ filters: input.filters, sorters: input.sorters, 'for-subadmin': input.forSubadmin, 'for-segment-ids': input.forSegmentIds, 'include-unsegmented': input.includeUnsegmented, limit: input.limit, offset: input.offset, count: input.count })}`, + init: { method: 'GET' }, + }), + 'list' + ) + case 'sailpoint_get_role': + return executeRequest( + credentials, + context, + (hosts) => ({ + url: `${hosts.apiBaseUrl}/roles/v1/${encodeId(input.id)}`, + init: { method: 'GET' }, + }), + 'role' + ) + case 'sailpoint_get_role_entitlements': + return executeRequest( + credentials, + context, + (hosts) => ({ + url: `${hosts.apiBaseUrl}/roles/v1/${encodeId(input.id)}/entitlements${queryString({ filters: input.filters, sorters: input.sorters, limit: input.limit, offset: input.offset, count: input.count })}`, + init: { method: 'GET' }, + }), + 'list' + ) + case 'sailpoint_list_access_profiles': + return executeRequest( + credentials, + context, + (hosts) => ({ + url: `${hosts.apiBaseUrl}/access-profiles/v1${queryString({ filters: input.filters, sorters: input.sorters, 'for-subadmin': input.forSubadmin, 'for-segment-ids': input.forSegmentIds, 'include-unsegmented': input.includeUnsegmented, limit: input.limit, offset: input.offset, count: input.count })}`, + init: { method: 'GET' }, + }), + 'list' + ) + case 'sailpoint_get_access_profile': + return executeRequest( + credentials, + context, + (hosts) => ({ + url: `${hosts.apiBaseUrl}/access-profiles/v1/${encodeId(input.id)}`, + init: { method: 'GET' }, + }), + 'accessProfile' + ) + case 'sailpoint_get_access_profile_entitlements': + return executeRequest( + credentials, + context, + (hosts) => ({ + url: `${hosts.apiBaseUrl}/access-profiles/v1/${encodeId(input.id)}/entitlements${queryString({ filters: input.filters, sorters: input.sorters, limit: input.limit, offset: input.offset, count: input.count })}`, + init: { method: 'GET' }, + }), + 'list' + ) + case 'sailpoint_list_sources': + return executeRequest( + credentials, + context, + (hosts) => ({ + url: `${hosts.apiBaseUrl}/sources/v1${queryString({ filters: input.filters, sorters: input.sorters, 'for-subadmin': input.forSubadmin, includeIDNSource: input.includeIDNSource, limit: input.limit, offset: input.offset, count: input.count })}`, + init: { method: 'GET' }, + }), + 'list' + ) + case 'sailpoint_get_source': + return executeRequest( + credentials, + context, + (hosts) => ({ + url: `${hosts.apiBaseUrl}/sources/v1/${encodeId(input.id)}`, + init: { method: 'GET' }, + }), + 'source' + ) + case 'sailpoint_list_account_activities': + return executeRequest( + credentials, + context, + (hosts) => ({ + url: `${hosts.apiBaseUrl}/account-activities/v1${queryString({ 'requested-for': input.requestedFor, 'requested-by': input.requestedBy, 'regarding-identity': input.regardingIdentity, filters: input.filters, sorters: input.sorters, limit: input.limit, offset: input.offset, count: input.count })}`, + init: { method: 'GET' }, + }), + 'list' + ) + case 'sailpoint_get_account_activity': + return executeRequest( + credentials, + context, + (hosts) => ({ + url: `${hosts.apiBaseUrl}/account-activities/v1/${encodeId(input.id)}`, + init: { method: 'GET' }, + }), + 'accountActivity' + ) + case 'sailpoint_list_campaigns': + return executeRequest( + credentials, + context, + (hosts) => ({ + url: `${hosts.apiBaseUrl}/campaigns/v1${queryString({ detail: input.detail, filters: input.filters, sorters: input.sorters, limit: input.limit, offset: input.offset, count: input.count })}`, + init: { method: 'GET' }, + }), + 'list' + ) + case 'sailpoint_get_campaign': + return executeRequest( + credentials, + context, + (hosts) => ({ + url: `${hosts.apiBaseUrl}/campaigns/v1/${encodeId(input.id)}${queryString({ detail: input.detail })}`, + init: { method: 'GET' }, + }), + 'campaign' + ) + case 'sailpoint_list_certifications': + return executeRequest( + credentials, + context, + (hosts) => ({ + url: `${hosts.apiBaseUrl}/certifications/v1${queryString({ 'reviewer-identity': input.reviewerIdentity, filters: input.filters, sorters: input.sorters, limit: input.limit, offset: input.offset, count: input.count })}`, + init: { method: 'GET' }, + }), + 'list' + ) + case 'sailpoint_get_certification': + return executeRequest( + credentials, + context, + (hosts) => ({ + url: `${hosts.apiBaseUrl}/certifications/v1/${encodeId(input.id)}`, + init: { method: 'GET' }, + }), + 'certification' + ) + case 'sailpoint_list_certification_review_items': + return executeRequest( + credentials, + context, + (hosts) => ({ + url: `${hosts.apiBaseUrl}/certifications/v1/${encodeId(input.id)}/access-review-items${queryString({ filters: input.filters, sorters: input.sorters, entitlements: input.entitlements, 'access-profiles': input.accessProfiles, roles: input.roles, limit: input.limit, offset: input.offset, count: input.count })}`, + init: { method: 'GET' }, + }), + 'list' + ) + case 'sailpoint_decide_certification_review_items': + return executeRequest( + credentials, + context, + (hosts) => ({ + url: `${hosts.apiBaseUrl}/certifications/v1/${encodeId(input.id)}/decide`, + init: jsonRequest(input.decisions), + }), + 'certification' + ) + case 'sailpoint_sign_off_certification': + return executeRequest( + credentials, + context, + (hosts) => ({ + url: `${hosts.apiBaseUrl}/certifications/v1/${encodeId(input.id)}/sign-off`, + init: { method: 'POST' }, + }), + 'certification' + ) + case 'sailpoint_request_access': + return executeRequest( + credentials, + context, + (hosts) => ({ + url: `${hosts.apiBaseUrl}/access-requests/v1`, + init: jsonRequest(accessRequestBody(input)), + }), + 'request-access' + ) + case 'sailpoint_get_account_selections': + return executeRequest( + credentials, + context, + (hosts) => ({ + url: `${hosts.apiBaseUrl}/access-requests/v1/accounts-selection`, + init: { + ...jsonRequest(accessRequestBody(input)), + headers: { + 'Content-Type': 'application/json', + 'X-SailPoint-Experimental': 'true', + }, + }, + }), + 'accountSelections' + ) + case 'sailpoint_get_access_request_config': + return executeRequest( + credentials, + context, + (hosts) => ({ + url: `${hosts.apiBaseUrl}/access-request-config/v2`, + init: { method: 'GET' }, + }), + 'accessRequestConfig' + ) + case 'sailpoint_cancel_access_request': + return executeRequest( + credentials, + context, + (hosts) => ({ + url: `${hosts.apiBaseUrl}/access-requests/v1/cancel`, + init: jsonRequest({ + accountActivityId: input.accountActivityId, + comment: input.comment, + }), + }), + 'write' + ) + case 'sailpoint_get_access_request_status': + return executeRequest( + credentials, + context, + (hosts) => ({ + url: `${hosts.apiBaseUrl}/access-request-status/v1${queryString({ 'requested-for': input.requestedFor, 'requested-by': input.requestedBy, 'regarding-identity': input.regardingIdentity, 'assigned-to': input.assignedTo, 'request-state': input.requestState, filters: input.filters, sorters: input.sorters, limit: input.limit, offset: input.offset, count: input.count })}`, + init: { method: 'GET' }, + }), + 'list' + ) + case 'sailpoint_list_pending_access_request_approvals': + return executeRequest( + credentials, + context, + (hosts) => ({ + url: `${hosts.apiBaseUrl}/access-request-approvals/v1/pending${queryString({ 'owner-id': input.ownerId, filters: input.filters, sorters: input.sorters, limit: input.limit, offset: input.offset, count: input.count })}`, + init: { method: 'GET' }, + }), + 'list' + ) + case 'sailpoint_approve_access_request': + case 'sailpoint_reject_access_request': { + const action = input.operation === 'sailpoint_approve_access_request' ? 'approve' : 'reject' + const init = input.comment + ? jsonRequest({ comment: input.comment }) + : { method: 'POST' as const } + return executeRequest( + credentials, + context, + (hosts) => ({ + url: `${hosts.apiBaseUrl}/access-request-approvals/v1/${encodeId(input.approvalId)}/${action}`, + init, + }), + 'write' + ) + } + case 'sailpoint_get_task_status': + return executeRequest( + credentials, + context, + (hosts) => ({ + url: `${hosts.apiBaseUrl}/task-status/v1/${encodeId(input.id)}`, + init: { method: 'GET' }, + }), + 'task' + ) + case 'sailpoint_load_accounts': + case 'sailpoint_load_entitlements': + return executeLoad(input, credentials, context) + } +} diff --git a/apps/sim/lib/internal/sailpoint/schema.ts b/apps/sim/lib/internal/sailpoint/schema.ts new file mode 100644 index 00000000000..f7462bb2a9f --- /dev/null +++ b/apps/sim/lib/internal/sailpoint/schema.ts @@ -0,0 +1,820 @@ +import { isRecordLike } from '@sim/utils/object' +import { z } from 'zod' +import { FileInputSchema } from '@/lib/uploads/utils/file-schemas' + +const MAX_ID_LENGTH = 1024 +const MAX_FILTER_LENGTH = 20_000 +const MAX_QUERY_LENGTH = 100_000 +const MAX_METADATA_ENTRIES = 100 +const STANDARD_LIMIT_MAX = 250 +const SEARCH_LIMIT_MAX = 10_000 +const ROLE_LIMIT_MAX = 50 + +function parseJson(value: unknown): unknown { + if (typeof value !== 'string') return value + try { + return JSON.parse(value) + } catch { + return value + } +} + +const requiredString = (label: string, max = MAX_ID_LENGTH) => + z.string().trim().min(1, `${label} is required`).max(max, `${label} is too long`) + +const optionalString = (max = MAX_FILTER_LENGTH) => z.string().max(max).optional() + +const baseFields = { + clientId: requiredString('Client ID'), + clientSecret: requiredString('Client Secret', 8192), + tenant: requiredString('Tenant', 253), +} + +const offsetField = z.coerce.number().int().min(0).optional() +const countField = z.boolean().optional() +const limitField = (max: number) => z.coerce.number().int().min(0).max(max).optional() +const pagination = (max = STANDARD_LIMIT_MAX) => ({ + limit: limitField(max), + offset: offsetField, + count: countField, +}) +const listFields = (max = STANDARD_LIMIT_MAX) => ({ + filters: optionalString(), + sorters: optionalString(), + ...pagination(max), +}) + +const stringListField = z.preprocess( + parseJson, + z.union([z.array(z.string().max(MAX_FILTER_LENGTH)).max(100), z.string()]).optional() +) + +const jsonObjectField = z.preprocess(parseJson, z.record(z.string(), z.unknown())).optional() +const searchQuerySchema = z.object({ + query: z.string().max(MAX_QUERY_LENGTH).optional(), + fields: z.string().max(MAX_FILTER_LENGTH).optional(), + timeZone: z.string().max(255).optional(), + innerHit: z.record(z.string(), z.unknown()).optional(), +}) +const textQuerySchema = z.object({ + terms: z.array(z.string().max(MAX_QUERY_LENGTH)).min(1).max(100), + fields: z.array(z.string().max(MAX_FILTER_LENGTH)).min(1).max(100), + matchAny: z.boolean().optional(), + contains: z.boolean().optional(), +}) +const typeAheadQuerySchema = z.object({ + query: z.string().max(MAX_QUERY_LENGTH), + field: z.string().max(MAX_FILTER_LENGTH), + nestedType: z.string().max(MAX_FILTER_LENGTH).optional(), + maxExpansions: z.coerce.number().int().min(1).max(1000).optional(), + size: z.coerce.number().int().min(1).optional(), + sort: z.string().max(255).optional(), + sortByValue: z.boolean().optional(), +}) +const queryResultFilterSchema = z.object({ + includes: z.array(z.string().max(MAX_FILTER_LENGTH)).max(1000).optional(), + excludes: z.array(z.string().max(MAX_FILTER_LENGTH)).max(1000).optional(), +}) +const searchFields = { + indices: stringListField, + queryType: z.enum(['DSL', 'SAILPOINT', 'TEXT', 'TYPEAHEAD']).optional(), + queryVersion: z.string().max(64).optional(), + query: z + .preprocess(parseJson, z.union([z.string().max(MAX_QUERY_LENGTH), searchQuerySchema])) + .optional(), + queryDsl: jsonObjectField, + textQuery: z.preprocess(parseJson, textQuerySchema).optional(), + typeAheadQuery: z.preprocess(parseJson, typeAheadQuerySchema).optional(), + includeNested: z.boolean().optional(), + queryResultFilter: z.preprocess(parseJson, queryResultFilterSchema).optional(), + aggregationType: z.enum(['DSL', 'SAILPOINT']).optional(), + aggregationsVersion: z.string().max(64).optional(), + aggregationsDsl: jsonObjectField, + aggregations: jsonObjectField, + sort: stringListField, + searchAfter: stringListField, + filters: jsonObjectField, +} + +function validateSearchQuerySelection( + value: Record, + ctx: z.RefinementCtx, + queryRequired: boolean +): void { + const hasQueryInput = + value.query !== undefined || + value.queryDsl !== undefined || + value.textQuery !== undefined || + value.typeAheadQuery !== undefined + if (!queryRequired && value.queryType === undefined && !hasQueryInput) return + + const queryType = value.queryType ?? 'SAILPOINT' + const requiredField = { + DSL: 'queryDsl', + SAILPOINT: 'query', + TEXT: 'textQuery', + TYPEAHEAD: 'typeAheadQuery', + }[queryType as 'DSL' | 'SAILPOINT' | 'TEXT' | 'TYPEAHEAD'] + if (value[requiredField] === undefined) { + ctx.addIssue({ + code: z.ZodIssueCode.custom, + path: [requiredField], + message: `${requiredField} is required when queryType is ${queryType}`, + }) + } +} + +const metadataSchema = z + .record(z.string().max(1024), z.string().max(10_000)) + .refine((value) => Object.keys(value).length <= MAX_METADATA_ENTRIES, { + message: `Metadata may contain at most ${MAX_METADATA_ENTRIES} entries`, + }) + +const requestedItemSchema = z.object({ + type: z.enum(['ACCESS_PROFILE', 'ROLE', 'ENTITLEMENT']), + id: requiredString('Requested item ID'), + comment: optionalString(10_000), + removeDate: z.string().datetime({ offset: true }).optional(), + startDate: z.string().datetime({ offset: true }).optional(), + assignmentId: optionalString(MAX_ID_LENGTH).nullable(), + nativeIdentity: optionalString(10_000).nullable(), + formInstanceId: optionalString(MAX_ID_LENGTH).nullable(), + clientMetadata: metadataSchema.optional(), +}) + +const sourceItemRefSchema = z.object({ + sourceId: z.string().max(MAX_ID_LENGTH).nullable().optional(), + accounts: z + .array( + z + .object({ + accountUuid: z.string().trim().min(1).max(MAX_ID_LENGTH).nullable().optional(), + nativeIdentity: z.string().trim().min(1).max(10_000).optional(), + }) + .superRefine((value, ctx) => { + if (!value.accountUuid && !value.nativeIdentity) { + ctx.addIssue({ + code: z.ZodIssueCode.custom, + path: ['accountUuid'], + message: 'accountUuid or nativeIdentity is required', + }) + } + }) + ) + .max(100) + .nullable() + .optional(), +}) + +const nestedRequestedItemSchema = requestedItemSchema.omit({ assignmentId: true }).extend({ + accountSelection: z.array(sourceItemRefSchema).max(100).nullable().optional(), +}) + +const requestedForWithItemsSchema = z.object({ + identityId: requiredString('Identity ID'), + identityType: z.enum(['HUMAN', 'MACHINE']).optional(), + requestedItems: z.array(nestedRequestedItemSchema).min(1).max(250), +}) + +const reviewDecisionSchema = z + .object({ + id: requiredString('Review item ID'), + decision: z.enum(['APPROVE', 'REVOKE']), + proposedEndDate: z.string().datetime({ offset: true }).optional(), + bulk: z.boolean(), + recommendation: z + .object({ + recommendation: z.string().nullable().optional(), + reasons: z.array(z.string().max(10_000)).max(100).optional(), + timestamp: z.string().datetime({ offset: true }).optional(), + }) + .nullable() + .optional(), + comments: optionalString(10_000), + }) + .superRefine((value, ctx) => { + if (value.decision !== 'REVOKE' && value.proposedEndDate) { + ctx.addIssue({ + code: z.ZodIssueCode.custom, + path: ['proposedEndDate'], + message: 'proposedEndDate is only allowed for REVOKE decisions', + }) + } + }) + +function operationSchema(operation: T, fields: S) { + return z.object({ ...baseFields, operation: z.literal(operation), ...fields }) +} + +const schemas = { + sailpoint_search: operationSchema('sailpoint_search', { + ...searchFields, + ...pagination(SEARCH_LIMIT_MAX), + }).superRefine((value, ctx) => validateSearchQuerySelection(value, ctx, true)), + sailpoint_search_count: operationSchema('sailpoint_search_count', { + ...searchFields, + }).superRefine((value, ctx) => validateSearchQuerySelection(value, ctx, true)), + sailpoint_search_aggregate: operationSchema('sailpoint_search_aggregate', { + ...searchFields, + ...pagination(), + }).superRefine((value, ctx) => { + validateSearchQuerySelection(value, ctx, false) + const hasAggregationsDsl = + isRecordLike(value.aggregationsDsl) && Object.keys(value.aggregationsDsl).length > 0 + const hasAggregations = + isRecordLike(value.aggregations) && Object.keys(value.aggregations).length > 0 + if (!hasAggregationsDsl && !hasAggregations) { + ctx.addIssue({ + code: z.ZodIssueCode.custom, + path: ['aggregationsDsl'], + message: 'aggregationsDsl or aggregations must be a non-empty object', + }) + } + }), + sailpoint_list_identities: operationSchema('sailpoint_list_identities', { + ...listFields(), + defaultFilter: z.enum(['CORRELATED_ONLY', 'NONE']).optional(), + }), + sailpoint_get_identity: operationSchema('sailpoint_get_identity', { + id: requiredString('Identity ID'), + }), + sailpoint_list_identity_entitlements: operationSchema('sailpoint_list_identity_entitlements', { + id: requiredString('Identity ID'), + ...pagination(), + }), + sailpoint_list_accounts: operationSchema('sailpoint_list_accounts', { + ...listFields(), + detailLevel: z.enum(['SLIM', 'FULL']).optional(), + }), + sailpoint_get_account: operationSchema('sailpoint_get_account', { + id: requiredString('Account ID'), + }), + sailpoint_get_account_entitlements: operationSchema('sailpoint_get_account_entitlements', { + id: requiredString('Account ID'), + ...pagination(), + }), + sailpoint_list_entitlements: operationSchema('sailpoint_list_entitlements', { + ...listFields(), + segmentedForIdentity: optionalString(MAX_ID_LENGTH), + forSegmentIds: optionalString(MAX_FILTER_LENGTH), + includeUnsegmented: z.boolean().optional(), + searchAfter: optionalString(MAX_FILTER_LENGTH), + }).superRefine((value, ctx) => { + if (value.includeUnsegmented === false && !value.forSegmentIds && !value.segmentedForIdentity) { + ctx.addIssue({ + code: z.ZodIssueCode.custom, + path: ['includeUnsegmented'], + message: 'includeUnsegmented=false requires forSegmentIds or segmentedForIdentity', + }) + } + }), + sailpoint_get_entitlement: operationSchema('sailpoint_get_entitlement', { + id: requiredString('Entitlement ID'), + }), + sailpoint_list_roles: operationSchema('sailpoint_list_roles', { + ...listFields(ROLE_LIMIT_MAX), + forSubadmin: optionalString(MAX_ID_LENGTH), + forSegmentIds: optionalString(MAX_FILTER_LENGTH), + includeUnsegmented: z.boolean().optional(), + }).superRefine((value, ctx) => { + if (value.includeUnsegmented === false && !value.forSegmentIds) { + ctx.addIssue({ + code: z.ZodIssueCode.custom, + path: ['includeUnsegmented'], + message: 'includeUnsegmented=false requires forSegmentIds', + }) + } + }), + sailpoint_get_role: operationSchema('sailpoint_get_role', { id: requiredString('Role ID') }), + sailpoint_get_role_entitlements: operationSchema('sailpoint_get_role_entitlements', { + id: requiredString('Role ID'), + ...listFields(ROLE_LIMIT_MAX), + }), + sailpoint_list_access_profiles: operationSchema('sailpoint_list_access_profiles', { + ...listFields(), + forSubadmin: optionalString(MAX_ID_LENGTH), + forSegmentIds: optionalString(MAX_FILTER_LENGTH), + includeUnsegmented: z.boolean().optional(), + }).superRefine((value, ctx) => { + if (value.includeUnsegmented === false && !value.forSegmentIds) { + ctx.addIssue({ + code: z.ZodIssueCode.custom, + path: ['includeUnsegmented'], + message: 'includeUnsegmented=false requires forSegmentIds', + }) + } + }), + sailpoint_get_access_profile: operationSchema('sailpoint_get_access_profile', { + id: requiredString('Access Profile ID'), + }), + sailpoint_get_access_profile_entitlements: operationSchema( + 'sailpoint_get_access_profile_entitlements', + { id: requiredString('Access Profile ID'), ...listFields() } + ), + sailpoint_list_sources: operationSchema('sailpoint_list_sources', { + ...listFields(), + forSubadmin: optionalString(MAX_ID_LENGTH), + includeIDNSource: z.boolean().optional(), + }), + sailpoint_get_source: operationSchema('sailpoint_get_source', { + id: requiredString('Source ID'), + }), + sailpoint_list_account_activities: operationSchema('sailpoint_list_account_activities', { + ...listFields(), + requestedFor: optionalString(MAX_ID_LENGTH), + requestedBy: optionalString(MAX_ID_LENGTH), + regardingIdentity: optionalString(MAX_ID_LENGTH), + }).superRefine((value, ctx) => { + if (value.regardingIdentity && (value.requestedFor || value.requestedBy)) { + ctx.addIssue({ + code: z.ZodIssueCode.custom, + path: ['regardingIdentity'], + message: 'regardingIdentity cannot be combined with requestedFor or requestedBy', + }) + } + }), + sailpoint_get_account_activity: operationSchema('sailpoint_get_account_activity', { + id: requiredString('Account activity ID'), + }), + sailpoint_list_campaigns: operationSchema('sailpoint_list_campaigns', { + ...listFields(), + detail: z.enum(['SLIM', 'FULL']).optional(), + }), + sailpoint_get_campaign: operationSchema('sailpoint_get_campaign', { + id: requiredString('Campaign ID'), + detail: z.enum(['SLIM', 'FULL']).optional(), + }), + sailpoint_list_certifications: operationSchema('sailpoint_list_certifications', { + ...listFields(), + reviewerIdentity: optionalString(MAX_ID_LENGTH), + }), + sailpoint_get_certification: operationSchema('sailpoint_get_certification', { + id: requiredString('Certification ID'), + }), + sailpoint_list_certification_review_items: operationSchema( + 'sailpoint_list_certification_review_items', + { + id: requiredString('Certification ID'), + ...listFields(), + entitlements: optionalString(MAX_FILTER_LENGTH), + accessProfiles: optionalString(MAX_FILTER_LENGTH), + roles: optionalString(MAX_FILTER_LENGTH), + } + ).superRefine((value, ctx) => { + const specialized = [value.entitlements, value.accessProfiles, value.roles].filter(Boolean) + if (specialized.length > 1) { + ctx.addIssue({ + code: z.ZodIssueCode.custom, + path: ['entitlements'], + message: 'Only one of entitlements, accessProfiles, or roles may be provided', + }) + } + }), + sailpoint_decide_certification_review_items: operationSchema( + 'sailpoint_decide_certification_review_items', + { + id: requiredString('Certification ID'), + decisions: z.preprocess(parseJson, z.array(reviewDecisionSchema).min(1).max(250)), + } + ), + sailpoint_sign_off_certification: operationSchema('sailpoint_sign_off_certification', { + id: requiredString('Certification ID'), + }), + sailpoint_get_entitlement_request_config: operationSchema( + 'sailpoint_get_entitlement_request_config', + { id: requiredString('Entitlement ID') } + ), + sailpoint_get_access_request_config: operationSchema('sailpoint_get_access_request_config', {}), + sailpoint_get_account_selections: operationSchema('sailpoint_get_account_selections', { + requestedFor: z + .preprocess(parseJson, z.array(requiredString('Identity ID')).max(250)) + .optional(), + requestedItems: z.preprocess(parseJson, z.array(requestedItemSchema).min(1).max(25)).optional(), + requestedForWithRequestedItems: z + .preprocess(parseJson, z.array(requestedForWithItemsSchema).min(1).max(10)) + .optional(), + requestType: z.enum(['GRANT_ACCESS', 'REVOKE_ACCESS', 'MODIFY_ACCESS']).optional(), + clientMetadata: z.preprocess(parseJson, metadataSchema).optional(), + }).superRefine((value, ctx) => { + const usesFlat = value.requestedFor !== undefined || value.requestedItems !== undefined + const usesNested = value.requestedForWithRequestedItems !== undefined + if (usesFlat === usesNested) { + ctx.addIssue({ + code: z.ZodIssueCode.custom, + path: ['requestedFor'], + message: + 'Provide requestedFor with requestedItems, or requestedForWithRequestedItems, but not both', + }) + return + } + if (usesFlat && (!value.requestedFor?.length || !value.requestedItems?.length)) { + ctx.addIssue({ + code: z.ZodIssueCode.custom, + path: ['requestedItems'], + message: 'requestedFor and requestedItems must both be non-empty', + }) + } + const requestType = value.requestType ?? 'GRANT_ACCESS' + if (requestType === 'REVOKE_ACCESS' && value.requestedFor) { + if (value.requestedFor.length !== 1) { + ctx.addIssue({ + code: z.ZodIssueCode.custom, + path: ['requestedFor'], + message: 'REVOKE_ACCESS supports exactly one identity', + }) + } + const entitlementCount = + value.requestedItems?.filter((item) => item.type === 'ENTITLEMENT').length ?? 0 + if (entitlementCount > 1) { + ctx.addIssue({ + code: z.ZodIssueCode.custom, + path: ['requestedItems'], + message: 'REVOKE_ACCESS supports at most one entitlement item', + }) + } + value.requestedItems?.forEach((item, index) => { + if (!item.comment) { + ctx.addIssue({ + code: z.ZodIssueCode.custom, + path: ['requestedItems', index, 'comment'], + message: 'comment is required for REVOKE_ACCESS', + }) + } + if (item.startDate) { + ctx.addIssue({ + code: z.ZodIssueCode.custom, + path: ['requestedItems', index, 'startDate'], + message: 'startDate is not allowed for REVOKE_ACCESS', + }) + } + }) + } + if (requestType !== 'REVOKE_ACCESS' && value.requestedItems) { + const entitlementCount = value.requestedItems.filter( + (item) => item.type === 'ENTITLEMENT' + ).length + if (entitlementCount > 0 && (value.requestedFor?.length ?? 0) > 10) { + ctx.addIssue({ + code: z.ZodIssueCode.custom, + path: ['requestedFor'], + message: 'Entitlement requests support at most 10 identities', + }) + } + } + if (value.requestedForWithRequestedItems) { + const identityTypes = new Set( + value.requestedForWithRequestedItems.map((entry) => entry.identityType ?? 'HUMAN') + ) + if (identityTypes.size > 1) { + ctx.addIssue({ + code: z.ZodIssueCode.custom, + path: ['requestedForWithRequestedItems'], + message: 'Human and machine identities cannot be mixed in one request', + }) + } + if (requestType === 'REVOKE_ACCESS' && !identityTypes.has('MACHINE')) { + ctx.addIssue({ + code: z.ZodIssueCode.custom, + path: ['requestedForWithRequestedItems'], + message: 'Human REVOKE_ACCESS must use requestedFor and requestedItems', + }) + } + const entitlementCount = value.requestedForWithRequestedItems.reduce( + (total, entry) => + total + entry.requestedItems.filter((item) => item.type === 'ENTITLEMENT').length, + 0 + ) + if (requestType !== 'REVOKE_ACCESS' && entitlementCount > 25) { + ctx.addIssue({ + code: z.ZodIssueCode.custom, + path: ['requestedForWithRequestedItems'], + message: 'Entitlement requests support at most 25 entitlement items', + }) + } + if (identityTypes.has('MACHINE')) { + if (requestType === 'REVOKE_ACCESS' && value.requestedForWithRequestedItems.length !== 1) { + ctx.addIssue({ + code: z.ZodIssueCode.custom, + path: ['requestedForWithRequestedItems'], + message: 'Machine REVOKE_ACCESS requires exactly one machine identity', + }) + } + if (requestType === 'REVOKE_ACCESS' && entitlementCount > 1) { + ctx.addIssue({ + code: z.ZodIssueCode.custom, + path: ['requestedForWithRequestedItems'], + message: 'REVOKE_ACCESS supports at most one entitlement item', + }) + } + value.requestedForWithRequestedItems.forEach((entry, entryIndex) => { + entry.requestedItems.forEach((item, itemIndex) => { + const itemPath = [ + 'requestedForWithRequestedItems', + entryIndex, + 'requestedItems', + itemIndex, + ] + if (item.type !== 'ENTITLEMENT') { + ctx.addIssue({ + code: z.ZodIssueCode.custom, + path: [...itemPath, 'type'], + message: 'Machine identity requests support entitlement items only', + }) + } + if (item.formInstanceId) { + ctx.addIssue({ + code: z.ZodIssueCode.custom, + path: [...itemPath, 'formInstanceId'], + message: 'Machine identity requests do not support formInstanceId', + }) + } + if (requestType === 'REVOKE_ACCESS' && item.accountSelection) { + ctx.addIssue({ + code: z.ZodIssueCode.custom, + path: [...itemPath, 'accountSelection'], + message: 'Machine identity revoke requests cannot include accountSelection', + }) + } + if (requestType === 'REVOKE_ACCESS' && !item.comment) { + ctx.addIssue({ + code: z.ZodIssueCode.custom, + path: [...itemPath, 'comment'], + message: 'comment is required for REVOKE_ACCESS', + }) + } + if (requestType === 'REVOKE_ACCESS' && item.startDate) { + ctx.addIssue({ + code: z.ZodIssueCode.custom, + path: [...itemPath, 'startDate'], + message: 'startDate is not allowed for REVOKE_ACCESS', + }) + } + if (requestType === 'MODIFY_ACCESS' && !item.startDate && !item.removeDate) { + ctx.addIssue({ + code: z.ZodIssueCode.custom, + path: itemPath, + message: 'Machine MODIFY_ACCESS requires startDate or removeDate', + }) + } + }) + }) + } + } + }), + sailpoint_request_access: operationSchema('sailpoint_request_access', { + requestedFor: z + .preprocess(parseJson, z.array(requiredString('Identity ID')).max(250)) + .optional(), + requestedItems: z + .preprocess(parseJson, z.array(requestedItemSchema).min(1).max(250)) + .optional(), + requestedForWithRequestedItems: z + .preprocess(parseJson, z.array(requestedForWithItemsSchema).min(1).max(10)) + .optional(), + requestType: z.enum(['GRANT_ACCESS', 'REVOKE_ACCESS', 'MODIFY_ACCESS']).optional(), + clientMetadata: z.preprocess(parseJson, metadataSchema).optional(), + }).superRefine((value, ctx) => { + const usesFlat = value.requestedFor !== undefined || value.requestedItems !== undefined + const usesNested = value.requestedForWithRequestedItems !== undefined + if (usesFlat === usesNested) { + ctx.addIssue({ + code: z.ZodIssueCode.custom, + path: ['requestedFor'], + message: + 'Provide requestedFor with requestedItems, or requestedForWithRequestedItems, but not both', + }) + return + } + if (usesFlat && (!value.requestedFor?.length || !value.requestedItems?.length)) { + ctx.addIssue({ + code: z.ZodIssueCode.custom, + path: ['requestedItems'], + message: 'requestedFor and requestedItems must both be non-empty', + }) + } + if ((value.requestType ?? 'GRANT_ACCESS') === 'REVOKE_ACCESS' && value.requestedFor) { + if (value.requestedFor.length !== 1) { + ctx.addIssue({ + code: z.ZodIssueCode.custom, + path: ['requestedFor'], + message: 'REVOKE_ACCESS supports exactly one identity', + }) + } + value.requestedItems?.forEach((item, index) => { + if (!item.comment) { + ctx.addIssue({ + code: z.ZodIssueCode.custom, + path: ['requestedItems', index, 'comment'], + message: 'comment is required for REVOKE_ACCESS', + }) + } + if (item.startDate) { + ctx.addIssue({ + code: z.ZodIssueCode.custom, + path: ['requestedItems', index, 'startDate'], + message: 'startDate is not allowed for REVOKE_ACCESS', + }) + } + }) + const entitlementCount = + value.requestedItems?.filter((item) => item.type === 'ENTITLEMENT').length ?? 0 + if (entitlementCount > 1) { + ctx.addIssue({ + code: z.ZodIssueCode.custom, + path: ['requestedItems'], + message: 'REVOKE_ACCESS supports at most one entitlement item', + }) + } + } + if ((value.requestType ?? 'GRANT_ACCESS') !== 'REVOKE_ACCESS' && value.requestedItems) { + const entitlementCount = value.requestedItems.filter( + (item) => item.type === 'ENTITLEMENT' + ).length + if (entitlementCount > 25) { + ctx.addIssue({ + code: z.ZodIssueCode.custom, + path: ['requestedItems'], + message: 'Entitlement requests support at most 25 entitlement items', + }) + } + if (entitlementCount > 0 && (value.requestedFor?.length ?? 0) > 10) { + ctx.addIssue({ + code: z.ZodIssueCode.custom, + path: ['requestedFor'], + message: 'Entitlement requests support at most 10 identities', + }) + } + } + if (value.requestedForWithRequestedItems) { + const identityTypes = new Set( + value.requestedForWithRequestedItems.map((entry) => entry.identityType ?? 'HUMAN') + ) + if (identityTypes.size > 1) { + ctx.addIssue({ + code: z.ZodIssueCode.custom, + path: ['requestedForWithRequestedItems'], + message: 'Human and machine identities cannot be mixed in one request', + }) + } + const requestType = value.requestType ?? 'GRANT_ACCESS' + const entitlementCount = value.requestedForWithRequestedItems.reduce( + (total, entry) => + total + entry.requestedItems.filter((item) => item.type === 'ENTITLEMENT').length, + 0 + ) + if (requestType !== 'REVOKE_ACCESS' && entitlementCount > 25) { + ctx.addIssue({ + code: z.ZodIssueCode.custom, + path: ['requestedForWithRequestedItems'], + message: 'Entitlement requests support at most 25 entitlement items', + }) + } + if (identityTypes.has('MACHINE')) { + if (requestType === 'REVOKE_ACCESS' && value.requestedForWithRequestedItems.length !== 1) { + ctx.addIssue({ + code: z.ZodIssueCode.custom, + path: ['requestedForWithRequestedItems'], + message: 'Machine REVOKE_ACCESS requires exactly one machine identity', + }) + } + if (requestType === 'REVOKE_ACCESS' && entitlementCount > 1) { + ctx.addIssue({ + code: z.ZodIssueCode.custom, + path: ['requestedForWithRequestedItems'], + message: 'REVOKE_ACCESS supports at most one entitlement item', + }) + } + value.requestedForWithRequestedItems.forEach((entry, entryIndex) => { + entry.requestedItems.forEach((item, itemIndex) => { + const path = ['requestedForWithRequestedItems', entryIndex, 'requestedItems', itemIndex] + if (item.type !== 'ENTITLEMENT') { + ctx.addIssue({ + code: z.ZodIssueCode.custom, + path: [...path, 'type'], + message: 'Machine identity requests support entitlement items only', + }) + } + if (item.formInstanceId) { + ctx.addIssue({ + code: z.ZodIssueCode.custom, + path: [...path, 'formInstanceId'], + message: 'Machine identity requests do not support formInstanceId', + }) + } + if (requestType === 'REVOKE_ACCESS' && item.accountSelection) { + ctx.addIssue({ + code: z.ZodIssueCode.custom, + path: [...path, 'accountSelection'], + message: 'Machine identity revoke requests cannot include accountSelection', + }) + } + if (requestType === 'REVOKE_ACCESS' && !item.comment) { + ctx.addIssue({ + code: z.ZodIssueCode.custom, + path: [...path, 'comment'], + message: 'comment is required for REVOKE_ACCESS', + }) + } + if (requestType === 'REVOKE_ACCESS' && item.startDate) { + ctx.addIssue({ + code: z.ZodIssueCode.custom, + path: [...path, 'startDate'], + message: 'startDate is not allowed for REVOKE_ACCESS', + }) + } + if (requestType !== 'REVOKE_ACCESS') { + const selection = item.accountSelection + if ( + !selection || + selection.length !== 1 || + !selection[0]?.accounts || + selection[0].accounts.length !== 1 + ) { + ctx.addIssue({ + code: z.ZodIssueCode.custom, + path: [...path, 'accountSelection'], + message: + 'Machine identity grant and modify items require exactly one source and one account selection', + }) + } + } + if (requestType === 'MODIFY_ACCESS' && !item.startDate && !item.removeDate) { + ctx.addIssue({ + code: z.ZodIssueCode.custom, + path, + message: 'Machine identity modify items require startDate or removeDate', + }) + } + }) + }) + } else if ((value.requestType ?? 'GRANT_ACCESS') === 'REVOKE_ACCESS') { + ctx.addIssue({ + code: z.ZodIssueCode.custom, + path: ['requestedForWithRequestedItems'], + message: 'Human revoke requests must use requestedFor and requestedItems', + }) + } + } + }), + sailpoint_cancel_access_request: operationSchema('sailpoint_cancel_access_request', { + accountActivityId: requiredString('Account activity ID'), + comment: requiredString('Comment', 10_000), + }), + sailpoint_get_access_request_status: operationSchema('sailpoint_get_access_request_status', { + ...listFields(), + requestedFor: optionalString(MAX_ID_LENGTH), + requestedBy: optionalString(MAX_ID_LENGTH), + regardingIdentity: optionalString(MAX_ID_LENGTH), + assignedTo: optionalString(MAX_ID_LENGTH), + requestState: z.literal('EXECUTING').optional(), + }).superRefine((value, ctx) => { + if (value.regardingIdentity && (value.requestedFor || value.requestedBy)) { + ctx.addIssue({ + code: z.ZodIssueCode.custom, + path: ['regardingIdentity'], + message: 'regardingIdentity cannot be combined with requestedFor or requestedBy', + }) + } + }), + sailpoint_list_pending_access_request_approvals: operationSchema( + 'sailpoint_list_pending_access_request_approvals', + { ownerId: optionalString(MAX_ID_LENGTH), ...listFields() } + ), + sailpoint_approve_access_request: operationSchema('sailpoint_approve_access_request', { + approvalId: requiredString('Approval ID'), + comment: optionalString(10_000), + }), + sailpoint_reject_access_request: operationSchema('sailpoint_reject_access_request', { + approvalId: requiredString('Approval ID'), + comment: requiredString('Comment', 10_000), + }), + sailpoint_get_task_status: operationSchema('sailpoint_get_task_status', { + id: requiredString('Task ID'), + }), + sailpoint_load_accounts: operationSchema('sailpoint_load_accounts', { + sourceId: requiredString('Source ID'), + file: FileInputSchema.optional().nullable(), + disableOptimization: z.boolean().optional(), + }), + sailpoint_load_entitlements: operationSchema('sailpoint_load_entitlements', { + sourceId: requiredString('Source ID'), + file: FileInputSchema.optional().nullable(), + }), +} as const + +export type SailPointOperationId = keyof typeof schemas + +export const SAILPOINT_OPERATION_IDS = Object.freeze(Object.keys(schemas) as SailPointOperationId[]) + +export type SailPointInput = z.output<(typeof schemas)[SailPointOperationId]> + +export function parseSailPointInput( + toolId: string, + input: unknown +): { success: true; data: SailPointInput } | { success: false; error: z.ZodError } | null { + const schema = schemas[toolId as SailPointOperationId] + if (!schema) return null + const parsed = schema.safeParse(input) + if (!parsed.success) return parsed + return { success: true, data: parsed.data as SailPointInput } +} diff --git a/apps/sim/lib/internal/sap-concur/client.ts b/apps/sim/lib/internal/sap-concur/client.ts index 31c436e85fc..bb77379101a 100644 --- a/apps/sim/lib/internal/sap-concur/client.ts +++ b/apps/sim/lib/internal/sap-concur/client.ts @@ -307,6 +307,7 @@ async function requestAccessToken( const response = await secureFetchWithValidation( tokenUrl, { + profile: 'configuredEndpoint', method: 'POST', headers: { 'Content-Type': 'application/x-www-form-urlencoded', @@ -426,6 +427,7 @@ export async function invokeSapConcur( const response = await secureFetchWithValidation( url, { + profile: 'configuredEndpoint', method: input.method, headers, body: hasBody @@ -498,6 +500,7 @@ export async function invokeSapConcurMultipart( const response = await secureFetchWithValidation( url, { + profile: 'configuredEndpoint', method: 'POST', headers, body: bodyBuffer, diff --git a/apps/sim/lib/internal/sap-s4hana/client.ts b/apps/sim/lib/internal/sap-s4hana/client.ts index f4bd96fc4c3..2cce17bb257 100644 --- a/apps/sim/lib/internal/sap-s4hana/client.ts +++ b/apps/sim/lib/internal/sap-s4hana/client.ts @@ -73,6 +73,7 @@ export async function fetchSapAccessToken( const response = await secureFetchWithValidation( tokenUrl, { + profile: 'configuredEndpoint', method: 'POST', headers: { Authorization: `Basic ${basic}`, @@ -164,6 +165,7 @@ export async function fetchSapCsrf( const response = await secureFetchWithValidation( buildOdataUrl(input, '/$metadata'), { + profile: 'configuredEndpoint', method: 'GET', headers: { Authorization: buildAuthHeader(input, accessToken), @@ -210,6 +212,7 @@ export async function callSapOdata( const response = await secureFetchWithValidation( buildOdataUrl(input), { + profile: 'configuredEndpoint', method: input.method, headers, body: hasBody ? JSON.stringify(input.body) : undefined, diff --git a/apps/sim/lib/internal/servicenow/client.ts b/apps/sim/lib/internal/servicenow/client.ts index e880dc9ee7a..415afc0db90 100644 --- a/apps/sim/lib/internal/servicenow/client.ts +++ b/apps/sim/lib/internal/servicenow/client.ts @@ -30,6 +30,7 @@ export async function uploadServiceNowAttachment( const response = await secureFetchWithValidation( `${baseUrl}/api/now/attachment/file?${params.toString()}`, { + profile: 'configuredEndpoint', method: 'POST', headers: { Authorization: createBasicAuthHeader(input.username, input.password), diff --git a/apps/sim/lib/internal/sharepoint/client.test.ts b/apps/sim/lib/internal/sharepoint/client.test.ts index 98f255b1481..d9c0a6cb7ca 100644 --- a/apps/sim/lib/internal/sharepoint/client.test.ts +++ b/apps/sim/lib/internal/sharepoint/client.test.ts @@ -42,6 +42,7 @@ describe('SharePointClient', () => { { headers: { Authorization: 'Bearer token' }, stripAuthOnRedirect: true, + profile: 'configuredEndpoint', maxResponseBytes: MAX_FILE_SIZE, signal: controller.signal, } @@ -63,6 +64,7 @@ describe('SharePointClient', () => { expect(mocks.validatedFetch).toHaveBeenCalledWith( 'https://graph.microsoft.com/upload', { + profile: 'contentFetch', method: 'PUT', headers: { Authorization: 'Bearer token', diff --git a/apps/sim/lib/internal/sharepoint/client.ts b/apps/sim/lib/internal/sharepoint/client.ts index 2697463f462..fee7d105641 100644 --- a/apps/sim/lib/internal/sharepoint/client.ts +++ b/apps/sim/lib/internal/sharepoint/client.ts @@ -94,19 +94,21 @@ export class SharePointClient { return `Bearer ${this.accessToken}` } + /** Every URL reaching here is a fixed Microsoft Graph endpoint. */ private async pinnedFetch( url: string, paramName: string, - options: SecureFetchOptions + options: Omit ): Promise { this.signal?.throwIfAborted() - const validation = await validateUrlWithDNS(url, paramName) + const validation = await validateUrlWithDNS(url, paramName, 'configuredEndpoint') this.signal?.throwIfAborted() - if (!validation.isValid || !validation.resolvedIP) { + if (!validation.isValid) { throw new SharePointGraphError(validation.error || `Invalid ${paramName}`, 400) } return secureFetchWithPinnedIP(url, validation.resolvedIP, { ...options, + profile: 'configuredEndpoint', signal: this.signal, }) } @@ -163,6 +165,7 @@ export class SharePointClient { const response = await secureFetchWithValidation( url, { + profile: 'contentFetch', method: 'PUT', headers: { Authorization: this.authorization, diff --git a/apps/sim/lib/internal/slack/operations.test.ts b/apps/sim/lib/internal/slack/operations.test.ts index 7d97783f171..8f269c2f2ae 100644 --- a/apps/sim/lib/internal/slack/operations.test.ts +++ b/apps/sim/lib/internal/slack/operations.test.ts @@ -229,6 +229,7 @@ describe('Slack operations', () => { '93.184.216.34', { headers: { Authorization: 'Bearer token' }, + profile: 'contentFetch', maxResponseBytes: MAX_FILE_SIZE, signal: controller.signal, } diff --git a/apps/sim/lib/internal/slack/operations.ts b/apps/sim/lib/internal/slack/operations.ts index b3649cdfdc4..5ee5762a857 100644 --- a/apps/sim/lib/internal/slack/operations.ts +++ b/apps/sim/lib/internal/slack/operations.ts @@ -325,6 +325,7 @@ async function uploadSlackFiles( const uploaded = await secureFetchWithValidation( uploadUrl, { + profile: 'contentFetch', method: 'POST', body: file.buffer, maxResponseBytes: 64 * 1024, @@ -444,10 +445,11 @@ export async function executeSlackDownload(input: SlackDownloadBody, signal?: Ab const urlPrivate = slackString(file, 'url_private') if (!urlPrivate) failure(400, 'File does not have a download URL') const downloadUrl = urlPrivate - const validation = await validateUrlWithDNS(downloadUrl, 'urlPrivate') + const validation = await validateUrlWithDNS(downloadUrl, 'urlPrivate', 'contentFetch') signal?.throwIfAborted() if (!validation.isValid) failure(400, validation.error || 'Invalid Slack file URL') - const response = await secureFetchWithPinnedIP(downloadUrl, validation.resolvedIP!, { + const response = await secureFetchWithPinnedIP(downloadUrl, validation.resolvedIP, { + profile: 'contentFetch', headers: { Authorization: `Bearer ${input.accessToken}` }, maxResponseBytes: MAX_FILE_SIZE, signal, diff --git a/apps/sim/lib/internal/stagehand/operations.ts b/apps/sim/lib/internal/stagehand/operations.ts index e12f9f5f66d..7cafab86303 100644 --- a/apps/sim/lib/internal/stagehand/operations.ts +++ b/apps/sim/lib/internal/stagehand/operations.ts @@ -130,7 +130,7 @@ export async function executeStagehandAgent( try { const startUrl = normalizeStagehandUrl(input.startUrl) - const urlValidation = await validateUrlWithDNS(startUrl, 'startUrl') + const urlValidation = await validateUrlWithDNS(startUrl, 'startUrl', 'requestTarget') context.signal?.throwIfAborted() if (!urlValidation.isValid) { return Response.json({ error: urlValidation.error }, { status: 400 }) @@ -251,7 +251,7 @@ export async function executeStagehandExtract( try { const url = normalizeStagehandUrl(input.url) - const urlValidation = await validateUrlWithDNS(url, 'url') + const urlValidation = await validateUrlWithDNS(url, 'url', 'requestTarget') context.signal?.throwIfAborted() if (!urlValidation.isValid) { return Response.json({ error: urlValidation.error }, { status: 400 }) diff --git a/apps/sim/lib/internal/stt/execute-tool.test.ts b/apps/sim/lib/internal/stt/execute-tool.test.ts index 79340a99fe1..d451ce8164c 100644 --- a/apps/sim/lib/internal/stt/execute-tool.test.ts +++ b/apps/sim/lib/internal/stt/execute-tool.test.ts @@ -45,6 +45,7 @@ vi.mock('@/lib/audio/extractor', () => ({ extractAudioFromVideo: vi.fn(), })) +import { extractAudioFromVideo, isVideoFile } from '@/lib/audio/extractor' import { executeSttTool } from '@/lib/internal/stt/execute-tool' import type { InternalToolOperationCall } from '@/lib/internal/tool-operations/types' @@ -161,6 +162,31 @@ describe('executeSttTool', () => { expect(data.transcript).toBe('hello world') }) + it('forwards tool cancellation through video audio extraction', async () => { + const controller = new AbortController() + vi.mocked(isVideoFile).mockReturnValueOnce(true) + vi.mocked(extractAudioFromVideo).mockResolvedValueOnce({ + buffer: Buffer.from('converted-audio'), + duration: 1, + format: 'mp3', + size: 15, + }) + inputValidationMockFns.mockSecureFetchWithPinnedIP.mockResolvedValueOnce( + mockSecureFetchResponse({ contentType: 'video/mp4' }) + ) + + const response = await executeSttTool( + createVerifiedSttRequest(baseBody, { signal: controller.signal }) + ) + + expect(response.status).toBe(200) + expect(extractAudioFromVideo).toHaveBeenCalledWith( + expect.any(Buffer), + 'video/mp4', + expect.objectContaining({ signal: controller.signal }) + ) + }) + it('rejects an authenticated but incomplete private provenance envelope before downloading', async () => { const response = await executeSttTool( createSttRequest( diff --git a/apps/sim/lib/internal/stt/operations.ts b/apps/sim/lib/internal/stt/operations.ts index fb319967e94..5785fa9beaa 100644 --- a/apps/sim/lib/internal/stt/operations.ts +++ b/apps/sim/lib/internal/stt/operations.ts @@ -3,6 +3,7 @@ import { getErrorMessage } from '@sim/utils/errors' import { sleep } from '@sim/utils/helpers' import { extractAudioFromVideo, isVideoFile } from '@/lib/audio/extractor' import { getMaxExecutionTimeout } from '@/lib/core/execution-limits' +import type { EgressProfile } from '@/lib/core/security/egress/profiles' import { secureFetchWithPinnedIP, validateUrlWithDNS, @@ -255,12 +256,18 @@ export async function executeSttOperation( } } - const urlValidation = await validateUrlWithDNS(audioUrl, 'audioUrl') + // A caller-supplied audio URL is content; a resolved internal one is a + // presigned URL against Sim's own storage, which on a self-hosted + // deployment legitimately sits on a private address. + const audioProfile: EgressProfile = internalAudioUrl ? 'configuredEndpoint' : 'contentFetch' + + const urlValidation = await validateUrlWithDNS(audioUrl, 'audioUrl', audioProfile) if (!urlValidation.isValid) { return Response.json({ error: urlValidation.error }, { status: 400 }) } - const response = await secureFetchWithPinnedIP(audioUrl, urlValidation.resolvedIP!, { + const response = await secureFetchWithPinnedIP(audioUrl, urlValidation.resolvedIP, { + profile: audioProfile, method: 'GET', maxResponseBytes: MAX_FILE_SIZE, signal, @@ -289,13 +296,21 @@ export async function executeSttOperation( outputFormat: 'mp3', sampleRate: 16000, channels: 1, + signal, }) signal?.throwIfAborted() audioBuffer = extracted.buffer audioMimeType = 'audio/mpeg' audioFileName = audioFileName.replace(/\.[^.]+$/, '.mp3') } catch (error) { + signal?.throwIfAborted() logger.error(`[${requestId}] Video extraction failed:`, error) + if (isPayloadSizeLimitError(error)) { + return Response.json( + { error: 'Extracted audio exceeds the maximum supported size' }, + { status: 413 } + ) + } return Response.json( { error: `Failed to extract audio from video: ${getErrorMessage(error, 'Unknown error')}`, diff --git a/apps/sim/lib/internal/table/read-schema.test.ts b/apps/sim/lib/internal/table/read-schema.test.ts index ff438f37705..bcd14a1e24a 100644 --- a/apps/sim/lib/internal/table/read-schema.test.ts +++ b/apps/sim/lib/internal/table/read-schema.test.ts @@ -76,6 +76,44 @@ describe('readTableSchemaAsExecutor', () => { }) }) + /** + * A select column's cardinality decides which filter operators it accepts, so + * LLM enrichment needs it to name the right subset. It is carried only for + * select columns, where it means something. + */ + it('carries select cardinality through and omits it elsewhere', async () => { + mocks.readTable.mockResolvedValue({ + table: { + name: 'Transactions', + schema: { + columns: [ + { id: 'column-category', name: 'category', type: 'select' }, + { id: 'column-tags', name: 'tags', type: 'select', multiple: true }, + { id: 'column-amount', name: 'amount', type: 'number' }, + ], + }, + }, + }) + + const result = await readTableSchemaAsExecutor({ + tableId: 'table-1', + context: { + workflowId: 'workflow-1', + executorDelegationOrigin: { + subjectUserId: 'user-1', + workflowId: 'workflow-1', + executionId: 'execution-1', + }, + }, + }) + + expect(result.columns).toEqual([ + { name: 'category', type: 'select', multiple: false }, + { name: 'tags', type: 'select', multiple: true }, + { name: 'amount', type: 'number' }, + ]) + }) + it('fails closed when canonical schema metadata is malformed', async () => { mocks.readTable.mockResolvedValueOnce({ table: { name: 'Customers', schema: { columns: [{ name: 'email', type: 'unknown' }] } }, diff --git a/apps/sim/lib/internal/table/read-schema.ts b/apps/sim/lib/internal/table/read-schema.ts index b1559501e87..259b72db9d9 100644 --- a/apps/sim/lib/internal/table/read-schema.ts +++ b/apps/sim/lib/internal/table/read-schema.ts @@ -32,7 +32,12 @@ export async function readTableSchemaAsExecutor({ if (typeof column.name !== 'string' || !isColumnType(column.type)) { throw new Error(`Invalid table column ${index} while enriching schema for ${tableId}`) } - return { name: column.name, type: column.type } + // `multiple` is a select-only concern (it decides which filter operators the + // column accepts), so it is carried only where it means something rather + // than stamped onto every column. + return column.type === 'select' + ? { name: column.name, type: column.type, multiple: column.multiple === true } + : { name: column.name, type: column.type } }) return { name: table.name, columns } diff --git a/apps/sim/lib/internal/textract/document-input.ts b/apps/sim/lib/internal/textract/document-input.ts index ebd82384cc4..a771d3679a9 100644 --- a/apps/sim/lib/internal/textract/document-input.ts +++ b/apps/sim/lib/internal/textract/document-input.ts @@ -1,6 +1,7 @@ import type { createLogger } from '@sim/logger' import { getErrorMessage } from '@sim/utils/errors' import { NextResponse } from 'next/server' +import type { EgressProfile } from '@/lib/core/security/egress/profiles' import { validateS3BucketName } from '@/lib/core/security/input-validation' import { secureFetchWithPinnedIP, @@ -36,17 +37,25 @@ export type ResolveDocumentResult = | { ok: true; document: ResolvedDocument } | { ok: false; response: NextResponse } +/** + * `profile` distinguishes the two kinds of URL that reach here: a document URL + * the caller supplied, and a presigned URL Sim minted against its own configured + * object storage — which on a self-hosted deployment legitimately points at a + * private or loopback MinIO. + */ async function fetchDocumentBytes( url: string, + profile: EgressProfile, signal?: AbortSignal ): Promise<{ bytes: Buffer; contentType: string }> { signal?.throwIfAborted() - const urlValidation = await validateUrlWithDNS(url, 'Document URL') + const urlValidation = await validateUrlWithDNS(url, 'Document URL', profile) if (!urlValidation.isValid) { throw new TextractOperationError(urlValidation.error || 'Invalid document URL', 400) } - const response = await secureFetchWithPinnedIP(url, urlValidation.resolvedIP!, { + const response = await secureFetchWithPinnedIP(url, urlValidation.resolvedIP, { + profile, method: 'GET', signal, }) @@ -159,7 +168,7 @@ export async function resolveDocumentInput( ), } } else { - const urlValidation = await validateUrlWithDNS(fileUrl, 'Document URL') + const urlValidation = await validateUrlWithDNS(fileUrl, 'Document URL', 'contentFetch') if (!urlValidation.isValid) { logger.warn(`[${requestId}] SSRF attempt blocked`, { userId, @@ -176,7 +185,11 @@ export async function resolveDocumentInput( } } - const fetched = await fetchDocumentBytes(fileUrl, signal) + const fetched = await fetchDocumentBytes( + fileUrl, + isInternalFilePath ? 'configuredEndpoint' : 'contentFetch', + signal + ) return { ok: true, document: { diff --git a/apps/sim/lib/internal/tool-operations/registry.server.test.ts b/apps/sim/lib/internal/tool-operations/registry.server.test.ts index 978bdc8c84e..04b446272c6 100644 --- a/apps/sim/lib/internal/tool-operations/registry.server.test.ts +++ b/apps/sim/lib/internal/tool-operations/registry.server.test.ts @@ -1,18 +1,21 @@ /** * @vitest-environment node */ -import { describe, expect, it, vi } from 'vitest' +import { describe, expect, it } from 'vitest' import { getInternalToolOperationHandler, getRegisteredInternalToolOperationIds, isInternalToolOperationRegistered, } from '@/lib/internal/tool-operations/registry.server' -import { tools } from '@/tools/registry' import { getToolIds } from '@/tools/tool-ids' -import { isInternalToolConfig } from '@/tools/types' - -vi.unmock('@/tools/registry') +/** + * Registration is checked against the generated tool ids rather than the + * executable registry, whose import costs more than every handler load below + * combined. The converse — that every operation-backed tool in the registry has + * a handler here — is the in-process half of the transport partition sweep in + * `tools/request-transport.test.ts`, which already pays for that registry. + */ describe('internal tool operation registry', () => { it('registers only canonical internal tool definitions with loadable handlers', async () => { const registeredIds = getRegisteredInternalToolOperationIds() @@ -22,29 +25,15 @@ describe('internal tool operation registry', () => { for (const toolId of registeredIds) { expect(canonicalIds.has(toolId), `Missing canonical tool definition for ${toolId}`).toBe(true) - expect(await getInternalToolOperationHandler(toolId)).toBeTypeOf('function') + } + const handlers = await Promise.all(registeredIds.map(getInternalToolOperationHandler)) + for (const [index, handler] of handlers.entries()) { + expect(handler, `${registeredIds[index]} has no loadable handler`).toBeTypeOf('function') } // Cost scales with the number of registered internal tools, so this budget has to grow // with the registry rather than sit just above the current total. }, 90_000) - it('registers every operation-backed tool and keeps it free of HTTP request metadata', async () => { - const operationTools = Object.entries(tools).filter(([, tool]) => isInternalToolConfig(tool)) - - expect(operationTools.length).toBeGreaterThan(0) - for (const [toolId, tool] of operationTools) { - expect(tool.request, `${toolId} must not declare an HTTP request`).toBeUndefined() - expect(tool.operation.input, `${toolId} must materialize its operation input`).toBeTypeOf( - 'function' - ) - if (toolId === 'function_execute' || toolId === 'workflow_executor') continue - expect( - isInternalToolOperationRegistered(toolId), - `${toolId} is missing its in-process operation handler` - ).toBe(true) - } - }) - it('loads dynamic MCP operations without an HTTP route', async () => { expect(isInternalToolOperationRegistered('mcp-server-id-tool-name')).toBe(true) expect(await getInternalToolOperationHandler('mcp-server-id-tool-name')).toBeTypeOf('function') diff --git a/apps/sim/lib/internal/tool-operations/registry.server.ts b/apps/sim/lib/internal/tool-operations/registry.server.ts index c84c5735af6..d0dda35cbe2 100644 --- a/apps/sim/lib/internal/tool-operations/registry.server.ts +++ b/apps/sim/lib/internal/tool-operations/registry.server.ts @@ -828,6 +828,7 @@ const FILE_TOOL_IDS = [ 'file_write', 'file_get', 'file_read', + 'file_search', 'file_get_content', 'file_compress', 'file_decompress', @@ -968,6 +969,49 @@ const MICROSOFT_TEAMS_TOOL_IDS = [ const BREX_TOOL_IDS = ['brex_match_receipt', 'brex_upload_receipt'] as const +const SAILPOINT_TOOL_IDS = [ + 'sailpoint_approve_access_request', + 'sailpoint_cancel_access_request', + 'sailpoint_decide_certification_review_items', + 'sailpoint_get_access_request_config', + 'sailpoint_get_account_selections', + 'sailpoint_get_access_profile', + 'sailpoint_get_access_profile_entitlements', + 'sailpoint_get_access_request_status', + 'sailpoint_get_account', + 'sailpoint_get_account_activity', + 'sailpoint_get_account_entitlements', + 'sailpoint_get_campaign', + 'sailpoint_get_certification', + 'sailpoint_get_entitlement', + 'sailpoint_get_entitlement_request_config', + 'sailpoint_get_identity', + 'sailpoint_get_role', + 'sailpoint_get_role_entitlements', + 'sailpoint_get_source', + 'sailpoint_get_task_status', + 'sailpoint_list_access_profiles', + 'sailpoint_list_account_activities', + 'sailpoint_list_accounts', + 'sailpoint_list_campaigns', + 'sailpoint_list_certification_review_items', + 'sailpoint_list_certifications', + 'sailpoint_list_entitlements', + 'sailpoint_list_identities', + 'sailpoint_list_identity_entitlements', + 'sailpoint_list_pending_access_request_approvals', + 'sailpoint_list_roles', + 'sailpoint_list_sources', + 'sailpoint_load_accounts', + 'sailpoint_load_entitlements', + 'sailpoint_reject_access_request', + 'sailpoint_request_access', + 'sailpoint_search', + 'sailpoint_search_aggregate', + 'sailpoint_search_count', + 'sailpoint_sign_off_certification', +] as const + const LATEX_TOOL_IDS = ['latex_compile'] as const const ONEDRIVE_TOOL_IDS = ['onedrive_download', 'onedrive_upload'] as const @@ -1199,6 +1243,8 @@ const DISCORD_TOOL_IDS = ['discord_send_message'] as const const LINQ_TOOL_IDS = ['linq_create_attachment'] as const +const ASHBY_TOOL_IDS = ['ashby_upload_candidate_file', 'ashby_upload_resume'] as const + const MICROSOFT_DATAVERSE_TOOL_IDS = ['microsoft_dataverse_upload_file'] as const const SERVICENOW_TOOL_IDS = ['servicenow_upload_attachment'] as const @@ -1230,6 +1276,10 @@ function registerFamily( const handlerLoaders = new Map() +registerFamily(handlerLoaders, ASHBY_TOOL_IDS, async () => { + return (await import('@/lib/internal/ashby/execute-tool')).executeAshbyTool +}) + registerFamily(handlerLoaders, STS_TOOL_IDS, async () => { return (await import('@/lib/internal/sts/execute-tool')).executeStsTool }) @@ -1488,6 +1538,9 @@ registerFamily(handlerLoaders, MICROSOFT_TEAMS_TOOL_IDS, async () => { registerFamily(handlerLoaders, BREX_TOOL_IDS, async () => { return (await import('@/lib/internal/brex/execute-tool')).executeBrexTool }) +registerFamily(handlerLoaders, SAILPOINT_TOOL_IDS, async () => { + return (await import('@/lib/internal/sailpoint/execute-tool')).executeSailPointTool +}) registerFamily(handlerLoaders, LATEX_TOOL_IDS, async () => { return (await import('@/lib/internal/latex/execute-tool')).executeLatexTool }) diff --git a/apps/sim/lib/internal/twilio-voice/operations.ts b/apps/sim/lib/internal/twilio-voice/operations.ts index a94793bcf1c..7acc77c44e8 100644 --- a/apps/sim/lib/internal/twilio-voice/operations.ts +++ b/apps/sim/lib/internal/twilio-voice/operations.ts @@ -50,12 +50,13 @@ async function fetchPinned( context: TwilioVoiceOperationContext, maxResponseBytes: number ) { - const validation = await validateUrlWithDNS(url, label) + const validation = await validateUrlWithDNS(url, label, 'configuredEndpoint') context.signal?.throwIfAborted() - if (!validation.isValid || !validation.resolvedIP) { + if (!validation.isValid) { throw new TwilioVoiceOperationError(validation.error || `Invalid ${label}`, 400) } return secureFetchWithPinnedIP(url, validation.resolvedIP, { + profile: 'configuredEndpoint', method: 'GET', headers: { Authorization: authHeader }, maxResponseBytes, diff --git a/apps/sim/lib/internal/typeform/operations.ts b/apps/sim/lib/internal/typeform/operations.ts index 32ade526aeb..0d5ada8984d 100644 --- a/apps/sim/lib/internal/typeform/operations.ts +++ b/apps/sim/lib/internal/typeform/operations.ts @@ -44,12 +44,13 @@ export async function downloadTypeformFile( ): Promise { context.signal?.throwIfAborted() const fileUrl = buildTypeformFileUrl(input) - const validation = await validateUrlWithDNS(fileUrl, 'typeformFileUrl') + const validation = await validateUrlWithDNS(fileUrl, 'typeformFileUrl', 'configuredEndpoint') context.signal?.throwIfAborted() - if (!validation.isValid || !validation.resolvedIP) { + if (!validation.isValid) { throw new TypeformOperationError(validation.error || 'Invalid Typeform file URL', 400) } const response = await secureFetchWithPinnedIP(fileUrl, validation.resolvedIP, { + profile: 'configuredEndpoint', headers: { Authorization: `Bearer ${input.apiKey}` }, maxResponseBytes: MAX_TYPEFORM_FILE_BYTES, signal: context.signal, diff --git a/apps/sim/lib/internal/vision/client.test.ts b/apps/sim/lib/internal/vision/client.test.ts index 78d52407f0f..d88a147695b 100644 --- a/apps/sim/lib/internal/vision/client.test.ts +++ b/apps/sim/lib/internal/vision/client.test.ts @@ -159,6 +159,7 @@ describe('Vision client', () => { 'https://images.example.com/a.png', '203.0.113.10', { + profile: 'contentFetch', method: 'GET', maxResponseBytes: MAX_BUFFERED_TRANSFER_BYTES, signal: controller.signal, diff --git a/apps/sim/lib/internal/vision/client.ts b/apps/sim/lib/internal/vision/client.ts index 7ff0a164d28..e3cdc7703bb 100644 --- a/apps/sim/lib/internal/vision/client.ts +++ b/apps/sim/lib/internal/vision/client.ts @@ -1,6 +1,7 @@ import { GoogleGenAI } from '@google/genai' import { createLogger } from '@sim/logger' import { isRecordLike } from '@sim/utils/object' +import type { EgressProfile } from '@/lib/core/security/egress/profiles' import { MAX_JSON_API_RESPONSE_BYTES, secureFetchWithPinnedIP, @@ -24,6 +25,7 @@ export interface VisionClientInput { model: string prompt: string remoteImageResolvedIP?: string + remoteImageProfile?: EgressProfile } export interface VisionAnalysisResult { @@ -94,6 +96,7 @@ async function fetchGeminiImage(input: VisionClientInput, signal?: AbortSignal): } const response = await secureFetchWithPinnedIP(input.imageSource, input.remoteImageResolvedIP, { + profile: input.remoteImageProfile ?? 'contentFetch', method: 'GET', maxResponseBytes: MAX_BUFFERED_TRANSFER_BYTES, signal, diff --git a/apps/sim/lib/internal/vision/operations.test.ts b/apps/sim/lib/internal/vision/operations.test.ts index 8f8f6a9280e..f9a15a0f0c0 100644 --- a/apps/sim/lib/internal/vision/operations.test.ts +++ b/apps/sim/lib/internal/vision/operations.test.ts @@ -216,9 +216,13 @@ describe('Vision operations', () => { expect(mocks.isModelSafeWorkspaceFileKey).toHaveBeenCalledWith( 'workspace/workspace-1/image.png' ) + // A resolved internal file URL is a presigned URL against Sim's own + // storage, which on a self-hosted deployment legitimately sits on a private + // address — so it is judged as a configured endpoint, not as content. expect(mocks.validateUrlWithDNS).toHaveBeenCalledWith( 'https://storage.example.com/image.png', - 'imageUrl' + 'imageUrl', + 'configuredEndpoint' ) expect(mocks.analyzeVision).toHaveBeenCalledWith( expect.objectContaining({ diff --git a/apps/sim/lib/internal/vision/operations.ts b/apps/sim/lib/internal/vision/operations.ts index 7c4c219d0d3..3736d321d03 100644 --- a/apps/sim/lib/internal/vision/operations.ts +++ b/apps/sim/lib/internal/vision/operations.ts @@ -1,5 +1,6 @@ import { createLogger } from '@sim/logger' import { getErrorMessage } from '@sim/utils/errors' +import type { EgressProfile } from '@/lib/core/security/egress/profiles' import { validateUrlWithDNS } from '@/lib/core/security/input-validation.server' import { validateOpaqueModelInputProvenance } from '@/lib/execution/model-input-provenance' import { analyzeVision, type VisionAnalysisResult } from '@/lib/internal/vision/client' @@ -35,6 +36,7 @@ interface ResolvedImage { source: string contentType?: string resolvedIP?: string + profile?: EgressProfile } function fail(message: string, status: number, body?: Record): never { @@ -84,7 +86,8 @@ async function resolveUrlImage( if (source.startsWith('/') && !isInternalFileUrl(source)) { fail('Invalid file path. Only uploaded files are supported for internal paths.', 400) } - if (isInternalFileUrl(source)) { + const internal = isInternalFileUrl(source) + if (internal) { context.signal?.throwIfAborted() const resolution = await resolveInternalFileUrl( source, @@ -100,8 +103,13 @@ async function resolveUrlImage( } } + // A caller-supplied image URL is content; a resolved internal one is a + // presigned URL against Sim's own storage, which on a self-hosted deployment + // legitimately sits on a private address. + const profile: EgressProfile = internal ? 'configuredEndpoint' : 'contentFetch' + context.signal?.throwIfAborted() - const validation = await validateUrlWithDNS(source, 'imageUrl') + const validation = await validateUrlWithDNS(source, 'imageUrl', profile) context.signal?.throwIfAborted() if (!validation.isValid) { fail(validation.error || 'Invalid image URL', 400, { @@ -109,7 +117,7 @@ async function resolveUrlImage( error: validation.error, }) } - return { source, resolvedIP: validation.resolvedIP } + return { source, resolvedIP: validation.resolvedIP, profile } } export async function executeVisionOperation( @@ -137,6 +145,7 @@ export async function executeVisionOperation( model: input.model, prompt: input.prompt || DEFAULT_PROMPT, remoteImageResolvedIP: image.resolvedIP, + remoteImageProfile: image.profile, }, context.signal ) diff --git a/apps/sim/lib/internal/whatsapp/operations.ts b/apps/sim/lib/internal/whatsapp/operations.ts index 1af0d6f6042..2ddfef539de 100644 --- a/apps/sim/lib/internal/whatsapp/operations.ts +++ b/apps/sim/lib/internal/whatsapp/operations.ts @@ -221,11 +221,12 @@ export async function executeWhatsAppGetMedia( ) } - const urlValidation = await validateUrlWithDNS(metadata.url, 'mediaUrl') + const urlValidation = await validateUrlWithDNS(metadata.url, 'mediaUrl', 'contentFetch') if (!urlValidation.isValid) { return failureResponse(`Invalid WhatsApp media URL: ${urlValidation.error}`, 502) } - const mediaResponse = await secureFetchWithPinnedIP(metadata.url, urlValidation.resolvedIP!, { + const mediaResponse = await secureFetchWithPinnedIP(metadata.url, urlValidation.resolvedIP, { + profile: 'contentFetch', method: 'GET', headers: { Authorization: authorization, 'User-Agent': DOWNLOAD_USER_AGENT }, maxResponseBytes: WHATSAPP_MEDIA_MAX_BYTES, diff --git a/apps/sim/lib/internal/windchill/client.ts b/apps/sim/lib/internal/windchill/client.ts index dc5f43bf60b..2b0059c37b7 100644 --- a/apps/sim/lib/internal/windchill/client.ts +++ b/apps/sim/lib/internal/windchill/client.ts @@ -116,6 +116,7 @@ export async function createWindchillSession( const response = await secureFetchWithValidation( `${ptcRoot(params.baseUrl)}/PTC/GetCSRFToken()`, { + profile: 'configuredEndpoint', method: 'GET', headers: { Authorization: createBasicAuthHeader(params.username, params.password), @@ -172,6 +173,7 @@ export async function windchillMutationRequest({ const response = await secureFetchWithValidation( url, { + profile: 'configuredEndpoint', method, headers, body: body === undefined ? undefined : JSON.stringify(body), @@ -318,6 +320,9 @@ export async function uploadWindchillContent({ const stageTwoResponse = await secureFetchWithValidation( descriptor.replicaUrl, { + // Windchill hands this URL back in the Stage 1 response, so it is + // response-derived rather than configured. + profile: 'contentFetch', method: 'POST', headers: { 'Content-Type': multipart.contentType, Accept: 'application/json' }, body: multipart.body, @@ -377,6 +382,7 @@ export async function resolveWindchillContentUrl({ const response = await secureFetchWithValidation( `${contentPath}/PTC.ApplicationData/Content/URL`, { + profile: 'configuredEndpoint', method: 'GET', headers: { Authorization: createBasicAuthHeader(params.username, params.password), @@ -435,6 +441,7 @@ export async function downloadWindchillContent({ const response = await secureFetchWithValidation( url, { + profile: 'configuredEndpoint', method: 'GET', headers: { Authorization: createBasicAuthHeader(params.username, params.password) }, stripAuthOnRedirect: true, diff --git a/apps/sim/lib/internal/zoho-desk/operations.ts b/apps/sim/lib/internal/zoho-desk/operations.ts index 21279980c70..8cb3f45a30e 100644 --- a/apps/sim/lib/internal/zoho-desk/operations.ts +++ b/apps/sim/lib/internal/zoho-desk/operations.ts @@ -37,6 +37,7 @@ export async function getZohoDeskAttachment( } const response = await secureFetchWithValidation(downloadUrl.toString(), { + profile: 'contentFetch', method: 'GET', headers: buildZohoDeskHeaders({ accessToken: input.accessToken, orgId: input.orgId }), timeout: 30_000, diff --git a/apps/sim/lib/internal/zoom/operations.ts b/apps/sim/lib/internal/zoom/operations.ts index 9bf64c3e8ff..1860b06e667 100644 --- a/apps/sim/lib/internal/zoom/operations.ts +++ b/apps/sim/lib/internal/zoom/operations.ts @@ -67,13 +67,14 @@ export async function getZoomMeetingRecordings( if (input.ttl) query.set('ttl', String(input.ttl)) const baseUrl = `https://api.zoom.us/v2/meetings/${encodeURIComponent(input.meetingId)}/recordings` const apiUrl = query.size > 0 ? `${baseUrl}?${query}` : baseUrl - const validation = await validateUrlWithDNS(apiUrl, 'apiUrl') + const validation = await validateUrlWithDNS(apiUrl, 'apiUrl', 'configuredEndpoint') context.signal?.throwIfAborted() - if (!validation.isValid || !validation.resolvedIP) { + if (!validation.isValid) { throw new ZoomOperationError(validation.error || 'Invalid Zoom API URL', 400) } const response = await secureFetchWithPinnedIP(apiUrl, validation.resolvedIP, { + profile: 'configuredEndpoint', method: 'GET', headers: { 'Content-Type': 'application/json', @@ -101,12 +102,17 @@ export async function getZoomMeetingRecordings( 413 ) } - const fileValidation = await validateUrlWithDNS(file.download_url, 'downloadUrl') - if (!fileValidation.isValid || !fileValidation.resolvedIP) continue + const fileValidation = await validateUrlWithDNS( + file.download_url, + 'downloadUrl', + 'contentFetch' + ) + if (!fileValidation.isValid) continue const downloadResponse = await secureFetchWithPinnedIP( file.download_url, fileValidation.resolvedIP, { + profile: 'contentFetch', method: 'GET', headers: { Authorization: `Bearer ${input.accessToken}` }, maxResponseBytes: remainingBytes, diff --git a/apps/sim/lib/internal/zoominfo/client.ts b/apps/sim/lib/internal/zoominfo/client.ts index b908cfce267..d98e19ea868 100644 --- a/apps/sim/lib/internal/zoominfo/client.ts +++ b/apps/sim/lib/internal/zoominfo/client.ts @@ -74,6 +74,7 @@ async function fetchAccessToken( const response = await secureFetchWithValidation( tokenUrl, { + profile: 'configuredEndpoint', method: 'POST', headers: { Authorization: `Basic ${basic}`, @@ -163,6 +164,7 @@ async function invokeZoomInfo( const response = await secureFetchWithValidation( url, { + profile: 'configuredEndpoint', method: input.method, headers, body: hasBody diff --git a/apps/sim/lib/invitations/core.test.ts b/apps/sim/lib/invitations/core.test.ts index ec014758f1c..0d28507befd 100644 --- a/apps/sim/lib/invitations/core.test.ts +++ b/apps/sim/lib/invitations/core.test.ts @@ -91,6 +91,7 @@ vi.mock('@sim/audit', () => auditMock) import { acceptInvitation, rejectInvitation, + resolveInvitationAdmissionOrganizationId, revokeInvitationAsAdmin, updateInvitation, } from '@/lib/invitations/core' @@ -2181,3 +2182,128 @@ describe('locked invitation mutations', () => { expect(dbChainMockFns.set).not.toHaveBeenCalled() }) }) + +/** + * What an invitation ADMITS TO, which is not its `kind`. The send-capability + * gates read this so they cannot let an invitation carry a member into an + * organization whose group withholds invitations, and so they cannot demand an + * organization's permission for an invitation that joins nobody to it. + */ +describe('resolveInvitationAdmissionOrganizationId', () => { + const invitation = { + id: 'invitation-1', + kind: 'workspace' as const, + email: 'invitee@example.com', + organizationId: 'organization-1', + membershipIntent: 'internal' as const, + inviterId: 'inviter-1', + role: 'member', + status: 'pending' as const, + token: 'token-1', + expiresAt: new Date('2026-12-01T00:00:00.000Z'), + createdAt: new Date('2026-01-01T00:00:00.000Z'), + updatedAt: new Date('2026-01-01T00:00:00.000Z'), + grants: [ + { + id: 'grant-1', + workspaceId: 'workspace-1', + permission: 'read' as const, + workspaceName: 'Workspace', + }, + ], + organizationName: null, + inviterName: null, + inviterEmail: null, + } + + beforeEach(() => { + vi.clearAllMocks() + resetDbChainMock() + mockGetUserOrganization.mockResolvedValue(null) + mockGetWorkspaceWithOwner.mockResolvedValue({ + id: 'workspace-1', + organizationId: 'organization-1', + billedAccountUserId: 'owner-1', + }) + }) + + it('names the organization a granted workspace belongs to, for a workspace invitation', async () => { + expect(await resolveInvitationAdmissionOrganizationId(invitation)).toBe('organization-1') + }) + + it('names nobody when the granted workspace belongs to no organization', async () => { + mockGetWorkspaceWithOwner.mockResolvedValue({ + id: 'workspace-1', + organizationId: null, + billedAccountUserId: 'owner-1', + }) + + expect(await resolveInvitationAdmissionOrganizationId(invitation)).toBeNull() + }) + + /** + * The workspace moved after the invite went out. Acceptance escalates into the + * new organization only when the inviter currently holds admin standing there, + * so the gate has to ask the same question of the same organization. + */ + it('follows a moved workspace into its live organization when the inviter may escalate', async () => { + mockGetWorkspaceWithOwner.mockResolvedValue({ + id: 'workspace-1', + organizationId: 'organization-2', + billedAccountUserId: 'owner-1', + }) + mockGetUserOrganization.mockResolvedValue({ + organizationId: 'organization-2', + role: 'admin', + }) + + expect(await resolveInvitationAdmissionOrganizationId(invitation)).toBe('organization-2') + }) + + it('names nobody when the escalation acceptance would refuse is the only join on offer', async () => { + mockGetWorkspaceWithOwner.mockResolvedValue({ + id: 'workspace-1', + organizationId: 'organization-2', + billedAccountUserId: 'owner-1', + }) + mockGetUserOrganization.mockResolvedValue({ + organizationId: 'organization-2', + role: 'member', + }) + + expect(await resolveInvitationAdmissionOrganizationId(invitation)).toBeNull() + }) + + /** + * An organization invitation joins its STAMPED organization whatever its + * granted workspaces do — the join target is never re-derived from a workspace + * whose organization can change after send. + */ + it('keeps an organization invitation on its stamped organization', async () => { + mockGetWorkspaceWithOwner.mockResolvedValue({ + id: 'workspace-1', + organizationId: 'organization-2', + billedAccountUserId: 'owner-1', + }) + + expect( + await resolveInvitationAdmissionOrganizationId({ ...invitation, kind: 'organization' }) + ).toBe('organization-1') + }) + + it('names nobody for an external invitation, which creates no membership', async () => { + expect( + await resolveInvitationAdmissionOrganizationId({ + ...invitation, + membershipIntent: 'external', + }) + ).toBeNull() + expect(mockGetWorkspaceWithOwner).not.toHaveBeenCalled() + }) + + it('falls back to the stamped organization for an invitation with no grants', async () => { + expect(await resolveInvitationAdmissionOrganizationId({ ...invitation, grants: [] })).toBe( + 'organization-1' + ) + }) +}) diff --git a/apps/sim/lib/invitations/core.ts b/apps/sim/lib/invitations/core.ts index 40b2026a0ca..f0cb61d32c2 100644 --- a/apps/sim/lib/invitations/core.ts +++ b/apps/sim/lib/invitations/core.ts @@ -250,6 +250,27 @@ export function isInvitationExpired(inv: Pick return new Date() > new Date(inv.expiresAt) } +/** + * The organization acceptance will land the invitee's membership in, before the + * gates that can downgrade the join to external. + * + * Workspace-kind invitations take the granted workspace's LIVE organization — + * the workspace is what was shared, and its organization can change after the + * invite goes out. Organization-kind invitations take their STAMPED one, which + * a granted workspace's move must never redirect. Acceptance, the accept-screen + * preview, and the resend gate all read the target here so none of them can + * disagree about which organization an invitation admits to. + */ +function invitationJoinTargetOrganizationId( + inv: Pick, + primaryWorkspace: Pick | null +): string | null { + if (inv.kind === 'workspace' && inv.grants.length > 0 && primaryWorkspace) { + return primaryWorkspace.organizationId + } + return inv.organizationId +} + /** * A workspace invitation only escalates into an EXISTING organization when * that organization matches what was stamped at send time — a workspace that @@ -281,6 +302,40 @@ async function stampedOrganizationAllowsEscalation( ) } +/** + * The organization ACCEPTANCE of this invitation would admit the invitee to, or + * `null` when acceptance creates no membership anywhere. + * + * Read by the send-capability gates, which have to key on what an invitation + * ADMITS TO rather than on its `kind`: a workspace-kind invitation whose granted + * workspace belongs to an organization joins the invitee to that organization + * exactly as an organization-kind one does ({@link acceptLockedInvitation} + * creates the member row from this same target), so gating those on their grants + * alone would let a workspace group that permits invitations carry a member into + * an organization whose default group withholds them. + * + * Mirrors acceptance's own decision, one clause at a time: an external + * membership intent creates no member row, and an escalation the stamped + * organization does not allow is downgraded to external before one is created. + * Both are read through the predicates acceptance uses, so a change there + * reaches this gate too. The reads are unlocked — a race resolves at accept + * time, where the locks are. + */ +export async function resolveInvitationAdmissionOrganizationId( + inv: InvitationWithGrants, + executor?: DbOrTx +): Promise { + if (inv.membershipIntent === 'external') return null + const primaryGrantWorkspaceId = inv.grants[0]?.workspaceId + const primaryWorkspace = primaryGrantWorkspaceId + ? await getWorkspaceWithOwner(primaryGrantWorkspaceId, executor ? { executor } : undefined) + : null + const organizationId = invitationJoinTargetOrganizationId(inv, primaryWorkspace) + if (!organizationId) return null + if (!(await stampedOrganizationAllowsEscalation(inv, organizationId, executor ?? db))) return null + return organizationId +} + /** * True when a member-role organization invitation still has at least one * granted workspace inside the organization it was stamped with. All grants @@ -362,18 +417,12 @@ export async function getInvitationJoinPreview( workspaceIdsToMove: [], }) - let workspaceOrganizationId = inv.organizationId - let billedAccountUserId: string | null = null const primaryGrantWorkspaceId = inv.grants[0]?.workspaceId - if (primaryGrantWorkspaceId) { - const primaryWorkspace = await getWorkspaceWithOwner(primaryGrantWorkspaceId) - if (primaryWorkspace) { - billedAccountUserId = primaryWorkspace.billedAccountUserId - if (inv.kind === 'workspace') { - workspaceOrganizationId = primaryWorkspace.organizationId - } - } - } + const primaryWorkspace = primaryGrantWorkspaceId + ? await getWorkspaceWithOwner(primaryGrantWorkspaceId) + : null + const billedAccountUserId = primaryWorkspace?.billedAccountUserId ?? null + const workspaceOrganizationId = invitationJoinTargetOrganizationId(inv, primaryWorkspace) /** * Personal-workspace invites only produce an organization through billing's @@ -858,11 +907,10 @@ async function acceptLockedInvitation( */ const primaryGrant = inv.grants[0] let billingOwnerUserId = inv.inviterId - let workspaceOrganizationId = inv.organizationId if (primaryGrant && lockPlan.primaryWorkspace && inv.kind === 'workspace') { billingOwnerUserId = lockPlan.primaryWorkspace.billedAccountUserId - workspaceOrganizationId = lockPlan.primaryWorkspace.organizationId } + const workspaceOrganizationId = invitationJoinTargetOrganizationId(inv, lockPlan.primaryWorkspace) if ( shouldJoinOrganization && diff --git a/apps/sim/lib/invitations/workspace-invitations.test.ts b/apps/sim/lib/invitations/workspace-invitations.test.ts index 6cebbf6ae47..afe9de882cc 100644 --- a/apps/sim/lib/invitations/workspace-invitations.test.ts +++ b/apps/sim/lib/invitations/workspace-invitations.test.ts @@ -112,7 +112,13 @@ vi.mock('@/ee/access-control/utils/permission-check', () => ({ validateInvitationsAllowed: vi.fn(), })) -import { createWorkspaceInvitation } from '@/lib/invitations/workspace-invitations' +import { + createWorkspaceInvitation, + prepareWorkspaceInvitationContext, +} from '@/lib/invitations/workspace-invitations' +import { hasWorkspaceAdminAccess } from '@/lib/workspaces/permissions/utils' +import { getWorkspaceInvitePolicy } from '@/lib/workspaces/policy' +import { validateInvitationsAllowed } from '@/ee/access-control/utils/permission-check' function queueWhereResponses(responses: unknown[][]) { const queue = [...responses] @@ -682,3 +688,56 @@ describe('createWorkspaceInvitation', () => { ).rejects.toThrow('invitation changed concurrently') }) }) + +/** + * The capability runs after the role check, never before it. Refusing on + * `invitations.send` first would answer a non-admin with a distinct `403` + * naming an organization setting, which tells a bystander in the same + * organization how another workspace's permission group is configured. + */ +describe('prepareWorkspaceInvitationContext refusal ordering', () => { + beforeEach(() => { + vi.clearAllMocks() + resetDbChainMock() + vi.mocked(validateInvitationsAllowed).mockResolvedValue(undefined) + vi.mocked(getWorkspaceInvitePolicy).mockResolvedValue({ + allowed: true, + reason: null, + requiresSeat: false, + organizationId: 'org-1', + upgradeRequired: false, + } as unknown as Awaited>) + }) + + it('refuses a non-admin on the role before consulting the permission group', async () => { + vi.mocked(hasWorkspaceAdminAccess).mockResolvedValue(false) + vi.mocked(validateInvitationsAllowed).mockRejectedValue( + new Error('Sending invitations is not available under your permission group') + ) + + await expect( + prepareWorkspaceInvitationContext({ + workspaceIds: ['ws-1'], + inviterId: 'outsider-1', + inviterName: 'Outsider', + }) + ).rejects.toThrow('You need admin permissions to invite users') + expect(validateInvitationsAllowed).not.toHaveBeenCalled() + }) + + it('still refuses an admin whose permission group withholds invitations', async () => { + vi.mocked(hasWorkspaceAdminAccess).mockResolvedValue(true) + vi.mocked(validateInvitationsAllowed).mockRejectedValue( + new Error('Sending invitations is not available under your permission group') + ) + + await expect( + prepareWorkspaceInvitationContext({ + workspaceIds: ['ws-1'], + inviterId: 'user-1', + inviterName: 'Owner', + }) + ).rejects.toThrow('Sending invitations is not available under your permission group') + expect(validateInvitationsAllowed).toHaveBeenCalledWith('user-1', 'ws-1') + }) +}) diff --git a/apps/sim/lib/invitations/workspace-invitations.ts b/apps/sim/lib/invitations/workspace-invitations.ts index 03e19257890..c96c3934fbe 100644 --- a/apps/sim/lib/invitations/workspace-invitations.ts +++ b/apps/sim/lib/invitations/workspace-invitations.ts @@ -206,8 +206,6 @@ export async function prepareWorkspaceInvitationContext({ const targets: WorkspaceInvitationTarget[] = [] for (const workspaceId of uniqueWorkspaceIds) { - await validateInvitationsAllowed(inviterId, workspaceId) - const isAdmin = await hasWorkspaceAdminAccess(inviterId, workspaceId) if (!isAdmin) { throw new WorkspaceInvitationError({ @@ -216,6 +214,16 @@ export async function prepareWorkspaceInvitationContext({ }) } + /** + * permission-group-enforced: invitations.send — after the admin check, not + * before it. The refusal names an organization setting, so answering it to + * someone with no admin reach into `workspaceId` would tell a bystander in + * the same organization how another workspace's group is configured. The + * role check is also the cheaper of the two and names the remedy the caller + * can actually act on. + */ + await validateInvitationsAllowed(inviterId, workspaceId) + const workspaceDetails = await getWorkspaceWithOwner(workspaceId) if (!workspaceDetails) { throw new WorkspaceInvitationError({ message: 'Workspace not found', status: 404 }) diff --git a/apps/sim/lib/knowledge/application/connectors.test.ts b/apps/sim/lib/knowledge/application/connectors.test.ts index 3c2abeac045..599f7f42ba3 100644 --- a/apps/sim/lib/knowledge/application/connectors.test.ts +++ b/apps/sim/lib/knowledge/application/connectors.test.ts @@ -21,6 +21,7 @@ const mocks = vi.hoisted(() => ({ refreshToken: vi.fn(), validateConnectorConfig: vi.fn(), recordAudit: vi.fn(), + getUserPermissionConfig: vi.fn(), })) vi.mock('@sim/audit', () => ({ @@ -67,6 +68,10 @@ vi.mock('@/lib/oauth/credential-service', () => ({ refreshAccessTokenIfNeeded: mocks.refreshToken, })) +vi.mock('@/lib/permission-groups/resolve.server', () => ({ + getUserPermissionConfig: mocks.getUserPermissionConfig, +})) + vi.mock('@/connectors/registry.server', () => ({ CONNECTOR_REGISTRY: { confluence: { @@ -84,6 +89,8 @@ import { updateKnowledgeConnector, updateKnowledgeConnectorDocuments, } from '@/lib/knowledge/application/connectors' +import { capabilityRefusal } from '@/lib/permission-groups/capability-assertions' +import { DEFAULT_PERMISSION_GROUP_CONFIG } from '@/lib/permission-groups/fields' const crossWorkspaceContext = { workspaceId: 'workspace-b', @@ -141,6 +148,7 @@ describe('knowledge connector application use cases', () => { mocks.refreshToken.mockResolvedValue('access-token') mocks.validateConnectorConfig.mockResolvedValue({ valid: true }) mocks.resolveBilling.mockResolvedValue(BILLING) + mocks.getUserPermissionConfig.mockResolvedValue(null) }) afterAll(resetDbChainMock) @@ -661,4 +669,166 @@ describe('knowledge connector application use cases', () => { ) } ) + + describe('connector allow-list', () => { + const sameWorkspaceContext = { + ...crossWorkspaceContext, + workspaceId: 'workspace-a', + knowledgeBaseId: 'knowledge-a', + knowledgeBase: { id: 'knowledge-a', name: 'Workspace A docs' }, + } + + const createInput = { + knowledgeBaseId: 'knowledge-a', + assertedWorkspaceId: 'workspace-a', + connectorType: 'confluence', + credentialId: 'credential-1', + sourceConfig: {}, + syncIntervalMinutes: 1440, + resolveBillingAttribution: mocks.resolveBilling, + } + + beforeEach(() => { + mocks.resolveKnowledgeBase.mockResolvedValue(sameWorkspaceContext) + }) + + function allowOnly(connectorTypes: string[] | null) { + mocks.getUserPermissionConfig.mockResolvedValue({ + ...DEFAULT_PERMISSION_GROUP_CONFIG, + allowedKnowledgeConnectors: connectorTypes, + }) + } + + it('refuses a connector the group withholds, before the connector is created', async () => { + allowOnly(['google_drive']) + + await expect( + createKnowledgeConnector.execute({ principal: delegatedPrincipal, input: createInput }) + ).rejects.toMatchObject({ + code: 'forbidden', + message: capabilityRefusal('knowledge.connectors'), + }) + + expect(mocks.createConnector).not.toHaveBeenCalled() + expect(mocks.recordAudit).not.toHaveBeenCalled() + }) + + it.each([ + ['the group names it', ['confluence', 'google_drive']], + ['the group restricts nothing', null], + ])('permits a connector when %s', async (_case, allowed) => { + allowOnly(allowed as string[] | null) + mocks.createConnector.mockResolvedValueOnce({ + success: true, + connector: { id: 'connector-a', connectorType: 'confluence', syncIntervalMinutes: 1440 }, + }) + + const result = await createKnowledgeConnector.execute({ + principal: delegatedPrincipal, + input: createInput, + }) + + expect(result.connector.id).toBe('connector-a') + expect(mocks.createConnector).toHaveBeenCalledTimes(1) + }) + + it('leaves an ungoverned caller unaffected', async () => { + mocks.getUserPermissionConfig.mockResolvedValue(null) + mocks.createConnector.mockResolvedValueOnce({ + success: true, + connector: { id: 'connector-a', connectorType: 'confluence', syncIntervalMinutes: 1440 }, + }) + + await createKnowledgeConnector.execute({ + principal: delegatedPrincipal, + input: createInput, + }) + + expect(mocks.createConnector).toHaveBeenCalledTimes(1) + }) + + /** + * A manual sync re-runs the pull, so an admin who has since removed the + * source from the allowlist has withdrawn it. The type comes off the + * persisted connector, which is the only place the request names it. + */ + describe('manual sync', () => { + const syncInput = { + knowledgeBaseId: 'knowledge-a', + connectorId: 'connector-b', + assertedWorkspaceId: 'workspace-a', + resolveBillingAttribution: mocks.resolveBilling, + } + + beforeEach(() => { + mocks.resolveConnector.mockResolvedValue({ + ...connectorContext, + workspaceId: 'workspace-a', + knowledgeBaseId: 'knowledge-a', + knowledgeBase: { id: 'knowledge-a', name: 'Workspace A docs' }, + connector: { ...connectorContext.connector, knowledgeBaseId: 'knowledge-a' }, + }) + }) + + it('refuses a sync of a connector whose type the group no longer names', async () => { + allowOnly(['google_drive']) + + await expect( + syncKnowledgeConnector.execute({ principal: delegatedPrincipal, input: syncInput }) + ).rejects.toMatchObject({ + code: 'forbidden', + message: capabilityRefusal('knowledge.connectors'), + }) + + expect(mocks.syncConnector).not.toHaveBeenCalled() + expect(mocks.recordAudit).not.toHaveBeenCalled() + }) + + it('permits the sync while the group still names the persisted type', async () => { + allowOnly(['confluence']) + mocks.syncConnector.mockResolvedValueOnce({ success: true }) + + await syncKnowledgeConnector.execute({ + principal: delegatedPrincipal, + input: syncInput, + }) + + expect(mocks.syncConnector).toHaveBeenCalledTimes(1) + }) + + /** + * Pausing and deleting stay reachable: the point is to stop the member + * re-running the pull, never to strand the connector. + */ + it('still lets the same caller pause and delete the withheld connector', async () => { + allowOnly(['google_drive']) + mocks.updateConnector.mockResolvedValueOnce({ + success: true, + connector: { ...connectorContext.connector, knowledgeBaseId: 'knowledge-a' }, + }) + mocks.deleteConnector.mockResolvedValueOnce({ + success: true, + documentsDeleted: 0, + documentsKept: 1, + }) + + await updateKnowledgeConnector.execute({ + principal: delegatedPrincipal, + input: { + connectorId: 'connector-b', + assertedWorkspaceId: 'workspace-a', + updates: { status: 'paused' }, + resolveBillingAttribution: mocks.resolveBilling, + }, + }) + await deleteKnowledgeConnector.execute({ + principal: delegatedPrincipal, + input: { connectorId: 'connector-b', assertedWorkspaceId: 'workspace-a' }, + }) + + expect(mocks.updateConnector).toHaveBeenCalledTimes(1) + expect(mocks.deleteConnector).toHaveBeenCalledTimes(1) + }) + }) + }) }) diff --git a/apps/sim/lib/knowledge/application/connectors.ts b/apps/sim/lib/knowledge/application/connectors.ts index d659aece16b..5e704149b80 100644 --- a/apps/sim/lib/knowledge/application/connectors.ts +++ b/apps/sim/lib/knowledge/application/connectors.ts @@ -1,4 +1,5 @@ import { AuditAction, AuditResourceType } from '@sim/audit' +import { resolvePrincipalSubjectUserId } from '@sim/auth/principal' import { db } from '@sim/db' import { document, knowledgeConnector, knowledgeConnectorSyncLog } from '@sim/db/schema' import { and, asc, count, desc, eq, inArray, isNull } from 'drizzle-orm' @@ -41,6 +42,8 @@ import type { KnowledgeOrchestrationResult, } from '@/lib/knowledge/orchestration/shared' import { refreshAccessTokenIfNeeded } from '@/lib/oauth/credential-service' +import { CAPABILITY_RULES, refuseCapability } from '@/lib/permission-groups/capabilities' +import { resolvePermissionGroupConfig } from '@/lib/permission-groups/config-scope.server' interface KnowledgeConnectorApplicationInput { assertedWorkspaceId?: string @@ -102,6 +105,43 @@ export interface UpdateKnowledgeConnectorDocumentsInput extends ReadKnowledgeCon documentIds: string[] } +const CONNECTOR_ALLOWLIST_RULE = CAPABILITY_RULES['knowledge.connectors'] + +/** + * Refuses a connector the caller's permission group has not sanctioned. + * + * A connector pulls a whole external corpus into the workspace, so which source + * a member may attach is a per-request decision — the authorization funnel + * applies an operation's capability knowing only the principal, the workspace + * and the operation, and never sees `connectorType`. Hence the assertion here, + * ahead of the write, rather than a `capability` on `knowledge.connectors.create`. + * + * No-op when no permission group governs the caller, which is what keeps + * non-enterprise and ungoverned organizations unaffected. + * + * A permission group is a membership of users, so an actorless caller — a + * schedule, or a webhook with no external subject — resolves no group and + * passes through, exactly as the authorization funnel treats one. Requiring a + * subject here would turn every scheduled connector sync into a 500 rather than + * a refusal anyone could act on. + * + * Refused through {@link refuseCapability} so the sentence reads exactly like + * every other capability refusal. The error it throws is a + * `ForbiddenOperationError` carrying this rule's own detail code, so the status + * and error contract are the ones this already raised. + */ +async function assertConnectorTypeAllowed( + userId: string | undefined, + workspaceId: string, + connectorType: string +): Promise { + if (!userId) return + const config = await resolvePermissionGroupConfig(userId, workspaceId, undefined) + if (!config || !CONNECTOR_ALLOWLIST_RULE.deniedBy(config, connectorType)) return + + refuseCapability('knowledge.connectors') +} + function requireSuccessfulOutcome( outcome: KnowledgeOrchestrationResult, fallback: string @@ -293,6 +333,12 @@ export const createKnowledgeConnector = defineAuthorizedKnowledgeUseCase({ const requestId = generateRequestId() const workspaceId = requireConnectorWorkspaceId(context) const actingUserId = resolveKnowledgeAttributedUserId(principal, context) + // permission-group-enforced: knowledge.connectors — needs the request's connector id, which the funnel never sees + await assertConnectorTypeAllowed( + resolvePrincipalSubjectUserId(principal), + workspaceId, + input.connectorType + ) const outcome = await performCreateKnowledgeConnector({ knowledgeBase: connectorTarget(context), connectorType: input.connectorType, @@ -337,6 +383,13 @@ export const createKnowledgeConnector = defineAuthorizedKnowledgeUseCase({ }), }) +/** + * Deliberately not gated by `knowledge.connectors`: an update may change the + * source config, sync interval or status, never the connector type. The + * sanctioned-source decision was made when the connector was created, and + * re-asserting it here would strand an existing connector — including the + * ability to pause it — the moment an admin narrowed the allowlist. + */ export const updateKnowledgeConnector = defineAuthorizedKnowledgeUseCase({ operation: knowledgeOperations.updateConnector, resolveContext: ({ input }: { input: UpdateKnowledgeConnectorInput }) => @@ -440,12 +493,33 @@ export const deleteKnowledgeConnector = defineAuthorizedKnowledgeUseCase({ }), }) +/** + * Gated by `knowledge.connectors` on the *persisted* type, unlike + * {@link updateKnowledgeConnector}: a manual sync is a fresh act by a person + * pulling the external corpus in again, so an admin who has since removed the + * source from the allowlist has withdrawn it. Pausing and deleting stay + * available for the reason recorded on the update use case — nothing here + * strands a connector, it only stops a member re-running the pull by hand. + * + * Only the manual path passes through this use case. The scheduled continuation + * of an existing connector runs `executeSync` from the sync engine directly + * (`background/knowledge-connector-sync.ts`) and is untouched, matching the + * webhook precedent: passive continuation keeps running, a person re-initiating + * it is gated. An actorless caller resolves no group and passes through, as + * {@link assertConnectorTypeAllowed} documents. + */ export const syncKnowledgeConnector = defineAuthorizedKnowledgeUseCase({ operation: knowledgeOperations.syncConnector, resolveContext: ({ input }: { input: SyncKnowledgeConnectorInput }) => resolveActiveKnowledgeConnectorContext(input), async execute({ principal, input, context, request }) { const workspaceId = requireConnectorWorkspaceId(context) + // permission-group-enforced: knowledge.connectors — needs the persisted connector type, which the funnel never sees + await assertConnectorTypeAllowed( + resolvePrincipalSubjectUserId(principal), + workspaceId, + context.connector.connectorType + ) const outcome = await performSyncKnowledgeConnector({ knowledgeBase: connectorTarget(context), connectorId: context.connectorId, diff --git a/apps/sim/lib/knowledge/application/operations.test.ts b/apps/sim/lib/knowledge/application/operations.test.ts index 803a4223c3b..9180ee81346 100644 --- a/apps/sim/lib/knowledge/application/operations.test.ts +++ b/apps/sim/lib/knowledge/application/operations.test.ts @@ -156,4 +156,43 @@ describe('knowledge operation registry', () => { expect(knowledgeOperations.search.delegatedServices).toEqual(['copilot', 'executor']) expect(knowledgeOperations.uploadComplete.delegatedServices).toBeUndefined() }) + + it('withholds knowledge base creation separately from using existing ones', () => { + expect(knowledgeOperations.create.capability).toBe('knowledge.create') + for (const operation of [ + knowledgeOperations.list, + knowledgeOperations.read, + knowledgeOperations.search, + knowledgeOperations.update, + knowledgeOperations.delete, + knowledgeOperations.createFolder, + knowledgeOperations.createTag, + knowledgeOperations.createConnector, + ]) { + expect(operation.capability).toBe('knowledge.use') + } + }) + + it('withholds every path that carries caller-supplied document bytes', () => { + for (const operation of [ + knowledgeOperations.uploadDocument, + knowledgeOperations.uploadCreate, + knowledgeOperations.uploadParts, + knowledgeOperations.uploadComplete, + knowledgeOperations.uploadCancel, + ]) { + expect(operation.capability).toBe('knowledge.upload') + } + }) + + it('leaves the connector sync path on the shared knowledge capability', () => { + /** A connector's documents are the sanctioned source, so an upload ban must not reach them. */ + for (const operation of [ + knowledgeOperations.syncConnector, + knowledgeOperations.updateConnectorDocuments, + knowledgeOperations.addWorkspaceFiles, + ]) { + expect(operation.capability).toBe('knowledge.use') + } + }) }) diff --git a/apps/sim/lib/knowledge/application/operations.ts b/apps/sim/lib/knowledge/application/operations.ts index a4ffeb7567b..ddd8cdc6fe8 100644 --- a/apps/sim/lib/knowledge/application/operations.ts +++ b/apps/sim/lib/knowledge/application/operations.ts @@ -1,4 +1,5 @@ -import { defineWorkspaceOperation } from '@/lib/core/application' +import type { ApplicationOperation } from '@/lib/core/application' +import { assertOperationCapability, defineWorkspaceOperation } from '@/lib/core/application' const ALL_PRINCIPAL_POLICY = { principalKinds: ['session', 'personal_api_key', 'workspace_api_key', 'delegated'], @@ -42,30 +43,40 @@ export const knowledgeOperations = { id: 'knowledge.list', minimumRole: 'read', workspaceApiKey: 'allow', + capability: 'knowledge.use', ...ALL_PRINCIPAL_POLICY, }), read: defineWorkspaceOperation({ id: 'knowledge.read', minimumRole: 'read', workspaceApiKey: 'allow', + capability: 'knowledge.use', ...ALL_PRINCIPAL_POLICY, }), + /** + * The only operation that brings a knowledge base into existence, so it is the + * only one `knowledge.create` governs — a group may be allowed to query, + * populate and organize the bases it already has without opening new ones. + */ create: defineWorkspaceOperation({ id: 'knowledge.create', minimumRole: 'write', workspaceApiKey: 'allow', + capability: 'knowledge.create', ...ALL_PRINCIPAL_POLICY, }), update: defineWorkspaceOperation({ id: 'knowledge.update', minimumRole: 'write', workspaceApiKey: 'allow', + capability: 'knowledge.use', ...ALL_PRINCIPAL_POLICY, }), delete: defineWorkspaceOperation({ id: 'knowledge.delete', minimumRole: 'write', workspaceApiKey: 'allow', + capability: 'knowledge.use', ...ALL_PRINCIPAL_POLICY, }), /** @@ -79,162 +90,195 @@ export const knowledgeOperations = { id: 'knowledge.restore', minimumRole: 'write', workspaceApiKey: 'allow', + capability: 'knowledge.use', ...ALL_PRINCIPAL_POLICY, }), bulkMoveItems: defineWorkspaceOperation({ id: 'knowledge.bulk_move_items', minimumRole: 'write', workspaceApiKey: 'allow', + capability: 'knowledge.use', ...ALL_PRINCIPAL_POLICY, }), bulkDeleteItems: defineWorkspaceOperation({ id: 'knowledge.bulk_delete_items', minimumRole: 'write', workspaceApiKey: 'allow', + capability: 'knowledge.use', ...ALL_PRINCIPAL_POLICY, }), bulkDelete: defineWorkspaceOperation({ id: 'knowledge.bulk_delete', minimumRole: 'write', workspaceApiKey: 'allow', + capability: 'knowledge.use', ...ALL_PRINCIPAL_POLICY, }), renameByVfsPath: defineWorkspaceOperation({ id: 'knowledge.vfs.rename', minimumRole: 'write', workspaceApiKey: 'deny', + capability: 'knowledge.use', ...COPILOT_PRINCIPAL_POLICY, }), moveByVfsPath: defineWorkspaceOperation({ id: 'knowledge.vfs.move', minimumRole: 'write', workspaceApiKey: 'deny', + capability: 'knowledge.use', ...COPILOT_PRINCIPAL_POLICY, }), manageVfsFolders: defineWorkspaceOperation({ id: 'knowledge.vfs.folders.manage', minimumRole: 'write', workspaceApiKey: 'deny', + capability: 'knowledge.use', ...COPILOT_PRINCIPAL_POLICY, }), deleteByVfsPath: defineWorkspaceOperation({ id: 'knowledge.vfs.delete', minimumRole: 'write', workspaceApiKey: 'deny', + capability: 'knowledge.use', ...COPILOT_PRINCIPAL_POLICY, }), search: defineWorkspaceOperation({ id: 'knowledge.search', minimumRole: 'read', workspaceApiKey: 'allow', + capability: 'knowledge.use', ...ALL_PRINCIPAL_WITH_EXECUTOR_POLICY, }), listFolders: defineWorkspaceOperation({ id: 'knowledge.folders.list', minimumRole: 'read', workspaceApiKey: 'allow', + capability: 'knowledge.use', principalKinds: HTTP_PRINCIPAL_KINDS, }), createFolder: defineWorkspaceOperation({ id: 'knowledge.folders.create', minimumRole: 'write', workspaceApiKey: 'allow', + capability: 'knowledge.use', principalKinds: HTTP_PRINCIPAL_KINDS, }), relocateFolder: defineWorkspaceOperation({ id: 'knowledge.folders.relocate', minimumRole: 'write', workspaceApiKey: 'allow', + capability: 'knowledge.use', principalKinds: HTTP_PRINCIPAL_KINDS, }), deleteFolder: defineWorkspaceOperation({ id: 'knowledge.folders.delete', minimumRole: 'write', workspaceApiKey: 'allow', + capability: 'knowledge.use', principalKinds: HTTP_PRINCIPAL_KINDS, }), listDocuments: defineWorkspaceOperation({ id: 'knowledge.documents.list', minimumRole: 'read', workspaceApiKey: 'allow', + capability: 'knowledge.use', ...ALL_PRINCIPAL_WITH_EXECUTOR_POLICY, }), readDocument: defineWorkspaceOperation({ id: 'knowledge.documents.read', minimumRole: 'read', workspaceApiKey: 'allow', + capability: 'knowledge.use', ...ALL_PRINCIPAL_WITH_EXECUTOR_POLICY, }), + /** + * The single-request upload path: the caller hands over file bytes, so the + * document's provenance is whatever the caller chose. `knowledge.upload` is + * what an organization withholds to admit documents only from the connectors + * it sanctioned. + */ uploadDocument: defineWorkspaceOperation({ id: 'knowledge.documents.upload', minimumRole: 'write', workspaceApiKey: 'allow', + capability: 'knowledge.upload', ...ALL_PRINCIPAL_WITH_EXECUTOR_POLICY, }), addWorkspaceFiles: defineWorkspaceOperation({ id: 'knowledge.documents.add_workspace_files', minimumRole: 'write', workspaceApiKey: 'deny', + capability: 'knowledge.use', ...HUMAN_AND_COPILOT_PRINCIPAL_POLICY, }), deleteDocument: defineWorkspaceOperation({ id: 'knowledge.documents.delete', minimumRole: 'write', workspaceApiKey: 'allow', + capability: 'knowledge.use', ...ALL_PRINCIPAL_WITH_EXECUTOR_POLICY, }), bulkDeleteDocuments: defineWorkspaceOperation({ id: 'knowledge.documents.bulk_delete', minimumRole: 'write', workspaceApiKey: 'deny', + capability: 'knowledge.use', ...HUMAN_AND_COPILOT_PRINCIPAL_POLICY, }), updateDocument: defineWorkspaceOperation({ id: 'knowledge.documents.update', minimumRole: 'write', workspaceApiKey: 'deny', + capability: 'knowledge.use', ...HUMAN_COPILOT_AND_EXECUTOR_PRINCIPAL_POLICY, }), bulkDocuments: defineWorkspaceOperation({ id: 'knowledge.documents.bulk', minimumRole: 'write', workspaceApiKey: 'deny', + capability: 'knowledge.use', ...HUMAN_AND_COPILOT_PRINCIPAL_POLICY, }), listChunks: defineWorkspaceOperation({ id: 'knowledge.chunks.list', minimumRole: 'read', workspaceApiKey: 'deny', + capability: 'knowledge.use', ...HUMAN_COPILOT_AND_EXECUTOR_PRINCIPAL_POLICY, }), readChunk: defineWorkspaceOperation({ id: 'knowledge.chunks.read', minimumRole: 'read', workspaceApiKey: 'deny', + capability: 'knowledge.use', ...HUMAN_AND_COPILOT_PRINCIPAL_POLICY, }), createChunk: defineWorkspaceOperation({ id: 'knowledge.chunks.create', minimumRole: 'write', workspaceApiKey: 'deny', + capability: 'knowledge.use', ...HUMAN_COPILOT_AND_EXECUTOR_PRINCIPAL_POLICY, }), updateChunk: defineWorkspaceOperation({ id: 'knowledge.chunks.update', minimumRole: 'write', workspaceApiKey: 'deny', + capability: 'knowledge.use', ...HUMAN_COPILOT_AND_EXECUTOR_PRINCIPAL_POLICY, }), deleteChunk: defineWorkspaceOperation({ id: 'knowledge.chunks.delete', minimumRole: 'write', workspaceApiKey: 'deny', + capability: 'knowledge.use', ...HUMAN_COPILOT_AND_EXECUTOR_PRINCIPAL_POLICY, }), bulkChunks: defineWorkspaceOperation({ id: 'knowledge.chunks.bulk', minimumRole: 'write', workspaceApiKey: 'deny', + capability: 'knowledge.use', ...HUMAN_AND_COPILOT_PRINCIPAL_POLICY, }), /** @@ -247,42 +291,49 @@ export const knowledgeOperations = { id: 'knowledge.tags.list', minimumRole: 'read', workspaceApiKey: 'allow', + capability: 'knowledge.use', ...ALL_PRINCIPAL_WITH_EXECUTOR_POLICY, }), createTag: defineWorkspaceOperation({ id: 'knowledge.tags.create', minimumRole: 'write', workspaceApiKey: 'deny', + capability: 'knowledge.use', ...HUMAN_AND_COPILOT_PRINCIPAL_POLICY, }), updateTag: defineWorkspaceOperation({ id: 'knowledge.tags.update', minimumRole: 'write', workspaceApiKey: 'deny', + capability: 'knowledge.use', ...HUMAN_AND_COPILOT_PRINCIPAL_POLICY, }), deleteTag: defineWorkspaceOperation({ id: 'knowledge.tags.delete', minimumRole: 'write', workspaceApiKey: 'deny', + capability: 'knowledge.use', ...HUMAN_AND_COPILOT_PRINCIPAL_POLICY, }), readTagUsage: defineWorkspaceOperation({ id: 'knowledge.tags.read_usage', minimumRole: 'read', workspaceApiKey: 'deny', + capability: 'knowledge.use', ...HUMAN_AND_COPILOT_PRINCIPAL_POLICY, }), readDetailedTagUsage: defineWorkspaceOperation({ id: 'knowledge.tags.read_detailed_usage', minimumRole: 'read', workspaceApiKey: 'deny', + capability: 'knowledge.use', ...HUMAN_AND_COPILOT_PRINCIPAL_POLICY, }), readNextTagSlot: defineWorkspaceOperation({ id: 'knowledge.tags.read_next_slot', minimumRole: 'read', workspaceApiKey: 'deny', + capability: 'knowledge.use', ...HUMAN_AND_COPILOT_PRINCIPAL_POLICY, }), /** @@ -296,6 +347,7 @@ export const knowledgeOperations = { id: 'knowledge.tags.bulk_save', minimumRole: 'write', workspaceApiKey: 'deny', + capability: 'knowledge.use', ...HUMAN_AND_COPILOT_PRINCIPAL_POLICY, }), /** Removal over that same vocabulary — unused definitions, or all of them. */ @@ -303,88 +355,127 @@ export const knowledgeOperations = { id: 'knowledge.tags.cleanup', minimumRole: 'write', workspaceApiKey: 'deny', + capability: 'knowledge.use', ...HUMAN_AND_COPILOT_PRINCIPAL_POLICY, }), listConnectors: defineWorkspaceOperation({ id: 'knowledge.connectors.list', minimumRole: 'read', workspaceApiKey: 'deny', + capability: 'knowledge.use', ...HUMAN_COPILOT_AND_EXECUTOR_PRINCIPAL_POLICY, }), readConnector: defineWorkspaceOperation({ id: 'knowledge.connectors.read', minimumRole: 'read', workspaceApiKey: 'deny', + capability: 'knowledge.use', ...HUMAN_COPILOT_AND_EXECUTOR_PRINCIPAL_POLICY, }), createConnector: defineWorkspaceOperation({ id: 'knowledge.connectors.create', minimumRole: 'write', workspaceApiKey: 'deny', + capability: 'knowledge.use', ...HUMAN_AND_COPILOT_PRINCIPAL_POLICY, }), updateConnector: defineWorkspaceOperation({ id: 'knowledge.connectors.update', minimumRole: 'write', workspaceApiKey: 'deny', + capability: 'knowledge.use', ...HUMAN_AND_COPILOT_PRINCIPAL_POLICY, }), deleteConnector: defineWorkspaceOperation({ id: 'knowledge.connectors.delete', minimumRole: 'write', workspaceApiKey: 'deny', + capability: 'knowledge.use', ...HUMAN_AND_COPILOT_PRINCIPAL_POLICY, }), syncConnector: defineWorkspaceOperation({ id: 'knowledge.connectors.sync', minimumRole: 'write', workspaceApiKey: 'deny', + capability: 'knowledge.use', ...HUMAN_COPILOT_AND_EXECUTOR_PRINCIPAL_POLICY, }), listConnectorDocuments: defineWorkspaceOperation({ id: 'knowledge.connectors.documents.list', minimumRole: 'read', workspaceApiKey: 'deny', + capability: 'knowledge.use', ...HUMAN_AND_COPILOT_PRINCIPAL_POLICY, }), updateConnectorDocuments: defineWorkspaceOperation({ id: 'knowledge.connectors.documents.update', minimumRole: 'write', workspaceApiKey: 'deny', + capability: 'knowledge.use', ...HUMAN_AND_COPILOT_PRINCIPAL_POLICY, }), + /** + * The four session operations are one upload, split across requests only + * because a large file cannot arrive in one. They carry the same capability + * for that reason — including cancel, which would otherwise be the one open + * door into a surface the group was denied. + */ uploadCreate: defineWorkspaceOperation({ id: 'knowledge.documents.upload.create', minimumRole: 'write', workspaceApiKey: 'allow', + capability: 'knowledge.upload', principalKinds: HTTP_PRINCIPAL_KINDS, }), uploadParts: defineWorkspaceOperation({ id: 'knowledge.documents.upload.parts', minimumRole: 'write', workspaceApiKey: 'allow', + capability: 'knowledge.upload', principalKinds: HTTP_PRINCIPAL_KINDS, }), uploadComplete: defineWorkspaceOperation({ id: 'knowledge.documents.upload.complete', minimumRole: 'write', workspaceApiKey: 'allow', + capability: 'knowledge.upload', principalKinds: HTTP_PRINCIPAL_KINDS, }), uploadCancel: defineWorkspaceOperation({ id: 'knowledge.documents.upload.cancel', minimumRole: 'write', workspaceApiKey: 'allow', + capability: 'knowledge.upload', principalKinds: HTTP_PRINCIPAL_KINDS, }), } as const +/** + * The session-scoped entry points, which resolve a knowledge base first and then + * hand authorization to the workspace-scoped `knowledgeOperations` sibling that + * matches. The capability rides on that sibling, so each of these declares + * `'none'` — but declares it, rather than being minted from a bare object + * literal as they were, which is the form that kept them out of + * `check:permission-group-enforcement` entirely. + */ +function defineKnowledgeSessionOperation( + operation: ApplicationOperation +): ApplicationOperation { + assertOperationCapability(operation) + return Object.freeze(operation) +} + export const knowledgeSessionOperations = { - list: Object.freeze({ id: 'knowledge.session.list' as const }), - read: Object.freeze({ id: 'knowledge.session.read' as const }), - update: Object.freeze({ id: 'knowledge.session.update' as const }), - delete: Object.freeze({ id: 'knowledge.session.delete' as const }), - restore: Object.freeze({ id: 'knowledge.session.restore' as const }), + // permission-group-exempt: delegates to knowledgeOperations.list, which carries knowledge.use + list: defineKnowledgeSessionOperation({ id: 'knowledge.session.list', capability: 'none' }), + // permission-group-exempt: delegates to knowledgeOperations.read, which carries knowledge.use + read: defineKnowledgeSessionOperation({ id: 'knowledge.session.read', capability: 'none' }), + // permission-group-exempt: delegates to knowledgeOperations.update, which carries knowledge.use + update: defineKnowledgeSessionOperation({ id: 'knowledge.session.update', capability: 'none' }), + // permission-group-exempt: delegates to knowledgeOperations.delete, which carries knowledge.use + delete: defineKnowledgeSessionOperation({ id: 'knowledge.session.delete', capability: 'none' }), + // permission-group-exempt: delegates to knowledgeOperations.restore, which carries knowledge.use + restore: defineKnowledgeSessionOperation({ id: 'knowledge.session.restore', capability: 'none' }), } as const export type KnowledgeOperation = (typeof knowledgeOperations)[keyof typeof knowledgeOperations] diff --git a/apps/sim/lib/knowledge/connectors/sync-engine.test.ts b/apps/sim/lib/knowledge/connectors/sync-engine.test.ts index 12a5919fafd..0e459335f1f 100644 --- a/apps/sim/lib/knowledge/connectors/sync-engine.test.ts +++ b/apps/sim/lib/knowledge/connectors/sync-engine.test.ts @@ -1257,9 +1257,16 @@ describe('executeSync deferred hydration rate limits', () => { expect(dbChainMockFns.set).toHaveBeenCalledWith( expect.objectContaining({ status: 'error', - nextSyncAt: new Date(NOW.getTime() + 45 * 60 * 1000), + consecutiveFailures: 0, }) ) + const failureUpdate = dbChainMockFns.set.mock.calls.find( + ([update]) => update.status === 'error' + )?.[0] + expect(failureUpdate?.nextSyncAt.getTime()).toBeGreaterThanOrEqual( + NOW.getTime() + 45 * 60 * 1000 + ) + expect(failureUpdate?.nextSyncAt.getTime()).toBeLessThanOrEqual(NOW.getTime() + 46 * 60 * 1000) }) }) @@ -2389,6 +2396,43 @@ describe('buildSyncFailureUpdate', () => { }) }) +describe('buildSyncRateLimitUpdate', () => { + const now = new Date('2026-08-20T00:00:00.000Z') + + it('preserves the failure counter and schedules after the provider deadline', async () => { + const { buildSyncRateLimitUpdate } = await import('@/lib/knowledge/connectors/sync-engine') + const providerDelayMs = 45 * 60 * 1000 + const update = buildSyncRateLimitUpdate(now, 9, 'rate limited', providerDelayMs) + + expect(update.status).toBe('error') + expect(update.lastSyncError).toBe('rate limited') + expect(update.consecutiveFailures).toBe(9) + expect(update.nextSyncAt.getTime()).toBeGreaterThanOrEqual(now.getTime() + providerDelayMs) + expect(update.nextSyncAt.getTime()).toBeLessThanOrEqual( + now.getTime() + providerDelayMs + 60_000 + ) + }) + + it('uses a conservative fallback without consuming the breaker', async () => { + const { buildSyncRateLimitUpdate } = await import('@/lib/knowledge/connectors/sync-engine') + const update = buildSyncRateLimitUpdate(now, null, 'rate limited') + const fallbackMs = 30 * 60 * 1000 + + expect(update.consecutiveFailures).toBe(0) + expect(update.nextSyncAt.getTime()).toBeGreaterThanOrEqual(now.getTime() + fallbackMs) + expect(update.nextSyncAt.getTime()).toBeLessThanOrEqual(now.getTime() + fallbackMs + 60_000) + }) + + it('caps the provider deadline and releases the sync lease', async () => { + const { buildSyncRateLimitUpdate } = await import('@/lib/knowledge/connectors/sync-engine') + const update = buildSyncRateLimitUpdate(now, 4, 'rate limited', 30 * 24 * 60 * 60 * 1000) + + expect(update.nextSyncAt).toEqual(new Date(now.getTime() + 24 * 60 * 60 * 1000)) + expect(update.syncLockToken).toBeNull() + expect(update.syncLockLeaseAt).toBeNull() + }) +}) + describe('buildSyncCapacityUpdate', () => { it('requires operator action without consuming the transient-failure breaker', async () => { const { buildSyncCapacityUpdate } = await import('@/lib/knowledge/connectors/sync-engine') diff --git a/apps/sim/lib/knowledge/connectors/sync-engine.ts b/apps/sim/lib/knowledge/connectors/sync-engine.ts index 7c7b8d059a1..c501a529593 100644 --- a/apps/sim/lib/knowledge/connectors/sync-engine.ts +++ b/apps/sim/lib/knowledge/connectors/sync-engine.ts @@ -71,6 +71,8 @@ import { hasIndexablePayload } from '@/connectors/utils' const logger = createLogger('ConnectorSyncEngine') +const RATE_LIMIT_RETRY_JITTER_MAX_MS = 60_000 + /** * Raised when a run discovers mid-flight that it no longer holds its sync lock. * @@ -1358,6 +1360,40 @@ export function buildSyncFailureUpdate( } } +/** + * The connector row written after a provider positively identifies throttling. + * + * Structured throttling is a transient quota or availability condition, so it + * must not consume the breaker reserved for persistent connector failures. The + * provider deadline remains authoritative, with a short post-deadline jitter + * to avoid releasing every connector sharing the same quota window at once. + * When the provider omits a usable deadline, the first rung of the ordinary + * failure ladder provides a conservative fallback. + */ +export function buildSyncRateLimitUpdate( + now: Date, + previousFailures: number | null | undefined, + errorMessage: string, + retryAfterMs?: number +) { + const maximumBackoffMs = CONNECTOR_FAILURE_BACKOFF_CAP_MINUTES * 60 * 1000 + const providerBackoffMs = + typeof retryAfterMs === 'number' && Number.isFinite(retryAfterMs) && retryAfterMs > 0 + ? retryAfterMs + : connectorFailureBackoffMinutes(1) * 60 * 1000 + const jitterMs = randomInt(0, RATE_LIMIT_RETRY_JITTER_MAX_MS + 1) + + return { + status: 'error' as const, + lastSyncError: errorMessage, + nextSyncAt: new Date(now.getTime() + Math.min(providerBackoffMs + jitterMs, maximumBackoffMs)), + consecutiveFailures: previousFailures ?? 0, + syncLockToken: null, + syncLockLeaseAt: null, + updatedAt: now, + } +} + /** * A deterministic capacity rejection needs operator action, not an automatic * retry or the transient-failure circuit breaker. Keep its precise diagnostic, @@ -3181,6 +3217,7 @@ export async function executeSync( const errorMessage = toError(error).message const retryAfterMs = getRetryAfterMs(error) + const rateLimited = isRateLimitError(error) logger.error('Sync failed', { connectorId, error: errorMessage, @@ -3193,12 +3230,19 @@ export async function executeSync( const failureUpdate = error instanceof ConnectorSyncCapacityError ? buildSyncCapacityUpdate(new Date(), connector.consecutiveFailures, errorMessage) - : buildSyncFailureUpdate( - new Date(), - connector.consecutiveFailures, - errorMessage, - retryAfterMs - ) + : rateLimited + ? buildSyncRateLimitUpdate( + new Date(), + connector.consecutiveFailures, + errorMessage, + retryAfterMs + ) + : buildSyncFailureUpdate( + new Date(), + connector.consecutiveFailures, + errorMessage, + retryAfterMs + ) if (failureUpdate.status === 'disabled') { logger.warn('Connector disabled after repeated failures', { diff --git a/apps/sim/lib/knowledge/documents/document-processing-error.test.ts b/apps/sim/lib/knowledge/documents/document-processing-error.test.ts index 011f9ee689e..b90d8a1ba46 100644 --- a/apps/sim/lib/knowledge/documents/document-processing-error.test.ts +++ b/apps/sim/lib/knowledge/documents/document-processing-error.test.ts @@ -187,7 +187,7 @@ describe('document processing failure taxonomy', () => { new Error('Storage request timed out'), new Error('Database connection terminated unexpectedly'), new Error('Embedding provider returned 503'), - new TypeError('parseOfficeAsync is not a function'), + new TypeError('parseOffice is not a function'), ]) { expect(classifyDocumentProcessingFailure(error, 'Report.docx')).toMatchObject({ disposition: 'transient', diff --git a/apps/sim/lib/knowledge/documents/document-processor.ts b/apps/sim/lib/knowledge/documents/document-processor.ts index 68ffa2346c4..ed486c519a6 100644 --- a/apps/sim/lib/knowledge/documents/document-processor.ts +++ b/apps/sim/lib/knowledge/documents/document-processor.ts @@ -29,6 +29,7 @@ import { } from '@/lib/execution/model-input-provenance' import { parseBuffer } from '@/lib/file-parsers' import { decodeDataUriWithinLimit } from '@/lib/file-parsers/data-uri' +import { openPdfDocument } from '@/lib/file-parsers/pdfjs-server' import type { FileParseMetadata, FileParseResult } from '@/lib/file-parsers/types' import { MistralOperationError } from '@/lib/internal/mistral/errors' import { mistralParseInputSchema } from '@/lib/internal/mistral/input' @@ -99,11 +100,10 @@ const LEGACY_FORMAT_REPLACEMENTS: Record = { } async function getPdfPageCount(buffer: Buffer): Promise { - let pdf: Awaited> | undefined + let pdf: Awaited> | undefined try { - const { getDocumentProxy } = await import('unpdf') const uint8Array = new Uint8Array(buffer) - pdf = await getDocumentProxy(uint8Array) + pdf = await openPdfDocument(uint8Array) return pdf.numPages } catch (error) { logger.warn('Primary PDF page-count parser failed', { diff --git a/apps/sim/lib/knowledge/documents/secure-fetch.server.ts b/apps/sim/lib/knowledge/documents/secure-fetch.server.ts index 818174760af..24cd4814dc8 100644 --- a/apps/sim/lib/knowledge/documents/secure-fetch.server.ts +++ b/apps/sim/lib/knowledge/documents/secure-fetch.server.ts @@ -11,7 +11,6 @@ import { } from '@/lib/knowledge/documents/utils' export interface SecureFetchRetryOptions extends RetryOptions { - allowHttp?: boolean timeout?: number maxResponseBytes?: number } @@ -28,16 +27,15 @@ export interface SecureFetchRetryOptions extends RetryOptions { */ export async function secureFetchWithRetry( url: string, - options: SecureFetchOptions = {}, + options: SecureFetchOptions, retryOptions: SecureFetchRetryOptions = {} ): Promise { - const { allowHttp, timeout, maxResponseBytes, ...retry } = retryOptions + const { timeout, maxResponseBytes, ...retry } = retryOptions return retryWithExponentialBackoff(async () => { const response = await secureFetchWithValidation( url, { ...options, - ...(allowHttp !== undefined ? { allowHttp } : {}), ...(timeout !== undefined ? { timeout } : {}), ...(maxResponseBytes !== undefined ? { maxResponseBytes } : {}), }, diff --git a/apps/sim/lib/knowledge/documents/utils.test.ts b/apps/sim/lib/knowledge/documents/utils.test.ts index 03cf245c8c1..da505416f91 100644 --- a/apps/sim/lib/knowledge/documents/utils.test.ts +++ b/apps/sim/lib/knowledge/documents/utils.test.ts @@ -786,13 +786,18 @@ describe('secureFetchWithRetry', () => { const response = await secureFetchWithRetry('https://example.com/api', { method: 'GET', headers: { Accept: 'application/json' }, + profile: 'configuredEndpoint', }) expect(response.status).toBe(200) expect(mockSecureFetchWithValidation).toHaveBeenCalledTimes(1) const [url, options, paramName] = mockSecureFetchWithValidation.mock.calls[0] expect(url).toBe('https://example.com/api') - expect(options).toMatchObject({ method: 'GET', headers: { Accept: 'application/json' } }) + expect(options).toMatchObject({ + method: 'GET', + headers: { Accept: 'application/json' }, + profile: 'configuredEndpoint', + }) expect(paramName).toBe('url') }) @@ -802,7 +807,11 @@ describe('secureFetchWithRetry', () => { ) await expect( - secureFetchWithRetry('https://attacker.test', { method: 'GET' }, FAST_RETRY) + secureFetchWithRetry( + 'https://attacker.test', + { method: 'GET', profile: 'configuredEndpoint' }, + FAST_RETRY + ) ).rejects.toThrow('blocked IP address') expect(mockSecureFetchWithValidation).toHaveBeenCalledTimes(1) @@ -815,7 +824,7 @@ describe('secureFetchWithRetry', () => { const response = await secureFetchWithRetry( 'https://example.com/api', - { method: 'GET' }, + { method: 'GET', profile: 'configuredEndpoint' }, FAST_RETRY ) @@ -828,7 +837,7 @@ describe('secureFetchWithRetry', () => { const response = await secureFetchWithRetry( 'https://example.com/api', - { method: 'GET' }, + { method: 'GET', profile: 'configuredEndpoint' }, FAST_RETRY ) @@ -836,17 +845,21 @@ describe('secureFetchWithRetry', () => { expect(mockSecureFetchWithValidation).toHaveBeenCalledTimes(1) }) - it('forwards allowHttp / timeout / maxResponseBytes to the pinned fetch', async () => { + it('forwards the egress profile, timeout and maxResponseBytes to the pinned fetch', async () => { mockSecureFetchWithValidation.mockResolvedValue(fakeResponse(200)) await secureFetchWithRetry( 'http://localhost:9000', - { method: 'GET' }, - { allowHttp: true, timeout: 5000, maxResponseBytes: 1024, ...FAST_RETRY } + { method: 'GET', profile: 'configuredEndpoint' }, + { timeout: 5000, maxResponseBytes: 1024, ...FAST_RETRY } ) const [, options] = mockSecureFetchWithValidation.mock.calls[0] - expect(options).toMatchObject({ allowHttp: true, timeout: 5000, maxResponseBytes: 1024 }) + expect(options).toMatchObject({ + profile: 'configuredEndpoint', + timeout: 5000, + maxResponseBytes: 1024, + }) }) /** @@ -868,7 +881,7 @@ describe('secureFetchWithRetry', () => { const response = await secureFetchWithRetry( 'https://api.github.com/repos', - { method: 'GET' }, + { method: 'GET', profile: 'configuredEndpoint' }, FAST_RETRY ) @@ -883,7 +896,7 @@ describe('secureFetchWithRetry', () => { const response = await secureFetchWithRetry( 'https://api.github.com/repos', - { method: 'GET' }, + { method: 'GET', profile: 'configuredEndpoint' }, FAST_RETRY ) @@ -898,7 +911,7 @@ describe('secureFetchWithRetry', () => { const response = await secureFetchWithRetry( 'https://example.com/api', - { method: 'GET' }, + { method: 'GET', profile: 'configuredEndpoint' }, FAST_RETRY ) @@ -914,7 +927,7 @@ describe('secureFetchWithRetry', () => { const error = await secureFetchWithRetry( 'https://gitlab.example.com/api/v4/projects', - { method: 'GET' }, + { method: 'GET', profile: 'configuredEndpoint' }, { ...FAST_RETRY, maxRetries: 0 } ).then( () => undefined, diff --git a/apps/sim/lib/logs/application/get-public-log.ts b/apps/sim/lib/logs/application/get-public-log.ts index d99e2703d19..7b7f9406bff 100644 --- a/apps/sim/lib/logs/application/get-public-log.ts +++ b/apps/sim/lib/logs/application/get-public-log.ts @@ -1,3 +1,4 @@ +import { resolvePrincipalSubjectUserId } from '@sim/auth/principal' import type { CostLedger } from '@/lib/api/contracts/logs' import { defineAuthorizedWorkspaceUseCase } from '@/lib/core/application' import { OrchestrationError } from '@/lib/core/orchestration/types' @@ -8,6 +9,11 @@ import { logDelegationAuthorization } from '@/lib/logs/application/authorization import { logOperations } from '@/lib/logs/application/operations' import { buildCostLedger } from '@/lib/logs/cost-ledger' import { materializeExecutionDataForDisplay } from '@/lib/logs/execution/trace-store' +import { + logProjectionSubjectUserId, + projectExecutionData, + resolveLogFieldProjection, +} from '@/lib/logs/log-projection' import { getPublicWorkflowLog, getPublicWorkflowLogScope } from '@/lib/logs/public-queries' import { sanitizeExecutionSnapshotState } from '@/lib/logs/snapshot-sanitizer' import { @@ -89,6 +95,27 @@ export const getPublicLog = defineAuthorizedWorkspaceUseCase({ }, authorizationOptions: logDelegationAuthorization(), execute: async ({ principal, context }): Promise => { + /** + * Attribution and the projection subject in one value; a workspace API key + * represents no user and therefore reads the run whole. See + * `list-public-logs.ts` for why the key's creator is never substituted. + */ + const viewerUserId = resolvePrincipalSubjectUserId(principal) + + /** + * permission-group-enforced: logs.trace_spans + * permission-group-enforced: logs.cost + * + * The same projection the list and the internal detail path apply. Without + * it this route published the whole trace and the itemized ledger to a + * member whose group withholds both everywhere else. + */ + const projection = await resolveLogFieldProjection( + logProjectionSubjectUserId(principal), + context.workspaceId, + context.workspaceOrganizationId + ) + const log = await getPublicWorkflowLog( { column: 'executionId', value: context.executionId }, context.workspaceId @@ -110,22 +137,36 @@ export const getPublicLog = defineAuthorizedWorkspaceUseCase({ workspaceId: context.workspaceId, workflowId: log.workflowId, executionId: log.executionId, - userId: principal.kind === 'personal_api_key' ? principal.userId : undefined, + userId: viewerUserId, } ) - if (log.workflowUserId && !log.workflowOwnerEmail) { - throw new Error(`Unable to resolve workflow owner email for ${log.workflowUserId}`) - } - const costLedger = await buildCostLedger(log.executionId) + /** + * No assertion on `executedByEmail`. The owner-email version of this field + * could reasonably insist a non-null user id resolve to an email, because + * the workflow row's owner was expected to exist. The executing identity is + * read from attribution the run captured for itself, and a run that failed + * before resolving one legitimately has none — so null is an answer here, + * not a missing join. + */ + /** + * The ledger is the itemization of the very total `costTotal` reports, so a + * group withholding spend has to lose both — blanking the total alone would + * leave the caller able to sum the lines. + */ + const costLedger = projection.hideCostInfo ? null : await buildCostLedger(log.executionId) return { - log: { ...log, workflowState: sanitizeExecutionSnapshotState(log.workflowState) }, + log: { + ...log, + costTotal: projection.hideCostInfo ? null : log.costTotal, + workflowState: sanitizeExecutionSnapshotState(log.workflowState), + }, costLedger, workflowFolderPath: publicLogFolderPath( folderIndex.pathById, log.workflowFolderId, log.workflowName !== null ), - executionData, + executionData: projectExecutionData(executionData, projection) as Record, } }, }) diff --git a/apps/sim/lib/logs/application/list-logs.test.ts b/apps/sim/lib/logs/application/list-logs.test.ts new file mode 100644 index 00000000000..2e534ab544b --- /dev/null +++ b/apps/sim/lib/logs/application/list-logs.test.ts @@ -0,0 +1,159 @@ +/** + * @vitest-environment node + */ + +import type { Principal } from '@sim/auth/principal' +import { permissionGroupScopeMock, permissionGroupScopeMockFns } from '@sim/testing' +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const mocks = vi.hoisted(() => ({ + readLogs: vi.fn(), + resolveWorkspace: vi.fn(), + resolvePermission: vi.fn(), +})) + +const resolveGroupConfigMock = permissionGroupScopeMockFns.mockResolvePermissionGroupConfig + +vi.mock('@/lib/logs/list-logs', () => ({ + readLogs: mocks.readLogs, +})) + +vi.mock('@/lib/workspaces/application/workspace-context', () => ({ + resolveActiveWorkspaceApplicationContext: mocks.resolveWorkspace, +})) + +vi.mock('@sim/platform-authz/workspace', () => ({ + permissionSatisfies: (held: string | null, required: string) => + held === 'admin' || held === required || (held === 'write' && required === 'read'), + resolveEffectiveWorkspacePermission: mocks.resolvePermission, +})) + +vi.mock('@/lib/permission-groups/config-scope.server', () => permissionGroupScopeMock) + +import { listLogsUseCase } from '@/lib/logs/application/list-logs' +import { PermissionGroupCapabilityError } from '@/lib/permission-groups/capability-error' + +const WORKSPACE_ID = 'workspace-1' +const SESSION: Principal = { kind: 'session', userId: 'user-1', sessionId: 'session-1' } +const INPUT = { workspaceId: WORKSPACE_ID, limit: 100, sortBy: 'date', sortOrder: 'desc' } as never + +describe('listLogsUseCase', () => { + beforeEach(() => { + vi.clearAllMocks() + mocks.resolveWorkspace.mockResolvedValue({ + workspaceId: WORKSPACE_ID, + workspaceOrganizationId: null, + allowPersonalApiKeys: true, + }) + mocks.resolvePermission.mockResolvedValue('admin') + mocks.readLogs.mockResolvedValue({ data: [], nextCursor: null }) + resolveGroupConfigMock.mockResolvedValue(null) + }) + + /** + * A cost withheld on the detail but still printed on the list withholds + * nothing, so the same key has to reach both queries. + */ + it('tells the list query to withhold spend when the group does', async () => { + resolveGroupConfigMock.mockResolvedValue({ hideCostInfo: true }) + + await listLogsUseCase.execute({ principal: SESSION, input: INPUT }) + + expect(mocks.readLogs).toHaveBeenCalledWith(expect.objectContaining({ hideCostInfo: true })) + }) + + it('leaves spend in place when no group withholds it', async () => { + await listLogsUseCase.execute({ principal: SESSION, input: INPUT }) + + expect(mocks.readLogs).toHaveBeenCalledWith(expect.objectContaining({ hideCostInfo: false })) + }) + + /** + * Blanking the field is not enough on its own: `cost > X` answered faithfully + * is a bisection oracle over the very number that was withheld, and the sort + * leaks the same ranking more slowly. + */ + it.each([ + ['a cost sort', { sortBy: 'cost' }], + ['a cost filter', { costOperator: '>', costValue: 0.5 }], + ['an equality cost filter', { costOperator: '=', costValue: 0 }], + ])('refuses %s when the group withholds spend', async (_label, overrides) => { + resolveGroupConfigMock.mockResolvedValue({ hideCostInfo: true }) + + await expect( + listLogsUseCase.execute({ + principal: SESSION, + input: { ...(INPUT as object), ...overrides } as never, + }) + ).rejects.toBeInstanceOf(PermissionGroupCapabilityError) + expect(mocks.readLogs).not.toHaveBeenCalled() + }) + + it('answers the same cost query when no group withholds spend', async () => { + await listLogsUseCase.execute({ + principal: SESSION, + input: { ...(INPUT as object), sortBy: 'cost', costOperator: '>', costValue: 0.5 } as never, + }) + + expect(mocks.readLogs).toHaveBeenCalledWith(expect.objectContaining({ sortBy: 'cost' })) + }) + + /** A duration filter names nothing the group withholds, so it still answers. */ + it('leaves a duration filter alone under a spend-withholding group', async () => { + resolveGroupConfigMock.mockResolvedValue({ hideCostInfo: true }) + + await listLogsUseCase.execute({ + principal: SESSION, + input: { ...(INPUT as object), durationOperator: '>', durationValue: 100 } as never, + }) + + expect(mocks.readLogs).toHaveBeenCalledWith(expect.objectContaining({ hideCostInfo: true })) + }) + + /** + * An actorless run has no user, so there is no group to resolve — it reads its + * own workspace's logs whole rather than being handed a stand-in viewer. + */ + it('does not resolve a group for a principal with no subject', async () => { + await listLogsUseCase.execute({ + principal: { + kind: 'workspace_api_key', + workspaceId: WORKSPACE_ID, + keyId: 'key-1', + } as Principal, + input: INPUT, + }) + + expect(resolveGroupConfigMock).not.toHaveBeenCalled() + expect(mocks.readLogs).toHaveBeenCalledWith(expect.objectContaining({ hideCostInfo: false })) + }) + + /** + * An executor delegation names the person who triggered the run, but carries + * their role and none of their capabilities — the exemption the authorization + * funnel already applied on the way in. Projecting on them here would be a + * second, contrary decision about the same principal, and a cost-sorted read + * would not merely lose a column: `assertLogCostQueryAllowed` would refuse the + * run's own listing outright. + */ + it('reads whole for a run delegated by a member whose group withholds spend', async () => { + resolveGroupConfigMock.mockResolvedValue({ hideCostInfo: true }) + + await listLogsUseCase.execute({ + principal: { + kind: 'delegated', + serviceId: 'executor', + subjectUserId: 'user-1', + workspaceId: WORKSPACE_ID, + delegationId: 'delegation-1', + audience: 'sim:logs', + issuedAt: new Date('2026-01-01T00:00:00Z'), + expiresAt: new Date('2999-01-01T00:00:00Z'), + } as Principal, + input: { ...(INPUT as object), sortBy: 'cost' } as never, + }) + + expect(resolveGroupConfigMock).not.toHaveBeenCalled() + expect(mocks.readLogs).toHaveBeenCalledWith(expect.objectContaining({ hideCostInfo: false })) + }) +}) diff --git a/apps/sim/lib/logs/application/list-logs.ts b/apps/sim/lib/logs/application/list-logs.ts index 9fed67ced18..f4294009bd5 100644 --- a/apps/sim/lib/logs/application/list-logs.ts +++ b/apps/sim/lib/logs/application/list-logs.ts @@ -7,6 +7,11 @@ import { } from '@/lib/logs/application/authorization' import { logOperations } from '@/lib/logs/application/operations' import { type ListLogsParams, readLogs } from '@/lib/logs/list-logs' +import { + assertLogCostQueryAllowed, + logProjectionSubjectUserId, + resolveLogFieldProjection, +} from '@/lib/logs/log-projection' import { resolveActiveWorkspaceApplicationContext } from '@/lib/workspaces/application/workspace-context' const authorizedListLogsUseCase = defineAuthorizedWorkspaceUseCase({ @@ -14,8 +19,31 @@ const authorizedListLogsUseCase = defineAuthorizedWorkspaceUseCase({ resolveContext: ({ input }: { input: ListLogsParams }) => resolveActiveWorkspaceApplicationContext(input.workspaceId), authorizationOptions: logDelegationAuthorization(), - async execute({ input, context }) { - return readLogs({ ...input, workspaceId: context.workspaceId }) + async execute({ principal, input, context }) { + /** + * permission-group-enforced: logs.cost — the list carries the same run + * total the detail does, so withholding it only on the detail would hide + * nothing. A projection rather than a refusal, for the reason given in + * `read-log-detail.ts`, resolved through the shared helper every other log + * surface reads so the subject and the rule cannot drift apart here. + */ + const { hideCostInfo } = await resolveLogFieldProjection( + logProjectionSubjectUserId(principal), + context.workspaceId, + context.workspaceOrganizationId + ) + /** + * The list's own `sortBy=cost` and `costOperator`/`costValue` select on the + * very figure the row above blanks, so they have to be refused rather than + * answered — see {@link assertLogCostQueryAllowed}. + */ + assertLogCostQueryAllowed(input, { hideCostInfo }) + + return readLogs({ + ...input, + workspaceId: context.workspaceId, + hideCostInfo, + }) }, }) diff --git a/apps/sim/lib/logs/application/list-public-logs.ts b/apps/sim/lib/logs/application/list-public-logs.ts index 729f015c6c9..a5020c61538 100644 --- a/apps/sim/lib/logs/application/list-public-logs.ts +++ b/apps/sim/lib/logs/application/list-public-logs.ts @@ -1,3 +1,4 @@ +import { resolvePrincipalSubjectUserId } from '@sim/auth/principal' import type { CursorKey, ListSortOrder } from '@/lib/api/list-query' import { defineAuthorizedWorkspaceUseCase } from '@/lib/core/application' import { OrchestrationError } from '@/lib/core/orchestration/types' @@ -6,6 +7,13 @@ import { logDelegationAuthorization } from '@/lib/logs/application/authorization import { logOperations } from '@/lib/logs/application/operations' import { materializeExecutionDataForDisplay } from '@/lib/logs/execution/trace-store' import { resolveLogFolderScope } from '@/lib/logs/folder-scope' +import { + assertLogCostQueryAllowed, + type LogFieldProjection, + logProjectionSubjectUserId, + projectExecutionData, + resolveLogFieldProjection, +} from '@/lib/logs/log-projection' import type { LogFilters } from '@/lib/logs/public-filters' import { type PublicLogListRow, @@ -41,6 +49,20 @@ export interface ListPublicLogsResult { includeTraceSpans: boolean } +/** + * The row with its spend blanked when the viewer's group withholds it. + * + * Blanked on the row rather than in the presenter so a surface that reads + * `costTotal` or a job run's `cost` directly cannot report a figure the group + * withholds by forgetting to ask. The two branches spell the same column + * differently because the two tables do: a workflow run stores a `numeric` + * total, a job run a jsonb document. + */ +function projectRowSpend(log: PublicLogListRow, projection: LogFieldProjection): PublicLogListRow { + if (!projection.hideCostInfo) return log + return log.kind === 'job' ? { ...log, cost: null } : { ...log, costTotal: null } +} + export const listPublicLogs = defineAuthorizedWorkspaceUseCase({ operation: logOperations.list, resolveContext: async ({ input }: { input: ListPublicLogsInput }) => { @@ -50,11 +72,58 @@ export const listPublicLogs = defineAuthorizedWorkspaceUseCase({ }, authorizationOptions: logDelegationAuthorization(), execute: async ({ principal, input, context }): Promise => { + /** + * Attribution and the projection subject in one value: a workspace API key + * authorizes as the workspace and represents no user, so it resolves to + * `undefined` and reads the page whole. Substituting the key's creator would + * apply a bystander's group to every caller of a shared credential. + */ + const viewerUserId = resolvePrincipalSubjectUserId(principal) + + /** + * permission-group-enforced: logs.trace_spans + * permission-group-enforced: logs.cost + * + * A projection rather than a refusal, for the reason + * {@link resolveLogFieldProjection} gives — and applied here, in the use + * case, rather than in the v2 presenter, so the withholding cannot be lost + * by a second surface reading the same list. + */ + const projection = await resolveLogFieldProjection( + logProjectionSubjectUserId(principal), + context.workspaceId, + context.workspaceOrganizationId + ) + + /** + * Refused after the workspace role check above and before the read below: + * `minCost`/`maxCost` bisect the very total the rows blank, and + * `sortBy=cost` leaks the same figure as a ranking — see + * {@link assertLogCostQueryAllowed}. + */ + assertLogCostQueryAllowed( + { + sortBy: input.sortBy, + minCost: input.filters.minCost, + maxCost: input.filters.maxCost, + }, + projection + ) + const folderScope = input.folderPaths ? await resolveLogFolderScope(context.workspaceId, input.folderPaths) : undefined - const needsMaterialization = input.includeFinalOutput || input.includeTraceSpans + /** + * A group withholding execution detail turns both render flags off below, + * so every materialized payload would be projected and then dropped + * unread. Skipped here instead: materialization is an object-store read per + * row plus a secret projection over the whole trace, and paying for a page + * of them to discard the result is the most expensive way to withhold + * something. + */ + const needsMaterialization = + (input.includeFinalOutput || input.includeTraceSpans) && !projection.hideTraceSpans const { data, nextCursorKeys } = await readPublicLogPage({ filters: { ...input.filters, workspaceId: context.workspaceId }, limit: input.limit, @@ -66,7 +135,6 @@ export const listPublicLogs = defineAuthorizedWorkspaceUseCase({ cursorKeys: input.cursorKeys, }) - const userId = principal.kind === 'personal_api_key' ? principal.userId : undefined /** * Job runs carry no materializable execution data on this surface: their * `execution_data` is a job envelope rather than a workflow trace, and @@ -76,28 +144,40 @@ export const listPublicLogs = defineAuthorizedWorkspaceUseCase({ */ const items = needsMaterialization ? await mapWithConcurrency(data, MATERIALIZE_CONCURRENCY, async (log) => { - if (log.kind !== 'workflow' || !log.executionData) return { log } + const projectedLog = projectRowSpend(log, projection) + if (log.kind !== 'workflow' || !log.executionData) return { log: projectedLog } + const materialized = await materializeExecutionDataForDisplay( + log.executionData as Record, + { + workspaceId: log.workspaceId, + workflowId: log.workflowId, + executionId: log.executionId, + userId: viewerUserId, + } + ) return { - log, - executionData: await materializeExecutionDataForDisplay( - log.executionData as Record, - { - workspaceId: log.workspaceId, - workflowId: log.workflowId, - executionId: log.executionId, - userId, - } - ), + log: projectedLog, + executionData: projectExecutionData(materialized, projection) as Record< + string, + unknown + >, } }) - : data.map((log) => ({ log })) + : data.map((log) => ({ log: projectRowSpend(log, projection) })) + /** + * The render flags are narrowed rather than left for the presenter to + * re-check. `projectExecutionData` deletes the withheld payloads, but the + * presenter reads `executionData.traceSpans ?? []`, so a deleted array would + * come back as an empty one — present, and indistinguishable from a run + * whose spans aged out. Turning the flag off omits the field instead. + */ return { items, nextCursorKeys, includeFullDetails: input.includeFullDetails, - includeFinalOutput: input.includeFinalOutput, - includeTraceSpans: input.includeTraceSpans, + includeFinalOutput: input.includeFinalOutput && !projection.hideTraceSpans, + includeTraceSpans: input.includeTraceSpans && !projection.hideTraceSpans, } }, }) diff --git a/apps/sim/lib/logs/application/operations.ts b/apps/sim/lib/logs/application/operations.ts index 2d84652e988..e0b760eb506 100644 --- a/apps/sim/lib/logs/application/operations.ts +++ b/apps/sim/lib/logs/application/operations.ts @@ -7,28 +7,36 @@ const LOG_READER_PRINCIPAL_POLICY = { } as const export const logOperations = { + // permission-group-exempt: reading the log list is governed by workspace role; the group withholds fields inside a run — trace spans and cost — not the fact that it ran list: defineWorkspaceOperation({ id: 'logs.list', minimumRole: 'read', workspaceApiKey: 'allow', + capability: 'none', ...LOG_READER_PRINCIPAL_POLICY, }), + // permission-group-exempt: aggregate run counts carry no execution payload, so there is nothing here for a group to withhold readStats: defineWorkspaceOperation({ id: 'logs.read_stats', minimumRole: 'read', workspaceApiKey: 'allow', + capability: 'none', principalKinds: PUBLIC_API_PRINCIPAL_KINDS, }), + // permission-group-exempt: as with the list, the group projects trace spans and cost out of the response rather than refusing the read readDetail: defineWorkspaceOperation({ id: 'logs.read_detail', minimumRole: 'read', workspaceApiKey: 'allow', + capability: 'none', ...LOG_READER_PRINCIPAL_POLICY, }), + // permission-group-exempt: the executor reading its own run's snapshot mid-flight; refusing it would fail runs the group permits readExecutionSnapshot: defineWorkspaceOperation({ id: 'logs.read_execution_snapshot', minimumRole: 'read', workspaceApiKey: 'deny', + capability: 'none', principalKinds: ['session', 'delegated'], delegatedServices: ['executor'], }), diff --git a/apps/sim/lib/logs/application/public-log-projection.test.ts b/apps/sim/lib/logs/application/public-log-projection.test.ts new file mode 100644 index 00000000000..6532246d979 --- /dev/null +++ b/apps/sim/lib/logs/application/public-log-projection.test.ts @@ -0,0 +1,423 @@ +/** + * @vitest-environment node + * + * `logs.trace_spans` and `logs.cost` are PROJECTIONS, not gates — a group + * withholds those fields from the response rather than refusing the read, which + * is why `logOperations.list` and `logOperations.readDetail` correctly declare + * `capability: 'none'`. + * + * `/api/v2/logs` and `/api/v2/logs/{runId}` applied none of it: an enterprise + * member whose group hides spend or execution detail read both in full through + * a personal API key, while the same person was withheld them on the internal + * and v1 surfaces. These run the real use cases against the real + * `resolveLogFieldProjection` — the same helper `readLogDetail` and the v1 + * routes resolve their flags through — so they fail if this surface stops + * projecting. + */ +import { + permissionGroupScopeMock, + permissionGroupScopeMockFns, + resetPermissionGroupScopeMock, +} from '@sim/testing' +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const mocks = vi.hoisted(() => ({ + loadWorkspace: vi.fn(), + resolvePermission: vi.fn(), + getLogScope: vi.fn(), + getLog: vi.fn(), + listLogs: vi.fn(), + loadFolders: vi.fn(), + materialize: vi.fn(), + buildCostLedger: vi.fn(), + recordAudit: vi.fn(), +})) + +vi.mock('@/lib/permission-groups/config-scope.server', () => permissionGroupScopeMock) + +vi.mock('@/lib/workspaces/application/workspace-context', () => ({ + loadActiveWorkspaceApplicationContext: mocks.loadWorkspace, +})) + +vi.mock('@sim/platform-authz/workspace', () => ({ + permissionSatisfies: (permission: string | null, required: string) => + permission === 'admin' || permission === 'write' || permission === required, + resolveEffectiveWorkspacePermission: mocks.resolvePermission, +})) + +vi.mock('@/lib/logs/public-queries', () => ({ + getPublicWorkflowLogScope: mocks.getLogScope, + getPublicWorkflowLog: mocks.getLog, + readPublicLogPage: mocks.listLogs, +})) + +vi.mock('@/lib/folders/queries', () => ({ + loadActiveFolderPathIndex: mocks.loadFolders, +})) + +vi.mock('@/lib/logs/execution/trace-store', () => ({ + materializeExecutionDataForDisplay: mocks.materialize, +})) + +vi.mock('@/lib/logs/cost-ledger', () => ({ + buildCostLedger: mocks.buildCostLedger, +})) + +vi.mock('@/lib/logs/snapshot-sanitizer', () => ({ + sanitizeExecutionSnapshotState: (state: unknown) => state, +})) + +vi.mock('@sim/audit', () => ({ recordAudit: mocks.recordAudit })) + +import { getPublicLog } from '@/lib/logs/application/get-public-log' +import { listPublicLogs } from '@/lib/logs/application/list-public-logs' +import { DEFAULT_PERMISSION_GROUP_CONFIG } from '@/lib/permission-groups/fields' + +const WORKSPACE_ID = 'workspace-1' + +const workspaceContext = { + workspaceId: WORKSPACE_ID, + workspaceOrganizationId: 'organization-1', + allowPersonalApiKeys: true, + billedAccountUserId: 'billing-owner-1', +} + +const EXECUTION_DATA = { + /** + * The run-level roll-up every completed run carries. `models` is the + * per-model dollar breakdown, so a projection that blanks the total and + * leaves this published the finer figure it was hiding. + */ + tokens: { input: 60, output: 30, total: 90 }, + models: { 'gpt-4': { input: 0.4, output: 0.35, total: 0.75 } }, + finalOutput: { answer: 'a customer address' }, + workflowInput: { question: 'who?' }, + blockInput: { prompt: 'who?' }, + blockExecutions: [{ blockId: 'b1', cost: { total: 0.2 }, tokens: { total: 90 } }], + traceSpans: [ + { + id: 's1', + name: 'agent', + cost: { total: 0.5 }, + tokens: { total: 120 }, + children: [{ id: 's2', name: 'tool', cost: { total: 0.1 } }], + }, + ], +} + +const COST_LEDGER = { total: 0.75, items: [{ model: 'gpt-4', cost: 0.75 }] } + +const workflowLog = { + kind: 'workflow' as const, + id: 'log-1', + executionId: 'run-1', + workspaceId: WORKSPACE_ID, + workflowId: 'workflow-1', + workflowName: 'Support triage', + workflowFolderId: 'folder-1', + workflowUserId: 'owner-1', + workflowOwnerEmail: 'owner@example.com', + workflowState: { blocks: {} }, + costTotal: '0.75', + executionData: { pointer: true }, +} + +const jobLog = { + kind: 'job' as const, + executionId: 'job-1', + cost: { total: 0.4 }, + executionData: { pointer: true }, +} + +/** A person governed by a group; the group's own keys decide what is withheld. */ +const personalPrincipal = { + kind: 'personal_api_key' as const, + userId: 'user-9', + keyId: 'key-9', +} + +/** A workspace key has no user and therefore no group. */ +const workspacePrincipal = { + kind: 'workspace_api_key' as const, + workspaceId: WORKSPACE_ID, + keyId: 'key-1', +} + +function governedBy(overrides: Partial) { + permissionGroupScopeMockFns.mockResolvePermissionGroupConfig.mockResolvedValue({ + ...DEFAULT_PERMISSION_GROUP_CONFIG, + ...overrides, + }) +} + +function listInput(overrides: Record = {}) { + return { + workspaceId: WORKSPACE_ID, + filters: {}, + sortBy: 'startedAt' as const, + sortOrder: 'desc' as const, + cursorKeys: undefined, + limit: 50, + includeFullDetails: true, + includeFinalOutput: true, + includeTraceSpans: true, + includeJobRuns: false, + ...overrides, + } +} + +beforeEach(() => { + vi.clearAllMocks() + resetPermissionGroupScopeMock() + mocks.loadWorkspace.mockResolvedValue(workspaceContext) + mocks.resolvePermission.mockResolvedValue('read') + mocks.getLogScope.mockResolvedValue({ + executionId: 'run-1', + workspaceId: WORKSPACE_ID, + workflowId: 'workflow-1', + }) + mocks.getLog.mockResolvedValue(workflowLog) + mocks.listLogs.mockResolvedValue({ data: [workflowLog], nextCursorKeys: null }) + mocks.loadFolders.mockResolvedValue({ + idByPath: new Map([['/agents', 'folder-1']]), + pathById: new Map([['folder-1', '/agents']]), + }) + mocks.materialize.mockImplementation(async () => structuredClone(EXECUTION_DATA)) + mocks.buildCostLedger.mockResolvedValue(structuredClone(COST_LEDGER)) +}) + +describe('listPublicLogs field projection', () => { + it('blanks the run cost on the row when the group hides cost', async () => { + governedBy({ hideCostInfo: true }) + + const result = await listPublicLogs.execute({ + principal: personalPrincipal, + input: listInput(), + }) + + expect((result.items[0].log as { costTotal: string | null }).costTotal).toBeNull() + }) + + it('blanks a job run cost too, which the presenter reads from another column', async () => { + governedBy({ hideCostInfo: true }) + mocks.listLogs.mockResolvedValueOnce({ data: [jobLog], nextCursorKeys: null }) + + const result = await listPublicLogs.execute({ + principal: personalPrincipal, + input: listInput({ includeJobRuns: true }), + }) + + expect((result.items[0].log as { cost: unknown }).cost).toBeNull() + }) + + it('strips spend from the spans it still returns when only cost is hidden', async () => { + governedBy({ hideCostInfo: true }) + + const result = await listPublicLogs.execute({ + principal: personalPrincipal, + input: listInput(), + }) + const [span] = result.items[0].executionData?.traceSpans as Array> + + expect(result.items[0].executionData).not.toHaveProperty('models') + expect(span.name).toBe('agent') + expect(span).not.toHaveProperty('cost') + expect(span).not.toHaveProperty('tokens') + expect((span.children as Array>)[0]).not.toHaveProperty('cost') + }) + + /** + * The flags are what the presenter renders from. Deleting the payloads alone + * is not enough: it reads `executionData.traceSpans ?? []`, so a deleted + * array would come back as an empty one — present, and indistinguishable from + * a run whose spans aged out. + */ + it('withholds the execution payloads and turns off their render flags', async () => { + governedBy({ hideTraceSpans: true }) + + const result = await listPublicLogs.execute({ + principal: personalPrincipal, + input: listInput(), + }) + + expect(result.includeTraceSpans).toBe(false) + expect(result.includeFinalOutput).toBe(false) + expect(result.items[0].executionData).toBeUndefined() + }) + + /** + * The page is withheld whole, so the object-store read and the secret + * projection behind every payload buy nothing. Asserted on the read itself, + * not only on `materializeExecutionDataForDisplay`, because the column is + * what the work hangs off. + */ + it('materializes nothing when the group withholds execution detail', async () => { + governedBy({ hideTraceSpans: true }) + + await listPublicLogs.execute({ principal: personalPrincipal, input: listInput() }) + + expect(mocks.materialize).not.toHaveBeenCalled() + expect(mocks.listLogs).toHaveBeenCalledWith( + expect.objectContaining({ includeExecutionData: false }) + ) + }) + + it('still materializes for a group that withholds only spend', async () => { + governedBy({ hideCostInfo: true }) + + await listPublicLogs.execute({ principal: personalPrincipal, input: listInput() }) + + expect(mocks.materialize).toHaveBeenCalledTimes(1) + }) + + it('withholds nothing from a caller no group governs', async () => { + const result = await listPublicLogs.execute({ + principal: personalPrincipal, + input: listInput(), + }) + + expect((result.items[0].log as { costTotal: string | null }).costTotal).toBe('0.75') + expect(result.includeTraceSpans).toBe(true) + expect(result.items[0].executionData?.traceSpans).toHaveLength(1) + expect(result.items[0].executionData?.finalOutput).toEqual(EXECUTION_DATA.finalOutput) + }) + + /** + * A workspace API key authorizes as the workspace and represents no user, so + * there is no group to apply. Substituting the key's creator would govern + * every caller of a shared credential by a bystander's group. + */ + it('withholds nothing from a workspace API key and never resolves a group', async () => { + governedBy({ hideTraceSpans: true, hideCostInfo: true }) + + const result = await listPublicLogs.execute({ + principal: workspacePrincipal, + input: listInput(), + }) + + expect(permissionGroupScopeMockFns.mockResolvePermissionGroupConfig).not.toHaveBeenCalled() + expect((result.items[0].log as { costTotal: string | null }).costTotal).toBe('0.75') + expect(result.includeTraceSpans).toBe(true) + expect(result.items[0].executionData?.traceSpans).toHaveLength(1) + }) +}) + +/** + * Withholding the figure is not enough on its own: `minCost`/`maxCost` bisect + * it, and `sortBy=cost` reads it as a ranking. Refused rather than dropped — + * dropping the clause answers a question nobody asked. + */ +describe('listPublicLogs cost-selective queries', () => { + it.each([ + ['a cost sort', { sortBy: 'cost' as const }], + ['a minCost filter', { filters: { minCost: 0.5 } }], + ['a maxCost filter', { filters: { maxCost: 0.5 } }], + ])('refuses %s for a group that withholds spend', async (_label, overrides) => { + governedBy({ hideCostInfo: true }) + + await expect( + listPublicLogs.execute({ principal: personalPrincipal, input: listInput(overrides) }) + ).rejects.toMatchObject({ + code: 'forbidden', + detailCode: 'PERMISSION_GROUP_CAPABILITY_BLOCKED', + message: "Execution cost is not available under your organization's permission group", + }) + + expect(mocks.listLogs).not.toHaveBeenCalled() + }) + + it('answers a cost sort for a group that withholds nothing', async () => { + await listPublicLogs.execute({ + principal: personalPrincipal, + input: listInput({ sortBy: 'cost' as const }), + }) + + expect(mocks.listLogs).toHaveBeenCalledWith(expect.objectContaining({ sortBy: 'cost' })) + }) + + it('answers a cost filter for a workspace API key', async () => { + governedBy({ hideCostInfo: true }) + + await listPublicLogs.execute({ + principal: workspacePrincipal, + input: listInput({ filters: { minCost: 0.5 } }), + }) + + expect(mocks.listLogs).toHaveBeenCalledWith( + expect.objectContaining({ filters: expect.objectContaining({ minCost: 0.5 }) }) + ) + }) + + it('leaves a non-spend filter alone for a group that withholds spend', async () => { + governedBy({ hideCostInfo: true }) + + await listPublicLogs.execute({ + principal: personalPrincipal, + input: listInput({ filters: { minDurationMs: 100 } }), + }) + + expect(mocks.listLogs).toHaveBeenCalled() + }) +}) + +describe('getPublicLog field projection', () => { + it('withholds the run total and the itemized ledger when the group hides cost', async () => { + governedBy({ hideCostInfo: true }) + + const result = await getPublicLog.execute({ + principal: personalPrincipal, + input: { runId: 'run-1' }, + }) + + expect(result.log.costTotal).toBeNull() + expect(result.costLedger).toBeNull() + expect( + (result.executionData.traceSpans as Array>)[0] + ).not.toHaveProperty('cost') + expect( + (result.executionData.blockExecutions as Array>)[0] + ).not.toHaveProperty('tokens') + expect(result.executionData).not.toHaveProperty('tokens') + expect(result.executionData).not.toHaveProperty('models') + }) + + it('withholds the execution payloads when the group hides trace spans', async () => { + governedBy({ hideTraceSpans: true }) + + const result = await getPublicLog.execute({ + principal: personalPrincipal, + input: { runId: 'run-1' }, + }) + + expect(result.executionData).not.toHaveProperty('traceSpans') + expect(result.executionData).not.toHaveProperty('finalOutput') + expect(result.executionData).not.toHaveProperty('workflowInput') + expect(result.executionData).not.toHaveProperty('blockExecutions') + }) + + it('withholds nothing from a caller no group governs', async () => { + const result = await getPublicLog.execute({ + principal: personalPrincipal, + input: { runId: 'run-1' }, + }) + + expect(result.log.costTotal).toBe('0.75') + expect(result.costLedger).toEqual(COST_LEDGER) + expect(result.executionData.finalOutput).toEqual(EXECUTION_DATA.finalOutput) + expect(result.executionData.models).toEqual(EXECUTION_DATA.models) + }) + + it('withholds nothing from a workspace API key', async () => { + governedBy({ hideTraceSpans: true, hideCostInfo: true }) + + const result = await getPublicLog.execute({ + principal: workspacePrincipal, + input: { runId: 'run-1' }, + }) + + expect(permissionGroupScopeMockFns.mockResolvePermissionGroupConfig).not.toHaveBeenCalled() + expect(result.log.costTotal).toBe('0.75') + expect(result.costLedger).toEqual(COST_LEDGER) + expect(result.executionData.traceSpans).toHaveLength(1) + }) +}) diff --git a/apps/sim/lib/logs/application/read-execution-snapshot.test.ts b/apps/sim/lib/logs/application/read-execution-snapshot.test.ts new file mode 100644 index 00000000000..30df45ff087 --- /dev/null +++ b/apps/sim/lib/logs/application/read-execution-snapshot.test.ts @@ -0,0 +1,198 @@ +/** + * @vitest-environment node + * + * `logs.cost` is a PROJECTION, not a gate — `logOperations.readExecutionSnapshot` + * correctly declares `capability: 'none'`, and the run stays readable while its + * spend does not. + * + * The snapshot read applied none of it, on either of its two doors: the internal + * `/api/logs/execution/{executionId}` route and the `logs_get_execution` Copilot + * tool both presented `executionMetadata.cost` verbatim, so a member whose group + * hides spend read the run total here after being withheld it everywhere else. + * Projecting in the use case is what makes both doors inherit it, which is why + * these exercise the use case against the real `resolveLogFieldProjection`. + */ +import { + permissionGroupScopeMock, + permissionGroupScopeMockFns, + resetPermissionGroupScopeMock, +} from '@sim/testing' +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const mocks = vi.hoisted(() => ({ + select: vi.fn(), + resolveWorkspace: vi.fn(), + resolvePermission: vi.fn(), + materialize: vi.fn(), + hydrateChildTraces: vi.fn(), + recordAudit: vi.fn(), +})) + +vi.mock('@/lib/permission-groups/config-scope.server', () => permissionGroupScopeMock) + +vi.mock('@sim/db', () => ({ db: { select: mocks.select } })) + +vi.mock('@sim/audit', () => ({ + AuditAction: {}, + AuditResourceType: {}, + recordAudit: mocks.recordAudit, +})) + +vi.mock('@sim/platform-authz/workspace', () => ({ + permissionSatisfies: (actual: string | null, required: string) => + actual === 'admin' || actual === 'write' || actual === required, + resolveEffectiveWorkspacePermission: mocks.resolvePermission, +})) + +vi.mock('@/lib/workspaces/application/workspace-context', () => ({ + resolveActiveWorkspaceApplicationContext: mocks.resolveWorkspace, +})) + +vi.mock('@/lib/logs/execution/trace-store', () => ({ + materializeExecutionData: mocks.materialize, +})) + +vi.mock('@/lib/logs/execution/hydrate-child-traces', () => ({ + hydrateChildTraces: mocks.hydrateChildTraces, +})) + +import { readExecutionSnapshotUseCase } from '@/lib/logs/application/read-execution-snapshot' +import { DEFAULT_PERMISSION_GROUP_CONFIG } from '@/lib/permission-groups/fields' + +const WORKSPACE_ID = 'workspace-1' + +const workspaceContext = { + workspaceId: WORKSPACE_ID, + workspaceOrganizationId: 'organization-1', + allowPersonalApiKeys: true, + billedAccountUserId: 'billing-owner-1', +} + +const workflowRecord = { + id: 'log-1', + workflowId: 'workflow-1', + workspaceId: WORKSPACE_ID, + executionId: 'run-1', + stateSnapshotId: 'snapshot-1', + trigger: 'api', + startedAt: new Date('2026-08-05T12:00:00.000Z'), + endedAt: new Date('2026-08-05T12:00:01.000Z'), + totalDurationMs: 1000, + costTotal: '0.75', + executionData: null, +} + +const jobRecord = { + id: 'job-log-1', + workspaceId: WORKSPACE_ID, + executionId: 'job-1', + trigger: 'schedule', + startedAt: new Date('2026-08-05T12:00:00.000Z'), + endedAt: null, + totalDurationMs: null, + cost: { total: 0.75, input: 0.5, output: 0.25 }, +} + +/** + * Answers `db.select(...)` calls in order. The snapshot read walks the workflow + * log, then (only when that missed) the job log, then the state snapshot. + */ +function queueSelects(...results: unknown[][]): void { + for (const rows of results) { + mocks.select.mockReturnValueOnce({ + from: () => ({ where: () => ({ limit: () => Promise.resolve(rows) }) }), + }) + } +} + +const principal = { kind: 'session' as const, userId: 'user-1', sessionId: 'session-1' } +/** + * `logs.read_execution_snapshot` denies a workspace API key outright, so the + * subjectless caller that actually reaches this read is the executor delegation — + * which carries a workspace role but no capabilities, and must read whole. + */ +const executorPrincipal = { + kind: 'delegated' as const, + serviceId: 'executor' as const, + workspaceId: WORKSPACE_ID, + delegationId: 'delegation-1', + audience: 'sim:logs', + issuedAt: new Date(Date.now() - 60_000), + expiresAt: new Date(Date.now() + 60 * 60_000), + delegationContext: { + kind: 'workflow_execution' as const, + workflowId: 'workflow-1', + currentWorkflow: { mode: 'deployment' as const }, + /** Never the projection subject: it is compatibility policy, not the caller. */ + compatibilityActor: { kind: 'legacy_execution_user' as const, userId: 'user-1' }, + }, +} + +function read(actor: typeof principal | typeof executorPrincipal, executionId: string) { + return readExecutionSnapshotUseCase.execute({ + principal: actor, + input: { executionId }, + }) +} + +describe('readExecutionSnapshot spend projection', () => { + beforeEach(() => { + vi.clearAllMocks() + resetPermissionGroupScopeMock() + mocks.resolveWorkspace.mockResolvedValue(workspaceContext) + mocks.resolvePermission.mockResolvedValue('read') + mocks.materialize.mockResolvedValue(null) + }) + + it('reads the run total whole for a member no group governs', async () => { + queueSelects([workflowRecord], [{ id: 'snapshot-1', stateData: { blocks: {} } }]) + + const result = await read(principal, 'run-1') + + expect(result.executionMetadata.cost).toEqual({ total: 0.75 }) + }) + + it('withholds the run total from a member whose group hides spend', async () => { + permissionGroupScopeMockFns.mockResolvePermissionGroupConfig.mockResolvedValue({ + ...DEFAULT_PERMISSION_GROUP_CONFIG, + hideCostInfo: true, + }) + queueSelects([workflowRecord], [{ id: 'snapshot-1', stateData: { blocks: {} } }]) + + const result = await read(principal, 'run-1') + + expect(result.executionMetadata.cost).toBeNull() + expect(result.executionMetadata.trigger).toBe('api') + expect(result.workflowState).toEqual({ blocks: {} }) + }) + + /** A job run spells its spend as a jsonb document; the same rule covers it. */ + it("withholds a job run's spend document from a member whose group hides spend", async () => { + permissionGroupScopeMockFns.mockResolvePermissionGroupConfig.mockResolvedValue({ + ...DEFAULT_PERMISSION_GROUP_CONFIG, + hideCostInfo: true, + }) + queueSelects([], [jobRecord]) + + const result = await read(principal, 'job-1') + + expect(result.executionMetadata.cost).toBeNull() + }) + + /** + * A workspace API key authorizes as the workspace and represents no user, so + * it resolves to no subject — the key's creator is never substituted. + */ + it('reads whole and resolves no group for an executor delegation', async () => { + permissionGroupScopeMockFns.mockResolvePermissionGroupConfig.mockResolvedValue({ + ...DEFAULT_PERMISSION_GROUP_CONFIG, + hideCostInfo: true, + }) + queueSelects([workflowRecord], [{ id: 'snapshot-1', stateData: { blocks: {} } }]) + + const result = await read(executorPrincipal, 'run-1') + + expect(result.executionMetadata.cost).toEqual({ total: 0.75 }) + expect(permissionGroupScopeMockFns.mockResolvePermissionGroupConfig).not.toHaveBeenCalled() + }) +}) diff --git a/apps/sim/lib/logs/application/read-execution-snapshot.ts b/apps/sim/lib/logs/application/read-execution-snapshot.ts index c44f803852a..15d45119c63 100644 --- a/apps/sim/lib/logs/application/read-execution-snapshot.ts +++ b/apps/sim/lib/logs/application/read-execution-snapshot.ts @@ -12,6 +12,11 @@ import { import { logOperations } from '@/lib/logs/application/operations' import { hydrateChildTraces } from '@/lib/logs/execution/hydrate-child-traces' import { materializeExecutionData } from '@/lib/logs/execution/trace-store' +import { + logProjectionSubjectUserId, + projectCostTotal, + resolveLogFieldProjection, +} from '@/lib/logs/log-projection' import type { TraceSpan, WorkflowExecutionLog } from '@/lib/logs/types' import { type ActiveWorkspaceApplicationContext, @@ -133,6 +138,27 @@ const authorizedReadExecutionSnapshotUseCase = defineAuthorizedWorkspaceUseCase( async execute({ principal, input, context }): Promise { input.signal?.throwIfAborted() const record = context.record + + /** + * A projection rather than a refusal, resolved through the shared helper the + * log-detail and v1 paths read — see {@link resolveLogFieldProjection}. Applied + * here in the use case so both doors onto this read inherit it: the internal + * snapshot route and the `logs_get_execution` Copilot tool. + * + * `cost` is the only field on the withheld list this resource carries. The + * snapshot's other payloads are the workflow's own definition — its state + * snapshot and any child-workflow snapshots — which neither capability + * withholds, and the execution data is read only to collect child snapshot + * ids; no trace span, block execution, input or final output is returned. + * + * permission-group-enforced: logs.cost + */ + const projection = await resolveLogFieldProjection( + logProjectionSubjectUserId(principal), + context.workspaceId, + context.workspaceOrganizationId + ) + if (record.kind === 'job') { return { executionId: record.executionId, @@ -144,7 +170,7 @@ const authorizedReadExecutionSnapshotUseCase = defineAuthorizedWorkspaceUseCase( startedAt: record.startedAt.toISOString(), endedAt: record.endedAt?.toISOString(), totalDurationMs: record.totalDurationMs, - cost: record.cost || null, + cost: projection.hideCostInfo ? null : record.cost || null, }, } } @@ -208,7 +234,7 @@ const authorizedReadExecutionSnapshotUseCase = defineAuthorizedWorkspaceUseCase( startedAt: record.startedAt.toISOString(), endedAt: record.endedAt?.toISOString(), totalDurationMs: record.totalDurationMs, - cost: record.costTotal != null ? { total: Number(record.costTotal) } : null, + cost: projectCostTotal(record.costTotal, projection), }, } }, diff --git a/apps/sim/lib/logs/application/read-log-detail.test.ts b/apps/sim/lib/logs/application/read-log-detail.test.ts index ee291213706..8960be8fd09 100644 --- a/apps/sim/lib/logs/application/read-log-detail.test.ts +++ b/apps/sim/lib/logs/application/read-log-detail.test.ts @@ -4,7 +4,12 @@ import type { Principal } from '@sim/auth/principal' import { workflowExecutionLogs } from '@sim/db/schema' -import { queueTableRows, resetDbChainMock } from '@sim/testing' +import { + permissionGroupScopeMock, + permissionGroupScopeMockFns, + queueTableRows, + resetDbChainMock, +} from '@sim/testing' import { afterAll, beforeEach, describe, expect, it, vi } from 'vitest' const mocks = vi.hoisted(() => ({ @@ -13,6 +18,8 @@ const mocks = vi.hoisted(() => ({ resolvePermission: vi.fn(), })) +const resolveGroupConfigMock = permissionGroupScopeMockFns.mockResolvePermissionGroupConfig + vi.mock('@/lib/logs/fetch-log-detail', () => ({ readLogDetail: mocks.readLogDetail, })) @@ -21,6 +28,8 @@ vi.mock('@/lib/workspaces/application/workspace-context', () => ({ resolveActiveWorkspaceApplicationContext: mocks.resolveWorkspace, })) +vi.mock('@/lib/permission-groups/config-scope.server', () => permissionGroupScopeMock) + vi.mock('@sim/platform-authz/workspace', () => ({ permissionSatisfies: (held: string | null, required: string) => held === 'admin' || held === required || (held === 'write' && required === 'read'), @@ -93,6 +102,7 @@ describe('readLogDetailUseCase', () => { }) mocks.readLogDetail.mockResolvedValue({ id: 'log-1', executionId: EXECUTION_ID }) mocks.resolvePermission.mockResolvedValue('admin') + resolveGroupConfigMock.mockResolvedValue(null) }) afterAll(resetDbChainMock) @@ -123,4 +133,57 @@ describe('readLogDetailUseCase', () => { expect.objectContaining({ viewerUserId: 'user-1' }) ) }) + + /** + * A projection, not a refusal: the loader is still asked for the log, just + * told to leave the spend out of it. + */ + it('tells the loader to withhold spend when the group does', async () => { + queueLogRow() + resolveGroupConfigMock.mockResolvedValue({ hideCostInfo: true }) + + await readLogDetailUseCase.execute({ + principal: { kind: 'session', userId: 'user-1', sessionId: 'session-1' }, + input: { workspaceId: WORKSPACE_ID, lookupColumn: 'executionId', lookupValue: EXECUTION_ID }, + }) + + expect(mocks.readLogDetail).toHaveBeenCalledWith( + expect.objectContaining({ hideCostInfo: true }) + ) + }) + + /** + * The same person's group, reached through the run they triggered rather than + * through their own session. The delegation carries their role and none of + * their capabilities — `authorizeWorkspaceOperation` already passed it + * ungated — so projecting on it would withhold from a run on a group the + * funnel declined to apply. Attribution still names them. + */ + it('leaves spend in place for a run delegated by that same person', async () => { + queueLogRow() + resolveGroupConfigMock.mockResolvedValue({ hideCostInfo: true }) + + await readLogDetailUseCase.execute({ + principal: HUMAN_PRINCIPAL, + input: { workspaceId: WORKSPACE_ID, lookupColumn: 'executionId', lookupValue: EXECUTION_ID }, + }) + + expect(resolveGroupConfigMock).not.toHaveBeenCalled() + expect(mocks.readLogDetail).toHaveBeenCalledWith( + expect.objectContaining({ viewerUserId: 'user-1', hideCostInfo: false }) + ) + }) + + it('leaves spend in place when no group withholds it', async () => { + queueLogRow() + + await readLogDetailUseCase.execute({ + principal: HUMAN_PRINCIPAL, + input: { workspaceId: WORKSPACE_ID, lookupColumn: 'executionId', lookupValue: EXECUTION_ID }, + }) + + expect(mocks.readLogDetail).toHaveBeenCalledWith( + expect.objectContaining({ hideCostInfo: false }) + ) + }) }) diff --git a/apps/sim/lib/logs/application/read-log-detail.ts b/apps/sim/lib/logs/application/read-log-detail.ts index 326a3047ee1..8c2c0c0137f 100644 --- a/apps/sim/lib/logs/application/read-log-detail.ts +++ b/apps/sim/lib/logs/application/read-log-detail.ts @@ -11,6 +11,7 @@ import { } from '@/lib/logs/application/authorization' import { logOperations } from '@/lib/logs/application/operations' import { readLogDetail } from '@/lib/logs/fetch-log-detail' +import { logProjectionSubjectUserId, resolveLogFieldProjection } from '@/lib/logs/log-projection' import { type ActiveWorkspaceApplicationContext, resolveActiveWorkspaceApplicationContext, @@ -79,12 +80,26 @@ const authorizedReadLogDetailUseCase = defineAuthorizedWorkspaceUseCase({ input.signal?.throwIfAborted() // Attribution, not authorization: an actorless run (a schedule, or a webhook // with no external subject) reads its own workspace's logs with no user to name. + const viewerUserId = resolvePrincipalSubjectUserId(principal) + + /** + * A projection rather than a refusal: the log stays readable, its execution + * payloads and its spend do not. Resolved through the shared helper, which + * the v1 public API reads too — see {@link resolveLogFieldProjection}. + */ + const projection = await resolveLogFieldProjection( + logProjectionSubjectUserId(principal), + context.workspaceId, + context.workspaceOrganizationId + ) + const detail = await readLogDetail({ - viewerUserId: resolvePrincipalSubjectUserId(principal), + viewerUserId, workspaceId: context.workspaceId, lookupColumn: input.lookupColumn, lookupValue: input.lookupValue, signal: input.signal, + ...projection, }) input.signal?.throwIfAborted() if (!detail) throw new OrchestrationError('not_found', 'Not found') diff --git a/apps/sim/lib/logs/execution/hydrate-child-traces.test.ts b/apps/sim/lib/logs/execution/hydrate-child-traces.test.ts index 0af90346a48..58a0de0555c 100644 --- a/apps/sim/lib/logs/execution/hydrate-child-traces.test.ts +++ b/apps/sim/lib/logs/execution/hydrate-child-traces.test.ts @@ -32,7 +32,7 @@ vi.mock('@/lib/workspaces/permissions/utils', () => ({ vi.mock('@/lib/logs/execution/trace-store', () => ({ materializeExecutionDataForDisplay: mockMaterialize, - stripSpanCosts: (spans: unknown) => { + stripJoinedChildTraceSpend: (spans: unknown) => { if (!Array.isArray(spans)) return for (const span of spans) { if (span && typeof span === 'object') { diff --git a/apps/sim/lib/logs/execution/hydrate-child-traces.ts b/apps/sim/lib/logs/execution/hydrate-child-traces.ts index cb4281eb92e..b47f7adbf69 100644 --- a/apps/sim/lib/logs/execution/hydrate-child-traces.ts +++ b/apps/sim/lib/logs/execution/hydrate-child-traces.ts @@ -6,7 +6,7 @@ import { inArray } from 'drizzle-orm' import { flattenWorkflowChildren } from '@/lib/logs/execution/trace-spans/span-factory' import { materializeExecutionDataForDisplay, - stripSpanCosts, + stripJoinedChildTraceSpend, } from '@/lib/logs/execution/trace-store' import type { TraceSpan } from '@/lib/logs/types' @@ -261,10 +261,11 @@ export async function hydrateChildTraces( // The same flattening the in-process workflow-in-workflow path uses, so a // cross-workspace child nests identically to a local one. const children = flattenWorkflowChildren(childSpans) - // The child's spend is billed to the SOURCE workspace and was never rolled - // into this run's total, so leaving per-span cost here would make the - // waterfall's numbers contradict the run cost shown above it. - stripSpanCosts(children) + // The child's spend is billed to the SOURCE workspace and was never + // rolled into this run's total, so leaving any of it here would make the + // waterfall's numbers contradict the run cost shown above it. Tokens go + // with the dollars — see {@link stripJoinedChildTraceSpend}. + stripJoinedChildTraceSpend(children) span.children = children span.childTraceAccess = 'granted' diff --git a/apps/sim/lib/logs/execution/logging-session.ts b/apps/sim/lib/logs/execution/logging-session.ts index c2fe239248b..ece15ee9196 100644 --- a/apps/sim/lib/logs/execution/logging-session.ts +++ b/apps/sim/lib/logs/execution/logging-session.ts @@ -46,6 +46,7 @@ import type { SerializableExecutionState } from '@/executor/execution/types' import type { BlockLog } from '@/executor/types' import { projectResolvedSecretDiagnosticError } from '@/executor/utils/resolved-secret-content-projection' import { + emptyResolvedSecretTraceProvenance, isResolvedSecretTraceProvenanceV1, RESOLVED_SECRET_TRACE_CHECKPOINT_VERSION, type ResolvedSecretTraceProvenanceV1, @@ -124,10 +125,6 @@ function getActiveBlockDisplayProvenance( const logger = createLogger('LoggingSession') -function emptyResolvedSecretTraceProvenance(): ResolvedSecretTraceProvenanceV1 { - return { version: 1, complete: true, entries: [] } -} - type CompletionAttempt = 'complete' | 'error' | 'cancelled' | 'paused' export interface SecretSafeDisplayContent { diff --git a/apps/sim/lib/logs/execution/trace-store.test.ts b/apps/sim/lib/logs/execution/trace-store.test.ts index c73e65241a4..299cbfda3fa 100644 --- a/apps/sim/lib/logs/execution/trace-store.test.ts +++ b/apps/sim/lib/logs/execution/trace-store.test.ts @@ -25,14 +25,18 @@ vi.mock('@/lib/execution/payloads/store', () => ({ })) import { + copyTraceSpansWithoutCosts, externalizeExecutionData, materializeExecutionData, materializeExecutionDataForDisplayWithBlockOutputs, projectExecutionDataForDisplay, RESOLVED_SECRET_PROVENANCE_KEY, SECRET_PROJECTION_VERSION, + stripJoinedChildTraceSpend, + stripSpanCosts, TRACE_STORE_REF_KEY, } from '@/lib/logs/execution/trace-store' +import type { TraceSpan } from '@/lib/logs/types' const CONTEXT = { workspaceId: 'workspace-1', @@ -772,3 +776,171 @@ describe('stored provenance display reporting', () => { ) }) }) + +/** + * The two strips are not the same removal, and the difference is whether the + * result is written. + * + * `stripJoinedChildTraceSpend` is what stands between a joined cross-workspace + * child run and the parent's reader: the child's spend is billed to the SOURCE + * workspace and was never rolled into this run's total, so anything it leaves + * behind is spend the reader was never meant to see — and it never persists. + * `stripSpanCosts` runs inside `backfill-trace-spans.ts`, which stores what it + * returns, so anything IT clears is gone for every authorized reader of that run + * forever. Only the dollars belong in that set. + */ +function spanWithSpend() { + return [ + { + id: 'span-1', + name: 'agent', + cost: { total: 0.5 }, + tokens: { total: 900 }, + providerTiming: { + duration: 5, + segments: [ + { type: 'model', name: 'gpt-4', tokens: { total: 900 }, cost: { total: 0.5 } }, + { type: 'tool', name: 'search' }, + ], + }, + children: [ + { + id: 'span-2', + name: 'model', + cost: { total: 0.2 }, + tokens: { total: 400 }, + providerTiming: { segments: [{ type: 'model', tokens: { total: 400 } }] }, + }, + ], + }, + ] +} + +describe('stripJoinedChildTraceSpend', () => { + it('clears the span roll-up and the provider-timing segments that itemize it', () => { + const spans = spanWithSpend() + + stripJoinedChildTraceSpend(spans) + + expect(spans[0].cost).toBeUndefined() + expect(spans[0].tokens).toBeUndefined() + const [modelSegment, toolSegment] = spans[0].providerTiming.segments as Array< + Record + > + expect(modelSegment.tokens).toBeUndefined() + expect(modelSegment.cost).toBeUndefined() + // Structure and identity are what the waterfall renders; only spend goes. + expect(modelSegment).toMatchObject({ type: 'model', name: 'gpt-4' }) + expect(toolSegment).toMatchObject({ type: 'tool', name: 'search' }) + }) + + it('reaches the segments of nested children too', () => { + const spans = spanWithSpend() + + stripJoinedChildTraceSpend(spans) + + const child = spans[0].children[0] + expect(child.cost).toBeUndefined() + expect(child.tokens).toBeUndefined() + expect( + (child.providerTiming.segments as Array>)[0].tokens + ).toBeUndefined() + }) + + it('leaves a span with no provider timing alone', () => { + const spans = [{ id: 'span-1', name: 'api', cost: { total: 0.1 } }] + + expect(() => stripJoinedChildTraceSpend(spans)).not.toThrow() + expect(spans[0]).toMatchObject({ id: 'span-1', name: 'api' }) + }) +}) + +describe('stripSpanCosts', () => { + it('clears cost at both levels and through children', () => { + const spans = spanWithSpend() + + stripSpanCosts(spans) + + expect(spans[0].cost).toBeUndefined() + expect(spans[0].children[0].cost).toBeUndefined() + expect( + (spans[0].providerTiming.segments as Array>)[0].cost + ).toBeUndefined() + }) + + /** + * The migration stores what this returns. A legacy run's token counts are + * ordinary trace detail its authorized readers have always had, and the ledger + * — not the span — is where dollars live, so erasing them buys nothing and + * cannot be undone. + */ + it('keeps the token counts the migration is about to persist', () => { + const spans = spanWithSpend() + + stripSpanCosts(spans) + + expect(spans[0].tokens).toEqual({ total: 900 }) + expect(spans[0].children[0].tokens).toEqual({ total: 400 }) + const segments = spans[0].providerTiming.segments as Array> + expect(segments[0].tokens).toEqual({ total: 900 }) + expect( + (spans[0].children[0].providerTiming.segments as Array>)[0].tokens + ).toEqual({ total: 400 }) + }) +}) + +/** + * The COMPLETION write. `stripSpanCosts` only ever ran over legacy rows the + * backfill touched; every normal run went through this copy, which used to drop + * the span's own `cost` and leave the same dollars itemized underneath it in + * `providerTiming.segments`. Both writers now share one removal rule, so the + * two cannot answer differently about what a persisted span may carry. + */ +describe('copyTraceSpansWithoutCosts', () => { + it('clears the segment dollars the completion write used to persist', () => { + const spans = spanWithSpend() as unknown as TraceSpan[] + + const persisted = copyTraceSpansWithoutCosts(spans) + + const [span] = persisted as Array> + expect(span.cost).toBeUndefined() + expect(span.providerTiming.segments[0].cost).toBeUndefined() + expect(span.children[0].cost).toBeUndefined() + expect(span.children[0].providerTiming.segments[0].cost).toBeUndefined() + }) + + it('keeps the token counts and the segment identity a trace is read for', () => { + const spans = spanWithSpend() as unknown as TraceSpan[] + + const [span] = copyTraceSpansWithoutCosts(spans) as unknown as Array> + + expect(span.tokens).toEqual({ total: 900 }) + expect(span.children[0].tokens).toEqual({ total: 400 }) + expect(span.providerTiming.segments[0]).toMatchObject({ + type: 'model', + name: 'gpt-4', + tokens: { total: 900 }, + }) + expect(span.providerTiming.duration).toBe(5) + }) + + /** + * The strip runs in place, so the copy has to reach every node it writes to. + * Sharing the `providerTiming` with the caller would blank the segments of the + * spans the rest of the run still holds in memory. + */ + it('leaves the caller’s in-memory spans untouched', () => { + const spans = spanWithSpend() as unknown as TraceSpan[] + + copyTraceSpansWithoutCosts(spans) + + const [span] = spans as unknown as Array> + expect(span.cost).toEqual({ total: 0.5 }) + expect(span.providerTiming.segments[0].cost).toEqual({ total: 0.5 }) + expect(span.children[0].cost).toEqual({ total: 0.2 }) + }) + + it('returns undefined for no spans', () => { + expect(copyTraceSpansWithoutCosts(undefined)).toBeUndefined() + }) +}) diff --git a/apps/sim/lib/logs/execution/trace-store.ts b/apps/sim/lib/logs/execution/trace-store.ts index ff229f87345..4851910dab4 100644 --- a/apps/sim/lib/logs/execution/trace-store.ts +++ b/apps/sim/lib/logs/execution/trace-store.ts @@ -100,27 +100,127 @@ function workflowIdFromStorageKey(key: string | undefined): string | undefined { } /** - * Recursively removes `cost` from trace spans before persistence. Cost lives in - * exactly one place — the usage_log ledger — so persisted spans carry only - * structure, timing, and tokens (KTD7). Must run AFTER `calculateCostSummary` - * has consumed span costs in memory. + * Recursively removes spend from trace spans, in place. + * + * `tokens` is optional because the two callers withhold different things. + * Persistence withholds only dollars — cost lives in exactly one place, the + * usage_log ledger (KTD7), so a stored span carries structure, timing and + * tokens. A joined cross-workspace child run withholds the whole amount: its + * token counts are the same spend in another unit, recoverable by anyone who + * knows the model's rate. + * + * Must run AFTER `calculateCostSummary` has consumed span costs in memory. */ -export function stripSpanCosts(spans: unknown): void { +function stripSpanSpendFields(spans: unknown, options: { tokens: boolean }): void { if (!Array.isArray(spans)) return for (const span of spans) { if (!span || typeof span !== 'object') continue - const record = span as { cost?: unknown; children?: unknown } + const record = span as { + cost?: unknown + tokens?: unknown + children?: unknown + providerTiming?: unknown + } + if ('cost' in record) record.cost = undefined + if (options.tokens && 'tokens' in record) record.tokens = undefined + stripProviderTimingSegmentSpend(record.providerTiming, options) + if (Array.isArray(record.children)) stripSpanSpendFields(record.children, options) + } +} + +/** + * Removes per-span `cost` before persistence, leaving tokens in place. + * + * The one strip that WRITES: `backfill-trace-spans.ts` runs it over a legacy + * row's spans and stores the result, so anything it clears is gone for every + * authorized reader of that run, forever. Only cost belongs in that set — the + * ledger owns the dollars, and the spans have never been the place they live. + */ +export function stripSpanCosts(spans: unknown): void { + stripSpanSpendFields(spans, { tokens: false }) +} + +/** + * Removes cost AND token counts from a joined child run's spans, in memory. + * + * The child's spend is billed to the SOURCE workspace and was never rolled into + * the parent run's total, so leaving any of it would publish spend the reader + * was never meant to see and make the waterfall contradict the run cost above + * it. A read-time projection only: these spans are hydrated onto a response and + * never written back. + */ +export function stripJoinedChildTraceSpend(spans: unknown): void { + stripSpanSpendFields(spans, { tokens: true }) +} + +/** + * The same removal one level down, in `providerTiming.segments`. + * + * A `ProviderTimingSegment` carries its own `tokens` and `cost` — the per-model + * iteration breakdown behind the span's roll-up — so clearing the span alone + * left the whole figure itemized underneath it, which is strictly more than the + * span published in the first place. + */ +function stripProviderTimingSegmentSpend( + providerTiming: unknown, + options: { tokens: boolean } +): void { + if (!providerTiming || typeof providerTiming !== 'object') return + const segments = (providerTiming as { segments?: unknown }).segments + if (!Array.isArray(segments)) return + for (const segment of segments) { + if (!segment || typeof segment !== 'object') continue + const record = segment as { cost?: unknown; tokens?: unknown } if ('cost' in record) record.cost = undefined - if (Array.isArray(record.children)) stripSpanCosts(record.children) + if (options.tokens && 'tokens' in record) record.tokens = undefined } } -/** Creates a persistence-owned span tree with per-span cost fields removed. */ +/** + * Copies exactly the nodes {@link stripSpanSpendFields} writes to — each span, + * its children, its `providerTiming`, and that timing's segments — and shares + * every other value with the caller's tree. Enough isolation for the strip to + * run in place without reaching the in-memory spans the rest of the run still + * holds, and no deep clone of the payloads hanging off a span. + */ +function copySpanTreeForStrip(spans: TraceSpan[]): TraceSpan[] { + return spans.map((span) => { + const copy: TraceSpan = { ...span } + if (Array.isArray(copy.children)) copy.children = copySpanTreeForStrip(copy.children) + if (copy.providerTiming && typeof copy.providerTiming === 'object') { + const { segments } = copy.providerTiming + copy.providerTiming = { + ...copy.providerTiming, + ...(Array.isArray(segments) + ? { + segments: segments.map((segment) => + segment && typeof segment === 'object' ? { ...segment } : segment + ), + } + : {}), + } + } + return copy + }) +} + +/** + * Creates a persistence-owned span tree with spend removed, for the COMPLETION + * write. + * + * Runs the same {@link stripSpanCosts} the legacy backfill does, over a copy — + * one removal rule for both writers, which is the point: this used to drop the + * span's own `cost` and nothing else, so every completed run persisted the + * itemized dollars underneath it in `providerTiming.segments`, which the backfill + * had already learned to clear. Tokens survive, on both paths: the ledger owns + * the dollars, and a span's token counts are trace detail the reader is entitled + * to. + */ export function copyTraceSpansWithoutCosts(spans?: TraceSpan[]): TraceSpan[] | undefined { - return spans?.map(({ cost: _cost, children, ...span }) => ({ - ...span, - ...(children ? { children: copyTraceSpansWithoutCosts(children) } : {}), - })) + if (!spans) return undefined + const copy = copySpanTreeForStrip(spans) + stripSpanCosts(copy) + return copy } /** diff --git a/apps/sim/lib/logs/fetch-log-detail.test.ts b/apps/sim/lib/logs/fetch-log-detail.test.ts index 326032257bc..c4303867f52 100644 --- a/apps/sim/lib/logs/fetch-log-detail.test.ts +++ b/apps/sim/lib/logs/fetch-log-detail.test.ts @@ -23,8 +23,110 @@ vi.mock('@/lib/logs/execution-origin', () => ({ workflowExecutionOriginSql: () => ({ as: () => ({}) }), })) +import { workflowLogDetailSchema } from '@/lib/api/contracts/logs' import { readLogDetail } from '@/lib/logs/fetch-log-detail' +function queueWorkflowLogRow(overrides: Record = {}): void { + queueTableRows(workflowExecutionLogs, [ + { + id: 'log-1', + workflowId: 'workflow-1', + executionId: 'execution-1', + deploymentVersionId: null, + deploymentVersion: null, + deploymentVersionName: null, + level: 'info', + status: 'completed', + trigger: 'manual', + startedAt: new Date('2026-01-01T00:00:00.000Z'), + endedAt: new Date('2026-01-01T00:00:01.000Z'), + totalDurationMs: 1000, + executionData: {}, + costTotal: '1.25', + files: null, + createdAt: new Date('2026-01-01T00:00:00.000Z'), + workflowName: 'Workflow', + workflowDescription: null, + workflowFolderId: null, + workflowUserId: 'user-1', + workflowWorkspaceId: 'workspace-1', + workflowCreatedAt: new Date('2026-01-01T00:00:00.000Z'), + workflowUpdatedAt: new Date('2026-01-01T00:00:00.000Z'), + pausedStatus: null, + pausedTotalPauseCount: 0, + pausedResumedCount: 0, + executionOrigin: null, + ...overrides, + }, + ]) +} + +const SPEND_BEARING_EXECUTION_DATA = { + /** + * The run-level roll-up `buildCompletedExecutionData` writes on every + * completed run. `models` is the per-model dollar breakdown itself, so it is + * finer-grained than the total the projection blanks. + */ + tokens: { input: 500, output: 400, total: 900 }, + models: { + 'gpt-4': { input: 0.4, output: 0.35, total: 0.75, tokens: { total: 900 } }, + }, + cost: { total: 0.75 }, + traceSpans: [ + { + id: 'span-1', + name: 'Agent 1', + type: 'agent', + duration: 5, + startTime: '2026-01-01T00:00:00.000Z', + endTime: '2026-01-01T00:00:00.005Z', + cost: { total: 0.75 }, + tokens: { total: 900 }, + /** A span's own itemization, one level below its roll-up. */ + providerTiming: { + duration: 5, + startTime: '2026-01-01T00:00:00.000Z', + endTime: '2026-01-01T00:00:00.005Z', + segments: [ + { + type: 'model', + name: 'gpt-4', + startTime: 0, + endTime: 5, + duration: 5, + tokens: { total: 900 }, + cost: { total: 0.75 }, + }, + ], + }, + children: [ + { + id: 'span-2', + name: 'Model', + type: 'model', + cost: { total: 0.5 }, + tokens: { total: 400 }, + }, + ], + }, + ], + blockExecutions: [ + { + id: 'block-exec-1', + blockId: 'block-1', + blockName: 'Agent 1', + blockType: 'agent', + startedAt: '2026-01-01T00:00:00.000Z', + endedAt: '2026-01-01T00:00:00.005Z', + durationMs: 5, + status: 'success', + inputData: {}, + outputData: {}, + cost: { total: 0.75 }, + }, + ], +} + describe('readLogDetail', () => { beforeEach(() => { vi.clearAllMocks() @@ -154,4 +256,73 @@ describe('readLogDetail', () => { viewerUserId: undefined, }) }) + + describe("when the viewer's permission group withholds cost", () => { + beforeEach(() => { + queueTableRows(usageLog, []) + mocks.materializeExecutionData.mockResolvedValue( + structuredClone(SPEND_BEARING_EXECUTION_DATA) + ) + }) + + it('still returns a log the contract accepts, with every spend figure gone', async () => { + queueWorkflowLogRow() + + const result = await readLogDetail({ + viewerUserId: 'user-1', + workspaceId: 'workspace-1', + lookupColumn: 'id', + lookupValue: 'log-1', + hideCostInfo: true, + }) + + // The projection must stay inside the wire contract: a withheld log is + // still a log, and a client parsing the response cannot be made to fail. + expect(() => workflowLogDetailSchema.parse(result)).not.toThrow() + + expect(result?.cost).toBeNull() + expect(result).not.toHaveProperty('costLedger') + + const [span] = result?.executionData.traceSpans ?? [] + expect(span).not.toHaveProperty('cost') + expect(span).not.toHaveProperty('tokens') + // Nested spans carry their own figures; summing children would otherwise + // reconstruct exactly the total that was withheld. + expect(span?.children?.[0]).not.toHaveProperty('cost') + expect(result?.executionData.blockExecutions?.[0]).not.toHaveProperty('cost') + + // The run's own roll-up. `models` is the per-model dollar breakdown, so + // leaving it published the finest figure of all next to a blanked total. + expect(result?.executionData).not.toHaveProperty('tokens') + expect(result?.executionData).not.toHaveProperty('models') + expect(result?.executionData).not.toHaveProperty('cost') + + // Provider-timing segments itemize the span's own roll-up, so stripping + // the span alone leaves the amount recoverable one level down. + const [segment] = (span as { providerTiming?: { segments?: unknown[] } })?.providerTiming + ?.segments as Array> + expect(segment).toMatchObject({ name: 'gpt-4' }) + expect(segment).not.toHaveProperty('cost') + expect(segment).not.toHaveProperty('tokens') + + // Everything the restriction does not cover is untouched. + expect(result).toMatchObject({ id: 'log-1', status: 'completed' }) + expect(span).toMatchObject({ id: 'span-1', name: 'Agent 1' }) + }) + + it('reports the run total when the group does not withhold it', async () => { + queueWorkflowLogRow() + + const result = await readLogDetail({ + viewerUserId: 'user-1', + workspaceId: 'workspace-1', + lookupColumn: 'id', + lookupValue: 'log-1', + }) + + expect(result?.cost).toEqual({ total: 1.25 }) + expect(result?.executionData.traceSpans?.[0]).toHaveProperty('cost') + expect(result?.executionData).toHaveProperty('models') + }) + }) }) diff --git a/apps/sim/lib/logs/fetch-log-detail.ts b/apps/sim/lib/logs/fetch-log-detail.ts index 41a89227a12..d54b6182b13 100644 --- a/apps/sim/lib/logs/fetch-log-detail.ts +++ b/apps/sim/lib/logs/fetch-log-detail.ts @@ -40,6 +40,113 @@ interface FetchLogDetailArgs { lookupColumn: LookupColumn lookupValue: string signal?: AbortSignal + /** + * Whether the viewer's permission group withholds execution detail. Applied + * here rather than in the client, because the payloads it covers — trace + * spans, block inputs and outputs, the final output — are the customer data + * the restriction exists to withhold, and a hidden tab withholds nothing from + * a caller reading the route directly. + */ + hideTraceSpans?: boolean + /** + * Whether the viewer's permission group withholds spend. Applied here for the + * same reason as {@link FetchLogDetailArgs.hideTraceSpans}: the run total, the + * itemized ledger and the per-block and per-span costs are the figures the + * restriction exists to withhold, and a hidden column withholds nothing from a + * caller reading the route directly. + * + * Required for the same reason as {@link FetchLogDetailArgs.hideTraceSpans}. + */ + hideCostInfo: boolean +} + +/** + * Strips the execution payloads a permission group withholds. + * + * Deletes rather than relies on the schema: `executionDataDetailSchema` is a + * passthrough, so a field left in place would survive response validation. + * Applied before child traces are hydrated, so a withheld view does not pay for + * a cross-workspace join whose result it discards. + */ +export function withheldExecutionData( + executionData: Record +): Record { + const { + traceSpans: _traceSpans, + blockExecutions: _blockExecutions, + finalOutput: _finalOutput, + workflowInput: _workflowInput, + blockInput: _blockInput, + ...retained + } = executionData + return retained +} + +/** + * A `providerTiming` with the per-iteration spend stripped from its segments. + * + * A segment carries its own `cost` and `tokens` — the itemization behind the + * span's roll-up — so removing the span's fields alone leaves the finer + * breakdown in place, which withholds nothing. + */ +function withoutSegmentSpend(providerTiming: unknown): unknown { + if (!providerTiming || typeof providerTiming !== 'object' || Array.isArray(providerTiming)) { + return providerTiming + } + const record = providerTiming as Record + if (!Array.isArray(record.segments)) return providerTiming + return { + ...record, + segments: record.segments.map((segment) => { + if (!segment || typeof segment !== 'object' || Array.isArray(segment)) return segment + const { cost: _cost, tokens: _tokens, ...retained } = segment as Record + return retained + }), + } +} + +/** A span or block execution with the spend fields stripped from it. */ +function withoutSpend(entry: unknown): unknown { + if (!entry || typeof entry !== 'object' || Array.isArray(entry)) return entry + const { + cost: _cost, + tokens: _tokens, + children, + providerTiming, + ...rest + } = entry as Record + const retained = + providerTiming === undefined + ? rest + : { ...rest, providerTiming: withoutSegmentSpend(providerTiming) } + return Array.isArray(children) ? { ...retained, children: children.map(withoutSpend) } : retained +} + +/** + * Strips spend from the execution payloads a permission group withholds. + * + * Reaches into trace spans and block executions rather than only blanking the + * run total: both carry their own `cost` and `tokens`, and a viewer who can sum + * the spans has not been withheld anything. Deletes rather than relies on the + * schema, because `executionDataDetailSchema` is a passthrough and a span's own + * shape is a `catchall`, so a field left in place would survive validation. + * + * The run's own roll-up goes first. `buildCompletedExecutionData` writes + * `tokens` and `models` at the root of every completed run, and `models` is the + * per-model dollar breakdown itself — leaving it while stripping the spans + * published the finest-grained figure of all next to a blanked total. `cost` is + * dropped with them for the runs old enough to carry it inline. + */ +export function withheldSpendData(executionData: Record): Record { + const { tokens: _tokens, models: _models, cost: _cost, ...retained } = executionData + const projected: Record = { ...retained } + if (Array.isArray(projected.traceSpans)) { + projected.traceSpans = projected.traceSpans.map(withoutSpend) + } + if (Array.isArray(projected.blockExecutions)) { + projected.blockExecutions = projected.blockExecutions.map(withoutSpend) + } + return projected } /** @@ -56,6 +163,8 @@ export async function readLogDetail({ lookupColumn, lookupValue, signal, + hideTraceSpans, + hideCostInfo, }: FetchLogDetailArgs): Promise { signal?.throwIfAborted() const workflowMatch: SQL = @@ -82,7 +191,6 @@ export async function readLogDetail({ workflowName: workflow.name, workflowDescription: workflow.description, workflowFolderId: workflow.folderId, - workflowUserId: workflow.userId, workflowWorkspaceId: workflow.workspaceId, workflowCreatedAt: workflow.createdAt, workflowUpdatedAt: workflow.updatedAt, @@ -113,7 +221,6 @@ export async function readLogDetail({ name: log.workflowName, description: log.workflowDescription, folderId: log.workflowFolderId, - userId: log.workflowUserId, workspaceId: log.workflowWorkspaceId, createdAt: log.workflowCreatedAt?.toISOString() ?? null, updatedAt: log.workflowUpdatedAt?.toISOString() ?? null, @@ -128,13 +235,13 @@ export async function readLogDetail({ // Cost is sourced exclusively from the usage_log ledger (itemized breakdown) // and its cost_total projection (run total). The cost jsonb is never read. - const costLedger = await buildCostLedger(log.executionId) + const costLedger = hideCostInfo ? null : await buildCostLedger(log.executionId) signal?.throwIfAborted() const totalDollars = costLedger?.total ?? (log.costTotal != null ? Number(log.costTotal) : null) // Trace spans / heavy execution data may live in object storage; resolve the // pointer here (no-op for inline / pre-externalization rows). - const executionData = await materializeExecutionDataForDisplay( + const materialized = await materializeExecutionDataForDisplay( log.executionData as Record | null, { workspaceId, @@ -143,6 +250,8 @@ export async function readLogDetail({ userId: viewerUserId, } ) + const withheldPayloads = hideTraceSpans ? withheldExecutionData(materialized) : materialized + const executionData = hideCostInfo ? withheldSpendData(withheldPayloads) : withheldPayloads signal?.throwIfAborted() // A custom block's child ran in another workspace and kept its spans on its @@ -182,8 +291,8 @@ export async function readLogDetail({ createdAt: log.startedAt.toISOString(), workflow: workflowSummary, jobTitle: null, - cost: totalDollars != null ? { total: totalDollars } : null, - costLedger, + cost: hideCostInfo || totalDollars == null ? null : { total: totalDollars }, + ...(hideCostInfo ? {} : { costLedger }), pauseSummary: { status: log.pausedStatus ?? null, total: totalPauseCount, @@ -228,7 +337,7 @@ export async function readLogDetail({ const jobLog = jobRows[0] if (!jobLog) return null - const execData = await materializeExecutionDataForDisplay( + const materializedJobData = await materializeExecutionDataForDisplay( jobLog.executionData as Record | null, { workspaceId, @@ -237,6 +346,10 @@ export async function readLogDetail({ userId: viewerUserId, } ) + const withheldJobPayloads = hideTraceSpans + ? withheldExecutionData(materializedJobData) + : materializedJobData + const execData = hideCostInfo ? withheldSpendData(withheldJobPayloads) : withheldJobPayloads signal?.throwIfAborted() return workflowLogDetailSchema.parse({ id: jobLog.id, @@ -253,7 +366,7 @@ export async function readLogDetail({ createdAt: jobLog.startedAt.toISOString(), workflow: null, jobTitle: ((execData.trigger as Record | undefined)?.source as string) ?? null, - cost: jobCostTotal(jobLog.cost), + cost: hideCostInfo ? null : jobCostTotal(jobLog.cost), pauseSummary: { status: null, total: 0, resumed: 0 }, hasPendingPause: false, executionData: { diff --git a/apps/sim/lib/logs/list-logs.test.ts b/apps/sim/lib/logs/list-logs.test.ts index 75da730b2f0..3ab31ecc96e 100644 --- a/apps/sim/lib/logs/list-logs.test.ts +++ b/apps/sim/lib/logs/list-logs.test.ts @@ -183,3 +183,31 @@ describe('readLogs', () => { expect(result.data[0].workflowId).toBe('wf-1') }) }) + +describe('readLogs cost projection', () => { + beforeEach(() => { + vi.clearAllMocks() + resetDbChainMock() + }) + + /** + * The list carries the same run total the detail does, so a group that + * withholds spend on one and not the other has withheld nothing. + */ + it('blanks the run total on workflow and job summaries alike', async () => { + queueTableRows(workflowExecutionLogs, [workflowRow()]) + queueTableRows(jobExecutionLogs, [jobRow()]) + + const result = await readLogs(baseParams({ hideCostInfo: true })) + + expect(result.data).toHaveLength(2) + for (const summary of result.data) { + expect(summary.cost).toBeNull() + } + // Nothing else about the row is withheld. + expect(result.data.find((row) => row.id === 'log-1')).toMatchObject({ + executionId: 'exec-1', + duration: '1000ms', + }) + }) +}) diff --git a/apps/sim/lib/logs/list-logs.ts b/apps/sim/lib/logs/list-logs.ts index 074cdd4753b..2fd1baa5acb 100644 --- a/apps/sim/lib/logs/list-logs.ts +++ b/apps/sim/lib/logs/list-logs.ts @@ -39,18 +39,35 @@ import { encodeLogSortCursor, } from '@/lib/logs/sort-cursor' +/** What a caller asks for: the contract's query, plus cancellation. */ export type ListLogsParams = z.output & { signal?: AbortSignal } +/** + * What the query actually runs with — the request, plus the viewer's spend + * projection. + * + * `hideCostInfo` is required and lives on this type rather than on + * {@link ListLogsParams} for two reasons that pull the same way. It is resolved + * by the application use case and never read off the query, so a client cannot + * ask for a row it is not entitled to; and being required rather than defaulted + * to `false`, a caller of this read that forgets it fails to compile instead of + * quietly disclosing every run's cost. + */ +export type ReadLogsParams = ListLogsParams & { + hideCostInfo: boolean +} + type SortBy = 'date' | 'duration' | 'cost' | 'status' type SortOrder = 'asc' | 'desc' /** * Canonical logs list query after workspace authorization. */ -export async function readLogs(params: ListLogsParams): Promise { +export async function readLogs(params: ReadLogsParams): Promise { params.signal?.throwIfAborted() + const { hideCostInfo } = params const sortBy = params.sortBy as SortBy const sortOrder = params.sortOrder as SortOrder const cursor = params.cursor ? decodeLogSortCursor(params.cursor) : null @@ -61,7 +78,7 @@ export async function readLogs(params: ListLogsParams): Promise = (() => { switch (sortBy) { @@ -185,7 +202,6 @@ export async function readLogs(params: ListLogsParams): Promise { + if (!viewerUserId) return NO_LOG_FIELD_PROJECTION + + const config = await resolvePermissionGroupConfig(viewerUserId, workspaceId, organizationId) + return { + hideTraceSpans: capabilityDeniedBy('logs.trace_spans', config), + hideCostInfo: capabilityDeniedBy('logs.cost', config), + } +} + +/** + * Applies {@link LogFieldProjection} to a materialized execution payload. + * + * Both halves DELETE the withheld fields rather than leaving them for response + * validation to drop, because the log contracts are passthrough (and a span's + * own shape is a `catchall`), so a field left in place would survive the schema. + */ +export function projectExecutionData | null | undefined>( + executionData: T, + projection: LogFieldProjection +): T | Record { + if (!executionData) return executionData + const withoutPayloads = projection.hideTraceSpans + ? withheldExecutionData(executionData) + : executionData + return projection.hideCostInfo ? withheldSpendData(withoutPayloads) : withoutPayloads +} + +/** The run's cost total, or `null` when the group withholds spend. */ +export function projectCostTotal( + costTotal: unknown, + projection: LogFieldProjection +): { total: number } | null { + if (projection.hideCostInfo || costTotal == null) return null + return { total: Number(costTotal) } +} + +/** + * The spend-selecting halves of a log query, in the spellings the surfaces use. + * + * The first-party list spells its filter as an operator plus a value; the public + * adapters spell theirs as a `minCost`/`maxCost` pair. Both are read here so the + * rule lives once — a second copy is how one of them stops refusing. + */ +export interface LogCostQuerySurface { + sortBy?: string | null + costOperator?: string | null + costValue?: number | null + minCost?: number | null + maxCost?: number | null +} + +/** Whether the query orders or selects rows by run spend. */ +export function logQuerySelectsCost(query: LogCostQuerySurface): boolean { + if (query.sortBy === 'cost') return true + if (query.costOperator && query.costValue != null) return true + return query.minCost != null || query.maxCost != null +} + +/** + * Refuses a cost-ordered or cost-filtered query from a viewer whose group + * withholds spend. + * + * Withholding the *field* is not enough on its own: `cost > X` answered + * faithfully is an oracle, and a caller who can repeat it recovers every run's + * cost by bisection — with `includeTotal` they do not even have to read the + * rows. Ordering leaks the same thing more slowly, as a ranking. + * + * Refused rather than silently ignored. Dropping the clause would answer a + * question nobody asked — a list of every run under a `cost > 5` chip, in an + * order the caller did not request — and a wrong answer presented as the right + * one is worse than a refusal. The refusal discloses nothing new either: the + * workspace role check has already passed by the time this runs, so the caller + * is a member being told about their own group, not an outsider being handed an + * organization-configuration oracle. + * + * `logs.trace_spans` needs no counterpart. Nothing the trace projection + * withholds — `traceSpans`, `blockExecutions`, `finalOutput`, `workflowInput`, + * `blockInput` — is filterable or sortable on any log surface: `search` matches + * the execution id alone, and every sort key is a scalar column. + * + * permission-group-enforced: logs.cost + */ +export function assertLogCostQueryAllowed( + query: LogCostQuerySurface, + projection: Pick +): void { + if (!projection.hideCostInfo) return + if (!logQuerySelectsCost(query)) return + refuseCapability('logs.cost') +} diff --git a/apps/sim/lib/logs/public-queries.ts b/apps/sim/lib/logs/public-queries.ts index 98d70fc6af4..291f23fc098 100644 --- a/apps/sim/lib/logs/public-queries.ts +++ b/apps/sim/lib/logs/public-queries.ts @@ -9,6 +9,7 @@ import { workflowExecutionSnapshots, } from '@sim/db/schema' import { and, type Column, eq, sql } from 'drizzle-orm' +import { alias } from 'drizzle-orm/pg-core' import { type CursorKey, decimalKey, @@ -32,6 +33,9 @@ import { type LogFilters, } from '@/lib/logs/public-filters' +/** Distinguishes the workflow-owner join from the execution-actor join on `user`. */ +const workflowOwner = alias(user, 'workflow_owner') + export interface PublicLogCursor { startedAt: string id: string @@ -115,7 +119,6 @@ function workflowLogQuery(includeExecutionData: boolean) { workflowName: workflow.name, workflowDescription: workflow.description, workflowFolderId: workflow.folderId, - workflowUserId: workflow.userId, workflowWorkspaceId: workflow.workspaceId, workflowCreatedAt: workflow.createdAt, workflowUpdatedAt: workflow.updatedAt, @@ -476,8 +479,8 @@ export async function getPublicWorkflowLog(lookup: PublicWorkflowLogLookup, work workflowName: workflow.name, workflowDescription: workflow.description, workflowFolderId: workflow.folderId, - workflowUserId: workflow.userId, - workflowOwnerEmail: user.email, + executedByEmail: user.email, + workflowOwnerEmail: workflowOwner.email, workflowWorkspaceId: workflow.workspaceId, workflowCreatedAt: workflow.createdAt, workflowUpdatedAt: workflow.updatedAt, @@ -500,7 +503,29 @@ export async function getPublicWorkflowLog(lookup: PublicWorkflowLogLookup, work ) .leftJoin(pausedExecutions, eq(pausedExecutions.executionId, workflowExecutionLogs.executionId)) .leftJoin(workflow, eq(workflowExecutionLogs.workflowId, workflow.id)) - .leftJoin(user, eq(workflow.userId, user.id)) + /** + * The identity the run ACTED as, read from the attribution the run itself + * captured — not the workflow owner, who contributes nothing to a run beyond + * a personal-variable fallback and whose row can be reassigned long after the + * fact. Joining the immutable per-run actor keeps a historical log honest + * about who it ran as. + * + * Null on a run that failed before an actor was resolved (a webhook rejected + * during setup), which is the truthful answer for those: there was no actor. + */ + .leftJoin( + user, + eq(sql`${workflowExecutionLogs.executionData}->'billingAttribution'->>'actorUserId'`, user.id) + ) + /** + * Kept only to serve the deprecated `workflow.ownerEmail`, which was a + * required field of the published v2 log schema before `executedByEmail` + * replaced it. Removing it outright would break typed clients, so it stays + * until that field does. Aliased because the actor join above already holds + * `user` — the two identities coincide on an interactive run and diverge on + * every background one, which is the whole reason the field was replaced. + */ + .leftJoin(workflowOwner, eq(workflow.userId, workflowOwner.id)) .where( and( lookupCondition, diff --git a/apps/sim/lib/logs/types.ts b/apps/sim/lib/logs/types.ts index c351a14268d..80d9f7542a1 100644 --- a/apps/sim/lib/logs/types.ts +++ b/apps/sim/lib/logs/types.ts @@ -1,4 +1,4 @@ -import type { Edge } from 'reactflow' +import type { Edge } from '@xyflow/react' import type { BillingAttributionSnapshot } from '@/lib/billing/core/billing-attribution' import type { AsyncExecutionCorrelation } from '@/lib/core/async-jobs/types' import type { ParentIteration, SerializableExecutionState } from '@/executor/execution/types' diff --git a/apps/sim/lib/managed-agents/session-client.test.ts b/apps/sim/lib/managed-agents/session-client.test.ts index 4cf6184c9d5..9551989b448 100644 --- a/apps/sim/lib/managed-agents/session-client.test.ts +++ b/apps/sim/lib/managed-agents/session-client.test.ts @@ -8,6 +8,7 @@ import { deleteSession, listSessionEvents, listSessionEventsPage, + managedAgentsList, parseSessionSnapshot, resolvePendingToolGates, sendCustomToolResults, @@ -201,6 +202,94 @@ describe('listSessionEvents — ordering', () => { }) }) +describe('managedAgentsList — selector collection bounds', () => { + const originalFetch = global.fetch + afterEach(() => { + global.fetch = originalFetch + }) + + it('cancels a non-success response and conceals its provider body', async () => { + let cancelled = false + const stream = new ReadableStream({ + cancel() { + cancelled = true + }, + }) + global.fetch = vi.fn( + async () => new Response(stream, { status: 500, statusText: 'provider failure' }) + ) as unknown as typeof fetch + + await expect(managedAgentsList({ apiKey: 'sk-ant-fake', path: '/v1/agents' })).rejects.toThrow( + 'Managed Agents collection request failed' + ) + expect(cancelled).toBe(true) + }) + + it('cancels a declared oversized collection response before reading it', async () => { + let cancelled = false + const stream = new ReadableStream({ + cancel() { + cancelled = true + }, + }) + global.fetch = vi.fn( + async () => + new Response(stream, { + headers: { 'content-length': String(16 * 1024 * 1024 + 1) }, + }) + ) as unknown as typeof fetch + + await expect(managedAgentsList({ apiKey: 'sk-ant-fake', path: '/v1/agents' })).rejects.toThrow( + 'Managed Agents collection response is unavailable' + ) + expect(cancelled).toBe(true) + }) + + it('enforces the response-byte budget across the whole paginated collection', async () => { + const firstBody = `${JSON.stringify({ data: [{ id: 'agent-1' }], next_page: 'page-2' })}${' '.repeat(8 * 1024 * 1024)}` + let secondCancelled = false + const secondStream = new ReadableStream({ + cancel() { + secondCancelled = true + }, + }) + global.fetch = vi + .fn() + .mockResolvedValueOnce(new Response(firstBody)) + .mockResolvedValueOnce( + new Response(secondStream, { + headers: { 'content-length': String(9 * 1024 * 1024) }, + }) + ) as unknown as typeof fetch + + await expect(managedAgentsList({ apiKey: 'sk-ant-fake', path: '/v1/agents' })).rejects.toThrow( + 'Managed Agents collection response is unavailable' + ) + expect(global.fetch).toHaveBeenCalledTimes(2) + expect(secondCancelled).toBe(true) + }) + + it('preserves a caller cancellation instead of mapping it to a collection failure', async () => { + const controller = new AbortController() + const cancelled = new Error('caller cancelled') + global.fetch = vi.fn( + (_url: string, init?: RequestInit) => + new Promise((_resolve, reject) => { + init?.signal?.addEventListener('abort', () => reject(cancelled), { once: true }) + }) + ) as unknown as typeof fetch + + const result = managedAgentsList({ + apiKey: 'sk-ant-fake', + path: '/v1/agents', + signal: controller.signal, + }) + controller.abort() + + await expect(result).rejects.toBe(cancelled) + }) +}) + describe('buildSessionCreatePayload — initial_events', () => { it('seeds a single user.message so create+send is one call', () => { const payload = buildSessionCreatePayload({ ...BASE, initialMessage: 'hello there' }) diff --git a/apps/sim/lib/managed-agents/session-client.ts b/apps/sim/lib/managed-agents/session-client.ts index 388aab8e4a6..f7166ae847e 100644 --- a/apps/sim/lib/managed-agents/session-client.ts +++ b/apps/sim/lib/managed-agents/session-client.ts @@ -494,6 +494,99 @@ interface AnthropicListPage { */ const MAX_LIST_PAGES = 1000 +/** + * Block-editor collection reads are a separate trust boundary from runtime + * session history. Keep their provider egress bounded without changing the + * exhaustive event reads used by the run loop. + */ +const SELECTOR_LIST_TIMEOUT_MS = 30_000 +const MAX_SELECTOR_LIST_TOTAL_BYTES = 16 * 1024 * 1024 +const MAX_SELECTOR_LIST_PAGES = 100 + +async function cancelResponseBody(response: Response): Promise { + try { + await response.body?.cancel() + } catch { + // Best effort: cancellation must not replace the concealed provider error. + } +} + +async function readBoundedSelectorListJson( + response: Response, + maxBytes: number +): Promise<{ value: T; bytesRead: number }> { + const declaredLength = Number(response.headers.get('content-length')) + if (Number.isFinite(declaredLength) && declaredLength > maxBytes) { + await cancelResponseBody(response) + throw new Error('Managed Agents collection response is unavailable') + } + if (!response.body) throw new Error('Managed Agents collection response is unavailable') + + const reader = response.body.getReader() + const chunks: Uint8Array[] = [] + let total = 0 + while (true) { + const { done, value } = await reader.read() + if (done) break + total += value.byteLength + if (total > maxBytes) { + await reader.cancel().catch(() => undefined) + throw new Error('Managed Agents collection response is unavailable') + } + chunks.push(value) + } + + const bytes = new Uint8Array(total) + let offset = 0 + for (const chunk of chunks) { + bytes.set(chunk, offset) + offset += chunk.byteLength + } + return { value: JSON.parse(new TextDecoder().decode(bytes)) as T, bytesRead: total } +} + +async function fetchSelectorListPage( + input: SessionAuth & { + path: string + beta?: string + page: string | null + maxResponseBytes: number + } +): Promise<{ page: AnthropicListPage; bytesRead: number }> { + const url = new URL(`${ANTHROPIC_API_BASE}${input.path}`) + url.searchParams.set('limit', '100') + if (input.page) url.searchParams.set('page', input.page) + + let response: Response + try { + response = await fetch(url.toString(), { + method: 'GET', + headers: managedAgentsHeaders(input.apiKey, { beta: input.beta }), + redirect: 'error', + signal: input.signal, + }) + } catch (error) { + if (input.signal?.aborted) throw error + throw new Error('Managed Agents collection request failed') + } + + if (!response.ok) { + await cancelResponseBody(response) + throw new Error('Managed Agents collection request failed') + } + + try { + const result = await readBoundedSelectorListJson>( + response, + input.maxResponseBytes + ) + return { page: result.value, bytesRead: result.bytesRead } + } catch (error) { + if (input.signal?.aborted) throw error + throw new Error('Managed Agents collection response is unavailable') + } +} + async function listPaginated( input: SessionAuth & { path: string @@ -531,6 +624,7 @@ async function listPaginated( const resp = await fetch(url.toString(), { method: 'GET', headers: managedAgentsHeaders(input.apiKey, { beta: input.beta }), + redirect: 'error', signal: input.signal, }) if (!resp.ok) { @@ -648,12 +742,31 @@ function parseProcessedAt(value: string | null | undefined): number { export async function managedAgentsList( input: SessionAuth & { path: string; beta?: string } ): Promise { - return listPaginated({ - apiKey: input.apiKey, - signal: input.signal, - path: input.path, - beta: input.beta, - }) + const collected: T[] = [] + let page: string | null = null + let remainingBytes = MAX_SELECTOR_LIST_TOTAL_BYTES + const timeoutSignal = AbortSignal.timeout(SELECTOR_LIST_TIMEOUT_MS) + const signal = input.signal ? AbortSignal.any([input.signal, timeoutSignal]) : timeoutSignal + for ( + let pageCount = 0; + pageCount < MAX_SELECTOR_LIST_PAGES && collected.length < 2000; + pageCount++ + ) { + const result: { page: AnthropicListPage; bytesRead: number } = + await fetchSelectorListPage({ + ...input, + page, + signal, + maxResponseBytes: remainingBytes, + }) + remainingBytes -= result.bytesRead + const pageBody: AnthropicListPage = result.page + const items = Array.isArray(pageBody.data) ? pageBody.data : [] + collected.push(...items) + if (!pageBody.next_page || items.length === 0) break + page = pageBody.next_page + } + return collected.length > 2000 ? collected.slice(0, 2000) : collected } /** diff --git a/apps/sim/lib/mcp/application/operations.test.ts b/apps/sim/lib/mcp/application/operations.test.ts index 75218e8f402..f145601905f 100644 --- a/apps/sim/lib/mcp/application/operations.test.ts +++ b/apps/sim/lib/mcp/application/operations.test.ts @@ -1,8 +1,26 @@ /** * @vitest-environment node */ -import { describe, expect, it } from 'vitest' +import { permissionGroupScopeMock, permissionGroupScopeMockFns } from '@sim/testing' +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const mocks = vi.hoisted(() => ({ + resolvePermission: vi.fn(), +})) + +const resolveGroupConfigMock = permissionGroupScopeMockFns.mockResolvePermissionGroupConfig + +vi.mock('@sim/platform-authz/workspace', () => ({ + permissionSatisfies: () => true, + resolveEffectiveWorkspacePermission: mocks.resolvePermission, +})) + +vi.mock('@/lib/permission-groups/config-scope.server', () => permissionGroupScopeMock) + +import type { WorkspaceOperation } from '@/lib/core/application' +import { authorizeWorkspaceOperation, PermissionGroupCapabilityError } from '@/lib/core/application' import { mcpServerOperations } from '@/lib/mcp/application/operations' +import { DEFAULT_PERMISSION_GROUP_CONFIG } from '@/lib/permission-groups/fields' describe('MCP server operation registry', () => { it('requires a human subject for tool discovery', () => { @@ -103,3 +121,118 @@ describe('MCP server operation registry', () => { expect(new Set(ids).size).toBe(ids.length) }) }) + +const sessionPrincipal = { kind: 'session', userId: 'user-1', sessionId: 'session-1' } as const +const context = { + workspaceId: 'workspace-1', + workspaceOrganizationId: 'organization-1', + allowPersonalApiKeys: true, +} + +/** + * Every operation's capability, by name. + * + * Pinned as a whole map rather than derived from the registry, because the + * refusal tests below select their subjects with `operation.capability === x` — + * a filter over the very field under test. Dropping the capability from one + * operation would simply remove it from that filter and leave those tests green + * (`mcp_servers.create`, which registers a server and stores its credentials + * against the workspace, was verified to do exactly that). This map fails + * instead. + */ +const EXPECTED_CAPABILITIES: Record = { + list: 'mcp_tools.use', + read: 'mcp_tools.use', + create: 'mcp_tools.use', + register: 'mcp_tools.use', + update: 'mcp_tools.use', + reconfigure: 'mcp_tools.use', + delete: 'mcp_tools.use', + discoverTools: 'mcp_tools.use', + executeTool: 'mcp_tools.use', + listWorkflowDeployments: 'deploy.mcp', + readWorkflowDeploymentServer: 'deploy.mcp', + listWorkflowDeploymentTools: 'deploy.mcp', + createWorkflowDeploymentServer: 'deploy.mcp', + updateWorkflowDeploymentServer: 'deploy.mcp', + deleteWorkflowDeploymentServer: 'deploy.mcp', + deployWorkflowTool: 'deploy.mcp', + undeployWorkflowTool: 'deploy.mcp', +} + +describe('MCP operation capability declarations', () => { + it('declares a capability on every operation, by name', () => { + const declared = Object.fromEntries( + Object.entries(mcpServerOperations).map(([key, operation]) => [key, operation.capability]) + ) + + expect(declared).toEqual(EXPECTED_CAPABILITIES) + }) +}) + +/** `tools.execute` admits only the executor delegation, so a session cannot stand in for it. */ +function sessionReachable(capability: string) { + return Object.values(mcpServerOperations).filter( + (operation) => + operation.capability === capability && operation.principalKinds.includes('session') + ) +} + +/** + * The declaration is only half the gate; these prove the funnel actually + * refuses, so a capability could not be renamed into one nothing reads. + */ +describe('MCP operations under a withholding permission group', () => { + beforeEach(() => { + vi.clearAllMocks() + mocks.resolvePermission.mockResolvedValue('admin') + }) + + it('refuses every mcp_servers operation when the group blocks MCP tools', async () => { + resolveGroupConfigMock.mockResolvedValue({ + ...DEFAULT_PERMISSION_GROUP_CONFIG, + disableMcpTools: true, + }) + + const registryOperations = sessionReachable('mcp_tools.use') + expect(registryOperations.length).toBeGreaterThan(0) + + for (const operation of registryOperations) { + await expect( + authorizeWorkspaceOperation(sessionPrincipal, operation as WorkspaceOperation, context), + operation.id + ).rejects.toBeInstanceOf(PermissionGroupCapabilityError) + } + }) + + it('refuses every workflow-deployment operation when the group hides MCP deployment', async () => { + resolveGroupConfigMock.mockResolvedValue({ + ...DEFAULT_PERMISSION_GROUP_CONFIG, + hideDeployMcp: true, + }) + + const deploymentOperations = sessionReachable('deploy.mcp') + expect(deploymentOperations.length).toBeGreaterThan(0) + + for (const operation of deploymentOperations) { + await expect( + authorizeWorkspaceOperation(sessionPrincipal, operation as WorkspaceOperation, context), + operation.id + ).rejects.toBeInstanceOf(PermissionGroupCapabilityError) + } + }) + + it('allows the same operations when the group withholds neither', async () => { + resolveGroupConfigMock.mockResolvedValue(DEFAULT_PERMISSION_GROUP_CONFIG) + + for (const operation of [ + ...sessionReachable('mcp_tools.use'), + ...sessionReachable('deploy.mcp'), + ]) { + await expect( + authorizeWorkspaceOperation(sessionPrincipal, operation as WorkspaceOperation, context), + operation.id + ).resolves.toBeUndefined() + } + }) +}) diff --git a/apps/sim/lib/mcp/application/operations.ts b/apps/sim/lib/mcp/application/operations.ts index dc4ed4d2335..e5cba22f155 100644 --- a/apps/sim/lib/mcp/application/operations.ts +++ b/apps/sim/lib/mcp/application/operations.ts @@ -17,23 +17,40 @@ const EXECUTION_PRINCIPAL_POLICY = { delegatedServices: ['executor'], } as const +/** + * Two capabilities, because the family covers two different things. + * + * `mcp_servers.*` is the workspace's registry of external MCP servers — the + * connections an agent calls tools through — so every one of them declares + * `mcp_tools.use`. Gating only `tools.execute` would leave a group that blocks + * MCP tools able to keep registering servers and storing their credentials + * against the workspace, which is the accumulation the key exists to stop. + * + * `mcp_servers.workflow_deployments.*` is the opposite direction: publishing a + * workflow *as* an MCP server, which is what `hideDeployMcp` names. Reads carry + * `deploy.mcp` alongside the writes, so a group that withholds the deployment + * surface does not still answer with what is published on it. + */ export const mcpServerOperations = { list: defineWorkspaceOperation({ id: 'mcp_servers.list', minimumRole: 'read', workspaceApiKey: 'allow', + capability: 'mcp_tools.use', ...ALL_PRINCIPAL_POLICY, }), discoverTools: defineWorkspaceOperation({ id: 'mcp_servers.tools.discover', minimumRole: 'read', workspaceApiKey: 'deny', + capability: 'mcp_tools.use', ...DISCOVERY_PRINCIPAL_POLICY, }), executeTool: defineWorkspaceOperation({ id: 'mcp_servers.tools.execute', minimumRole: 'read', workspaceApiKey: 'deny', + capability: 'mcp_tools.use', ...EXECUTION_PRINCIPAL_POLICY, }), /** @@ -56,6 +73,7 @@ export const mcpServerOperations = { id: 'mcp_servers.workflow_deployments.list', minimumRole: 'read', workspaceApiKey: 'deny', + capability: 'deploy.mcp', ...HUMAN_PRINCIPAL_POLICY, }), /** @@ -70,18 +88,21 @@ export const mcpServerOperations = { id: 'mcp_servers.workflow_deployments.read_server', minimumRole: 'read', workspaceApiKey: 'deny', + capability: 'deploy.mcp', ...HUMAN_PRINCIPAL_POLICY, }), listWorkflowDeploymentTools: defineWorkspaceOperation({ id: 'mcp_servers.workflow_deployments.list_tools', minimumRole: 'read', workspaceApiKey: 'deny', + capability: 'deploy.mcp', ...HUMAN_PRINCIPAL_POLICY, }), createWorkflowDeploymentServer: defineWorkspaceOperation({ id: 'mcp_servers.workflow_deployments.create_server', minimumRole: 'admin', workspaceApiKey: 'deny', + capability: 'deploy.mcp', ...HUMAN_PRINCIPAL_POLICY, }), /** @@ -101,60 +122,70 @@ export const mcpServerOperations = { id: 'mcp_servers.workflow_deployments.update_server', minimumRole: 'admin', workspaceApiKey: 'deny', + capability: 'deploy.mcp', ...HUMAN_PRINCIPAL_POLICY, }), deleteWorkflowDeploymentServer: defineWorkspaceOperation({ id: 'mcp_servers.workflow_deployments.delete_server', minimumRole: 'admin', workspaceApiKey: 'deny', + capability: 'deploy.mcp', ...HUMAN_PRINCIPAL_POLICY, }), deployWorkflowTool: defineWorkspaceOperation({ id: 'mcp_servers.workflow_deployments.deploy_tool', minimumRole: 'admin', workspaceApiKey: 'deny', + capability: 'deploy.mcp', ...HUMAN_PRINCIPAL_POLICY, }), undeployWorkflowTool: defineWorkspaceOperation({ id: 'mcp_servers.workflow_deployments.undeploy_tool', minimumRole: 'admin', workspaceApiKey: 'deny', + capability: 'deploy.mcp', ...HUMAN_PRINCIPAL_POLICY, }), read: defineWorkspaceOperation({ id: 'mcp_servers.read', minimumRole: 'read', workspaceApiKey: 'allow', + capability: 'mcp_tools.use', ...ALL_PRINCIPAL_POLICY, }), create: defineWorkspaceOperation({ id: 'mcp_servers.create', minimumRole: 'write', workspaceApiKey: 'allow', + capability: 'mcp_tools.use', ...ALL_PRINCIPAL_POLICY, }), register: defineWorkspaceOperation({ id: 'mcp_servers.register', minimumRole: 'write', workspaceApiKey: 'allow', + capability: 'mcp_tools.use', ...ALL_PRINCIPAL_POLICY, }), update: defineWorkspaceOperation({ id: 'mcp_servers.update', minimumRole: 'write', workspaceApiKey: 'allow', + capability: 'mcp_tools.use', ...ALL_PRINCIPAL_POLICY, }), reconfigure: defineWorkspaceOperation({ id: 'mcp_servers.reconfigure', minimumRole: 'write', workspaceApiKey: 'allow', + capability: 'mcp_tools.use', ...ALL_PRINCIPAL_POLICY, }), delete: defineWorkspaceOperation({ id: 'mcp_servers.delete', minimumRole: 'write', workspaceApiKey: 'allow', + capability: 'mcp_tools.use', ...ALL_PRINCIPAL_POLICY, }), } as const diff --git a/apps/sim/lib/mcp/client.ts b/apps/sim/lib/mcp/client.ts index 5aefff7fc08..c9b2607feac 100644 --- a/apps/sim/lib/mcp/client.ts +++ b/apps/sim/lib/mcp/client.ts @@ -104,15 +104,15 @@ export class McpClient { throw new McpError('OAuth MCP server requires an authProvider') } const useOauth = this.config.authType === 'oauth' - // `resolvedIP` non-null signals the SSRF policy is active for this server (it is null in - // allowlist mode / localhost-on-self-hosted); the guard validates addresses per-connect. - // A private/loopback resolvedIP only reaches here on self-hosted (where the policy - // permits it) — the guarded lookup would filter it, so that case keeps the legacy pin - // to the validated address (old behavior + its anti-rebinding property). + // `resolvedIP` is null only when the hostname still carries an unresolved env-var + // reference, which is checked again once it resolves. Otherwise the guard validates + // addresses per-connect. A private/loopback resolvedIP only reaches here on a + // self-hosted deployment whose policy permits it, and that case pins to the address + // that was validated rather than to whatever the name resolves to next. const guarded = resolvedIP ? isPrivateIp(resolvedIP) - ? createPinnedPrivateMcpFetch(resolvedIP) - : createGuardedMcpFetch() + ? createPinnedPrivateMcpFetch(resolvedIP, this.config.url) + : createGuardedMcpFetch(this.config.url) : undefined this.closeGuardedTransport = guarded?.close this.transport = new StreamableHTTPClientTransport(new URL(this.config.url), { diff --git a/apps/sim/lib/mcp/domain-check.test.ts b/apps/sim/lib/mcp/domain-check.test.ts index aca421d7127..f68f9de7b0c 100644 --- a/apps/sim/lib/mcp/domain-check.test.ts +++ b/apps/sim/lib/mcp/domain-check.test.ts @@ -18,9 +18,11 @@ vi.mock('@/executor/utils/reference-validation', () => ({ import { isMcpDomainAllowed, + MCP_EGRESS_PROFILE, McpDnsResolutionError, McpDomainNotAllowedError, McpSsrfError, + OAUTH_EGRESS_PROFILE, validateMcpDomain, validateMcpServerSsrf, } from './domain-check' @@ -334,13 +336,13 @@ describe('validateMcpServerSsrf', () => { expect(mockDnsLookup).not.toHaveBeenCalled() }) - it('returns null for localhost URLs without DNS lookup', async () => { - await expect(validateMcpServerSsrf('http://localhost:3000/mcp')).resolves.toBeNull() - expect(mockDnsLookup).not.toHaveBeenCalled() + it('pins a localhost URL rather than leaving it unguarded', async () => { + mockDnsLookup.mockResolvedValue([{ address: '127.0.0.1', family: 4 }]) + await expect(validateMcpServerSsrf('http://localhost:3000/mcp')).resolves.toBe('127.0.0.1') }) - it('returns null for 127.0.0.1 literal without DNS lookup', async () => { - await expect(validateMcpServerSsrf('http://127.0.0.1:8080/mcp')).resolves.toBeNull() + it('pins a loopback literal without a DNS lookup', async () => { + await expect(validateMcpServerSsrf('http://127.0.0.1:8080/mcp')).resolves.toBe('127.0.0.1') expect(mockDnsLookup).not.toHaveBeenCalled() }) @@ -423,9 +425,22 @@ describe('validateMcpServerSsrf', () => { ) }) - it('returns resolved IP for URLs resolving to loopback on self-hosted (localhost alias)', async () => { + it('refuses a DNS alias that resolves to loopback unless it is allowlisted', async () => { + // The loopback carve-out keys off the hostname, so a name pointed at + // loopback is named in EGRESS_ALLOWED_HOSTS or it is not reachable. mockDnsLookup.mockResolvedValue([{ address: '127.0.0.1', family: 4 }]) - await expect(validateMcpServerSsrf('http://my-local-alias:3000/mcp')).resolves.toBe('127.0.0.1') + await expect(validateMcpServerSsrf('http://my-local-alias:3000/mcp')).rejects.toThrow( + McpSsrfError + ) + + setEnvFlags({ egressAllowedHosts: 'my-local-alias' }) + try { + await expect(validateMcpServerSsrf('http://my-local-alias:3000/mcp')).resolves.toBe( + '127.0.0.1' + ) + } finally { + setEnvFlags({ egressAllowedHosts: undefined }) + } }) it('throws for malformed URLs', async () => { @@ -462,13 +477,24 @@ describe('validateMcpServerSsrf', () => { }) it('pins public IP literals on hosted so redirects cannot escape', async () => { - await expect(validateMcpServerSsrf('http://93.184.216.34/mcp')).resolves.toBe('93.184.216.34') + await expect(validateMcpServerSsrf('https://93.184.216.34/mcp')).resolves.toBe( + '93.184.216.34' + ) expect(mockDnsLookup).not.toHaveBeenCalled() }) - it('skips loopback check on hosted when allowlist is configured', async () => { + it('refuses plain HTTP on hosted, where a credential would cross the wire in the clear', async () => { + await expect(validateMcpServerSsrf('http://93.184.216.34/mcp')).rejects.toThrow( + /must use https/ + ) + }) + + it('still refuses loopback on hosted when a domain allowlist is configured', async () => { + // The domain allowlist governs which domains may be used. It is not a + // substitute for the address check, which it used to disable entirely. mockGetAllowedMcpDomainsFromEnv.mockReturnValue(['localhost']) - await expect(validateMcpServerSsrf('http://localhost:3000/mcp')).resolves.toBeNull() + mockDnsLookup.mockResolvedValue([{ address: '127.0.0.1', family: 4 }]) + await expect(validateMcpServerSsrf('http://localhost:3000/mcp')).rejects.toThrow(McpSsrfError) }) it('still blocks RFC-1918 IP literals on hosted (regression)', async () => { @@ -499,26 +525,61 @@ describe('validateMcpServerSsrf', () => { setEnvFlags({ isHosted: false }) }) - it('still allows localhost URLs (returns null, no pinning needed)', async () => { - await expect(validateMcpServerSsrf('http://localhost:3000/mcp')).resolves.toBeNull() - }) - - it('still allows 127.0.0.1 URLs (returns null, no pinning needed)', async () => { - await expect(validateMcpServerSsrf('http://127.0.0.1:8080/mcp')).resolves.toBeNull() + it('still reaches a local MCP server, now pinned rather than unguarded', async () => { + mockDnsLookup.mockResolvedValue([{ address: '127.0.0.1', family: 4 }]) + await expect(validateMcpServerSsrf('http://localhost:3000/mcp')).resolves.toBe('127.0.0.1') + await expect(validateMcpServerSsrf('http://127.0.0.1:8080/mcp')).resolves.toBe('127.0.0.1') }) - it('returns resolved loopback IP for DNS aliases (caller pins)', async () => { - mockDnsLookup.mockResolvedValue([{ address: '127.0.0.1', family: 4 }]) - await expect(validateMcpServerSsrf('http://my-local-alias/mcp')).resolves.toBe('127.0.0.1') + it('reaches a private MCP server once the operator allowlists it', async () => { + setEnvFlags({ egressAllowedIpRanges: '10.0.0.0/8' }) + try { + await expect(validateMcpServerSsrf('http://10.0.0.9:3000/mcp')).resolves.toBe('10.0.0.9') + } finally { + setEnvFlags({ egressAllowedIpRanges: undefined }) + } }) }) - it('skips all checks when ALLOWED_MCP_DOMAINS is configured', async () => { + it('applies the address check even when ALLOWED_MCP_DOMAINS is configured', async () => { + // Configuring the domain list used to disable this entirely, which left an + // allowlisted domain free to redirect at anything, metadata included. mockGetAllowedMcpDomainsFromEnv.mockReturnValue(['internal.corp']) - await expect(validateMcpServerSsrf('http://10.0.0.1/mcp')).resolves.toBeNull() + await expect(validateMcpServerSsrf('http://10.0.0.1/mcp')).rejects.toThrow(McpSsrfError) + await expect(validateMcpServerSsrf('http://169.254.169.254/latest/meta-data/')).rejects.toThrow( + McpSsrfError + ) + }) +}) + +describe('the OAuth provenance', () => { + beforeEach(() => { + setEnvFlags({ isHosted: false }) + }) + + it('is contentFetch, so a hop the metadata names inherits nothing from the server', () => { + expect(OAUTH_EGRESS_PROFILE).toBe('contentFetch') + expect(MCP_EGRESS_PROFILE).toBe('selfHostedService') + }) + + it('refuses loopback that the configured-server provenance reaches', async () => { + mockDnsLookup.mockResolvedValue([{ address: '127.0.0.1', family: 4 }]) + await expect(validateMcpServerSsrf('http://localhost:3000/mcp')).resolves.toBe('127.0.0.1') await expect( - validateMcpServerSsrf('http://169.254.169.254/latest/meta-data/') - ).resolves.toBeNull() - expect(mockDnsLookup).not.toHaveBeenCalled() + validateMcpServerSsrf('http://localhost:3000/token', OAUTH_EGRESS_PROFILE) + ).rejects.toThrow(McpSsrfError) + }) + + it('ignores the operator allowlist that the configured-server provenance honors', async () => { + setEnvFlags({ egressAllowedIpRanges: '10.0.0.0/8' }) + try { + mockDnsLookup.mockResolvedValue([{ address: '10.0.0.9', family: 4 }]) + await expect(validateMcpServerSsrf('https://mcp.corp/mcp')).resolves.toBe('10.0.0.9') + await expect( + validateMcpServerSsrf('https://idp.corp/token', OAUTH_EGRESS_PROFILE) + ).rejects.toThrow(McpSsrfError) + } finally { + setEnvFlags({ egressAllowedIpRanges: undefined }) + } }) }) diff --git a/apps/sim/lib/mcp/domain-check.ts b/apps/sim/lib/mcp/domain-check.ts index 6a39f0a8aa2..8599e8ed6c6 100644 --- a/apps/sim/lib/mcp/domain-check.ts +++ b/apps/sim/lib/mcp/domain-check.ts @@ -1,12 +1,29 @@ import { createLogger } from '@sim/logger' -import { resolveHostAddresses } from '@sim/security/dns' -import { isIpLiteral, isLoopbackIp, isPrivateIp, unwrapIpv6Brackets } from '@sim/security/ssrf' -import { toError } from '@sim/utils/errors' -import { getAllowedMcpDomainsFromEnv, isHosted } from '@/lib/core/config/env-flags' +import { getAllowedMcpDomainsFromEnv } from '@/lib/core/config/env-flags' +import type { EgressProfile } from '@/lib/core/security/egress/profiles' +import { validateUrlWithDNS } from '@/lib/core/security/input-validation.server' import { createEnvVarPattern } from '@/executor/utils/reference-validation' const logger = createLogger('McpDomainCheck') +/** + * An MCP server URL is a configured endpoint for software commonly self-hosted: + * plain HTTP and an arbitrary port are ordinary, and reaching one on a private + * address is a matter of the operator naming it in the egress allowlist. + */ +export const MCP_EGRESS_PROFILE: EgressProfile = 'selfHostedService' + +/** + * Profile for an MCP OAuth leg — discovery, registration, token exchange, + * revocation. + * + * Every hop after the first takes its URL from authorization-server metadata, + * which the remote server controls. Treating those as configured endpoints would + * let a hostile server steer a leg at whatever the operator allowlisted for their + * own workflows, so they get the provenance they actually have. + */ +export const OAUTH_EGRESS_PROFILE: EgressProfile = 'contentFetch' + export class McpDomainNotAllowedError extends Error { constructor(domain: string) { super(`MCP server domain "${domain}" is not allowed by the server's ALLOWED_MCP_DOMAINS policy`) @@ -98,111 +115,47 @@ export function validateMcpDomain(url: string | undefined): void { } /** - * Returns true if the hostname is localhost or a loopback IP literal (full - * 127.0.0.0/8 range, or ::1). Expects IPv6 brackets to already be stripped. - */ -function isLocalhostHostname(hostname: string): boolean { - const clean = hostname.toLowerCase() - if (clean === 'localhost') return true - return isLoopbackIp(clean) -} - -/** - * Validates an MCP server URL against SSRF attacks by resolving DNS and - * rejecting private/reserved IP ranges (RFC-1918, link-local, cloud metadata). + * Validates an MCP server URL against the deployment's egress policy and returns + * the address to pin. * - * Only active when ALLOWED_MCP_DOMAINS is **not configured**. When an admin - * has set an explicit domain allowlist, they control which domains are - * reachable and private-network MCP servers are legitimate. Applying SSRF - * blocking on top of an admin-curated list would break self-hosted - * deployments where MCP servers run on internal networks. + * Domain governance (`ALLOWED_MCP_DOMAINS`) and this check are separate + * questions and both apply: an allowlisted domain still has to resolve somewhere + * the deployment permits. They used to be alternatives — configuring the domain + * list disabled this entirely — which left an allowlisted domain free to redirect + * anywhere, cloud metadata included. * - * Does NOT enforce protocol (HTTP is allowed) or block service ports — MCP - * servers legitimately run on HTTP and on arbitrary ports. - * - * Localhost/loopback is allowed for local dev MCP servers in self-hosted - * deployments, but blocked on the hosted environment (sim.ai) where users - * must not be able to reach the server's own loopback interface. - * URLs with env var references in the hostname are skipped — they will be - * validated after resolution at execution time. + * `profile` defaults to the configured-server one. An OAuth leg passes + * `contentFetch` instead, because those URLs come out of authorization-server + * metadata rather than from whoever configured the server. * - * Returns the resolved IP (or the literal itself for IP-literal URLs) as a - * non-null **policy signal**: the SSRF guard is active for this server. A public - * resolution selects the validate-at-connect guarded fetch — DNS-rebinding TOCTOU - * and redirect escapes are prevented by re-validating every socket connect and - * following redirects under per-hop validation (see `createSsrfGuardedMcpFetch` / - * `followRedirectsGuarded`), NOT by pinning to this address. The value is literally - * pinned only for the self-hosted private/loopback carve-out (a policy-permitted - * DNS alias the guarded lookup would otherwise filter). Returns null when the guard - * is unnecessary or impossible: no URL, allowlist-only mode, env-var hostnames - * (validated later), and localhost on self-hosted (no rebinding risk against a - * fixed loopback). + * Returns null when there is no URL yet, or when the hostname still contains an + * unresolved env-var reference. That URL is checked again after resolution, at + * which point it takes the normal path. * - * @throws McpSsrfError if the URL resolves to a blocked IP address + * @throws McpSsrfError when the policy refuses the destination + * @throws McpDnsResolutionError when the hostname cannot be resolved */ -export async function validateMcpServerSsrf(url: string | undefined): Promise { +export async function validateMcpServerSsrf( + url: string | undefined, + profile: EgressProfile = MCP_EGRESS_PROFILE +): Promise { if (!url) return null - if (getAllowedMcpDomainsFromEnv() !== null) return null if (hasEnvVarInHostname(url)) return null - let hostname: string - try { - hostname = new URL(url).hostname - } catch { - throw new McpSsrfError('MCP server URL is not a valid URL') - } - - const cleanHostname = unwrapIpv6Brackets(hostname) - - if (isLocalhostHostname(cleanHostname)) { - if (isHosted) { - throw new McpSsrfError('MCP server URL cannot point to a loopback address') - } - return null - } + const validation = await validateUrlWithDNS(url, 'MCP server URL', profile) + if (validation.isValid) return validation.resolvedIP - if (isIpLiteral(cleanHostname)) { - if (isPrivateIp(cleanHostname)) { - throw new McpSsrfError('MCP server URL cannot point to a private or reserved IP address') + const error = validation.error + if (error.includes('could not be resolved')) { + let hostname = url + try { + hostname = new URL(url).hostname + } catch { + // Fall back to the raw URL in the message. } - // Public IP literal: pin to this exact address so the caller's pinned fetch - // (createPinnedFetch) keeps every redirect hop on it. Returning null here - // would fall back to the default fetch, which follows a 3xx redirect to a - // private/metadata host and escapes SSRF controls. - return cleanHostname + logger.warn('DNS lookup failed for MCP server URL', { hostname }) + throw new McpDnsResolutionError(hostname) } - - let addresses: string[] - let address: string - try { - const resolved = await resolveHostAddresses(cleanHostname) - addresses = resolved.addresses - address = resolved.preferred - } catch (error) { - logger.warn('DNS lookup failed for MCP server URL', { - hostname, - error: toError(error).message, - }) - throw new McpDnsResolutionError(cleanHostname) - } - - for (const candidate of addresses) { - if (isLoopbackIp(candidate)) { - if (isHosted) { - logger.warn('MCP server URL resolves to loopback address', { - hostname, - resolvedIP: candidate, - }) - throw new McpSsrfError('MCP server URL resolves to a loopback address') - } - } else if (isPrivateIp(candidate)) { - logger.warn('MCP server URL resolves to blocked IP address', { - hostname, - resolvedIP: candidate, - }) - throw new McpSsrfError('MCP server URL resolves to a blocked IP address') - } - } - - return address + logger.warn('MCP server URL refused by egress policy', { error }) + throw new McpSsrfError(error) } diff --git a/apps/sim/lib/mcp/middleware.test.ts b/apps/sim/lib/mcp/middleware.test.ts new file mode 100644 index 00000000000..6f1a15efecc --- /dev/null +++ b/apps/sim/lib/mcp/middleware.test.ts @@ -0,0 +1,163 @@ +/** + * @vitest-environment node + * + * The gate lives on the middleware, so this is where it is proved. Thirteen raw + * MCP management routes sit behind `withMcpAuth` and only the workflow-server + * create handler ever grew a capability check of its own; asserting per route + * would have reproduced exactly that, so these assertions are about the wrapper + * every one of them shares. + */ +import { permissionGroupScopeMock, permissionGroupScopeMockFns } from '@sim/testing' +import type { NextRequest } from 'next/server' +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const mocks = vi.hoisted(() => ({ + auth: vi.fn(), + permissions: vi.fn(), +})) + +vi.mock('@/lib/auth/hybrid', async () => { + const AuthType = { SESSION: 'session', API_KEY: 'api_key', INTERNAL_JWT: 'internal_jwt' } as const + return { + AuthType, + checkSessionOrInternalAuth: mocks.auth, + capabilityGovernedAuthUserId: (auth: { + userId?: string + authType?: string + apiKeyType?: string + }) => { + if (!auth?.userId) return null + if (auth.authType === AuthType.SESSION) return auth.userId + return auth.authType === AuthType.API_KEY && auth.apiKeyType === 'personal' + ? auth.userId + : null + }, + } +}) +vi.mock('@/lib/workspaces/permissions/utils', () => ({ + getUserEntityPermissions: mocks.permissions, +})) +vi.mock('@/lib/permission-groups/config-scope.server', () => permissionGroupScopeMock) + +import { withMcpAuth } from '@/lib/mcp/middleware' +import { DEFAULT_PERMISSION_GROUP_CONFIG } from '@/lib/permission-groups/fields' + +const resolveGroupConfigMock = permissionGroupScopeMockFns.mockResolvePermissionGroupConfig + +const handler = vi.fn(async () => Response.json({ ok: true }) as never) + +function request() { + return new Request('http://localhost:3000/api/mcp/anything?workspaceId=workspace-1', { + method: 'POST', + }) as NextRequest +} + +function call(capability: 'deploy.mcp' | 'mcp_tools.use' | 'none') { + return withMcpAuth('write', capability)(handler)(request(), { + params: Promise.resolve({}), + }) +} + +describe('withMcpAuth permission-group gate', () => { + beforeEach(() => { + vi.clearAllMocks() + mocks.auth.mockResolvedValue({ + success: true, + userId: 'user-1', + authType: 'session', + }) + mocks.permissions.mockResolvedValue('admin') + resolveGroupConfigMock.mockResolvedValue(null) + }) + + it('refuses a session caller whose group withholds the declared capability', async () => { + resolveGroupConfigMock.mockResolvedValue({ + ...DEFAULT_PERMISSION_GROUP_CONFIG, + hideDeployMcp: true, + }) + + const response = await call('deploy.mcp') + + expect(response.status).toBe(403) + await expect(response.json()).resolves.toMatchObject({ + error: "MCP server deployment is not available under your organization's permission group", + }) + expect(handler).not.toHaveBeenCalled() + }) + + /** + * The two capabilities are separate keys, so a group withholding one must not + * refuse a route declaring the other — that would make the gate a blanket MCP + * switch rather than the two doors `mcpServerOperations` describes. + */ + it('admits a route whose declared capability the group does not withhold', async () => { + resolveGroupConfigMock.mockResolvedValue({ + ...DEFAULT_PERMISSION_GROUP_CONFIG, + hideDeployMcp: true, + }) + + const response = await call('mcp_tools.use') + + expect(response.status).toBe(200) + expect(handler).toHaveBeenCalled() + }) + + it('admits a caller no permission group governs', async () => { + const response = await call('deploy.mcp') + + expect(response.status).toBe(200) + expect(handler).toHaveBeenCalled() + }) + + /** + * The executor exemption. An internal JWT's `userId` is the subject the + * executor embedded, so resolving it would hand the run actor's grants to a + * credential that bears no person — the same rule + * `capabilityGovernedAuthUserId` states for every other surface. + */ + it('passes a non-user-bearing internal JWT ungated', async () => { + mocks.auth.mockResolvedValue({ + success: true, + userId: 'user-1', + authType: 'internal_jwt', + }) + resolveGroupConfigMock.mockResolvedValue({ + ...DEFAULT_PERMISSION_GROUP_CONFIG, + hideDeployMcp: true, + }) + + const response = await call('deploy.mcp') + + expect(response.status).toBe(200) + expect(handler).toHaveBeenCalled() + expect(resolveGroupConfigMock).not.toHaveBeenCalled() + }) + + it('resolves no group at all for a route declaring no capability', async () => { + const response = await call('none') + + expect(response.status).toBe(200) + expect(resolveGroupConfigMock).not.toHaveBeenCalled() + }) + + /** + * A capability refusal handed to a non-member would confirm the workspace + * exists and name which modules the organization withholds; the role failure + * conceals both, so it has to come first. + */ + it('answers the role failure, not the capability refusal, for a non-member', async () => { + mocks.permissions.mockResolvedValue(null) + resolveGroupConfigMock.mockResolvedValue({ + ...DEFAULT_PERMISSION_GROUP_CONFIG, + hideDeployMcp: true, + }) + + const response = await call('deploy.mcp') + + expect(response.status).toBe(403) + await expect(response.json()).resolves.toMatchObject({ + error: 'Insufficient permissions', + }) + expect(resolveGroupConfigMock).not.toHaveBeenCalled() + }) +}) diff --git a/apps/sim/lib/mcp/middleware.ts b/apps/sim/lib/mcp/middleware.ts index 6987f57e960..88be4b04c39 100644 --- a/apps/sim/lib/mcp/middleware.ts +++ b/apps/sim/lib/mcp/middleware.ts @@ -2,7 +2,12 @@ import { createLogger } from '@sim/logger' import { type PermissionType, permissionSatisfies } from '@sim/platform-authz/workspace' import { toError } from '@sim/utils/errors' import type { NextRequest, NextResponse } from 'next/server' -import { type AuthTypeValue, checkSessionOrInternalAuth } from '@/lib/auth/hybrid' +import { + type AuthTypeValue, + capabilityGovernedAuthUserId, + checkSessionOrInternalAuth, + type AuthResult as HybridAuthResult, +} from '@/lib/auth/hybrid' import { generateRequestId } from '@/lib/core/utils/request' import { assertContentLengthWithinLimit, @@ -10,6 +15,11 @@ import { readStreamToBufferWithLimit, } from '@/lib/core/utils/stream-limits' import { createMcpErrorResponse } from '@/lib/mcp/utils' +import type { StaticPermissionGroupCapability } from '@/lib/permission-groups/capabilities' +import { + capabilityRefusal, + isWorkspaceCapabilityWithheld, +} from '@/lib/permission-groups/capability-assertions' import { getUserEntityPermissions } from '@/lib/workspaces/permissions/utils' const logger = createLogger('McpAuthMiddleware') @@ -18,6 +28,52 @@ const parsedBodies = new WeakMap() export type McpPermissionLevel = 'read' | 'write' | 'admin' +/** + * The permission-group capability an MCP management route requires, or `'none'` + * when no group governs it. + * + * Required at every call site, and `'none'` spelled out rather than omitted, + * for the same reason `capability` is required on `defineWorkspaceOperation` + * and on `V1RouteCapability` in the v1 middleware: an absent declaration cannot be told apart + * from an unreviewed one. That is exactly how this surface came to gate + * `deploy.mcp` on one of its thirteen routes — the create handler grew an + * inline check and its twelve siblings, including the one that flips a server + * public, silently did not. + * + * Each route's value is the one its `/api/v2` twin already declares in + * `mcpServerOperations`; this surface does not get a mapping of its own. + */ +export type McpRouteCapability = StaticPermissionGroupCapability | 'none' + +/** + * The permission-group gate for an MCP management route. + * + * Only a user-bearing credential carries capabilities. + * `checkSessionOrInternalAuth` rejects `x-api-key` outright, so the two kinds + * that reach here are a browser session and the executor's internal JWT — and + * the JWT's `userId` is the subject the executor embedded, a value that must not + * hand the run's actor's grants to a caller the executor exemption deliberately + * passes ungated. {@link capabilityGovernedAuthUserId} is the one place that + * distinction is read, so this cannot drift from the funnel's own rule. + * + * Never called before the role check. A capability refusal handed to a + * non-member would confirm the workspace exists and disclose which modules the + * organization withholds; the role failure conceals both. + */ +async function capabilityRefusalResponse( + auth: HybridAuthResult, + workspaceId: string, + capability: McpRouteCapability +): Promise { + if (capability === 'none') return null + const userId = capabilityGovernedAuthUserId(auth) + if (!userId) return null + if (!(await isWorkspaceCapabilityWithheld(userId, workspaceId, capability))) return null + + logger.warn('MCP request blocked by permission group', { workspaceId, userId, capability }) + return createMcpErrorResponse(null, capabilityRefusal(capability), 403) +} + export interface McpAuthContext { userId: string userName?: string | null @@ -118,7 +174,8 @@ export function mcpBodyReadErrorResponse( */ async function validateMcpAuth( request: NextRequest, - permissionLevel: McpPermissionLevel + permissionLevel: McpPermissionLevel, + capability: McpRouteCapability ): Promise { const requestId = generateRequestId() @@ -194,6 +251,11 @@ async function validateMcpAuth( } } + const capabilityFailure = await capabilityRefusalResponse(auth, workspaceId, capability) + if (capabilityFailure) { + return { success: false, errorResponse: capabilityFailure } + } + return { success: true, context: { @@ -246,18 +308,20 @@ function getPermissionErrorMessage(permissionLevel: McpPermissionLevel): string * Higher-order function that wraps MCP route handlers with authentication middleware * * @param permissionLevel - Required permission level ('read', 'write', or 'admin') + * @param capability - The permission-group capability the route requires, or + * `'none'` with a reason. See {@link McpRouteCapability}. * @returns Middleware wrapper function - * */ export function withMcpAuth>( - permissionLevel: McpPermissionLevel = 'read' + permissionLevel: McpPermissionLevel, + capability: McpRouteCapability ) { return function middleware(handler: McpRouteHandler) { return async function wrappedHandler( request: NextRequest, routeContext: { params: Promise } ): Promise { - const authResult = await validateMcpAuth(request, permissionLevel) + const authResult = await validateMcpAuth(request, permissionLevel, capability) if (!authResult.success) { return (authResult as AuthFailure).errorResponse diff --git a/apps/sim/lib/mcp/oauth/auth.ts b/apps/sim/lib/mcp/oauth/auth.ts index 2787486e546..092f90c5f3d 100644 --- a/apps/sim/lib/mcp/oauth/auth.ts +++ b/apps/sim/lib/mcp/oauth/auth.ts @@ -17,6 +17,6 @@ export function mcpAuthGuarded( ): ReturnType { return auth(provider, { ...options, - fetchFn: options.fetchFn ?? createSsrfGuardedMcpFetch(), + fetchFn: options.fetchFn ?? createSsrfGuardedMcpFetch({ serverUrl: String(options.serverUrl) }), }) } diff --git a/apps/sim/lib/mcp/oauth/probe.test.ts b/apps/sim/lib/mcp/oauth/probe.test.ts index 23551694cc2..3a6a2383c90 100644 --- a/apps/sim/lib/mcp/oauth/probe.test.ts +++ b/apps/sim/lib/mcp/oauth/probe.test.ts @@ -58,7 +58,9 @@ describe('detectMcpAuthType — connection pinning (SSRF / DNS-rebinding)', () = const authType = await detectMcpAuthType('https://rebind.example.com/mcp', '203.0.113.10') expect(authType).toBe('none') - expect(mockCreatePinnedFetchWithDispatcher).toHaveBeenCalledWith('203.0.113.10') + expect(mockCreatePinnedFetchWithDispatcher).toHaveBeenCalledWith('203.0.113.10', { + profile: 'selfHostedService', + }) expect(mockCreateSsrfGuardedMcpFetch).not.toHaveBeenCalled() expect(mockPinnedFetch).toHaveBeenCalledTimes(1) // The unpinned global fetch must never be used — that was the SSRF sink. diff --git a/apps/sim/lib/mcp/oauth/probe.ts b/apps/sim/lib/mcp/oauth/probe.ts index 72c6c593ced..c39f521fe01 100644 --- a/apps/sim/lib/mcp/oauth/probe.ts +++ b/apps/sim/lib/mcp/oauth/probe.ts @@ -3,6 +3,7 @@ import type { FetchLike } from '@modelcontextprotocol/sdk/shared/transport.js' import { createLogger } from '@sim/logger' import { isLoopbackHostname } from '@sim/security/hostnames' import { createPinnedFetchWithDispatcher } from '@/lib/core/security/input-validation.server' +import { MCP_EGRESS_PROFILE } from '@/lib/mcp/domain-check' import { createSsrfGuardedMcpFetch } from '@/lib/mcp/pinned-fetch' import type { McpAuthType } from '@/lib/mcp/types' @@ -35,8 +36,10 @@ export async function detectMcpAuthType( // Pre-validated IP → pin directly (we own the Agent); otherwise the SSRF-guarded fetch // self-manages its per-request Agent teardown. - const pinned = resolvedIP ? createPinnedFetchWithDispatcher(resolvedIP) : undefined - const probeFetch: FetchLike = pinned?.fetch ?? createSsrfGuardedMcpFetch() + const pinned = resolvedIP + ? createPinnedFetchWithDispatcher(resolvedIP, { profile: MCP_EGRESS_PROFILE }) + : undefined + const probeFetch: FetchLike = pinned?.fetch ?? createSsrfGuardedMcpFetch({ serverUrl: url }) const controller = new AbortController() const timer = setTimeout(() => controller.abort(), PROBE_TIMEOUT_MS) diff --git a/apps/sim/lib/mcp/oauth/revoke.test.ts b/apps/sim/lib/mcp/oauth/revoke.test.ts index ba91b2cad97..abfcea1b7b8 100644 --- a/apps/sim/lib/mcp/oauth/revoke.test.ts +++ b/apps/sim/lib/mcp/oauth/revoke.test.ts @@ -44,6 +44,9 @@ vi.mock('@sim/security/ssrf', () => ({ isPrivateIp: (ip: string) => ip.startsWith('127.') || ip.startsWith('10.') || ip === '::1', })) vi.mock('@/lib/mcp/domain-check', () => ({ + MCP_EGRESS_PROFILE: 'selfHostedService', + OAUTH_EGRESS_PROFILE: 'contentFetch', + McpSsrfError: class McpSsrfError extends Error {}, validateMcpServerSsrf: mockValidateMcpServerSsrf, })) vi.mock('@modelcontextprotocol/sdk/client/auth.js', () => ({ @@ -114,7 +117,7 @@ describe('revokeMcpOauthTokens — SSRF guard', () => { it('validates the attacker-controlled revocation_endpoint before issuing the request', async () => { await revokeMcpOauthTokens('server-1', 'workspace-1') - expect(mockValidateMcpServerSsrf).toHaveBeenCalledWith(BLOCKED_ENDPOINT) + expect(mockValidateMcpServerSsrf).toHaveBeenCalledWith(BLOCKED_ENDPOINT, 'contentFetch') }) it('never issues an outbound request to the blocked revocation endpoint', async () => { @@ -145,7 +148,10 @@ describe('revokeMcpOauthTokens — SSRF guard', () => { await revokeMcpOauthTokens('server-1', 'workspace-1') - expect(mockValidateMcpServerSsrf).toHaveBeenCalledWith(publicEndpoint) + // Same origin as the configured server, so it keeps that server's profile — + // the metadata pointed back at the host the operator already chose. The + // blocked-endpoint test above covers the cross-origin case, which does not. + expect(mockValidateMcpServerSsrf).toHaveBeenCalledWith(publicEndpoint, 'selfHostedService') const revokeCalls = mockUndiciFetch.mock.calls.filter((call) => { const target = typeof call[0] === 'string' ? call[0] : String(call[0]) return target === publicEndpoint diff --git a/apps/sim/lib/mcp/oauth/revoke.ts b/apps/sim/lib/mcp/oauth/revoke.ts index 89c9760d2c3..fc154978730 100644 --- a/apps/sim/lib/mcp/oauth/revoke.ts +++ b/apps/sim/lib/mcp/oauth/revoke.ts @@ -37,7 +37,7 @@ export async function revokeMcpOauthTokens( const row = await loadOauthRow({ mcpServerId }) if (!row?.tokens) return - const ssrfGuardedFetch = createSsrfGuardedMcpFetch() + const ssrfGuardedFetch = createSsrfGuardedMcpFetch({ serverUrl: server.url }) const info = await discoverOAuthServerInfo(server.url, { fetchFn: ssrfGuardedFetch }).catch( () => undefined ) diff --git a/apps/sim/lib/mcp/orchestration/server-lifecycle.test.ts b/apps/sim/lib/mcp/orchestration/server-lifecycle.test.ts index b6df53b6b12..489d9cf4d05 100644 --- a/apps/sim/lib/mcp/orchestration/server-lifecycle.test.ts +++ b/apps/sim/lib/mcp/orchestration/server-lifecycle.test.ts @@ -38,6 +38,8 @@ vi.mock('@sim/db/schema', () => ({ vi.mock('@sim/utils/id', () => ({ generateId: vi.fn() })) vi.mock('@/lib/core/security/encryption', () => encryptionMock) vi.mock('@/lib/mcp/domain-check', () => ({ + MCP_EGRESS_PROFILE: 'selfHostedService', + OAUTH_EGRESS_PROFILE: 'contentFetch', McpDnsResolutionError: class extends Error {}, McpDomainNotAllowedError: class extends Error {}, McpSsrfError: class extends Error {}, diff --git a/apps/sim/lib/mcp/pinned-fetch.test.ts b/apps/sim/lib/mcp/pinned-fetch.test.ts index 3d9d8518b8b..2316c4f6092 100644 --- a/apps/sim/lib/mcp/pinned-fetch.test.ts +++ b/apps/sim/lib/mcp/pinned-fetch.test.ts @@ -31,9 +31,13 @@ vi.mock('@sim/security/ssrf', () => ({ isPrivateIp: (ip: string) => ip.startsWith('127.') || ip.startsWith('10.') || ip === '::1', })) vi.mock('@/lib/mcp/domain-check', () => ({ + MCP_EGRESS_PROFILE: 'selfHostedService', + OAUTH_EGRESS_PROFILE: 'contentFetch', + McpSsrfError: class McpSsrfError extends Error {}, validateMcpServerSsrf: mockValidateMcpServerSsrf, })) +import { McpSsrfError } from '@/lib/mcp/domain-check' import { createGuardedMcpFetch, createSsrfGuardedMcpFetch } from '@/lib/mcp/pinned-fetch' /** The per-request guarded Agent is always built with a DoS-backstop response cap. */ @@ -55,7 +59,9 @@ describe('createGuardedMcpFetch', () => { // No dispatcher options: no `allowH2` opt-in (h1.1 default) and no Agent-level // maxResponseSize — the standalone GET SSE stream must stream unbounded (the body cap // is applied per-response to non-GET exchanges instead). - expect(mockCreateGuardedFetchWithDispatcher).toHaveBeenCalledWith() + expect(mockCreateGuardedFetchWithDispatcher).toHaveBeenCalledWith({ + profile: 'selfHostedService', + }) void close() expect(mockDestroy).toHaveBeenCalledTimes(1) @@ -93,6 +99,26 @@ describe('createGuardedMcpFetch', () => { expect(res.url).toBe('https://mcp.example/mcp') expect(res.redirected).toBe(true) }) + + it('judges a cross-origin SDK OAuth leg as content, never as the configured server', async () => { + // The MCP SDK reuses this fetch for its OAuth auth() legs, whose URLs come + // from the server's own metadata. A same-origin request stays on the + // persistent connect-time-validated transport; anything to another origin is + // validated per request under `contentFetch`, so the operator allowlist and + // the loopback carve-out of `selfHostedService` can never apply to it. + sentinelFetch.mockImplementation(async () => Response.json({ ok: true })) + mockValidateMcpServerSsrf.mockResolvedValue('203.0.113.10') + const { fetch: guarded } = createGuardedMcpFetch('https://mcp.example.com') + + await guarded('https://mcp.example.com/rpc', { method: 'POST' }) + expect(mockValidateMcpServerSsrf).not.toHaveBeenCalled() + + await guarded('https://auth.other.example/token', { method: 'POST' }) + expect(mockValidateMcpServerSsrf).toHaveBeenCalledWith( + 'https://auth.other.example/token', + 'contentFetch' + ) + }) }) describe('createSsrfGuardedMcpFetch', () => { @@ -111,7 +137,10 @@ describe('createSsrfGuardedMcpFetch', () => { const fetchLike = createSsrfGuardedMcpFetch() await fetchLike('https://attacker.example/revoke', { method: 'POST' }) - expect(mockValidateMcpServerSsrf).toHaveBeenCalledWith('https://attacker.example/revoke') + expect(mockValidateMcpServerSsrf).toHaveBeenCalledWith( + 'https://attacker.example/revoke', + 'contentFetch' + ) // The guarded Agent is always built with the DoS-backstop response-size cap. expect(mockCreateGuardedFetchWithDispatcher).toHaveBeenCalledWith(withResponseCap) expect(sentinelFetch).toHaveBeenCalledWith( @@ -180,27 +209,6 @@ describe('createSsrfGuardedMcpFetch', () => { expect(mockDestroy).toHaveBeenCalledTimes(1) }) - it('returns a streaming response live (un-buffered) over the unpinned fallback', async () => { - // resolvedIP null → global fetch; a text/event-stream reply (the auth-type probe) - // must be handed back as-is so the caller reads headers without draining the stream. - // Identity (same object) proves it was NOT re-wrapped into a buffered copy. - mockValidateMcpServerSsrf.mockResolvedValue(null) - const streamingRes = new Response(new ReadableStream({ start() {} }), { - headers: { 'content-type': 'text/event-stream' }, - }) - const globalFetch = vi.spyOn(globalThis, 'fetch').mockImplementation(async () => streamingRes) - try { - const fetchLike = createSsrfGuardedMcpFetch() - const res = await fetchLike('https://allowed.internal/mcp', { method: 'POST' }) - - expect(res).toBe(streamingRes) - // No per-request Agent on the unpinned path, so nothing to tear down. - expect(mockDestroy).not.toHaveBeenCalled() - } finally { - globalFetch.mockRestore() - } - }) - it('streams (does not buffer) a pinned text/event-stream reply and tears down after it drains', async () => { // The guard resolves the IP itself, so the probe's initialize over the guarded path // DOES get a pinned Agent. A streaming reply must still be handed back live (not @@ -266,7 +274,7 @@ describe('createSsrfGuardedMcpFetch', () => { signal?.addEventListener('abort', () => reject(signal.reason), { once: true }) }) ) - const fetchLike = createSsrfGuardedMcpFetch(5) + const fetchLike = createSsrfGuardedMcpFetch({ timeoutMs: 5 }) await expect(fetchLike('https://slow.example/token', { method: 'POST' })).rejects.toThrow( /timed out after 5ms/ @@ -281,7 +289,7 @@ describe('createSsrfGuardedMcpFetch', () => { sentinelFetch.mockImplementation( async () => new Response(new ReadableStream({ start() {} })) ) - const fetchLike = createSsrfGuardedMcpFetch(5) + const fetchLike = createSsrfGuardedMcpFetch({ timeoutMs: 5 }) await expect(fetchLike('https://slow-body.example/token', { method: 'POST' })).rejects.toThrow( /timed out after 5ms/ @@ -292,7 +300,7 @@ describe('createSsrfGuardedMcpFetch', () => { it('bounds a stalled SSRF/DNS validation by the deadline', async () => { // Validation never resolves (mimics a hanging dns.lookup, which takes no signal). mockValidateMcpServerSsrf.mockReturnValue(new Promise(() => {})) - const fetchLike = createSsrfGuardedMcpFetch(5) + const fetchLike = createSsrfGuardedMcpFetch({ timeoutMs: 5 }) await expect(fetchLike('https://slow-dns.example/token')).rejects.toThrow(/timed out after 5ms/) // Never got past validation, so no request was issued and no Agent was created. @@ -307,7 +315,7 @@ describe('createSsrfGuardedMcpFetch', () => { mockValidateMcpServerSsrf.mockRejectedValue(new Error('blocked late')) const controller = new AbortController() controller.abort(new Error('pre-aborted')) - const fetchLike = createSsrfGuardedMcpFetch(60_000) + const fetchLike = createSsrfGuardedMcpFetch({ timeoutMs: 60_000 }) await expect( fetchLike('https://slow.example/token', { signal: controller.signal }) @@ -321,7 +329,7 @@ describe('createSsrfGuardedMcpFetch', () => { // Validation hangs; the caller's abort — well before the 60s deadline — must settle it. mockValidateMcpServerSsrf.mockReturnValue(new Promise(() => {})) const controller = new AbortController() - const fetchLike = createSsrfGuardedMcpFetch(60_000) + const fetchLike = createSsrfGuardedMcpFetch({ timeoutMs: 60_000 }) const pending = fetchLike('https://slow-dns.example/token', { signal: controller.signal }) controller.abort(new Error('caller cancelled')) @@ -344,7 +352,7 @@ describe('createSsrfGuardedMcpFetch', () => { ) const controller = new AbortController() // Long deadline so the caller's abort — not the timeout — is what settles the request. - const fetchLike = createSsrfGuardedMcpFetch(60_000) + const fetchLike = createSsrfGuardedMcpFetch({ timeoutMs: 60_000 }) const pending = fetchLike('https://slow.example/token', { signal: controller.signal }) controller.abort(new Error('caller cancelled')) @@ -367,23 +375,27 @@ describe('createSsrfGuardedMcpFetch', () => { const fetchLike = createSsrfGuardedMcpFetch() await fetchLike(new URL('https://attacker.example/discover')) - expect(mockValidateMcpServerSsrf).toHaveBeenCalledWith('https://attacker.example/discover') + expect(mockValidateMcpServerSsrf).toHaveBeenCalledWith( + 'https://attacker.example/discover', + 'contentFetch' + ) expect(mockCreateGuardedFetchWithDispatcher).toHaveBeenCalledWith(withResponseCap) }) - it('falls back to global fetch when validation returns no IP', async () => { + it('refuses rather than falling back to an unguarded fetch when validation yields no IP', async () => { mockValidateMcpServerSsrf.mockResolvedValue(null) const globalFetch = vi .spyOn(globalThis, 'fetch') .mockImplementation(async () => new Response('ok')) try { const fetchLike = createSsrfGuardedMcpFetch() - await fetchLike('https://allowed.internal/mcp') + await expect(fetchLike('https://allowed.internal/mcp')).rejects.toThrow( + 'could not be validated' + ) + // No leg of the OAuth flow may reach the network unguarded. + expect(globalFetch).not.toHaveBeenCalled() expect(mockCreateGuardedFetchWithDispatcher).not.toHaveBeenCalled() - expect(globalFetch).toHaveBeenCalledTimes(1) - // No pinned Agent was created, so there is nothing to tear down. - expect(mockDestroy).not.toHaveBeenCalled() } finally { globalFetch.mockRestore() } @@ -391,22 +403,15 @@ describe('createSsrfGuardedMcpFetch', () => { }) describe('self-hosted private-resolution carve-out', () => { - it('keeps the legacy pin for a loopback-resolving host (guarded lookup would filter it)', async () => { - // Self-hosted DNS alias -> 127.0.0.1: policy allows it. The guarded lookup would - // strand the connect and an unguarded fallback would reopen rebinding — so this case - // pins to the validated address, preserving the old behavior and its security property. - mockValidateMcpServerSsrf.mockResolvedValue('127.0.0.1') - mockCreatePinnedFetchWithDispatcher.mockReturnValue({ - fetch: sentinelFetch, - dispatcher: { destroy: mockDestroy }, - }) - sentinelFetch.mockImplementation(async () => new Response('ok')) + it('refuses a loopback resolution instead of pinning to it', async () => { + // OAuth legs run under `contentFetch`, which vouches for nothing — so a + // hostile authorization server cannot steer a leg at the deployment's own + // loopback, which the previous pinned carve-out would have permitted. + mockValidateMcpServerSsrf.mockRejectedValue(new McpSsrfError('blocked')) const fetchLike = createSsrfGuardedMcpFetch() - await fetchLike('https://my-local-alias/mcp') - expect(mockCreatePinnedFetchWithDispatcher).toHaveBeenCalledWith( - '127.0.0.1', - expect.objectContaining({ maxResponseSize: expect.any(Number) }) - ) + + await expect(fetchLike('https://my-local-alias/mcp')).rejects.toThrow(McpSsrfError) + expect(mockCreatePinnedFetchWithDispatcher).not.toHaveBeenCalled() expect(mockCreateGuardedFetchWithDispatcher).not.toHaveBeenCalled() }) }) diff --git a/apps/sim/lib/mcp/pinned-fetch.ts b/apps/sim/lib/mcp/pinned-fetch.ts index d5169a2b696..a8360236156 100644 --- a/apps/sim/lib/mcp/pinned-fetch.ts +++ b/apps/sim/lib/mcp/pinned-fetch.ts @@ -6,7 +6,12 @@ import { createPinnedFetchWithDispatcher, createSsrfGuardedFetchWithDispatcher, } from '@/lib/core/security/input-validation.server' -import { validateMcpServerSsrf } from '@/lib/mcp/domain-check' +import { + MCP_EGRESS_PROFILE, + McpSsrfError, + OAUTH_EGRESS_PROFILE, + validateMcpServerSsrf, +} from '@/lib/mcp/domain-check' import { McpError } from '@/lib/mcp/types' const logger = createLogger('McpOauthFetch') @@ -26,9 +31,9 @@ export interface GuardedMcpFetch { * against the private/reserved blocklist (validate-at-connect, the LibreChat * pattern), and redirects are followed manually with per-hop validation — an * IP-literal redirect target (which bypasses any connect-time lookup) is checked - * explicitly, and custom headers are dropped on cross-origin hops. This replaces - * the previous single-IP pin, which no reference MCP client uses and which welded - * every request to one address with no fallback. + * explicitly, and custom headers are dropped on cross-origin hops. Keeping the + * full address set rather than welding every request to one address is what lets + * the connection fall back across a server's records. * * Runs HTTP/1.1: we do not opt into undici's experimental `allowH2`, whose h2 path stalls * with headers-but-no-body on reused POST sessions (nodejs/undici #2311, #3433, #4143) — @@ -87,15 +92,56 @@ function capResponseBody(response: Response, maxBytes: number): Response { return wrapped } +/** The href of whatever the SDK handed the transport fetch. */ +function requestHref(input: string | URL | Request): string { + if (typeof input === 'string') return input + if (input instanceof URL) return input.href + if (input instanceof Request) return input.url + return String(input) +} + +/** + * Wraps the transport's same-origin fetch so a request to any other origin is + * judged as content and validated per request, never inheriting the configured + * server's privileges or its pinned address. + * + * The MCP SDK reuses the transport's own `fetch` for its internal OAuth `auth()` + * legs (protected-resource / authorization-server metadata, token exchange and + * refresh), whose URLs come from the server's own `WWW-Authenticate`/metadata — + * attacker-steerable. Without this split those legs would run under + * {@link MCP_EGRESS_PROFILE} (honors the allowlist, permits loopback off-hosted) + * or, worse, be pinned onto the server's private address. Same-origin requests — + * the JSON-RPC transport itself — keep the persistent guarded path. + */ +function splitByConfiguredOrigin( + sameOrigin: typeof fetch, + serverUrl: string | undefined +): typeof fetch { + if (!serverUrl || !URL.canParse(serverUrl)) return sameOrigin + const configuredOrigin = new URL(serverUrl).origin + const crossOrigin = createSsrfGuardedMcpFetch({ serverUrl }) + return async (input, init) => { + const target = requestHref(input) + const sameAsConfigured = URL.canParse(target) && new URL(target).origin === configuredOrigin + return sameAsConfigured ? sameOrigin(input, init) : crossOrigin(target, init) + } +} + /** - * Legacy single-IP pin, kept ONLY for self-hosted private/loopback resolutions - * (a DNS alias the policy explicitly permits): the guarded lookup would filter - * the address and strand the connect, while an unguarded fallback would reopen - * rebinding/redirect escape. Pinning to the validated address preserves the old - * behavior and its security property for exactly this carve-out. + * Single-IP pin, used for the self-hosted private/loopback resolutions the + * policy explicitly permits. The guarded transport would reach them too, but a + * destination vouched by hostname is vouched for wherever it points, so on this + * one path pinning to the address that was actually validated is the stricter + * choice: it holds the connection to that address rather than to whatever the + * name resolves to next. */ -export function createPinnedPrivateMcpFetch(resolvedIP: string): GuardedMcpFetch { - const { fetch: pinnedFetch, dispatcher } = createPinnedFetchWithDispatcher(resolvedIP) +export function createPinnedPrivateMcpFetch( + resolvedIP: string, + serverUrl?: string +): GuardedMcpFetch { + const { fetch: pinnedFetch, dispatcher } = createPinnedFetchWithDispatcher(resolvedIP, { + profile: MCP_EGRESS_PROFILE, + }) const capped: typeof fetch = async (input, init) => { const method = init?.method ?? (input instanceof Request ? input.method : 'GET') const response = await pinnedFetch(input, init) @@ -103,25 +149,23 @@ export function createPinnedPrivateMcpFetch(resolvedIP: string): GuardedMcpFetch ? response : capResponseBody(response, MAX_TRANSPORT_RESPONSE_BYTES) } - return { fetch: capped, close: () => dispatcher.destroy() } + return { + fetch: splitByConfiguredOrigin(capped, serverUrl), + close: () => dispatcher.destroy(), + } } -export function createGuardedMcpFetch(): GuardedMcpFetch { - const { fetch: guardedFetch, dispatcher } = createSsrfGuardedFetchWithDispatcher() +export function createGuardedMcpFetch(serverUrl?: string): GuardedMcpFetch { + const { fetch: guardedFetch, dispatcher } = createSsrfGuardedFetchWithDispatcher({ + profile: MCP_EGRESS_PROFILE, + }) // Per-request phase logging: a stalled transport request (e.g. a first `initialize` that hangs // to the client timeout) shows whether it stalls BEFORE response headers ("request" with no // "response headers" = connect/request stall) or AFTER ("response headers" then the SDK's // stream read stalls). Isolates the client-side first-connect stall. const instrumentedFetch: typeof fetch = async (input, init) => { const method = init?.method ?? (input instanceof Request ? input.method : 'GET') - const target = - typeof input === 'string' - ? input - : input instanceof URL - ? input.href - : input instanceof Request - ? input.url - : String(input) + const target = requestHref(input) const host = URL.canParse(target) ? new URL(target).host : target const startedAt = Date.now() transportLogger.info('MCP transport request', { host, method }) @@ -149,7 +193,10 @@ export function createGuardedMcpFetch(): GuardedMcpFetch { throw error } } - return { fetch: instrumentedFetch, close: () => dispatcher.destroy() } + return { + fetch: splitByConfiguredOrigin(instrumentedFetch, serverUrl), + close: () => dispatcher.destroy(), + } } /** @@ -277,10 +324,26 @@ function releaseStreamOnSettle( * @throws McpSsrfError if a request URL resolves to a blocked IP address * @throws McpError if a request exceeds `timeoutMs` */ -export function createSsrfGuardedMcpFetch(timeoutMs: number = OAUTH_FETCH_TIMEOUT_MS): FetchLike { +export function createSsrfGuardedMcpFetch( + options: { serverUrl?: string; timeoutMs?: number } = {} +): FetchLike { + const { serverUrl, timeoutMs = OAUTH_FETCH_TIMEOUT_MS } = options + // The origin the operator configured. A leg that stays on it is the server + // they chose; everything else was named by that server's metadata. + let configuredOrigin: string | undefined + if (serverUrl && URL.canParse(serverUrl)) configuredOrigin = new URL(serverUrl).origin + return (async (url, init) => { const target = typeof url === 'string' ? url : url.href const host = URL.canParse(target) ? new URL(target).host : target + const sameAsConfigured = + configuredOrigin !== undefined && + URL.canParse(target) && + new URL(target).origin === configuredOrigin + // The first hop is the configured server and keeps its privileges — a + // self-hosted MCP on an allowlisted private address must still be able to + // start discovery. Every hop the metadata names is judged as content. + const profile = sameAsConfigured ? MCP_EGRESS_PROFILE : OAUTH_EGRESS_PROFILE const startedAt = Date.now() const timeoutSignal = AbortSignal.timeout(timeoutMs) // Bound every phase — validation, request, body read — by the deadline + caller signal. @@ -289,28 +352,28 @@ export function createSsrfGuardedMcpFetch(timeoutMs: number = OAUTH_FETCH_TIMEOU let dispatcher: Agent | undefined try { logger.info('OAuth guarded fetch: validating', { host }) - const resolvedIP = await withDeadline(validateMcpServerSsrf(target), signal) - logger.info('OAuth guarded fetch: requesting', { host, guarded: Boolean(resolvedIP) }) - let response: Response - if (resolvedIP && isPrivateIp(resolvedIP)) { - // Self-hosted private/loopback resolution (policy-permitted): the guarded lookup - // would filter the address, and an unguarded fallback would reopen rebinding — - // keep the legacy pin to the validated address for exactly this case. - const pinned = createPinnedFetchWithDispatcher(resolvedIP, { - maxResponseSize: MAX_OAUTH_RESPONSE_BYTES, - }) - dispatcher = pinned.dispatcher - response = await withDeadline(pinned.fetch(url, { ...init, signal }), signal) - } else if (resolvedIP) { - const guarded = createSsrfGuardedFetchWithDispatcher({ - maxResponseSize: MAX_OAUTH_RESPONSE_BYTES, - }) - dispatcher = guarded.dispatcher - response = await withDeadline(guarded.fetch(url, { ...init, signal }), signal) - } else { - // No guard (self-hosted allowlist / localhost carve-out) — global fetch as before. - response = await withDeadline(globalThis.fetch(url, { ...init, signal }), signal) + const resolvedIP = await withDeadline(validateMcpServerSsrf(target, profile), signal) + if (!resolvedIP) { + // No leg of this flow may run unguarded. Under `contentFetch` the only + // way here is an unresolved env-var reference in the hostname, which is + // never a real authorization-server URL by the time OAuth runs. + throw new McpSsrfError('MCP OAuth request URL could not be validated') } + logger.info('OAuth guarded fetch: requesting', { host, configured: sameAsConfigured }) + // A private address only survives validation on the configured first hop. + // Pinning it holds the connection to the address that was validated, which + // a hostname-vouched destination would otherwise not be held to. + const transport = isPrivateIp(resolvedIP) + ? createPinnedFetchWithDispatcher(resolvedIP, { + profile, + maxResponseSize: MAX_OAUTH_RESPONSE_BYTES, + }) + : createSsrfGuardedFetchWithDispatcher({ + profile, + maxResponseSize: MAX_OAUTH_RESPONSE_BYTES, + }) + dispatcher = transport.dispatcher + const response = await withDeadline(transport.fetch(url, { ...init, signal }), signal) // The probe's `initialize` can stream (text/event-stream); hand it back live so the // buffer doesn't drain/stall it. Every OAuth leg is single-shot JSON and is buffered. const contentType = response.headers.get('content-type') ?? '' diff --git a/apps/sim/lib/mcp/service-pool.test.ts b/apps/sim/lib/mcp/service-pool.test.ts index 2903558d561..3f1566b7ab9 100644 --- a/apps/sim/lib/mcp/service-pool.test.ts +++ b/apps/sim/lib/mcp/service-pool.test.ts @@ -101,6 +101,9 @@ const SERVER_ROW = { } vi.mock('@/lib/mcp/domain-check', () => ({ + MCP_EGRESS_PROFILE: 'selfHostedService', + OAUTH_EGRESS_PROFILE: 'contentFetch', + McpSsrfError: class McpSsrfError extends Error {}, isMcpDomainAllowed: () => true, validateMcpDomain: () => {}, validateMcpServerSsrf: async () => '203.0.113.10', diff --git a/apps/sim/lib/mcp/service.test.ts b/apps/sim/lib/mcp/service.test.ts index bf7706168a6..8b297d5272d 100644 --- a/apps/sim/lib/mcp/service.test.ts +++ b/apps/sim/lib/mcp/service.test.ts @@ -99,6 +99,9 @@ vi.mock('@/lib/mcp/connection-manager', () => ({ })) vi.mock('@/lib/mcp/domain-check', () => ({ + MCP_EGRESS_PROFILE: 'selfHostedService', + OAUTH_EGRESS_PROFILE: 'contentFetch', + McpSsrfError: class McpSsrfError extends Error {}, isMcpDomainAllowed: (...args: unknown[]) => mockIsDomainAllowed(...args), validateMcpDomain: (...args: unknown[]) => mockValidateDomain(...args), validateMcpServerSsrf: (...args: unknown[]) => mockValidateSsrf(...args), diff --git a/apps/sim/lib/media/falai.ts b/apps/sim/lib/media/falai.ts index 21a36dfbad9..db4dc0a6152 100644 --- a/apps/sim/lib/media/falai.ts +++ b/apps/sim/lib/media/falai.ts @@ -202,12 +202,13 @@ export async function downloadFalMedia( return { contentType: match[1], buffer } } - const validation = await validateUrlWithDNS(url, 'mediaUrl') - if (!validation.isValid || !validation.resolvedIP) { - throw new Error(validation.error || 'Generated media URL failed validation') + const validation = await validateUrlWithDNS(url, 'mediaUrl', 'contentFetch') + if (!validation.isValid) { + throw new Error(validation.error) } const response = await secureFetchWithPinnedIP(url, validation.resolvedIP, { + profile: 'contentFetch', method: 'GET', maxResponseBytes: MAX_MEDIA_BYTES, }) diff --git a/apps/sim/lib/media/ffmpeg-process.ts b/apps/sim/lib/media/ffmpeg-process.ts new file mode 100644 index 00000000000..f014832ca14 --- /dev/null +++ b/apps/sim/lib/media/ffmpeg-process.ts @@ -0,0 +1,74 @@ +import { execFile, execFileSync } from 'node:child_process' +import { toError } from '@sim/utils/errors' + +const DEFAULT_MAX_OUTPUT_BYTES = 1024 * 1024 + +/** + * Base flags for every ffmpeg invocation. `-y`/`-nostdin` replace + * fluent-ffmpeg's implicit overwrite flag: without them an existing output + * makes ffmpeg prompt on an unread stdin pipe and hang until the timeout + * kills it. + */ +export const FFMPEG_BASE_ARGS = ['-y', '-nostdin'] as const + +export interface ExecutableOptions { + cwd?: string + maxOutputBytes?: number + signal?: AbortSignal + timeoutMs: number +} + +export interface ExecutableResult { + stdout: string + stderr: string +} + +/** Resolve an executable without invoking a shell. */ +export function resolveExecutable(binary: string): string | null { + try { + const locator = process.platform === 'win32' ? 'where' : 'which' + const stdout = execFileSync(locator, [binary], { encoding: 'utf-8' }) + return stdout.trim().split(/\r?\n/)[0] || null + } catch { + return null + } +} + +/** + * Run an executable with a shell-free argument vector and bounded output. + * + * `execFile` owns the timeout and abort handling, including killing the child. + * `maxBuffer` bounds both stdout and stderr, which prevents FFmpeg diagnostics + * from growing with an adversarial or badly corrupted input. + */ +export function runExecutable( + executable: string, + args: string[], + options: ExecutableOptions +): Promise { + return new Promise((resolve, reject) => { + try { + execFile( + executable, + args, + { + cwd: options.cwd, + encoding: 'utf-8', + killSignal: 'SIGKILL', + maxBuffer: options.maxOutputBytes ?? DEFAULT_MAX_OUTPUT_BYTES, + signal: options.signal, + timeout: options.timeoutMs, + }, + (error, stdout, stderr) => { + if (error) { + reject(Object.assign(error, { stderr, stdout })) + return + } + resolve({ stdout, stderr }) + } + ) + } catch (error) { + reject(toError(error)) + } + }) +} diff --git a/apps/sim/lib/media/ffmpeg.test.ts b/apps/sim/lib/media/ffmpeg.test.ts index a5477267ceb..8c0cc0dfb9a 100644 --- a/apps/sim/lib/media/ffmpeg.test.ts +++ b/apps/sim/lib/media/ffmpeg.test.ts @@ -5,83 +5,81 @@ import fs from 'node:fs' import path from 'node:path' import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' -const { capturedVideoFilters, capturedCaptions, killSignals, probeReport, command, saves } = - vi.hoisted(() => ({ - capturedVideoFilters: [] as string[], - capturedCaptions: [] as string[], - killSignals: [] as string[], - probeReport: { json: '{"streams":[],"format":{}}' }, - command: { hang: false }, - saves: { waiters: [] as Array<() => void> }, - })) - -vi.mock('node:child_process', () => ({ - execSync: (cmd: string) => - String(cmd).includes('ffprobe') ? '/usr/bin/ffprobe\n' : '/usr/bin/ffmpeg\n', - // `promisify` honors this symbol the same way it does for the real execFile, - // so the module under test destructures `{ stdout }` exactly as in production. - execFile: Object.assign(() => undefined, { - [Symbol.for('nodejs.util.promisify.custom')]: async () => ({ - stdout: probeReport.json, - stderr: '', - }), - }), +const { + capturedArgs, + capturedVideoFilters, + capturedCaptions, + killSignals, + probeReport, + command, + saves, +} = vi.hoisted(() => ({ + capturedArgs: [] as string[][], + capturedVideoFilters: [] as string[], + capturedCaptions: [] as string[], + killSignals: [] as string[], + probeReport: { json: '{"streams":[],"format":{}}' }, + command: { hang: false }, + saves: { waiters: [] as Array<() => void> }, })) -vi.mock('fluent-ffmpeg', () => { - const makeCommand = (cwd?: string) => { - const handlers: Record void> = {} - const cmd: Record = {} - const chain = (fn?: (arg: unknown) => void) => (arg?: unknown) => { - fn?.(arg) - return cmd +vi.mock('node:child_process', () => ({ + execFileSync: (_executable: string, args: string[]) => + args[0] === 'ffprobe' ? '/usr/bin/ffprobe\n' : '/usr/bin/ffmpeg\n', + execFile: ( + executable: string, + args: string[], + options: { cwd?: string; killSignal?: string; signal?: AbortSignal; timeout?: number }, + callback: (error: Error | null, stdout: string, stderr: string) => void + ) => { + if (executable.includes('ffprobe')) { + callback(null, probeReport.json, '') + return } - cmd.input = chain() - cmd.inputOptions = chain() - cmd.outputOptions = chain() - cmd.complexFilter = chain() - cmd.audioFilters = chain() - cmd.noVideo = chain() - cmd.setStartTime = chain() - cmd.setDuration = chain() - cmd.seekInput = chain() - cmd.frames = chain() - cmd.kill = chain((signal) => { - killSignals.push(String(signal)) - }) - cmd.videoFilters = chain((arg) => { - const filter = String(arg) + + capturedArgs.push(args) + const filterIndex = args.indexOf('-vf') + if (filterIndex >= 0) { + const filter = args[filterIndex + 1] capturedVideoFilters.push(filter) - // The caption is a bare relative filename resolved against the command's cwd (the - // temp dir). Read it back while it still exists to prove the raw caption never - // reached the filtergraph string. const match = filter.match(/textfile=([^:]+)/) - if (match && cwd) { - capturedCaptions.push(fs.readFileSync(path.join(cwd, match[1]), 'utf-8')) + if (match && options.cwd) { + capturedCaptions.push(fs.readFileSync(path.join(options.cwd, match[1]), 'utf-8')) } - }) - cmd.on = (event: string, handler: (...args: unknown[]) => void) => { - handlers[event] = handler - return cmd } - cmd.save = (outputPath: string) => { - for (const resolve of saves.waiters.splice(0)) resolve() - // A hung command never emits `end`, standing in for an encode that outlives - // the request that asked for it. - if (command.hang) return cmd - fs.writeFileSync(outputPath, Buffer.from('stub-output')) - handlers.end?.() - return cmd + for (const resolve of saves.waiters.splice(0)) resolve() + + let settled = false + let timer: ReturnType | undefined + const finish = (error: Error | null) => { + if (settled) return + settled = true + if (timer) clearTimeout(timer) + options.signal?.removeEventListener('abort', onAbort) + callback(error, '', error ? 'stub failure' : '') } - return cmd - } - const ffmpeg = ((_input?: unknown, options?: { cwd?: string }) => - makeCommand(options?.cwd)) as unknown as Record & (() => unknown) - ;(ffmpeg as Record).setFfmpegPath = () => {} - return { default: ffmpeg } -}) + const onAbort = () => { + killSignals.push(options.killSignal || 'SIGTERM') + const error = new Error('aborted') + error.name = 'AbortError' + finish(error) + } + options.signal?.addEventListener('abort', onAbort, { once: true }) + + if (command.hang) { + timer = setTimeout(() => { + killSignals.push(options.killSignal || 'SIGTERM') + finish(Object.assign(new Error('timed out'), { killed: true })) + }, options.timeout) + return + } + + fs.writeFileSync(args.at(-1) as string, Buffer.from('stub-output')) + finish(null) + }, +})) -import { runFfmpegOperation } from '@/lib/media/ffmpeg' +import { extFromMime, runFfmpegOperation } from '@/lib/media/ffmpeg' const videoInput = { buffer: Buffer.from('fake-video-bytes'), @@ -89,12 +87,40 @@ const videoInput = { name: 'clip.mp4', } +describe('extFromMime', () => { + it('normalizes parameters and casing before selecting a known extension', () => { + expect(extFromMime(' Video/MP4; codecs=avc1 ')).toBe('mp4') + }) + + it.each(['constructor', '__proto__', 'toString'])( + 'does not resolve inherited object properties as MIME types: %s', + (mimeType) => { + expect(extFromMime(mimeType)).toBe('bin') + } + ) + + it.each(['video/../../../../escaped', 'video/..\\..\\..\\..\\escaped', 'video/mp4\\..\\escaped'])( + 'rejects path syntax in an unknown MIME subtype: %s', + (mimeType) => { + expect(extFromMime(mimeType)).toBe('bin') + } + ) + + it('never returns separators from extra MIME path segments', () => { + const extension = extFromMime('video/mp4/../../escaped') + + expect(extension).toBe('mp4') + expect(extension).not.toMatch(/[\\/]/) + }) +}) + /** Resolves once the next FFmpeg command reaches `.save()`, i.e. once it is running. */ function nextSave(): Promise { return new Promise((resolve) => saves.waiters.push(resolve)) } beforeEach(() => { + capturedArgs.length = 0 capturedVideoFilters.length = 0 capturedCaptions.length = 0 killSignals.length = 0 @@ -282,6 +308,65 @@ describe('runFfmpegOperation output format', () => { }) }) +describe('runFfmpegOperation argument vectors', () => { + it('loops the second overlay input without involving a shell', async () => { + await runFfmpegOperation('overlay_audio', [videoInput, videoInput], { loopToVideo: true }) + + expect(capturedArgs[0]).toEqual([ + '-y', + '-nostdin', + '-i', + expect.stringMatching(/in-0\.mp4$/), + '-stream_loop', + '-1', + '-i', + expect.stringMatching(/in-1\.mp4$/), + '-map', + '0:v:0', + '-map', + '1:a:0', + '-c:v', + 'copy', + '-c:a', + 'aac', + '-shortest', + expect.stringMatching(/out\.mp4$/), + ]) + }) + + it('keeps trim seek and duration as distinct arguments', async () => { + await runFfmpegOperation('trim', [videoInput], { end: 5.5, start: 1.25 }) + + expect(capturedArgs[0]).toEqual([ + '-y', + '-nostdin', + '-i', + expect.stringMatching(/in-0\.mp4$/), + '-ss', + '1.25', + '-t', + '4.25', + expect.stringMatching(/out\.mp4$/), + ]) + }) + + it('selects one thumbnail frame at the requested input time', async () => { + await runFfmpegOperation('thumbnail', [videoInput], { start: 3 }) + + expect(capturedArgs[0]).toEqual([ + '-y', + '-nostdin', + '-ss', + '3', + '-i', + expect.stringMatching(/in-0\.mp4$/), + '-frames:v', + '1', + expect.stringMatching(/out\.jpg$/), + ]) + }) +}) + describe('runFfmpegOperation process bounds', () => { it('kills a command that outlives the operation budget', async () => { vi.useFakeTimers({ toFake: ['setTimeout', 'clearTimeout'] }) diff --git a/apps/sim/lib/media/ffmpeg.ts b/apps/sim/lib/media/ffmpeg.ts index 6ec74914f6b..16c3f396250 100644 --- a/apps/sim/lib/media/ffmpeg.ts +++ b/apps/sim/lib/media/ffmpeg.ts @@ -1,18 +1,14 @@ -import { execFile, execSync } from 'node:child_process' import fs from 'node:fs/promises' import os from 'node:os' import path from 'node:path' -import { promisify } from 'node:util' import { createLogger } from '@sim/logger' import { toError } from '@sim/utils/errors' -import ffmpeg from 'fluent-ffmpeg' import { MAX_MEDIA_BYTES } from '@/lib/media/falai' import { FFMPEG_LIMITS } from '@/lib/media/ffmpeg-limits' +import { FFMPEG_BASE_ARGS, resolveExecutable, runExecutable } from '@/lib/media/ffmpeg-process' const logger = createLogger('MediaFfmpeg') -const execFileAsync = promisify(execFile) - const INSTALL_HINT = 'Install: brew install ffmpeg (macOS) / apk add ffmpeg (Alpine) / apt-get install ffmpeg (Ubuntu)' @@ -42,12 +38,7 @@ let ffmpegPath: string | null = null let ffprobePath: string | null = null function resolveBinary(binary: string): string | null { - try { - const cmd = process.platform === 'win32' ? `where ${binary}` : `which ${binary}` - return execSync(cmd, { encoding: 'utf-8' }).trim().split('\n')[0] || null - } catch { - return null - } + return resolveExecutable(binary) } /** Lazy system FFmpeg binary resolution, mirroring lib/audio/extractor.ts. */ @@ -56,8 +47,7 @@ function ensureFfmpeg(): void { binariesInitialized = true ffmpegPath = resolveBinary('ffmpeg') ffprobePath = resolveBinary('ffprobe') - if (ffmpegPath) ffmpeg.setFfmpegPath(ffmpegPath) - else logger.warn('[FFmpeg] No FFmpeg binary found at init time') + if (!ffmpegPath) logger.warn('[FFmpeg] No FFmpeg binary found at init time') } if (!ffmpegPath) throw new Error(`FFmpeg not found. ${INSTALL_HINT}`) } @@ -191,7 +181,13 @@ const EXT_TO_MIME: Record = { } function extFromMime(mime: string): string { - return MIME_TO_EXT[mime] || mime.split('/')[1] || 'bin' + const normalizedMime = mime.split(';', 1)[0].trim().toLowerCase() + if (Object.hasOwn(MIME_TO_EXT, normalizedMime)) { + return MIME_TO_EXT[normalizedMime] + } + + const subtype = normalizedMime.split('/')[1] + return subtype && /^[a-z0-9][a-z0-9.+_-]{0,63}$/.test(subtype) ? subtype : 'bin' } function mimeFromExt(ext: string): string { @@ -281,75 +277,35 @@ async function writeInput(dir: string, file: MediaFile, index: number): Promise< * the instance's cores for as long as it likes and survives the request that * asked for it. */ -function runCommand( +async function runCommand( ctx: FfmpegRunContext, - command: ffmpeg.FfmpegCommand, - outputPath: string + args: string[], + outputPath: string, + cwd?: string ): Promise { - return new Promise((resolve, reject) => { - if (ctx.signal?.aborted) { - reject(new Error(CANCELLED_MESSAGE)) - return - } - const remaining = ctx.deadlineAt - Date.now() - if (remaining <= 0) { - reject(new Error(timedOutMessage())) - return - } - - let settled = false - const timer = setTimeout(() => terminate(new Error(timedOutMessage())), remaining) + if (ctx.signal?.aborted) throw new Error(CANCELLED_MESSAGE) + const remaining = ctx.deadlineAt - Date.now() + if (remaining <= 0) throw new Error(timedOutMessage()) - function cleanup() { - clearTimeout(timer) - ctx.signal?.removeEventListener('abort', onAbort) - } - function succeed() { - if (settled) return - settled = true - cleanup() - resolve() - } - function fail(error: Error) { - if (settled) return - settled = true - cleanup() - reject(error) - } - /** - * SIGKILL rather than SIGTERM: the process being torn down is either wedged - * or deliberately expensive, and neither deserves a chance to ignore it. - * Settling first makes the `error` event FFmpeg emits on death a no-op, so - * the caller sees why we killed it instead of "killed with signal SIGKILL". - */ - function terminate(error: Error) { - if (settled) return - settled = true - cleanup() - try { - command.kill('SIGKILL') - } catch { - // Already exited — nothing to signal. - } - reject(error) - } - function onAbort() { - terminate(new Error(CANCELLED_MESSAGE)) + ensureFfmpeg() + try { + await runExecutable(ffmpegPath as string, [...FFMPEG_BASE_ARGS, ...args, outputPath], { + cwd, + maxOutputBytes: PROBE_MAX_OUTPUT_BYTES, + signal: ctx.signal, + timeoutMs: remaining, + }) + } catch (error) { + const failure = error as NodeJS.ErrnoException & { killed?: boolean; stderr?: string } + if (failure.name === 'AbortError' || ctx.signal?.aborted) { + throw new Error(CANCELLED_MESSAGE) } - - ctx.signal?.addEventListener('abort', onAbort, { once: true }) - - try { - command - .on('end', () => succeed()) - .on('error', (err) => fail(new Error(`FFmpeg error: ${err.message}`))) - .save(outputPath) - } catch (error) { - // Settle through `fail` rather than letting the executor throw, so the - // deadline timer is cleared instead of holding the event loop open. - fail(toError(error)) + if (failure.code === 'ERR_CHILD_PROCESS_STDIO_MAXBUFFER') { + throw new Error('FFmpeg error: process output was too large to read') } - }) + if (failure.killed) throw new Error(timedOutMessage()) + throw new Error(`FFmpeg error: ${failure.stderr?.trim() || toError(error).message}`) + } } interface FfprobeReport { @@ -363,10 +319,9 @@ interface FfprobeReport { } /** - * Probe with `execFile` rather than `fluent-ffmpeg`'s static `ffprobe`, which - * hands back no process handle: its callback can only be raced, leaving a - * wedged prober alive on the instance. `concat` probes once per input, so that - * leak scales with the request. + * Probe through the shared `execFile` boundary so the child is killed on a + * deadline or cancellation and its diagnostic output is bounded. `concat` + * probes once per input, so an unbounded prober would scale with the request. */ async function probeFile(ctx: FfmpegRunContext, filePath: string): Promise { const binary = ensureFfprobe() @@ -375,15 +330,13 @@ async function probeFile(ctx: FfmpegRunContext, filePath: string): Promise { if (inputPaths.length < 2) throw new Error('overlay_audio requires [video, audio]') const outputPath = path.join(dir, 'out.mp4') - const command = ffmpeg().input(inputPaths[0]) + const args = ['-i', inputPaths[0]] if (options.loopToVideo) { - command.input(inputPaths[1]).inputOptions(['-stream_loop', '-1']) + args.push('-stream_loop', '-1', '-i', inputPaths[1]) } else { - command.input(inputPaths[1]) + args.push('-i', inputPaths[1]) } - command.outputOptions([ - '-map', - '0:v:0', - '-map', - '1:a:0', - '-c:v', - 'copy', - '-c:a', - 'aac', - '-shortest', - ]) - await runCommand(ctx, command, outputPath) + args.push('-map', '0:v:0', '-map', '1:a:0', '-c:v', 'copy', '-c:a', 'aac', '-shortest') + await runCommand(ctx, args, outputPath) return readOut(outputPath, 'mp4') } @@ -533,16 +476,16 @@ async function mixAudio( const outputPath = path.join(dir, 'out.mp3') const voiceVol = options.volume ?? 1 const musicVol = options.musicVolume ?? 0.3 - const command = ffmpeg() - .input(inputPaths[0]) - .input(inputPaths[1]) - .complexFilter([ - `[0:a]volume=${voiceVol}[v]`, - `[1:a]volume=${musicVol}[m]`, - `[v][m]amix=inputs=2:duration=longest:dropout_transition=0[a]`, - ]) - .outputOptions(['-map', '[a]']) - await runCommand(ctx, command, outputPath) + const filter = [ + `[0:a]volume=${voiceVol}[v]`, + `[1:a]volume=${musicVol}[m]`, + `[v][m]amix=inputs=2:duration=longest:dropout_transition=0[a]`, + ].join(';') + await runCommand( + ctx, + ['-i', inputPaths[0], '-i', inputPaths[1], '-filter_complex', filter, '-map', '[a]'], + outputPath + ) return readOut(outputPath, 'mp3') } @@ -573,47 +516,50 @@ async function concat( const normalized: string[] = [] for (let i = 0; i < inputPaths.length; i++) { const out = path.join(dir, `norm-${i}.mp4`) - const cmd = ffmpeg().input(inputPaths[i]) + const args = ['-i', inputPaths[i]] const maps: string[] = ['-map', '0:v:0'] const extra: string[] = [] if (probes[i].hasAudio) { maps.push('-map', '0:a:0') } else { - cmd - .input('anullsrc=channel_layout=stereo:sample_rate=48000') - .inputOptions(['-f', 'lavfi', '-t', String(probes[i].durationSeconds || 1)]) + args.push( + '-f', + 'lavfi', + '-t', + String(probes[i].durationSeconds || 1), + '-i', + 'anullsrc=channel_layout=stereo:sample_rate=48000' + ) maps.push('-map', '1:a:0') extra.push('-shortest') } - cmd - .videoFilters( - `scale=${width}:${height}:force_original_aspect_ratio=decrease,pad=${width}:${height}:(ow-iw)/2:(oh-ih)/2,setsar=1,fps=${fps},format=yuv420p` - ) - .outputOptions([ - ...maps, - '-c:v', - 'libx264', - '-preset', - 'medium', - '-crf', - '18', - '-pix_fmt', - 'yuv420p', - '-r', - String(fps), - '-video_track_timescale', - '90000', - '-c:a', - 'aac', - '-b:a', - '192k', - '-ar', - '48000', - '-ac', - '2', - ...extra, - ]) - await runCommand(ctx, cmd, out) + args.push( + '-vf', + `scale=${width}:${height}:force_original_aspect_ratio=decrease,pad=${width}:${height}:(ow-iw)/2:(oh-ih)/2,setsar=1,fps=${fps},format=yuv420p`, + ...maps, + '-c:v', + 'libx264', + '-preset', + 'medium', + '-crf', + '18', + '-pix_fmt', + 'yuv420p', + '-r', + String(fps), + '-video_track_timescale', + '90000', + '-c:a', + 'aac', + '-b:a', + '192k', + '-ar', + '48000', + '-ac', + '2', + ...extra + ) + await runCommand(ctx, args, out) normalized.push(out) } @@ -624,11 +570,11 @@ async function concat( normalized.map((p) => `file '${p.replace(/'/g, "'\\''")}'`).join('\n') ) const outputPath = path.join(dir, 'out.mp4') - const concatCmd = ffmpeg() - .input(listPath) - .inputOptions(['-f', 'concat', '-safe', '0']) - .outputOptions(['-c', 'copy', '-movflags', '+faststart']) - await runCommand(ctx, concatCmd, outputPath) + await runCommand( + ctx, + ['-f', 'concat', '-safe', '0', '-i', listPath, '-c', 'copy', '-movflags', '+faststart'], + outputPath + ) return readOut(outputPath, 'mp4') } @@ -642,11 +588,13 @@ async function trim( const ext = extFromMime(input.mimeType) const outputPath = path.join(dir, `out.${ext}`) const start = options.start ?? 0 - const command = ffmpeg(inputPath).setStartTime(start) + // Output-side -ss (after -i) preserves fluent-ffmpeg's setStartTime + // semantics: frame-accurate trim starts instead of keyframe-snapped ones. + const args = ['-i', inputPath, '-ss', String(start)] if (options.end !== undefined) { - command.setDuration(Math.max(0, options.end - start)) + args.push('-t', String(Math.max(0, options.end - start))) } - await runCommand(ctx, command, outputPath) + await runCommand(ctx, args, outputPath) return readOut(outputPath, ext) } @@ -734,12 +682,18 @@ async function scalePad( } const outputPath = path.join(dir, 'out.mp4') - const command = ffmpeg(inputPath) - .videoFilters( - `scale=${width}:${height}:force_original_aspect_ratio=decrease,pad=${width}:${height}:(ow-iw)/2:(oh-ih)/2,setsar=1` - ) - .outputOptions(['-c:a', 'copy']) - await runCommand(ctx, command, outputPath) + await runCommand( + ctx, + [ + '-i', + inputPath, + '-vf', + `scale=${width}:${height}:force_original_aspect_ratio=decrease,pad=${width}:${height}:(ow-iw)/2:(oh-ih)/2,setsar=1`, + '-c:a', + 'copy', + ], + outputPath + ) return readOut(outputPath, 'mp4') } @@ -752,12 +706,24 @@ async function overlayImage( if (inputPaths.length < 2) throw new Error('overlay_image requires [video, image]') const xy = OVERLAY_POSITION[options.position || 'top-right'] || OVERLAY_POSITION['top-right'] const outputPath = path.join(dir, 'out.mp4') - const command = ffmpeg() - .input(inputPaths[0]) - .input(inputPaths[1]) - .complexFilter([`[0:v][1:v]overlay=${xy}[v]`]) - .outputOptions(['-map', '[v]', '-map', '0:a?', '-c:a', 'copy']) - await runCommand(ctx, command, outputPath) + await runCommand( + ctx, + [ + '-i', + inputPaths[0], + '-i', + inputPaths[1], + '-filter_complex', + `[0:v][1:v]overlay=${xy}[v]`, + '-map', + '[v]', + '-map', + '0:a?', + '-c:a', + 'copy', + ], + outputPath + ) return readOut(outputPath, 'mp4') } @@ -798,10 +764,12 @@ async function addText( `y=${pos.y}`, ].join(':') const outputPath = path.join(dir, 'out.mp4') - const command = ffmpeg(inputPath, { cwd: dir }) - .videoFilters(`drawtext=${drawtext}`) - .outputOptions(['-c:a', 'copy']) - await runCommand(ctx, command, outputPath) + await runCommand( + ctx, + ['-i', inputPath, '-vf', `drawtext=${drawtext}`, '-c:a', 'copy'], + outputPath, + dir + ) return readOut(outputPath, 'mp4') } @@ -819,12 +787,18 @@ async function fade( const isVideo = input.mimeType.startsWith('video/') || probe.hasVideo const ext = isVideo ? 'mp4' : extFromMime(input.mimeType) const outputPath = path.join(dir, `out.${ext}`) - const command = ffmpeg(inputPath) + const args = ['-i', inputPath] if (isVideo) { - command.videoFilters([`fade=t=in:st=0:d=${fadeDur}`, `fade=t=out:st=${outStart}:d=${fadeDur}`]) + args.push( + '-vf', + [`fade=t=in:st=0:d=${fadeDur}`, `fade=t=out:st=${outStart}:d=${fadeDur}`].join(',') + ) } - command.audioFilters([`afade=t=in:st=0:d=${fadeDur}`, `afade=t=out:st=${outStart}:d=${fadeDur}`]) - await runCommand(ctx, command, outputPath) + args.push( + '-af', + [`afade=t=in:st=0:d=${fadeDur}`, `afade=t=out:st=${outStart}:d=${fadeDur}`].join(',') + ) + await runCommand(ctx, args, outputPath) return readOut(outputPath, ext) } @@ -836,8 +810,7 @@ async function extractAudio( ): Promise { const ext = (options.format || 'mp3').toLowerCase() const outputPath = outputPathForExt(dir, ext) - const command = ffmpeg(inputPath).noVideo() - await runCommand(ctx, command, outputPath) + await runCommand(ctx, ['-i', inputPath, '-vn'], outputPath) return readOut(outputPath, ext) } @@ -850,7 +823,7 @@ async function convert( if (!options.format) throw new Error('convert requires a target format') const ext = options.format.toLowerCase() const outputPath = outputPathForExt(dir, ext) - await runCommand(ctx, ffmpeg(inputPath), outputPath) + await runCommand(ctx, ['-i', inputPath], outputPath) return readOut(outputPath, ext) } @@ -861,10 +834,11 @@ async function thumbnail( options: FfmpegOptions ): Promise { const outputPath = path.join(dir, 'out.jpg') - const command = ffmpeg(inputPath) - .seekInput(options.start ?? 0) - .frames(1) - await runCommand(ctx, command, outputPath) + await runCommand( + ctx, + ['-ss', String(options.start ?? 0), '-i', inputPath, '-frames:v', '1'], + outputPath + ) return readOut(outputPath, 'jpg') } diff --git a/apps/sim/lib/memory/application/operations.ts b/apps/sim/lib/memory/application/operations.ts index 253df541c3d..3a99abef426 100644 --- a/apps/sim/lib/memory/application/operations.ts +++ b/apps/sim/lib/memory/application/operations.ts @@ -5,11 +5,18 @@ const MEMORY_EXECUTOR_PRINCIPAL_POLICY = { delegatedServices: ['executor'], } as const +/** + * Memory is the executor's own store: an Agent block writes and reads it inside + * a run the workspace already authorized, and no permission-group key names it. + * A gate here would fail runs the group permits rather than withhold a + * capability from a member, so all four operations declare `'none'`. + */ function readOperation(id: Id) { return defineWorkspaceOperation({ id, minimumRole: 'read', workspaceApiKey: 'deny', + capability: 'none', ...MEMORY_EXECUTOR_PRINCIPAL_POLICY, }) } @@ -19,13 +26,18 @@ function writeOperation(id: Id) { id, minimumRole: 'write', workspaceApiKey: 'deny', + capability: 'none', ...MEMORY_EXECUTOR_PRINCIPAL_POLICY, }) } export const memoryOperations = { + // permission-group-exempt: the executor's own per-run store; no group key names it, and refusing would fail runs the group allows list: readOperation('memory.list'), + // permission-group-exempt: the executor's own per-run store; no group key names it, and refusing would fail runs the group allows read: readOperation('memory.read'), + // permission-group-exempt: the executor's own per-run store; no group key names it, and refusing would fail runs the group allows append: writeOperation('memory.append'), + // permission-group-exempt: the executor's own per-run store; no group key names it, and refusing would fail runs the group allows delete: writeOperation('memory.delete'), } as const diff --git a/apps/sim/lib/oauth/__tests__/terminal-errors.test.ts b/apps/sim/lib/oauth/__tests__/terminal-errors.test.ts index 9755c64b2d7..cb4434e7dc5 100644 --- a/apps/sim/lib/oauth/__tests__/terminal-errors.test.ts +++ b/apps/sim/lib/oauth/__tests__/terminal-errors.test.ts @@ -3,6 +3,10 @@ */ import { redisConfigMockFns } from '@sim/testing' import { beforeEach, describe, expect, it, vi } from 'vitest' +import { + clearOAuthRefreshDeadFlag, + getOAuthRefreshCoordinationIdentity, +} from '@/lib/oauth/refresh-coordination' import { clearDeadFlag, getRecentTerminalError, @@ -74,6 +78,27 @@ describe('markCredentialDead / getRecentTerminalError / clearDeadFlag', () => { expect(await getRecentTerminalError('acc-1')).toBeNull() }) + it.each(['account-1', 'slack:T08CM6ZNYBE'])( + 'reconnect clears the matching private refresh flag for %s', + async (scopeKey) => { + const redis = createFakeRedis() + redisConfigMockFns.mockGetRedisClient.mockReturnValue(redis as never) + const identity = getOAuthRefreshCoordinationIdentity(scopeKey) + + await markCredentialDead(identity, 'invalid_refresh_token') + await clearOAuthRefreshDeadFlag(scopeKey) + + expect(redis.set).toHaveBeenCalledWith( + `oauth:dead:${identity}`, + 'invalid_refresh_token', + 'EX', + 3600 + ) + expect(redis.del).toHaveBeenCalledWith(`oauth:dead:${identity}`) + expect(identity).not.toContain(scopeKey) + } + ) + it('all functions are no-ops when Redis is unavailable', async () => { await expect(markCredentialDead('acc-1', 'code')).resolves.toBeUndefined() await expect(getRecentTerminalError('acc-1')).resolves.toBeNull() diff --git a/apps/sim/lib/oauth/credential-service.test.ts b/apps/sim/lib/oauth/credential-service.test.ts new file mode 100644 index 00000000000..337b56aa435 --- /dev/null +++ b/apps/sim/lib/oauth/credential-service.test.ts @@ -0,0 +1,202 @@ +/** + * @vitest-environment node + */ +import { account, credential } from '@sim/db/schema' +import { queueTableRows, resetDbChainMock } from '@sim/testing' +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const mocks = vi.hoisted(() => ({ + coalesceLocally: vi.fn(), + getFreshestSlackChain: vi.fn(), + getRecentTerminalError: vi.fn(), + logger: { + info: vi.fn(), + warn: vi.fn(), + error: vi.fn(), + debug: vi.fn(), + trace: vi.fn(), + fatal: vi.fn(), + }, + refreshOAuthToken: vi.fn(), + withLeaderLock: vi.fn(), +})) + +vi.mock('@sim/logger', () => ({ + createLogger: vi.fn(() => mocks.logger), +})) + +vi.mock('@/lib/concurrency/singleflight', () => ({ + coalesceLocally: mocks.coalesceLocally, +})) + +vi.mock('@/lib/concurrency/leader-lock', () => ({ + withLeaderLock: mocks.withLeaderLock, +})) + +vi.mock('@/lib/oauth/instagram', () => ({ + isInstagramProvider: vi.fn(() => false), + shouldProactivelyRefreshInstagramToken: vi.fn(() => false), +})) + +vi.mock('@/lib/oauth/microsoft', () => ({ + getMicrosoftRefreshTokenExpiry: vi.fn(), + isMicrosoftProvider: vi.fn(() => false), + PROACTIVE_REFRESH_THRESHOLD_DAYS: 7, +})) + +vi.mock('@/lib/oauth/oauth', () => ({ + OAUTH_PROVIDERS: {}, + refreshOAuthToken: mocks.refreshOAuthToken, +})) + +vi.mock('@/lib/oauth/slack', () => ({ + extractSlackTeamId: (value: string | null | undefined) => + value?.match(/^([TE][A-Z0-9]+)-/)?.[1] ?? null, + fanOutSlackTokenChain: vi.fn(), + getFreshestSlackChain: mocks.getFreshestSlackChain, + hasSlackChainMoved: vi.fn(() => false), + isSlackProvider: (providerId: string) => providerId === 'slack', +})) + +vi.mock('@/lib/oauth/terminal-errors', () => ({ + getRecentTerminalError: mocks.getRecentTerminalError, + isTerminalRefreshError: vi.fn(() => false), + markCredentialDead: vi.fn(), +})) + +import { resolveCredentialTokenBundle } from '@/lib/oauth/credential-service' + +const RAW_CREDENTIAL_ID = 'credential-raw-secret-id' +const RAW_ACCOUNT_ID = 'account-raw-secret-id' +const RAW_USER_ID = 'user-raw-secret-id' +const RAW_SLACK_TEAM_ID = 'TSECRET123' +const RAW_PROVIDER_ERROR = 'provider returned raw private failure text' + +interface RefreshObservation { + cacheKey: string + coalescingKey: string + lockKey: string + logs: string +} + +async function observeRefresh( + providerId: 'google' | 'slack', + privacyMode?: 'selector' +): Promise { + resetDbChainMock() + vi.clearAllMocks() + mocks.getRecentTerminalError.mockResolvedValue(null) + mocks.coalesceLocally.mockImplementation(async (_key: string, producer: () => Promise) => + producer() + ) + mocks.withLeaderLock.mockImplementation(async (options: { onLeader: () => Promise }) => + options.onLeader() + ) + mocks.getFreshestSlackChain.mockResolvedValue({ + accessToken: null, + refreshToken: 'refresh-token', + accessTokenExpiresAt: new Date(0), + chainVersion: new Date(0), + }) + mocks.refreshOAuthToken.mockRejectedValue(new Error(RAW_PROVIDER_ERROR)) + + queueTableRows(credential, [ + { + id: RAW_CREDENTIAL_ID, + type: 'oauth', + accountId: RAW_ACCOUNT_ID, + workspaceId: 'workspace-1', + providerId: null, + }, + ]) + queueTableRows(account, [ + { + id: RAW_ACCOUNT_ID, + accountId: + providerId === 'slack' + ? `${RAW_SLACK_TEAM_ID}-usr_USECRET-connection` + : 'provider-account-id', + providerId, + userId: RAW_USER_ID, + accessToken: null, + refreshToken: 'refresh-token', + accessTokenExpiresAt: new Date(0), + refreshTokenExpiresAt: null, + updatedAt: new Date(0), + }, + ]) + + await expect( + resolveCredentialTokenBundle( + RAW_CREDENTIAL_ID, + RAW_USER_ID, + 'selector-execution', + undefined, + undefined, + privacyMode ? { privacyMode } : undefined + ) + ).resolves.toBeNull() + + return { + cacheKey: mocks.getRecentTerminalError.mock.calls[0][0], + coalescingKey: mocks.coalesceLocally.mock.calls[0][0], + lockKey: mocks.withLeaderLock.mock.calls[0][0].key, + logs: JSON.stringify([ + ...mocks.logger.info.mock.calls, + ...mocks.logger.warn.mock.calls, + ...mocks.logger.error.mock.calls, + ]), + } +} + +describe('resolveCredentialTokenBundle selector privacy', () => { + beforeEach(() => { + vi.clearAllMocks() + resetDbChainMock() + }) + + it('HMACs OAuth and Slack refresh identities and suppresses raw identifiers and provider errors', async () => { + for (const providerId of ['google', 'slack'] as const) { + const observed = await observeRefresh(providerId, 'selector') + const serializedKeys = JSON.stringify([ + observed.cacheKey, + observed.coalescingKey, + observed.lockKey, + ]) + + expect(observed.coalescingKey).toBe(observed.lockKey) + expect(observed.coalescingKey).toMatch(/^oauth:refresh:[A-Za-z0-9_-]{40,}$/) + for (const privateValue of [ + RAW_CREDENTIAL_ID, + RAW_ACCOUNT_ID, + RAW_USER_ID, + RAW_SLACK_TEAM_ID, + RAW_PROVIDER_ERROR, + ]) { + expect(serializedKeys).not.toContain(privateValue) + expect(observed.logs).not.toContain(privateValue) + } + } + }) + + it('shares private refresh coordination across privacy modes without changing ordinary diagnostics', async () => { + const privateGoogle = await observeRefresh('google', 'selector') + const google = await observeRefresh('google') + expect(google.cacheKey).toBe(privateGoogle.cacheKey) + expect(google.coalescingKey).toBe(privateGoogle.coalescingKey) + expect(google.lockKey).toBe(google.coalescingKey) + expect(google.coalescingKey).not.toContain(RAW_ACCOUNT_ID) + expect(google.logs).toContain(RAW_ACCOUNT_ID) + expect(google.logs).toContain(RAW_USER_ID) + expect(google.logs).toContain(RAW_PROVIDER_ERROR) + + const privateSlack = await observeRefresh('slack', 'selector') + const slack = await observeRefresh('slack') + expect(slack.cacheKey).toBe(privateSlack.cacheKey) + expect(slack.coalescingKey).toBe(privateSlack.coalescingKey) + expect(slack.lockKey).toBe(slack.coalescingKey) + expect(slack.coalescingKey).not.toContain(RAW_SLACK_TEAM_ID) + expect(slack.logs).toContain(RAW_SLACK_TEAM_ID) + expect(slack.logs).toContain(RAW_PROVIDER_ERROR) + }) +}) diff --git a/apps/sim/lib/oauth/credential-service.ts b/apps/sim/lib/oauth/credential-service.ts index a92468715e6..0962cf8cb56 100644 --- a/apps/sim/lib/oauth/credential-service.ts +++ b/apps/sim/lib/oauth/credential-service.ts @@ -1,4 +1,4 @@ -import { createSign } from 'crypto' +import { createHmac, createSign } from 'crypto' import { db } from '@sim/db' import { account, credential } from '@sim/db/schema' import { createLogger } from '@sim/logger' @@ -6,6 +6,7 @@ import { getPostgresErrorCode, toError } from '@sim/utils/errors' import { and, desc, eq } from 'drizzle-orm' import { withLeaderLock } from '@/lib/concurrency/leader-lock' import { coalesceLocally } from '@/lib/concurrency/singleflight' +import { env } from '@/lib/core/config/env' import { decryptSecret } from '@/lib/core/security/encryption' import { isClientCredentialAccountProviderId } from '@/lib/credentials/client-credential-accounts/descriptors' import { @@ -27,6 +28,7 @@ import { PROACTIVE_REFRESH_THRESHOLD_DAYS, } from '@/lib/oauth/microsoft' import { refreshOAuthToken } from '@/lib/oauth/oauth' +import { getOAuthRefreshCoordinationIdentity } from '@/lib/oauth/refresh-coordination' import { extractSlackTeamId, fanOutSlackTokenChain, @@ -48,6 +50,24 @@ import { const logger = createLogger('OAuthCredentialService') +export interface CredentialTokenResolutionOptions { + /** + * Selector execution may receive a credential/account id through a hidden + * workspace environment reference. In that mode identifiers are omitted + * from diagnostics. Refresh coordination identities are private in every + * mode so selector and ordinary calls share the same locks and dead flags. + */ + privacyMode?: 'selector' +} + +function privateCredentialIdentity(namespace: string, value: string): string { + return createHmac('sha256', env.ENCRYPTION_KEY) + .update(namespace) + .update('\0') + .update(value) + .digest('base64url') +} + export class ServiceAccountTokenError extends Error { constructor( public readonly statusCode: number, @@ -160,7 +180,8 @@ const SA_EXCLUDED_SCOPES = new Set([ export async function getServiceAccountToken( credentialId: string, scopes: string[], - impersonateEmail?: string + impersonateEmail?: string, + options?: CredentialTokenResolutionOptions ): Promise { const [credentialRow] = await db .select({ @@ -203,12 +224,22 @@ export async function getServiceAccountToken( payload.sub = impersonateEmail } - logger.info('Service account JWT payload', { - iss: keyData.client_email, - sub: impersonateEmail || '(none)', - scopes: filteredScopes.join(' '), - aud: tokenUri, - }) + logger.info( + 'Service account JWT payload', + options?.privacyMode === 'selector' + ? { + hasIssuer: Boolean(keyData.client_email), + hasSubject: Boolean(impersonateEmail), + scopeCount: filteredScopes.length, + hasAudience: Boolean(tokenUri), + } + : { + iss: keyData.client_email, + sub: impersonateEmail || '(none)', + scopes: filteredScopes.join(' '), + aud: tokenUri, + } + ) const toBase64Url = (obj: unknown) => Buffer.from(JSON.stringify(obj)).toString('base64url') @@ -222,6 +253,7 @@ export async function getServiceAccountToken( const response = await fetch(tokenUri, { method: 'POST', + redirect: 'error', headers: { 'Content-Type': 'application/x-www-form-urlencoded' }, body: new URLSearchParams({ grant_type: 'urn:ietf:params:oauth:grant-type:jwt-bearer', @@ -233,21 +265,23 @@ export async function getServiceAccountToken( const errorBody = await response.text() logger.error('Service account token exchange failed', { status: response.status, - body: errorBody, + ...(options?.privacyMode === 'selector' ? {} : { body: errorBody }), }) let description = `Token exchange failed: ${response.status}` - try { - const parsed = JSON.parse(errorBody) as { error_description?: string } - if (parsed.error_description) { - const raw = parsed.error_description - if (raw.includes('SignatureException') || raw.includes('Invalid signature')) { - description = 'Invalid account credentials.' - } else { - description = raw + if (options?.privacyMode !== 'selector') { + try { + const parsed = JSON.parse(errorBody) as { error_description?: string } + if (parsed.error_description) { + const raw = parsed.error_description + if (raw.includes('SignatureException') || raw.includes('Invalid signature')) { + description = 'Invalid account credentials.' + } else { + description = raw + } } + } catch { + // use default description } - } catch { - // use default description } throw new ServiceAccountTokenError(response.status, description) } @@ -491,9 +525,14 @@ function secretFingerprintOf(encryptedServiceAccountKey: string): string { */ async function resolveClientCredentialAccountToken( credentialId: string, - providerId: string + providerId: string, + options?: CredentialTokenResolutionOptions ): Promise { - return coalesceLocally(`ccsa:${credentialId}`, async () => { + const cacheIdentity = + options?.privacyMode === 'selector' + ? privateCredentialIdentity('selector-client-credential', credentialId) + : credentialId + return coalesceLocally(`ccsa:${cacheIdentity}`, async () => { pruneExpiredClientCredentialCaches(Date.now()) const [credentialRow] = await db .select({ encryptedServiceAccountKey: credential.encryptedServiceAccountKey }) @@ -501,13 +540,13 @@ async function resolveClientCredentialAccountToken( .where(eq(credential.id, credentialId)) .limit(1) if (!credentialRow?.encryptedServiceAccountKey) { - clientCredentialTokenCache.delete(credentialId) - clientCredentialMintFailureCache.delete(credentialId) + clientCredentialTokenCache.delete(cacheIdentity) + clientCredentialMintFailureCache.delete(cacheIdentity) throw new Error('Client-credential service account secret not found') } const secretFingerprint = secretFingerprintOf(credentialRow.encryptedServiceAccountKey) - const cached = clientCredentialTokenCache.get(credentialId) + const cached = clientCredentialTokenCache.get(cacheIdentity) if ( cached && cached.secretFingerprint === secretFingerprint && @@ -520,7 +559,7 @@ async function resolveClientCredentialAccountToken( } } - const failed = clientCredentialMintFailureCache.get(credentialId) + const failed = clientCredentialMintFailureCache.get(cacheIdentity) if ( failed && failed.secretFingerprint === secretFingerprint && @@ -528,7 +567,7 @@ async function resolveClientCredentialAccountToken( ) { throw failed.error } - clientCredentialMintFailureCache.delete(credentialId) + clientCredentialMintFailureCache.delete(cacheIdentity) try { const { decrypted } = await decryptSecret(credentialRow.encryptedServiceAccountKey) @@ -551,7 +590,7 @@ async function resolveClientCredentialAccountToken( }, { skipIdentity: true } ) - clientCredentialTokenCache.set(credentialId, { + clientCredentialTokenCache.set(cacheIdentity, { accessToken: mint.accessToken, expiresAtMs: Date.now() + mint.expiresInSeconds * 1000, secretFingerprint, @@ -564,8 +603,8 @@ async function resolveClientCredentialAccountToken( apiDomain: mint.apiDomain, } } catch (error) { - clientCredentialMintFailureCache.set(credentialId, { - error, + clientCredentialMintFailureCache.set(cacheIdentity, { + error: options?.privacyMode === 'selector' ? new Error('Credential mint failed') : error, secretFingerprint, expiresAtMs: Date.now() + CLIENT_CREDENTIAL_MINT_FAILURE_TTL_MS, }) @@ -577,6 +616,7 @@ async function resolveClientCredentialAccountToken( interface ServiceAccountTokenOptions { scopes?: string[] impersonateEmail?: string + privacyMode?: 'selector' } type ServiceAccountTokenResolver = ( @@ -601,11 +641,18 @@ const SERVICE_ACCOUNT_TOKEN_RESOLVERS: Record { + [GOOGLE_SERVICE_ACCOUNT_PROVIDER_ID]: async ( + credentialId, + { scopes, impersonateEmail, privacyMode } + ) => { if (!scopes?.length) { throw new Error('Scopes are required for service account credentials') } - return { accessToken: await getServiceAccountToken(credentialId, scopes, impersonateEmail) } + return { + accessToken: await getServiceAccountToken(credentialId, scopes, impersonateEmail, { + privacyMode, + }), + } }, } @@ -620,7 +667,8 @@ export async function resolveServiceAccountToken( credentialId: string, providerId: string | null | undefined, scopes?: string[], - impersonateEmail?: string + impersonateEmail?: string, + options?: CredentialTokenResolutionOptions ): Promise { if (providerId && isTokenServiceAccountProviderId(providerId)) { const secret = await getTokenServiceAccountSecret(credentialId, providerId) @@ -632,7 +680,7 @@ export async function resolveServiceAccountToken( } } if (providerId && isClientCredentialAccountProviderId(providerId)) { - return resolveClientCredentialAccountToken(credentialId, providerId) + return resolveClientCredentialAccountToken(credentialId, providerId, options) } const resolver = providerId && Object.hasOwn(SERVICE_ACCOUNT_TOKEN_RESOLVERS, providerId) @@ -641,7 +689,7 @@ export async function resolveServiceAccountToken( if (!resolver) { throw new Error(`Unsupported service-account provider: ${providerId ?? 'unknown'}`) } - return resolver(credentialId, { scopes, impersonateEmail }) + return resolver(credentialId, { scopes, impersonateEmail, ...options }) } /** @@ -708,6 +756,7 @@ interface CoalescedRefreshOptions { providerAccountId?: string | null requestId?: string userId?: string + privacyMode?: 'selector' } /** @@ -731,6 +780,7 @@ async function performCoalescedRefresh({ providerAccountId, requestId, userId, + privacyMode, }: CoalescedRefreshOptions): Promise { /** * Slack bot tokens are per-installation (team × app): every account row for @@ -738,14 +788,15 @@ async function performCoalescedRefresh({ * dead-flagged, and written per installation rather than per row. */ const slackTeamId = isSlackProvider(providerId) ? extractSlackTeamId(providerAccountId) : null - const scopeKey = slackTeamId ? `slack:${slackTeamId}` : accountId + const rawScopeKey = slackTeamId ? `slack:${slackTeamId}` : accountId + const scopeKey = getOAuthRefreshCoordinationIdentity(rawScopeKey) const logContext = { ...(requestId ? { requestId } : {}), - ...(userId ? { userId } : {}), - ...(slackTeamId ? { slackTeamId } : {}), + ...(privacyMode === 'selector' || !userId ? {} : { userId }), + ...(privacyMode === 'selector' || !slackTeamId ? {} : { slackTeamId }), providerId, - accountId, + ...(privacyMode === 'selector' ? {} : { accountId }), } const deadCode = await getRecentTerminalError(scopeKey) @@ -805,6 +856,7 @@ async function performCoalescedRefresh({ logger.error('Failed to refresh token', { ...logContext, errorCode: result.errorCode, + message: result.message, }) if (result.errorCode && isTerminalRefreshError(result.errorCode)) { // A refresh that lost a race with a concurrent connect fails with @@ -856,7 +908,7 @@ async function performCoalescedRefresh({ } catch (error) { logger.error('Refresh failed inside leader path', { ...logContext, - error: toError(error).message, + ...(privacyMode === 'selector' ? {} : { error: toError(error).message }), }) return null } @@ -883,7 +935,7 @@ async function performCoalescedRefresh({ } catch (error) { logger.warn('Follower DB read failed during refresh poll', { ...logContext, - error: toError(error).message, + ...(privacyMode === 'selector' ? {} : { error: toError(error).message }), }) return null } @@ -896,7 +948,7 @@ async function performCoalescedRefresh({ } catch (error) { logger.error('Coalesced refresh did not settle', { ...logContext, - error: toError(error).message, + ...(privacyMode === 'selector' ? {} : { error: toError(error).message }), }) return null } @@ -976,12 +1028,13 @@ export async function getOAuthToken(userId: string, providerId: string): Promise * Pipedrive's `x-api-token`. OAuth credentials resolve with `accessToken` * only. */ -export async function resolveCredentialAccessToken( +export async function resolveCredentialTokenBundle( credentialId: string, userId: string, requestId: string, scopes?: string[], - impersonateEmail?: string + impersonateEmail?: string, + options?: CredentialTokenResolutionOptions ): Promise { const resolved = await resolveOAuthAccountId(credentialId) if (!resolved) { @@ -994,7 +1047,8 @@ export async function resolveCredentialAccessToken( resolved.credentialId, resolved.providerId, scopes, - impersonateEmail + impersonateEmail, + options ) } @@ -1052,6 +1106,7 @@ export async function resolveCredentialAccessToken( providerAccountId: credential.accountId, requestId, userId: credential.userId, + privacyMode: options?.privacyMode, }) if (fresh) return { accessToken: fresh } @@ -1076,7 +1131,7 @@ export async function resolveCredentialAccessToken( /** * Refreshes an OAuth token if needed based on credential information. * Also handles service account credentials by generating a JWT-based token. - * Thin string wrapper over {@link resolveCredentialAccessToken}. + * Thin string wrapper over {@link resolveCredentialTokenBundle}. * @param credentialId The ID of the credential to check and potentially refresh * @param userId The user ID who owns the credential (for security verification) * @param requestId Request ID for log correlation @@ -1088,14 +1143,16 @@ export async function refreshAccessTokenIfNeeded( userId: string, requestId: string, scopes?: string[], - impersonateEmail?: string + impersonateEmail?: string, + options?: CredentialTokenResolutionOptions ): Promise { - const result = await resolveCredentialAccessToken( + const result = await resolveCredentialTokenBundle( credentialId, userId, requestId, scopes, - impersonateEmail + impersonateEmail, + options ) return result?.accessToken ?? null } diff --git a/apps/sim/lib/oauth/oauth.test.ts b/apps/sim/lib/oauth/oauth.test.ts index 51ce0371382..3aadd1da130 100644 --- a/apps/sim/lib/oauth/oauth.test.ts +++ b/apps/sim/lib/oauth/oauth.test.ts @@ -1,5 +1,5 @@ -import { getOAuth2Tokens } from '@better-auth/core/oauth2' import { createMockFetch, resetEnvMock, setEnv } from '@sim/testing' +import { getOAuth2Tokens } from 'better-auth/oauth2' import { afterAll, beforeAll, describe, expect, it, vi } from 'vitest' beforeAll(() => { @@ -78,6 +78,9 @@ import { } from '@/lib/oauth' import { REDDIT_USER_AGENT } from '@/tools/reddit/constants' +/** Compares real icon components by identity; the global `@/components/icons` stub in vitest.setup.ts would make that vacuous. */ +vi.unmock('@/components/icons') + /** * Default OAuth token response for successful requests. */ diff --git a/apps/sim/lib/oauth/refresh-coordination.ts b/apps/sim/lib/oauth/refresh-coordination.ts new file mode 100644 index 00000000000..0f1f7466122 --- /dev/null +++ b/apps/sim/lib/oauth/refresh-coordination.ts @@ -0,0 +1,21 @@ +import { createHmac } from 'crypto' +import { env } from '@/lib/core/config/env' +import { clearDeadFlag } from '@/lib/oauth/terminal-errors' + +/** + * Returns the private identity shared by OAuth refresh locks and terminal-error + * flags. Callers choose the semantic scope: an account row for ordinary OAuth, + * or `slack:${teamId}` for a Slack installation. + */ +export function getOAuthRefreshCoordinationIdentity(scopeKey: string): string { + return createHmac('sha256', env.ENCRYPTION_KEY) + .update('oauth-refresh') + .update('\0') + .update(scopeKey) + .digest('base64url') +} + +/** Clears the terminal-error flag written by the refresh path for one raw scope. */ +export function clearOAuthRefreshDeadFlag(scopeKey: string): Promise { + return clearDeadFlag(getOAuthRefreshCoordinationIdentity(scopeKey)) +} diff --git a/apps/sim/lib/oauth/token-resolution.test.ts b/apps/sim/lib/oauth/token-resolution.test.ts index 611604f4144..e06ee3c9c8d 100644 --- a/apps/sim/lib/oauth/token-resolution.test.ts +++ b/apps/sim/lib/oauth/token-resolution.test.ts @@ -5,14 +5,20 @@ import { beforeEach, describe, expect, it, vi } from 'vitest' const { mockAuthorizeCredentialUseForAuth, + mockCaptureServerEvent, + mockExecuteManagedToken, mockGetCredential, + mockGetToolMetadata, mockRecordAudit, mockRefreshTokenIfNeeded, mockResolveOAuthAccountId, mockResolveServiceAccountToken, } = vi.hoisted(() => ({ mockAuthorizeCredentialUseForAuth: vi.fn(), + mockCaptureServerEvent: vi.fn(), + mockExecuteManagedToken: vi.fn(), mockGetCredential: vi.fn(), + mockGetToolMetadata: vi.fn(), mockRecordAudit: vi.fn(), mockRefreshTokenIfNeeded: vi.fn(), mockResolveOAuthAccountId: vi.fn(), @@ -37,18 +43,55 @@ vi.mock('@/lib/oauth/credential-service', () => ({ })) vi.mock('@/lib/posthog/server', () => ({ - captureServerEvent: vi.fn(), + captureServerEvent: mockCaptureServerEvent, })) +vi.mock('@/lib/credentials/application/managed-oauth-delegation', () => ({ + InvalidManagedOAuthDelegationError: class InvalidManagedOAuthDelegationError extends Error { + constructor() { + super('Managed credential execution requires valid workflow delegation') + this.name = 'InvalidManagedOAuthDelegationError' + } + }, + authenticateManagedOAuthDelegation: vi.fn(), +})) + +vi.mock('@/lib/credentials/application/resolve-managed-oauth-token', () => ({ + resolveManagedOAuthCredentialToken: { execute: mockExecuteManagedToken }, +})) + +vi.mock('@/lib/credentials/managed-oauth', () => ({ + ManagedOAuthCredentialError: class ManagedOAuthCredentialError extends Error { + constructor( + message: string, + readonly code: string, + readonly statusCode: number + ) { + super(message) + this.name = 'ManagedOAuthCredentialError' + } + }, +})) + +vi.mock('@/tools/metadata', () => ({ + getToolMetadata: mockGetToolMetadata, +})) + +vi.mock('@/lib/oauth/utils', () => ({ + getCanonicalScopesForProvider: vi.fn().mockReturnValue([]), +})) + +import { OrchestrationError } from '@/lib/core/orchestration/types' +import { InvalidManagedOAuthDelegationError } from '@/lib/credentials/application/managed-oauth-delegation' +import { ManagedOAuthCredentialError } from '@/lib/credentials/managed-oauth' import { TokenServiceAccountValidationError } from '@/lib/credentials/token-service-accounts/errors' -import { resolveCredentialToken } from '@/lib/oauth/token-resolution' +import { resolveCredentialAccessToken, resolveCredentialToken } from '@/lib/oauth/token-resolution' const INTERNAL_AUTH = { success: true, userId: 'user-1', authType: 'internal_jwt' } as const describe('resolveCredentialToken', () => { beforeEach(() => { vi.clearAllMocks() - mockResolveOAuthAccountId.mockResolvedValue(null) }) it('fails closed when the credential is not authorized', async () => { @@ -59,6 +102,7 @@ describe('resolveCredentialToken', () => { const result = await resolveCredentialToken(INTERNAL_AUTH, { requestId: 'req-1', + resolvedCredential: null, credentialId: 'cred-1', }) @@ -80,7 +124,7 @@ describe('resolveCredentialToken', () => { const result = await resolveCredentialToken( { success: true, authType: 'internal_jwt' }, - { requestId: 'req-1', credentialId: 'cred-1' } + { requestId: 'req-1', credentialId: 'cred-1', resolvedCredential: null } ) expect(result).toEqual({ ok: false, status: 403, error: 'Authentication required' }) @@ -103,11 +147,15 @@ describe('resolveCredentialToken', () => { const result = await resolveCredentialToken(INTERNAL_AUTH, { requestId: 'req-1', + resolvedCredential: null, credentialId: 'cred-1', workflowId: 'wf-1', }) - expect(result).toEqual({ ok: true, token: { accessToken: 'fresh', idToken: 'id-token' } }) + expect(result).toEqual({ + ok: true, + token: { accessToken: 'fresh', credentialType: 'oauth', idToken: 'id-token' }, + }) expect(mockGetCredential).toHaveBeenCalledWith('req-1', 'account-1', 'owner-1') expect(mockRefreshTokenIfNeeded).toHaveBeenCalled() expect(mockRecordAudit).toHaveBeenCalledWith( @@ -130,6 +178,7 @@ describe('resolveCredentialToken', () => { const result = await resolveCredentialToken(INTERNAL_AUTH, { requestId: 'req-1', + resolvedCredential: null, credentialId: 'cred-1', }) @@ -154,12 +203,14 @@ describe('resolveCredentialToken', () => { await expect( resolveCredentialToken(INTERNAL_AUTH, { requestId: 'req-1', + resolvedCredential: null, credentialId: 'cred-1', }) ).resolves.toEqual({ ok: true, token: { accessToken: 'fresh', + credentialType: 'oauth', idToken: undefined, instanceUrl: 'https://contoso.api.crm.dynamics.com', }, @@ -183,11 +234,12 @@ describe('resolveCredentialToken', () => { await expect( resolveCredentialToken(INTERNAL_AUTH, { requestId: 'req-1', + resolvedCredential: null, credentialId: 'cred-1', }) ).resolves.toEqual({ ok: true, - token: { accessToken: 'fresh', idToken: undefined }, + token: { accessToken: 'fresh', credentialType: 'oauth', idToken: undefined }, }) expect(mockRefreshTokenIfNeeded).toHaveBeenCalled() }) @@ -203,6 +255,7 @@ describe('resolveCredentialToken', () => { const result = await resolveCredentialToken(INTERNAL_AUTH, { requestId: 'req-1', + resolvedCredential: null, credentialId: 'cred-1', }) @@ -211,19 +264,19 @@ describe('resolveCredentialToken', () => { }) it('authorizes service-account credentials before minting a token', async () => { - mockResolveOAuthAccountId.mockResolvedValue({ - credentialType: 'service_account', - credentialId: 'sa-1', - providerId: 'google', - workspaceId: 'ws-1', - accountId: '', - usedCredentialTable: true, - }) mockAuthorizeCredentialUseForAuth.mockResolvedValue({ ok: false, error: 'Unauthorized' }) const result = await resolveCredentialToken(INTERNAL_AUTH, { requestId: 'req-1', credentialId: 'cred-1', + resolvedCredential: { + credentialType: 'service_account', + credentialId: 'sa-1', + providerId: 'google', + workspaceId: 'ws-1', + accountId: '', + usedCredentialTable: true, + }, }) expect(result).toEqual({ ok: false, status: 403, error: 'Unauthorized' }) @@ -231,14 +284,6 @@ describe('resolveCredentialToken', () => { }) it('surfaces the classified service-account failure code', async () => { - mockResolveOAuthAccountId.mockResolvedValue({ - credentialType: 'service_account', - credentialId: 'sa-1', - providerId: 'atlassian', - workspaceId: 'ws-1', - accountId: '', - usedCredentialTable: true, - }) mockAuthorizeCredentialUseForAuth.mockResolvedValue({ ok: true, requesterUserId: 'user-1' }) mockResolveServiceAccountToken.mockRejectedValue( new TokenServiceAccountValidationError('invalid_credentials', 401) @@ -247,6 +292,14 @@ describe('resolveCredentialToken', () => { const result = await resolveCredentialToken(INTERNAL_AUTH, { requestId: 'req-1', credentialId: 'cred-1', + resolvedCredential: { + credentialType: 'service_account', + credentialId: 'sa-1', + providerId: 'atlassian', + workspaceId: 'ws-1', + accountId: '', + usedCredentialTable: true, + }, }) expect(result).toEqual({ @@ -260,6 +313,7 @@ describe('resolveCredentialToken', () => { it('rejects a malformed impersonation subject before touching the credential', async () => { const result = await resolveCredentialToken(INTERNAL_AUTH, { requestId: 'req-1', + resolvedCredential: null, credentialId: 'cred-1', impersonateEmail: 'not-an-email', }) @@ -268,3 +322,297 @@ describe('resolveCredentialToken', () => { expect(mockAuthorizeCredentialUseForAuth).not.toHaveBeenCalled() }) }) + +const MANAGED_RESOLVED = { + credentialType: 'managed_oauth', + credentialId: 'managed-1', + providerId: 'google', + workspaceId: 'ws-1', + accountId: '', + usedCredentialTable: true, +} as const + +const EXECUTOR_PRINCIPAL = { + kind: 'delegated', + serviceId: 'executor', + subjectUserId: 'user-1', + workspaceId: 'ws-1', +} as never + +describe('resolveCredentialAccessToken', () => { + const authenticate = vi.fn() + const resolveManagedPrincipal = vi.fn() + + beforeEach(() => { + vi.clearAllMocks() + mockResolveOAuthAccountId.mockResolvedValue(null) + authenticate.mockResolvedValue(INTERNAL_AUTH) + resolveManagedPrincipal.mockResolvedValue(EXECUTOR_PRINCIPAL) + mockGetToolMetadata.mockReturnValue({ + oauth: { required: true, provider: 'google', requiredScopes: ['scope-a'] }, + }) + }) + + it('authenticates and delegates non-managed credentials without a second account lookup', async () => { + mockAuthorizeCredentialUseForAuth.mockResolvedValue({ + ok: true, + requesterUserId: 'user-1', + credentialOwnerUserId: 'owner-1', + workspaceId: 'ws-1', + resolvedCredentialId: 'account-1', + }) + mockGetCredential.mockResolvedValue({ providerId: 'google' }) + mockRefreshTokenIfNeeded.mockResolvedValue({ accessToken: 'fresh', refreshed: false }) + + const result = await resolveCredentialAccessToken({ + requestId: 'req-1', + credentialId: 'cred-1', + workflowId: 'wf-1', + callerUserId: 'user-1', + authenticate, + resolveManagedPrincipal, + }) + + expect(result).toEqual({ + ok: true, + token: { accessToken: 'fresh', credentialType: 'oauth', idToken: undefined }, + }) + expect(authenticate).toHaveBeenCalledTimes(1) + expect(resolveManagedPrincipal).not.toHaveBeenCalled() + expect(mockResolveOAuthAccountId).toHaveBeenCalledTimes(1) + expect(mockAuthorizeCredentialUseForAuth).toHaveBeenCalledWith(INTERNAL_AUTH, { + credentialId: 'cred-1', + workflowId: 'wf-1', + callerUserId: 'user-1', + }) + }) + + it('treats an empty impersonation subject as absent', async () => { + mockAuthorizeCredentialUseForAuth.mockResolvedValue({ + ok: true, + requesterUserId: 'user-1', + credentialOwnerUserId: 'owner-1', + workspaceId: 'ws-1', + resolvedCredentialId: 'account-1', + }) + mockGetCredential.mockResolvedValue({ providerId: 'google' }) + mockRefreshTokenIfNeeded.mockResolvedValue({ accessToken: 'fresh', refreshed: false }) + + const result = await resolveCredentialAccessToken({ + requestId: 'req-1', + credentialId: 'cred-1', + impersonateEmail: '', + authenticate, + }) + + expect(result).toEqual({ + ok: true, + token: { accessToken: 'fresh', credentialType: 'oauth', idToken: undefined }, + }) + }) + + it('rejects a managed credential when no delegation resolver is wired', async () => { + mockResolveOAuthAccountId.mockResolvedValue(MANAGED_RESOLVED) + + const result = await resolveCredentialAccessToken({ + requestId: 'req-1', + credentialId: 'cred-1', + toolId: 'gmail_send', + authenticate, + }) + + expect(result).toEqual({ + ok: false, + status: 403, + code: 'MANAGED_CREDENTIAL_DELEGATION_REQUIRED', + error: 'Managed credentials can only be used by an authenticated workflow execution', + }) + expect(authenticate).not.toHaveBeenCalled() + }) + + it('maps an invalid delegation to 401 with its message', async () => { + mockResolveOAuthAccountId.mockResolvedValue(MANAGED_RESOLVED) + resolveManagedPrincipal.mockRejectedValue(new InvalidManagedOAuthDelegationError()) + + const result = await resolveCredentialAccessToken({ + requestId: 'req-1', + credentialId: 'cred-1', + toolId: 'gmail_send', + authenticate, + resolveManagedPrincipal, + }) + + expect(result).toEqual({ + ok: false, + status: 401, + code: 'MANAGED_CREDENTIAL_DELEGATION_INVALID', + error: 'Managed credential execution requires valid workflow delegation', + }) + expect(resolveManagedPrincipal).toHaveBeenCalledWith('managed-1') + }) + + it('rethrows unexpected delegation resolver failures', async () => { + mockResolveOAuthAccountId.mockResolvedValue(MANAGED_RESOLVED) + resolveManagedPrincipal.mockRejectedValue(new Error('db unavailable')) + + await expect( + resolveCredentialAccessToken({ + requestId: 'req-1', + credentialId: 'cred-1', + toolId: 'gmail_send', + authenticate, + resolveManagedPrincipal, + }) + ).rejects.toThrow('db unavailable') + }) + + it('requires a tool id for managed credentials', async () => { + mockResolveOAuthAccountId.mockResolvedValue(MANAGED_RESOLVED) + + const result = await resolveCredentialAccessToken({ + requestId: 'req-1', + credentialId: 'cred-1', + authenticate, + resolveManagedPrincipal, + }) + + expect(result).toEqual({ + ok: false, + status: 400, + code: 'MANAGED_CREDENTIAL_TOOL_REQUIRED', + error: 'A tool ID is required to use a managed credential', + }) + }) + + it('rejects tools without managed OAuth support', async () => { + mockResolveOAuthAccountId.mockResolvedValue(MANAGED_RESOLVED) + mockGetToolMetadata.mockReturnValue({ oauth: undefined }) + + const result = await resolveCredentialAccessToken({ + requestId: 'req-1', + credentialId: 'cred-1', + toolId: 'http_request', + authenticate, + resolveManagedPrincipal, + }) + + expect(result).toEqual({ + ok: false, + status: 500, + code: 'MANAGED_CREDENTIAL_TOOL_UNSUPPORTED', + error: 'This tool is not configured to use managed credentials', + }) + expect(mockExecuteManagedToken).not.toHaveBeenCalled() + }) + + it('rejects tools whose scope policy is empty', async () => { + mockResolveOAuthAccountId.mockResolvedValue(MANAGED_RESOLVED) + mockGetToolMetadata.mockReturnValue({ oauth: { required: true, provider: 'google' } }) + + const result = await resolveCredentialAccessToken({ + requestId: 'req-1', + credentialId: 'cred-1', + toolId: 'gmail_send', + authenticate, + resolveManagedPrincipal, + }) + + expect(result).toEqual({ + ok: false, + status: 500, + code: 'MANAGED_CREDENTIAL_TOOL_UNSUPPORTED', + error: 'This tool is not configured to use managed credentials', + }) + }) + + it('resolves a managed credential through the use case and records analytics', async () => { + mockResolveOAuthAccountId.mockResolvedValue(MANAGED_RESOLVED) + mockExecuteManagedToken.mockResolvedValue({ accessToken: 'managed-token', idToken: 'id-1' }) + const auditRequest = { headers: { get: () => null } } + + const result = await resolveCredentialAccessToken({ + requestId: 'req-1', + credentialId: 'cred-1', + toolId: 'gmail_send', + auditRequest, + authenticate, + resolveManagedPrincipal, + }) + + expect(result).toEqual({ + ok: true, + token: { + accessToken: 'managed-token', + credentialType: 'managed_oauth', + idToken: 'id-1', + }, + }) + expect(mockExecuteManagedToken).toHaveBeenCalledWith({ + principal: EXECUTOR_PRINCIPAL, + input: { + credentialId: 'managed-1', + expectedProviderId: 'google', + requiredScopes: ['scope-a'], + toolId: 'gmail_send', + }, + request: auditRequest, + }) + expect(mockCaptureServerEvent).toHaveBeenCalledWith( + 'user-1', + 'credential_used', + expect.objectContaining({ credential_type: 'managed_oauth', provider_id: 'google' }), + { groups: { workspace: 'ws-1' } } + ) + expect(authenticate).not.toHaveBeenCalled() + }) + + it('projects managed credential rejections with their code and status', async () => { + mockResolveOAuthAccountId.mockResolvedValue(MANAGED_RESOLVED) + mockExecuteManagedToken.mockRejectedValue( + new ( + ManagedOAuthCredentialError as never as new ( + message: string, + code: string, + statusCode: number + ) => Error + )('Credential is disabled', 'MANAGED_CREDENTIAL_DISABLED', 403) + ) + + const result = await resolveCredentialAccessToken({ + requestId: 'req-1', + credentialId: 'cred-1', + toolId: 'gmail_send', + authenticate, + resolveManagedPrincipal, + }) + + expect(result).toEqual({ + ok: false, + status: 403, + code: 'MANAGED_CREDENTIAL_DISABLED', + error: 'Credential is disabled', + }) + }) + + it('projects orchestration failures as managed unauthorized', async () => { + mockResolveOAuthAccountId.mockResolvedValue(MANAGED_RESOLVED) + mockExecuteManagedToken.mockRejectedValue( + new OrchestrationError('not_found', 'Managed credential not found') + ) + + const result = await resolveCredentialAccessToken({ + requestId: 'req-1', + credentialId: 'cred-1', + toolId: 'gmail_send', + authenticate, + resolveManagedPrincipal, + }) + + expect(result).toEqual({ + ok: false, + status: 404, + code: 'MANAGED_CREDENTIAL_UNAUTHORIZED', + error: 'Managed credential not found', + }) + }) +}) diff --git a/apps/sim/lib/oauth/token-resolution.ts b/apps/sim/lib/oauth/token-resolution.ts index 501d69f40a1..ba903ef7eb5 100644 --- a/apps/sim/lib/oauth/token-resolution.ts +++ b/apps/sim/lib/oauth/token-resolution.ts @@ -1,4 +1,8 @@ import { AuditAction, AuditResourceType, recordAudit } from '@sim/audit' +import { + resolvePrincipalSubject, + type WorkflowExecutionDelegatedPrincipal, +} from '@sim/auth/principal' import { createLogger } from '@sim/logger' import { impersonateEmailSchema, @@ -6,6 +10,10 @@ import { } from '@/lib/api/contracts/oauth-connections' import { authorizeCredentialUseForAuth } from '@/lib/auth/credential-access' import type { AuthResult } from '@/lib/auth/hybrid' +import { asOrchestrationError, statusForOrchestrationError } from '@/lib/core/orchestration/types' +import { InvalidManagedOAuthDelegationError } from '@/lib/credentials/application/managed-oauth-delegation' +import { resolveManagedOAuthCredentialToken } from '@/lib/credentials/application/resolve-managed-oauth-token' +import { ManagedOAuthCredentialError } from '@/lib/credentials/managed-oauth' import { TokenServiceAccountValidationError } from '@/lib/credentials/token-service-accounts/errors' import { getCredential, @@ -19,7 +27,9 @@ import { MICROSOFT_DATAVERSE_PROVIDER_ID, } from '@/lib/oauth/microsoft-dataverse' import { extractSalesforceInstanceUrl, isSalesforceOAuthProviderId } from '@/lib/oauth/salesforce' +import { getCanonicalScopesForProvider } from '@/lib/oauth/utils' import { captureServerEvent } from '@/lib/posthog/server' +import { getToolMetadata } from '@/tools/metadata' import { extractZohoDeskBaseFromScope } from '@/tools/zoho_desk/host-allowlist' const logger = createLogger('OAuthTokenResolution') @@ -50,8 +60,8 @@ export interface ResolveCredentialTokenInput { */ callerUserId?: string auditRequest?: CredentialAuditRequest - /** Reuses a credential lookup already performed by the route's managed-OAuth dispatch. */ - resolvedCredential?: ResolvedCredential | null + /** Credential lookup already performed by {@link resolveCredentialAccessToken}'s dispatch. */ + resolvedCredential: ResolvedCredential | null } export type ResolveCredentialTokenResult = @@ -62,7 +72,7 @@ export type ResolveCredentialTokenResult = * Emits the semantic "credential used" trail for one resolved credential. * Both the audit row and the analytics event are fire-and-forget. */ -function recordCredentialAccess(params: { +export function recordCredentialAccess(params: { actorId: string workspaceId: string | null resourceId: string @@ -118,6 +128,7 @@ function buildOAuthTokenPayload( return { accessToken, + credentialType: 'oauth', idToken: credential.idToken || undefined, ...(instanceUrl && { instanceUrl }), ...(apiDomain && { apiDomain }), @@ -160,9 +171,10 @@ export async function completeOAuthCredentialToken(params: { } /** - * Authorized application operation behind `POST /api/auth/oauth/token`. Every surface that - * needs a credential token — the route and the in-process tool executor — goes through - * here, so authorization, refresh, and audit cannot drift between them. + * Resolves a plain OAuth or service-account credential to a token for an + * authenticated caller. Managed OAuth credentials are dispatched one level up by + * {@link resolveCredentialAccessToken}, which every server surface goes through, + * so authorization, refresh, and audit cannot drift between surfaces. * * @param auth Result of authenticating the caller (session or internal JWT). */ @@ -191,16 +203,12 @@ export async function resolveCredentialToken( return { ok: false, status: 400, error: 'impersonateEmail must be a valid email address' } } - /** - * Both branches below authorize with the same arguments, and neither read depends - * on the other, so they resolve together — this runs per credentialed tool call. - */ - const [resolved, authz] = await Promise.all([ - input.resolvedCredential === undefined - ? resolveOAuthAccountId(credentialId) - : input.resolvedCredential, - authorizeCredentialUseForAuth(auth, { credentialId, workflowId, callerUserId }), - ]) + const resolved = input.resolvedCredential + const authz = await authorizeCredentialUseForAuth(auth, { + credentialId, + workflowId, + callerUserId, + }) if (resolved?.credentialType === 'service_account' && resolved.credentialId) { if (!authz.ok) { @@ -233,6 +241,7 @@ export async function resolveCredentialToken( ok: true, token: { accessToken: result.accessToken, + credentialType: 'service_account', cloudId: result.cloudId, domain: result.domain, instanceUrl: result.instanceUrl, @@ -306,3 +315,166 @@ export async function resolveCredentialToken( return { ok: false, status: 500, error: 'Internal server error' } } } + +export interface ResolveCredentialAccessTokenInput + extends Omit { + /** Tool consuming the token; required by the managed-OAuth scope policy. */ + toolId?: string + /** + * Authenticates the caller for non-managed credentials. Invoked only when the + * credential is not managed OAuth, which authenticates through delegation instead. + */ + authenticate: () => AuthResult | Promise + /** + * Proves a workflow-execution delegation for one managed credential. The route + * verifies the delegation JWT header; the executor binds its delegation origin + * in-process. Absent, managed credentials are rejected with + * `MANAGED_CREDENTIAL_DELEGATION_REQUIRED`. Must throw + * {@link InvalidManagedOAuthDelegationError} on an invalid delegation. + */ + resolveManagedPrincipal?: (credentialId: string) => Promise +} + +/** + * Authorized application dispatch behind `POST /api/auth/oauth/token`. Every server + * surface that needs a credential token — the route and the in-process tool + * executor — goes through here, so the managed / service-account / plain-OAuth + * dispatch, authorization, refresh, audit, and analytics cannot drift between them. + */ +export async function resolveCredentialAccessToken( + input: ResolveCredentialAccessTokenInput +): Promise { + const { requestId, credentialId, toolId, auditRequest } = input + + const resolved = credentialId ? await resolveOAuthAccountId(credentialId) : null + + if (resolved?.credentialType !== 'managed_oauth' || !resolved.credentialId) { + const auth = await input.authenticate() + return resolveCredentialToken(auth, { + requestId, + credentialId, + workflowId: input.workflowId, + scopes: input.scopes, + /** + * In-process callers forward raw subblock state, where an untouched + * field is '' — treated as absent, matching what the wire contract + * (which rejects '') and the old truthy guards always produced. + */ + impersonateEmail: input.impersonateEmail || undefined, + callerUserId: input.callerUserId, + auditRequest, + resolvedCredential: resolved, + }) + } + + if (!input.resolveManagedPrincipal) { + return { + ok: false, + status: 403, + code: 'MANAGED_CREDENTIAL_DELEGATION_REQUIRED', + error: 'Managed credentials can only be used by an authenticated workflow execution', + } + } + + let principal: WorkflowExecutionDelegatedPrincipal + try { + principal = await input.resolveManagedPrincipal(resolved.credentialId) + } catch (error) { + if (!(error instanceof InvalidManagedOAuthDelegationError)) throw error + return { + ok: false, + status: 401, + code: 'MANAGED_CREDENTIAL_DELEGATION_INVALID', + error: error.message, + } + } + + if (!toolId) { + return { + ok: false, + status: 400, + code: 'MANAGED_CREDENTIAL_TOOL_REQUIRED', + error: 'A tool ID is required to use a managed credential', + } + } + + const toolMetadata = getToolMetadata(toolId) + if (!toolMetadata?.oauth?.required) { + logger.error(`[${requestId}] Tool is not configured for managed OAuth`, { toolId }) + return { + ok: false, + status: 500, + code: 'MANAGED_CREDENTIAL_TOOL_UNSUPPORTED', + error: 'This tool is not configured to use managed credentials', + } + } + const requiredScopes = + toolMetadata.oauth.requiredScopes ?? getCanonicalScopesForProvider(toolMetadata.oauth.provider) + if (requiredScopes.length === 0) { + logger.error(`[${requestId}] Tool has no trusted OAuth scope policy`, { + toolId, + providerId: toolMetadata.oauth.provider, + }) + return { + ok: false, + status: 500, + code: 'MANAGED_CREDENTIAL_TOOL_UNSUPPORTED', + error: 'This tool is not configured to use managed credentials', + } + } + + try { + const result = await resolveManagedOAuthCredentialToken.execute({ + principal, + input: { + credentialId: resolved.credentialId, + expectedProviderId: toolMetadata.oauth.provider, + requiredScopes, + toolId, + }, + request: auditRequest, + }) + + const subject = resolvePrincipalSubject(principal) + if (subject?.kind === 'sim_user') { + captureServerEvent( + subject.userId, + 'credential_used', + { + credential_type: 'managed_oauth', + provider_id: toolMetadata.oauth.provider, + workspace_id: principal.workspaceId, + }, + { groups: { workspace: principal.workspaceId } } + ) + } + + return { + ok: true, + token: { + accessToken: result.accessToken, + credentialType: 'managed_oauth', + idToken: result.idToken, + }, + } + } catch (error) { + if (error instanceof ManagedOAuthCredentialError) { + logger.warn(`[${requestId}] Managed OAuth credential rejected`, { + credentialId: resolved.credentialId, + code: error.code, + }) + return { ok: false, status: error.statusCode, code: error.code, error: error.message } + } + + const orchestrationError = asOrchestrationError(error) + if (orchestrationError) { + return { + ok: false, + status: statusForOrchestrationError(orchestrationError.code), + code: 'MANAGED_CREDENTIAL_UNAUTHORIZED', + error: orchestrationError.message, + } + } + throw error + } +} diff --git a/apps/sim/lib/oauth/utils.ts b/apps/sim/lib/oauth/utils.ts index fd4fb11d73c..e0a091a126e 100644 --- a/apps/sim/lib/oauth/utils.ts +++ b/apps/sim/lib/oauth/utils.ts @@ -313,7 +313,8 @@ export const SCOPE_DESCRIPTIONS: Record = { 'groups:write': 'Create, archive, and manage private channels', 'chat:write': 'Send messages', 'chat:write.public': 'Post to public channels', - 'assistant:write': 'Set assistant thread status, title, and suggested prompts', + 'chat:write.customize': 'Customize message username and icon', + 'assistant:write': 'Manage assistant status, titles, and suggested prompts', 'im:write': 'Send direct messages', 'im:history': 'Read direct message history', 'im:read': 'View direct message channels', diff --git a/apps/sim/lib/permission-groups/block-access.test.ts b/apps/sim/lib/permission-groups/block-access.test.ts new file mode 100644 index 00000000000..88514b10888 --- /dev/null +++ b/apps/sim/lib/permission-groups/block-access.test.ts @@ -0,0 +1,123 @@ +/** + * @vitest-environment node + */ +import { beforeEach, describe, expect, it, vi } from 'vitest' +import { + isAccessControlAllowlistRow, + isBlockTypeAccessControlExempt, +} from '@/lib/permission-groups/block-access' +import { getBlock } from '@/blocks/registry' + +const mockGetBlock = getBlock as unknown as ReturnType + +interface FakeBlock { + hideFromToolbar?: boolean + sunset?: { status: 'legacy' | 'deprecated'; replacedBy?: string } +} + +/** + * Only `hideFromToolbar` is read from here: the successor half of the decision + * comes from the generated map, so every id used below is a real one whose real + * successor the assertion depends on. + */ +function registry(blocks: Record) { + mockGetBlock.mockImplementation((type: string) => blocks[type]) +} + +describe('isBlockTypeAccessControlExempt', () => { + beforeEach(() => { + vi.clearAllMocks() + }) + + it('exempts the universal entry point', () => { + registry({}) + + expect(isBlockTypeAccessControlExempt('start_trigger')).toBe(true) + }) + + /** + * The bypass this closes: a legacy block is fully functional, so an allowlist + * naming only the current version used to be satisfied by the retired one. + */ + it('does not exempt a superseded block, which is judged as its successor', () => { + registry({ + slack: { hideFromToolbar: true, sunset: { status: 'legacy', replacedBy: 'slack_v2' } }, + slack_v2: {}, + }) + + expect(isBlockTypeAccessControlExempt('slack')).toBe(false) + }) + + /** + * A retired block with no successor has no row in the editor and nothing to + * be permitted as, so denying it would break older workflows an admin could + * not have rescued. + */ + it('exempts a retired block with no successor', () => { + registry({ thinking: { hideFromToolbar: true } }) + + expect(isBlockTypeAccessControlExempt('thinking')).toBe(true) + }) + + it('does not exempt a current block', () => { + registry({ slack_v2: {} }) + + expect(isBlockTypeAccessControlExempt('slack_v2')).toBe(false) + }) + + /** + * The editor never offers `start_trigger` as an allowlist row, so a retired + * entry point judged as its successor would be refused by every active + * allowlist — breaking every saved workflow that still carries one. + */ + it('exempts a retired entry point, whose successor is the universal one', () => { + registry({ + starter: { hideFromToolbar: true, sunset: { status: 'legacy', replacedBy: 'start_trigger' } }, + manual_trigger: { + hideFromToolbar: true, + sunset: { status: 'legacy', replacedBy: 'start_trigger' }, + }, + start_trigger: {}, + }) + + expect(isBlockTypeAccessControlExempt('starter')).toBe(true) + expect(isBlockTypeAccessControlExempt('manual_trigger')).toBe(true) + }) +}) + +describe('isAccessControlAllowlistRow', () => { + beforeEach(() => { + vi.clearAllMocks() + }) + + /** + * The bug this closes: the editor renders only visible blocks but used to + * materialize an allowlist from every non-exempt one. Unchecking `slack_v2` + * on a previously-unrestricted group therefore wrote `slack` into the stored + * list, and the runtime resolves `slack` to `slack_v2` — re-allowing exactly + * the integration the admin had just denied. + */ + it('is not a row for a superseded block, which has no row of its own', () => { + registry({ + slack: { hideFromToolbar: true, sunset: { status: 'legacy', replacedBy: 'slack_v2' } }, + slack_v2: {}, + }) + + expect(isAccessControlAllowlistRow('slack')).toBe(false) + expect(isBlockTypeAccessControlExempt('slack')).toBe(false) + }) + + it('is a row for a current block', () => { + registry({ slack_v2: {} }) + + expect(isAccessControlAllowlistRow('slack_v2')).toBe(true) + }) + + /** Exempt block types are decided by no row at all. */ + it('is not a row for an exempt block', () => { + registry({ thinking: { hideFromToolbar: true }, start_trigger: {} }) + + expect(isAccessControlAllowlistRow('thinking')).toBe(false) + expect(isAccessControlAllowlistRow('start_trigger')).toBe(false) + }) +}) diff --git a/apps/sim/lib/permission-groups/block-access.ts b/apps/sim/lib/permission-groups/block-access.ts index 884f952665a..e18a2068ae1 100644 --- a/apps/sim/lib/permission-groups/block-access.ts +++ b/apps/sim/lib/permission-groups/block-access.ts @@ -1,22 +1,66 @@ +import { resolveAccessControlBlockType } from '@/lib/permission-groups/integration-allowlist' import { getBlock } from '@/blocks/registry' +/** + * The universal workflow entry point. Every retired entry point resolves to it, + * and it is never an allowlist row, so both it and anything that resolves to it + * are exempt. + */ +const UNIVERSAL_ENTRY_POINT = 'start_trigger' + /** * Block types that bypass permission-group access control entirely. * - * Two kinds of blocks are exempt: - * - `start_trigger`: the universal workflow entry point. A workflow must always - * be startable regardless of the configured integration allowlist. - * - Legacy blocks (`hideFromToolbar: true`): superseded integration versions and - * deprecated blocks. They never appear in the toolbar or the Access Control - * admin list, so admins cannot allowlist them — yet they may still live inside - * older workflows. Exempting them keeps those workflows runnable instead of - * silently blocking blocks the admin had no way to permit. + * Three kinds are exempt: + * - `start_trigger`: the universal workflow entry point. A workflow must be + * startable whatever the integration allowlist says. + * - A retired block with no successor. It is hidden from the toolbar and from + * the Access Control editor, so an admin has no row to permit it on and + * nothing to permit it *as*; denying it would silently break the older + * workflows still carrying it. + * - A retired entry point — `starter`, `manual_trigger`, `api_trigger`, + * `chat_trigger` — whose successor is `start_trigger`. It is judged as the + * universal entry point, and the universal entry point is exempt, so it must + * be too. The editor never offers `start_trigger` as an allowlist row, so + * without this every active allowlist refuses every workflow still carrying + * an old starter block. * - * This is the single source of truth shared by both the runtime enforcement - * paths and the Access Control admin UI so the "hidden from the list" set and - * the "skipped by enforcement" set never drift apart. + * A *superseded* block is deliberately not exempt. Legacy `slack` talks to + * Slack exactly as `slack_v2` does, so exempting it let an allowlist naming + * `slack_v2` be satisfied by `slack` — reachable through workflow import, the + * API, or a Copilot-built workflow, and invisible to the admin who configured + * the allowlist. It is judged as its successor instead; see + * {@link resolveAccessControlBlockType}. + * + * Shared by the runtime enforcement paths and the Access Control editor, so the + * set that is hidden and the set that is skipped cannot drift apart. */ export function isBlockTypeAccessControlExempt(blockType: string): boolean { - if (blockType === 'start_trigger') return true - return getBlock(blockType)?.hideFromToolbar === true + if (blockType === UNIVERSAL_ENTRY_POINT) return true + const block = getBlock(blockType) + if (block?.hideFromToolbar !== true) return false + const successor = resolveAccessControlBlockType(blockType) + return successor === blockType || successor === UNIVERSAL_ENTRY_POINT +} + +/** + * Whether `blockType` is a row in the Access Control editor's allowlist + * universe — the set the editor materializes an allowlist from and compares + * against to collapse one back to `null`. + * + * Narrower than {@link isBlockTypeAccessControlExempt} on purpose. A superseded + * block must stay non-exempt at runtime (legacy `slack` reaches Slack and is + * judged as `slack_v2`), but it must not be an editor row: the editor renders + * only visible blocks, so an admin narrowing a previously-unrestricted + * allowlist by unchecking `slack_v2` would still write the hidden `slack` into + * it — and canonical resolution then reads that entry as `slack_v2` and allows + * the very integration the admin just denied. + * + * Viewer-independent, like the exemption: it reads the pure registry and the + * generated successor map, never the visibility projection, so a preview block + * gated for the acting admin stays in the universe and keeps its stored grant. + */ +export function isAccessControlAllowlistRow(blockType: string): boolean { + if (isBlockTypeAccessControlExempt(blockType)) return false + return resolveAccessControlBlockType(blockType) === blockType } diff --git a/apps/sim/lib/permission-groups/block-successors.generated.ts b/apps/sim/lib/permission-groups/block-successors.generated.ts new file mode 100644 index 00000000000..d4f7c30c858 --- /dev/null +++ b/apps/sim/lib/permission-groups/block-successors.generated.ts @@ -0,0 +1,52 @@ +/** + * Generated by `bun run generate:block-successors` from the block registry. + * Do not edit this file directly. + * + * Maps a retired block type to the *terminal* type an access-control decision + * about it is made against — `sunset.replacedBy`, followed transitively. It + * exists as a generated projection because `lib/permission-groups/` may not + * import `blocks/`; see `scripts/generate-block-successors.ts`. + */ +export const BLOCK_ACCESS_SUCCESSORS: Record = { + api_trigger: 'start_trigger', + chat_trigger: 'start_trigger', + confluence: 'confluence_v2', + cursor: 'cursor_v2', + extend: 'extend_v2', + file: 'file_v5', + file_v2: 'file_v5', + file_v3: 'file_v5', + file_v4: 'file_v5', + fireflies: 'fireflies_v2', + github: 'github_v2', + gmail: 'gmail_v2', + google_calendar: 'google_calendar_v2', + google_sheets: 'google_sheets_v2', + google_slides: 'google_slides_v2', + grain: 'grain_v2', + human_in_the_loop: 'human_in_the_loop_v2', + image_generator: 'image_generator_v2', + input_trigger: 'start_trigger', + intercom: 'intercom_v2', + kalshi: 'kalshi_v2', + linear: 'linear_v2', + logs: 'logs_v2', + manual_trigger: 'start_trigger', + microsoft_excel: 'microsoft_excel_v2', + mistral_parse: 'mistral_parse_v3', + mistral_parse_v2: 'mistral_parse_v3', + notion: 'notion_v2', + openai: 'embeddings', + pulse: 'pulse_v2', + reducto: 'reducto_v2', + router: 'router_v2', + sharepoint: 'sharepoint_v2', + slack: 'slack_v2', + starter: 'start_trigger', + stt: 'stt_v2', + table: 'table_v2', + textract: 'textract_v2', + video_generator: 'video_generator_v3', + video_generator_v2: 'video_generator_v3', + workflow: 'workflow_input', +} diff --git a/apps/sim/lib/permission-groups/capabilities.test.ts b/apps/sim/lib/permission-groups/capabilities.test.ts new file mode 100644 index 00000000000..30e19dc4cf0 --- /dev/null +++ b/apps/sim/lib/permission-groups/capabilities.test.ts @@ -0,0 +1,49 @@ +/** + * @vitest-environment node + */ + +import { describe, expect, it } from 'vitest' +import { CAPABILITY_RULES } from '@/lib/permission-groups/capabilities' +import { + DEFAULT_PERMISSION_GROUP_CONFIG, + type PermissionGroupConfig, +} from '@/lib/permission-groups/fields' + +function configWith(overrides: Partial): PermissionGroupConfig { + return { ...DEFAULT_PERMISSION_GROUP_CONFIG, ...overrides } +} + +describe('knowledge capability rules', () => { + const create = CAPABILITY_RULES['knowledge.create'] + const upload = CAPABILITY_RULES['knowledge.upload'] + const connectors = CAPABILITY_RULES['knowledge.connectors'] + + it('permits creation and upload under the unrestricted config', () => { + expect(create.deniedBy(DEFAULT_PERMISSION_GROUP_CONFIG)).toBe(false) + expect(upload.deniedBy(DEFAULT_PERMISSION_GROUP_CONFIG)).toBe(false) + }) + + it('withholds creation and upload from their own keys', () => { + expect(create.deniedBy(configWith({ disableKnowledgeBaseCreation: true }))).toBe(true) + expect(upload.deniedBy(configWith({ disableKnowledgeBaseFileUpload: true }))).toBe(true) + }) + + it('subsumes the module-wide key, since an operation declares only one capability', () => { + const hidden = configWith({ hideKnowledgeBaseTab: true }) + expect(create.deniedBy(hidden)).toBe(true) + expect(upload.deniedBy(hidden)).toBe(true) + }) + + it('reads the connector allow-list as a named set, with null meaning unrestricted', () => { + expect(connectors.deniedBy(DEFAULT_PERMISSION_GROUP_CONFIG, 'confluence')).toBe(false) + + const narrowed = configWith({ allowedKnowledgeConnectors: ['google_drive'] }) + expect(connectors.deniedBy(narrowed, 'google_drive')).toBe(false) + expect(connectors.deniedBy(narrowed, 'confluence')).toBe(true) + }) + + it('withholds every connector when the allow-list is emptied rather than cleared', () => { + const emptied = configWith({ allowedKnowledgeConnectors: [] }) + expect(connectors.deniedBy(emptied, 'google_drive')).toBe(true) + }) +}) diff --git a/apps/sim/lib/permission-groups/capabilities.ts b/apps/sim/lib/permission-groups/capabilities.ts new file mode 100644 index 00000000000..c6e67edae20 --- /dev/null +++ b/apps/sim/lib/permission-groups/capabilities.ts @@ -0,0 +1,444 @@ +import type { ForbiddenDetailCode } from '@/lib/core/application/forbidden' +import { PermissionGroupCapabilityError } from '@/lib/permission-groups/capability-error' +import type { + PermissionGroupConfig, + PermissionGroupConfigKey, +} from '@/lib/permission-groups/fields' + +/** + * Every capability a permission group can withhold. + * + * Domain-shaped, because the declaration site is an operation + * (`tables.rows.create`) while the config is surface-shaped (`hideTablesTab`). + * {@link CAPABILITY_RULES} is the only place the two vocabularies meet — without + * it every domain's operations module would restate a config key, and the + * mapping would drift the first time one was renamed. + * + * A closed union rather than a predicate on the operation: operations are + * frozen policy data, and a closure cannot be logged, compared, or read by + * `check:permission-group-enforcement`. Fifty table operations naming one + * capability is fifty identical strings, not fifty identical functions. + */ +export const CAPABILITY_IDS = [ + 'knowledge.use', + 'tables.use', + 'files.use', + 'inbox.use', + 'copilot.use', + 'secrets.manage', + 'api_keys.manage', + 'integrations.manage', + 'deploy.api', + 'deploy.mcp', + 'deploy.chat', + 'deploy.chat.auth_mode', + 'file_share.publish', + 'file_share.auth_mode', + 'public_api.use', + 'invitations.send', + 'mcp_tools.use', + 'custom_tools.use', + 'skills.use', + 'logs.trace_spans', + 'personal_api_key.use', + 'logs.export', + 'logs.cost', + 'knowledge.create', + 'knowledge.upload', + 'knowledge.connectors', + 'tables.create', + 'tables.export', + 'files.bulk_download', + 'credentials.personal', + 'workspace.create', + 'organization.member_directory', + 'cli.use', + 'triggers.webhook', + 'copilot.tool_auto_approval', +] as const + +export type PermissionGroupCapability = (typeof CAPABILITY_IDS)[number] + +interface CapabilityRuleBase { + /** The config keys this rule reads, so the audit can prove a key is enforced. */ + readonly configKeys: readonly PermissionGroupConfigKey[] + readonly detailCode: ForbiddenDetailCode + /** + * The subject of the shared refusal sentence — ` is not available + * under your organization's permission group` — so it is written as a + * singular noun or gerund phrase that agrees with the verb. + */ + readonly describe: string +} + +/** + * Decidable from the config alone, so the authorization funnel can apply it + * knowing only the principal, the workspace, and the operation. + */ +export interface StaticCapabilityRule extends CapabilityRuleBase { + readonly kind: 'static' + deniedBy(config: PermissionGroupConfig): boolean +} + +/** + * Needs a value only the request carries. The funnel never sees request input, + * and widening the authorization context to carry it would reach every use case + * for the sake of two keys, so these are asserted from inside `execute` and held + * to account by the audit's annotation rule instead. They are not valid as an + * operation's declared capability. + */ +export interface ParameterizedCapabilityRule extends CapabilityRuleBase { + readonly kind: 'parameterized' + deniedBy(config: PermissionGroupConfig, parameter: string): boolean +} + +export type CapabilityRule = StaticCapabilityRule | ParameterizedCapabilityRule + +/** + * The one sentence every capability refusal uses, wherever it is raised. + * + * Shared so the funnel, a raw route gating inline, and a parameterized rule + * asserted from a use case cannot word the same refusal three ways. Each rule's + * `describe` is written to read as this sentence's subject. + */ +export function capabilityRefusal(capability: PermissionGroupCapability): string { + return `${CAPABILITY_RULES[capability].describe} is not available under your organization's permission group` +} + +/** + * Throws {@link capabilityRefusal} as the error the surfaces project. + * + * Accepts any capability, static or parameterized, because a parameterized one + * is refused from a call site rather than by the funnel and still has to read + * identically. + */ +export function refuseCapability(capability: PermissionGroupCapability): never { + throw new PermissionGroupCapabilityError( + capability, + CAPABILITY_RULES[capability].detailCode, + capabilityRefusal(capability) + ) +} + +/** An allowlist denies a member when it is set and does not name it; `null` names everything. */ +function allowlistDenies(allowed: readonly string[] | null, member: string): boolean { + return allowed !== null && !allowed.includes(member) +} + +/** + * What each capability means in terms of the stored config. + * + * `satisfies` rather than an annotation, so adding a capability still fails to + * compile until it is given a rule — the same completeness gate + * `FORBIDDEN_DETAIL_CODE_DESCRIPTIONS` uses — while each entry keeps its own + * `kind`. Annotating would widen every entry to `CapabilityRule`, and + * {@link StaticPermissionGroupCapability} would then resolve to `never`, + * silently rejecting every capability an operation tried to declare. + */ +export const CAPABILITY_RULES = { + 'knowledge.use': { + kind: 'static', + configKeys: ['hideKnowledgeBaseTab'], + detailCode: 'PERMISSION_GROUP_CAPABILITY_BLOCKED', + describe: 'The Knowledge Base module', + deniedBy: (config) => config.hideKnowledgeBaseTab, + }, + 'tables.use': { + kind: 'static', + configKeys: ['hideTablesTab'], + detailCode: 'PERMISSION_GROUP_CAPABILITY_BLOCKED', + describe: 'The Tables module', + deniedBy: (config) => config.hideTablesTab, + }, + 'files.use': { + kind: 'static', + configKeys: ['hideFilesTab'], + detailCode: 'PERMISSION_GROUP_CAPABILITY_BLOCKED', + describe: 'The Files module', + deniedBy: (config) => config.hideFilesTab, + }, + 'inbox.use': { + kind: 'static', + configKeys: ['hideInboxTab'], + detailCode: 'PERMISSION_GROUP_CAPABILITY_BLOCKED', + describe: 'The inbox', + deniedBy: (config) => config.hideInboxTab, + }, + 'copilot.use': { + kind: 'static', + configKeys: ['hideCopilot'], + detailCode: 'PERMISSION_GROUP_CAPABILITY_BLOCKED', + describe: 'Chat', + deniedBy: (config) => config.hideCopilot, + }, + 'secrets.manage': { + kind: 'static', + configKeys: ['hideSecretsTab'], + detailCode: 'PERMISSION_GROUP_CAPABILITY_BLOCKED', + describe: 'Managing secrets', + deniedBy: (config) => config.hideSecretsTab, + }, + 'api_keys.manage': { + kind: 'static', + configKeys: ['hideApiKeysTab'], + detailCode: 'PERMISSION_GROUP_CAPABILITY_BLOCKED', + describe: 'Managing API keys', + deniedBy: (config) => config.hideApiKeysTab, + }, + 'integrations.manage': { + kind: 'static', + configKeys: ['hideIntegrationsTab'], + detailCode: 'PERMISSION_GROUP_CAPABILITY_BLOCKED', + describe: 'Managing integrations', + deniedBy: (config) => config.hideIntegrationsTab, + }, + 'deploy.api': { + kind: 'static', + configKeys: ['hideDeployApi'], + detailCode: 'PERMISSION_GROUP_CAPABILITY_BLOCKED', + describe: 'API deployment', + deniedBy: (config) => config.hideDeployApi, + }, + 'deploy.mcp': { + kind: 'static', + configKeys: ['hideDeployMcp'], + detailCode: 'PERMISSION_GROUP_CAPABILITY_BLOCKED', + describe: 'MCP server deployment', + deniedBy: (config) => config.hideDeployMcp, + }, + 'deploy.chat': { + kind: 'static', + configKeys: ['hideDeployChatbot'], + detailCode: 'PERMISSION_GROUP_CAPABILITY_BLOCKED', + describe: 'Chat deployment', + deniedBy: (config) => config.hideDeployChatbot, + }, + 'deploy.chat.auth_mode': { + kind: 'parameterized', + configKeys: ['allowedChatDeployAuthTypes'], + detailCode: 'CHAT_AUTH_MODE_NOT_PERMITTED', + describe: 'This chat authentication mode', + deniedBy: (config, mode) => allowlistDenies(config.allowedChatDeployAuthTypes, mode), + }, + 'file_share.publish': { + kind: 'static', + configKeys: ['disablePublicFileSharing'], + detailCode: 'PUBLIC_SHARING_NOT_ALLOWED', + describe: 'Public file sharing', + deniedBy: (config) => config.disablePublicFileSharing, + }, + 'file_share.auth_mode': { + kind: 'parameterized', + configKeys: ['allowedFileShareAuthTypes'], + detailCode: 'PUBLIC_SHARING_NOT_ALLOWED', + describe: 'This file-share authentication mode', + deniedBy: (config, mode) => allowlistDenies(config.allowedFileShareAuthTypes, mode), + }, + 'public_api.use': { + kind: 'static', + configKeys: ['disablePublicApi'], + detailCode: 'PERMISSION_GROUP_CAPABILITY_BLOCKED', + describe: 'Public API access', + deniedBy: (config) => config.disablePublicApi, + }, + 'invitations.send': { + kind: 'static', + configKeys: ['disableInvitations'], + detailCode: 'PERMISSION_GROUP_CAPABILITY_BLOCKED', + describe: 'Sending invitations', + deniedBy: (config) => config.disableInvitations, + }, + 'mcp_tools.use': { + kind: 'static', + configKeys: ['disableMcpTools'], + detailCode: 'PERMISSION_GROUP_CAPABILITY_BLOCKED', + describe: 'Calling MCP tools', + deniedBy: (config) => config.disableMcpTools, + }, + 'custom_tools.use': { + kind: 'static', + configKeys: ['disableCustomTools'], + detailCode: 'PERMISSION_GROUP_CAPABILITY_BLOCKED', + describe: 'Calling custom tools', + deniedBy: (config) => config.disableCustomTools, + }, + 'skills.use': { + kind: 'static', + configKeys: ['disableSkills'], + detailCode: 'PERMISSION_GROUP_CAPABILITY_BLOCKED', + describe: 'Loading skills', + deniedBy: (config) => config.disableSkills, + }, + 'logs.trace_spans': { + kind: 'static', + configKeys: ['hideTraceSpans'], + detailCode: 'PERMISSION_GROUP_CAPABILITY_BLOCKED', + describe: 'The per-block execution trace', + deniedBy: (config) => config.hideTraceSpans, + }, + /** + * Not declarable on an operation: it refuses a *principal kind* rather than a + * capability of the resource, so it applies to every operation a personal key + * could reach. Asserted in the authorization funnel's personal-key branch. + */ + 'personal_api_key.use': { + kind: 'static', + configKeys: ['disablePersonalApiKeys'], + detailCode: 'PERSONAL_API_KEYS_DISABLED', + describe: 'Using a personal API key', + deniedBy: (config) => config.disablePersonalApiKeys, + }, + 'logs.export': { + kind: 'static', + configKeys: ['disableLogExport'], + detailCode: 'PERMISSION_GROUP_CAPABILITY_BLOCKED', + describe: 'Exporting execution logs', + deniedBy: (config) => config.disableLogExport, + }, + 'logs.cost': { + kind: 'static', + configKeys: ['hideCostInfo'], + detailCode: 'PERMISSION_GROUP_CAPABILITY_BLOCKED', + describe: 'Execution cost', + deniedBy: (config) => config.hideCostInfo, + }, + /** + * Also reads `hideKnowledgeBaseTab`, because an operation declares exactly one + * capability: moving knowledge-base creation off `knowledge.use` would + * otherwise let a group that withheld the whole module still create one + * through the API. The narrower capability has to subsume the broader. + */ + 'knowledge.create': { + kind: 'static', + configKeys: ['disableKnowledgeBaseCreation', 'hideKnowledgeBaseTab'], + detailCode: 'PERMISSION_GROUP_CAPABILITY_BLOCKED', + describe: 'Creating a knowledge base', + deniedBy: (config) => config.disableKnowledgeBaseCreation || config.hideKnowledgeBaseTab, + }, + /** Subsumes `knowledge.use` for the same reason as `knowledge.create`. */ + 'knowledge.upload': { + kind: 'static', + configKeys: ['disableKnowledgeBaseFileUpload', 'hideKnowledgeBaseTab'], + detailCode: 'PERMISSION_GROUP_CAPABILITY_BLOCKED', + describe: 'Uploading documents to a knowledge base', + deniedBy: (config) => config.disableKnowledgeBaseFileUpload || config.hideKnowledgeBaseTab, + }, + /** + * Parameterized on the connector id, because the decision is which source a + * member may sync — a connector pulls a whole external corpus into the + * workspace, and an organization that sanctions Drive rarely sanctions every + * one of the other sixty. + */ + 'knowledge.connectors': { + kind: 'parameterized', + configKeys: ['allowedKnowledgeConnectors'], + detailCode: 'PERMISSION_GROUP_CAPABILITY_BLOCKED', + describe: 'This knowledge base connector', + deniedBy: (config, connectorType) => + allowlistDenies(config.allowedKnowledgeConnectors, connectorType), + }, + /** + * Also reads `hideTablesTab`, for the reason `knowledge.create` does: an + * operation declares exactly one capability, so the narrower one replacing the + * broader one on `tables.create` would otherwise let a group that withholds + * the whole module still create tables through the API. + */ + 'tables.create': { + kind: 'static', + configKeys: ['disableTableCreation', 'hideTablesTab'], + detailCode: 'PERMISSION_GROUP_CAPABILITY_BLOCKED', + describe: 'Creating a table', + deniedBy: (config) => config.disableTableCreation || config.hideTablesTab, + }, + /** Subsumes `tables.use` for the same reason as `tables.create`. */ + 'tables.export': { + kind: 'static', + configKeys: ['disableTableExport', 'hideTablesTab'], + detailCode: 'PERMISSION_GROUP_CAPABILITY_BLOCKED', + describe: 'Exporting a table', + deniedBy: (config) => config.disableTableExport || config.hideTablesTab, + }, + 'files.bulk_download': { + kind: 'static', + configKeys: ['disableBulkFileDownload'], + detailCode: 'PERMISSION_GROUP_CAPABILITY_BLOCKED', + describe: 'Downloading files in bulk', + deniedBy: (config) => config.disableBulkFileDownload, + }, + 'credentials.personal': { + kind: 'static', + configKeys: ['disablePersonalCredentials'], + detailCode: 'PERMISSION_GROUP_CAPABILITY_BLOCKED', + describe: 'Connecting personal credentials', + deniedBy: (config) => config.disablePersonalCredentials, + }, + 'workspace.create': { + kind: 'static', + configKeys: ['disableWorkspaceCreation'], + detailCode: 'PERMISSION_GROUP_CAPABILITY_BLOCKED', + describe: 'Creating a workspace', + deniedBy: (config) => config.disableWorkspaceCreation, + }, + 'organization.member_directory': { + kind: 'static', + configKeys: ['hideOrgMemberDirectory'], + detailCode: 'PERMISSION_GROUP_CAPABILITY_BLOCKED', + describe: 'The organization member directory', + deniedBy: (config) => config.hideOrgMemberDirectory, + }, + 'cli.use': { + kind: 'static', + configKeys: ['disableCliAccess'], + detailCode: 'PERMISSION_GROUP_CAPABILITY_BLOCKED', + describe: 'CLI access', + deniedBy: (config) => config.disableCliAccess, + }, + 'triggers.webhook': { + kind: 'static', + configKeys: ['disableWebhookTriggers'], + detailCode: 'PERMISSION_GROUP_CAPABILITY_BLOCKED', + describe: 'Creating a webhook trigger', + deniedBy: (config) => config.disableWebhookTriggers, + }, + 'copilot.tool_auto_approval': { + kind: 'static', + configKeys: ['disableToolAutoApproval'], + detailCode: 'PERMISSION_GROUP_CAPABILITY_BLOCKED', + describe: 'Silencing a tool confirmation', + deniedBy: (config) => config.disableToolAutoApproval, + }, +} satisfies { readonly [K in PermissionGroupCapability]: CapabilityRule } + +/** + * The capabilities an operation may declare — the static ones. A parameterized + * rule needs a request value the funnel cannot see, so naming one on an + * operation would silently never fire. + */ +export type StaticPermissionGroupCapability = { + [K in PermissionGroupCapability]: (typeof CAPABILITY_RULES)[K] extends StaticCapabilityRule + ? K + : never +}[PermissionGroupCapability] + +/** + * Proof that the static/parameterized split resolves. + * + * `StaticPermissionGroupCapability` reads each rule's own `kind`, so annotating + * {@link CAPABILITY_RULES} instead of using `satisfies` would widen every entry + * and collapse this type to `never` — at which point no operation could declare + * any capability and every gate would silently never fire. Nothing at runtime + * would look wrong, so it is asserted here. + * + * The aliases below are deliberately unexported: the constraint on + * {@link Assert} is checked where the alias is declared, so an export bought + * nothing but the appearance of a consumer that never existed. They are unused + * on purpose, and deleting one deletes the proof. + */ +type Assert = T + +type AssertsStaticCapabilityResolves = Assert< + 'tables.use' extends StaticPermissionGroupCapability ? true : false +> +type AssertsParameterizedCapabilityIsExcluded = Assert< + 'deploy.chat.auth_mode' extends StaticPermissionGroupCapability ? false : true +> diff --git a/apps/sim/lib/permission-groups/capability-assertions.ts b/apps/sim/lib/permission-groups/capability-assertions.ts new file mode 100644 index 00000000000..9d5c51f5506 --- /dev/null +++ b/apps/sim/lib/permission-groups/capability-assertions.ts @@ -0,0 +1,92 @@ +import { + CAPABILITY_RULES, + refuseCapability, + type StaticPermissionGroupCapability, +} from '@/lib/permission-groups/capabilities' +import { resolvePermissionGroupConfig } from '@/lib/permission-groups/config-scope.server' +import type { PermissionGroupConfig } from '@/lib/permission-groups/fields' +import { getUserPermissionConfigForOrganization } from '@/lib/permission-groups/resolve.server' + +/** + * Re-exported so a caller that gates inline reaches the refusal sentence and the + * assertions through one module; {@link CAPABILITY_RULES} remains its only + * definition. + */ +export { capabilityRefusal } from '@/lib/permission-groups/capabilities' + +/** + * The one way to ask whether a permission group withholds a capability. + * + * The authorization funnel decides from the operation alone, which is right when + * the capability describes the whole operation. Three cases fall outside it: a + * decision that depends on request input (one download is a single file, the + * next a folder tree), a raw route that predates the operation boundary, and an + * organization-level action with no workspace. All of them come through here, so + * the decision always reads {@link CAPABILITY_RULES} rather than a config key + * spelled out at a call site — where a renamed key would silently stop denying + * anything, and the refusal wording would drift from the funnel's. + */ +export function capabilityDeniedBy( + capability: StaticPermissionGroupCapability, + config: PermissionGroupConfig | null +): boolean { + if (!config) return false + const rule = CAPABILITY_RULES[capability] + return rule.kind === 'static' && rule.deniedBy(config) +} + +/** + * Throws when `userId`'s group in `workspaceId` withholds `capability`. + * + * A no-op when no group governs the user, so a personal workspace or a + * non-enterprise organization is unaffected. Pass `organizationId` when the + * caller has already loaded the workspace; omitting it costs one lookup, and + * both forms share the same per-request memo either way. + */ +export async function assertWorkspaceCapability( + userId: string, + workspaceId: string, + capability: StaticPermissionGroupCapability, + organizationId?: string | null +): Promise { + const config = await resolvePermissionGroupConfig(userId, workspaceId, organizationId) + if (capabilityDeniedBy(capability, config)) refuseCapability(capability) +} + +/** + * Whether the capability is withheld, without throwing. + * + * For a caller that must answer rather than refuse — a raw handler rendering its + * own response shape, or a policy that reports a structured decision. + */ +export async function isWorkspaceCapabilityWithheld( + userId: string, + workspaceId: string, + capability: StaticPermissionGroupCapability, + organizationId?: string | null +): Promise { + return capabilityDeniedBy( + capability, + await resolvePermissionGroupConfig(userId, workspaceId, organizationId) + ) +} + +/** + * The organization-scoped counterpart of {@link isWorkspaceCapabilityWithheld}. + * + * Outside the per-request memo on purpose. That memo is keyed by user and + * workspace, and this decision is keyed by organization alone, so sharing it + * would need a second key vocabulary in the store. No request asks an + * organization-scoped capability twice — every call site gates one + * organization-level act — so the memo would never be hit. Key it if that + * changes. + */ +export async function isOrganizationCapabilityWithheld( + organizationId: string, + capability: StaticPermissionGroupCapability +): Promise { + return capabilityDeniedBy( + capability, + await getUserPermissionConfigForOrganization(organizationId) + ) +} diff --git a/apps/sim/lib/permission-groups/capability-error.ts b/apps/sim/lib/permission-groups/capability-error.ts new file mode 100644 index 00000000000..a470a7592b6 --- /dev/null +++ b/apps/sim/lib/permission-groups/capability-error.ts @@ -0,0 +1,26 @@ +import { type ForbiddenDetailCode, ForbiddenOperationError } from '@/lib/core/application/forbidden' +import type { PermissionGroupCapability } from '@/lib/permission-groups/capabilities' + +/** + * The caller's permission group withholds a capability the request needs. + * + * Carries the capability so a log line or an audit entry can name it; the + * message names it for the caller. The detail code comes from the capability's + * own rule rather than being fixed here — the closed code set is closed over + * remedies, so the handful of capabilities with a remedy of their own (a chat + * auth mode, public sharing, personal API keys) carry a code of their own and + * the rest share the generic one. + * + * Lives here rather than beside the authorization funnel so the assertion + * helpers can throw it without importing the funnel, which imports them. + */ +export class PermissionGroupCapabilityError extends ForbiddenOperationError { + constructor( + readonly capability: PermissionGroupCapability, + detailCode: ForbiddenDetailCode, + message: string + ) { + super(detailCode, message) + this.name = 'PermissionGroupCapabilityError' + } +} diff --git a/apps/sim/lib/permission-groups/capability-response.ts b/apps/sim/lib/permission-groups/capability-response.ts new file mode 100644 index 00000000000..a1d28471aaa --- /dev/null +++ b/apps/sim/lib/permission-groups/capability-response.ts @@ -0,0 +1,31 @@ +import { NextResponse } from 'next/server' +import { + CAPABILITY_RULES, + capabilityRefusal, + type PermissionGroupCapability, +} from '@/lib/permission-groups/capabilities' + +/** + * The 403 a raw route returns when a permission group withholds a capability, + * as opposed to the caller's role being too low. + * + * One builder so the sentence and the detail code cannot drift between the + * routes that gate through a shared access check, the ones that assert inline, + * and the ones that catch {@link PermissionGroupCapabilityError} and render its + * capability. The detail code is read off the rule rather than spelled out at + * the call site — four capabilities carry a more specific one, and a literal + * would report them as the generic block. + * + * The v1 public API renders its own `{ error: { code, message } }` envelope and + * is deliberately not converged here; see `resolveCapabilityRefusal` in + * `app/api/v1/middleware.ts`. + */ +export function capabilityRefusalResponse(capability: PermissionGroupCapability): NextResponse { + return NextResponse.json( + { + error: capabilityRefusal(capability), + details: { code: CAPABILITY_RULES[capability].detailCode }, + }, + { status: 403 } + ) +} diff --git a/apps/sim/lib/permission-groups/config-scope.server.test.ts b/apps/sim/lib/permission-groups/config-scope.server.test.ts new file mode 100644 index 00000000000..7f4f473e885 --- /dev/null +++ b/apps/sim/lib/permission-groups/config-scope.server.test.ts @@ -0,0 +1,65 @@ +/** + * @vitest-environment node + */ +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const { mockGetUserPermissionConfig, mockResolveVerifiedContext } = vi.hoisted(() => ({ + mockGetUserPermissionConfig: vi.fn(), + mockResolveVerifiedContext: vi.fn(), +})) + +vi.mock('react', () => ({ cache: (fn: F) => fn })) + +vi.mock('@/lib/permission-groups/resolve.server', () => ({ + getUserPermissionConfig: mockGetUserPermissionConfig, + resolveVerifiedUserAccessControlContext: mockResolveVerifiedContext, +})) + +import { resolvePermissionGroupConfig } from '@/lib/permission-groups/config-scope.server' +import { withPermissionGroupScope } from '@/lib/permission-groups/request-scope.server' + +const CONFIG = { hideTablesTab: true } + +describe('resolvePermissionGroupConfig scope memo', () => { + beforeEach(() => { + vi.clearAllMocks() + mockGetUserPermissionConfig.mockResolvedValue(CONFIG) + mockResolveVerifiedContext.mockResolvedValue({ config: CONFIG }) + }) + + /** + * The key omits `organizationId` because a caller may only pass the + * organization of the workspace it names, so the two arms resolve the same + * group. Adding it to the key would split the cache and query twice. + */ + it('shares one query between the looked-up and the already-loaded form', async () => { + const [first, second] = await withPermissionGroupScope(() => + Promise.all([ + resolvePermissionGroupConfig('user-1', 'workspace-1', undefined), + resolvePermissionGroupConfig('user-1', 'workspace-1', 'org-1'), + ]) + ) + + expect(first).toBe(second) + expect( + mockGetUserPermissionConfig.mock.calls.length + mockResolveVerifiedContext.mock.calls.length + ).toBe(1) + }) + + it('resolves a different user or workspace separately', async () => { + await withPermissionGroupScope(async () => { + await resolvePermissionGroupConfig('user-1', 'workspace-1', 'org-1') + await resolvePermissionGroupConfig('user-2', 'workspace-1', 'org-1') + await resolvePermissionGroupConfig('user-1', 'workspace-2', 'org-1') + }) + + expect(mockResolveVerifiedContext).toHaveBeenCalledTimes(3) + }) + + it('still answers outside a scope, without memoizing', async () => { + await resolvePermissionGroupConfig('user-1', 'workspace-1', 'org-1') + await resolvePermissionGroupConfig('user-1', 'workspace-1', 'org-1') + + expect(mockResolveVerifiedContext).toHaveBeenCalledTimes(2) + }) +}) diff --git a/apps/sim/lib/permission-groups/config-scope.server.ts b/apps/sim/lib/permission-groups/config-scope.server.ts new file mode 100644 index 00000000000..42d480c46c7 --- /dev/null +++ b/apps/sim/lib/permission-groups/config-scope.server.ts @@ -0,0 +1,68 @@ +import { cache } from 'react' +import type { PermissionGroupConfig } from '@/lib/permission-groups/fields' +import type { PermissionGroupScopeKey } from '@/lib/permission-groups/request-scope.server' +import { getPermissionGroupConfigStore } from '@/lib/permission-groups/request-scope.server' +import { + getUserPermissionConfig, + resolveVerifiedUserAccessControlContext, +} from '@/lib/permission-groups/resolve.server' + +/** + * Memoized for a React server request, so an RSC render that resolves the same + * viewer twice still makes one query. Both paths delegate here, so the scope and + * the React cache can never disagree. + */ +const resolveCached = cache( + async ( + userId: string, + workspaceId: string, + organizationId: string | null | undefined + ): Promise => + organizationId === undefined + ? await getUserPermissionConfig(userId, workspaceId) + : (await resolveVerifiedUserAccessControlContext(userId, workspaceId, organizationId)).config +) + +/** + * The permission-group config governing `userId` in `workspaceId`, resolved at + * most once per scope. + * + * Caches the promise rather than the value, so concurrent callers share one + * query instead of racing to start several. Caches `null` too — "no group + * governs this user" is the common answer and the one least worth re-asking. + * + * Outside a scope this degrades to the React memo, and outside a request to a + * direct call: slower, never wrong. + * + * Pass `undefined` for `organizationId` when the caller has not already loaded + * the workspace — a raw route, typically. The resolver looks it up, and both + * forms share this memo, so a request that mixes them still queries once. + * + * `organizationId` is deliberately NOT part of the key, and adding it would + * split the cache and double the queries for no gain. Both arms end in + * `resolveUserAccessControlContextForOrganization(userId, workspaceId, org)`; + * they differ only in where `org` came from, and a caller may only pass the + * organization of the very workspace it names — it is a value it loaded off + * that workspace, not an independent argument. So `organizationId` is a + * function of `workspaceId`, and the key already carries it. A caller that + * passed some *other* organization would be resolving the wrong group with or + * without this memo, and would fail open (no group in that organization targets + * this workspace, so nothing restricts); that is a call-site invariant, which + * is why the parameter is documented as "already loaded" rather than free. + */ +export function resolvePermissionGroupConfig( + userId: string, + workspaceId: string, + organizationId: string | null | undefined +): Promise { + const store = getPermissionGroupConfigStore() + if (!store) return resolveCached(userId, workspaceId, organizationId) + + const key: PermissionGroupScopeKey = `${userId}:${workspaceId}` + const existing = store.get(key) + if (existing) return existing + + const pending = resolveCached(userId, workspaceId, organizationId) + store.set(key, pending) + return pending +} diff --git a/apps/sim/lib/permission-groups/constraints.ts b/apps/sim/lib/permission-groups/constraints.ts new file mode 100644 index 00000000000..f87bde2ec06 --- /dev/null +++ b/apps/sim/lib/permission-groups/constraints.ts @@ -0,0 +1,8 @@ +export const PERMISSION_GROUP_CONSTRAINTS = { + organizationName: 'permission_group_organization_name_unique', + organizationDefault: 'permission_group_organization_default_unique', +} as const + +export const PERMISSION_GROUP_MEMBER_CONSTRAINTS = { + groupUser: 'permission_group_member_group_user_unique', +} as const diff --git a/apps/sim/lib/permission-groups/features.test.ts b/apps/sim/lib/permission-groups/features.test.ts index c7496830cd3..45211533c16 100644 --- a/apps/sim/lib/permission-groups/features.test.ts +++ b/apps/sim/lib/permission-groups/features.test.ts @@ -4,12 +4,13 @@ import { describe, expect, it } from 'vitest' import { getActivePermissionGroupRestrictions, + isFeatureInertForGroup, PLATFORM_FEATURES, } from '@/lib/permission-groups/features' import { DEFAULT_PERMISSION_GROUP_CONFIG, type PermissionGroupConfig, -} from '@/lib/permission-groups/types' +} from '@/lib/permission-groups/fields' describe('getActivePermissionGroupRestrictions', () => { it('returns no restrictions for an absent or unrestricted config', () => { @@ -95,3 +96,80 @@ describe('getActivePermissionGroupRestrictions', () => { } ) }) + +/** + * The editor renders every boolean key on every group, so a key whose + * capability is read from the organization's *default* group is a checkbox that + * does nothing on any other group. `scope` is what lets the editor say so, and + * this pins the membership of each class: a new key that reads the default + * group has to be added here deliberately, rather than shipping as a silently + * inert checkbox. + * + * `workspace-or-organization` is the honest third answer. `api_keys.manage`, + * `cli.use`, `integrations.manage`, `invitations.send` and + * `personal_api_key.use` each have a workspace-scoped path that reads the group + * being edited *and* an account-level path that falls back to the default + * group, so they are neither inert nor purely local — marking them + * `organization` would tell an admin their workspace restriction does not apply + * when it does. + */ +describe('platform feature scope', () => { + function keysWithScope(scope: string): string[] { + return PLATFORM_FEATURES.filter((feature) => feature.scope === scope) + .map((feature) => feature.configKey) + .sort() + } + + it('reads exactly two keys from the organization default group alone', () => { + expect(keysWithScope('organization')).toEqual([ + 'disableWorkspaceCreation', + 'hideOrgMemberDirectory', + ]) + }) + + it('reads exactly five keys from both a workspace group and the default group', () => { + expect(keysWithScope('workspace-or-organization')).toEqual([ + 'disableCliAccess', + 'disableInvitations', + 'disablePersonalApiKeys', + 'hideApiKeysTab', + 'hideIntegrationsTab', + ]) + }) + + it('gives every feature a scope', () => { + for (const feature of PLATFORM_FEATURES) { + expect( + ['workspace', 'organization', 'workspace-or-organization'], + `${feature.configKey} declares no known scope` + ).toContain(feature.scope) + } + }) +}) + +describe('isFeatureInertForGroup', () => { + function feature(configKey: string) { + const found = PLATFORM_FEATURES.find((f) => f.configKey === configKey) + if (!found) throw new Error(`No platform feature for ${configKey}`) + return found + } + + it('makes an organization-scoped key inert on a non-default group', () => { + expect(isFeatureInertForGroup(feature('hideOrgMemberDirectory'), false)).toBe(true) + expect(isFeatureInertForGroup(feature('disableWorkspaceCreation'), false)).toBe(true) + }) + + it('leaves an organization-scoped key editable on the default group', () => { + expect(isFeatureInertForGroup(feature('hideOrgMemberDirectory'), true)).toBe(false) + }) + + /** + * The dual-scope keys have a workspace path that reads the group being + * edited, so making them inert would withhold a restriction that does apply. + */ + it('never makes a workspace or dual-scope key inert', () => { + expect(isFeatureInertForGroup(feature('hideApiKeysTab'), false)).toBe(false) + expect(isFeatureInertForGroup(feature('disableCliAccess'), false)).toBe(false) + expect(isFeatureInertForGroup(feature('hideTraceSpans'), false)).toBe(false) + }) +}) diff --git a/apps/sim/lib/permission-groups/features.ts b/apps/sim/lib/permission-groups/features.ts index 78aa4f79545..642921fa3ca 100644 --- a/apps/sim/lib/permission-groups/features.ts +++ b/apps/sim/lib/permission-groups/features.ts @@ -1,6 +1,11 @@ -import type { PermissionGroupConfig } from '@/lib/permission-groups/types' +import { + PERMISSION_GROUP_FIELDS, + type PermissionGroupCapabilityScope, + type PermissionGroupConfig, + type PermissionGroupConfigKey, +} from '@/lib/permission-groups/fields' -type BooleanPermissionGroupConfigKey = { +export type BooleanPermissionGroupConfigKey = { [Key in keyof PermissionGroupConfig]: PermissionGroupConfig[Key] extends boolean ? Key : never }[keyof PermissionGroupConfig] @@ -10,6 +15,33 @@ export interface PermissionGroupPlatformFeature { category: string configKey: BooleanPermissionGroupConfigKey hint: string + /** See {@link PermissionGroupCapabilityScope}. */ + scope: PermissionGroupCapabilityScope +} + +/** + * The note a group editor shows beside an organization-scoped row. + * + * One sentence, in one place, so the editor and any other surface that has to + * explain the row say the same thing. + */ +export const ORGANIZATION_SCOPED_FEATURE_NOTE = + "Read from the organization's default group, so it applies organization-wide no matter which group sets it." + +/** + * Whether a group editor can decide this key at all. + * + * An `organization`-scoped key is read from the organization's default group, so + * on any other group the checkbox writes a value nothing will ever read. The + * editor renders those rows inert and every bulk action skips them, both from + * this one predicate — a second copy is how the row and the "Select All" it sits + * under would come to disagree. + */ +export function isFeatureInertForGroup( + feature: PermissionGroupPlatformFeature, + groupIsDefault: boolean +): boolean { + return feature.scope === 'organization' && !groupIsDefault } export interface ActivePermissionGroupRestriction { @@ -17,151 +49,49 @@ export interface ActivePermissionGroupRestriction { description: string } -/** Render order for the platform-feature category sections; unlisted ones follow. */ +/** + * Render order for the platform-feature category sections; unlisted ones follow. + * + * Named after what a group withholds rather than after the surface the key once + * hid, for the reason `PlatformFeatureMeta.hint` in `fields.ts` gives at length. + */ export const PLATFORM_CATEGORY_ORDER: readonly string[] = [ - 'Sidebar', - 'Deploy Tabs', - 'Chat', - 'Collaboration', - 'Workflow Panel', + 'Modules', + 'Knowledge Base', + 'Tables', + 'Files', + 'Deployment', 'Tools', - 'Features', - 'Settings Tabs', 'Logs', - 'Files', + 'Collaboration', + 'Credentials & Access', ] as const -/** User-facing descriptions shared by the Access Control editor and live permission context. */ -export const PLATFORM_FEATURES = [ - { - id: 'hide-knowledge-base', - label: 'Knowledge Base', - category: 'Sidebar', - configKey: 'hideKnowledgeBaseTab', - hint: 'Hide the Knowledge Base module from the sidebar.', - }, - { - id: 'hide-tables', - label: 'Tables', - category: 'Sidebar', - configKey: 'hideTablesTab', - hint: 'Hide the Tables module from the sidebar.', - }, - { - id: 'hide-copilot', - label: 'Chat', - category: 'Workflow Panel', - configKey: 'hideCopilot', - hint: 'Hide the Chat panel so users cannot build or edit with natural language.', - }, - { - id: 'hide-integrations', - label: 'Integrations', - category: 'Settings Tabs', - configKey: 'hideIntegrationsTab', - hint: 'Hide the Integrations settings tab (OAuth connections).', - }, - { - id: 'hide-secrets', - label: 'Secrets', - category: 'Settings Tabs', - configKey: 'hideSecretsTab', - hint: 'Hide the Secrets (environment variables) settings tab.', - }, - { - id: 'hide-api-keys', - label: 'API Keys', - category: 'Settings Tabs', - configKey: 'hideApiKeysTab', - hint: 'Hide the API Keys settings tab.', - }, - { - id: 'hide-files', - label: 'Files', - category: 'Settings Tabs', - configKey: 'hideFilesTab', - hint: 'Hide the Files settings tab.', - }, - { - id: 'hide-deploy-api', - label: 'API', - category: 'Deploy Tabs', - configKey: 'hideDeployApi', - hint: 'Hide the API deployment option.', - }, - { - id: 'hide-deploy-mcp', - label: 'MCP', - category: 'Deploy Tabs', - configKey: 'hideDeployMcp', - hint: 'Hide the MCP server deployment option.', - }, - { - id: 'disable-mcp', - label: 'MCP Tools', - category: 'Tools', - configKey: 'disableMcpTools', - hint: 'Block agents from calling MCP tools.', - }, - { - id: 'disable-custom-tools', - label: 'Custom Tools', - category: 'Tools', - configKey: 'disableCustomTools', - hint: 'Block agents from calling user-defined custom tools.', - }, - { - id: 'disable-skills', - label: 'Skills', - category: 'Tools', - configKey: 'disableSkills', - hint: 'Block agents from loading skills.', - }, - { - id: 'hide-trace-spans', - label: 'Trace Spans', - category: 'Logs', - configKey: 'hideTraceSpans', - hint: 'Hide per-block trace spans in logs.', - }, - { - id: 'disable-invitations', - label: 'Invitations', - category: 'Collaboration', - configKey: 'disableInvitations', - hint: 'Prevent users from inviting others to workspaces.', - }, - { - id: 'hide-inbox', - label: 'Sim Mailer', - category: 'Features', - configKey: 'hideInboxTab', - hint: 'Hide the Sim Mailer inbox.', - }, - { - id: 'disable-public-api', - label: 'Public API', - category: 'Features', - configKey: 'disablePublicApi', - hint: 'Disable public API access to deployed workflows.', - }, - { - id: 'hide-deploy-chatbot', - label: 'Deployment', - category: 'Chat', - configKey: 'hideDeployChatbot', - hint: 'Hide the chat deployment option.', - }, - { - id: 'disable-public-file-sharing', - label: 'Public Sharing', - category: 'Files', - configKey: 'disablePublicFileSharing', - hint: 'Disable public file-share links.', - }, -] as const satisfies readonly PermissionGroupPlatformFeature[] +const FIELD_ENTRIES = Object.entries(PERMISSION_GROUP_FIELDS) as Array< + [PermissionGroupConfigKey, (typeof PERMISSION_GROUP_FIELDS)[PermissionGroupConfigKey]] +> -/** Returns only restrictions that actively constrain the current user. */ +/** + * The boolean toggles the Access Control editor renders, in registry order. + * + * Derived rather than listed, so a boolean key cannot reach the config without + * reaching the editor — an unrendered key is one an admin can neither set nor + * see, which is how a restriction ends up applying with nothing to explain it. + */ +export const PLATFORM_FEATURES: readonly PermissionGroupPlatformFeature[] = FIELD_ENTRIES.flatMap( + ([key, field]) => + field.kind === 'boolean-restriction' + ? [{ ...field.feature, configKey: key as BooleanPermissionGroupConfigKey }] + : [] +) + +/** + * Returns only restrictions that actively constrain the current user. + * + * Two passes, allowlists and denylists before booleans, because the resulting + * prose is what the Copilot context and the group roster read: reordering it + * would rewrite text that surfaces to users for no reason. + */ export function getActivePermissionGroupRestrictions( config: PermissionGroupConfig | null ): ActivePermissionGroupRestriction[] { @@ -169,53 +99,16 @@ export function getActivePermissionGroupRestrictions( const restrictions: ActivePermissionGroupRestriction[] = [] - if (config.allowedIntegrations !== null) { - restrictions.push({ - key: 'allowedIntegrations', - description: - config.allowedIntegrations.length > 0 - ? 'Integrations and blocks are limited to effectiveConfig.allowedIntegrations.' - : 'No non-exempt integrations or blocks are allowed.', - }) - } - if (config.allowedModelProviders !== null) { - restrictions.push({ - key: 'allowedModelProviders', - description: - config.allowedModelProviders.length > 0 - ? 'Model providers are limited to effectiveConfig.allowedModelProviders.' - : 'No model providers are allowed.', - }) - } - if (config.deniedModels.length > 0) { - restrictions.push({ - key: 'deniedModels', - description: 'Models listed in effectiveConfig.deniedModels are blocked.', - }) - } - if (config.deniedTools.length > 0) { - restrictions.push({ - key: 'deniedTools', - description: 'Integration tools listed in effectiveConfig.deniedTools are blocked.', - }) - } - if (config.allowedFileShareAuthTypes !== null) { - restrictions.push({ - key: 'allowedFileShareAuthTypes', - description: - config.allowedFileShareAuthTypes.length > 0 - ? 'Public file-share authentication is limited to effectiveConfig.allowedFileShareAuthTypes.' - : 'No public file-share authentication modes are allowed.', - }) - } - if (config.allowedChatDeployAuthTypes !== null) { - restrictions.push({ - key: 'allowedChatDeployAuthTypes', - description: - config.allowedChatDeployAuthTypes.length > 0 - ? 'Chat deployment authentication is limited to effectiveConfig.allowedChatDeployAuthTypes.' - : 'No chat deployment authentication modes are allowed.', - }) + for (const [key, field] of FIELD_ENTRIES) { + const value = config[key] + if (field.kind === 'allowlist' && Array.isArray(value)) { + restrictions.push({ + key, + description: value.length > 0 ? field.phrasing.limited : field.phrasing.empty, + }) + } else if (field.kind === 'denylist' && Array.isArray(value) && value.length > 0) { + restrictions.push({ key, description: field.phrasing }) + } } for (const feature of PLATFORM_FEATURES) { diff --git a/apps/sim/lib/permission-groups/fields.test.ts b/apps/sim/lib/permission-groups/fields.test.ts new file mode 100644 index 00000000000..e5ae2c82d0d --- /dev/null +++ b/apps/sim/lib/permission-groups/fields.test.ts @@ -0,0 +1,352 @@ +/** + * @vitest-environment node + */ +import { describe, expect, it } from 'vitest' +import { permissionGroupFullConfigSchema } from '@/lib/api/contracts/permission-groups' +import { PLATFORM_FEATURES } from '@/lib/permission-groups/features' +import { + DEFAULT_PERMISSION_GROUP_CONFIG, + type PermissionGroupConfig, + parsePermissionGroupConfig, + permissionGroupConfigSchema, +} from '@/lib/permission-groups/fields' + +/** + * The coercion corpus, pinned against the hand-written parser before it is + * replaced by a derived one. + * + * Every row states what a stored `jsonb` value coerces to today. A derived + * implementation has to reproduce this table exactly, so any row that changes + * in a later diff is a deliberate semantic decision someone has to defend + * rather than a silent regression. + */ +interface CoercionFixture { + name: string + input: unknown + expected: PermissionGroupConfig +} + +const fixtures: readonly CoercionFixture[] = [ + { name: 'null', input: null, expected: DEFAULT_PERMISSION_GROUP_CONFIG }, + { name: 'undefined', input: undefined, expected: DEFAULT_PERMISSION_GROUP_CONFIG }, + /** + * `typeof [] === 'object'`, so an array-valued column falls through the + * object guard and coerces to defaults rather than throwing. A derived + * parser built on `z.object()` throws here unless it guards `Array.isArray`. + */ + { name: 'an array (jsonb [])', input: [], expected: DEFAULT_PERMISSION_GROUP_CONFIG }, + { name: 'a string', input: 'nope', expected: DEFAULT_PERMISSION_GROUP_CONFIG }, + { name: 'a number', input: 7, expected: DEFAULT_PERMISSION_GROUP_CONFIG }, + { name: 'an empty object', input: {}, expected: DEFAULT_PERMISSION_GROUP_CONFIG }, + { + name: 'unknown keys, which are dropped', + input: { bogus: 1, hideCopilot: true }, + expected: { ...DEFAULT_PERMISSION_GROUP_CONFIG, hideCopilot: true }, + }, + { + name: 'a boolean given a string', + input: { hideCopilot: 'yes' }, + expected: DEFAULT_PERMISSION_GROUP_CONFIG, + }, + { + name: 'a boolean given null', + input: { hideTablesTab: null }, + expected: DEFAULT_PERMISSION_GROUP_CONFIG, + }, + { + name: 'a boolean given false explicitly', + input: { hideFilesTab: false }, + expected: DEFAULT_PERMISSION_GROUP_CONFIG, + }, + { + name: 'a denylist with mixed members, keeping the strings', + input: { deniedTools: ['slack_canvas', 42, null, { a: 1 }] }, + expected: { ...DEFAULT_PERMISSION_GROUP_CONFIG, deniedTools: ['slack_canvas'] }, + }, + { + name: 'a denylist given an object', + input: { deniedModels: {} }, + expected: DEFAULT_PERMISSION_GROUP_CONFIG, + }, + { + name: 'a denylist given a string', + input: { deniedModels: 'gpt-4o' }, + expected: DEFAULT_PERMISSION_GROUP_CONFIG, + }, + { + name: 'auth types with an invalid member, keeping the valid ones', + input: { allowedFileShareAuthTypes: ['sso', 'bogus', 'password'] }, + expected: { + ...DEFAULT_PERMISSION_GROUP_CONFIG, + allowedFileShareAuthTypes: ['sso', 'password'], + }, + }, + { + name: 'auth types given a bare string', + input: { allowedChatDeployAuthTypes: 'sso' }, + expected: DEFAULT_PERMISSION_GROUP_CONFIG, + }, + { + name: 'auth types emptied, which denies every mode', + input: { allowedChatDeployAuthTypes: [] }, + expected: { ...DEFAULT_PERMISSION_GROUP_CONFIG, allowedChatDeployAuthTypes: [] }, + }, + /** + * An emptied allowlist denies everything while `null` allows everything, so + * the two must never collapse into one another. + */ + { + name: 'an emptied allowlist, which denies every integration', + input: { allowedIntegrations: [] }, + expected: { ...DEFAULT_PERMISSION_GROUP_CONFIG, allowedIntegrations: [] }, + }, + { + name: 'an explicitly null allowlist', + input: { allowedModelProviders: null }, + expected: DEFAULT_PERMISSION_GROUP_CONFIG, + }, + { + name: 'an allowlist given a string', + input: { allowedIntegrations: 'slack' }, + expected: DEFAULT_PERMISSION_GROUP_CONFIG, + }, + /** + * The allowlists used to be the only keys that skipped element validation, so + * a `string[]`-typed field could hold a number and the read schema then + * refused the config it produced. Filtering keeps the members that parse and + * fails closed. + */ + { + name: 'an allowlist with a non-string member, keeping the strings', + input: { allowedIntegrations: ['slack', 42] }, + expected: { ...DEFAULT_PERMISSION_GROUP_CONFIG, allowedIntegrations: ['slack'] }, + }, + { + name: 'a fully populated config', + input: { + allowedIntegrations: ['slack_v2'], + allowedModelProviders: ['anthropic'], + deniedModels: ['gpt-4o'], + deniedTools: ['slack_canvas'], + hideTraceSpans: true, + hideKnowledgeBaseTab: true, + hideTablesTab: true, + hideCopilot: true, + hideIntegrationsTab: true, + hideSecretsTab: true, + hideApiKeysTab: true, + hideInboxTab: true, + hideFilesTab: true, + disableMcpTools: true, + disableCustomTools: true, + disableSkills: true, + disableInvitations: true, + disablePublicApi: true, + disablePublicFileSharing: true, + allowedFileShareAuthTypes: ['sso'], + hideDeployApi: true, + hideDeployMcp: true, + hideDeployChatbot: true, + allowedChatDeployAuthTypes: ['password'], + disablePersonalApiKeys: true, + disableLogExport: true, + hideCostInfo: true, + disableKnowledgeBaseCreation: true, + disableKnowledgeBaseFileUpload: true, + allowedKnowledgeConnectors: ['google_drive'], + disableTableCreation: true, + disableTableExport: true, + disableBulkFileDownload: true, + disablePersonalCredentials: true, + disableWorkspaceCreation: true, + hideOrgMemberDirectory: true, + disableCliAccess: true, + disableWebhookTriggers: true, + disableToolAutoApproval: true, + }, + expected: { + allowedIntegrations: ['slack_v2'], + allowedModelProviders: ['anthropic'], + deniedModels: ['gpt-4o'], + deniedTools: ['slack_canvas'], + hideTraceSpans: true, + hideKnowledgeBaseTab: true, + hideTablesTab: true, + hideCopilot: true, + hideIntegrationsTab: true, + hideSecretsTab: true, + hideApiKeysTab: true, + hideInboxTab: true, + hideFilesTab: true, + disableMcpTools: true, + disableCustomTools: true, + disableSkills: true, + disableInvitations: true, + disablePublicApi: true, + disablePublicFileSharing: true, + allowedFileShareAuthTypes: ['sso'], + hideDeployApi: true, + hideDeployMcp: true, + hideDeployChatbot: true, + allowedChatDeployAuthTypes: ['password'], + disablePersonalApiKeys: true, + disableLogExport: true, + hideCostInfo: true, + disableKnowledgeBaseCreation: true, + disableKnowledgeBaseFileUpload: true, + allowedKnowledgeConnectors: ['google_drive'], + disableTableCreation: true, + disableTableExport: true, + disableBulkFileDownload: true, + disablePersonalCredentials: true, + disableWorkspaceCreation: true, + hideOrgMemberDirectory: true, + disableCliAccess: true, + disableWebhookTriggers: true, + disableToolAutoApproval: true, + }, + }, +] + +describe('parsePermissionGroupConfig', () => { + it.each(fixtures)('coerces $name', ({ input, expected }) => { + expect(parsePermissionGroupConfig(input)).toEqual(expected) + }) + + it.each(fixtures)('emits every key in wire order for $name', ({ input }) => { + expect(Object.keys(parsePermissionGroupConfig(input))).toEqual( + Object.keys(DEFAULT_PERMISSION_GROUP_CONFIG) + ) + }) + + it.each(fixtures)('produces a config the read schema accepts for $name', ({ input }) => { + const parsed = structuredClone(parsePermissionGroupConfig(input)) + expect(permissionGroupFullConfigSchema.safeParse(parsed).success).toBe(true) + }) + + /** + * The allowlists used to skip element validation, so a corrupted row coerced + * to a value `permissionGroupFullConfigSchema` then refused — the route + * reading it failed response validation instead of returning a usable + * allowlist. Filtering is fail-closed: the members that parse survive, and a + * corrupt one narrows the allowlist rather than voiding it. + */ + it('narrows a corrupted allowlist instead of voiding it', () => { + const parsed = parsePermissionGroupConfig({ allowedIntegrations: ['slack', 42] }) + expect(parsed.allowedIntegrations).toEqual(['slack']) + expect(permissionGroupFullConfigSchema.safeParse(structuredClone(parsed)).success).toBe(true) + }) + + it('is idempotent', () => { + for (const { input } of fixtures) { + const once = parsePermissionGroupConfig(input) + expect(parsePermissionGroupConfig(structuredClone(once))).toEqual(once) + } + }) +}) + +/** + * A deterministic generator, seeded so a failure reproduces from the printed + * seed alone. A fixed corpus pins the cases we thought of; this covers the + * shapes we did not, and asserts only invariants so it stays meaningful after + * the parser is reimplemented. + */ +function createRandom(seed: number): () => number { + let state = seed + return () => { + state = (state * 1664525 + 1013904223) % 0x100000000 + return state / 0x100000000 + } +} + +const MALFORMED_VALUES: readonly unknown[] = [ + undefined, + null, + true, + false, + 0, + 1, + 'sso', + '', + [], + ['slack'], + ['slack', 42], + ['sso', 'bogus'], + [null], + [{}], + {}, + { nested: true }, + Number.NaN, +] + +describe('parsePermissionGroupConfig invariants', () => { + const configKeys = Object.keys(DEFAULT_PERMISSION_GROUP_CONFIG) + + it('holds over randomly malformed configs', () => { + const seed = 0x5eed + const random = createRandom(seed) + + for (let iteration = 0; iteration < 2000; iteration++) { + const input: Record = {} + for (const key of configKeys) { + if (random() < 0.35) continue + input[key] = MALFORMED_VALUES[Math.floor(random() * MALFORMED_VALUES.length)] + } + + const parsed = parsePermissionGroupConfig(input) + const context = `seed ${seed}, iteration ${iteration}, input ${JSON.stringify(input)}` + + expect(Object.keys(parsed), context).toEqual(configKeys) + expect(parsePermissionGroupConfig(structuredClone(parsed)), context).toEqual(parsed) + expect( + permissionGroupFullConfigSchema.safeParse(structuredClone(parsed)).success, + context + ).toBe(true) + } + }) +}) + +describe('permission group config key coverage', () => { + it('declares the same keys in the write schema, the defaults, and the read schema', () => { + expect(Object.keys(permissionGroupConfigSchema.shape)).toEqual( + Object.keys(DEFAULT_PERMISSION_GROUP_CONFIG) + ) + expect(Object.keys(permissionGroupFullConfigSchema.shape)).toEqual( + Object.keys(DEFAULT_PERMISSION_GROUP_CONFIG) + ) + }) + + it('registers every boolean config key as a platform feature', () => { + const booleanKeys = Object.entries(DEFAULT_PERMISSION_GROUP_CONFIG) + .filter(([, value]) => typeof value === 'boolean') + .map(([key]) => key) + + expect([...PLATFORM_FEATURES.map((feature) => feature.configKey)].sort()).toEqual( + [...booleanKeys].sort() + ) + }) + + /** + * Each key gates an act that names no workspace, so each is read from the + * organization's default group only — a group scoped to specific workspaces + * cannot deny an account-level login, a workspace that does not exist yet, or + * a roster read that belongs to the organization rather than to any one + * workspace. The editor still offers the checkbox on such a group, so the + * hint is the only place an admin learns where it applies; all three shipped + * saying nothing, and a hint that omits it is a checkbox that silently + * enforces nothing wherever an admin is most likely to tick it. + */ + it.each(['disableWorkspaceCreation', 'disableCliAccess', 'hideOrgMemberDirectory'] as const)( + "tells an admin that %s is read from the organization's default group", + (configKey) => { + const feature = PLATFORM_FEATURES.find((entry) => entry.configKey === configKey) + + expect(feature?.hint).toContain("organization's default group") + } + ) + + it('gives every platform feature a unique id', () => { + const ids = PLATFORM_FEATURES.map((feature) => feature.id) + expect(new Set(ids).size).toBe(ids.length) + }) +}) diff --git a/apps/sim/lib/permission-groups/fields.ts b/apps/sim/lib/permission-groups/fields.ts new file mode 100644 index 00000000000..79b2d92e6fa --- /dev/null +++ b/apps/sim/lib/permission-groups/fields.ts @@ -0,0 +1,595 @@ +import { z } from 'zod' + +/** + * Auth modes a public file share or a chat deployment can use; admins may + * restrict the allowed subset. The two surfaces share the same four modes. + */ +export const FILE_SHARE_AUTH_TYPES = ['public', 'password', 'email', 'sso'] as const + +const shareAuthType = z.enum(FILE_SHARE_AUTH_TYPES) + +/** + * Which mechanism refuses a request when this key is set. + * + * - `capability`: an operation declares a capability whose rule reads the key, + * so the authorization funnel refuses before the use case runs. + * - `executor`: the key is read per block, tool or model at execution time by + * `assertPermissionsAllowed`. It governs what a run may *do*, which no + * operation-level gate can express. + * - `ui-only`: the key hides a surface without withholding it, so a caller that + * skips the UI still reaches the API. + * + * Declared rather than inferred, because `ui-only` is the value an admin is + * most likely to mistake for a control — twelve keys shipped that way. + * + * Not derived from the capability rules, which also name config keys, and not + * worth collapsing into them: `capabilities.ts` imports this module for + * `PermissionGroupConfigKey`, so the dependency runs one way only, and the two + * are different facts anyway — a field is storage plus admin UI, a rule is a + * decision — related many-to-many (`knowledge.create` reads two keys; + * `hideKnowledgeBaseTab` is read by three rules). This value is the single fact + * they share, and `check:permission-group-enforcement` is what keeps it honest + * in both directions. + */ +type PermissionGroupEnforcement = 'capability' | 'executor' | 'ui-only' + +/** + * Which group a key's capability is actually read from when it is enforced. + * + * - `workspace`: resolved from the group governing the caller in the workspace + * the request names, so the group being edited is the group that applies. + * - `organization`: resolved from the organization's *default* group, because + * the act names no workspace (creating one, reading the member directory). + * Setting it on any other group changes nothing at all. + * - `workspace-or-organization`: both, on different paths — a workspace-scoped + * act reads this group, and the same capability's account-level path + * (minting a key, an organization-wide invitation) falls back to the default + * group. + * + * Declared because the editor renders every key on every group, and two of them + * are read from one group no matter which is open. That was disclosed only in a + * hint an admin has to hover, so the checkbox looked like it did something on + * the group in front of them; `scope` is what lets the editor say so in the row + * itself. Required rather than optional so the next organization-scoped key + * ships marked instead of inheriting the majority answer by omission. + */ +export type PermissionGroupCapabilityScope = + | 'workspace' + | 'organization' + | 'workspace-or-organization' + +/** The admin-editor descriptor for a boolean key, rendered from the registry. */ +interface PlatformFeatureMeta { + readonly id: string + readonly label: string + readonly category: string + /** See {@link PermissionGroupCapabilityScope}. */ + readonly scope: PermissionGroupCapabilityScope + /** + * What the key withholds, in one or two short sentences. Read twice — as the + * editor's hint and as the prose `getActivePermissionGroupRestrictions` + * reports for an active restriction — so it states the restriction rather + * than what remains permitted, which reads backwards in the second place. + * + * It must describe the *access* withheld, never a surface hidden. Every key + * with `enforcement: 'capability'` refuses at the API, so a hint that says + * "hide from the sidebar" tells an admin they are tidying a nav bar when they + * are revoking a module. Twelve keys shipped worded that way while they were + * genuinely cosmetic; the wording outlived the behavior. + */ + readonly hint: string +} + +/** Prose for an allowlist, which reads differently narrowed than emptied. */ +interface AllowlistPhrasing { + readonly limited: string + readonly empty: string +} + +/** + * Coerces an untrusted value into an array of `item`, element by element. + * + * Deliberately not `z.array(item).catch(fallback)`: `.catch` is whole-value + * tolerant, so one bad member would discard every good one. For an allowlist + * that is also a fail-open change, because the fallback is `null` and `null` + * means unrestricted — a partially corrupt allowlist would stop restricting + * anything. Filtering keeps the surviving members and fails closed. + */ +function tolerantArray[] | null>( + item: TItem, + fallback: TFallback +): z.ZodType[] | TFallback> { + return z.unknown().transform((raw) => { + if (!Array.isArray(raw)) return fallback + return raw.flatMap((entry) => { + const parsed = item.safeParse(entry) + return parsed.success ? [parsed.data as z.infer] : [] + }) + }) +} + +interface BooleanRestrictionField { + readonly kind: 'boolean-restriction' + readonly writeSchema: z.ZodOptional + readonly readSchema: z.ZodBoolean + readonly tolerantSchema: z.ZodType + readonly default: boolean + readonly enforcement: PermissionGroupEnforcement + readonly feature: PlatformFeatureMeta +} + +interface AllowlistField { + readonly kind: 'allowlist' + readonly writeSchema: z.ZodOptional>> + readonly readSchema: z.ZodNullable> + readonly tolerantSchema: z.ZodType[] | null> + readonly default: null + readonly enforcement: PermissionGroupEnforcement + readonly phrasing: AllowlistPhrasing +} + +interface DenylistField { + readonly kind: 'denylist' + readonly writeSchema: z.ZodOptional> + readonly readSchema: z.ZodDefault> + readonly tolerantSchema: z.ZodType[]> + readonly default: never[] + readonly enforcement: PermissionGroupEnforcement + readonly phrasing: string +} + +/** + * The structural shape every entry satisfies. Deliberately loose in its schema + * members: a concrete `AllowlistField` is not reliably assignable to + * `AllowlistField` through zod's internals, and widening the registry + * to this type would erase the per-key output types the config depends on. It + * exists for `satisfies`, never as an annotation. + */ +type PermissionGroupField = { + readonly kind: 'boolean-restriction' | 'allowlist' | 'denylist' + readonly writeSchema: z.ZodType + readonly readSchema: z.ZodType + readonly tolerantSchema: z.ZodType + readonly default: unknown + readonly enforcement: PermissionGroupEnforcement +} + +function booleanRestriction( + enforcement: PermissionGroupEnforcement, + feature: PlatformFeatureMeta +): BooleanRestrictionField { + const schema = z.boolean() + return { + kind: 'boolean-restriction', + writeSchema: schema.optional(), + readSchema: schema, + tolerantSchema: schema.catch(false), + default: false, + enforcement, + feature, + } +} + +function allowlist( + item: TItem, + enforcement: PermissionGroupEnforcement, + phrasing: AllowlistPhrasing +): AllowlistField { + const schema = z.array(item).nullable() + return { + kind: 'allowlist', + writeSchema: schema.optional(), + readSchema: schema, + tolerantSchema: tolerantArray(item, null), + default: null, + enforcement, + phrasing, + } +} + +function denylist( + item: TItem, + enforcement: PermissionGroupEnforcement, + phrasing: string +): DenylistField { + const schema = z.array(item) + return { + kind: 'denylist', + writeSchema: schema.optional(), + readSchema: schema.default([]), + tolerantSchema: tolerantArray(item, [] as never[]), + default: [], + enforcement, + phrasing, + } +} + +/** + * Every permission-group config key, in wire order. + * + * Declaration order here is the key order of `PermissionGroupConfig`, of both + * zod schemas, and of every config JSON that crosses the API boundary. The + * group editor's dirty check compares stringified configs, so reordering + * entries is a breaking change. + * + * Adding a key here adds it to the write schema, the read schema, the type, the + * defaults, the tolerant parser and — for a boolean — the admin editor. It does + * not add enforcement: `enforcement` names the mechanism that refuses, and + * `check:permission-group-enforcement` refuses a key that claims one it does + * not have. + */ +export const PERMISSION_GROUP_FIELDS = { + allowedIntegrations: allowlist(z.string(), 'executor', { + limited: 'Integrations and blocks are limited to effectiveConfig.allowedIntegrations.', + empty: 'No non-exempt integrations or blocks are allowed.', + }), + allowedModelProviders: allowlist(z.string(), 'executor', { + limited: 'Model providers are limited to effectiveConfig.allowedModelProviders.', + empty: 'No model providers are allowed.', + }), + deniedModels: denylist( + z.string(), + 'executor', + 'Models listed in effectiveConfig.deniedModels are blocked.' + ), + deniedTools: denylist( + z.string(), + 'executor', + 'Integration tools listed in effectiveConfig.deniedTools are blocked.' + ), + hideTraceSpans: booleanRestriction('capability', { + scope: 'workspace', + id: 'hide-trace-spans', + label: 'Trace Spans', + category: 'Logs', + hint: 'Withhold per-block trace spans from logs and from the API.', + }), + hideKnowledgeBaseTab: booleanRestriction('capability', { + scope: 'workspace', + id: 'hide-knowledge-base', + label: 'Knowledge Base', + category: 'Knowledge Base', + hint: 'Revoke the Knowledge Base module. Members cannot open, search, or query any knowledge base.', + }), + hideTablesTab: booleanRestriction('capability', { + scope: 'workspace', + id: 'hide-tables', + label: 'Tables', + category: 'Tables', + hint: 'Revoke the Tables module. Members cannot read or write any table.', + }), + hideCopilot: booleanRestriction('capability', { + scope: 'workspace', + id: 'hide-copilot', + label: 'Chat', + category: 'Modules', + hint: 'Revoke Chat. Members cannot ask Sim to build or edit anything.', + }), + hideIntegrationsTab: booleanRestriction('capability', { + scope: 'workspace-or-organization', + id: 'hide-integrations', + label: 'Integrations', + category: 'Credentials & Access', + hint: 'Revoke integration connections. Members cannot view, add, or remove an OAuth connection.', + }), + hideSecretsTab: booleanRestriction('capability', { + scope: 'workspace', + id: 'hide-secrets', + label: 'Secrets', + category: 'Credentials & Access', + hint: 'Revoke secrets. Members cannot read, add, or change a workspace environment variable.', + }), + hideApiKeysTab: booleanRestriction('capability', { + scope: 'workspace-or-organization', + id: 'hide-api-keys', + label: 'API Keys', + category: 'Credentials & Access', + hint: 'Revoke workspace API keys. Members cannot list, create, or revoke one.', + }), + hideInboxTab: booleanRestriction('capability', { + scope: 'workspace', + id: 'hide-inbox', + label: 'Sim Mailer', + category: 'Modules', + hint: 'Revoke the Sim Mailer inbox. Members cannot read or send mail.', + }), + hideFilesTab: booleanRestriction('capability', { + scope: 'workspace', + id: 'hide-files', + label: 'Files', + category: 'Files', + hint: 'Revoke the Files module. Members cannot list, upload, or download workspace files.', + }), + disableMcpTools: booleanRestriction('capability', { + scope: 'workspace', + id: 'disable-mcp', + label: 'MCP Tools', + category: 'Tools', + hint: 'Block agents from calling MCP tools.', + }), + disableCustomTools: booleanRestriction('capability', { + scope: 'workspace', + id: 'disable-custom-tools', + label: 'Custom Tools', + category: 'Tools', + hint: 'Block agents from calling user-defined custom tools.', + }), + disableSkills: booleanRestriction('capability', { + scope: 'workspace', + id: 'disable-skills', + label: 'Skills', + category: 'Tools', + hint: 'Block agents from loading skills.', + }), + disableInvitations: booleanRestriction('capability', { + scope: 'workspace-or-organization', + id: 'disable-invitations', + label: 'Invitations', + category: 'Collaboration', + hint: 'Prevent inviting anyone to a workspace or to the organization.', + }), + disablePublicApi: booleanRestriction('capability', { + scope: 'workspace', + id: 'disable-public-api', + label: 'Public API', + category: 'Deployment', + hint: 'Revoke public API access. Calls to a deployed workflow are refused.', + }), + disablePublicFileSharing: booleanRestriction('capability', { + scope: 'workspace', + id: 'disable-public-file-sharing', + label: 'Public Sharing', + category: 'Files', + hint: 'Revoke public file sharing. Members cannot create a share link.', + }), + allowedFileShareAuthTypes: allowlist(shareAuthType, 'capability', { + limited: + 'Public file-share authentication is limited to effectiveConfig.allowedFileShareAuthTypes.', + empty: 'No public file-share authentication modes are allowed.', + }), + hideDeployApi: booleanRestriction('capability', { + scope: 'workspace', + id: 'hide-deploy-api', + label: 'API Deployment', + category: 'Deployment', + hint: 'Prevent deploying a workflow as an API endpoint.', + }), + hideDeployMcp: booleanRestriction('capability', { + scope: 'workspace', + id: 'hide-deploy-mcp', + label: 'MCP Server', + category: 'Deployment', + hint: 'Prevent exposing a workflow as an MCP server.', + }), + hideDeployChatbot: booleanRestriction('capability', { + scope: 'workspace', + id: 'hide-deploy-chatbot', + label: 'Chat Deployment', + category: 'Deployment', + hint: 'Prevent publishing a workflow as a chat.', + }), + allowedChatDeployAuthTypes: allowlist(shareAuthType, 'capability', { + limited: + 'Chat deployment authentication is limited to effectiveConfig.allowedChatDeployAuthTypes.', + empty: 'No chat deployment authentication modes are allowed.', + }), + /** + * Appended rather than grouped with the other restrictions: declaration order + * is the wire order, and moving an existing key would read as an unsaved + * change in every open group editor. + */ + disablePersonalApiKeys: booleanRestriction('capability', { + scope: 'workspace-or-organization', + id: 'disable-personal-api-keys', + label: 'Personal API Keys', + category: 'Credentials & Access', + hint: 'Prevent members from using a personal API key against this workspace.', + }), + disableLogExport: booleanRestriction('capability', { + scope: 'workspace', + id: 'disable-log-export', + label: 'Log Export', + category: 'Logs', + hint: 'Prevent downloading execution logs as a CSV.', + }), + hideCostInfo: booleanRestriction('capability', { + scope: 'workspace', + id: 'hide-cost-info', + label: 'Execution Cost', + category: 'Logs', + hint: 'Withhold execution cost. Logs and member exports omit cost and token spend; organization-level data drains, configurable by org admins only, are not projected.', + }), + disableKnowledgeBaseCreation: booleanRestriction('capability', { + scope: 'workspace', + id: 'disable-knowledge-base-creation', + label: 'Knowledge Base Creation', + category: 'Knowledge Base', + hint: 'Prevent creating knowledge bases, leaving existing ones queryable.', + }), + disableKnowledgeBaseFileUpload: booleanRestriction('capability', { + scope: 'workspace', + id: 'disable-knowledge-base-upload', + label: 'Knowledge Base Uploads', + category: 'Knowledge Base', + hint: 'Prevent uploading local documents, leaving sanctioned connectors as the only source.', + }), + allowedKnowledgeConnectors: allowlist(z.string(), 'capability', { + limited: 'Knowledge base connectors are limited to effectiveConfig.allowedKnowledgeConnectors.', + empty: 'No knowledge base connectors are allowed.', + }), + disableTableCreation: booleanRestriction('capability', { + scope: 'workspace', + id: 'disable-table-creation', + label: 'Table Creation', + category: 'Tables', + hint: 'Prevent creating tables, leaving existing ones usable.', + }), + disableTableExport: booleanRestriction('capability', { + scope: 'workspace', + id: 'disable-table-export', + label: 'Table Export', + category: 'Tables', + hint: 'Prevent downloading a whole table as CSV or JSON.', + }), + disableBulkFileDownload: booleanRestriction('capability', { + scope: 'workspace', + id: 'disable-bulk-file-download', + label: 'Bulk Download', + category: 'Files', + hint: 'Prevent downloading folders as an archive.', + }), + disablePersonalCredentials: booleanRestriction('capability', { + scope: 'workspace', + id: 'disable-personal-credentials', + label: 'Personal Credentials', + category: 'Credentials & Access', + hint: 'Prevent connecting personal credentials, leaving only workspace-shared ones.', + }), + disableWorkspaceCreation: booleanRestriction('capability', { + scope: 'organization', + id: 'disable-workspace-creation', + label: 'Workspace Creation', + category: 'Collaboration', + hint: "Prevent creating new workspaces, which no existing group would govern. Read from the organization's default group, because creating a workspace names none.", + }), + hideOrgMemberDirectory: booleanRestriction('capability', { + scope: 'organization', + id: 'hide-org-member-directory', + label: 'Member Directory', + category: 'Collaboration', + hint: "Withhold the member directory. Members cannot see the names or email addresses of other members. Read from the organization's default group, because the directory belongs to the organization and names no workspace.", + }), + disableCliAccess: booleanRestriction('capability', { + scope: 'workspace-or-organization', + id: 'disable-cli-access', + label: 'CLI Access', + category: 'Credentials & Access', + hint: "Prevent approving a CLI login, which mints a key for the public API. A login naming one of this group's workspaces is refused; an account-level login names none, so it is read from the organization's default group.", + }), + disableWebhookTriggers: booleanRestriction('capability', { + scope: 'workspace', + id: 'disable-webhook-triggers', + label: 'Webhook Triggers', + category: 'Deployment', + hint: 'Prevent making a workflow reachable from an inbound webhook.', + }), + disableToolAutoApproval: booleanRestriction('capability', { + scope: 'workspace', + id: 'disable-tool-auto-approval', + label: 'Tool Auto-Approval', + category: 'Tools', + hint: 'Prevent silencing a tool confirmation, so every call is confirmed again.', + }), +} satisfies Record + +export type PermissionGroupFields = typeof PERMISSION_GROUP_FIELDS +export type PermissionGroupConfigKey = keyof PermissionGroupFields + +type DerivedPermissionGroupConfig = { + [K in PermissionGroupConfigKey]: z.infer +} + +/** + * The effective permission-group configuration. + * + * Declared as an interface over the derived shape so it keeps a stable name in + * errors and hovers rather than expanding to a mapped type at every use site. + */ +export interface PermissionGroupConfig extends DerivedPermissionGroupConfig {} + +/** The per-field properties that each project into one whole-config shape. */ +type FieldProjection = 'writeSchema' | 'readSchema' | 'tolerantSchema' | 'default' + +/** + * Collects one property from every field, preserving declaration order. + * + * `Object.fromEntries` widens to an index signature, so the result is asserted + * back to the mapped type. The assertion is safe by construction — the value at + * each key is that key's own entry read at a fixed property — but TypeScript + * cannot follow that correspondence through a union of field kinds, so it is + * stated once here rather than at each of the four shapes. + */ +function collectFieldProperty

( + property: P +): { [K in PermissionGroupConfigKey]: PermissionGroupFields[K][P] } { + const entries = Object.entries(PERMISSION_GROUP_FIELDS).map(([key, field]) => [ + key, + field[property], + ]) + return Object.fromEntries(entries) as { + [K in PermissionGroupConfigKey]: PermissionGroupFields[K][P] + } +} + +/** The PATCH shape: every key optional, so a partial config is a legal write. */ +export const permissionGroupWriteShape = collectFieldProperty('writeSchema') + +/** + * The config a create or update body may carry: every key optional, so a caller + * patches only what it means to change. The route merges the result over the + * group's stored config, which is what heals a row written before a key + * existed. + */ +export const permissionGroupConfigSchema = z.object(permissionGroupWriteShape) + +/** The wire shape: every key present, denylists defaulted. */ +export const permissionGroupReadShape = collectFieldProperty('readSchema') + +const tolerantConfigSchema = z.object(collectFieldProperty('tolerantSchema')) + +/** + * The unrestricted config: allowlists `null` (everything permitted), denylists + * empty, restrictions off. Assigned rather than asserted, so each field's + * declared default has to actually satisfy that key's config type. + */ +export const DEFAULT_PERMISSION_GROUP_CONFIG: PermissionGroupConfig = + collectFieldProperty('default') + +/** + * Coerces an untrusted stored config into a complete, well-typed one. + * + * Never throws: every field is total, and the guard covers the shapes a `jsonb` + * column can hold that `z.object()` refuses. `Array.isArray` is load-bearing — + * `typeof [] === 'object'`, so an array-valued column would otherwise reach + * `.parse` and throw where it used to coerce to defaults. + */ +export function parsePermissionGroupConfig(config: unknown): PermissionGroupConfig { + if (!config || typeof config !== 'object' || Array.isArray(config)) { + return DEFAULT_PERMISSION_GROUP_CONFIG + } + return tolerantConfigSchema.parse(config) +} + +/** + * Compile-time proof that deriving the config from the registry did not widen + * it. Every consumer reads these keys expecting a precise type, and a zod + * generic that degraded to `unknown` would be invisible at runtime — the values + * would still be right, so no test would fail, while every call site quietly + * lost its narrowing. Declared here rather than in a `.test.ts` because + * type-check excludes test files. + */ +type Exact = [A] extends [B] ? ([B] extends [A] ? true : false) : false + +/** + * Fails to compile unless `T` is exactly `true`, which is what makes the aliases + * below load-bearing. They are deliberately unexported: the constraint is + * checked where the alias is declared, so an export bought nothing but the + * appearance of a consumer that never existed. Nothing may import them; they + * are unused on purpose, and deleting one deletes the proof. + */ +type Assert = T + +type AssertsAllowlistStaysPrecise = Assert< + Exact +> +type AssertsDenylistStaysPrecise = Assert> +type AssertsRestrictionStaysPrecise = Assert> +type AssertsAuthTypesStayPrecise = Assert< + Exact< + PermissionGroupConfig['allowedFileShareAuthTypes'], + (typeof FILE_SHARE_AUTH_TYPES)[number][] | null + > +> +type AssertsParserReturnsTheConfig = Assert< + Exact, PermissionGroupConfig> +> diff --git a/apps/sim/lib/permission-groups/integration-allowlist.test.ts b/apps/sim/lib/permission-groups/integration-allowlist.test.ts index 314bdf9c86e..fd183581ac3 100644 --- a/apps/sim/lib/permission-groups/integration-allowlist.test.ts +++ b/apps/sim/lib/permission-groups/integration-allowlist.test.ts @@ -1,16 +1,145 @@ +/** + * @vitest-environment node + * + * These helpers read the generated successor map rather than the block + * registry, so every id below is a real one and the assertions are about the + * repository's actual lifecycle facts: `slack` was replaced by `slack_v2`, + * `notion` by `notion_v2`, `file` by `file_v5`. + */ import { describe, expect, it } from 'vitest' -import { intersectIntegrationAllowlists } from '@/lib/permission-groups/integration-allowlist' +import { + intersectAccessControlAllowlists, + intersectIntegrationAllowlists, + resolveAccessControlBlockType, + toAccessControlAllowlist, +} from '@/lib/permission-groups/integration-allowlist' + +describe('resolveAccessControlBlockType', () => { + it('judges a superseded block as its successor', () => { + expect(resolveAccessControlBlockType('slack')).toBe('slack_v2') + }) + + /** The map is flattened, so a chain costs one lookup and never a partial hop. */ + it('answers the terminal version of a chain, not an intermediate one', () => { + expect(resolveAccessControlBlockType('file')).toBe('file_v5') + expect(resolveAccessControlBlockType('file_v3')).toBe('file_v5') + }) + + it('leaves a current block alone', () => { + expect(resolveAccessControlBlockType('slack_v2')).toBe('slack_v2') + }) + + /** + * A retired block with no successor keeps its own identity, which is the only + * id an admin can permit it under. + */ + it('keeps its own identity when nothing replaced it', () => { + expect(resolveAccessControlBlockType('thinking')).toBe('thinking') + }) + + it('accepts the dashed spelling the registry also normalizes', () => { + expect(resolveAccessControlBlockType('google-sheets')).toBe('google_sheets_v2') + }) + + /** + * `allowedIntegrations` is admin-supplied jsonb and `ALLOWED_INTEGRATIONS` is + * hand-written, so an arbitrary string reaches the successor map. An + * inherited key must stay an ordinary unresolved id rather than answering + * with `Object.prototype`'s function. + */ + it('leaves an object-prototype key alone', () => { + expect(resolveAccessControlBlockType('constructor')).toBe('constructor') + expect(resolveAccessControlBlockType('toString')).toBe('toString') + expect(resolveAccessControlBlockType('__proto__')).toBe('__proto__') + }) +}) + +describe('toAccessControlAllowlist', () => { + it('keeps an unrestricted allowlist unrestricted', () => { + expect(toAccessControlAllowlist(null)).toBeNull() + }) + + /** + * `ALLOWED_INTEGRATIONS` is written by hand against whatever ids its author + * knows, so a deployment that permitted `slack` must not refuse `slack_v2`. + */ + it('judges a policy entry naming a retired id as its successor', () => { + const allowlist = toAccessControlAllowlist(['Slack']) + + expect(allowlist?.has('slack_v2')).toBe(true) + expect(allowlist?.has('slack')).toBe(false) + }) + + it('denies everything for an empty allowlist', () => { + expect(toAccessControlAllowlist([])?.size).toBe(0) + }) + + /** + * A prototype key used to resolve to an inherited function and throw on + * `.toLowerCase()`, turning one configured string into a 500 on every + * enforcement path that read the group. + */ + it('indexes an object-prototype entry as an ordinary block type', () => { + const allowlist = toAccessControlAllowlist(['constructor', 'slack']) + + expect(allowlist?.has('constructor')).toBe(true) + expect(allowlist?.has('slack_v2')).toBe(true) + }) +}) + +describe('intersectAccessControlAllowlists', () => { + /** + * The two policies are written independently — `ALLOWED_INTEGRATIONS` by hand + * against whatever ids its author knew, the group through an editor that only + * offers current ones — so they routinely name the same integration by + * different vintages. Intersecting before resolving leaves those disjoint, + * which refuses an integration both policies allow. + */ + it('intersects a retired id against its successor', () => { + expect([...(intersectAccessControlAllowlists(['slack'], ['slack_v2']) ?? [])]).toEqual([ + 'slack_v2', + ]) + expect([...(intersectAccessControlAllowlists(['slack_v2'], ['slack']) ?? [])]).toEqual([ + 'slack_v2', + ]) + }) + + it('keeps either side null as unrestricted', () => { + expect([...(intersectAccessControlAllowlists(null, ['notion']) ?? [])]).toEqual(['notion_v2']) + expect([...(intersectAccessControlAllowlists(['notion'], null) ?? [])]).toEqual(['notion_v2']) + expect(intersectAccessControlAllowlists(null, null)).toBeNull() + }) + + it('keeps an empty policy denying everything', () => { + expect(intersectAccessControlAllowlists([], ['notion'])?.size).toBe(0) + }) + + it('drops an integration only one policy names', () => { + expect([...(intersectAccessControlAllowlists(['notion', 'gmail'], ['gmail']) ?? [])]).toEqual([ + 'gmail_v2', + ]) + }) +}) describe('intersectIntegrationAllowlists', () => { it('uses the configured list when the other policy is unrestricted', () => { - expect(intersectIntegrationAllowlists(null, ['Slack'])).toEqual(['slack']) - expect(intersectIntegrationAllowlists(['Notion'], null)).toEqual(['notion']) + expect(intersectIntegrationAllowlists(null, ['Slack'])).toEqual(['slack_v2']) + expect(intersectIntegrationAllowlists(['Notion'], null)).toEqual(['notion_v2']) expect(intersectIntegrationAllowlists(null, null)).toBeNull() }) + /** + * The list form must canonicalize identically to the set form, or the config + * the catalogs carry and the gate the block path applies would disagree about + * the same two policies. + */ + it('keeps a mixed-vintage integration both policies allow', () => { + expect(intersectIntegrationAllowlists(['slack'], ['slack_v2'])).toEqual(['slack_v2']) + }) + it('keeps only integrations allowed by both policies', () => { expect(intersectIntegrationAllowlists(['Slack', 'Notion'], ['notion', 'gmail'])).toEqual([ - 'notion', + 'notion_v2', ]) }) diff --git a/apps/sim/lib/permission-groups/integration-allowlist.ts b/apps/sim/lib/permission-groups/integration-allowlist.ts index dd38b4ce07d..8f1789c48ef 100644 --- a/apps/sim/lib/permission-groups/integration-allowlist.ts +++ b/apps/sim/lib/permission-groups/integration-allowlist.ts @@ -1,29 +1,100 @@ +import { BLOCK_ACCESS_SUCCESSORS } from '@/lib/permission-groups/block-successors.generated' + /** - * Intersects integration allowlists from independent policy layers. - * `null` means unrestricted, while an empty array denies every integration. + * The block type an allowlist decision about `blockType` is really made against. + * + * A superseded version resolves to the successor its `sunset.replacedBy` names, + * transitively, so allowing or denying an integration covers every version of + * it. Without this an admin would have to know each retired id and deny it + * individually — and could not, since the editor only offers the current ones. + * + * A retired block with no successor keeps its own identity and appears in the + * editor under it, which is the only way an admin can decide about it at all. + * + * Reads the generated projection of the registry rather than the registry + * itself: this module sits under the authorization funnel, which + * `scripts/check-application-graph.ts` forbids from importing `blocks/`. + * `check:block-successors` fails the build when the projection drifts. */ -export function intersectIntegrationAllowlists( - first: readonly string[] | null, - second: readonly string[] | null -): string[] | null { - const normalizedFirst = first?.map((integration) => integration.toLowerCase()) ?? null - const normalizedSecond = second?.map((integration) => integration.toLowerCase()) ?? null - - if (normalizedFirst === null) return normalizedSecond - if (normalizedSecond === null) return normalizedFirst +export function resolveAccessControlBlockType(blockType: string): string { + return ownSuccessor(blockType) ?? ownSuccessor(blockType.replace(/-/g, '_')) ?? blockType +} - const secondSet = new Set(normalizedSecond) - return normalizedFirst.filter((integration) => secondSet.has(integration)) +/** + * Reads the successor map by its own keys only. + * + * The generated map is an object literal with an intact prototype, so a bare + * bracket lookup answers `constructor`, `toString`, `valueOf` and friends with + * an inherited function. The ids reaching here come from admin-supplied jsonb + * (`allowedIntegrations`) and from `ALLOWED_INTEGRATIONS`, so a group naming + * `constructor` made {@link toAccessControlAllowlist} call `.toLowerCase()` on + * a function and throw — an unclassified 500 on every enforcement path that + * read that group. `getBlock` guards the registry the same way for the same + * reason. + */ +function ownSuccessor(blockType: string): string | undefined { + return Object.hasOwn(BLOCK_ACCESS_SUCCESSORS, blockType) + ? BLOCK_ACCESS_SUCCESSORS[blockType] + : undefined } /** - * The lowercased block types an allowlist permits, indexed for membership tests. - * `null` stays `null` — unrestricted, not "nothing allowed". + * The allowlist, indexed for membership tests against the block type an + * allowlist decision is made against. `null` stays `null` — unrestricted, not + * "nothing allowed". + * + * Both sides have to be normalized or they compare different vocabularies. A + * policy list can name a retired id: `ALLOWED_INTEGRATIONS` is written by hand + * against whatever ids the author knows, so `ALLOWED_INTEGRATIONS=slack` is the + * expected way to permit Slack. The checked type is always successor-resolved, + * so without normalizing the policy the deployment that permitted `slack` would + * refuse every `slack_v2` block in it. */ -export function toAllowedIntegrationTypes( +export function toAccessControlAllowlist( allowedIntegrations: readonly string[] | null ): ReadonlySet | null { return allowedIntegrations - ? new Set(allowedIntegrations.map((integration) => integration.toLowerCase())) + ? new Set( + allowedIntegrations.map((integration) => + resolveAccessControlBlockType(integration.toLowerCase()).toLowerCase() + ) + ) : null } + +/** + * Intersects two independent integration policies in the *resolved* vocabulary. + * + * Each side is canonicalized before the intersection, not after. A policy list + * can name a retired id while the other names its successor — + * `ALLOWED_INTEGRATIONS=slack` against a group naming `slack_v2` — and folding + * only case leaves those two ids disjoint, intersecting to nothing and hiding + * an integration both policies allow. `null` stays unrestricted on either side. + */ +export function intersectAccessControlAllowlists( + first: readonly string[] | null, + second: readonly string[] | null +): ReadonlySet | null { + const resolvedFirst = toAccessControlAllowlist(first) + const resolvedSecond = toAccessControlAllowlist(second) + if (resolvedFirst === null) return resolvedSecond + if (resolvedSecond === null) return resolvedFirst + return new Set([...resolvedFirst].filter((type) => resolvedSecond.has(type))) +} + +/** + * Intersects integration allowlists from independent policy layers, as a list. + * `null` means unrestricted, while an empty array denies every integration. + * + * The list form of {@link intersectAccessControlAllowlists}, for the callers + * that carry the effective policy on a `PermissionGroupConfig`. It canonicalizes + * for the same reason and the result is in the same resolved vocabulary, so + * callers must successor-resolve the type they check against it. + */ +export function intersectIntegrationAllowlists( + first: readonly string[] | null, + second: readonly string[] | null +): string[] | null { + const intersection = intersectAccessControlAllowlists(first, second) + return intersection === null ? null : [...intersection] +} diff --git a/apps/sim/lib/permission-groups/model-access.ts b/apps/sim/lib/permission-groups/model-access.ts index b677f96f15b..b78f98fd4e2 100644 --- a/apps/sim/lib/permission-groups/model-access.ts +++ b/apps/sim/lib/permission-groups/model-access.ts @@ -1,4 +1,4 @@ -import type { PermissionGroupConfig } from '@/lib/permission-groups/types' +import type { PermissionGroupConfig } from '@/lib/permission-groups/fields' import { findProviderFromModel } from '@/providers/utils' /** Decides whether the caller's permission group allows a concrete model id. */ diff --git a/apps/sim/lib/permission-groups/queries.ts b/apps/sim/lib/permission-groups/queries.ts index 06bc3e96eb9..f61ef57e394 100644 --- a/apps/sim/lib/permission-groups/queries.ts +++ b/apps/sim/lib/permission-groups/queries.ts @@ -6,8 +6,11 @@ import { workspace, } from '@sim/db/schema' import { asc, count, desc, eq, inArray } from 'drizzle-orm' -import { getActivePermissionGroupRestrictions } from '@/lib/permission-groups/features' -import type { PermissionGroupConfig } from '@/lib/permission-groups/types' +import { + type ActivePermissionGroupRestriction, + getActivePermissionGroupRestrictions, +} from '@/lib/permission-groups/features' +import type { PermissionGroupConfig } from '@/lib/permission-groups/fields' /** A workspace reference (id + display name). */ export interface OrgWorkspaceRef { @@ -38,7 +41,7 @@ export interface PermissionGroupRosterEntry { isDefault: boolean memberCount: number workspaces: OrgWorkspaceRef[] - activeRestrictions: Array<{ key: string; description: string }> + activeRestrictions: ActivePermissionGroupRestriction[] } /** diff --git a/apps/sim/lib/permission-groups/request-scope.server.ts b/apps/sim/lib/permission-groups/request-scope.server.ts new file mode 100644 index 00000000000..c972449dd5f --- /dev/null +++ b/apps/sim/lib/permission-groups/request-scope.server.ts @@ -0,0 +1,67 @@ +import type { PermissionGroupConfig } from '@/lib/permission-groups/fields' + +/** + * A resolution key, `userId:workspaceId`. `organizationId` is deliberately not + * part of it — see `resolvePermissionGroupConfig` for why. + * + * Named for the scope rather than the config, so it cannot be mistaken for + * `PermissionGroupConfigKey` in `fields.ts`, which is a config *field* name. + */ +export type PermissionGroupScopeKey = `${string}:${string}` + +/** + * The per-scope memo: a resolution key to the in-flight resolution for it. + * + * Holds the promise rather than the resolved value so N concurrent + * authorizations share one query instead of racing to start several. + */ +export type PermissionGroupConfigStore = Map< + PermissionGroupScopeKey, + Promise +> + +interface Storage { + getStore(): T | undefined + run(store: T, fn: () => R): R +} + +/** + * AsyncLocalStorage is only available in Node.js. Parts of this graph reach the + * Edge runtime, so fall back to a no-op that simply runs the callback — the + * resolver then degrades to its React `cache()` memo, which is slower but never + * wrong. + */ +let storage: Storage + +if (typeof globalThis.process !== 'undefined' && globalThis.process.versions?.node) { + // eslint-disable-next-line @typescript-eslint/no-require-imports + const { AsyncLocalStorage } = require('node:async_hooks') as typeof import('node:async_hooks') + storage = new AsyncLocalStorage() +} else { + storage = { + getStore: () => undefined, + run: (_store: PermissionGroupConfigStore, fn: () => R) => fn(), + } +} + +/** + * Establishes one permission-group memo for everything a request or job does. + * + * A request that authorizes several operations — a bulk mutation, a route that + * runs two use cases — would otherwise resolve the same group once per + * operation. Nesting this inside the request context means every route handler + * gets it without threading a parameter through every operation. + * + * This module is deliberately free of runtime imports. `withRouteHandler` wraps + * every route in the app, so anything reachable from here is loaded by every + * route and every route test; the resolver that fills the store lives in + * `config-scope.server.ts`, which only the gate call sites import. + */ +export function withPermissionGroupScope(run: () => R): R { + return storage.run(new Map(), run) +} + +/** The memo for the current scope, or undefined when running outside one. */ +export function getPermissionGroupConfigStore(): PermissionGroupConfigStore | undefined { + return storage.getStore() +} diff --git a/apps/sim/lib/permission-groups/resolve.server.test.ts b/apps/sim/lib/permission-groups/resolve.server.test.ts new file mode 100644 index 00000000000..12e4176cb14 --- /dev/null +++ b/apps/sim/lib/permission-groups/resolve.server.test.ts @@ -0,0 +1,104 @@ +/** + * @vitest-environment node + */ +import { resetEnvFlagsMock, setEnvFlags } from '@sim/testing' +import { afterAll, beforeEach, describe, expect, it, vi } from 'vitest' + +const { mockIsOrganizationOnEnterprisePlan, mockGetWorkspaceWithOwner } = vi.hoisted(() => ({ + mockIsOrganizationOnEnterprisePlan: vi.fn(), + mockGetWorkspaceWithOwner: vi.fn(), +})) + +vi.mock('@/lib/billing/core/subscription', () => ({ + isOrganizationOnEnterprisePlan: mockIsOrganizationOnEnterprisePlan, +})) + +vi.mock('@/lib/workspaces/permissions/utils', () => ({ + getWorkspaceWithOwner: mockGetWorkspaceWithOwner, +})) + +import { + getUserPermissionConfig, + getUserPermissionConfigForOrganization, + resolveVerifiedUserAccessControlContext, +} from '@/lib/permission-groups/resolve.server' + +const ORGANIZATION_ID = 'org-1' +const USER_ID = 'user-1' +const WORKSPACE_ID = 'workspace-1' + +/** + * Stands in for the entitlement resolver's two regimes: it answers `false` for + * the lenient default — which is exactly what a swallowed billing outage looks + * like — and rejects only for a caller that asked to throw. A resolution path + * that drops the `'throw'` argument therefore reads the outage as "not + * entitled" and these tests go red. + */ +function entitlementReadFails(): void { + mockIsOrganizationOnEnterprisePlan.mockImplementation( + async (_organizationId: string, onError?: string) => { + if (onError === 'throw') throw new Error('billing database unavailable') + return false + } + ) +} + +describe('permission-group resolution under a failed entitlement read', () => { + beforeEach(() => { + vi.clearAllMocks() + setEnvFlags({ isHosted: true, isAccessControlEnabled: true }) + mockGetWorkspaceWithOwner.mockResolvedValue({ organizationId: ORGANIZATION_ID }) + }) + + afterAll(resetEnvFlagsMock) + + /** + * `config: null` is not a stricter answer — it means every capability allowed + * and every allowlist off. Resolving it from a billing-read failure would + * turn the whole regime off for the request, so the failure has to surface. + */ + it('rejects rather than resolving an unrestricted context for a verified workspace', async () => { + entitlementReadFails() + + await expect( + resolveVerifiedUserAccessControlContext(USER_ID, WORKSPACE_ID, ORGANIZATION_ID) + ).rejects.toThrow('billing database unavailable') + expect(mockIsOrganizationOnEnterprisePlan).toHaveBeenCalledWith(ORGANIZATION_ID, 'throw') + }) + + it('rejects rather than resolving a null config from the workspace-lookup path', async () => { + entitlementReadFails() + + await expect(getUserPermissionConfig(USER_ID, WORKSPACE_ID)).rejects.toThrow( + 'billing database unavailable' + ) + }) + + it('rejects rather than resolving a null config for the organization-addressed path', async () => { + entitlementReadFails() + + await expect(getUserPermissionConfigForOrganization(ORGANIZATION_ID)).rejects.toThrow( + 'billing database unavailable' + ) + expect(mockIsOrganizationOnEnterprisePlan).toHaveBeenCalledWith(ORGANIZATION_ID, 'throw') + }) + + /** + * The fail-closed policy must not turn a genuine plan lapse into an error: + * an organization that simply is not on the plan still resolves to an + * inactive context. + */ + it('still resolves an inactive context when the organization is genuinely unentitled', async () => { + mockIsOrganizationOnEnterprisePlan.mockResolvedValue(false) + + await expect( + resolveVerifiedUserAccessControlContext(USER_ID, WORKSPACE_ID, ORGANIZATION_ID) + ).resolves.toEqual({ + organizationId: ORGANIZATION_ID, + entitled: false, + permissionGroup: null, + config: null, + }) + await expect(getUserPermissionConfigForOrganization(ORGANIZATION_ID)).resolves.toBeNull() + }) +}) diff --git a/apps/sim/lib/permission-groups/resolve.server.ts b/apps/sim/lib/permission-groups/resolve.server.ts new file mode 100644 index 00000000000..ec5beb74d0e --- /dev/null +++ b/apps/sim/lib/permission-groups/resolve.server.ts @@ -0,0 +1,296 @@ +/** + * Resolves the permission group governing a user, and its config. + * + * Lives here rather than in `ee/access-control` because the authorization + * funnel reads it: `capability-assertions.ts` and `config-scope.server.ts` sit + * under `@/lib/core/application`, which ~24 domain `operations.ts` modules + * import. `ee/access-control/utils/permission-check.ts` also holds the model, + * block and tool gates, and those reach the provider registry, the block + * registry and the billing barrel — a graph no authorization decision should + * load. Splitting resolution out is what keeps the funnel light; + * `scripts/check-application-graph.ts` fails the build if the edge returns. + * + * `permission-check.ts` re-exports these, so the surfaces that read every + * validator from one module are unaffected. + */ +import { db } from '@sim/db' +import { permissionGroup, permissionGroupMember, permissionGroupWorkspace } from '@sim/db/schema' +import { and, asc, eq, sql } from 'drizzle-orm' +import { isOrganizationOnEnterprisePlan } from '@/lib/billing/core/subscription' +import { + getAllowedIntegrationsFromEnv, + isAccessControlEnabled, + isHosted, +} from '@/lib/core/config/env-flags' +import { + DEFAULT_PERMISSION_GROUP_CONFIG, + type PermissionGroupConfig, + parsePermissionGroupConfig, +} from '@/lib/permission-groups/fields' +import { intersectIntegrationAllowlists } from '@/lib/permission-groups/integration-allowlist' +import { getWorkspaceWithOwner } from '@/lib/workspaces/permissions/utils' + +/** + * Merges the env allowlist into a permission config. + * + * Returns null only when neither layer restricts anything. Otherwise the group's + * own allowlist is intersected with the env one by + * {@link intersectIntegrationAllowlists}, which canonicalizes both sides — case + * *and* successor — before intersecting. Both matter here: a stored config + * reaches this function straight off the wire, where the contract permits any + * casing, and the two layers are written independently, so one can name a + * retired id (`ALLOWED_INTEGRATIONS=slack`) while the other names its successor + * (`slack_v2`). Intersecting those textually yields the empty allowlist, which + * refuses an integration both layers allow. The result is therefore in the + * resolved vocabulary, and callers must judge a block type through + * `resolveAccessControlBlockType` before testing membership. + */ +export function mergeEnvAllowlist( + config: PermissionGroupConfig | null +): PermissionGroupConfig | null { + const envAllowlist = getAllowedIntegrationsFromEnv() + if (config === null && envAllowlist === null) return null + + const base = config ?? DEFAULT_PERMISSION_GROUP_CONFIG + return { + ...base, + allowedIntegrations: intersectIntegrationAllowlists(base.allowedIntegrations, envAllowlist), + } +} + +/** + * The permission group that governs a user in a given context, with its parsed + * config. Shared by the executor path and the `/api/permission-groups/user` + * route so resolution never drifts between the two. + */ +export interface ResolvedPermissionGroup { + permissionGroupId: string + groupName: string + resolution: 'explicit-member' | 'all-members' | 'default' + config: PermissionGroupConfig +} + +export interface UserAccessControlContext { + organizationId: string | null + entitled: boolean + permissionGroup: { + id: string + name: string + resolution: ResolvedPermissionGroup['resolution'] + } | null + config: PermissionGroupConfig | null +} + +function inactiveUserAccessControlContext(organizationId: string | null): UserAccessControlContext { + return { + organizationId, + entitled: false, + permissionGroup: null, + config: mergeEnvAllowlist(null), + } +} + +/** The organization's single default group (`isDefault`), or `null`. */ +async function resolveDefaultGroup( + organizationId: string +): Promise { + const [defaultGroup] = await db + .select({ + id: permissionGroup.id, + name: permissionGroup.name, + config: permissionGroup.config, + }) + .from(permissionGroup) + .where( + and(eq(permissionGroup.organizationId, organizationId), eq(permissionGroup.isDefault, true)) + ) + .limit(1) + + if (!defaultGroup) { + return null + } + + return { + permissionGroupId: defaultGroup.id, + groupName: defaultGroup.name, + resolution: 'default', + config: parsePermissionGroupConfig(defaultGroup.config), + } +} + +/** + * Resolve the group governing `userId` in `workspaceId` (which belongs to + * `organizationId`). One effective group per workspace, by precedence: + * 1. a non-default group targeting this workspace that `userId` is an explicit + * member of, else + * 2. a non-default group targeting this workspace that has no explicit members + * — governs all members of the workspace, including external members, else + * 3. the organization's default group (also governs external members), else + * 4. `null` (unrestricted). + * + * Assignment-time conflict checks keep this unambiguous: at most one all-members + * group per workspace, and a user is an explicit member of at most one group per + * workspace. If an overlap nonetheless exists, the oldest group wins — rows are + * ordered by `created_at` (then `id`). + * + * Callers gate on enterprise entitlement before invoking this and merge the env + * allowlist afterwards. + */ +export async function resolveWorkspaceGroup( + userId: string, + organizationId: string, + workspaceId: string +): Promise { + const rows = await db + .select({ + id: permissionGroup.id, + name: permissionGroup.name, + config: permissionGroup.config, + isMember: sql`exists ( + select 1 from ${permissionGroupMember} + where ${permissionGroupMember.permissionGroupId} = ${permissionGroup.id} + and ${permissionGroupMember.userId} = ${userId} + )`, + hasMembers: sql`exists ( + select 1 from ${permissionGroupMember} + where ${permissionGroupMember.permissionGroupId} = ${permissionGroup.id} + )`, + }) + .from(permissionGroup) + .innerJoin( + permissionGroupWorkspace, + and( + eq(permissionGroupWorkspace.permissionGroupId, permissionGroup.id), + eq(permissionGroupWorkspace.workspaceId, workspaceId) + ) + ) + .where( + and(eq(permissionGroup.organizationId, organizationId), eq(permissionGroup.isDefault, false)) + ) + .orderBy(asc(permissionGroup.createdAt), asc(permissionGroup.id)) + + const explicitMemberGroup = rows.find((row) => row.isMember) + const winner = explicitMemberGroup ?? rows.find((row) => !row.hasMembers) + + if (winner) { + return { + permissionGroupId: winner.id, + groupName: winner.name, + resolution: explicitMemberGroup ? 'explicit-member' : 'all-members', + config: parsePermissionGroupConfig(winner.config), + } + } + + return resolveDefaultGroup(organizationId) +} + +/** + * Resolve the effective permission-group config for a user in the context of a + * specific workspace. The workspace is mapped to its organization and the + * governing group is resolved with specific-over-all precedence. + * + * Returns `null` (after env merge) when the workspace has no organization, the + * organization isn't on an enterprise plan, or no group governs the user. + * + * The env-level integration allowlist is always merged last so self-hosted + * deployments can constrain integrations without touching the DB. + */ +async function resolveUserAccessControlContextForOrganization( + userId: string, + workspaceId: string, + organizationId: string | null +): Promise { + if (!organizationId) return inactiveUserAccessControlContext(null) + + /** + * `'throw'` because an unentitled organization resolves to `config: null`, + * and `null` is not a smaller permission set — it is *no* permission group at + * all: every capability allowed, every allowlist off. Under the lenient + * default a single subscription-read failure would be indistinguishable from + * a genuine plan lapse and would turn the whole regime off for the request. + * Throwing surfaces the outage as an error instead. + */ + const isEnterprise = await isOrganizationOnEnterprisePlan(organizationId, 'throw') + if (!isEnterprise) { + return inactiveUserAccessControlContext(organizationId) + } + + const resolved = await resolveWorkspaceGroup(userId, organizationId, workspaceId) + return { + organizationId, + entitled: true, + permissionGroup: resolved + ? { + id: resolved.permissionGroupId, + name: resolved.groupName, + resolution: resolved.resolution, + } + : null, + config: mergeEnvAllowlist(resolved?.config ?? null), + } +} + +/** + * Resolves Access Control from an organization ID obtained from an already + * access-checked workspace. This function does not independently authorize the + * user for the workspace; callers must establish that boundary first. + */ +export async function resolveVerifiedUserAccessControlContext( + userId: string, + workspaceId: string, + organizationId: string | null +): Promise { + if (!isHosted && !isAccessControlEnabled) { + return inactiveUserAccessControlContext(null) + } + return resolveUserAccessControlContextForOrganization(userId, workspaceId, organizationId) +} + +/** + * The unverified counterpart of {@link resolveVerifiedUserAccessControlContext}: + * it loads the workspace itself to learn the owning organization. + * + * For the callers that have not already access-checked the workspace — a raw + * route, typically. Everything else holds the organization id already and + * should pass it, rather than paying for a second lookup of a value it has. + */ +export async function getUserPermissionConfig( + userId: string, + workspaceId: string +): Promise { + if (!isHosted && !isAccessControlEnabled) { + return mergeEnvAllowlist(null) + } + + const workspace = await getWorkspaceWithOwner(workspaceId, { includeArchived: true }) + const context = await resolveUserAccessControlContextForOrganization( + userId, + workspaceId, + workspace?.organizationId ?? null + ) + return context.config +} + +/** + * Org-addressed variant of {@link getUserPermissionConfig}. Use when only the + * organization is known (e.g. organization-level invitations). Non-default + * groups target specific workspaces and never gate organization-level actions, + * so this resolves the organization's default group — which governs everyone not + * covered by a workspace group. + */ +export async function getUserPermissionConfigForOrganization( + organizationId: string +): Promise { + if (!isHosted && !isAccessControlEnabled) { + return mergeEnvAllowlist(null) + } + + /** `'throw'` for the same reason as in {@link resolveUserAccessControlContextForOrganization}. */ + const isEnterprise = await isOrganizationOnEnterprisePlan(organizationId, 'throw') + if (!isEnterprise) { + return mergeEnvAllowlist(null) + } + + const resolved = await resolveDefaultGroup(organizationId) + return mergeEnvAllowlist(resolved?.config ?? null) +} diff --git a/apps/sim/lib/permission-groups/types.ts b/apps/sim/lib/permission-groups/types.ts deleted file mode 100644 index 15bdb9773f9..00000000000 --- a/apps/sim/lib/permission-groups/types.ts +++ /dev/null @@ -1,159 +0,0 @@ -import { z } from 'zod' -import type { ShareAuthType } from '@/lib/api/contracts/public-shares' - -/** - * Auth modes a public file share or a chat deployment can use; admins may - * restrict the allowed subset. The two surfaces share the same four modes. - */ -export const FILE_SHARE_AUTH_TYPES = ['public', 'password', 'email', 'sso'] as const - -export const PERMISSION_GROUP_CONSTRAINTS = { - organizationName: 'permission_group_organization_name_unique', - organizationDefault: 'permission_group_organization_default_unique', -} as const - -export const PERMISSION_GROUP_MEMBER_CONSTRAINTS = { - groupUser: 'permission_group_member_group_user_unique', -} as const - -export const permissionGroupConfigSchema = z.object({ - allowedIntegrations: z.array(z.string()).nullable().optional(), - allowedModelProviders: z.array(z.string()).nullable().optional(), - deniedModels: z.array(z.string()).optional(), - deniedTools: z.array(z.string()).optional(), - hideTraceSpans: z.boolean().optional(), - hideKnowledgeBaseTab: z.boolean().optional(), - hideTablesTab: z.boolean().optional(), - hideCopilot: z.boolean().optional(), - hideIntegrationsTab: z.boolean().optional(), - hideSecretsTab: z.boolean().optional(), - hideApiKeysTab: z.boolean().optional(), - hideInboxTab: z.boolean().optional(), - hideFilesTab: z.boolean().optional(), - disableMcpTools: z.boolean().optional(), - disableCustomTools: z.boolean().optional(), - disableSkills: z.boolean().optional(), - disableInvitations: z.boolean().optional(), - disablePublicApi: z.boolean().optional(), - disablePublicFileSharing: z.boolean().optional(), - allowedFileShareAuthTypes: z.array(z.enum(FILE_SHARE_AUTH_TYPES)).nullable().optional(), - hideDeployApi: z.boolean().optional(), - hideDeployMcp: z.boolean().optional(), - hideDeployChatbot: z.boolean().optional(), - allowedChatDeployAuthTypes: z.array(z.enum(FILE_SHARE_AUTH_TYPES)).nullable().optional(), -}) - -export interface PermissionGroupConfig { - allowedIntegrations: string[] | null - allowedModelProviders: string[] | null - /** - * Fully-qualified model IDs (e.g. `ollama/llama3`, `gpt-4o`) blocked for this - * group, checked after `allowedModelProviders`. Empty means nothing is blocked. - */ - deniedModels: string[] - /** - * Snake_case tool IDs (e.g. `slack_canvas`) blocked for this group, checked - * after the block-level `allowedIntegrations` gate. Lets an admin allow an - * integration but deny specific operations within it. Empty means nothing is - * blocked. - */ - deniedTools: string[] - hideTraceSpans: boolean - hideKnowledgeBaseTab: boolean - hideTablesTab: boolean - hideCopilot: boolean - hideIntegrationsTab: boolean - hideSecretsTab: boolean - hideApiKeysTab: boolean - hideInboxTab: boolean - hideFilesTab: boolean - disableMcpTools: boolean - disableCustomTools: boolean - disableSkills: boolean - disableInvitations: boolean - disablePublicApi: boolean - disablePublicFileSharing: boolean - /** Allowed public-file-share auth modes; `null` means all are allowed. */ - allowedFileShareAuthTypes: ShareAuthType[] | null - hideDeployApi: boolean - hideDeployMcp: boolean - hideDeployChatbot: boolean - /** Allowed chat-deployment auth modes; `null` means all are allowed. */ - allowedChatDeployAuthTypes: ShareAuthType[] | null -} - -export const DEFAULT_PERMISSION_GROUP_CONFIG: PermissionGroupConfig = { - allowedIntegrations: null, - allowedModelProviders: null, - deniedModels: [], - deniedTools: [], - hideTraceSpans: false, - hideKnowledgeBaseTab: false, - hideTablesTab: false, - hideCopilot: false, - hideIntegrationsTab: false, - hideSecretsTab: false, - hideApiKeysTab: false, - hideInboxTab: false, - hideFilesTab: false, - disableMcpTools: false, - disableCustomTools: false, - disableSkills: false, - disableInvitations: false, - disablePublicApi: false, - disablePublicFileSharing: false, - allowedFileShareAuthTypes: null, - hideDeployApi: false, - hideDeployMcp: false, - hideDeployChatbot: false, - allowedChatDeployAuthTypes: null, -} - -export function parsePermissionGroupConfig(config: unknown): PermissionGroupConfig { - if (!config || typeof config !== 'object') { - return DEFAULT_PERMISSION_GROUP_CONFIG - } - - const c = config as Record - - return { - allowedIntegrations: Array.isArray(c.allowedIntegrations) ? c.allowedIntegrations : null, - allowedModelProviders: Array.isArray(c.allowedModelProviders) ? c.allowedModelProviders : null, - deniedModels: Array.isArray(c.deniedModels) - ? c.deniedModels.filter((m): m is string => typeof m === 'string') - : [], - deniedTools: Array.isArray(c.deniedTools) - ? c.deniedTools.filter((t): t is string => typeof t === 'string') - : [], - hideTraceSpans: typeof c.hideTraceSpans === 'boolean' ? c.hideTraceSpans : false, - hideKnowledgeBaseTab: - typeof c.hideKnowledgeBaseTab === 'boolean' ? c.hideKnowledgeBaseTab : false, - hideTablesTab: typeof c.hideTablesTab === 'boolean' ? c.hideTablesTab : false, - hideCopilot: typeof c.hideCopilot === 'boolean' ? c.hideCopilot : false, - hideIntegrationsTab: typeof c.hideIntegrationsTab === 'boolean' ? c.hideIntegrationsTab : false, - hideSecretsTab: typeof c.hideSecretsTab === 'boolean' ? c.hideSecretsTab : false, - hideApiKeysTab: typeof c.hideApiKeysTab === 'boolean' ? c.hideApiKeysTab : false, - hideInboxTab: typeof c.hideInboxTab === 'boolean' ? c.hideInboxTab : false, - hideFilesTab: typeof c.hideFilesTab === 'boolean' ? c.hideFilesTab : false, - disableMcpTools: typeof c.disableMcpTools === 'boolean' ? c.disableMcpTools : false, - disableCustomTools: typeof c.disableCustomTools === 'boolean' ? c.disableCustomTools : false, - disableSkills: typeof c.disableSkills === 'boolean' ? c.disableSkills : false, - disableInvitations: typeof c.disableInvitations === 'boolean' ? c.disableInvitations : false, - disablePublicApi: typeof c.disablePublicApi === 'boolean' ? c.disablePublicApi : false, - disablePublicFileSharing: - typeof c.disablePublicFileSharing === 'boolean' ? c.disablePublicFileSharing : false, - allowedFileShareAuthTypes: Array.isArray(c.allowedFileShareAuthTypes) - ? c.allowedFileShareAuthTypes.filter((t): t is ShareAuthType => - (FILE_SHARE_AUTH_TYPES as readonly string[]).includes(t as string) - ) - : null, - hideDeployApi: typeof c.hideDeployApi === 'boolean' ? c.hideDeployApi : false, - hideDeployMcp: typeof c.hideDeployMcp === 'boolean' ? c.hideDeployMcp : false, - hideDeployChatbot: typeof c.hideDeployChatbot === 'boolean' ? c.hideDeployChatbot : false, - allowedChatDeployAuthTypes: Array.isArray(c.allowedChatDeployAuthTypes) - ? c.allowedChatDeployAuthTypes.filter((t): t is ShareAuthType => - (FILE_SHARE_AUTH_TYPES as readonly string[]).includes(t as string) - ) - : null, - } -} diff --git a/apps/sim/lib/permission-groups/user-scope.server.ts b/apps/sim/lib/permission-groups/user-scope.server.ts new file mode 100644 index 00000000000..e160ec1076a --- /dev/null +++ b/apps/sim/lib/permission-groups/user-scope.server.ts @@ -0,0 +1,37 @@ +import { getUserOrganization } from '@/lib/billing/organizations/membership' +import type { StaticPermissionGroupCapability } from '@/lib/permission-groups/capabilities' +import { + isOrganizationCapabilityWithheld, + isWorkspaceCapabilityWithheld, +} from '@/lib/permission-groups/capability-assertions' + +/** + * Whether the group governing `userId` withholds `capability`, for an action + * that may or may not name a workspace. + * + * A workspace-scoped action is governed by the group targeting that workspace. + * A user-global one — a personal API key, a CLI login with no workspace — falls + * back to the organization's default group rather than going ungoverned, which + * would leave the narrower scope as the unguarded one. + * + * Shared so that fallback cannot drift between the surfaces that mint the same + * credential: `/api/users/me/api-keys`, `/api/cli/auth/approve`. It restates no + * capability of its own — each caller names the one it enforces, and carries + * the `permission-group-enforced:` annotation for it. + * + * Not in `capability-assertions.ts` on purpose: reading the caller's + * organization membership reaches the billing graph, and that module is a + * guarded root of `check:application-graph` precisely so the authorization + * funnel never loads it. + */ +export async function isCapabilityWithheldForUser( + userId: string, + capability: StaticPermissionGroupCapability, + workspaceId?: string +): Promise { + if (workspaceId) return isWorkspaceCapabilityWithheld(userId, workspaceId, capability) + + const membership = await getUserOrganization(userId) + if (!membership?.organizationId) return false + return isOrganizationCapabilityWithheld(membership.organizationId, capability) +} diff --git a/apps/sim/lib/platform-context/application/operations.ts b/apps/sim/lib/platform-context/application/operations.ts index 36c650a064f..80658e1acc9 100644 --- a/apps/sim/lib/platform-context/application/operations.ts +++ b/apps/sim/lib/platform-context/application/operations.ts @@ -5,17 +5,28 @@ const LIVE_PLATFORM_CONTEXT_PRINCIPAL_POLICY = { delegatedServices: ['copilot'], } as const +/** + * What Sim reads about itself before it can answer at all: the workspace's plan, + * its seat and usage state, and whether the organization is on the enterprise + * tier. No permission-group key names them, and a member whose group withheld + * them would get an agent that cannot tell them why anything is unavailable — + * withholding the description of a restriction is not the same as applying one. + */ export const platformContextOperations = { + // permission-group-exempt: the plan and usage state every answer is framed against; withholding it blanks the agent rather than restricting it readAccountBilling: defineWorkspaceOperation({ id: 'platform_context.account_billing.read', minimumRole: 'read', workspaceApiKey: 'deny', + capability: 'none', ...LIVE_PLATFORM_CONTEXT_PRINCIPAL_POLICY, }), + // permission-group-exempt: reports which enterprise features the organization has, the frame the restrictions themselves are described in readEnterpriseContext: defineWorkspaceOperation({ id: 'platform_context.enterprise.read', minimumRole: 'read', workspaceApiKey: 'deny', + capability: 'none', ...LIVE_PLATFORM_CONTEXT_PRINCIPAL_POLICY, }), } as const diff --git a/apps/sim/lib/platform-context/application/platform-context-use-cases.test.ts b/apps/sim/lib/platform-context/application/platform-context-use-cases.test.ts index 64319782d72..44976130c67 100644 --- a/apps/sim/lib/platform-context/application/platform-context-use-cases.test.ts +++ b/apps/sim/lib/platform-context/application/platform-context-use-cases.test.ts @@ -34,11 +34,11 @@ vi.mock('@/lib/workspaces/host-context', () => ({ getWorkspaceHostContextForViewer: mocks.getWorkspaceHostContextForViewer, })) -vi.mock('@/ee/access-control/utils/permission-check', () => ({ +vi.mock('@/lib/permission-groups/resolve.server', () => ({ resolveVerifiedUserAccessControlContext: mocks.resolveVerifiedUserAccessControlContext, })) -import { DEFAULT_PERMISSION_GROUP_CONFIG } from '@/lib/permission-groups/types' +import { DEFAULT_PERMISSION_GROUP_CONFIG } from '@/lib/permission-groups/fields' import { readAccountBilling } from '@/lib/platform-context/application/read-account-billing' import { readEnterpriseContext } from '@/lib/platform-context/application/read-enterprise-context' diff --git a/apps/sim/lib/platform-context/application/read-enterprise-context.ts b/apps/sim/lib/platform-context/application/read-enterprise-context.ts index dab02b17e5d..cbca10f2ca6 100644 --- a/apps/sim/lib/platform-context/application/read-enterprise-context.ts +++ b/apps/sim/lib/platform-context/application/read-enterprise-context.ts @@ -2,6 +2,7 @@ import { requirePrincipalSubjectUserId } from '@sim/auth/principal' import { permissionSatisfies } from '@sim/platform-authz/workspace' import { defineAuthorizedWorkspaceUseCase } from '@/lib/core/application' import { OrchestrationError } from '@/lib/core/orchestration/types' +import { capabilityDeniedBy } from '@/lib/permission-groups/capability-assertions' import { getActivePermissionGroupRestrictions } from '@/lib/permission-groups/features' import { platformContextDelegationPolicy } from '@/lib/platform-context/application/authorization' import { resolvePlatformContextWorkspace } from '@/lib/platform-context/application/context' @@ -46,9 +47,9 @@ export const readEnterpriseContext = defineAuthorizedWorkspaceUseCase({ const canWrite = permissionSatisfies(hostContext.viewer.permission, 'write') const canAdmin = permissionSatisfies(hostContext.viewer.permission, 'admin') const allDeploymentSurfacesHidden = - accessControl.config?.hideDeployApi === true && - accessControl.config.hideDeployMcp === true && - accessControl.config.hideDeployChatbot === true + capabilityDeniedBy('deploy.api', accessControl.config) && + capabilityDeniedBy('deploy.mcp', accessControl.config) && + capabilityDeniedBy('deploy.chat', accessControl.config) return { workspace: { diff --git a/apps/sim/lib/pptx-renderer/renderer/chart-renderer.ts b/apps/sim/lib/pptx-renderer/renderer/chart-renderer.ts index c0b4a7dae31..c59a8fdaefa 100644 --- a/apps/sim/lib/pptx-renderer/renderer/chart-renderer.ts +++ b/apps/sim/lib/pptx-renderer/renderer/chart-renderer.ts @@ -6,7 +6,8 @@ const logger = createLogger('PptxChartRenderer') * Chart renderer — converts OOXML chart XML into ECharts visualizations. */ -import * as echarts from 'echarts' +import type * as echarts from 'echarts' +import { format, graphic, init } from '@/lib/pptx-renderer/renderer/echarts-runtime' import type { ChartNodeData } from '../model/nodes/chart-node' import type { SafeXmlNode } from '../parser/xml-parser' import { cssFontStack } from '../utils/font-stack' @@ -402,7 +403,7 @@ function buildEChartsGradient(gradFill: SafeXmlNode, ctx: RenderContext): object const x1 = 0.5 + 0.5 * Math.cos(rad) const y1 = 0.5 + 0.5 * Math.sin(rad) - return new echarts.graphic.LinearGradient(x0, y0, x1, y1, stops) + return new graphic.LinearGradient(x0, y0, x1, y1, stops) } /** @@ -2065,7 +2066,7 @@ function buildBubbleChartOption( // ECharts escapes the markup it builds itself; a hand-built one escapes its own. formatter: (params: unknown) => { const p = params as { seriesName: string; value: number[] } - const name = echarts.format.encodeHTML(p.seriesName) + const name = format.encodeHTML(p.seriesName) return `${name}
x: ${p.value[0]}, y: ${p.value[1]}, size: ${p.value[2]}` }, }, @@ -3360,7 +3361,7 @@ function initChart( chartInstances?: Set ): void { try { - const chart = echarts.init(container) + const chart = init(container) chart.setOption(option) chartInstances?.add(chart) diff --git a/apps/sim/lib/pptx-renderer/renderer/echarts-runtime.smoke.test.ts b/apps/sim/lib/pptx-renderer/renderer/echarts-runtime.smoke.test.ts new file mode 100644 index 00000000000..dc76f3d8889 --- /dev/null +++ b/apps/sim/lib/pptx-renderer/renderer/echarts-runtime.smoke.test.ts @@ -0,0 +1,41 @@ +/** + * @vitest-environment node + */ +import { use } from 'echarts/core' +import { SVGRenderer } from 'echarts/renderers' +import { describe, expect, it } from 'vitest' +import { init } from '@/lib/pptx-renderer/renderer/echarts-runtime' + +use([SVGRenderer]) + +describe('PPTX ECharts runtime smoke', () => { + it('renders representative registered charts and components', () => { + const chart = init(null, null, { + renderer: 'svg', + ssr: true, + width: 400, + height: 300, + }) + + try { + chart.setOption({ + title: { text: 'Quarterly results' }, + tooltip: {}, + legend: {}, + xAxis: { type: 'category', data: ['Q1', 'Q2'] }, + yAxis: { type: 'value' }, + series: [ + { name: 'Revenue', type: 'bar', data: [12, 18] }, + { name: 'Target', type: 'line', data: [10, 16] }, + ], + }) + + const svg = chart.renderToSVGString() + expect(svg).toContain(' { + const modules = { + barChart: Symbol('BarChart'), + candlestickChart: Symbol('CandlestickChart'), + customChart: Symbol('CustomChart'), + lineChart: Symbol('LineChart'), + pieChart: Symbol('PieChart'), + radarChart: Symbol('RadarChart'), + scatterChart: Symbol('ScatterChart'), + axisPointerComponent: Symbol('AxisPointerComponent'), + gridComponent: Symbol('GridComponent'), + legendComponent: Symbol('LegendComponent'), + radarComponent: Symbol('RadarComponent'), + titleComponent: Symbol('TitleComponent'), + tooltipComponent: Symbol('TooltipComponent'), + labelLayout: Symbol('LabelLayout'), + legacyGridContainLabel: Symbol('LegacyGridContainLabel'), + canvasRenderer: Symbol('CanvasRenderer'), + } + + return { + format: { encodeHTML: vi.fn() }, + graphic: { LinearGradient: vi.fn() }, + init: vi.fn(), + modules, + use: vi.fn(), + } +}) + +vi.mock('echarts/charts', () => ({ + BarChart: mocks.modules.barChart, + CandlestickChart: mocks.modules.candlestickChart, + CustomChart: mocks.modules.customChart, + LineChart: mocks.modules.lineChart, + PieChart: mocks.modules.pieChart, + RadarChart: mocks.modules.radarChart, + ScatterChart: mocks.modules.scatterChart, +})) + +vi.mock('echarts/components', () => ({ + AxisPointerComponent: mocks.modules.axisPointerComponent, + GridComponent: mocks.modules.gridComponent, + LegendComponent: mocks.modules.legendComponent, + RadarComponent: mocks.modules.radarComponent, + TitleComponent: mocks.modules.titleComponent, + TooltipComponent: mocks.modules.tooltipComponent, +})) + +vi.mock('echarts/core', () => ({ + format: mocks.format, + graphic: mocks.graphic, + init: mocks.init, + use: mocks.use, +})) + +vi.mock('echarts/features', () => ({ + LabelLayout: mocks.modules.labelLayout, + LegacyGridContainLabel: mocks.modules.legacyGridContainLabel, +})) + +vi.mock('echarts/renderers', () => ({ + CanvasRenderer: mocks.modules.canvasRenderer, +})) + +import { format, graphic, init } from '@/lib/pptx-renderer/renderer/echarts-runtime' + +describe('PPTX ECharts runtime', () => { + it('registers only the chart modules used by PPTX rendering', () => { + expect(mocks.use).toHaveBeenCalledOnce() + expect(mocks.use).toHaveBeenCalledWith([ + mocks.modules.barChart, + mocks.modules.candlestickChart, + mocks.modules.customChart, + mocks.modules.lineChart, + mocks.modules.pieChart, + mocks.modules.radarChart, + mocks.modules.scatterChart, + mocks.modules.axisPointerComponent, + mocks.modules.gridComponent, + mocks.modules.legendComponent, + mocks.modules.radarComponent, + mocks.modules.titleComponent, + mocks.modules.tooltipComponent, + mocks.modules.labelLayout, + mocks.modules.legacyGridContainLabel, + mocks.modules.canvasRenderer, + ]) + }) + + it('exposes the core helpers consumed by the chart renderer', () => { + expect(format).toBe(mocks.format) + expect(graphic).toBe(mocks.graphic) + expect(init).toBe(mocks.init) + }) +}) diff --git a/apps/sim/lib/pptx-renderer/renderer/echarts-runtime.ts b/apps/sim/lib/pptx-renderer/renderer/echarts-runtime.ts new file mode 100644 index 00000000000..e5f86f2453c --- /dev/null +++ b/apps/sim/lib/pptx-renderer/renderer/echarts-runtime.ts @@ -0,0 +1,41 @@ +import { + BarChart, + CandlestickChart, + CustomChart, + LineChart, + PieChart, + RadarChart, + ScatterChart, +} from 'echarts/charts' +import { + AxisPointerComponent, + GridComponent, + LegendComponent, + RadarComponent, + TitleComponent, + TooltipComponent, +} from 'echarts/components' +import { format, graphic, init, use } from 'echarts/core' +import { LabelLayout, LegacyGridContainLabel } from 'echarts/features' +import { CanvasRenderer } from 'echarts/renderers' + +use([ + BarChart, + CandlestickChart, + CustomChart, + LineChart, + PieChart, + RadarChart, + ScatterChart, + AxisPointerComponent, + GridComponent, + LegendComponent, + RadarComponent, + TitleComponent, + TooltipComponent, + LabelLayout, + LegacyGridContainLabel, + CanvasRenderer, +]) + +export { format, graphic, init } diff --git a/apps/sim/lib/realtime/notify.ts b/apps/sim/lib/realtime/notify.ts index f1d9846919b..2374a8d5580 100644 --- a/apps/sim/lib/realtime/notify.ts +++ b/apps/sim/lib/realtime/notify.ts @@ -18,33 +18,30 @@ const NOTIFY_TIMEOUT_MS = 2000 const APPLY_EDIT_TIMEOUT_MS = FILE_DOC_TIMEOUTS.applyEditMs /** - * Best-effort fan-out to the realtime server that a workspace's file tree changed, - * so every browser currently viewing that workspace's files refetches. File - * mutations happen over the HTTP API (not the socket); this is a lossy liveness - * signal — a dropped notification only degrades to stale-until-refetch. - * - * Never throws. Callers `await` it (rather than fire-and-forget) so the fetch is - * guaranteed to dispatch before a Node route handler returns — a floating promise - * can be dropped after the response is sent. It is a normally-sub-millisecond - * local call and is hard-bounded to {@link NOTIFY_TIMEOUT_MS}, so it adds that + * POST one workspace list-changed signal (`/api/workspace--changed`) to the realtime server, + * which fans it out to every socket in that workspace's live-list room so their browser refetches. + * Lossy — a dropped notification only degrades to stale-until-refetch. Never throws. Callers + * `await` it (rather than fire-and-forget) so the fetch is guaranteed to dispatch before a Node + * route handler returns — a floating promise can be dropped after the response is sent. It is a + * normally-sub-millisecond local call, hard-bounded to {@link NOTIFY_TIMEOUT_MS}, so it adds that * latency only when the socket pod is unreachable. */ -export async function notifyWorkspaceFilesChanged(workspaceId: string): Promise { +async function postWorkspaceListChanged(endpoint: string, workspaceId: string): Promise { try { - const response = await fetch(`${getSocketServerUrl()}/api/workspace-files-changed`, { + const response = await fetch(`${getSocketServerUrl()}/api/${endpoint}`, { method: 'POST', headers: { 'Content-Type': 'application/json', 'x-api-key': env.INTERNAL_API_SECRET }, body: JSON.stringify({ workspaceId }), signal: AbortSignal.timeout(NOTIFY_TIMEOUT_MS), }) if (!response.ok) { - logger.warn('workspace-files-changed notify failed', { + logger.warn(`${endpoint} notify failed`, { workspaceId, status: response.status, }) } } catch (error) { - logger.warn('workspace-files-changed notify error', { + logger.warn(`${endpoint} notify error`, { workspaceId, error: getErrorMessage(error), }) @@ -52,36 +49,34 @@ export async function notifyWorkspaceFilesChanged(workspaceId: string): Promise< } /** - * Best-effort fan-out to the realtime server that a workspace's table list changed (a table was - * created, renamed, moved, deleted, or restored), so every browser currently viewing that - * workspace's tables refetches. The list-level counterpart to {@link notifyWorkspaceFilesChanged}; - * table mutations happen server-side (HTTP routes AND copilot), so this fires from the shared table - * service, not a socket. Lossy — a dropped notification only degrades to stale-until-refetch. - * - * Never throws. Callers `await` it so the fetch is guaranteed to dispatch before the mutation - * returns; hard-bounded to {@link NOTIFY_TIMEOUT_MS}, so it adds that latency only when the socket - * pod is unreachable. + * Best-effort fan-out that a workspace's file tree changed, so every viewer of that workspace's + * files refetches. See {@link postWorkspaceListChanged} for the shared lossy/never-throws contract. */ -export async function notifyWorkspaceTablesChanged(workspaceId: string): Promise { - try { - const response = await fetch(`${getSocketServerUrl()}/api/workspace-tables-changed`, { - method: 'POST', - headers: { 'Content-Type': 'application/json', 'x-api-key': env.INTERNAL_API_SECRET }, - body: JSON.stringify({ workspaceId }), - signal: AbortSignal.timeout(NOTIFY_TIMEOUT_MS), - }) - if (!response.ok) { - logger.warn('workspace-tables-changed notify failed', { - workspaceId, - status: response.status, - }) - } - } catch (error) { - logger.warn('workspace-tables-changed notify error', { - workspaceId, - error: getErrorMessage(error), - }) - } +export function notifyWorkspaceFilesChanged(workspaceId: string): Promise { + return postWorkspaceListChanged('workspace-files-changed', workspaceId) +} + +/** + * Best-effort fan-out that a workspace's table list changed (a table was created, renamed, moved, + * deleted, or restored), so every viewer of that workspace's tables refetches. Fires from the + * shared table service, so it covers every surface (HTTP routes AND copilot). See + * {@link postWorkspaceListChanged} for the shared lossy/never-throws contract. + */ +export function notifyWorkspaceTablesChanged(workspaceId: string): Promise { + return postWorkspaceListChanged('workspace-tables-changed', workspaceId) +} + +/** + * Best-effort fan-out that a workspace's workflow registry changed (a workflow was created, + * renamed, moved, deleted, duplicated, imported, restored, or reordered, or a workflow folder + * changed), so every viewer's sidebar workflow list refetches. The list-level counterpart to the + * per-workflow editor notifications ({@link notifyWorkflowUpdated}): those only reach sockets with + * that workflow's canvas open, while this reaches everyone in the workspace. Fires from the + * workflow application use cases, so it covers every surface (UI, CLI, copilot, API). See + * {@link postWorkspaceListChanged} for the shared lossy/never-throws contract. + */ +export function notifyWorkspaceWorkflowsChanged(workspaceId: string): Promise { + return postWorkspaceListChanged('workspace-workflows-changed', workspaceId) } /** Best-effort fan-out that invalidates open editors for one durably changed workflow. */ @@ -149,12 +144,13 @@ export async function notifyWorkflowReverted(workflowId: string, timestamp: numb * (create/rename/move/delete/restore) for one of these must fan out the same list-changed signal as a * direct resource mutation, because a new/renamed/removed folder changes what that resource's browser * shows. Extend this map as more resource lists adopt an invalidation room — `file` and - * `knowledge_base` currently refetch through their own paths, and `workflow` has no such list room. + * `knowledge_base` currently refetch through their own paths. */ const FOLDER_RESOURCE_NOTIFIERS: Partial< Record Promise> > = { table: notifyWorkspaceTablesChanged, + workflow: notifyWorkspaceWorkflowsChanged, } /** diff --git a/apps/sim/lib/secrets/application/operations.ts b/apps/sim/lib/secrets/application/operations.ts index 00f408fd84e..b629a1014dd 100644 --- a/apps/sim/lib/secrets/application/operations.ts +++ b/apps/sim/lib/secrets/application/operations.ts @@ -7,18 +7,21 @@ export const secretOperations = { id: 'secrets.list', minimumRole: 'read', workspaceApiKey: 'deny', + capability: 'secrets.manage', principalKinds: HUMAN_API_PRINCIPAL_KINDS, }), set: defineWorkspaceOperation({ id: 'secrets.set', minimumRole: 'read', workspaceApiKey: 'deny', + capability: 'secrets.manage', principalKinds: HUMAN_API_PRINCIPAL_KINDS, }), delete: defineWorkspaceOperation({ id: 'secrets.delete', minimumRole: 'read', workspaceApiKey: 'deny', + capability: 'secrets.manage', principalKinds: HUMAN_API_PRINCIPAL_KINDS, }), /** @@ -29,6 +32,7 @@ export const secretOperations = { id: 'secrets.usage', minimumRole: 'read', workspaceApiKey: 'deny', + capability: 'secrets.manage', principalKinds: HUMAN_API_PRINCIPAL_KINDS, }), /** @@ -40,6 +44,7 @@ export const secretOperations = { id: 'secrets.references', minimumRole: 'read', workspaceApiKey: 'deny', + capability: 'secrets.manage', principalKinds: HUMAN_API_PRINCIPAL_KINDS, }), } as const diff --git a/apps/sim/lib/selectors/application/execute-selector.test.ts b/apps/sim/lib/selectors/application/execute-selector.test.ts new file mode 100644 index 00000000000..60f75064fa7 --- /dev/null +++ b/apps/sim/lib/selectors/application/execute-selector.test.ts @@ -0,0 +1,688 @@ +/** + * @vitest-environment node + */ +import { permissionGroupScopeMock, permissionGroupScopeMockFns } from '@sim/testing' +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const mocks = vi.hoisted(() => ({ + events: [] as string[], + authorizeCredential: vi.fn(), + executeAttachment: vi.fn(), + getAttachment: vi.fn(), + logger: { + info: vi.fn(), + warn: vi.fn(), + error: vi.fn(), + debug: vi.fn(), + }, + recordCredentialAccess: vi.fn(), + resolvePermission: vi.fn(), + resolveReferences: vi.fn(), + resolveScope: vi.fn(), + sanitize: vi.fn(), +})) + +vi.mock('@sim/audit', () => ({ recordAudit: vi.fn() })) + +vi.mock('@sim/logger', () => ({ + createLogger: vi.fn(() => mocks.logger), +})) + +vi.mock('@sim/platform-authz/workspace', () => ({ + permissionSatisfies: (actual: string | null, required: string) => { + const rank = { read: 1, write: 2, admin: 3 } as const + return ( + actual !== null && rank[actual as keyof typeof rank] >= rank[required as keyof typeof rank] + ) + }, + resolveEffectiveWorkspacePermission: mocks.resolvePermission, +})) + +vi.mock('@/lib/selectors/application/resolve-scope', () => ({ + resolveSelectorApplicationContext: mocks.resolveScope, +})) + +vi.mock('@/lib/oauth/token-resolution', () => ({ + recordCredentialAccess: mocks.recordCredentialAccess, +})) + +vi.mock('@/lib/selectors/server/credentials', () => ({ + authorizeSelectorCredential: mocks.authorizeCredential, +})) + +vi.mock('@/lib/selectors/server/references', () => ({ + resolveSelectorReferences: mocks.resolveReferences, +})) + +vi.mock('@/lib/selectors/server/registry', () => ({ + getServerSelectorAttachment: mocks.getAttachment, +})) + +vi.mock('@/lib/selectors/server/sanitize', () => ({ + sanitizeSelectorResult: mocks.sanitize, +})) + +vi.mock('@/lib/permission-groups/config-scope.server', () => permissionGroupScopeMock) + +const mockResolvePermissionGroupConfig = + permissionGroupScopeMockFns.mockResolvePermissionGroupConfig + +import { selectorScopeSchema } from '@/lib/api/contracts/selectors/execute' +import { DEFAULT_PERMISSION_GROUP_CONFIG } from '@/lib/permission-groups/fields' +import { executeSelector } from '@/lib/selectors/application/execute-selector' +import { getSelectorManifestEntry } from '@/lib/selectors/manifest' +import { + SelectorContextUnavailableError, + SelectorOptionsUnavailableError, +} from '@/lib/selectors/server/errors' +import type { ExecuteServerSelectorArgs } from '@/lib/selectors/server/types' +import { IntegrationNotAllowedError } from '@/ee/access-control/utils/permission-check' + +const principal = { kind: 'session' as const, userId: 'user-1', sessionId: 'session-1' } +const scope = { kind: 'workspace' as const, workspaceId: 'workspace-1' } + +function execute(inputOverrides: Record = {}) { + return executeSelector.execute({ + principal, + input: { + selectorKey: 'gmail.labels', + scope, + context: { oauthCredential: '{{GMAIL_CREDENTIAL_ID}}' }, + request: { kind: 'list' as const }, + ...inputOverrides, + }, + }) +} + +describe('executeSelector', () => { + beforeEach(() => { + vi.clearAllMocks() + mocks.events.length = 0 + mocks.resolveScope.mockImplementation(async () => { + mocks.events.push('canonical-scope') + return { + workspaceId: 'workspace-1', + workspaceOrganizationId: null, + allowPersonalApiKeys: true, + selectorKey: 'gmail.labels', + selectorManifest: getSelectorManifestEntry('gmail.labels'), + selectorScope: scope, + } + }) + mocks.resolvePermission.mockImplementation(async () => { + mocks.events.push('workspace-authorization') + return 'read' + }) + mocks.resolveReferences.mockImplementation(async () => { + mocks.events.push('reference-resolution') + return { + context: { oauthCredential: 'credential-1' }, + request: { kind: 'list' }, + references: new Map(), + } + }) + mocks.authorizeCredential.mockImplementation(async () => { + mocks.events.push('credential-authorization') + return { suppliedId: 'credential-1' } + }) + mocks.executeAttachment.mockImplementation(async () => { + mocks.events.push('provider-execution') + return { kind: 'list', items: [{ id: 'label-1', label: 'Inbox' }] } + }) + mocks.getAttachment.mockReturnValue({ + destination: 'fixed', + credential: { kind: 'stored', field: 'oauthCredential', serviceIds: ['gmail'] }, + execute: mocks.executeAttachment, + }) + mocks.sanitize.mockImplementation((result) => { + mocks.events.push('sanitization') + return result + }) + mockResolvePermissionGroupConfig.mockResolvedValue(null) + }) + + it('authorizes canonical scope before references, credentials, and provider execution', async () => { + await expect(execute()).resolves.toEqual({ + kind: 'list', + items: [{ id: 'label-1', label: 'Inbox' }], + }) + + expect(mocks.events).toEqual([ + 'canonical-scope', + 'workspace-authorization', + 'reference-resolution', + 'credential-authorization', + 'provider-execution', + 'sanitization', + ]) + }) + + /** + * The picker is a use of the integration, not a neutral list: it reaches the + * provider's API with the caller's credential. The authorization funnel never + * sees which integration a selector key stands for, so the allowlist decision + * is asserted from the use case instead. + */ + it('refuses a selector whose integration the permission group excludes', async () => { + mockResolvePermissionGroupConfig.mockResolvedValue({ + ...DEFAULT_PERMISSION_GROUP_CONFIG, + allowedIntegrations: ['slack_v2'], + }) + + await expect(execute()).rejects.toBeInstanceOf(IntegrationNotAllowedError) + + expect(mocks.events).toEqual([ + 'canonical-scope', + 'workspace-authorization', + 'reference-resolution', + 'credential-authorization', + ]) + }) + + it('executes a selector whose integration the permission group names', async () => { + mockResolvePermissionGroupConfig.mockResolvedValue({ + ...DEFAULT_PERMISSION_GROUP_CONFIG, + allowedIntegrations: ['gmail_v2'], + }) + + await expect(execute()).resolves.toMatchObject({ kind: 'list' }) + expect(mocks.executeAttachment).toHaveBeenCalledTimes(1) + }) + + /** + * `serviceIds` names which credentials a selector accepts, not which resource + * it reads. `google.drive` accepts a Drive, Docs, Sheets or Forms connection + * because all four carry Drive scope, but it only ever calls the Drive API. + * Judging the accepted set let a group that permits `google_sheets_v2` and + * excludes `google_drive` read Drive through it. + */ + it('refuses a multi-service selector whose own resource is excluded', async () => { + mocks.authorizeCredential.mockImplementation(async () => { + mocks.events.push('credential-authorization') + return { suppliedId: 'credential-1', providerId: 'google-sheets' } + }) + mocks.getAttachment.mockReturnValue({ + destination: 'fixed', + credential: { + kind: 'stored', + field: 'oauthCredential', + serviceIds: ['google-drive', 'google-docs', 'google-sheets', 'google-forms'], + resourceServiceId: 'google-drive', + }, + execute: mocks.executeAttachment, + }) + mockResolvePermissionGroupConfig.mockResolvedValue({ + ...DEFAULT_PERMISSION_GROUP_CONFIG, + allowedIntegrations: ['google_sheets_v2'], + }) + + await expect(execute()).rejects.toBeInstanceOf(IntegrationNotAllowedError) + expect(mocks.executeAttachment).not.toHaveBeenCalled() + }) + + /** + * The same selector, with its own resource permitted. The credential is a + * Sheets one and `google_sheets_v2` is *not* allowed, which is deliberate: + * the credential narrows nothing, because the API the selector reaches is the + * only thing the allowlist has an opinion about. + */ + it('allows a multi-service selector whose own resource is permitted', async () => { + mocks.authorizeCredential.mockImplementation(async () => { + mocks.events.push('credential-authorization') + return { suppliedId: 'credential-1', providerId: 'google-sheets' } + }) + mocks.getAttachment.mockReturnValue({ + destination: 'fixed', + credential: { + kind: 'stored', + field: 'oauthCredential', + serviceIds: ['google-drive', 'google-docs', 'google-sheets', 'google-forms'], + resourceServiceId: 'google-drive', + }, + execute: mocks.executeAttachment, + }) + mockResolvePermissionGroupConfig.mockResolvedValue({ + ...DEFAULT_PERMISSION_GROUP_CONFIG, + allowedIntegrations: ['google_drive'], + }) + + await expect(execute()).resolves.toMatchObject({ kind: 'list' }) + expect(mocks.executeAttachment).toHaveBeenCalledTimes(1) + }) + + /** The SharePoint/Excel pair reads SharePoint, whatever credential opened it. */ + it('refuses a sharepoint selector when only the excel half is allowed', async () => { + mocks.authorizeCredential.mockImplementation(async () => { + mocks.events.push('credential-authorization') + return { suppliedId: 'credential-1', providerId: 'microsoft-excel' } + }) + mocks.getAttachment.mockReturnValue({ + destination: 'fixed', + credential: { + kind: 'stored', + field: 'oauthCredential', + serviceIds: ['sharepoint', 'microsoft-excel'], + resourceServiceId: 'sharepoint', + }, + execute: mocks.executeAttachment, + }) + mockResolvePermissionGroupConfig.mockResolvedValue({ + ...DEFAULT_PERMISSION_GROUP_CONFIG, + allowedIntegrations: ['microsoft_excel_v2'], + }) + + await expect(execute()).rejects.toBeInstanceOf(IntegrationNotAllowedError) + expect(mocks.executeAttachment).not.toHaveBeenCalled() + }) + + /** + * The hole this closes: a selector authenticated from raw context fields + * (CloudWatch's AWS keys, IMAP's host and password) carries no credential + * policy, so the gate used to resolve it to an empty service list and return + * without checking — reaching the third party with the caller's keys under an + * allowlist that never named it. + */ + it('refuses a raw-context selector whose declared integration is excluded', async () => { + mocks.getAttachment.mockReturnValue({ + destination: 'fixed', + integrationBlockTypes: ['cloudwatch'], + execute: mocks.executeAttachment, + }) + mockResolvePermissionGroupConfig.mockResolvedValue({ + ...DEFAULT_PERMISSION_GROUP_CONFIG, + allowedIntegrations: ['slack_v2'], + }) + + await expect(execute()).rejects.toBeInstanceOf(IntegrationNotAllowedError) + expect(mocks.executeAttachment).not.toHaveBeenCalled() + }) + + it('executes a raw-context selector whose declared integration is permitted', async () => { + mocks.getAttachment.mockReturnValue({ + destination: 'fixed', + integrationBlockTypes: ['cloudwatch'], + execute: mocks.executeAttachment, + }) + mockResolvePermissionGroupConfig.mockResolvedValue({ + ...DEFAULT_PERMISSION_GROUP_CONFIG, + allowedIntegrations: ['cloudwatch'], + }) + + await expect(execute()).resolves.toMatchObject({ kind: 'list' }) + expect(mocks.executeAttachment).toHaveBeenCalledTimes(1) + }) + + /** + * An API-key integration owns no OAuth catalog entry, so its service id maps + * to no block type. The declaration is what gives the allowlist something to + * judge, and it must win over the catalog. + */ + it('refuses an api-key selector whose declared integration is excluded', async () => { + mocks.getAttachment.mockReturnValue({ + destination: 'fixed', + credential: { kind: 'stored', field: 'oauthCredential', serviceIds: ['snowflake'] }, + integrationBlockTypes: ['snowflake'], + execute: mocks.executeAttachment, + }) + mockResolvePermissionGroupConfig.mockResolvedValue({ + ...DEFAULT_PERMISSION_GROUP_CONFIG, + allowedIntegrations: ['slack_v2'], + }) + + await expect(execute()).rejects.toBeInstanceOf(IntegrationNotAllowedError) + expect(mocks.executeAttachment).not.toHaveBeenCalled() + }) + + /** + * A selector with no integration identity is not an integration: an internal + * selector declares no credential policy at all, so an allowlist that names + * nothing still leaves workspace files and knowledge bases pickable. + */ + it('passes through a selector that carries no credential policy', async () => { + mocks.getAttachment.mockReturnValue({ + destination: 'fixed', + execute: mocks.executeAttachment, + }) + mockResolvePermissionGroupConfig.mockResolvedValue({ + ...DEFAULT_PERMISSION_GROUP_CONFIG, + allowedIntegrations: [], + }) + + await expect(execute()).resolves.toMatchObject({ kind: 'list' }) + expect(mocks.executeAttachment).toHaveBeenCalledTimes(1) + }) + + /** No group governs the caller, so nothing narrows the allowlist. */ + it('executes when no permission group governs the caller', async () => { + mockResolvePermissionGroupConfig.mockResolvedValue(null) + + await expect(execute()).resolves.toMatchObject({ kind: 'list' }) + expect(mocks.executeAttachment).toHaveBeenCalledTimes(1) + }) + + it('prepares non-fixed destinations after credential authorization and before provider execution', async () => { + const prepare = vi.fn(async () => { + mocks.events.push('destination-preparation') + return { baseUrl: 'https://credential-bound.example.com' } + }) + mocks.getAttachment.mockReturnValueOnce({ + destination: { kind: 'credential-bound', prepare }, + credential: { kind: 'stored', field: 'oauthCredential', serviceIds: ['gmail'] }, + execute: vi.fn(async (_args: ExecuteServerSelectorArgs, preparedDestination: unknown) => { + mocks.events.push('provider-execution') + expect(preparedDestination).toEqual({ + baseUrl: 'https://credential-bound.example.com', + }) + return { kind: 'list', items: [{ id: 'label-1', label: 'Inbox' }] } + }), + }) + + await expect(execute()).resolves.toMatchObject({ kind: 'list' }) + + expect(mocks.events).toEqual([ + 'canonical-scope', + 'workspace-authorization', + 'reference-resolution', + 'credential-authorization', + 'destination-preparation', + 'provider-execution', + 'sanitization', + ]) + }) + + it('binds the request signal to credentials and does not present a late provider result', async () => { + const controller = new AbortController() + let markProviderStarted!: () => void + let finishProvider!: (result: { kind: 'list'; items: never[] }) => void + const providerStarted = new Promise((resolve) => { + markProviderStarted = resolve + }) + mocks.executeAttachment.mockImplementationOnce( + (args: ExecuteServerSelectorArgs) => + new Promise((resolve) => { + expect(args.credential?.signal).toBe(controller.signal) + markProviderStarted() + finishProvider = resolve + }) + ) + + const pending = execute({ signal: controller.signal }) + await providerStarted + + const abortReason = new DOMException('Selector request canceled', 'AbortError') + controller.abort(abortReason) + finishProvider({ kind: 'list', items: [] }) + + await expect(pending).rejects.toBe(abortReason) + expect(mocks.sanitize).not.toHaveBeenCalled() + expect(mocks.logger.info).not.toHaveBeenCalledWith('Executed selector', expect.anything()) + }) + + it('records legacy service-account use once with its trusted provider id', async () => { + mocks.authorizeCredential.mockResolvedValueOnce({ + suppliedId: 'credential-1', + providerId: 'atlassian-service-account', + access: { + ok: true, + credentialOwnerUserId: 'owner-1', + resolvedCredentialId: 'resolved-credential-1', + credentialType: 'service_account', + }, + }) + mocks.getAttachment.mockReturnValueOnce({ + destination: 'fixed', + auditCredentialUse: true, + credential: { kind: 'stored', field: 'oauthCredential', serviceIds: ['jira'] }, + execute: vi.fn(async (args: ExecuteServerSelectorArgs) => { + args.recordCredentialUse?.('jira') + args.recordCredentialUse?.('jira') + return { kind: 'list', items: [{ id: 'project-1', label: 'Project' }] } + }), + }) + + await execute({ auditRequest: { headers: { get: vi.fn(() => null) } } }) + + expect(mocks.recordCredentialAccess).toHaveBeenCalledOnce() + expect(mocks.recordCredentialAccess).toHaveBeenCalledWith( + expect.objectContaining({ + actorId: 'user-1', + workspaceId: 'workspace-1', + resourceId: 'resolved-credential-1', + providerId: 'atlassian-service-account', + credentialType: 'service_account', + }) + ) + expect(JSON.stringify(mocks.recordCredentialAccess.mock.calls)).not.toContain( + 'GMAIL_CREDENTIAL_ID' + ) + }) + + it('exposes safe truncation state without diagnostic details', async () => { + const { sanitizeSelectorResult } = await vi.importActual< + typeof import('@/lib/selectors/server/sanitize') + >('@/lib/selectors/server/sanitize') + mocks.executeAttachment.mockResolvedValueOnce({ + kind: 'list', + items: [{ id: 'label-1', label: 'Inbox' }], + diagnostics: { truncated: { reason: 'provider-cap', pages: 10, limit: 2_000 } }, + }) + mocks.sanitize.mockImplementationOnce(sanitizeSelectorResult) + + const result = await execute() + + expect(result).toEqual({ + kind: 'list', + items: [{ id: 'label-1', label: 'Inbox' }], + truncated: true, + }) + expect(result).not.toHaveProperty('diagnostics') + expect(JSON.stringify(result)).not.toContain('provider-cap') + expect(mocks.logger.warn).toHaveBeenCalledWith( + 'Selector provider result reached a configured cap', + expect.objectContaining({ + selectorKey: 'gmail.labels', + reason: 'provider-cap', + pages: 10, + limit: 2_000, + }) + ) + }) + + it('rejects extra context and unsupported capabilities before secret resolution', async () => { + await expect( + execute({ + context: { oauthCredential: 'credential-1', domain: 'tenant.example.com' }, + request: { kind: 'list', search: 'private query' }, + }) + ).rejects.toEqual(new SelectorContextUnavailableError()) + + expect(mocks.events).toEqual(['canonical-scope', 'workspace-authorization']) + expect(mocks.resolveReferences).not.toHaveBeenCalled() + expect(mocks.authorizeCredential).not.toHaveBeenCalled() + expect(mocks.executeAttachment).not.toHaveBeenCalled() + }) + + it('rejects oversized resolved context before credentials, destinations, or providers', async () => { + const prepare = vi.fn(async () => ({ baseUrl: 'https://example.com' })) + mocks.getAttachment.mockReturnValueOnce({ + destination: { kind: 'credential-bound', prepare }, + credential: { kind: 'stored', field: 'oauthCredential', serviceIds: ['gmail'] }, + execute: mocks.executeAttachment, + }) + mocks.resolveReferences.mockImplementationOnce(async () => { + mocks.events.push('reference-resolution') + return { + context: { oauthCredential: 'x'.repeat(16_385) }, + request: { kind: 'list' }, + references: new Map(), + } + }) + + await expect(execute()).rejects.toEqual(new SelectorContextUnavailableError()) + + expect(mocks.authorizeCredential).not.toHaveBeenCalled() + expect(prepare).not.toHaveBeenCalled() + expect(mocks.executeAttachment).not.toHaveBeenCalled() + expect(mocks.logger.warn).not.toHaveBeenCalled() + }) + + it.each([ + ['empty', ''], + ['oversized', 'x'.repeat(16_385)], + ])( + 'rejects %s resolved detail ids before credentials, destinations, or providers', + async (_case, resolvedId) => { + const prepare = vi.fn(async () => ({ baseUrl: 'https://example.com' })) + mocks.resolveScope.mockImplementationOnce(async () => { + mocks.events.push('canonical-scope') + return { + workspaceId: 'workspace-1', + workspaceOrganizationId: null, + allowPersonalApiKeys: true, + selectorKey: 'google.drive', + selectorManifest: getSelectorManifestEntry('google.drive'), + selectorScope: scope, + } + }) + mocks.getAttachment.mockReturnValueOnce({ + destination: { kind: 'credential-bound', prepare }, + credential: { kind: 'stored', field: 'oauthCredential', serviceIds: ['google-drive'] }, + execute: mocks.executeAttachment, + }) + mocks.resolveReferences.mockImplementationOnce(async () => { + mocks.events.push('reference-resolution') + return { + context: { oauthCredential: 'credential-1' }, + request: { kind: 'detail', id: resolvedId }, + references: new Map(), + } + }) + + await expect( + execute({ + selectorKey: 'google.drive', + request: { kind: 'detail', id: '{{GOOGLE_FILE_ID}}' }, + }) + ).rejects.toEqual(new SelectorContextUnavailableError()) + + expect(mocks.authorizeCredential).not.toHaveBeenCalled() + expect(prepare).not.toHaveBeenCalled() + expect(mocks.executeAttachment).not.toHaveBeenCalled() + expect(mocks.logger.warn).not.toHaveBeenCalled() + } + ) + + it('projects provider failures to a safe error and never logs request context', async () => { + mocks.executeAttachment.mockRejectedValueOnce( + new Error('upstream leaked selector-secret-canary for {{GMAIL_CREDENTIAL_ID}}') + ) + + await expect(execute()).rejects.toEqual(new SelectorOptionsUnavailableError()) + + expect(mocks.logger.warn).toHaveBeenCalledOnce() + const logged = JSON.stringify(mocks.logger.warn.mock.calls) + expect(logged).not.toContain('selector-secret-canary') + expect(logged).not.toContain('GMAIL_CREDENTIAL_ID') + expect(logged).not.toContain('credential-1') + expect(logged).not.toContain('context') + }) + + it.each([ + { + name: 'restores exact detail-id repeats after sanitization', + referenceName: 'GOOGLE_FILE_ID', + resolvedId: 'resolved-file-id', + }, + { + name: 'restores a reference whose spelling overlaps its resolved ID', + referenceName: 'ID', + resolvedId: 'ID', + }, + ])('$name', async ({ referenceName, resolvedId }) => { + const { sanitizeSelectorResult } = await vi.importActual< + typeof import('@/lib/selectors/server/sanitize') + >('@/lib/selectors/server/sanitize') + const originalId = `{{${referenceName}}}` + + mocks.resolveScope.mockImplementationOnce(async () => { + mocks.events.push('canonical-scope') + return { + workspaceId: 'workspace-1', + workspaceOrganizationId: null, + allowPersonalApiKeys: true, + selectorKey: 'google.drive', + selectorManifest: getSelectorManifestEntry('google.drive'), + selectorScope: scope, + } + }) + mocks.resolveReferences.mockImplementationOnce(async ({ protectedValues }) => { + mocks.events.push('reference-resolution') + protectedValues.add(resolvedId) + return { + context: { oauthCredential: 'credential-1' }, + request: { kind: 'detail', id: resolvedId }, + references: new Map([ + [ + 'request.id', + { + field: 'request.id', + name: referenceName, + scope: 'workspace', + visible: false, + }, + ], + ]), + } + }) + mocks.executeAttachment.mockImplementationOnce(async () => { + mocks.events.push('provider-execution') + return { + kind: 'detail', + item: { + id: resolvedId, + label: resolvedId, + meta: { resourceId: resolvedId, mimeType: 'application/pdf' }, + }, + } + }) + mocks.sanitize.mockImplementationOnce((result, protectedValues, options) => { + mocks.events.push('sanitization') + expect(result).toEqual({ + kind: 'detail', + item: { + id: resolvedId, + label: resolvedId, + meta: { resourceId: resolvedId, mimeType: 'application/pdf' }, + }, + }) + expect(protectedValues.contains(resolvedId)).toBe(true) + expect(options).toEqual({ allowedDetailExactProtectedValue: resolvedId }) + return sanitizeSelectorResult(result, protectedValues, options) + }) + + await expect( + execute({ + selectorKey: 'google.drive', + request: { kind: 'detail', id: originalId }, + }) + ).resolves.toEqual({ + kind: 'detail', + item: { + id: originalId, + label: originalId, + meta: { resourceId: originalId, mimeType: 'application/pdf' }, + }, + }) + }) +}) + +describe('selector scope contract', () => { + it('rejects workflow ids longer than 128 characters', () => { + expect( + selectorScopeSchema.safeParse({ kind: 'workflow', workflowId: 'w'.repeat(128) }).success + ).toBe(true) + expect( + selectorScopeSchema.safeParse({ kind: 'workflow', workflowId: 'w'.repeat(129) }).success + ).toBe(false) + }) +}) diff --git a/apps/sim/lib/selectors/application/execute-selector.ts b/apps/sim/lib/selectors/application/execute-selector.ts new file mode 100644 index 00000000000..11c6fdaef3d --- /dev/null +++ b/apps/sim/lib/selectors/application/execute-selector.ts @@ -0,0 +1,300 @@ +import { createLogger } from '@sim/logger' +import { + type ExecuteSelectorRequest, + selectorContextSchema, + selectorRequestSchema, +} from '@/lib/api/contracts/selectors/execute' +import { defineAuthorizedWorkspaceUseCase } from '@/lib/core/application' +import { type CredentialAuditRequest, recordCredentialAccess } from '@/lib/oauth/token-resolution' +import { selectorOperations } from '@/lib/selectors/application/operations' +import { + resolveSelectorApplicationContext, + type SelectorApplicationContext, +} from '@/lib/selectors/application/resolve-scope' +import { isSelectorReady, type ServerSelectorKey } from '@/lib/selectors/manifest' +import { authorizeSelectorCredential } from '@/lib/selectors/server/credentials' +import { + SelectorConnectionUnavailableError, + SelectorContextUnavailableError, + SelectorOptionsUnavailableError, +} from '@/lib/selectors/server/errors' +import { + assertSelectorIntegrationAllowed, + selectorIntegrationBlockTypes, +} from '@/lib/selectors/server/integration-access' +import { createSelectorProtectedValues } from '@/lib/selectors/server/protected-values' +import { resolveSelectorReferences } from '@/lib/selectors/server/references' +import { getServerSelectorAttachment } from '@/lib/selectors/server/registry' +import { sanitizeSelectorResult } from '@/lib/selectors/server/sanitize' +import type { ResolvedSelectorReference } from '@/lib/selectors/server/types' +import type { SelectorExecutionResult, SelectorRequest } from '@/lib/selectors/types' +import { IntegrationNotAllowedError } from '@/ee/access-control/utils/permission-check' + +const logger = createLogger('ExecuteSelector') + +export interface ExecuteSelectorInput extends ExecuteSelectorRequest { + signal?: AbortSignal + auditRequest?: CredentialAuditRequest +} + +function validateAuthorizedInput( + input: ExecuteSelectorInput, + context: SelectorApplicationContext +): void { + const manifest = context.selectorManifest + if (!manifest.scopeKinds.includes(input.scope.kind)) { + throw new SelectorContextUnavailableError() + } + + if (input.request.kind === 'detail' && !manifest.supportsDetail) { + throw new SelectorContextUnavailableError() + } + if ( + input.request.kind === 'list' && + ((input.request.search !== undefined && !manifest.supportsSearch) || + (input.request.cursor !== undefined && manifest.listMode !== 'paginated')) + ) { + throw new SelectorContextUnavailableError() + } + + const allowedContext = new Set(manifest.context.allowed) + if (Object.keys(input.context).some((field) => !allowedContext.has(field))) { + throw new SelectorContextUnavailableError() + } + if (!isSelectorReady(input.selectorKey, input.context)) { + throw new SelectorContextUnavailableError() + } +} + +function restoreReferencedDetailValues(input: { + originalRequest: SelectorRequest + resolvedRequest: SelectorRequest + result: SelectorExecutionResult + references: ReadonlyMap +}): SelectorExecutionResult { + if ( + input.originalRequest.kind !== 'detail' || + input.resolvedRequest.kind !== 'detail' || + input.result.kind !== 'detail' || + !input.result.item || + !input.references.has('request.id') + ) { + return input.result + } + + const item = input.result.item + const resolvedId = input.resolvedRequest.id + const originalId = input.originalRequest.id + const meta = item.meta + ? Object.fromEntries( + Object.entries(item.meta).map(([key, value]) => [ + key, + value === resolvedId ? originalId : value, + ]) + ) + : undefined + + return { + kind: 'detail', + item: { + ...item, + id: originalId, + label: item.label === resolvedId ? originalId : item.label, + ...(meta ? { meta } : {}), + }, + } +} + +function getReferencedDetailResolvedId(input: { + originalRequest: SelectorRequest + resolvedRequest: SelectorRequest + references: ReadonlyMap +}): string | undefined { + if ( + input.originalRequest.kind !== 'detail' || + input.resolvedRequest.kind !== 'detail' || + !input.references.has('request.id') + ) { + return undefined + } + return input.resolvedRequest.id +} + +async function executeAuthorizedSelector(args: { + principal: { kind: 'session'; userId: string; sessionId: string } + input: ExecuteSelectorInput + context: SelectorApplicationContext +}): Promise { + const startedAt = Date.now() + const protectedValues = createSelectorProtectedValues() + + try { + const attachment = getServerSelectorAttachment(args.input.selectorKey as ServerSelectorKey) + const resolved = await resolveSelectorReferences({ + selectorKey: args.input.selectorKey as ServerSelectorKey, + context: args.input.context, + request: args.input.request, + requesterUserId: args.principal.userId, + workspaceId: args.context.workspaceId, + protectedValues, + }) + const parsedContext = selectorContextSchema.safeParse(resolved.context) + const parsedRequest = selectorRequestSchema.safeParse(resolved.request) + if (!parsedContext.success || !parsedRequest.success) { + throw new SelectorContextUnavailableError() + } + const resolvedContext = parsedContext.data + const resolvedRequest = parsedRequest.data + + if (!isSelectorReady(args.input.selectorKey, resolvedContext)) { + throw new SelectorContextUnavailableError() + } + + const credential = attachment.credential + ? { + ...(await authorizeSelectorCredential({ + principal: args.principal, + context: resolvedContext, + scope: args.input.scope, + workspaceId: args.context.workspaceId, + policy: attachment.credential, + protectedValues, + references: resolved.references, + })), + signal: args.input.signal, + } + : undefined + + /** + * Enforces the permission group's `allowedIntegrations` decision, which the + * funnel cannot apply because it never sees which integration a selector + * reaches. Not a `permission-group-enforced:` annotation because that names + * a capability, and this key's enforcement mechanism is `executor`, not + * `capability`. + * + * Judged against the selector's own resource — the API it calls — not the + * set of credentials it accepts, and not the bound credential's provider. + * A selector the OAuth catalog cannot identify (raw-context credentials, an + * API-key integration) declares its block types instead of resolving to + * none and passing untested. Placed before the provider call so a denied + * integration is never reached. + */ + await assertSelectorIntegrationAllowed({ + principal: args.principal, + workspaceId: args.context.workspaceId, + blockTypes: selectorIntegrationBlockTypes(attachment), + }) + + const credentialAccess = credential?.access + let credentialUseRecorded = false + const recordCredentialUse = + attachment.auditCredentialUse && credentialAccess?.resolvedCredentialId + ? (providerId: string) => { + if (credentialUseRecorded) return + credentialUseRecorded = true + recordCredentialAccess({ + actorId: args.principal.userId, + workspaceId: args.context.workspaceId, + resourceId: credentialAccess.resolvedCredentialId!, + providerId: credential?.providerId ?? providerId, + credentialType: + credentialAccess.credentialType === 'service_account' ? 'service_account' : 'oauth', + auditRequest: args.input.auditRequest, + }) + } + : undefined + + const selectorArgs = { + selectorKey: args.input.selectorKey as ServerSelectorKey, + context: resolvedContext, + request: resolvedRequest, + scope: args.input.scope, + workspaceId: args.context.workspaceId, + principal: args.principal, + requesterUserId: args.principal.userId, + credential, + references: resolved.references, + signal: args.input.signal, + protectedValues, + ...(recordCredentialUse ? { recordCredentialUse } : {}), + } + const preparedDestination = + attachment.destination === 'fixed' + ? undefined + : await attachment.destination.prepare(selectorArgs) + const providerResult = await attachment.execute(selectorArgs, preparedDestination) + args.input.signal?.throwIfAborted() + if (providerResult.diagnostics?.truncated) { + logger.warn('Selector provider result reached a configured cap', { + selectorKey: args.input.selectorKey, + requestKind: args.input.request.kind, + scopeKind: args.input.scope.kind, + workspaceId: args.context.workspaceId, + reason: providerResult.diagnostics.truncated.reason, + limit: providerResult.diagnostics.truncated.limit, + pages: providerResult.diagnostics.truncated.pages, + }) + } + const referencedDetailResolvedId = getReferencedDetailResolvedId({ + originalRequest: args.input.request, + resolvedRequest, + references: resolved.references, + }) + const sanitizedProviderResult = sanitizeSelectorResult( + providerResult, + protectedValues, + referencedDetailResolvedId + ? { allowedDetailExactProtectedValue: referencedDetailResolvedId } + : undefined + ) + const result = restoreReferencedDetailValues({ + originalRequest: args.input.request, + resolvedRequest, + result: sanitizedProviderResult, + references: resolved.references, + }) + + logger.info('Executed selector', { + selectorKey: args.input.selectorKey, + requestKind: args.input.request.kind, + scopeKind: args.input.scope.kind, + workspaceId: args.context.workspaceId, + workflowId: args.input.scope.kind === 'workflow' ? args.input.scope.workflowId : undefined, + durationMs: Date.now() - startedAt, + itemCount: result.kind === 'list' ? result.items.length : result.item ? 1 : 0, + }) + return result + } catch (error) { + if (args.input.signal?.aborted) throw error + if ( + error instanceof SelectorContextUnavailableError || + error instanceof SelectorConnectionUnavailableError || + error instanceof SelectorOptionsUnavailableError || + // A refusal, not a provider failure: it reaches the caller as its own 403 + // rather than being folded into "Options unavailable". + error instanceof IntegrationNotAllowedError + ) { + throw error + } + logger.warn('Selector provider execution failed', { + selectorKey: args.input.selectorKey, + requestKind: args.input.request.kind, + scopeKind: args.input.scope.kind, + workspaceId: args.context.workspaceId, + durationMs: Date.now() - startedAt, + }) + throw new SelectorOptionsUnavailableError() + } +} + +export const executeSelector = defineAuthorizedWorkspaceUseCase({ + operation: selectorOperations.execute, + resolveContext: ({ input }) => + resolveSelectorApplicationContext({ + selectorKey: input.selectorKey as ServerSelectorKey, + scope: input.scope, + }), + authorizationOptions: {}, + authorizeResource: ({ input, context }) => validateAuthorizedInput(input, context), + execute: executeAuthorizedSelector, +}) diff --git a/apps/sim/lib/selectors/application/operations.ts b/apps/sim/lib/selectors/application/operations.ts new file mode 100644 index 00000000000..2e91b477da8 --- /dev/null +++ b/apps/sim/lib/selectors/application/operations.ts @@ -0,0 +1,12 @@ +import { defineWorkspaceOperation } from '@/lib/core/application' + +export const selectorOperations = { + // permission-group-exempt: no static capability names selector browsing — credential access is authorized per credential, and per-integration denial is the parameterized allowedIntegrations key, which the funnel cannot apply because it never sees which integration a selector reaches. That decision is enforced from the use case by assertSelectorIntegrationAllowed, against the selector's own resource, ahead of the provider call. + execute: defineWorkspaceOperation({ + id: 'selectors.execute', + minimumRole: 'read', + workspaceApiKey: 'deny', + principalKinds: ['session'], + capability: 'none', + }), +} as const diff --git a/apps/sim/lib/selectors/application/resolve-scope.ts b/apps/sim/lib/selectors/application/resolve-scope.ts new file mode 100644 index 00000000000..33fb54c6006 --- /dev/null +++ b/apps/sim/lib/selectors/application/resolve-scope.ts @@ -0,0 +1,41 @@ +import { getSelectorManifestEntry, type ServerSelectorKey } from '@/lib/selectors/manifest' +import { SelectorContextUnavailableError } from '@/lib/selectors/server/errors' +import type { SelectorManifestEntry, SelectorScope } from '@/lib/selectors/types' +import type { ActiveWorkflowApplicationContext } from '@/lib/workflows/application/context' +import { resolveActiveWorkflowApplicationContext } from '@/lib/workflows/application/context' +import type { ActiveWorkspaceApplicationContext } from '@/lib/workspaces/application/workspace-context' +import { resolveActiveWorkspaceApplicationContext } from '@/lib/workspaces/application/workspace-context' + +export type SelectorApplicationContext = ( + | ActiveWorkflowApplicationContext + | ActiveWorkspaceApplicationContext +) & { + selectorKey: ServerSelectorKey + selectorManifest: SelectorManifestEntry + selectorScope: SelectorScope +} + +export async function resolveSelectorApplicationContext(input: { + selectorKey: ServerSelectorKey + scope: SelectorScope +}): Promise { + const selectorManifest = getSelectorManifestEntry(input.selectorKey) + if (selectorManifest.classification === 'local') { + throw new SelectorContextUnavailableError() + } + + const workspaceContext = + input.scope.kind === 'workflow' + ? await resolveActiveWorkflowApplicationContext({ + workflowId: input.scope.workflowId, + assertedWorkspaceId: input.scope.workspaceId, + }) + : await resolveActiveWorkspaceApplicationContext(input.scope.workspaceId) + + return { + ...workspaceContext, + selectorKey: input.selectorKey, + selectorManifest, + selectorScope: input.scope, + } +} diff --git a/apps/sim/lib/selectors/client/execute-selector.test.ts b/apps/sim/lib/selectors/client/execute-selector.test.ts new file mode 100644 index 00000000000..4e9bfeedc41 --- /dev/null +++ b/apps/sim/lib/selectors/client/execute-selector.test.ts @@ -0,0 +1,57 @@ +/** + * @vitest-environment node + */ +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const { mockRequestJson } = vi.hoisted(() => ({ + mockRequestJson: vi.fn(), +})) + +vi.mock('@/lib/api/client/request', () => ({ requestJson: mockRequestJson })) + +import { loadAllSelectorOptions } from '@/lib/selectors/client/execute-selector' +import { MAX_SELECTOR_OPTIONS, MAX_SELECTOR_PAGES } from '@/lib/selectors/limits' + +const input = { + selectorKey: 'bitbucket.workspaces' as const, + scope: { kind: 'workspace' as const, workspaceId: 'workspace-1' }, + context: { oauthCredential: 'credential-1' }, +} + +describe('loadAllSelectorOptions', () => { + beforeEach(() => { + vi.clearAllMocks() + }) + + it('distinguishes a complete boundary-sized catalog from a capped page walk', async () => { + mockRequestJson.mockResolvedValueOnce({ + kind: 'list', + items: Array.from({ length: MAX_SELECTOR_OPTIONS }, (_, index) => ({ + id: `option-${index}`, + label: `Option ${index}`, + })), + }) + + const complete = await loadAllSelectorOptions(input) + + expect(complete.items).toHaveLength(MAX_SELECTOR_OPTIONS) + expect(complete.truncated).toBe(false) + + mockRequestJson.mockReset() + mockRequestJson.mockImplementation(async (...args: unknown[]) => { + const options = args[1] as { body: { request: { cursor?: string } } } + const page = Number(options.body.request.cursor ?? '0') + return { + kind: 'list', + items: [{ id: `page-${page}`, label: `Page ${page}` }], + nextCursor: String(page + 1), + } + }) + + const capped = await loadAllSelectorOptions(input) + + expect(mockRequestJson).toHaveBeenCalledTimes(MAX_SELECTOR_PAGES) + expect(capped.items).toHaveLength(MAX_SELECTOR_PAGES) + expect(capped.truncated).toBe(true) + }) +}) diff --git a/apps/sim/lib/selectors/client/execute-selector.ts b/apps/sim/lib/selectors/client/execute-selector.ts new file mode 100644 index 00000000000..3ff00336e9f --- /dev/null +++ b/apps/sim/lib/selectors/client/execute-selector.ts @@ -0,0 +1,90 @@ +'use client' + +import { requestJson } from '@/lib/api/client/request' +import { executeSelectorContract } from '@/lib/api/contracts/selectors/execute' +import { localSelectorAttachments } from '@/lib/selectors/client/local' +import { MAX_SELECTOR_OPTIONS, MAX_SELECTOR_PAGES } from '@/lib/selectors/limits' +import { + getSelectorManifestEntry, + type LocalSelectorKey, + type SelectorKey, +} from '@/lib/selectors/manifest' +import type { + SafeSelectorOption, + SelectorContext, + SelectorExecutionResult, + SelectorRequest, + SelectorScope, +} from '@/lib/selectors/types' + +export interface ExecuteSelectorClientInput { + selectorKey: SelectorKey + scope?: SelectorScope + context: SelectorContext + request: SelectorRequest + signal?: AbortSignal +} + +export interface LoadedSelectorOptions { + items: SafeSelectorOption[] + truncated: boolean +} + +export async function executeSelectorRequest( + input: ExecuteSelectorClientInput +): Promise { + const manifest = getSelectorManifestEntry(input.selectorKey) + if (manifest.classification === 'local') { + if (input.request.kind !== 'list') return { kind: 'detail', item: null } + return localSelectorAttachments[input.selectorKey as LocalSelectorKey]() + } + if (!input.scope) throw new Error('Selector scope is required') + return requestJson(executeSelectorContract, { + body: { + selectorKey: input.selectorKey, + scope: input.scope, + context: input.context, + request: input.request, + }, + signal: input.signal, + }) +} + +export async function loadAllSelectorOptions( + input: Omit & { search?: string } +): Promise { + const supportsSearch = getSelectorManifestEntry(input.selectorKey).supportsSearch + const items: SafeSelectorOption[] = [] + const seen = new Set() + let providerTruncated = false + let cursor: string | undefined + for (let page = 0; page < MAX_SELECTOR_PAGES; page += 1) { + const result = await executeSelectorRequest({ + ...input, + request: { + kind: 'list', + ...(supportsSearch && input.search !== undefined ? { search: input.search } : {}), + ...(cursor ? { cursor } : {}), + }, + }) + if (result.kind !== 'list') throw new Error('Selector returned an unexpected detail result') + providerTruncated ||= result.truncated === true + for (const [index, item] of result.items.entries()) { + if (seen.has(item.id)) continue + seen.add(item.id) + items.push(item) + if (items.length >= MAX_SELECTOR_OPTIONS) { + const omittedUniqueOption = result.items + .slice(index + 1) + .some((candidate) => !seen.has(candidate.id)) + return { + items, + truncated: providerTruncated || omittedUniqueOption || result.nextCursor !== undefined, + } + } + } + cursor = result.nextCursor + if (!cursor) return { items, truncated: providerTruncated } + } + return { items, truncated: providerTruncated || cursor !== undefined } +} diff --git a/apps/sim/lib/selectors/client/local.ts b/apps/sim/lib/selectors/client/local.ts new file mode 100644 index 00000000000..d74a463218d --- /dev/null +++ b/apps/sim/lib/selectors/client/local.ts @@ -0,0 +1,25 @@ +'use client' + +import type { LocalSelectorKey } from '@/lib/selectors/manifest' +import type { SelectorExecutionResult } from '@/lib/selectors/types' + +type LocalSelectorAttachment = () => Promise + +export const localSelectorAttachments = { + 'workspace.triggerTypes': async () => { + const { getTriggerOptions } = await import('@/lib/logs/get-trigger-options') + const valuesByLabel = new Map() + for (const option of getTriggerOptions()) { + const values = valuesByLabel.get(option.label) + if (values) values.push(option.value) + else valuesByLabel.set(option.label, [option.value]) + } + return { + kind: 'list', + items: Array.from(valuesByLabel, ([label, values]) => ({ + id: values.join(','), + label, + })), + } + }, +} satisfies Record diff --git a/apps/sim/lib/selectors/context.ts b/apps/sim/lib/selectors/context.ts new file mode 100644 index 00000000000..b750899de2c --- /dev/null +++ b/apps/sim/lib/selectors/context.ts @@ -0,0 +1,187 @@ +import { getSelectorManifestEntry, type SelectorKey } from '@/lib/selectors/manifest' +import { + type SelectorContext, + type SelectorContextKey, + selectorContextKeys, +} from '@/lib/selectors/types' +import { + buildCanonicalIndex, + buildSubBlockValues, + type CanonicalModeOverrides, + evaluateSubBlockCondition, + resolveActiveCanonicalValue, +} from '@/lib/workflows/subblocks/visibility' +import { getBlock } from '@/blocks' +import type { SubBlockConfig } from '@/blocks/types' +import { isReference } from '@/executor/constants' +import type { SubBlockState } from '@/stores/workflows/workflow/types' + +export const SELECTOR_CONTEXT_FIELDS = new Set(selectorContextKeys) +const EXPLICIT_SELECTOR_HINT_FIELDS = ['impersonateUserEmail'] as const + +function isSurfaceSubBlock(subBlock: SubBlockConfig, triggerMode: boolean): boolean { + const triggerField = subBlock.mode === 'trigger' || subBlock.mode === 'trigger-advanced' + return triggerMode ? triggerField : !triggerField +} + +export function getSelectorContextSubBlocks( + subBlocks: SubBlockConfig[], + values: Record, + triggerMode = false +): SubBlockConfig[] { + return subBlocks.filter( + (subBlock) => + isSurfaceSubBlock(subBlock, triggerMode) && + evaluateSubBlockCondition(subBlock.condition, values) + ) +} + +function toContextValue(value: unknown): string | undefined { + if (value === null || value === undefined) return undefined + const normalized = typeof value === 'string' ? value : String(value) + if (!normalized || isReference(normalized) || /<[^<>]+>/.test(normalized)) return undefined + return normalized +} + +export function projectSelectorContext( + selectorKey: SelectorKey, + candidate: object +): SelectorContext { + const manifest = getSelectorManifestEntry(selectorKey) + const allowed = new Set(manifest.context.allowed) + const source = candidate as Record + const projectedCandidate: Record = { ...source } + + if (projectedCandidate.oauthCredential === undefined) { + projectedCandidate.oauthCredential = + source.credential ?? + source.botCredential ?? + source.customBotCredential ?? + source.manualBotCredential + } + for (const [target, sourceFields] of Object.entries(manifest.context.sourceFields ?? {})) { + for (const sourceField of sourceFields ?? []) { + const value = toContextValue(source[sourceField]) + if (value === undefined) continue + projectedCandidate[target] = value + break + } + } + + const context: SelectorContext = {} + for (const [field, value] of Object.entries(projectedCandidate)) { + if (!allowed.has(field)) continue + const normalized = toContextValue(value) + if (normalized !== undefined) context[field as SelectorContextKey] = normalized + } + return context +} + +export interface BuildSelectorRawContextInput { + selectorKey: SelectorKey + blockType: string + subBlocks: Record + dependsOn?: readonly string[] + canonicalModes?: CanonicalModeOverrides + triggerMode?: boolean + staticContext?: Readonly> +} + +export interface BuildSelectorContextFromValuesInput { + selectorKey: SelectorKey + contextConfigs: SubBlockConfig[] + values: Record + dependsOn?: readonly string[] + canonicalIndex?: ReturnType + canonicalModes?: CanonicalModeOverrides + staticContext?: Readonly> +} + +/** Shared active-value projection used by every selector surface. */ +export function buildSelectorContextFromValues( + input: BuildSelectorContextFromValuesInput +): SelectorContext { + const manifest = getSelectorManifestEntry(input.selectorKey) + const allowed = new Set(manifest.context.allowed) + const canonicalIndex = input.canonicalIndex ?? buildCanonicalIndex(input.contextConfigs) + const configById = new Map(input.contextConfigs.map((config) => [config.id, config])) + const dependencies = input.dependsOn ? new Set(input.dependsOn) : null + const candidate: Record = { ...(input.staticContext ?? {}) } + const resolvedGroups = new Set() + + const includeSubBlock = (subBlockId: string, canonicalId?: string): boolean => { + if (!dependencies) return true + return ( + dependencies.has(subBlockId) || (canonicalId !== undefined && dependencies.has(canonicalId)) + ) + } + + const includeValue = (subBlockId: string, value: unknown) => { + const config = configById.get(subBlockId) + if (!config) return + const canonicalId = canonicalIndex.canonicalIdBySubBlockId[subBlockId] + if (!includeSubBlock(subBlockId, canonicalId)) return + + if (canonicalId) { + if (resolvedGroups.has(canonicalId)) return + resolvedGroups.add(canonicalId) + candidate[canonicalId] = resolveActiveCanonicalValue( + canonicalIndex.groupsById[canonicalId], + input.values, + input.canonicalModes + ) + return + } + candidate[subBlockId] = value + } + + if (dependencies) { + for (const dependency of dependencies) { + const canonicalId = + canonicalIndex.groupsById[dependency]?.canonicalId ?? + canonicalIndex.canonicalIdBySubBlockId[dependency] + if (canonicalId) { + if (resolvedGroups.has(canonicalId)) continue + resolvedGroups.add(canonicalId) + candidate[canonicalId] = resolveActiveCanonicalValue( + canonicalIndex.groupsById[canonicalId], + input.values, + input.canonicalModes + ) + continue + } + if (!configById.has(dependency)) continue + candidate[dependency] = input.values[dependency] + } + } else { + for (const [subBlockId, value] of Object.entries(input.values)) { + includeValue(subBlockId, value) + } + } + + for (const hint of EXPLICIT_SELECTOR_HINT_FIELDS) { + if (allowed.has(hint)) candidate[hint] = input.values[hint] + } + + return projectSelectorContext(input.selectorKey, candidate) +} + +export function buildSelectorRawContext(input: BuildSelectorRawContextInput): SelectorContext { + const blockConfig = getBlock(input.blockType) + if (!blockConfig) return projectSelectorContext(input.selectorKey, input.staticContext ?? {}) + + const values = buildSubBlockValues(input.subBlocks) + const contextConfigs = getSelectorContextSubBlocks( + blockConfig.subBlocks, + values, + input.triggerMode + ) + return buildSelectorContextFromValues({ + selectorKey: input.selectorKey, + contextConfigs, + values, + dependsOn: input.dependsOn, + canonicalModes: input.canonicalModes, + staticContext: input.staticContext, + }) +} diff --git a/apps/sim/lib/selectors/limits.ts b/apps/sim/lib/selectors/limits.ts new file mode 100644 index 00000000000..24ce8b47984 --- /dev/null +++ b/apps/sim/lib/selectors/limits.ts @@ -0,0 +1,2 @@ +export const MAX_SELECTOR_OPTIONS = 10_000 +export const MAX_SELECTOR_PAGES = 200 diff --git a/apps/sim/lib/selectors/manifest.test.ts b/apps/sim/lib/selectors/manifest.test.ts new file mode 100644 index 00000000000..df9322d33e3 --- /dev/null +++ b/apps/sim/lib/selectors/manifest.test.ts @@ -0,0 +1,127 @@ +import { describe, expect, it } from 'vitest' +import { localSelectorAttachments } from '@/lib/selectors/client/local' +import { selectorManifest } from '@/lib/selectors/manifest' +import { serverSelectorRegistry } from '@/lib/selectors/server/registry' + +describe('selector manifest', () => { + it('keeps the completed migration inventory exhaustive and legacy-free', () => { + const classifications = Object.values(selectorManifest).map((entry) => entry.classification) + const count = (classification: (typeof classifications)[number]) => + classifications.filter((value) => value === classification).length + + expect(Object.keys(selectorManifest)).toHaveLength(94) + expect(count('provider-server')).toBe(82) + expect(count('internal-server')).toBe(11) + expect(count('local')).toBe(1) + expect(classifications).not.toContain('provider-legacy') + }) + + it('attaches every manifest key exactly once on its declared execution side', () => { + const entries = Object.entries(selectorManifest) + const expectedServerKeys = entries + .filter(([, entry]) => entry.classification !== 'local') + .map(([key]) => key) + .sort() + const expectedLocalKeys = entries + .filter(([, entry]) => entry.classification === 'local') + .map(([key]) => key) + .sort() + + expect(Object.keys(serverSelectorRegistry).sort()).toEqual(expectedServerKeys) + expect(Object.keys(localSelectorAttachments).sort()).toEqual(expectedLocalKeys) + + const providerKeys = entries + .filter(([, entry]) => entry.classification === 'provider-server') + .map(([key]) => key) + const rawConnectionKeys = providerKeys.filter( + (key) => !serverSelectorRegistry[key as keyof typeof serverSelectorRegistry].credential + ) + expect(providerKeys).toHaveLength(82) + expect(rawConnectionKeys.sort()).toEqual([ + 'cloudwatch.logGroups', + 'cloudwatch.logStreams', + 'imap.mailboxes', + ]) + }) + + it('keeps shared Microsoft selectors bound only to their intended credential families', () => { + expect(serverSelectorRegistry['onedrive.files'].credential?.serviceIds).toEqual(['onedrive']) + expect(serverSelectorRegistry['onedrive.folders'].credential?.serviceIds).toEqual([ + 'onedrive', + 'microsoft-word', + ]) + expect(serverSelectorRegistry['sharepoint.lists'].credential?.serviceIds).toEqual([ + 'sharepoint', + ]) + expect(serverSelectorRegistry['sharepoint.sites'].credential?.serviceIds).toEqual([ + 'sharepoint', + 'microsoft-excel', + ]) + }) + + it('declares both CloudWatch selectors as paginated', () => { + expect(selectorManifest['cloudwatch.logGroups'].listMode).toBe('paginated') + expect(selectorManifest['cloudwatch.logStreams'].listMode).toBe('paginated') + }) + + /** + * `serviceIds` names which credentials a selector accepts; the integration + * allowlist has to judge which resource it *reaches*, and for a shared + * provider API those differ. A multi-service declaration that named no + * resource fell back to "any accepted service is allowed", which let a group + * permitting `google_sheets_v2` read Drive through `google.drive`. + */ + it('makes every multi-service selector name the resource it reaches', () => { + for (const [key, attachment] of Object.entries(serverSelectorRegistry)) { + const credential = attachment.credential + if (!credential || credential.serviceIds.length < 2) continue + + expect(credential.resourceServiceId, `${key} declares no resourceServiceId`).toBeDefined() + expect(credential.serviceIds).toContain(credential.resourceServiceId) + } + }) + + it('pins the resource each shared-provider selector reaches', () => { + expect(serverSelectorRegistry['google.drive'].credential?.resourceServiceId).toBe( + 'google-drive' + ) + expect(serverSelectorRegistry['onedrive.folders'].credential?.resourceServiceId).toBe( + 'onedrive' + ) + expect(serverSelectorRegistry['sharepoint.sites'].credential?.resourceServiceId).toBe( + 'sharepoint' + ) + }) + + it('requires executable preparation for every non-fixed destination', () => { + const preparedDestinations = Object.values(serverSelectorRegistry).filter( + (attachment) => attachment.destination !== 'fixed' + ) + + expect(preparedDestinations).toHaveLength(13) + for (const attachment of preparedDestinations) { + expect(attachment.destination).toEqual( + expect.objectContaining({ + kind: expect.stringMatching(/^(credential-bound|user-controlled)$/), + prepare: expect.any(Function), + }) + ) + } + }) + + it('preserves credential-use auditing only for the seven legacy-audited selectors', () => { + const auditedKeys = Object.entries(serverSelectorRegistry) + .flatMap(([key, attachment]) => (attachment.auditCredentialUse ? [key] : [])) + .sort() + + expect(auditedKeys).toEqual([ + 'confluence.pages', + 'jira.issues', + 'jira.projects', + 'managedAgent.agents', + 'managedAgent.environments', + 'managedAgent.memoryStores', + 'managedAgent.vaults', + ]) + }) +}) diff --git a/apps/sim/lib/selectors/manifest.ts b/apps/sim/lib/selectors/manifest.ts new file mode 100644 index 00000000000..172fcfa0ac5 --- /dev/null +++ b/apps/sim/lib/selectors/manifest.ts @@ -0,0 +1,417 @@ +import type { + SelectorContextKey, + SelectorManifestEntry, + SelectorReadiness, +} from '@/lib/selectors/types' + +export const DEFAULT_SELECTOR_STALE_TIME = 30_000 +export const DEFAULT_SELECTOR_DETAIL_STALE_TIME = 300_000 +export const STANDARD_SELECTOR_STALE_TIME = 60_000 +export const SEARCH_SELECTOR_STALE_TIME = 15_000 + +const SERVER_SCOPE_KINDS = ['workflow', 'workspace'] as const + +interface ServerManifestOptions { + readiness?: SelectorReadiness + sensitive?: readonly SelectorContextKey[] + sourceFields?: Partial> + listMode?: 'flat' | 'paginated' + search?: boolean + detail?: boolean + unknownDetail?: boolean + staleTime?: number +} + +function providerSelector( + extraContext: readonly SelectorContextKey[] = [], + options: ServerManifestOptions = {} +): SelectorManifestEntry { + return { + classification: 'provider-server', + context: { + allowed: ['oauthCredential', ...extraContext], + readiness: options.readiness ?? { all: ['oauthCredential'] }, + ...(options.sensitive ? { sensitive: options.sensitive } : {}), + ...(options.sourceFields ? { sourceFields: options.sourceFields } : {}), + }, + scopeKinds: SERVER_SCOPE_KINDS, + listMode: options.listMode ?? 'flat', + supportsSearch: options.search ?? false, + supportsDetail: options.detail ?? false, + resolvesUnknownIds: options.unknownDetail ?? false, + staleTime: options.staleTime ?? STANDARD_SELECTOR_STALE_TIME, + } +} + +function rawProviderSelector( + context: readonly SelectorContextKey[], + options: ServerManifestOptions +): SelectorManifestEntry { + return { + classification: 'provider-server', + context: { + allowed: context, + ...(options.readiness ? { readiness: options.readiness } : {}), + ...(options.sensitive ? { sensitive: options.sensitive } : {}), + ...(options.sourceFields ? { sourceFields: options.sourceFields } : {}), + }, + scopeKinds: SERVER_SCOPE_KINDS, + listMode: options.listMode ?? 'flat', + supportsSearch: options.search ?? false, + supportsDetail: options.detail ?? false, + resolvesUnknownIds: options.unknownDetail ?? false, + staleTime: options.staleTime ?? STANDARD_SELECTOR_STALE_TIME, + } +} + +function internalSelector( + context: readonly SelectorContextKey[] = [], + options: ServerManifestOptions = {} +): SelectorManifestEntry { + return { + classification: 'internal-server', + context: { + allowed: context, + ...(options.readiness ? { readiness: options.readiness } : {}), + ...(options.sensitive ? { sensitive: options.sensitive } : {}), + ...(options.sourceFields ? { sourceFields: options.sourceFields } : {}), + }, + scopeKinds: SERVER_SCOPE_KINDS, + listMode: options.listMode ?? 'flat', + supportsSearch: options.search ?? false, + supportsDetail: options.detail ?? false, + resolvesUnknownIds: options.unknownDetail ?? false, + staleTime: options.staleTime ?? STANDARD_SELECTOR_STALE_TIME, + } +} + +export const selectorManifest = { + 'airtable.bases': providerSelector([], { detail: true }), + 'airtable.tables': providerSelector(['baseId'], { + readiness: { all: ['oauthCredential', 'baseId'] }, + detail: true, + }), + 'asana.workspaces': providerSelector([], { detail: true }), + 'attio.lists': providerSelector([], { detail: true }), + 'attio.objects': providerSelector([], { detail: true }), + 'bigquery.datasets': providerSelector(['projectId', 'impersonateUserEmail'], { + readiness: { all: ['oauthCredential', 'projectId'] }, + listMode: 'paginated', + detail: true, + }), + 'bigquery.tables': providerSelector(['projectId', 'datasetId', 'impersonateUserEmail'], { + readiness: { all: ['oauthCredential', 'projectId', 'datasetId'] }, + listMode: 'paginated', + detail: true, + }), + 'bitbucket.workspaces': providerSelector([], { listMode: 'paginated', detail: true }), + 'bitbucket.repositories': providerSelector(['workspaceSlug'], { + readiness: { all: ['oauthCredential', 'workspaceSlug'] }, + listMode: 'paginated', + detail: true, + }), + 'calcom.eventTypes': providerSelector([], { detail: true }), + 'calcom.schedules': providerSelector([], { detail: true }), + 'clickup.workspaces': providerSelector(), + 'clickup.spaces': providerSelector(['teamId'], { + readiness: { all: ['oauthCredential', 'teamId'] }, + }), + 'clickup.folders': providerSelector(['spaceId', 'listSpaceId'], { + readiness: { all: ['oauthCredential'], any: ['spaceId', 'listSpaceId'] }, + }), + 'clickup.lists': providerSelector(['folderId', 'spaceId', 'listSpaceId'], { + readiness: { + all: ['oauthCredential'], + any: ['folderId', 'spaceId', 'listSpaceId'], + }, + }), + 'confluence.spaces': providerSelector(['domain'], { + readiness: { all: ['oauthCredential', 'domain'] }, + listMode: 'paginated', + detail: true, + unknownDetail: true, + }), + 'confluence.spacesById': providerSelector(['domain'], { + readiness: { all: ['oauthCredential', 'domain'] }, + listMode: 'paginated', + detail: true, + unknownDetail: true, + }), + 'confluence.pages': providerSelector(['domain'], { + readiness: { all: ['oauthCredential', 'domain'] }, + search: true, + detail: true, + }), + 'google.tasks.lists': providerSelector(['impersonateUserEmail'], { + listMode: 'paginated', + detail: true, + }), + 'gmail.labels': providerSelector(['impersonateUserEmail']), + 'google.calendar': providerSelector(['impersonateUserEmail'], { + listMode: 'paginated', + detail: true, + }), + 'google.drive': providerSelector(['mimeType', 'fileId', 'impersonateUserEmail'], { + listMode: 'paginated', + search: true, + detail: true, + staleTime: SEARCH_SELECTOR_STALE_TIME, + }), + 'google.sheets': providerSelector(['spreadsheetId', 'impersonateUserEmail'], { + readiness: { all: ['oauthCredential', 'spreadsheetId'] }, + }), + 'harmonic.savedSearches': providerSelector([], { detail: true, unknownDetail: true }), + 'hubspot.lists': providerSelector([], { listMode: 'paginated', search: true, detail: true }), + 'hubspot.owners': providerSelector([], { listMode: 'paginated', detail: true }), + 'hubspot.pipelines': providerSelector(['objectType', 'customObjectTypeId']), + 'hubspot.pipelineStages': providerSelector(['objectType', 'customObjectTypeId', 'pipelineId'], { + readiness: { all: ['oauthCredential', 'pipelineId'] }, + }), + 'hubspot.properties': providerSelector(['objectType', 'customObjectTypeId']), + 'jsm.requestTypes': providerSelector(['domain', 'serviceDeskId'], { + readiness: { all: ['oauthCredential', 'domain', 'serviceDeskId'] }, + detail: true, + }), + 'jsm.serviceDesks': providerSelector(['domain'], { + readiness: { all: ['oauthCredential', 'domain'] }, + detail: true, + }), + 'microsoft.planner.plans': providerSelector([], { listMode: 'paginated', detail: true }), + 'notion.databases': providerSelector([], { detail: true }), + 'notion.pages': providerSelector([], { detail: true }), + 'netsuite.recordTypes': providerSelector(['jobId'], { + detail: true, + unknownDetail: true, + }), + 'netsuite.asyncTasks': providerSelector(['jobId'], { + readiness: { all: ['oauthCredential', 'jobId'] }, + detail: true, + unknownDetail: true, + }), + 'pipedrive.pipelines': providerSelector([], { detail: true }), + 'sharepoint.lists': providerSelector(['siteId'], { + readiness: { all: ['oauthCredential', 'siteId'] }, + listMode: 'paginated', + detail: true, + }), + 'trello.boards': providerSelector([], { detail: true }), + 'zoho_desk.organizations': providerSelector(), + 'zoho_desk.departments': providerSelector(['orgId'], { + readiness: { all: ['oauthCredential', 'orgId'] }, + }), + 'zoho_desk.agents': providerSelector(['orgId'], { + readiness: { all: ['oauthCredential', 'orgId'] }, + }), + 'zoom.meetings': providerSelector([], { listMode: 'paginated', detail: true }), + 'slack.channels': providerSelector([], { + sourceFields: { oauthCredential: ['botToken'] }, + listMode: 'paginated', + detail: true, + }), + 'snowflake.databases': providerSelector(['database', 'schema'], { + detail: true, + unknownDetail: true, + }), + 'snowflake.schemas': providerSelector(['database', 'schema'], { + readiness: { all: ['oauthCredential', 'database'] }, + detail: true, + unknownDetail: true, + }), + 'snowflake.tables': providerSelector(['database', 'schema'], { + readiness: { all: ['oauthCredential', 'database', 'schema'] }, + detail: true, + unknownDetail: true, + }), + 'snowflake.warehouses': providerSelector(['database', 'schema'], { + detail: true, + unknownDetail: true, + }), + 'snowflake.roles': providerSelector(['database', 'schema'], { + detail: true, + unknownDetail: true, + }), + 'snowflake.fileFormats': providerSelector(['database', 'schema'], { + readiness: { all: ['oauthCredential', 'database', 'schema'] }, + detail: true, + unknownDetail: true, + }), + 'snowflake.procedures': providerSelector(['database', 'schema'], { + readiness: { all: ['oauthCredential', 'database', 'schema'] }, + detail: true, + unknownDetail: true, + }), + 'slack.users': providerSelector([], { + sourceFields: { oauthCredential: ['botToken'] }, + listMode: 'paginated', + detail: true, + }), + 'outlook.folders': providerSelector([], { listMode: 'paginated', detail: true }), + 'outlook.calendars': providerSelector([], { listMode: 'paginated', detail: true }), + 'microsoft.teams': providerSelector([], { listMode: 'paginated', detail: true }), + 'microsoft.chats': providerSelector([], { listMode: 'paginated', detail: true }), + 'microsoft.channels': providerSelector(['teamId'], { + readiness: { all: ['oauthCredential', 'teamId'] }, + listMode: 'paginated', + detail: true, + }), + 'microsoft.planner': providerSelector(['planId'], { + readiness: { all: ['oauthCredential', 'planId'] }, + listMode: 'paginated', + detail: true, + }), + 'onedrive.files': providerSelector(['mimeType'], { listMode: 'paginated', detail: true }), + 'onedrive.folders': providerSelector(['driveId'], { listMode: 'paginated', detail: true }), + 'sharepoint.sites': providerSelector([], { + listMode: 'paginated', + search: true, + detail: true, + }), + 'microsoft.excel': providerSelector(['driveId'], { + listMode: 'paginated', + search: true, + detail: true, + }), + 'microsoft.excel.drives': providerSelector(['siteId'], { + readiness: { all: ['oauthCredential', 'siteId'] }, + listMode: 'paginated', + detail: true, + }), + 'microsoft.excel.sheets': providerSelector(['driveId', 'spreadsheetId'], { + readiness: { all: ['oauthCredential', 'spreadsheetId'] }, + listMode: 'paginated', + }), + 'microsoft.word': providerSelector(['driveId'], { + listMode: 'paginated', + search: true, + detail: true, + }), + 'wealthbox.contacts': providerSelector([], { search: true }), + 'jira.issues': providerSelector(['domain', 'projectId'], { + readiness: { all: ['oauthCredential', 'domain'] }, + search: true, + detail: true, + staleTime: SEARCH_SELECTOR_STALE_TIME, + }), + 'jira.projects': providerSelector(['domain'], { + readiness: { all: ['oauthCredential', 'domain'] }, + listMode: 'paginated', + search: true, + detail: true, + }), + 'linear.projects': providerSelector(['teamId'], { + readiness: { all: ['oauthCredential', 'teamId'] }, + listMode: 'paginated', + detail: true, + }), + 'linear.teams': providerSelector([], { listMode: 'paginated', detail: true }), + 'monday.boards': providerSelector([], { detail: true }), + 'monday.groups': providerSelector(['boardId'], { + readiness: { all: ['oauthCredential', 'boardId'] }, + detail: true, + }), + 'webflow.sites': providerSelector(), + 'webflow.collections': providerSelector(['siteId'], { + readiness: { all: ['oauthCredential', 'siteId'] }, + }), + 'webflow.items': providerSelector(['collectionId'], { + readiness: { all: ['oauthCredential', 'collectionId'] }, + listMode: 'paginated', + search: true, + detail: true, + staleTime: SEARCH_SELECTOR_STALE_TIME, + }), + 'cloudwatch.logGroups': rawProviderSelector( + ['awsAccessKeyId', 'awsSecretAccessKey', 'awsRegion'], + { + readiness: { all: ['awsAccessKeyId', 'awsSecretAccessKey', 'awsRegion'] }, + sensitive: ['awsAccessKeyId', 'awsSecretAccessKey'], + listMode: 'paginated', + search: true, + detail: true, + } + ), + 'cloudwatch.logStreams': rawProviderSelector( + ['awsAccessKeyId', 'awsSecretAccessKey', 'awsRegion', 'logGroupName'], + { + readiness: { + all: ['awsAccessKeyId', 'awsSecretAccessKey', 'awsRegion', 'logGroupName'], + }, + sensitive: ['awsAccessKeyId', 'awsSecretAccessKey'], + listMode: 'paginated', + search: true, + detail: true, + } + ), + 'imap.mailboxes': rawProviderSelector(['host', 'port', 'secure', 'username', 'password'], { + readiness: { all: ['host', 'username', 'password'] }, + sensitive: ['username', 'password'], + }), + 'managedAgent.agents': providerSelector(), + 'managedAgent.environments': providerSelector(['environmentType']), + 'managedAgent.vaults': providerSelector(), + 'managedAgent.memoryStores': providerSelector(), + 'knowledge.documents': internalSelector(['knowledgeBaseId'], { + readiness: { all: ['knowledgeBaseId'] }, + listMode: 'paginated', + search: true, + detail: true, + }), + 'sim.workflows': internalSelector(['excludeWorkflowId'], { detail: true }), + 'table.columns': internalSelector(['tableId'], { + readiness: { all: ['tableId'] }, + detail: true, + staleTime: 0, + }), + 'table.outputColumns': internalSelector(['tableId'], { + readiness: { all: ['tableId'] }, + detail: true, + staleTime: 0, + }), + 'workspace.credentialProviders': internalSelector([], { detail: true }), + 'workspace.credentialGroups': internalSelector([], { detail: true }), + 'workspace.credentialGroupProviders': internalSelector(['credentialGroupId'], { + detail: true, + }), + 'workspace.secretNames': internalSelector(), + 'workspace.rawSecretNames': internalSelector(), + 'workspace.sandboxes': internalSelector(['language'], { detail: true }), + 'providers.openrouterEmbeddingModels': internalSelector(), + 'workspace.triggerTypes': { + classification: 'local', + context: { allowed: [] }, + scopeKinds: [], + listMode: 'flat', + supportsSearch: false, + supportsDetail: false, + resolvesUnknownIds: false, + staleTime: STANDARD_SELECTOR_STALE_TIME, + }, +} as const satisfies Record + +export type SelectorKey = keyof typeof selectorManifest +export type ServerSelectorKey = { + [K in SelectorKey]: (typeof selectorManifest)[K]['classification'] extends 'local' ? never : K +}[SelectorKey] +export type ProviderSelectorKey = { + [K in SelectorKey]: (typeof selectorManifest)[K]['classification'] extends 'provider-server' + ? K + : never +}[SelectorKey] +export type InternalSelectorKey = { + [K in SelectorKey]: (typeof selectorManifest)[K]['classification'] extends 'internal-server' + ? K + : never +}[SelectorKey] +export type LocalSelectorKey = Exclude + +export function getSelectorManifestEntry(key: SelectorKey): SelectorManifestEntry { + return selectorManifest[key] +} + +export function isSelectorReady(key: SelectorKey, context: Record): boolean { + const readiness = getSelectorManifestEntry(key).context.readiness + if (!readiness) return true + if (readiness.all?.some((field) => !context[field])) return false + if (readiness.any?.length && !readiness.any.some((field) => Boolean(context[field]))) return false + return true +} diff --git a/apps/sim/lib/selectors/server/credentials.test.ts b/apps/sim/lib/selectors/server/credentials.test.ts new file mode 100644 index 00000000000..17138d2079a --- /dev/null +++ b/apps/sim/lib/selectors/server/credentials.test.ts @@ -0,0 +1,259 @@ +/** + * @vitest-environment node + */ + +import { credential } from '@sim/db/schema' +import { queueTableRows, resetDbChainMock } from '@sim/testing' +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const mocks = vi.hoisted(() => ({ + authorizeCredentialUse: vi.fn(), + credentialProviderMatchesService: vi.fn(), + getServiceConfig: vi.fn(), + resolveCredentialTokenBundle: vi.fn(), +})) + +vi.mock('@/lib/auth/credential-access', () => ({ + authorizeCredentialUseForAuth: mocks.authorizeCredentialUse, +})) + +vi.mock('@/lib/oauth/credential-service', () => ({ + resolveCredentialTokenBundle: mocks.resolveCredentialTokenBundle, +})) + +vi.mock('@/lib/oauth/utils', () => ({ + credentialProviderMatchesService: mocks.credentialProviderMatchesService, + getServiceConfigByServiceId: mocks.getServiceConfig, +})) + +import { + authorizeSelectorCredential, + resolveSelectorOAuthAccessToken, +} from '@/lib/selectors/server/credentials' +import { SelectorConnectionUnavailableError } from '@/lib/selectors/server/errors' +import { createSelectorProtectedValues } from '@/lib/selectors/server/protected-values' + +const principal = { kind: 'session' as const, userId: 'user-1', sessionId: 'session-1' } +const policy = { + kind: 'stored' as const, + field: 'oauthCredential' as const, + serviceIds: ['gmail'], +} + +function authorize(): Promise { + return authorizeSelectorCredential({ + principal, + context: { oauthCredential: 'credential-1' }, + scope: { kind: 'workspace', workspaceId: 'workspace-1' }, + workspaceId: 'workspace-1', + policy, + protectedValues: createSelectorProtectedValues(), + references: new Map(), + }) +} + +describe('authorizeSelectorCredential', () => { + beforeEach(() => { + vi.clearAllMocks() + resetDbChainMock() + mocks.getServiceConfig.mockReturnValue({ id: 'gmail' }) + }) + + it('conceals a credential authorized in a different workspace', async () => { + mocks.authorizeCredentialUse.mockResolvedValue({ + ok: true, + workspaceId: 'workspace-2', + credentialOwnerUserId: 'owner-1', + resolvedCredentialId: 'credential-1', + }) + + await expect(authorize()).rejects.toEqual(new SelectorConnectionUnavailableError()) + expect(mocks.credentialProviderMatchesService).not.toHaveBeenCalled() + }) + + it('pins workspace-scoped credential authorization before legacy account resolution', async () => { + mocks.authorizeCredentialUse.mockResolvedValue({ + ok: true, + workspaceId: 'workspace-1', + credentialOwnerUserId: 'owner-1', + resolvedCredentialId: 'account-1', + }) + queueTableRows(credential, [{ accountId: 'account-1', providerId: 'google' }]) + mocks.credentialProviderMatchesService.mockReturnValue(true) + + await expect(authorize()).resolves.toMatchObject({ suppliedId: 'credential-1' }) + expect(mocks.authorizeCredentialUse).toHaveBeenCalledWith( + expect.anything(), + expect.objectContaining({ + credentialId: 'credential-1', + workspaceId: 'workspace-1', + }) + ) + }) + + it('promotes a hidden fixed token to an authentication secret at every length', async () => { + const protectedValues = createSelectorProtectedValues() + + await expect( + authorizeSelectorCredential({ + principal, + context: { oauthCredential: 'xoxb-a' }, + scope: { kind: 'workspace', workspaceId: 'workspace-1' }, + workspaceId: 'workspace-1', + policy: { + kind: 'stored-or-fixed-token', + field: 'oauthCredential', + serviceIds: ['slack'], + tokenPrefixes: ['xoxb-'], + }, + protectedValues, + references: new Map([ + [ + 'oauthCredential', + { + field: 'oauthCredential', + name: 'SLACK_BOT_TOKEN', + scope: 'workspace', + visible: false, + }, + ], + ]), + }) + ).resolves.toMatchObject({ fixedToken: 'xoxb-a' }) + + expect(protectedValues.contains('prefix-xoxb-a-suffix')).toBe(true) + expect(mocks.authorizeCredentialUse).not.toHaveBeenCalled() + }) + + it('conceals a stored credential whose trusted provider does not match the selector service', async () => { + mocks.authorizeCredentialUse.mockResolvedValue({ + ok: true, + workspaceId: 'workspace-1', + credentialOwnerUserId: 'owner-1', + resolvedCredentialId: 'credential-1', + }) + queueTableRows(credential, [{ accountId: 'account-1', providerId: 'microsoft' }]) + mocks.credentialProviderMatchesService.mockReturnValue(false) + + await expect(authorize()).rejects.toEqual(new SelectorConnectionUnavailableError()) + expect(mocks.credentialProviderMatchesService).toHaveBeenCalledWith('microsoft', { + id: 'gmail', + }) + }) +}) + +describe('resolveSelectorOAuthAccessToken', () => { + beforeEach(() => vi.clearAllMocks()) + + it('rejects only the canceled waiter while shared credential work serves another caller', async () => { + let resolveShared!: (value: { accessToken: string }) => void + const sharedResolution = new Promise<{ accessToken: string }>((resolve) => { + resolveShared = resolve + }) + mocks.resolveCredentialTokenBundle.mockReturnValue(sharedResolution) + const canceledController = new AbortController() + const liveController = new AbortController() + const canceledProtectedValues = createSelectorProtectedValues() + const liveProtectedValues = createSelectorProtectedValues() + const canceledRecordUse = vi.fn() + const liveRecordUse = vi.fn() + const access = { + ok: true as const, + credentialOwnerUserId: 'owner-1', + resolvedCredentialId: 'credential-1', + } + + const canceledWaiter = resolveSelectorOAuthAccessToken({ + credential: { + suppliedId: 'credential-1', + access, + signal: canceledController.signal, + }, + serviceId: 'gmail', + protectedValues: canceledProtectedValues, + recordCredentialUse: canceledRecordUse, + }) + const liveWaiter = resolveSelectorOAuthAccessToken({ + credential: { + suppliedId: 'credential-1', + access, + signal: liveController.signal, + }, + serviceId: 'gmail', + protectedValues: liveProtectedValues, + recordCredentialUse: liveRecordUse, + }) + + const abortReason = new DOMException('Selector request canceled', 'AbortError') + canceledController.abort(abortReason) + await expect(canceledWaiter).rejects.toBe(abortReason) + + resolveShared({ accessToken: 'shared-access-token' }) + await expect(liveWaiter).resolves.toBe('shared-access-token') + + expect(canceledRecordUse).not.toHaveBeenCalled() + expect(canceledProtectedValues.contains('shared-access-token')).toBe(false) + expect(liveRecordUse).toHaveBeenCalledOnce() + expect(liveProtectedValues.contains('shared-access-token')).toBe(true) + expect(mocks.resolveCredentialTokenBundle).toHaveBeenCalledTimes(2) + for (const call of mocks.resolveCredentialTokenBundle.mock.calls) { + expect(call[5]).toEqual({ privacyMode: 'selector' }) + expect(call).not.toContain(canceledController.signal) + expect(call).not.toContain(liveController.signal) + } + }) + + it('does not start credential resolution for an already canceled selector', async () => { + const controller = new AbortController() + const abortReason = new DOMException('Selector request canceled', 'AbortError') + controller.abort(abortReason) + + await expect( + resolveSelectorOAuthAccessToken({ + credential: { + suppliedId: 'credential-1', + access: { + ok: true, + credentialOwnerUserId: 'owner-1', + resolvedCredentialId: 'credential-1', + }, + signal: controller.signal, + }, + serviceId: 'gmail', + protectedValues: createSelectorProtectedValues(), + }) + ).rejects.toBe(abortReason) + + expect(mocks.resolveCredentialTokenBundle).not.toHaveBeenCalled() + }) + + it('rechecks cancellation before consuming a fulfilled credential result', async () => { + mocks.resolveCredentialTokenBundle.mockResolvedValue({ + accessToken: 'fulfilled-access-token', + }) + const controller = new AbortController() + const protectedValues = createSelectorProtectedValues() + const recordCredentialUse = vi.fn() + const abortReason = new DOMException('Selector request canceled', 'AbortError') + + const pending = resolveSelectorOAuthAccessToken({ + credential: { + suppliedId: 'credential-1', + access: { + ok: true, + credentialOwnerUserId: 'owner-1', + resolvedCredentialId: 'credential-1', + }, + signal: controller.signal, + }, + serviceId: 'gmail', + protectedValues, + recordCredentialUse, + }) + queueMicrotask(() => controller.abort(abortReason)) + + await expect(pending).rejects.toBe(abortReason) + expect(protectedValues.contains('fulfilled-access-token')).toBe(false) + expect(recordCredentialUse).not.toHaveBeenCalled() + }) +}) diff --git a/apps/sim/lib/selectors/server/credentials.ts b/apps/sim/lib/selectors/server/credentials.ts new file mode 100644 index 00000000000..2bc68c62275 --- /dev/null +++ b/apps/sim/lib/selectors/server/credentials.ts @@ -0,0 +1,183 @@ +import type { SessionPrincipal } from '@sim/auth/principal' +import { db } from '@sim/db' +import { account, credential } from '@sim/db/schema' +import { and, eq } from 'drizzle-orm' +import { + authorizeCredentialUseForAuth, + type CredentialAccessResult, +} from '@/lib/auth/credential-access' +import { AuthType } from '@/lib/auth/hybrid' +import { resolveCredentialTokenBundle } from '@/lib/oauth/credential-service' +import { credentialProviderMatchesService, getServiceConfigByServiceId } from '@/lib/oauth/utils' +import { SelectorConnectionUnavailableError } from '@/lib/selectors/server/errors' +import type { + AuthorizedSelectorCredential, + ResolvedSelectorReference, + SelectorCredentialPolicy, + SelectorProtectedValues, +} from '@/lib/selectors/server/types' +import type { SelectorContext, SelectorScope } from '@/lib/selectors/types' + +function selectorAbortReason(signal: AbortSignal): unknown { + return signal.reason ?? new DOMException('The operation was aborted.', 'AbortError') +} + +/** + * Makes one selector's wait abortable without attaching its signal to shared + * refresh or mint work that may still be serving other callers. + */ +export function waitForSelectorCredentialResolution( + resolution: Promise, + signal?: AbortSignal +): Promise { + if (!signal) return resolution + if (signal.aborted) return Promise.reject(selectorAbortReason(signal)) + + return new Promise((resolve, reject) => { + const onAbort = () => { + signal.removeEventListener('abort', onAbort) + reject(selectorAbortReason(signal)) + } + signal.addEventListener('abort', onAbort, { once: true }) + resolution.then( + (value) => { + signal.removeEventListener('abort', onAbort) + resolve(value) + }, + (error) => { + signal.removeEventListener('abort', onAbort) + reject(error) + } + ) + if (signal.aborted) onAbort() + }) +} + +async function resolveCredentialProviderId(input: { + credentialId: string + credentialOwnerUserId: string +}): Promise { + const [credentialRow] = await db + .select({ accountId: credential.accountId, providerId: credential.providerId }) + .from(credential) + .where(eq(credential.id, input.credentialId)) + .limit(1) + + let providerId = credentialRow?.providerId ?? null + const accountId = credentialRow?.accountId ?? input.credentialId + if (!providerId) { + const [accountRow] = await db + .select({ providerId: account.providerId }) + .from(account) + .where(and(eq(account.id, accountId), eq(account.userId, input.credentialOwnerUserId))) + .limit(1) + providerId = accountRow?.providerId ?? null + } + + return providerId +} + +async function requireCredentialProviderBinding( + credentialId: string, + access: CredentialAccessResult, + serviceIds: readonly string[] +): Promise { + if (!access.credentialOwnerUserId) throw new SelectorConnectionUnavailableError() + const providerId = await resolveCredentialProviderId({ + credentialId, + credentialOwnerUserId: access.credentialOwnerUserId, + }) + if (!providerId) throw new SelectorConnectionUnavailableError() + for (const serviceId of serviceIds) { + const service = getServiceConfigByServiceId(serviceId) + if (service && credentialProviderMatchesService(providerId, service)) return providerId + } + throw new SelectorConnectionUnavailableError() +} + +export async function authorizeSelectorCredential(input: { + principal: SessionPrincipal + context: SelectorContext + scope: SelectorScope + workspaceId: string + policy: SelectorCredentialPolicy + protectedValues: SelectorProtectedValues + references: ReadonlyMap +}): Promise { + const suppliedId = input.context[input.policy.field] + if (!suppliedId) throw new SelectorConnectionUnavailableError() + + if ( + input.policy.kind === 'stored-or-fixed-token' && + input.policy.tokenPrefixes.some((prefix) => suppliedId.startsWith(prefix)) + ) { + const reference = input.references.get(input.policy.field) + if (reference && !reference.visible) { + input.protectedValues.add(suppliedId, 'secret') + } + return { suppliedId, fixedToken: suppliedId } + } + + const access = await authorizeCredentialUseForAuth( + { + success: true, + userId: input.principal.userId, + authType: AuthType.SESSION, + }, + { + credentialId: suppliedId, + ...(input.scope.kind === 'workflow' ? { workflowId: input.scope.workflowId } : {}), + ...(input.scope.kind === 'workspace' ? { workspaceId: input.workspaceId } : {}), + } + ) + if (!access.ok || access.workspaceId !== input.workspaceId) { + throw new SelectorConnectionUnavailableError() + } + input.protectedValues.add(access.resolvedCredentialId, 'reference') + + const providerId = await requireCredentialProviderBinding( + suppliedId, + access, + input.policy.serviceIds + ) + return { suppliedId, access, providerId } +} + +export async function resolveSelectorOAuthAccessToken(input: { + credential: AuthorizedSelectorCredential + serviceId: string + scopes?: readonly string[] + impersonateEmail?: string + protectedValues: SelectorProtectedValues + recordCredentialUse?: (providerId: string) => void +}): Promise { + input.credential.signal?.throwIfAborted() + if (input.credential.fixedToken) return input.credential.fixedToken + + const access = input.credential.access + if (!access?.credentialOwnerUserId || !access.resolvedCredentialId) { + throw new SelectorConnectionUnavailableError() + } + + const result = await waitForSelectorCredentialResolution( + resolveCredentialTokenBundle( + input.credential.suppliedId, + access.credentialOwnerUserId, + 'selector-execution', + input.scopes ? [...input.scopes] : undefined, + input.impersonateEmail, + { privacyMode: 'selector' } + ), + input.credential.signal + ) + input.credential.signal?.throwIfAborted() + const token = result?.accessToken + + if (!token) throw new SelectorConnectionUnavailableError() + input.protectedValues.add(token) + input.protectedValues.add(result.domain, 'reference') + input.protectedValues.add(result.instanceUrl, 'reference') + input.protectedValues.add(result.apiDomain, 'reference') + input.recordCredentialUse?.(input.credential.providerId ?? input.serviceId) + return token +} diff --git a/apps/sim/lib/selectors/server/errors.ts b/apps/sim/lib/selectors/server/errors.ts new file mode 100644 index 00000000000..6ebbe841d73 --- /dev/null +++ b/apps/sim/lib/selectors/server/errors.ts @@ -0,0 +1,26 @@ +export class SelectorContextUnavailableError extends Error { + constructor() { + super('Context unavailable') + this.name = 'SelectorContextUnavailableError' + } +} + +export class SelectorConnectionUnavailableError extends Error { + readonly status: 401 | 403 + + constructor(status: 401 | 403 = 403) { + super('Connection unavailable') + this.name = 'SelectorConnectionUnavailableError' + this.status = status + } +} + +export class SelectorOptionsUnavailableError extends Error { + readonly status: 429 | 502 + + constructor(status: 429 | 502 = 502) { + super('Options unavailable') + this.name = 'SelectorOptionsUnavailableError' + this.status = status + } +} diff --git a/apps/sim/lib/selectors/server/integration-access.test.ts b/apps/sim/lib/selectors/server/integration-access.test.ts new file mode 100644 index 00000000000..82aa9da94d2 --- /dev/null +++ b/apps/sim/lib/selectors/server/integration-access.test.ts @@ -0,0 +1,69 @@ +/** + * @vitest-environment node + */ +import { describe, expect, it } from 'vitest' +import { selectorManifest } from '@/lib/selectors/manifest' +import { selectorIntegrationBlockTypes } from '@/lib/selectors/server/integration-access' +import { serverSelectorRegistry } from '@/lib/selectors/server/registry' + +describe('selectorIntegrationBlockTypes', () => { + /** + * The gate passes a selector with no integration identity, so an identity it + * cannot derive is a silent hole: `POST /api/selectors/execute` reaches the + * third party with the caller's credentials and the group's + * `allowedIntegrations` never gets a say. Every selector the manifest calls + * `provider-server` must therefore resolve to at least one block type, either + * through the OAuth credential catalog or by declaring one. + */ + it('gives every provider selector an integration identity', () => { + const ungated = Object.entries(serverSelectorRegistry) + .filter(([key]) => selectorManifest[key as keyof typeof selectorManifest]) + .filter( + ([key, attachment]) => + selectorManifest[key as keyof typeof selectorManifest].classification === + 'provider-server' && selectorIntegrationBlockTypes(attachment).length === 0 + ) + .map(([key]) => key) + + expect(ungated).toEqual([]) + }) + + /** + * The other half of the same rule: an internal selector reads Sim's own + * workspace data, so it is not an integration and nothing gates it. + */ + it('gives an internal selector no integration identity', () => { + const internal = Object.entries(serverSelectorRegistry).filter( + ([key]) => + selectorManifest[key as keyof typeof selectorManifest]?.classification === 'internal-server' + ) + + expect(internal.length).toBeGreaterThan(0) + for (const [key, attachment] of internal) { + expect([key, selectorIntegrationBlockTypes(attachment)]).toEqual([key, []]) + } + }) + + /** A declaration wins over the catalog, which is what an API-key selector needs. */ + it('prefers a declared block type over the credential catalog', () => { + expect( + selectorIntegrationBlockTypes({ + credential: { kind: 'stored', field: 'oauthCredential', serviceIds: ['gmail'] }, + integrationBlockTypes: ['snowflake'], + }) + ).toEqual(['snowflake']) + }) + + it('derives the block type from the credential resource when none is declared', () => { + expect( + selectorIntegrationBlockTypes({ + credential: { + kind: 'stored', + field: 'oauthCredential', + serviceIds: ['google-drive', 'google-sheets'], + resourceServiceId: 'google-drive', + }, + }) + ).toContain('google_drive') + }) +}) diff --git a/apps/sim/lib/selectors/server/integration-access.ts b/apps/sim/lib/selectors/server/integration-access.ts new file mode 100644 index 00000000000..a1fc5a513cb --- /dev/null +++ b/apps/sim/lib/selectors/server/integration-access.ts @@ -0,0 +1,106 @@ +import type { Principal } from '@sim/auth/principal' +import { getIntegrationTypesForOAuthServiceId } from '@sim/deployment-config/integration-availability' +import { createLogger } from '@sim/logger' +import { allowedIntegrationTypes } from '@/lib/integrations/principal-scope.server' +import { isBlockTypeAccessControlExempt } from '@/lib/permission-groups/block-access' +import { resolveAccessControlBlockType } from '@/lib/permission-groups/integration-allowlist' +import type { + SelectorCredentialPolicy, + ServerSelectorAttachment, +} from '@/lib/selectors/server/types' +import { IntegrationNotAllowedError } from '@/ee/access-control/utils/permission-check' + +const logger = createLogger('SelectorIntegrationAccess') + +/** + * The OAuth service a selector execution actually reaches — its own resource + * rather than the set of credentials it accepts. See + * {@link SelectorCredentialPolicy} for why those two differ and why the bound + * credential's provider id is never consulted. + */ +function selectorResourceServiceIds(policy: SelectorCredentialPolicy): readonly string[] { + return policy.resourceServiceId ? [policy.resourceServiceId] : policy.serviceIds +} + +/** + * The block types an allowlist decision about this selector is made against. + * + * Two independent sources, because the OAuth credential catalog cannot identify + * every selector that reaches a third-party API; the declared + * `integrationBlockTypes` cover the shapes it misses and win over the catalog + * when both are present. See + * {@link ServerSelectorAttachment.integrationBlockTypes} for which shapes those + * are. + * + * An empty result means "no integration identity", which is a pass. That is + * reserved for the internal selectors — workspace files, knowledge bases, + * tables — which read only Sim's own data. `integration-access.test.ts` keeps + * every provider selector out of it: "gives every provider selector an + * integration identity" walks the whole manifest and fails on the first + * provider-backed selector this function answers with an empty list. + */ +export function selectorIntegrationBlockTypes( + attachment: Pick +): readonly string[] { + if (attachment.integrationBlockTypes?.length) return attachment.integrationBlockTypes + if (!attachment.credential) return [] + return selectorResourceServiceIds(attachment.credential).flatMap((serviceId) => + getIntegrationTypesForOAuthServiceId(serviceId) + ) +} + +/** + * Refuses a selector execution whose integration the caller's permission group + * does not permit. + * + * `POST /api/selectors/execute` reaches a provider's API with the caller's + * credential, so it is a use of the integration and not merely a picker. The + * authorization funnel cannot apply the rule: `allowedIntegrations` is a + * parameterized decision about *which* integration, and the funnel knows only + * the principal, the workspace and the operation. Hence the assertion here, + * ahead of the provider call, exactly as `knowledge.connectors` is asserted + * ahead of the connector write. + * + * The decision is the one the block-access path makes. `allowedIntegrationTypes` + * is the shared gate — it intersects the caller's permission group with the + * deployment's `ALLOWED_INTEGRATIONS`, contributes no group half for a principal + * that stands for no person, and canonicalizes each half through + * `resolveAccessControlBlockType` *before* intersecting, so a group naming + * `slack_v2` and a deployment naming `slack` still meet. The checked side is + * successor-resolved the same way, so a group naming `slack` and a selector + * bound to `slack_v2` match. + * + * A `null` allowlist, a caller no group governs, and a selector with no + * integration identity all pass through; see {@link selectorIntegrationBlockTypes} + * for why the last of those is reserved for the internal selectors. + * + * One service can still map to several block types — the `google-drive` entry + * authenticates both `google_drive` and `google_slides_v2` — and any of them + * satisfies the check. That is the catalog's own shared-service convention and + * not a widening: both block types hold the same Drive scope on the same + * credential, so permitting either already grants the access. + */ +export async function assertSelectorIntegrationAllowed(input: { + principal: Principal + workspaceId: string + blockTypes: readonly string[] +}): Promise { + const blockTypes = input.blockTypes + if (blockTypes.length === 0) return + + const allowlist = await allowedIntegrationTypes(input.principal, input.workspaceId) + if (allowlist === null) return + + const allowed = blockTypes.some( + (blockType) => + isBlockTypeAccessControlExempt(blockType) || + allowlist.has(resolveAccessControlBlockType(blockType).toLowerCase()) + ) + if (allowed) return + + logger.warn('Selector integration blocked by integration allowlist', { + workspaceId: input.workspaceId, + blockTypes, + }) + throw new IntegrationNotAllowedError(blockTypes[0]) +} diff --git a/apps/sim/lib/selectors/server/internal.test.ts b/apps/sim/lib/selectors/server/internal.test.ts new file mode 100644 index 00000000000..f085c147afe --- /dev/null +++ b/apps/sim/lib/selectors/server/internal.test.ts @@ -0,0 +1,152 @@ +/** + * @vitest-environment node + */ +import { environmentUtilsMockFns, resetEnvironmentUtilsMock } from '@sim/testing' +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const mockListWorkflows = vi.hoisted(() => vi.fn()) +const mockFetchOpenRouterEmbeddingModelCatalog = vi.hoisted(() => vi.fn()) + +vi.mock('@/lib/workflows/application/list-workflows', () => ({ + listWorkflows: { execute: mockListWorkflows }, +})) + +vi.mock('@/lib/embeddings/openrouter-model-catalog.server', () => ({ + fetchOpenRouterEmbeddingModelCatalog: mockFetchOpenRouterEmbeddingModelCatalog, +})) + +import { SelectorOptionsUnavailableError } from '@/lib/selectors/server/errors' +import { internalSelectorAttachments } from '@/lib/selectors/server/internal' +import { createSelectorProtectedValues } from '@/lib/selectors/server/protected-values' +import type { ExecuteServerSelectorArgs } from '@/lib/selectors/server/types' + +function workflowArgs(): ExecuteServerSelectorArgs { + return { + selectorKey: 'sim.workflows', + context: {}, + request: { kind: 'list' }, + scope: { kind: 'workspace', workspaceId: 'workspace-1' }, + workspaceId: 'workspace-1', + principal: { kind: 'session', userId: 'user-1', sessionId: 'session-1' }, + requesterUserId: 'user-1', + references: new Map(), + protectedValues: createSelectorProtectedValues(), + } +} + +describe('workspace.secretNames selector', () => { + beforeEach(() => { + vi.clearAllMocks() + resetEnvironmentUtilsMock() + }) + + it('returns the ACL-filtered names without loading the decrypted environment snapshot', async () => { + environmentUtilsMockFns.mockGetEffectiveEnvironmentVariableNames.mockResolvedValue([ + 'PERSONAL_KEY', + 'SHARED_KEY', + ]) + + await expect( + internalSelectorAttachments['workspace.secretNames'].execute({ + selectorKey: 'workspace.secretNames', + context: {}, + request: { kind: 'list' }, + scope: { kind: 'workspace', workspaceId: 'workspace-1' }, + workspaceId: 'workspace-1', + principal: { kind: 'session', userId: 'user-1', sessionId: 'session-1' }, + requesterUserId: 'user-1', + references: new Map(), + protectedValues: createSelectorProtectedValues(), + }) + ).resolves.toEqual({ + kind: 'list', + items: [ + { id: 'PERSONAL_KEY', label: 'PERSONAL_KEY' }, + { id: 'SHARED_KEY', label: 'SHARED_KEY' }, + ], + }) + + expect(environmentUtilsMockFns.mockGetEffectiveEnvironmentVariableNames).toHaveBeenCalledWith( + 'user-1', + 'workspace-1' + ) + expect(environmentUtilsMockFns.mockGetEffectiveEnvironmentSnapshot).not.toHaveBeenCalled() + }) +}) + +describe('sim.workflows selector', () => { + beforeEach(() => { + vi.clearAllMocks() + }) + + it('continues beyond the former 5,000-workflow limit', async () => { + for (let page = 0; page < 20; page += 1) { + mockListWorkflows.mockResolvedValueOnce({ + workflows: [], + nextCursorKeys: [`page-${page + 1}`], + }) + } + mockListWorkflows.mockResolvedValueOnce({ + workflows: [{ id: 'workflow-late', name: 'Late workflow', folderPath: '/' }], + nextCursorKeys: null, + }) + + await expect( + internalSelectorAttachments['sim.workflows'].execute(workflowArgs()) + ).resolves.toEqual({ + kind: 'list', + items: [{ id: 'workflow-late', label: 'Late workflow' }], + }) + expect(mockListWorkflows).toHaveBeenCalledTimes(21) + }) + + it('fails rather than returning a partial list after 10,000 workflows', async () => { + const workflowPage = Array.from({ length: 250 }, (_, index) => ({ + id: `workflow-${index}`, + name: `Workflow ${index}`, + folderPath: '/', + })) + mockListWorkflows.mockResolvedValue({ + workflows: workflowPage, + nextCursorKeys: ['more'], + }) + + await expect( + internalSelectorAttachments['sim.workflows'].execute(workflowArgs()) + ).rejects.toBeInstanceOf(SelectorOptionsUnavailableError) + expect(mockListWorkflows).toHaveBeenCalledTimes(40) + }) +}) + +describe('providers.openrouterEmbeddingModels selector', () => { + beforeEach(() => { + vi.clearAllMocks() + resetEnvironmentUtilsMock() + }) + + it('passes the selector signal to the OpenRouter catalog fetch', async () => { + const controller = new AbortController() + mockFetchOpenRouterEmbeddingModelCatalog.mockResolvedValue([ + { id: 'openai/text-embedding-3-small', maxInputTokens: 8_191 }, + ]) + + await expect( + internalSelectorAttachments['providers.openrouterEmbeddingModels'].execute({ + ...workflowArgs(), + selectorKey: 'providers.openrouterEmbeddingModels', + signal: controller.signal, + }) + ).resolves.toEqual({ + kind: 'list', + items: [ + { + id: 'openai/text-embedding-3-small', + label: 'openai/text-embedding-3-small', + }, + ], + }) + + expect(mockFetchOpenRouterEmbeddingModelCatalog).toHaveBeenCalledOnce() + expect(mockFetchOpenRouterEmbeddingModelCatalog).toHaveBeenCalledWith(controller.signal) + }) +}) diff --git a/apps/sim/lib/selectors/server/internal.ts b/apps/sim/lib/selectors/server/internal.ts new file mode 100644 index 00000000000..4a5241cdf27 --- /dev/null +++ b/apps/sim/lib/selectors/server/internal.ts @@ -0,0 +1,309 @@ +import { listCredentialGroupSettings } from '@/lib/credential-groups/application/manage-groups' +import { getCredentialGroupProviderService } from '@/lib/credential-groups/providers' +import { listInternalCredentials } from '@/lib/credentials/application/credential-crud' +import { fetchOpenRouterEmbeddingModelCatalog } from '@/lib/embeddings/openrouter-model-catalog.server' +import { getEffectiveEnvironmentVariableNames } from '@/lib/environment/utils' +import { listWorkspaceSandboxes } from '@/lib/execution/remote-sandbox/workspace-sandboxes' +import { + listKnowledgeDocuments, + readKnowledgeDocument, +} from '@/lib/knowledge/application/documents' +import { getServiceConfigByProviderId } from '@/lib/oauth/utils' +import type { InternalSelectorKey } from '@/lib/selectors/manifest' +import { SelectorOptionsUnavailableError } from '@/lib/selectors/server/errors' +import { + detailSelectorResult, + type ExecuteServerSelectorArgs, + listSelectorResult, + type ServerSelectorAttachmentMap, +} from '@/lib/selectors/server/types' +import { readTableUseCase } from '@/lib/table/application/tables' +import { getColumnId } from '@/lib/table/column-keys' +import { listWorkflows } from '@/lib/workflows/application/list-workflows' +import { filterBlacklistedModels, isProviderBlacklisted } from '@/providers/utils' + +const WORKFLOW_PAGE_SIZE = 250 +const MAX_WORKFLOWS = 10_000 +const MAX_WORKFLOW_PAGES = MAX_WORKFLOWS / WORKFLOW_PAGE_SIZE +const KNOWLEDGE_PAGE_SIZE = 100 + +function labelWorkflow( + workflow: { id: string; name: string | null; folderPath: string }, + duplicateNames: ReadonlySet +): string { + const base = workflow.name || `Workflow ${workflow.id.slice(0, 8)}` + if (!duplicateNames.has(base)) return base + const folder = + workflow.folderPath === '/' ? 'Root' : workflow.folderPath.slice(1).replaceAll('/', ' / ') + return `${base} (${folder})` +} + +async function loadWorkflows( + args: Parameters<(typeof listWorkflows)['execute']>[0]['principal'], + workspaceId: string +) { + const workflows: Array< + Awaited>['workflows'][number] + > = [] + let cursorKeys: Awaited>['nextCursorKeys'] = null + for (let page = 0; page < MAX_WORKFLOW_PAGES; page += 1) { + const result = await listWorkflows.execute({ + principal: args, + input: { + workspaceId, + scope: 'active', + deployedOnly: false, + sortBy: 'updatedAt', + sortOrder: 'desc', + limit: WORKFLOW_PAGE_SIZE, + ...(cursorKeys ? { cursorKeys } : {}), + }, + }) + workflows.push(...result.workflows) + cursorKeys = result.nextCursorKeys + if (!cursorKeys) break + } + if (cursorKeys) throw new SelectorOptionsUnavailableError() + return workflows +} + +async function loadCredentialGroups( + principal: Parameters<(typeof listCredentialGroupSettings)['execute']>[0]['principal'], + workspaceId: string +) { + return (await listCredentialGroupSettings.execute({ principal, input: { workspaceId } })) + .credentialGroups +} + +export const internalSelectorAttachments = { + 'knowledge.documents': { + destination: 'fixed', + async execute(args: ExecuteServerSelectorArgs) { + const knowledgeBaseId = args.context.knowledgeBaseId! + if (args.request.kind === 'detail') { + const result = await readKnowledgeDocument.execute({ + principal: args.principal, + input: { + knowledgeBaseId, + documentId: args.request.id, + assertedWorkspaceId: args.workspaceId, + }, + }) + return detailSelectorResult({ + id: result.document.id, + label: result.document.filename, + }) + } + + const offset = args.request.cursor ? Number(args.request.cursor) : 0 + if (!Number.isSafeInteger(offset) || offset < 0) throw new Error('Invalid selector cursor') + const result = await listKnowledgeDocuments.execute({ + principal: args.principal, + input: { + knowledgeBaseId, + assertedWorkspaceId: args.workspaceId, + enabledFilter: 'all', + search: args.request.search, + limit: KNOWLEDGE_PAGE_SIZE, + offset, + sortBy: 'filename', + sortOrder: 'asc', + }, + }) + const nextOffset = result.pagination.offset + result.pagination.limit + return listSelectorResult( + result.documents.map((document) => ({ id: document.id, label: document.filename })), + result.pagination.hasMore ? String(nextOffset) : undefined + ) + }, + }, + 'sim.workflows': { + destination: 'fixed', + async execute(args: ExecuteServerSelectorArgs) { + const workflows = (await loadWorkflows(args.principal, args.workspaceId)).filter( + (workflow) => workflow.id !== args.context.excludeWorkflowId + ) + const names = workflows.map( + (workflow) => workflow.name || `Workflow ${workflow.id.slice(0, 8)}` + ) + const seen = new Set() + const duplicates = new Set() + for (const name of names) { + if (seen.has(name)) duplicates.add(name) + seen.add(name) + } + const options = workflows + .map((workflow) => ({ + id: workflow.id, + label: labelWorkflow(workflow, duplicates), + })) + .sort((left, right) => left.label.localeCompare(right.label)) + if (args.request.kind === 'detail') { + const detailId = args.request.id + return detailSelectorResult(options.find((option) => option.id === detailId) ?? null) + } + return listSelectorResult(options) + }, + }, + 'table.columns': { + destination: 'fixed', + async execute(args: ExecuteServerSelectorArgs) { + const { table } = await readTableUseCase.execute({ + principal: args.principal, + input: { tableId: args.context.tableId!, workspaceId: args.workspaceId }, + }) + const options = (table.schema?.columns ?? []) + .filter((column) => column.unique) + .map((column) => ({ id: getColumnId(column), label: column.name })) + if (args.request.kind === 'detail') { + const detailId = args.request.id + return detailSelectorResult(options.find((option) => option.id === detailId) ?? null) + } + return listSelectorResult(options) + }, + }, + 'table.outputColumns': { + destination: 'fixed', + async execute(args: ExecuteServerSelectorArgs) { + const { table } = await readTableUseCase.execute({ + principal: args.principal, + input: { tableId: args.context.tableId!, workspaceId: args.workspaceId }, + }) + const options = (table.schema?.columns ?? []).map((column) => ({ + id: getColumnId(column), + label: column.name, + })) + if (args.request.kind === 'detail') { + const detailId = args.request.id + return detailSelectorResult(options.find((option) => option.id === detailId) ?? null) + } + return listSelectorResult(options) + }, + }, + 'workspace.credentialProviders': { + destination: 'fixed', + async execute(args: ExecuteServerSelectorArgs) { + const result = await listInternalCredentials.execute({ + principal: args.principal, + input: { workspaceId: args.workspaceId, type: 'oauth' }, + }) + if (result.mode !== 'list') throw new Error('Unexpected credential lookup result') + const seen = new Set() + const options = result.credentials + .flatMap((credential) => { + if (!credential.providerId || seen.has(credential.providerId)) return [] + seen.add(credential.providerId) + const service = getServiceConfigByProviderId(credential.providerId) + return [{ id: credential.providerId, label: service?.name ?? credential.providerId }] + }) + .sort((left, right) => left.label.localeCompare(right.label)) + if (args.request.kind === 'detail') { + const detailId = args.request.id + return detailSelectorResult( + options.find((option) => option.id === detailId) ?? { + id: detailId, + label: getServiceConfigByProviderId(detailId)?.name ?? detailId, + } + ) + } + return listSelectorResult(options) + }, + }, + 'workspace.credentialGroups': { + destination: 'fixed', + async execute(args: ExecuteServerSelectorArgs) { + const options = (await loadCredentialGroups(args.principal, args.workspaceId)) + .filter((group) => group.status === 'active') + .map((group) => ({ id: group.id, label: group.name })) + .sort((left, right) => left.label.localeCompare(right.label)) + if (args.request.kind === 'detail') { + const detailId = args.request.id + return detailSelectorResult(options.find((option) => option.id === detailId) ?? null) + } + return listSelectorResult(options) + }, + }, + 'workspace.credentialGroupProviders': { + destination: 'fixed', + async execute(args: ExecuteServerSelectorArgs) { + const group = (await loadCredentialGroups(args.principal, args.workspaceId)).find( + (candidate) => candidate.id === args.context.credentialGroupId + ) + const options = (group?.options ?? []) + .filter((option) => option.status === 'active') + .map((option) => { + const service = getCredentialGroupProviderService(option.provider) + return { id: service.providerId, label: service.name } + }) + .sort((left, right) => left.label.localeCompare(right.label)) + if (args.request.kind === 'detail') { + const detailId = args.request.id + return detailSelectorResult(options.find((option) => option.id === detailId) ?? null) + } + return listSelectorResult(options) + }, + }, + 'workspace.secretNames': { + destination: 'fixed', + async execute(args: ExecuteServerSelectorArgs) { + const names = await getEffectiveEnvironmentVariableNames( + args.requesterUserId, + args.workspaceId + ) + return listSelectorResult(names.map((name) => ({ id: name, label: name }))) + }, + }, + 'workspace.rawSecretNames': { + destination: 'fixed', + async execute(args: ExecuteServerSelectorArgs) { + const result = await listInternalCredentials.execute({ + principal: args.principal, + input: { workspaceId: args.workspaceId }, + }) + if (result.mode !== 'list') throw new Error('Unexpected credential lookup result') + const names = new Set( + result.credentials.flatMap((credential) => + (credential.type === 'env_workspace' || credential.type === 'env_personal') && + credential.role === 'admin' && + credential.envKey + ? [credential.envKey] + : [] + ) + ) + return listSelectorResult([...names].sort().map((name) => ({ id: name, label: name }))) + }, + }, + 'workspace.sandboxes': { + destination: 'fixed', + async execute(args: ExecuteServerSelectorArgs) { + const sandboxes = await listWorkspaceSandboxes(args.workspaceId) + const language = args.context.language + if (args.request.kind === 'detail') { + const detailId = args.request.id + const sandbox = sandboxes.find((candidate) => candidate.id === detailId) + if (!sandbox) return detailSelectorResult(null) + const wrongLanguage = + (language === 'python' || language === 'javascript') && sandbox.language !== language + return detailSelectorResult({ + id: sandbox.id, + label: wrongLanguage ? `${sandbox.name} · wrong language for this block` : sandbox.name, + }) + } + return listSelectorResult( + sandboxes + .filter((sandbox) => !language || language === 'shell' || sandbox.language === language) + .map((sandbox) => ({ id: sandbox.id, label: sandbox.name })) + ) + }, + }, + 'providers.openrouterEmbeddingModels': { + destination: 'fixed', + async execute(args: ExecuteServerSelectorArgs) { + if (isProviderBlacklisted('openrouter')) return listSelectorResult([]) + const models = filterBlacklistedModels( + (await fetchOpenRouterEmbeddingModelCatalog(args.signal)).map((model) => model.id) + ) + return listSelectorResult([...new Set(models)].map((model) => ({ id: model, label: model }))) + }, + }, +} as const satisfies ServerSelectorAttachmentMap diff --git a/apps/sim/lib/selectors/server/option-budget.test.ts b/apps/sim/lib/selectors/server/option-budget.test.ts new file mode 100644 index 00000000000..26dae102c29 --- /dev/null +++ b/apps/sim/lib/selectors/server/option-budget.test.ts @@ -0,0 +1,36 @@ +/** + * @vitest-environment node + */ +import { describe, expect, it } from 'vitest' +import { MAX_SELECTOR_OPTIONS } from '@/lib/selectors/limits' +import { appendSelectorOptions } from '@/lib/selectors/server/option-budget' +import { listSelectorResult } from '@/lib/selectors/server/types' + +describe('selector option budget', () => { + it('appends only the entries that fit', () => { + const target = [1, 2] + + expect(appendSelectorOptions(target, [3, 4, 5], 4)).toEqual({ + full: true, + overflow: true, + }) + expect(target).toEqual([1, 2, 3, 4]) + }) + + it('defensively returns a bounded partial list with diagnostics', () => { + const result = listSelectorResult( + Array.from({ length: MAX_SELECTOR_OPTIONS + 1 }, (_, index) => ({ + id: String(index), + label: `Option ${index}`, + })) + ) + + expect(result).toMatchObject({ + kind: 'list', + diagnostics: { + truncated: { reason: 'provider-cap', limit: MAX_SELECTOR_OPTIONS }, + }, + }) + expect(result.kind === 'list' ? result.items : []).toHaveLength(MAX_SELECTOR_OPTIONS) + }) +}) diff --git a/apps/sim/lib/selectors/server/option-budget.ts b/apps/sim/lib/selectors/server/option-budget.ts new file mode 100644 index 00000000000..40d090e4958 --- /dev/null +++ b/apps/sim/lib/selectors/server/option-budget.ts @@ -0,0 +1,23 @@ +import { MAX_SELECTOR_OPTIONS } from '@/lib/selectors/limits' + +export interface SelectorOptionBudgetAppendResult { + full: boolean + overflow: boolean +} + +/** Appends only the options that fit in the selector response budget. */ +export function appendSelectorOptions( + target: T[], + incoming: readonly T[], + limit = MAX_SELECTOR_OPTIONS +): SelectorOptionBudgetAppendResult { + const remaining = Math.max(0, limit - target.length) + const accepted = Math.min(remaining, incoming.length) + for (let index = 0; index < accepted; index++) { + target.push(incoming[index]) + } + return { + full: target.length >= limit, + overflow: incoming.length > accepted, + } +} diff --git a/apps/sim/lib/selectors/server/protected-values.ts b/apps/sim/lib/selectors/server/protected-values.ts new file mode 100644 index 00000000000..db0526be574 --- /dev/null +++ b/apps/sim/lib/selectors/server/protected-values.ts @@ -0,0 +1,33 @@ +import type { + SelectorProtectedValueKind, + SelectorProtectedValues, +} from '@/lib/selectors/server/types' +import { isNonIdentifyingSecretLiteral } from '@/executor/utils/resolved-secret-match-policy' + +export function createSelectorProtectedValues(): SelectorProtectedValues { + const values = new Map() + + function contains(value: string, allowedExactValue?: string): boolean { + for (const [protectedValue, kind] of values) { + if (value === protectedValue) { + if (protectedValue !== allowedExactValue) return true + continue + } + if (kind === 'secret' || !isNonIdentifyingSecretLiteral(protectedValue)) { + if (value.includes(protectedValue)) return true + } + } + return false + } + + return { + add(value, kind = 'secret') { + if (!value) return + const current = values.get(value) + if (current === 'secret') return + values.set(value, kind) + }, + contains: (value) => contains(value), + containsExceptExact: (value, allowedExactValue) => contains(value, allowedExactValue), + } +} diff --git a/apps/sim/lib/selectors/server/providers/airtable.ts b/apps/sim/lib/selectors/server/providers/airtable.ts new file mode 100644 index 00000000000..23dfa9cee43 --- /dev/null +++ b/apps/sim/lib/selectors/server/providers/airtable.ts @@ -0,0 +1,148 @@ +import { z } from 'zod' +import { MAX_SELECTOR_OPTIONS } from '@/lib/selectors/limits' +import type { ServerSelectorKey } from '@/lib/selectors/manifest' +import { resolveSelectorOAuthAccessToken } from '@/lib/selectors/server/credentials' +import { + SelectorConnectionUnavailableError, + SelectorContextUnavailableError, + SelectorOptionsUnavailableError, +} from '@/lib/selectors/server/errors' +import { appendSelectorOptions } from '@/lib/selectors/server/option-budget' +import { flatSelectorResult } from '@/lib/selectors/server/providers/flat-results' +import { fetchProviderJson } from '@/lib/selectors/server/providers/provider-http' +import type { + ExecuteServerSelectorArgs, + ServerSelectorAttachmentMap, +} from '@/lib/selectors/server/types' + +type AirtableSelectorKey = Extract + +const AIRTABLE_MAX_BASE_PAGES = 50 +const AIRTABLE_BASES_URL = 'https://api.airtable.com/v0/meta/bases' + +const airtableBaseSchema = z.object({ + id: z.string().min(1), + name: z.string().min(1), +}) + +const airtableBasesPageSchema = z.object({ + bases: z.array(airtableBaseSchema).max(1_000).optional(), + offset: z.string().min(1).max(4_096).optional(), +}) + +const airtableTablesResponseSchema = z.object({ + tables: z.array(airtableBaseSchema).max(10_000).optional(), +}) + +function requireCredential(args: ExecuteServerSelectorArgs) { + if (!args.credential) throw new SelectorConnectionUnavailableError() + return args.credential +} + +async function getAccessToken(args: ExecuteServerSelectorArgs): Promise { + return resolveSelectorOAuthAccessToken({ + credential: requireCredential(args), + serviceId: 'airtable', + protectedValues: args.protectedValues, + }) +} + +async function listBases(args: ExecuteServerSelectorArgs, accessToken: string) { + const bases: z.infer[] = [] + let offset: string | undefined + let truncated = false + + for (let page = 0; page < AIRTABLE_MAX_BASE_PAGES; page++) { + const url = new URL(AIRTABLE_BASES_URL) + if (offset) url.searchParams.set('offset', offset) + + const body = await fetchProviderJson(url, { + headers: { + Authorization: `Bearer ${accessToken}`, + Accept: 'application/json', + }, + redirect: 'error', + signal: args.signal, + }) + const parsed = airtableBasesPageSchema.safeParse(body) + if (!parsed.success) throw new SelectorOptionsUnavailableError() + + const appended = appendSelectorOptions(bases, parsed.data.bases ?? []) + offset = parsed.data.offset + if (!offset) { + if (appended.overflow) truncated = true + break + } + if (appended.full || page === AIRTABLE_MAX_BASE_PAGES - 1) { + truncated = true + break + } + } + + return { + items: bases.map((base) => ({ id: base.id, label: base.name })), + truncated, + } +} + +async function listTables(args: ExecuteServerSelectorArgs) { + const baseId = args.context.baseId + if (!baseId || !/^app[A-Za-z0-9]{14}$/.test(baseId)) { + throw new SelectorContextUnavailableError() + } + const accessToken = await getAccessToken(args) + + const body = await fetchProviderJson( + `https://api.airtable.com/v0/meta/bases/${encodeURIComponent(baseId)}/tables`, + { + headers: { + Authorization: `Bearer ${accessToken}`, + Accept: 'application/json', + }, + redirect: 'error', + signal: args.signal, + } + ) + const parsed = airtableTablesResponseSchema.safeParse(body) + if (!parsed.success) throw new SelectorOptionsUnavailableError() + + return (parsed.data.tables ?? []).map((table) => ({ id: table.id, label: table.name })) +} + +const credential = { + kind: 'stored', + field: 'oauthCredential', + serviceIds: ['airtable'], +} as const + +export const airtableSelectorAttachments = { + 'airtable.bases': { + credential, + destination: 'fixed', + async execute(args) { + const accessToken = await getAccessToken(args) + const { items, truncated } = await listBases(args, accessToken) + return flatSelectorResult( + args.request, + items, + true, + truncated + ? { + truncated: { + reason: 'provider-cap', + limit: MAX_SELECTOR_OPTIONS, + pages: AIRTABLE_MAX_BASE_PAGES, + }, + } + : undefined + ) + }, + }, + 'airtable.tables': { + credential, + destination: 'fixed', + async execute(args) { + return flatSelectorResult(args.request, await listTables(args), true) + }, + }, +} satisfies ServerSelectorAttachmentMap diff --git a/apps/sim/lib/selectors/server/providers/asana.ts b/apps/sim/lib/selectors/server/providers/asana.ts new file mode 100644 index 00000000000..e325f9eb037 --- /dev/null +++ b/apps/sim/lib/selectors/server/providers/asana.ts @@ -0,0 +1,95 @@ +import { z } from 'zod' +import type { ServerSelectorKey } from '@/lib/selectors/manifest' +import { resolveSelectorOAuthAccessToken } from '@/lib/selectors/server/credentials' +import { + SelectorConnectionUnavailableError, + SelectorOptionsUnavailableError, +} from '@/lib/selectors/server/errors' +import { flatSelectorResult } from '@/lib/selectors/server/providers/flat-results' +import { fetchProviderJson } from '@/lib/selectors/server/providers/provider-http' +import type { + ExecuteServerSelectorArgs, + ServerSelectorAttachmentMap, +} from '@/lib/selectors/server/types' + +type AsanaSelectorKey = Extract + +const ASANA_WORKSPACES_URL = 'https://app.asana.com/api/1.0/workspaces' +const ASANA_PAGE_LIMIT = 100 +const ASANA_MAX_PAGES = 50 + +const asanaWorkspaceSchema = z.object({ + gid: z.string().min(1), + name: z.string().min(1), +}) + +const asanaPageSchema = z.object({ + data: z.array(asanaWorkspaceSchema).max(ASANA_PAGE_LIMIT).optional(), + next_page: z + .object({ offset: z.string().min(1).max(4_096) }) + .nullable() + .optional(), +}) + +function requireCredential(args: ExecuteServerSelectorArgs) { + if (!args.credential) throw new SelectorConnectionUnavailableError() + return args.credential +} + +async function listWorkspaces(args: ExecuteServerSelectorArgs) { + const accessToken = await resolveSelectorOAuthAccessToken({ + credential: requireCredential(args), + serviceId: 'asana', + protectedValues: args.protectedValues, + }) + const workspaces: z.infer[] = [] + let offset: string | undefined + let truncated = false + + for (let page = 0; page < ASANA_MAX_PAGES; page++) { + const url = new URL(ASANA_WORKSPACES_URL) + url.searchParams.set('limit', String(ASANA_PAGE_LIMIT)) + if (offset) url.searchParams.set('offset', offset) + + const body = await fetchProviderJson(url, { + headers: { + Authorization: `Bearer ${accessToken}`, + Accept: 'application/json', + }, + redirect: 'error', + signal: args.signal, + }) + const parsed = asanaPageSchema.safeParse(body) + if (!parsed.success) throw new SelectorOptionsUnavailableError() + + workspaces.push(...(parsed.data.data ?? [])) + offset = parsed.data.next_page?.offset + if (!offset) break + if (page === ASANA_MAX_PAGES - 1) truncated = true + } + + return { + items: workspaces.map((workspace) => ({ id: workspace.gid, label: workspace.name })), + truncated, + } +} + +export const asanaSelectorAttachments = { + 'asana.workspaces': { + credential: { + kind: 'stored', + field: 'oauthCredential', + serviceIds: ['asana'], + }, + destination: 'fixed', + async execute(args) { + const { items, truncated } = await listWorkspaces(args) + return flatSelectorResult( + args.request, + items, + true, + truncated ? { truncated: { reason: 'provider-cap', pages: ASANA_MAX_PAGES } } : undefined + ) + }, + }, +} satisfies ServerSelectorAttachmentMap diff --git a/apps/sim/lib/selectors/server/providers/atlassian.test.ts b/apps/sim/lib/selectors/server/providers/atlassian.test.ts new file mode 100644 index 00000000000..0f9e1a79995 --- /dev/null +++ b/apps/sim/lib/selectors/server/providers/atlassian.test.ts @@ -0,0 +1,147 @@ +/** + * @vitest-environment node + */ +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const { mockFetchProviderJsonWithStatus, mockRetryWithExponentialBackoff } = vi.hoisted(() => ({ + mockFetchProviderJsonWithStatus: vi.fn(), + mockRetryWithExponentialBackoff: vi.fn(), +})) + +vi.mock('@/lib/selectors/server/providers/provider-http', () => ({ + fetchProviderJsonWithStatus: mockFetchProviderJsonWithStatus, + RetryableProviderNetworkError: class RetryableProviderNetworkError extends Error {}, +})) + +vi.mock('@/lib/knowledge/documents/utils', () => ({ + retryWithExponentialBackoff: mockRetryWithExponentialBackoff, +})) + +import { + SelectorConnectionUnavailableError, + SelectorOptionsUnavailableError, +} from '@/lib/selectors/server/errors' +import { resolveSelectorAtlassianCloudId } from '@/lib/selectors/server/providers/atlassian' +import { RetryableProviderNetworkError } from '@/lib/selectors/server/providers/provider-http' + +describe('Atlassian server selector authentication', () => { + beforeEach(() => { + vi.clearAllMocks() + mockRetryWithExponentialBackoff.mockImplementation(async (operation: () => Promise) => + operation() + ) + }) + + it('accepts only a service-account cloud id bound to the selected domain', async () => { + await expect( + resolveSelectorAtlassianCloudId({ + accessToken: 'server-only-token', + domain: 'https://ACME.atlassian.net/', + providedCloudId: 'cloud-1', + providedDomain: 'acme.atlassian.net', + product: 'Jira', + }) + ).resolves.toBe('cloud-1') + expect(mockFetchProviderJsonWithStatus).not.toHaveBeenCalled() + + await expect( + resolveSelectorAtlassianCloudId({ + accessToken: 'server-only-token', + domain: 'other.atlassian.net', + providedCloudId: 'cloud-1', + providedDomain: 'acme.atlassian.net', + product: 'Jira', + }) + ).rejects.toBeInstanceOf(SelectorConnectionUnavailableError) + expect(mockFetchProviderJsonWithStatus).not.toHaveBeenCalled() + }) + + it('passes only transient statuses into the bounded retry path', async () => { + mockRetryWithExponentialBackoff.mockImplementation( + async (operation: () => Promise) => { + await expect(operation()).rejects.toMatchObject({ status: 503, retryAfterMs: 1 }) + return operation() + } + ) + mockFetchProviderJsonWithStatus + .mockResolvedValueOnce({ ok: false, status: 503, retryAfterMs: 1 }) + .mockResolvedValueOnce({ + ok: true, + status: 200, + data: [{ id: 'cloud-1', url: 'acme.atlassian.net' }], + }) + + await expect( + resolveSelectorAtlassianCloudId({ + accessToken: 'server-only-token', + domain: 'acme.atlassian.net', + product: 'Jira', + }) + ).resolves.toBe('cloud-1') + + expect(mockFetchProviderJsonWithStatus).toHaveBeenCalledTimes(2) + const retryOptions = mockRetryWithExponentialBackoff.mock.calls[0]?.[1] + expect(retryOptions).toMatchObject({ signal: undefined }) + expect(retryOptions.retryCondition(new DOMException('timed out', 'TimeoutError'))).toBe(true) + expect(retryOptions.retryCondition(new DOMException('cancelled', 'AbortError'))).toBe(false) + expect(mockFetchProviderJsonWithStatus.mock.calls[0]?.[1]?.signal).toBeInstanceOf(AbortSignal) + }) + + it('retries a concealed network failure without retrying caller cancellation', async () => { + mockRetryWithExponentialBackoff.mockImplementation( + async (operation: () => Promise) => { + await expect(operation()).rejects.toBeInstanceOf(RetryableProviderNetworkError) + return operation() + } + ) + mockFetchProviderJsonWithStatus + .mockRejectedValueOnce(new RetryableProviderNetworkError()) + .mockResolvedValueOnce({ + ok: true, + status: 200, + data: [{ id: 'cloud-1', url: 'acme.atlassian.net' }], + }) + + await expect( + resolveSelectorAtlassianCloudId({ + accessToken: 'server-only-token', + domain: 'acme.atlassian.net', + product: 'Jira', + }) + ).resolves.toBe('cloud-1') + + const retryOptions = mockRetryWithExponentialBackoff.mock.calls[0]?.[1] + expect(retryOptions.retryCondition(new RetryableProviderNetworkError())).toBe(true) + expect(mockFetchProviderJsonWithStatus.mock.calls[0]?.[2]).toMatchObject({ + passthroughNetworkErrors: true, + }) + }) + + it('preserves caller cancellation through discovery retries', async () => { + const controller = new AbortController() + const abortError = new DOMException('The operation was aborted', 'AbortError') + controller.abort(abortError) + mockRetryWithExponentialBackoff.mockRejectedValueOnce(abortError) + + await expect( + resolveSelectorAtlassianCloudId({ + accessToken: 'server-only-token', + domain: 'acme.atlassian.net', + product: 'Jira', + signal: controller.signal, + }) + ).rejects.toBe(abortError) + }) + + it('preserves the safe rate-limit category after retries are exhausted', async () => { + mockFetchProviderJsonWithStatus.mockResolvedValueOnce({ ok: false, status: 429 }) + + await expect( + resolveSelectorAtlassianCloudId({ + accessToken: 'server-only-token', + domain: 'acme.atlassian.net', + product: 'Jira', + }) + ).rejects.toEqual(new SelectorOptionsUnavailableError(429)) + }) +}) diff --git a/apps/sim/lib/selectors/server/providers/atlassian.ts b/apps/sim/lib/selectors/server/providers/atlassian.ts new file mode 100644 index 00000000000..d690bca8d4b --- /dev/null +++ b/apps/sim/lib/selectors/server/providers/atlassian.ts @@ -0,0 +1,143 @@ +import { normalizeAtlassianSiteUrl, selectAtlassianCloudId } from '@/lib/atlassian/discovery' +import { retryWithExponentialBackoff } from '@/lib/knowledge/documents/utils' +import { + SelectorConnectionUnavailableError, + SelectorContextUnavailableError, + SelectorOptionsUnavailableError, +} from '@/lib/selectors/server/errors' +import { + fetchProviderJsonWithStatus, + RetryableProviderNetworkError, +} from '@/lib/selectors/server/providers/provider-http' + +const ATLASSIAN_ACCESSIBLE_RESOURCES_URL = + 'https://api.atlassian.com/oauth/token/accessible-resources' +const ATLASSIAN_CLOUD_ID_PATTERN = /^[A-Za-z0-9_-]{1,100}$/ +const ATLASSIAN_DISCOVERY_ATTEMPT_TIMEOUT_MS = 5_000 + +interface AtlassianAccessibleResource { + id?: string + url?: string +} + +const ATLASSIAN_SELECTOR_RETRY_OPTIONS = { + maxRetries: 3, + initialDelayMs: 500, + maxDelayMs: 8_000, +} as const + +/** + * Intentionally generic: retry diagnostics must never include a provider body, + * selected domain, or credential-derived value. + */ +class RetryableAtlassianSelectorError extends Error { + readonly status: number + readonly retryAfterMs?: number + + constructor(status: number, retryAfterMs?: number) { + super('Atlassian selector request unavailable') + this.name = 'RetryableAtlassianSelectorError' + this.status = status + this.retryAfterMs = retryAfterMs + } +} + +function requireCloudId(value: string): string { + if (!ATLASSIAN_CLOUD_ID_PATTERN.test(value)) { + throw new SelectorOptionsUnavailableError() + } + return value +} + +/** + * Resolves an Atlassian cloud id without putting a reference-resolved domain in + * the shared discovery cache key. The endpoint is fixed and provider failures + * are deliberately collapsed before they reach the selector response boundary. + */ +export async function resolveSelectorAtlassianCloudId(input: { + accessToken: string + domain: string | undefined + providedCloudId?: string + providedDomain?: string + product: 'Jira' | 'Confluence' + signal?: AbortSignal +}): Promise { + if (input.providedCloudId) { + const contextDomain = input.domain?.trim() + const credentialDomain = input.providedDomain?.trim() + if ( + !contextDomain || + !credentialDomain || + normalizeAtlassianSiteUrl(contextDomain) !== normalizeAtlassianSiteUrl(credentialDomain) + ) { + throw new SelectorConnectionUnavailableError() + } + return requireCloudId(input.providedCloudId) + } + + const domain = input.domain?.trim() + if (!domain) throw new SelectorContextUnavailableError() + + let resources: AtlassianAccessibleResource[] + try { + resources = await retryWithExponentialBackoff( + async () => { + const attemptTimeout = AbortSignal.timeout(ATLASSIAN_DISCOVERY_ATTEMPT_TIMEOUT_MS) + const attemptSignal = input.signal + ? AbortSignal.any([input.signal, attemptTimeout]) + : attemptTimeout + const response = await fetchProviderJsonWithStatus( + ATLASSIAN_ACCESSIBLE_RESOURCES_URL, + { + headers: { + Authorization: `Bearer ${input.accessToken}`, + Accept: 'application/json', + }, + redirect: 'error', + signal: attemptSignal, + }, + { + passthroughStatus: (status) => status === 429 || status >= 500, + passthroughNetworkErrors: true, + } + ) + if (response.ok) return response.data + throw new RetryableAtlassianSelectorError(response.status, response.retryAfterMs) + }, + { + ...ATLASSIAN_SELECTOR_RETRY_OPTIONS, + signal: input.signal, + retryCondition: (error) => + (error instanceof RetryableAtlassianSelectorError && + (error.status === 429 || error.status >= 500)) || + error instanceof RetryableProviderNetworkError || + (error instanceof Error && error.name === 'TimeoutError'), + } + ) + } catch (error) { + if (input.signal?.aborted) throw error + if ( + error instanceof SelectorConnectionUnavailableError || + error instanceof SelectorContextUnavailableError || + error instanceof SelectorOptionsUnavailableError + ) { + throw error + } + if (error instanceof RetryableAtlassianSelectorError) { + throw new SelectorOptionsUnavailableError(error.status === 429 ? 429 : 502) + } + throw new SelectorOptionsUnavailableError() + } + + try { + return requireCloudId(selectAtlassianCloudId(resources, domain, input.product)) + } catch (error) { + if ( + error instanceof SelectorContextUnavailableError || + error instanceof SelectorOptionsUnavailableError + ) { + throw error + } + throw new SelectorOptionsUnavailableError() + } +} diff --git a/apps/sim/lib/selectors/server/providers/attio.ts b/apps/sim/lib/selectors/server/providers/attio.ts new file mode 100644 index 00000000000..062ac8ac6b7 --- /dev/null +++ b/apps/sim/lib/selectors/server/providers/attio.ts @@ -0,0 +1,90 @@ +import { z } from 'zod' +import type { ServerSelectorKey } from '@/lib/selectors/manifest' +import { resolveSelectorOAuthAccessToken } from '@/lib/selectors/server/credentials' +import { + SelectorConnectionUnavailableError, + SelectorOptionsUnavailableError, +} from '@/lib/selectors/server/errors' +import { flatSelectorResult } from '@/lib/selectors/server/providers/flat-results' +import { fetchProviderJson } from '@/lib/selectors/server/providers/provider-http' +import type { + ExecuteServerSelectorArgs, + ServerSelectorAttachmentMap, +} from '@/lib/selectors/server/types' + +type AttioSelectorKey = Extract + +const attioListSchema = z.object({ + api_slug: z.string().min(1), + name: z.string().min(1), +}) + +const attioObjectSchema = z.object({ + api_slug: z.string().min(1), + singular_noun: z.string().min(1), +}) + +function responseSchema(item: T) { + return z.object({ data: z.array(item).max(10_000).optional() }) +} + +function requireCredential(args: ExecuteServerSelectorArgs) { + if (!args.credential) throw new SelectorConnectionUnavailableError() + return args.credential +} + +async function accessToken(args: ExecuteServerSelectorArgs) { + return resolveSelectorOAuthAccessToken({ + credential: requireCredential(args), + serviceId: 'attio', + protectedValues: args.protectedValues, + }) +} + +async function fetchAttioOptions(args: ExecuteServerSelectorArgs, kind: 'lists' | 'objects') { + const token = await accessToken(args) + const body = await fetchProviderJson(`https://api.attio.com/v2/${kind}`, { + headers: { + Authorization: `Bearer ${token}`, + Accept: 'application/json', + }, + redirect: 'error', + signal: args.signal, + }) + + if (kind === 'lists') { + const parsed = responseSchema(attioListSchema).safeParse(body) + if (!parsed.success) throw new SelectorOptionsUnavailableError() + return (parsed.data.data ?? []).map((list) => ({ id: list.api_slug, label: list.name })) + } + + const parsed = responseSchema(attioObjectSchema).safeParse(body) + if (!parsed.success) throw new SelectorOptionsUnavailableError() + return (parsed.data.data ?? []).map((object) => ({ + id: object.api_slug, + label: object.singular_noun, + })) +} + +const credential = { + kind: 'stored', + field: 'oauthCredential', + serviceIds: ['attio'], +} as const + +export const attioSelectorAttachments = { + 'attio.lists': { + credential, + destination: 'fixed', + async execute(args) { + return flatSelectorResult(args.request, await fetchAttioOptions(args, 'lists'), true) + }, + }, + 'attio.objects': { + credential, + destination: 'fixed', + async execute(args) { + return flatSelectorResult(args.request, await fetchAttioOptions(args, 'objects'), true) + }, + }, +} satisfies ServerSelectorAttachmentMap diff --git a/apps/sim/lib/selectors/server/providers/bigquery.test.ts b/apps/sim/lib/selectors/server/providers/bigquery.test.ts new file mode 100644 index 00000000000..87840705600 --- /dev/null +++ b/apps/sim/lib/selectors/server/providers/bigquery.test.ts @@ -0,0 +1,223 @@ +/** + * @vitest-environment node + */ +import { afterAll, beforeEach, describe, expect, it, vi } from 'vitest' + +const { mockFetch, mockResolveSelectorOAuthAccessToken } = vi.hoisted(() => ({ + mockFetch: vi.fn(), + mockResolveSelectorOAuthAccessToken: vi.fn(), +})) + +vi.mock('@/lib/selectors/server/credentials', () => ({ + resolveSelectorOAuthAccessToken: mockResolveSelectorOAuthAccessToken, +})) + +import { createSelectorProtectedValues } from '@/lib/selectors/server/protected-values' +import { bigQuerySelectorAttachments } from '@/lib/selectors/server/providers/bigquery' +import type { ExecuteServerSelectorArgs } from '@/lib/selectors/server/types' + +const PROJECT_ID = 'selector-test' +const DATASET_ID = 'analytics' + +interface DatasetFixture { + datasetReference: { projectId: string; datasetId: string } + friendlyName: string +} + +interface TableFixture { + tableReference: { projectId: string; datasetId: string; tableId: string } + friendlyName: string +} + +function args( + selectorKey: 'bigquery.datasets' | 'bigquery.tables', + request: ExecuteServerSelectorArgs['request'] +): ExecuteServerSelectorArgs { + return { + selectorKey, + context: { + oauthCredential: 'credential-1', + projectId: PROJECT_ID, + ...(selectorKey === 'bigquery.tables' ? { datasetId: DATASET_ID } : {}), + }, + request, + scope: { kind: 'workspace', workspaceId: 'workspace-1' }, + workspaceId: 'workspace-1', + principal: { kind: 'session', userId: 'user-1', sessionId: 'session-1' }, + requesterUserId: 'user-1', + credential: { suppliedId: 'credential-1' }, + references: new Map(), + protectedValues: createSelectorProtectedValues(), + } +} + +function dataset(datasetId: string, friendlyName: string, projectId = PROJECT_ID): DatasetFixture { + return { + datasetReference: { projectId, datasetId }, + friendlyName, + } +} + +function table(tableId: string, friendlyName: string): TableFixture { + return { + tableReference: { projectId: PROJECT_ID, datasetId: DATASET_ID, tableId }, + friendlyName, + } +} + +describe('BigQuery server selector adapters', () => { + beforeEach(() => { + vi.clearAllMocks() + vi.stubGlobal('fetch', mockFetch) + mockResolveSelectorOAuthAccessToken.mockResolvedValue('server-only-token') + }) + + afterAll(() => vi.unstubAllGlobals()) + + it('returns one dataset page and forwards its continuation token on demand', async () => { + mockFetch + .mockResolvedValueOnce( + new Response( + JSON.stringify({ + datasets: [dataset('dataset_1', 'Dataset 1')], + nextPageToken: 'dataset-page-2', + }), + { status: 200 } + ) + ) + .mockResolvedValueOnce( + new Response(JSON.stringify({ datasets: [dataset('dataset_2', 'Dataset 2')] }), { + status: 200, + }) + ) + + await expect( + bigQuerySelectorAttachments['bigquery.datasets'].execute( + args('bigquery.datasets', { kind: 'list' }) + ) + ).resolves.toEqual({ + kind: 'list', + items: [{ id: 'dataset_1', label: 'Dataset 1' }], + nextCursor: 'dataset-page-2', + }) + await expect( + bigQuerySelectorAttachments['bigquery.datasets'].execute( + args('bigquery.datasets', { kind: 'list', cursor: 'dataset-page-2' }) + ) + ).resolves.toEqual({ + kind: 'list', + items: [{ id: 'dataset_2', label: 'Dataset 2' }], + }) + + const firstUrl = new URL(String(mockFetch.mock.calls[0]?.[0])) + const secondUrl = new URL(String(mockFetch.mock.calls[1]?.[0])) + expect(firstUrl.searchParams.get('maxResults')).toBe('200') + expect(firstUrl.searchParams.has('pageToken')).toBe(false) + expect(secondUrl.searchParams.get('pageToken')).toBe('dataset-page-2') + expect(mockFetch).toHaveBeenCalledTimes(2) + }) + + it('returns one table page and forwards its continuation token on demand', async () => { + mockFetch + .mockResolvedValueOnce( + new Response( + JSON.stringify({ + tables: [table('table_1', 'Table 1')], + nextPageToken: 'table-page-2', + }), + { status: 200 } + ) + ) + .mockResolvedValueOnce( + new Response(JSON.stringify({ tables: [table('table_2', 'Table 2')] }), { + status: 200, + }) + ) + + await expect( + bigQuerySelectorAttachments['bigquery.tables'].execute( + args('bigquery.tables', { kind: 'list' }) + ) + ).resolves.toEqual({ + kind: 'list', + items: [{ id: 'table_1', label: 'Table 1' }], + nextCursor: 'table-page-2', + }) + await expect( + bigQuerySelectorAttachments['bigquery.tables'].execute( + args('bigquery.tables', { kind: 'list', cursor: 'table-page-2' }) + ) + ).resolves.toEqual({ + kind: 'list', + items: [{ id: 'table_2', label: 'Table 2' }], + }) + + const firstUrl = new URL(String(mockFetch.mock.calls[0]?.[0])) + const secondUrl = new URL(String(mockFetch.mock.calls[1]?.[0])) + expect(firstUrl.searchParams.get('maxResults')).toBe('200') + expect(firstUrl.searchParams.has('pageToken')).toBe(false) + expect(secondUrl.searchParams.get('pageToken')).toBe('table-page-2') + expect(mockFetch).toHaveBeenCalledTimes(2) + }) + + it('hydrates a selected dataset in a legacy domain-scoped project directly by id', async () => { + const projectId = 'example.com:selector-test' + mockFetch.mockResolvedValueOnce( + new Response(JSON.stringify(dataset('saved_dataset', 'Saved Dataset', projectId)), { + status: 200, + }) + ) + + const detailArgs = args('bigquery.datasets', { kind: 'detail', id: 'saved_dataset' }) + detailArgs.context.projectId = projectId + + await expect( + bigQuerySelectorAttachments['bigquery.datasets'].execute(detailArgs) + ).resolves.toEqual({ + kind: 'detail', + item: { id: 'saved_dataset', label: 'Saved Dataset' }, + }) + + const url = new URL(String(mockFetch.mock.calls[0]?.[0])) + expect(url.pathname).toBe( + '/bigquery/v2/projects/example.com%3Aselector-test/datasets/saved_dataset' + ) + expect(url.searchParams.get('datasetView')).toBe('METADATA') + expect(mockFetch).toHaveBeenCalledTimes(1) + }) + + it('hydrates a selected table directly by its encoded id', async () => { + const tableId = 'Sales Table-Δ' + mockFetch.mockResolvedValueOnce( + new Response(JSON.stringify(table(tableId, 'Saved Table')), { status: 200 }) + ) + + await expect( + bigQuerySelectorAttachments['bigquery.tables'].execute( + args('bigquery.tables', { kind: 'detail', id: tableId }) + ) + ).resolves.toEqual({ + kind: 'detail', + item: { id: tableId, label: 'Saved Table' }, + }) + + const url = new URL(String(mockFetch.mock.calls[0]?.[0])) + expect(url.pathname).toBe( + `/bigquery/v2/projects/${PROJECT_ID}/datasets/${DATASET_ID}/tables/Sales%20Table-%CE%94` + ) + expect(url.searchParams.get('view')).toBe('BASIC') + expect(mockFetch).toHaveBeenCalledTimes(1) + }) + + it.each([ + ['bigquery.datasets', { kind: 'detail', id: 'missing_dataset' }], + ['bigquery.tables', { kind: 'detail', id: 'missing_table' }], + ] as const)('returns null when %s detail is missing', async (selectorKey, request) => { + mockFetch.mockResolvedValueOnce(new Response(null, { status: 404 })) + + await expect( + bigQuerySelectorAttachments[selectorKey].execute(args(selectorKey, request)) + ).resolves.toEqual({ kind: 'detail', item: null }) + expect(mockFetch).toHaveBeenCalledTimes(1) + }) +}) diff --git a/apps/sim/lib/selectors/server/providers/bigquery.ts b/apps/sim/lib/selectors/server/providers/bigquery.ts new file mode 100644 index 00000000000..5efa62e389d --- /dev/null +++ b/apps/sim/lib/selectors/server/providers/bigquery.ts @@ -0,0 +1,284 @@ +import { z } from 'zod' +import { getScopesForService } from '@/lib/oauth/utils' +import type { ServerSelectorKey } from '@/lib/selectors/manifest' +import { resolveSelectorOAuthAccessToken } from '@/lib/selectors/server/credentials' +import { + SelectorConnectionUnavailableError, + SelectorContextUnavailableError, + SelectorOptionsUnavailableError, +} from '@/lib/selectors/server/errors' +import { + fetchProviderJson, + fetchProviderJsonWithStatus, +} from '@/lib/selectors/server/providers/provider-http' +import { + detailSelectorResult, + type ExecuteServerSelectorArgs, + listSelectorResult, + requireListRequest, + type ServerSelectorAttachmentMap, + type ServerSelectorExecutionResult, +} from '@/lib/selectors/server/types' + +type BigQuerySelectorKey = Extract + +const BIGQUERY_PAGE_SIZE = 200 +const BIGQUERY_CURSOR_MAX_LENGTH = 4_096 +const BIGQUERY_SCOPES = getScopesForService('google-bigquery') +/** Standard project IDs plus Google's legacy `domain.tld:project-id` form. */ +const PROJECT_ID_PATTERN = /^([a-z][a-z0-9.-]{0,61}[a-z0-9]:)?[a-z][a-z0-9-]{4,28}[a-z0-9]$/ +const DATASET_ID_PATTERN = /^[A-Za-z0-9_]{1,1024}$/ +const TABLE_ID_PATTERN = /^[\p{L}\p{M}\p{N}\p{Pc}\p{Pd}\p{Zs}]+$/u + +const projectIdSchema = z.string().regex(PROJECT_ID_PATTERN) +const datasetIdSchema = z.string().regex(DATASET_ID_PATTERN) +const tableIdSchema = z.string().superRefine((value, context) => { + if ( + !value || + new TextEncoder().encode(value).byteLength > 1_024 || + !TABLE_ID_PATTERN.test(value) + ) { + context.addIssue({ code: 'custom', message: 'Invalid BigQuery table ID' }) + } +}) + +const bigQueryDatasetSchema = z.object({ + datasetReference: z.object({ + datasetId: datasetIdSchema, + projectId: projectIdSchema, + }), + friendlyName: z.string().optional(), +}) + +const bigQueryTableSchema = z.object({ + tableReference: z.object({ + datasetId: datasetIdSchema, + projectId: projectIdSchema, + tableId: tableIdSchema, + }), + friendlyName: z.string().optional(), +}) + +const datasetsPageSchema = z.object({ + datasets: z.array(bigQueryDatasetSchema).max(BIGQUERY_PAGE_SIZE).optional(), + nextPageToken: z.string().min(1).max(4_096).optional(), +}) + +const tablesPageSchema = z.object({ + tables: z.array(bigQueryTableSchema).max(BIGQUERY_PAGE_SIZE).optional(), + nextPageToken: z.string().min(1).max(4_096).optional(), +}) + +function requireCredential( + args: ExecuteServerSelectorArgs +): NonNullable { + if (!args.credential) throw new SelectorConnectionUnavailableError() + return args.credential +} + +function requireProjectId(value: string | undefined): string { + const parsed = projectIdSchema.safeParse(value?.trim()) + if (!parsed.success) throw new SelectorContextUnavailableError() + return parsed.data +} + +function requireDatasetId(value: string | undefined): string { + const parsed = datasetIdSchema.safeParse(value?.trim()) + if (!parsed.success) throw new SelectorContextUnavailableError() + return parsed.data +} + +function requireTableId(value: string): string { + const parsed = tableIdSchema.safeParse(value) + if (!parsed.success) throw new SelectorContextUnavailableError() + return parsed.data +} + +function requireCursor(value: string | undefined): string | undefined { + if (value === undefined) return undefined + if (!value.trim() || value.length > BIGQUERY_CURSOR_MAX_LENGTH) { + throw new SelectorContextUnavailableError() + } + return value +} + +async function getAccessToken(args: ExecuteServerSelectorArgs): Promise { + return resolveSelectorOAuthAccessToken({ + credential: requireCredential(args), + serviceId: 'google-bigquery', + scopes: BIGQUERY_SCOPES, + impersonateEmail: args.context.impersonateUserEmail, + protectedValues: args.protectedValues, + }) +} + +function requestHeaders(accessToken: string): Record { + return { + Authorization: `Bearer ${accessToken}`, + Accept: 'application/json', + } +} + +async function listDatasets( + args: ExecuteServerSelectorArgs +): Promise { + const request = requireListRequest(args.selectorKey, args.request) + const projectId = requireProjectId(args.context.projectId) + const cursor = requireCursor(request.cursor) + const accessToken = await getAccessToken(args) + const url = new URL( + `https://bigquery.googleapis.com/bigquery/v2/projects/${encodeURIComponent(projectId)}/datasets` + ) + url.searchParams.set('maxResults', String(BIGQUERY_PAGE_SIZE)) + if (cursor) url.searchParams.set('pageToken', cursor) + + const body = await fetchProviderJson(url, { + headers: requestHeaders(accessToken), + redirect: 'error', + signal: args.signal, + }) + const parsed = datasetsPageSchema.safeParse(body) + if (!parsed.success) throw new SelectorOptionsUnavailableError() + if (parsed.data.datasets?.some((dataset) => dataset.datasetReference.projectId !== projectId)) { + throw new SelectorOptionsUnavailableError() + } + + return listSelectorResult( + (parsed.data.datasets ?? []).map((dataset) => ({ + id: dataset.datasetReference.datasetId, + label: dataset.friendlyName || dataset.datasetReference.datasetId, + })), + parsed.data.nextPageToken + ) +} + +async function listTables(args: ExecuteServerSelectorArgs): Promise { + const request = requireListRequest(args.selectorKey, args.request) + const projectId = requireProjectId(args.context.projectId) + const datasetId = requireDatasetId(args.context.datasetId) + const cursor = requireCursor(request.cursor) + const accessToken = await getAccessToken(args) + const url = new URL( + `https://bigquery.googleapis.com/bigquery/v2/projects/${encodeURIComponent(projectId)}/datasets/${encodeURIComponent(datasetId)}/tables` + ) + url.searchParams.set('maxResults', String(BIGQUERY_PAGE_SIZE)) + if (cursor) url.searchParams.set('pageToken', cursor) + + const body = await fetchProviderJson(url, { + headers: requestHeaders(accessToken), + redirect: 'error', + signal: args.signal, + }) + const parsed = tablesPageSchema.safeParse(body) + if (!parsed.success) throw new SelectorOptionsUnavailableError() + if ( + parsed.data.tables?.some( + (table) => + table.tableReference.projectId !== projectId || table.tableReference.datasetId !== datasetId + ) + ) { + throw new SelectorOptionsUnavailableError() + } + + return listSelectorResult( + (parsed.data.tables ?? []).map((table) => ({ + id: table.tableReference.tableId, + label: table.friendlyName || table.tableReference.tableId, + })), + parsed.data.nextPageToken + ) +} + +async function getDataset(args: ExecuteServerSelectorArgs): Promise { + if (args.request.kind !== 'detail') throw new SelectorOptionsUnavailableError() + const projectId = requireProjectId(args.context.projectId) + const datasetId = requireDatasetId(args.request.id) + const accessToken = await getAccessToken(args) + const url = new URL( + `https://bigquery.googleapis.com/bigquery/v2/projects/${encodeURIComponent(projectId)}/datasets/${encodeURIComponent(datasetId)}` + ) + url.searchParams.set('datasetView', 'METADATA') + + const result = await fetchProviderJsonWithStatus( + url, + { + headers: requestHeaders(accessToken), + redirect: 'error', + signal: args.signal, + }, + { passthroughStatuses: [404] } + ) + if (!result.ok) return detailSelectorResult(null) + + const parsed = bigQueryDatasetSchema.safeParse(result.data) + if ( + !parsed.success || + parsed.data.datasetReference.projectId !== projectId || + parsed.data.datasetReference.datasetId !== datasetId + ) { + throw new SelectorOptionsUnavailableError() + } + return detailSelectorResult({ + id: datasetId, + label: parsed.data.friendlyName || datasetId, + }) +} + +async function getTable(args: ExecuteServerSelectorArgs): Promise { + if (args.request.kind !== 'detail') throw new SelectorOptionsUnavailableError() + const projectId = requireProjectId(args.context.projectId) + const datasetId = requireDatasetId(args.context.datasetId) + const tableId = requireTableId(args.request.id) + const accessToken = await getAccessToken(args) + const url = new URL( + `https://bigquery.googleapis.com/bigquery/v2/projects/${encodeURIComponent(projectId)}/datasets/${encodeURIComponent(datasetId)}/tables/${encodeURIComponent(tableId)}` + ) + url.searchParams.set('view', 'BASIC') + + const result = await fetchProviderJsonWithStatus( + url, + { + headers: requestHeaders(accessToken), + redirect: 'error', + signal: args.signal, + }, + { passthroughStatuses: [404] } + ) + if (!result.ok) return detailSelectorResult(null) + + const parsed = bigQueryTableSchema.safeParse(result.data) + if ( + !parsed.success || + parsed.data.tableReference.projectId !== projectId || + parsed.data.tableReference.datasetId !== datasetId || + parsed.data.tableReference.tableId !== tableId + ) { + throw new SelectorOptionsUnavailableError() + } + return detailSelectorResult({ id: tableId, label: parsed.data.friendlyName || tableId }) +} + +const credential = { + kind: 'stored', + field: 'oauthCredential', + serviceIds: ['google-bigquery'], +} as const + +export const bigQuerySelectorAttachments = { + 'bigquery.datasets': { + credential, + destination: 'fixed', + async execute(args) { + if (args.request.kind === 'detail') return getDataset(args) + return listDatasets(args) + }, + }, + 'bigquery.tables': { + credential, + destination: 'fixed', + async execute(args) { + if (args.request.kind === 'detail') return getTable(args) + return listTables(args) + }, + }, +} satisfies ServerSelectorAttachmentMap diff --git a/apps/sim/lib/selectors/server/providers/bitbucket.test.ts b/apps/sim/lib/selectors/server/providers/bitbucket.test.ts new file mode 100644 index 00000000000..0f90de7d37d --- /dev/null +++ b/apps/sim/lib/selectors/server/providers/bitbucket.test.ts @@ -0,0 +1,200 @@ +/** + * @vitest-environment node + */ +import { afterAll, beforeEach, describe, expect, it, vi } from 'vitest' + +const { mockFetch, mockResolveSelectorOAuthAccessToken } = vi.hoisted(() => ({ + mockFetch: vi.fn(), + mockResolveSelectorOAuthAccessToken: vi.fn(), +})) + +vi.mock('@/lib/selectors/server/credentials', () => ({ + resolveSelectorOAuthAccessToken: mockResolveSelectorOAuthAccessToken, +})) + +import { SelectorContextUnavailableError } from '@/lib/selectors/server/errors' +import { createSelectorProtectedValues } from '@/lib/selectors/server/protected-values' +import { bitbucketSelectorAttachments } from '@/lib/selectors/server/providers/bitbucket' +import type { ExecuteServerSelectorArgs } from '@/lib/selectors/server/types' + +function repositoryArgs( + overrides: Partial = {} +): ExecuteServerSelectorArgs { + return { + selectorKey: 'bitbucket.repositories', + context: { oauthCredential: 'credential-1', workspaceSlug: 'acme-platform' }, + request: { kind: 'list' }, + scope: { kind: 'workspace', workspaceId: 'workspace-1' }, + workspaceId: 'workspace-1', + principal: { kind: 'session', userId: 'user-1', sessionId: 'session-1' }, + requesterUserId: 'user-1', + credential: { suppliedId: 'credential-1' }, + references: new Map(), + protectedValues: createSelectorProtectedValues(), + ...overrides, + } +} + +function providerResponse(body: unknown): Response { + return new Response(JSON.stringify(body), { + status: 200, + headers: { 'content-type': 'application/json' }, + }) +} + +describe('Bitbucket server selector adapters', () => { + beforeEach(() => { + vi.clearAllMocks() + vi.stubGlobal('fetch', mockFetch) + mockResolveSelectorOAuthAccessToken.mockResolvedValue('server-only-token') + }) + + afterAll(() => vi.unstubAllGlobals()) + + it('keeps a referenced workspace server-only while returning an origin-bound page cursor', async () => { + mockFetch.mockResolvedValueOnce( + providerResponse({ + values: [ + { + slug: 'payments-api', + uuid: '{repository-uuid}', + name: 'Payments API', + full_name: 'acme-platform/payments-api', + }, + ], + next: 'https://api.bitbucket.org/2.0/repositories/acme-platform?page=2&pagelen=100', + }) + ) + + const result = await bitbucketSelectorAttachments['bitbucket.repositories'].execute( + repositoryArgs({ + references: new Map([ + [ + 'workspaceSlug', + { + field: 'workspaceSlug', + name: 'BITBUCKET_WORKSPACE', + scope: 'workspace', + visible: false, + }, + ], + ]), + }) + ) + + expect(result).toEqual({ + kind: 'list', + items: [ + { + id: 'payments-api', + label: 'Payments API', + meta: { slug: 'payments-api', uuid: '{repository-uuid}' }, + }, + ], + nextCursor: 'page=2', + }) + const requestUrl = new URL(String(mockFetch.mock.calls[0]?.[0])) + expect(requestUrl.origin).toBe('https://api.bitbucket.org') + expect(requestUrl.pathname).toBe('/2.0/repositories/acme-platform') + expect(new Headers(mockFetch.mock.calls[0]?.[1]?.headers).get('Authorization')).toBe( + 'Bearer server-only-token' + ) + }) + + it('rejects a cursor that attempts to select another destination before resolving a token', async () => { + await expect( + bitbucketSelectorAttachments['bitbucket.repositories'].execute( + repositoryArgs({ + request: { + kind: 'list', + cursor: 'https://evil.example/2.0/repositories/acme-platform?page=2', + }, + }) + ) + ).rejects.toBeInstanceOf(SelectorContextUnavailableError) + + expect(mockResolveSelectorOAuthAccessToken).not.toHaveBeenCalled() + expect(mockFetch).not.toHaveBeenCalled() + }) + + it('hydrates a selected repository without traversing its workspace pages', async () => { + mockFetch.mockResolvedValueOnce( + providerResponse({ + slug: 'payments-api', + uuid: '{repository-uuid}', + name: 'Payments API', + full_name: 'acme-platform/payments-api', + }) + ) + + await expect( + bitbucketSelectorAttachments['bitbucket.repositories'].execute( + repositoryArgs({ request: { kind: 'detail', id: 'payments-api' } }) + ) + ).resolves.toEqual({ + kind: 'detail', + item: { + id: 'payments-api', + label: 'Payments API', + meta: { + slug: 'payments-api', + uuid: '{repository-uuid}', + fullName: 'acme-platform/payments-api', + workspaceSlug: 'acme-platform', + }, + }, + }) + expect(new URL(String(mockFetch.mock.calls[0]?.[0])).pathname).toBe( + '/2.0/repositories/acme-platform/payments-api' + ) + expect(mockFetch).toHaveBeenCalledTimes(1) + }) + + it.each([ + ['braced', '{a15fb181-db1f-48f7-b41f-e1eff06929d6}'], + ['unbraced', 'a15fb181-db1f-48f7-b41f-e1eff06929d6'], + ])('resolves a %s workspace UUID before listing its repositories', async (_, workspaceUuid) => { + const providerUuid = '{a15fb181-db1f-48f7-b41f-e1eff06929d6}' + mockFetch + .mockResolvedValueOnce( + providerResponse({ slug: 'acme-platform', uuid: providerUuid, name: 'Acme' }) + ) + .mockResolvedValueOnce(providerResponse({ values: [] })) + + await expect( + bitbucketSelectorAttachments['bitbucket.repositories'].execute( + repositoryArgs({ + context: { oauthCredential: 'credential-1', workspaceSlug: workspaceUuid }, + }) + ) + ).resolves.toEqual({ kind: 'list', items: [] }) + + expect(new URL(String(mockFetch.mock.calls[0]?.[0])).pathname).toBe( + `/2.0/workspaces/${encodeURIComponent(providerUuid)}` + ) + expect(new URL(String(mockFetch.mock.calls[1]?.[0])).pathname).toBe( + '/2.0/repositories/acme-platform' + ) + }) + + it('preserves a repository UUID while hydrating its canonical slug', async () => { + const repositoryUuid = '{470c176d-3574-44ea-bb41-89e8638bcca4}' + mockFetch.mockResolvedValueOnce( + providerResponse({ + slug: 'payments-api', + uuid: repositoryUuid, + name: 'Payments API', + full_name: 'acme-platform/payments-api', + }) + ) + + await expect( + bitbucketSelectorAttachments['bitbucket.repositories'].execute( + repositoryArgs({ request: { kind: 'detail', id: repositoryUuid } }) + ) + ).resolves.toMatchObject({ + kind: 'detail', + item: { id: repositoryUuid, label: 'Payments API', meta: { slug: 'payments-api' } }, + }) + }) +}) diff --git a/apps/sim/lib/selectors/server/providers/bitbucket.ts b/apps/sim/lib/selectors/server/providers/bitbucket.ts new file mode 100644 index 00000000000..99b1b112b51 --- /dev/null +++ b/apps/sim/lib/selectors/server/providers/bitbucket.ts @@ -0,0 +1,390 @@ +import { z } from 'zod' +import type { ServerSelectorKey } from '@/lib/selectors/manifest' +import { resolveSelectorOAuthAccessToken } from '@/lib/selectors/server/credentials' +import { + SelectorConnectionUnavailableError, + SelectorContextUnavailableError, + SelectorOptionsUnavailableError, +} from '@/lib/selectors/server/errors' +import { fetchProviderJson } from '@/lib/selectors/server/providers/provider-http' +import { + detailSelectorResult, + type ExecuteServerSelectorArgs, + listSelectorResult, + requireListRequest, + type ServerSelectorAttachmentMap, +} from '@/lib/selectors/server/types' + +type BitbucketSelectorKey = Extract< + ServerSelectorKey, + 'bitbucket.workspaces' | 'bitbucket.repositories' +> + +const BITBUCKET_API_ORIGIN = 'https://api.bitbucket.org' +const BITBUCKET_WORKSPACES_PATH = '/2.0/user/workspaces' +const BITBUCKET_REPOSITORIES_PATH = '/2.0/repositories' +const BITBUCKET_PAGE_SIZE = 100 +const BITBUCKET_CURSOR_MAX_LENGTH = 4_096 +const BITBUCKET_WORKSPACE_FIELDS = + 'values.administrator,values.workspace.slug,values.workspace.uuid,values.workspace.name,next' +const BITBUCKET_REPOSITORY_FIELDS = 'values.slug,values.uuid,values.name,values.full_name,next' + +const bitbucketSlugSchema = z.string().trim().min(1).max(255) +const bitbucketUuidSchema = z.string().trim().min(1).max(100) +const bitbucketNameSchema = z.string().trim().min(1).max(512) +const workspaceUuidPattern = + /^(?:\{[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}\}|[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})$/i +const workspaceSlugPattern = /^[a-z0-9][a-z0-9_-]*$/i + +const workspaceSlugSchema = bitbucketSlugSchema.refine( + (slug) => workspaceSlugPattern.test(slug) && !workspaceUuidPattern.test(slug) +) +const workspaceUuidSchema = bitbucketUuidSchema.refine((uuid) => workspaceUuidPattern.test(uuid)) +const workspaceIdentifierSchema = z.union([workspaceSlugSchema, workspaceUuidSchema]) +const repositorySlugSchema = bitbucketSlugSchema.max(62) +const repositoryIdentifierSchema = z.union([repositorySlugSchema, workspaceUuidSchema]) + +const workspacePageSchema = z.object({ + values: z + .array( + z.object({ + administrator: z.boolean(), + workspace: z.object({ + slug: workspaceSlugSchema, + uuid: bitbucketUuidSchema, + name: bitbucketNameSchema.optional(), + }), + }) + ) + .max(BITBUCKET_PAGE_SIZE), + next: z.string().min(1).max(BITBUCKET_CURSOR_MAX_LENGTH).optional(), +}) + +const repositoryPageSchema = z.object({ + values: z + .array( + z.object({ + slug: repositorySlugSchema.optional(), + uuid: bitbucketUuidSchema, + name: bitbucketNameSchema.optional(), + full_name: bitbucketNameSchema, + }) + ) + .max(BITBUCKET_PAGE_SIZE), + next: z.string().min(1).max(BITBUCKET_CURSOR_MAX_LENGTH).optional(), +}) + +const workspaceDetailSchema = z.object({ + slug: workspaceSlugSchema, + uuid: bitbucketUuidSchema, + name: bitbucketNameSchema.optional(), +}) + +const repositoryDetailSchema = z.object({ + slug: repositorySlugSchema.optional(), + uuid: bitbucketUuidSchema, + name: bitbucketNameSchema.optional(), + full_name: bitbucketNameSchema, +}) + +function requireCredential(args: ExecuteServerSelectorArgs) { + if (!args.credential) throw new SelectorConnectionUnavailableError() + return args.credential +} + +function requireWorkspaceIdentifier(value: string | undefined): string { + const parsed = workspaceIdentifierSchema.safeParse(value) + if (!parsed.success) throw new SelectorContextUnavailableError() + return parsed.data +} + +function isBitbucketUuid(value: string): boolean { + return workspaceUuidPattern.test(value) +} + +function normalizeBitbucketUuid(value: string): string { + return value.replace(/^\{?|\}?$/g, '').toLowerCase() +} + +function formatBitbucketUuid(value: string): string { + return `{${normalizeBitbucketUuid(value)}}` +} + +function requireCursorParams(cursor: string): URLSearchParams { + if (!cursor || cursor.length > BITBUCKET_CURSOR_MAX_LENGTH) { + throw new SelectorContextUnavailableError() + } + + const input = new URLSearchParams(cursor) + const output = new URLSearchParams() + for (const [key, value] of input) { + if (key === 'page') { + if (!/^[1-9][0-9]{0,8}$/.test(value) || output.has(key)) { + throw new SelectorContextUnavailableError() + } + output.set(key, value) + continue + } + if (key === 'after') { + if (!value || value.length > 512 || output.has(key)) { + throw new SelectorContextUnavailableError() + } + output.set(key, value) + continue + } + throw new SelectorContextUnavailableError() + } + + if (output.size === 0) throw new SelectorContextUnavailableError() + return output +} + +function encodeNextCursor(next: string, expectedPath: string): string { + let url: URL + try { + url = new URL(next) + } catch { + throw new SelectorOptionsUnavailableError() + } + + if ( + url.origin !== BITBUCKET_API_ORIGIN || + url.username || + url.password || + url.hash || + url.pathname.toLowerCase() !== expectedPath.toLowerCase() + ) { + throw new SelectorOptionsUnavailableError() + } + + const cursor = new URLSearchParams() + for (const [key, value] of url.searchParams) { + if (key === 'page' || key === 'after') cursor.append(key, value) + else if (key === 'pagelen' && value !== String(BITBUCKET_PAGE_SIZE)) { + throw new SelectorOptionsUnavailableError() + } + } + + try { + return requireCursorParams(cursor.toString()).toString() + } catch { + throw new SelectorOptionsUnavailableError() + } +} + +function buildPageUrl(path: string, fields: string, cursor: string | undefined): URL { + const url = new URL(path, BITBUCKET_API_ORIGIN) + url.searchParams.set('pagelen', String(BITBUCKET_PAGE_SIZE)) + url.searchParams.set('fields', fields) + if (cursor) { + for (const [key, value] of requireCursorParams(cursor)) { + url.searchParams.set(key, value) + } + } + return url +} + +async function getAccessToken(args: ExecuteServerSelectorArgs): Promise { + return resolveSelectorOAuthAccessToken({ + credential: requireCredential(args), + serviceId: 'bitbucket', + protectedValues: args.protectedValues, + }) +} + +async function getWorkspace( + args: ExecuteServerSelectorArgs, + identifier: string, + accessToken: string +): Promise> { + const providerIdentifier = isBitbucketUuid(identifier) + ? formatBitbucketUuid(identifier) + : identifier + const url = new URL( + `/2.0/workspaces/${encodeURIComponent(providerIdentifier)}`, + BITBUCKET_API_ORIGIN + ) + url.searchParams.set('fields', 'slug,uuid,name') + const body = await fetchProviderJson(url, { + headers: { Authorization: `Bearer ${accessToken}`, Accept: 'application/json' }, + redirect: 'error', + signal: args.signal, + }) + const parsed = workspaceDetailSchema.safeParse(body) + if (!parsed.success) throw new SelectorOptionsUnavailableError() + if ( + isBitbucketUuid(identifier) + ? normalizeBitbucketUuid(parsed.data.uuid) !== normalizeBitbucketUuid(identifier) + : parsed.data.slug.toLowerCase() !== identifier.toLowerCase() + ) { + throw new SelectorOptionsUnavailableError() + } + return parsed.data +} + +async function resolveWorkspaceSlug( + args: ExecuteServerSelectorArgs, + identifier: string, + accessToken: string +): Promise { + if (!isBitbucketUuid(identifier)) return identifier + return (await getWorkspace(args, identifier, accessToken)).slug +} + +async function listWorkspaces(args: ExecuteServerSelectorArgs) { + const request = requireListRequest(args.selectorKey, args.request) + const url = buildPageUrl(BITBUCKET_WORKSPACES_PATH, BITBUCKET_WORKSPACE_FIELDS, request.cursor) + const accessToken = await getAccessToken(args) + const body = await fetchProviderJson(url, { + headers: { + Authorization: `Bearer ${accessToken}`, + Accept: 'application/json', + }, + redirect: 'error', + signal: args.signal, + }) + const parsed = workspacePageSchema.safeParse(body) + if (!parsed.success) throw new SelectorOptionsUnavailableError() + + return listSelectorResult( + parsed.data.values.map(({ administrator, workspace }) => ({ + id: workspace.slug, + label: workspace.name ?? workspace.slug, + meta: { + slug: workspace.slug, + uuid: workspace.uuid, + fullName: workspace.name ?? workspace.slug, + administrator, + }, + })), + parsed.data.next ? encodeNextCursor(parsed.data.next, BITBUCKET_WORKSPACES_PATH) : undefined + ) +} + +async function executeWorkspaces(args: ExecuteServerSelectorArgs) { + if (args.request.kind === 'list') return listWorkspaces(args) + const identifier = requireWorkspaceIdentifier(args.request.id) + const accessToken = await getAccessToken(args) + const workspace = await getWorkspace(args, identifier, accessToken) + return detailSelectorResult({ + id: identifier, + label: workspace.name ?? workspace.slug, + meta: { + slug: workspace.slug, + uuid: workspace.uuid, + fullName: workspace.name ?? workspace.slug, + }, + }) +} + +function normalizeRepository( + args: ExecuteServerSelectorArgs, + workspaceSlug: string, + repository: z.infer, + requestedId?: string +) { + const separator = repository.full_name.indexOf('/') + if (separator <= 0 || separator !== repository.full_name.lastIndexOf('/')) { + throw new SelectorOptionsUnavailableError() + } + const responseWorkspace = repository.full_name.slice(0, separator) + const fullNameSlug = repository.full_name.slice(separator + 1) + const slug = repository.slug ?? fullNameSlug + if ( + responseWorkspace.toLowerCase() !== workspaceSlug.toLowerCase() || + slug !== fullNameSlug || + !repositorySlugSchema.safeParse(slug).success + ) { + throw new SelectorOptionsUnavailableError() + } + + return { + id: requestedId ?? slug, + label: repository.name ?? slug, + meta: { + slug, + uuid: repository.uuid, + ...(!args.references.has('workspaceSlug') + ? { fullName: repository.full_name, workspaceSlug } + : {}), + }, + } +} + +async function listRepositories(args: ExecuteServerSelectorArgs) { + const request = requireListRequest(args.selectorKey, args.request) + if (request.cursor) requireCursorParams(request.cursor) + const workspaceIdentifier = requireWorkspaceIdentifier(args.context.workspaceSlug) + const accessToken = await getAccessToken(args) + const workspaceSlug = await resolveWorkspaceSlug(args, workspaceIdentifier, accessToken) + const path = `${BITBUCKET_REPOSITORIES_PATH}/${workspaceSlug}` + const url = buildPageUrl(path, BITBUCKET_REPOSITORY_FIELDS, request.cursor) + const body = await fetchProviderJson(url, { + headers: { + Authorization: `Bearer ${accessToken}`, + Accept: 'application/json', + }, + redirect: 'error', + signal: args.signal, + }) + const parsed = repositoryPageSchema.safeParse(body) + if (!parsed.success) throw new SelectorOptionsUnavailableError() + + const normalized = parsed.data.values.map((repository) => + normalizeRepository(args, workspaceSlug, repository) + ) + + return listSelectorResult( + normalized, + parsed.data.next ? encodeNextCursor(parsed.data.next, path) : undefined + ) +} + +async function executeRepositories(args: ExecuteServerSelectorArgs) { + if (args.request.kind === 'list') return listRepositories(args) + const workspaceIdentifier = requireWorkspaceIdentifier(args.context.workspaceSlug) + const repositoryIdentifier = repositoryIdentifierSchema.safeParse(args.request.id) + if (!repositoryIdentifier.success) throw new SelectorContextUnavailableError() + const accessToken = await getAccessToken(args) + const workspaceSlug = await resolveWorkspaceSlug(args, workspaceIdentifier, accessToken) + const url = new URL( + `/2.0/repositories/${encodeURIComponent(workspaceSlug)}/${encodeURIComponent(repositoryIdentifier.data)}`, + BITBUCKET_API_ORIGIN + ) + url.searchParams.set('fields', 'slug,uuid,name,full_name') + const body = await fetchProviderJson(url, { + headers: { Authorization: `Bearer ${accessToken}`, Accept: 'application/json' }, + redirect: 'error', + signal: args.signal, + }) + const parsed = repositoryDetailSchema.safeParse(body) + if (!parsed.success) throw new SelectorOptionsUnavailableError() + if ( + isBitbucketUuid(repositoryIdentifier.data) && + normalizeBitbucketUuid(parsed.data.uuid) !== normalizeBitbucketUuid(repositoryIdentifier.data) + ) { + throw new SelectorOptionsUnavailableError() + } + return detailSelectorResult( + normalizeRepository(args, workspaceSlug, parsed.data, repositoryIdentifier.data) + ) +} + +const credential = { + kind: 'stored', + field: 'oauthCredential', + serviceIds: ['bitbucket'], +} as const + +export const bitbucketSelectorAttachments = { + 'bitbucket.workspaces': { + credential, + destination: 'fixed', + execute: executeWorkspaces, + }, + 'bitbucket.repositories': { + credential, + destination: 'fixed', + execute: executeRepositories, + }, +} satisfies ServerSelectorAttachmentMap diff --git a/apps/sim/lib/selectors/server/providers/calcom.ts b/apps/sim/lib/selectors/server/providers/calcom.ts new file mode 100644 index 00000000000..3640ecb6c62 --- /dev/null +++ b/apps/sim/lib/selectors/server/providers/calcom.ts @@ -0,0 +1,107 @@ +import { z } from 'zod' +import type { ServerSelectorKey } from '@/lib/selectors/manifest' +import { resolveSelectorOAuthAccessToken } from '@/lib/selectors/server/credentials' +import { + SelectorConnectionUnavailableError, + SelectorOptionsUnavailableError, +} from '@/lib/selectors/server/errors' +import { flatSelectorResult } from '@/lib/selectors/server/providers/flat-results' +import { fetchProviderJson } from '@/lib/selectors/server/providers/provider-http' +import type { + ExecuteServerSelectorArgs, + ServerSelectorAttachmentMap, +} from '@/lib/selectors/server/types' + +type CalcomSelectorKey = Extract + +const calcomIdSchema = z.union([z.string().min(1), z.number().finite()]).transform(String) + +const eventTypesResponseSchema = z.object({ + data: z + .array( + z.object({ + id: calcomIdSchema, + title: z.string(), + slug: z.string().min(1), + }) + ) + .max(10_000) + .optional(), +}) + +const schedulesResponseSchema = z.object({ + data: z + .array( + z.object({ + id: calcomIdSchema, + name: z.string().min(1), + }) + ) + .max(10_000) + .optional(), +}) + +function requireCredential(args: ExecuteServerSelectorArgs) { + if (!args.credential) throw new SelectorConnectionUnavailableError() + return args.credential +} + +async function getAccessToken(args: ExecuteServerSelectorArgs): Promise { + return resolveSelectorOAuthAccessToken({ + credential: requireCredential(args), + serviceId: 'calcom', + protectedValues: args.protectedValues, + }) +} + +async function getOptions(args: ExecuteServerSelectorArgs, kind: 'event-types' | 'schedules') { + const accessToken = await getAccessToken(args) + const body = await fetchProviderJson(`https://api.cal.com/v2/${kind}`, { + headers: { + Authorization: `Bearer ${accessToken}`, + Accept: 'application/json', + 'cal-api-version': kind === 'event-types' ? '2024-06-14' : '2024-06-11', + }, + redirect: 'error', + signal: args.signal, + }) + + if (kind === 'event-types') { + const parsed = eventTypesResponseSchema.safeParse(body) + if (!parsed.success) throw new SelectorOptionsUnavailableError() + return (parsed.data.data ?? []).map((eventType) => ({ + id: eventType.id, + label: eventType.title || eventType.slug, + })) + } + + const parsed = schedulesResponseSchema.safeParse(body) + if (!parsed.success) throw new SelectorOptionsUnavailableError() + return (parsed.data.data ?? []).map((schedule) => ({ + id: schedule.id, + label: schedule.name, + })) +} + +const credential = { + kind: 'stored', + field: 'oauthCredential', + serviceIds: ['calcom'], +} as const + +export const calcomSelectorAttachments = { + 'calcom.eventTypes': { + credential, + destination: 'fixed', + async execute(args) { + return flatSelectorResult(args.request, await getOptions(args, 'event-types'), true) + }, + }, + 'calcom.schedules': { + credential, + destination: 'fixed', + async execute(args) { + return flatSelectorResult(args.request, await getOptions(args, 'schedules'), true) + }, + }, +} satisfies ServerSelectorAttachmentMap diff --git a/apps/sim/lib/selectors/server/providers/clickup.test.ts b/apps/sim/lib/selectors/server/providers/clickup.test.ts new file mode 100644 index 00000000000..bf2c216dda0 --- /dev/null +++ b/apps/sim/lib/selectors/server/providers/clickup.test.ts @@ -0,0 +1,70 @@ +/** + * @vitest-environment node + */ +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const { mockFetchProviderJson, mockResolveSelectorOAuthAccessToken } = vi.hoisted(() => ({ + mockFetchProviderJson: vi.fn(), + mockResolveSelectorOAuthAccessToken: vi.fn(), +})) + +vi.mock('@/lib/selectors/server/providers/provider-http', () => ({ + fetchProviderJson: mockFetchProviderJson, +})) + +vi.mock('@/lib/selectors/server/credentials', () => ({ + resolveSelectorOAuthAccessToken: mockResolveSelectorOAuthAccessToken, +})) + +import { createSelectorProtectedValues } from '@/lib/selectors/server/protected-values' +import { clickupSelectorAttachments } from '@/lib/selectors/server/providers/clickup' +import type { ExecuteServerSelectorArgs } from '@/lib/selectors/server/types' + +function listArgs(context: Record): ExecuteServerSelectorArgs { + return { + selectorKey: 'clickup.lists', + context, + request: { kind: 'list' }, + scope: { kind: 'workspace', workspaceId: 'workspace-1' }, + workspaceId: 'workspace-1', + principal: { kind: 'session', userId: 'user-1', sessionId: 'session-1' }, + requesterUserId: 'user-1', + credential: { suppliedId: 'credential-1' }, + references: new Map(), + protectedValues: createSelectorProtectedValues(), + } +} + +describe('ClickUp server selector adapters', () => { + beforeEach(() => { + vi.clearAllMocks() + mockResolveSelectorOAuthAccessToken.mockResolvedValue('server-only-token') + mockFetchProviderJson.mockResolvedValue({ lists: [] }) + }) + + it('trims saved dependency IDs and treats a whitespace-only folder as absent', async () => { + await clickupSelectorAttachments['clickup.lists'].execute( + listArgs({ folderId: ' ', spaceId: ' space-1 ' }) + ) + + expect(String(mockFetchProviderJson.mock.calls[0]?.[0])).toContain('/space/space-1/list') + + await clickupSelectorAttachments['clickup.lists'].execute( + listArgs({ folderId: ' folder-1 ', spaceId: 'space-1' }) + ) + + expect(String(mockFetchProviderJson.mock.calls[1]?.[0])).toContain('/folder/folder-1/list') + + const folderArgs = listArgs({ spaceId: ' ', listSpaceId: ' list-space-1 ' }) + folderArgs.selectorKey = 'clickup.folders' + await clickupSelectorAttachments['clickup.folders'].execute(folderArgs) + + expect(String(mockFetchProviderJson.mock.calls[2]?.[0])).toContain('/space/list-space-1/folder') + + await clickupSelectorAttachments['clickup.lists'].execute( + listArgs({ folderId: ' ', spaceId: ' ', listSpaceId: ' list-space-1 ' }) + ) + + expect(String(mockFetchProviderJson.mock.calls[3]?.[0])).toContain('/space/list-space-1/list') + }) +}) diff --git a/apps/sim/lib/selectors/server/providers/clickup.ts b/apps/sim/lib/selectors/server/providers/clickup.ts new file mode 100644 index 00000000000..0b506f815eb --- /dev/null +++ b/apps/sim/lib/selectors/server/providers/clickup.ts @@ -0,0 +1,138 @@ +import { z } from 'zod' +import type { ServerSelectorKey } from '@/lib/selectors/manifest' +import { resolveSelectorOAuthAccessToken } from '@/lib/selectors/server/credentials' +import { + SelectorConnectionUnavailableError, + SelectorContextUnavailableError, + SelectorOptionsUnavailableError, +} from '@/lib/selectors/server/errors' +import { fetchProviderJson } from '@/lib/selectors/server/providers/provider-http' +import { + type ExecuteServerSelectorArgs, + listSelectorResult, + requireListRequest, + type ServerSelectorAttachmentMap, +} from '@/lib/selectors/server/types' +import { CLICKUP_API_BASE_URL, clickupAuthorizationHeader } from '@/tools/clickup/shared' + +type ClickupSelectorKey = Extract< + ServerSelectorKey, + 'clickup.workspaces' | 'clickup.spaces' | 'clickup.folders' | 'clickup.lists' +> + +const clickupResourceSchema = z.object({ + id: z.union([z.string().min(1), z.number().finite()]).transform(String), + name: z.string().optional(), +}) + +const clickupResponseSchema = z.object({ + teams: z.array(clickupResourceSchema).max(10_000).optional(), + spaces: z.array(clickupResourceSchema).max(10_000).optional(), + folders: z.array(clickupResourceSchema).max(10_000).optional(), + lists: z.array(clickupResourceSchema).max(10_000).optional(), +}) + +type ClickupResponseField = 'teams' | 'spaces' | 'folders' | 'lists' + +const fallbackLabels: Record = { + teams: 'Workspace', + spaces: 'Space', + folders: 'Folder', + lists: 'List', +} + +function requireCredential(args: ExecuteServerSelectorArgs) { + if (!args.credential) throw new SelectorConnectionUnavailableError() + return args.credential +} + +function requireClickupId(value: string | undefined): string { + const normalized = value?.trim() + if (!normalized || normalized.length > 100 || !/^[A-Za-z0-9_-]+$/.test(normalized)) { + throw new SelectorContextUnavailableError() + } + return normalized +} + +async function getAccessToken(args: ExecuteServerSelectorArgs): Promise { + return resolveSelectorOAuthAccessToken({ + credential: requireCredential(args), + serviceId: 'clickup', + protectedValues: args.protectedValues, + }) +} + +async function fetchClickupOptions( + args: ExecuteServerSelectorArgs, + field: ClickupResponseField, + path: string +) { + requireListRequest(args.selectorKey, args.request) + const accessToken = await getAccessToken(args) + const body = await fetchProviderJson(`${CLICKUP_API_BASE_URL}${path}`, { + headers: { + Authorization: clickupAuthorizationHeader(accessToken), + Accept: 'application/json', + }, + redirect: 'error', + signal: args.signal, + }) + const parsed = clickupResponseSchema.safeParse(body) + if (!parsed.success) throw new SelectorOptionsUnavailableError() + + const resources = parsed.data[field] + return (resources ?? []).map((resource) => ({ + id: resource.id, + label: resource.name || `${fallbackLabels[field]} ${resource.id}`, + })) +} + +const credential = { + kind: 'stored', + field: 'oauthCredential', + serviceIds: ['clickup'], +} as const + +export const clickupSelectorAttachments = { + 'clickup.workspaces': { + credential, + destination: 'fixed', + async execute(args) { + return listSelectorResult(await fetchClickupOptions(args, 'teams', '/team')) + }, + }, + 'clickup.spaces': { + credential, + destination: 'fixed', + async execute(args) { + const teamId = requireClickupId(args.context.teamId) + return listSelectorResult( + await fetchClickupOptions(args, 'spaces', `/team/${encodeURIComponent(teamId)}/space`) + ) + }, + }, + 'clickup.folders': { + credential, + destination: 'fixed', + async execute(args) { + const spaceId = requireClickupId( + args.context.spaceId?.trim() || args.context.listSpaceId?.trim() + ) + return listSelectorResult( + await fetchClickupOptions(args, 'folders', `/space/${encodeURIComponent(spaceId)}/folder`) + ) + }, + }, + 'clickup.lists': { + credential, + destination: 'fixed', + async execute(args) { + const folderId = args.context.folderId?.trim() + const spaceId = args.context.spaceId?.trim() || args.context.listSpaceId?.trim() + const path = folderId + ? `/folder/${encodeURIComponent(requireClickupId(folderId))}/list` + : `/space/${encodeURIComponent(requireClickupId(spaceId))}/list` + return listSelectorResult(await fetchClickupOptions(args, 'lists', path)) + }, + }, +} satisfies ServerSelectorAttachmentMap diff --git a/apps/sim/lib/selectors/server/providers/cloudwatch.test.ts b/apps/sim/lib/selectors/server/providers/cloudwatch.test.ts new file mode 100644 index 00000000000..3cf9a6f2b08 --- /dev/null +++ b/apps/sim/lib/selectors/server/providers/cloudwatch.test.ts @@ -0,0 +1,157 @@ +/** + * @vitest-environment node + */ +import { CloudWatchLogsServiceException } from '@aws-sdk/client-cloudwatch-logs' +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const { mockListCloudWatchLogGroups, mockListCloudWatchLogStreams } = vi.hoisted(() => ({ + mockListCloudWatchLogGroups: vi.fn(), + mockListCloudWatchLogStreams: vi.fn(), +})) + +vi.mock('@/tools/cloudwatch/listing', () => ({ + listCloudWatchLogGroups: mockListCloudWatchLogGroups, + listCloudWatchLogStreams: mockListCloudWatchLogStreams, +})) + +import { createSelectorProtectedValues } from '@/lib/selectors/server/protected-values' +import { cloudWatchSelectorAttachments } from '@/lib/selectors/server/providers/cloudwatch' +import type { ExecuteServerSelectorArgs } from '@/lib/selectors/server/types' + +function logGroupArgs(signal?: AbortSignal): ExecuteServerSelectorArgs { + return { + selectorKey: 'cloudwatch.logGroups', + context: { + awsAccessKeyId: 'access-key', + awsSecretAccessKey: 'secret-key', + awsRegion: 'us-east-1', + }, + request: { kind: 'list' }, + scope: { kind: 'workspace', workspaceId: 'workspace-1' }, + workspaceId: 'workspace-1', + principal: { kind: 'session', userId: 'user-1', sessionId: 'session-1' }, + requesterUserId: 'user-1', + references: new Map(), + protectedValues: createSelectorProtectedValues(), + signal, + } +} + +function logStreamArgs(): ExecuteServerSelectorArgs { + const args = logGroupArgs() + args.selectorKey = 'cloudwatch.logStreams' + args.context.logGroupName = '/aws/lambda/example' + return args +} + +function cloudWatchError(status: number): CloudWatchLogsServiceException { + return new CloudWatchLogsServiceException({ + name: 'CloudWatchLogsError', + $fault: status >= 500 ? 'server' : 'client', + $metadata: { httpStatusCode: status }, + }) +} + +describe('CloudWatch server selector adapter', () => { + beforeEach(() => vi.clearAllMocks()) + + it.each([ + [401, 'SelectorConnectionUnavailableError', 401], + [403, 'SelectorConnectionUnavailableError', 403], + [429, 'SelectorOptionsUnavailableError', 429], + [500, 'SelectorOptionsUnavailableError', 502], + ] as const)( + 'maps trusted AWS status %i to the safe selector taxonomy', + async (status, name, safeStatus) => { + mockListCloudWatchLogGroups.mockRejectedValueOnce(cloudWatchError(status)) + + await expect( + cloudWatchSelectorAttachments['cloudwatch.logGroups'].execute(logGroupArgs()) + ).rejects.toMatchObject({ name, status: safeStatus }) + } + ) + + it('does not trust a status-shaped unknown error', async () => { + mockListCloudWatchLogGroups.mockRejectedValueOnce({ $metadata: { httpStatusCode: 401 } }) + + await expect( + cloudWatchSelectorAttachments['cloudwatch.logGroups'].execute(logGroupArgs()) + ).rejects.toMatchObject({ name: 'SelectorOptionsUnavailableError', status: 502 }) + }) + + it('preserves caller cancellation', async () => { + const controller = new AbortController() + const abortError = new DOMException('The operation was aborted', 'AbortError') + controller.abort(abortError) + mockListCloudWatchLogGroups.mockRejectedValueOnce(abortError) + + await expect( + cloudWatchSelectorAttachments['cloudwatch.logGroups'].execute(logGroupArgs(controller.signal)) + ).rejects.toBe(abortError) + }) + + it('returns null when a selected log group no longer exists', async () => { + mockListCloudWatchLogGroups.mockResolvedValue({ items: [], pages: 1, truncated: false }) + const args = logGroupArgs() + args.request = { kind: 'detail', id: '/aws/missing' } + + await expect( + cloudWatchSelectorAttachments['cloudwatch.logGroups'].execute(args) + ).resolves.toEqual({ kind: 'detail', item: null }) + expect(mockListCloudWatchLogGroups).toHaveBeenCalledWith( + expect.objectContaining({ prefix: '/aws/missing' }) + ) + }) + + it.each([ + { + selectorKey: 'cloudwatch.logGroups' as const, + mockListing: mockListCloudWatchLogGroups, + args: logGroupArgs, + providerField: 'logGroupName' as const, + binding: {}, + }, + { + selectorKey: 'cloudwatch.logStreams' as const, + mockListing: mockListCloudWatchLogStreams, + args: logStreamArgs, + providerField: 'logStreamName' as const, + binding: { logGroupName: '/aws/lambda/example' }, + }, + ])('forwards opaque cursors for $selectorKey', async (testCase) => { + testCase.mockListing.mockResolvedValueOnce({ + items: [{ [testCase.providerField]: 'target' }], + pages: 20, + truncated: true, + nextToken: 'opaque::next+=', + }) + const args = testCase.args() + args.request = { kind: 'list', search: 'target', cursor: 'opaque::start+=' } + + await expect( + cloudWatchSelectorAttachments[testCase.selectorKey].execute(args) + ).resolves.toEqual({ + kind: 'list', + items: [{ id: 'target', label: 'target' }], + nextCursor: 'opaque::next+=', + }) + expect(testCase.mockListing).toHaveBeenCalledWith( + expect.objectContaining({ + prefix: 'target', + nextToken: 'opaque::start+=', + ...testCase.binding, + }) + ) + }) + + it('rejects an invalid region before invoking the AWS listing helper', async () => { + const args = logGroupArgs() + args.context.awsRegion = 'not-a-region' + + await expect( + cloudWatchSelectorAttachments['cloudwatch.logGroups'].execute(args) + ).rejects.toMatchObject({ name: 'SelectorContextUnavailableError' }) + expect(mockListCloudWatchLogGroups).not.toHaveBeenCalled() + expect(mockListCloudWatchLogStreams).not.toHaveBeenCalled() + }) +}) diff --git a/apps/sim/lib/selectors/server/providers/cloudwatch.ts b/apps/sim/lib/selectors/server/providers/cloudwatch.ts new file mode 100644 index 00000000000..cc9dd3e7edd --- /dev/null +++ b/apps/sim/lib/selectors/server/providers/cloudwatch.ts @@ -0,0 +1,142 @@ +import { CloudWatchLogsServiceException } from '@aws-sdk/client-cloudwatch-logs' +import { validateAwsRegion } from '@/lib/core/security/input-validation' +import { + SelectorContextUnavailableError, + SelectorOptionsUnavailableError, +} from '@/lib/selectors/server/errors' +import { selectorProviderStatusError } from '@/lib/selectors/server/providers/provider-http' +import type { ServerSelectorAttachmentMap } from '@/lib/selectors/server/types' +import { detailSelectorResult, listSelectorResult } from '@/lib/selectors/server/types' +import { + type CloudWatchListingCredentials, + listCloudWatchLogGroups, + listCloudWatchLogStreams, +} from '@/tools/cloudwatch/listing' + +type CloudWatchSelectorKey = 'cloudwatch.logGroups' | 'cloudwatch.logStreams' + +function credentials(context: { + awsAccessKeyId?: string + awsSecretAccessKey?: string + awsRegion?: string +}): CloudWatchListingCredentials { + if ( + !context.awsAccessKeyId || + !context.awsSecretAccessKey || + !context.awsRegion || + !validateAwsRegion(context.awsRegion).isValid + ) { + throw new SelectorContextUnavailableError() + } + return { + accessKeyId: context.awsAccessKeyId, + secretAccessKey: context.awsSecretAccessKey, + region: context.awsRegion, + } +} + +async function executeCloudWatchListing( + signal: AbortSignal | undefined, + operation: () => Promise +): Promise { + try { + return await operation() + } catch (error) { + if (signal?.aborted) throw error + if ( + error instanceof CloudWatchLogsServiceException && + typeof error.$metadata.httpStatusCode === 'number' + ) { + throw selectorProviderStatusError(error.$metadata.httpStatusCode) + } + throw new SelectorOptionsUnavailableError() + } +} + +/** + * The integration this selector reaches. Declared rather than derived: the selector authenticates from raw AWS keys in the request context and + * carries no stored connection, so the OAuth credential catalog can identify + * nothing to gate it on. + */ +const integrationBlockTypes = ['cloudwatch'] as const + +export const cloudWatchSelectorAttachments = { + 'cloudwatch.logGroups': { + integrationBlockTypes, + destination: 'fixed', + async execute(args) { + const listingCredentials = credentials(args.context) + if (args.request.kind === 'detail') { + const detailId = args.request.id + const groups = await executeCloudWatchListing(args.signal, () => + listCloudWatchLogGroups({ + credentials: listingCredentials, + prefix: detailId, + signal: args.signal, + suppressTruncationLog: true, + }) + ) + const match = groups.items.find((group) => group.logGroupName === detailId) + return detailSelectorResult( + match?.logGroupName ? { id: match.logGroupName, label: match.logGroupName } : null + ) + } + const { search, cursor } = args.request + const groups = await executeCloudWatchListing(args.signal, () => + listCloudWatchLogGroups({ + credentials: listingCredentials, + prefix: search, + nextToken: cursor, + signal: args.signal, + suppressTruncationLog: true, + }) + ) + return listSelectorResult( + groups.items + .filter((group) => group.logGroupName) + .map((group) => ({ id: group.logGroupName, label: group.logGroupName })), + groups.nextToken + ) + }, + }, + 'cloudwatch.logStreams': { + integrationBlockTypes, + destination: 'fixed', + async execute(args) { + const listingCredentials = credentials(args.context) + if (args.request.kind === 'detail') { + const detailId = args.request.id + const streams = await executeCloudWatchListing(args.signal, () => + listCloudWatchLogStreams({ + credentials: listingCredentials, + logGroupName: args.context.logGroupName!, + prefix: detailId, + signal: args.signal, + suppressTruncationLog: true, + }) + ) + const match = streams.items.find((stream) => stream.logStreamName === detailId) + return detailSelectorResult( + match?.logStreamName ? { id: match.logStreamName, label: match.logStreamName } : null + ) + } + const { search, cursor } = args.request + const streams = await executeCloudWatchListing(args.signal, () => + listCloudWatchLogStreams({ + credentials: listingCredentials, + logGroupName: args.context.logGroupName!, + prefix: search, + nextToken: cursor, + signal: args.signal, + suppressTruncationLog: true, + }) + ) + return listSelectorResult( + streams.items + .filter((stream) => stream.logStreamName) + .map((stream) => ({ id: stream.logStreamName, label: stream.logStreamName })), + streams.nextToken + ) + }, + }, +} satisfies ServerSelectorAttachmentMap diff --git a/apps/sim/lib/selectors/server/providers/confluence.test.ts b/apps/sim/lib/selectors/server/providers/confluence.test.ts new file mode 100644 index 00000000000..07e6d87ec37 --- /dev/null +++ b/apps/sim/lib/selectors/server/providers/confluence.test.ts @@ -0,0 +1,200 @@ +/** + * @vitest-environment node + */ +import { afterAll, beforeEach, describe, expect, it, vi } from 'vitest' + +const { mockFetch, mockResolveCredentialBundle, mockResolveCloudId } = vi.hoisted(() => ({ + mockFetch: vi.fn(), + mockResolveCredentialBundle: vi.fn(), + mockResolveCloudId: vi.fn(), +})) + +vi.mock('@/lib/selectors/server/providers/credential-bundle', () => ({ + resolveSelectorCredentialBundle: mockResolveCredentialBundle, +})) + +vi.mock('@/lib/selectors/server/providers/atlassian', () => ({ + resolveSelectorAtlassianCloudId: mockResolveCloudId, +})) + +import { + SelectorConnectionUnavailableError, + SelectorOptionsUnavailableError, +} from '@/lib/selectors/server/errors' +import { createSelectorProtectedValues } from '@/lib/selectors/server/protected-values' +import { confluenceSelectorAttachments } from '@/lib/selectors/server/providers/confluence' +import * as providerHttp from '@/lib/selectors/server/providers/provider-http' +import type { ExecuteServerSelectorArgs } from '@/lib/selectors/server/types' + +function pageDetailArgs(): ExecuteServerSelectorArgs { + return { + selectorKey: 'confluence.pages', + context: { oauthCredential: 'credential-1', domain: 'acme.atlassian.net' }, + request: { kind: 'detail', id: 'page-1' }, + scope: { kind: 'workspace', workspaceId: 'workspace-1' }, + workspaceId: 'workspace-1', + principal: { kind: 'session', userId: 'user-1', sessionId: 'session-1' }, + requesterUserId: 'user-1', + credential: { suppliedId: 'credential-1' }, + references: new Map(), + protectedValues: createSelectorProtectedValues(), + } +} + +function spaceDetailArgs(signal?: AbortSignal): ExecuteServerSelectorArgs { + return { + ...pageDetailArgs(), + selectorKey: 'confluence.spaces', + request: { kind: 'detail', id: 'ENG' }, + signal, + } +} + +function spaceIdDetailArgs(): ExecuteServerSelectorArgs { + return { + ...pageDetailArgs(), + selectorKey: 'confluence.spacesById', + request: { kind: 'detail', id: '12345' }, + } +} + +describe('Confluence server selector adapters', () => { + beforeEach(() => { + vi.clearAllMocks() + vi.stubGlobal('fetch', mockFetch) + mockResolveCredentialBundle.mockResolvedValue({ accessToken: 'server-only-token' }) + mockResolveCloudId.mockResolvedValue('cloud-1') + }) + + afterAll(() => vi.unstubAllGlobals()) + + it('hydrates page details through the bounded provider reader without requesting page bodies', async () => { + mockFetch.mockResolvedValueOnce( + new Response(JSON.stringify({ id: 'page-1', title: 'Architecture' }), { status: 200 }) + ) + + await expect( + confluenceSelectorAttachments['confluence.pages'].execute(pageDetailArgs()) + ).resolves.toEqual({ + kind: 'detail', + item: { id: 'page-1', label: 'Architecture' }, + }) + + const requestedUrl = String(mockFetch.mock.calls[0]?.[0]) + expect(requestedUrl).toBe( + 'https://api.atlassian.com/ex/confluence/cloud-1/wiki/api/v2/pages/page-1' + ) + expect(requestedUrl).not.toContain('body-format') + }) + + it('rejects an oversized page detail response before parsing it', async () => { + mockFetch.mockResolvedValueOnce( + new Response('{}', { + status: 200, + headers: { 'content-length': String(16 * 1024 * 1024 + 1) }, + }) + ) + + await expect( + confluenceSelectorAttachments['confluence.pages'].execute(pageDetailArgs()) + ).rejects.toBeInstanceOf(SelectorOptionsUnavailableError) + }) + + it('preserves caller cancellation while hydrating space details', async () => { + const controller = new AbortController() + const abortError = new DOMException('The operation was aborted', 'AbortError') + controller.abort(abortError) + mockFetch.mockRejectedValue(abortError) + + await expect( + confluenceSelectorAttachments['confluence.spaces'].execute(spaceDetailArgs(controller.signal)) + ).rejects.toBe(abortError) + }) + + it('hydrates block space selections by provider resource ID', async () => { + mockFetch.mockResolvedValueOnce( + new Response(JSON.stringify({ id: '12345', key: 'ENG', name: 'Engineering' }), { + status: 200, + }) + ) + + await expect( + confluenceSelectorAttachments['confluence.spacesById'].execute(spaceIdDetailArgs()) + ).resolves.toEqual({ + kind: 'detail', + item: { id: '12345', label: 'Engineering (ENG)' }, + }) + expect(String(mockFetch.mock.calls[0]?.[0])).toContain('/wiki/api/v2/spaces/12345') + }) + + it('hydrates a legacy numeric value in the key selector without rewriting it', async () => { + mockFetch.mockResolvedValueOnce( + new Response(JSON.stringify({ id: '12345', key: 'ENG', name: 'Engineering' }), { + status: 200, + }) + ) + + await expect( + confluenceSelectorAttachments['confluence.spaces'].execute({ + ...spaceDetailArgs(), + request: { kind: 'detail', id: '12345' }, + }) + ).resolves.toEqual({ + kind: 'detail', + item: { id: '12345', label: 'Engineering (ENG)' }, + }) + expect(String(mockFetch.mock.calls[0]?.[0])).toContain('/wiki/api/v2/spaces/12345') + }) + + it('projects provider IDs for block space lists while key selectors remain unchanged', async () => { + mockFetch + .mockResolvedValueOnce( + new Response( + JSON.stringify({ results: [{ id: '12345', key: 'ENG', name: 'Engineering' }] }), + { status: 200 } + ) + ) + .mockResolvedValueOnce( + new Response( + JSON.stringify({ results: [{ id: '12345', key: 'ENG', name: 'Engineering' }] }), + { status: 200 } + ) + ) + + const args = { ...spaceDetailArgs(), request: { kind: 'list' } as const } + await expect( + confluenceSelectorAttachments['confluence.spacesById'].execute({ + ...args, + selectorKey: 'confluence.spacesById', + }) + ).resolves.toMatchObject({ items: [{ id: '12345', label: 'Engineering (ENG)' }] }) + await expect( + confluenceSelectorAttachments['confluence.spaces'].execute(args) + ).resolves.toMatchObject({ items: [{ id: 'ENG', label: 'Engineering (ENG)' }] }) + }) + + it('preserves the first safe provider failure when both space detail requests fail', async () => { + mockFetch + .mockResolvedValueOnce(new Response(null, { status: 401 })) + .mockResolvedValueOnce(new Response(null, { status: 429 })) + + const result = confluenceSelectorAttachments['confluence.spaces'].execute(spaceDetailArgs()) + await expect(result).rejects.toBeInstanceOf(SelectorConnectionUnavailableError) + await expect(result).rejects.toMatchObject({ status: 401 }) + }) + + it('skips an arbitrary failure and preserves the next typed space-detail failure', async () => { + const fetchProviderJson = vi + .spyOn(providerHttp, 'fetchProviderJson') + .mockRejectedValueOnce(new Error('raw provider failure')) + .mockRejectedValueOnce(new SelectorConnectionUnavailableError(403)) + + const result = confluenceSelectorAttachments['confluence.spaces'].execute(spaceDetailArgs()) + await expect(result).rejects.toMatchObject({ + name: 'SelectorConnectionUnavailableError', + status: 403, + }) + + fetchProviderJson.mockRestore() + }) +}) diff --git a/apps/sim/lib/selectors/server/providers/confluence.ts b/apps/sim/lib/selectors/server/providers/confluence.ts new file mode 100644 index 00000000000..31a25f8dc58 --- /dev/null +++ b/apps/sim/lib/selectors/server/providers/confluence.ts @@ -0,0 +1,265 @@ +import { getScopesForService } from '@/lib/oauth/utils' +import type { ServerSelectorKey } from '@/lib/selectors/manifest' +import { + SelectorConnectionUnavailableError, + SelectorContextUnavailableError, + SelectorOptionsUnavailableError, +} from '@/lib/selectors/server/errors' +import { resolveSelectorAtlassianCloudId } from '@/lib/selectors/server/providers/atlassian' +import { resolveSelectorCredentialBundle } from '@/lib/selectors/server/providers/credential-bundle' +import { fetchProviderJson } from '@/lib/selectors/server/providers/provider-http' +import { + detailSelectorResult, + type ExecuteServerSelectorArgs, + listSelectorResult, + type ServerSelectorAttachmentMap, +} from '@/lib/selectors/server/types' + +type ConfluenceSelectorKey = Extract< + ServerSelectorKey, + 'confluence.spaces' | 'confluence.spacesById' | 'confluence.pages' +> + +const CONFLUENCE_SCOPES = getScopesForService('confluence') +const SPACE_PAGE_LIMIT = 250 +const PAGE_LIST_LIMIT = 50 + +type SpaceStatus = 'current' | 'archived' + +interface ConfluenceSpace { + id: string + name: string + key: string + status?: SpaceStatus +} + +interface ConfluenceSpacesResponse { + results?: ConfluenceSpace[] + _links?: { next?: string } +} + +interface ConfluencePage { + id: string + title: string +} + +interface ConfluencePagesResponse { + results?: ConfluencePage[] +} + +function isPublicSelectorError( + error: unknown +): error is + | SelectorContextUnavailableError + | SelectorConnectionUnavailableError + | SelectorOptionsUnavailableError { + return ( + error instanceof SelectorContextUnavailableError || + error instanceof SelectorConnectionUnavailableError || + error instanceof SelectorOptionsUnavailableError + ) +} + +function parseSpaceCursor(raw: string | undefined): { status: SpaceStatus; inner?: string } { + if (!raw) return { status: 'current' } + const separator = raw.indexOf(':') + if (separator < 0) return { status: 'current' } + const status = raw.slice(0, separator) === 'archived' ? 'archived' : 'current' + const inner = raw.slice(separator + 1) + return { status, ...(inner ? { inner } : {}) } +} + +function spaceOption( + space: ConfluenceSpace, + fallbackStatus: SpaceStatus, + identifier: 'key' | 'id' +) { + const status = space.status ?? fallbackStatus + const base = `${space.name} (${space.key})` + return { + id: identifier === 'id' ? space.id : space.key, + label: status === 'archived' ? `${base} — archived` : base, + } +} + +async function resolveConfluenceAuth(args: ExecuteServerSelectorArgs) { + const domain = args.context.domain + if (!domain) throw new SelectorContextUnavailableError() + + const bundle = await resolveSelectorCredentialBundle({ + credential: args.credential, + scopes: CONFLUENCE_SCOPES, + protectedValues: args.protectedValues, + recordCredentialUse: args.recordCredentialUse, + providerId: 'confluence', + }) + const cloudId = await resolveSelectorAtlassianCloudId({ + accessToken: bundle.accessToken, + domain, + providedCloudId: bundle.cloudId, + providedDomain: bundle.domain, + product: 'Confluence', + signal: args.signal, + }) + return { accessToken: bundle.accessToken, cloudId } +} + +async function requestSpaces(input: { + accessToken: string + cloudId: string + params: URLSearchParams + signal?: AbortSignal +}): Promise { + const url = new URL(`https://api.atlassian.com/ex/confluence/${input.cloudId}/wiki/api/v2/spaces`) + url.search = input.params.toString() + return fetchProviderJson(url, { + headers: { Accept: 'application/json', Authorization: `Bearer ${input.accessToken}` }, + signal: input.signal, + }) +} + +async function executeSpaces(args: ExecuteServerSelectorArgs, identifier: 'key' | 'id') { + const auth = await resolveConfluenceAuth(args) + + if (args.request.kind === 'detail') { + const requestedId = args.request.id.trim() + if (!requestedId || requestedId.length > 255) throw new SelectorContextUnavailableError() + if (/^[1-9][0-9]{0,19}$/.test(requestedId)) { + const space = await fetchProviderJson( + `https://api.atlassian.com/ex/confluence/${auth.cloudId}/wiki/api/v2/spaces/${requestedId}`, + { + headers: { Accept: 'application/json', Authorization: `Bearer ${auth.accessToken}` }, + signal: args.signal, + } + ) + if (!space.id || !space.key || !space.name) throw new SelectorOptionsUnavailableError() + return detailSelectorResult({ + ...spaceOption(space, space.status ?? 'current', identifier), + id: requestedId, + }) + } + + const key = requestedId + const paramsFor = (status: SpaceStatus) => + new URLSearchParams({ + keys: key, + limit: String(SPACE_PAGE_LIMIT), + status, + }) + const [current, archived] = await Promise.allSettled([ + requestSpaces({ ...auth, params: paramsFor('current'), signal: args.signal }), + requestSpaces({ ...auth, params: paramsFor('archived'), signal: args.signal }), + ]) + args.signal?.throwIfAborted() + if (current.status === 'rejected' && archived.status === 'rejected') { + for (const result of [current, archived]) { + if (isPublicSelectorError(result.reason)) throw result.reason + } + throw new SelectorOptionsUnavailableError() + } + const spaces = [ + ...(current.status === 'fulfilled' + ? (current.value.results ?? []).map((space) => ({ space, status: 'current' as const })) + : []), + ...(archived.status === 'fulfilled' + ? (archived.value.results ?? []).map((space) => ({ space, status: 'archived' as const })) + : []), + ] + const match = spaces.find(({ space }) => space.key === key) + return detailSelectorResult( + match + ? { + ...spaceOption(match.space, match.status, identifier), + id: requestedId, + } + : null + ) + } + + const { status, inner } = parseSpaceCursor(args.request.cursor) + const params = new URLSearchParams({ limit: String(SPACE_PAGE_LIMIT), status }) + if (inner) params.set('cursor', inner) + const data = await requestSpaces({ ...auth, params, signal: args.signal }) + + let nextInner: string | undefined + if (data._links?.next) { + try { + nextInner = + new URL(data._links.next, 'https://api.atlassian.com').searchParams.get('cursor') || + undefined + } catch { + nextInner = undefined + } + } + const nextCursor = nextInner + ? `${status}:${nextInner}` + : status === 'current' + ? 'archived:' + : undefined + return listSelectorResult( + (data.results ?? []).map((space) => spaceOption(space, status, identifier)), + nextCursor + ) +} + +async function executePages(args: ExecuteServerSelectorArgs) { + const auth = await resolveConfluenceAuth(args) + if (args.request.kind === 'detail') { + const pageId = args.request.id.trim() + if (!/^[A-Za-z0-9_-]{1,255}$/.test(pageId)) { + throw new SelectorContextUnavailableError() + } + const page = await fetchProviderJson( + `https://api.atlassian.com/ex/confluence/${auth.cloudId}/wiki/api/v2/pages/${pageId}`, + { + headers: { + Accept: 'application/json', + Authorization: `Bearer ${auth.accessToken}`, + }, + signal: args.signal, + } + ) + if (!page.id || !page.title) throw new SelectorOptionsUnavailableError() + return detailSelectorResult({ id: page.id, label: page.title }) + } + + const url = new URL(`https://api.atlassian.com/ex/confluence/${auth.cloudId}/wiki/api/v2/pages`) + url.searchParams.set('limit', String(PAGE_LIST_LIMIT)) + if (args.request.search) url.searchParams.set('title', args.request.search) + const data = await fetchProviderJson(url, { + headers: { + 'Content-Type': 'application/json', + Authorization: `Bearer ${auth.accessToken}`, + }, + signal: args.signal, + }) + return listSelectorResult( + (data.results ?? []) + .filter((page) => page.id && page.title) + .map((page) => ({ + id: page.id, + label: page.title, + })) + ) +} + +const credential = { kind: 'stored', field: 'oauthCredential', serviceIds: ['confluence'] } as const + +export const confluenceSelectorAttachments = { + 'confluence.spaces': { + credential, + destination: 'fixed', + execute: (args) => executeSpaces(args, 'key'), + }, + 'confluence.spacesById': { + credential, + destination: 'fixed', + execute: (args) => executeSpaces(args, 'id'), + }, + 'confluence.pages': { + credential, + destination: 'fixed', + auditCredentialUse: true, + execute: executePages, + }, +} satisfies ServerSelectorAttachmentMap diff --git a/apps/sim/lib/selectors/server/providers/credential-bundle.test.ts b/apps/sim/lib/selectors/server/providers/credential-bundle.test.ts new file mode 100644 index 00000000000..78607b9b596 --- /dev/null +++ b/apps/sim/lib/selectors/server/providers/credential-bundle.test.ts @@ -0,0 +1,101 @@ +/** + * @vitest-environment node + */ +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const mockResolveCredentialAccessToken = vi.hoisted(() => vi.fn()) + +vi.mock('@/lib/oauth/credential-service', () => ({ + resolveCredentialTokenBundle: mockResolveCredentialAccessToken, +})) + +import { createSelectorProtectedValues } from '@/lib/selectors/server/protected-values' +import { resolveSelectorCredentialBundle } from '@/lib/selectors/server/providers/credential-bundle' + +describe('selector credential bundles', () => { + beforeEach(() => vi.clearAllMocks()) + + it('protects short credential-bound cloud ids as exact identifiers', async () => { + mockResolveCredentialAccessToken.mockResolvedValue({ + accessToken: 'server-only-token', + cloudId: 'cloud-1', + domain: 'acme.atlassian.net', + }) + const protectedValues = createSelectorProtectedValues() + + await expect( + resolveSelectorCredentialBundle({ + credential: { + suppliedId: 'credential-1', + access: { ok: true, credentialOwnerUserId: 'owner-1' }, + }, + protectedValues, + }) + ).resolves.toMatchObject({ cloudId: 'cloud-1' }) + + expect(protectedValues.contains('cloud-1')).toBe(true) + expect(protectedValues.contains('prefix-cloud-1-suffix')).toBe(false) + }) + + it('preserves a selector abort without canceling the shared resolution', async () => { + let resolveShared!: (value: { accessToken: string }) => void + const sharedResolution = new Promise<{ accessToken: string }>((resolve) => { + resolveShared = resolve + }) + mockResolveCredentialAccessToken.mockReturnValue(sharedResolution) + const controller = new AbortController() + const protectedValues = createSelectorProtectedValues() + const pending = resolveSelectorCredentialBundle({ + credential: { + suppliedId: 'credential-1', + access: { ok: true, credentialOwnerUserId: 'owner-1' }, + signal: controller.signal, + }, + protectedValues, + }) + const abortReason = new DOMException('Selector request canceled', 'AbortError') + + controller.abort(abortReason) + await expect(pending).rejects.toBe(abortReason) + + resolveShared({ accessToken: 'shared-access-token' }) + await Promise.resolve() + expect(protectedValues.contains('shared-access-token')).toBe(false) + expect(mockResolveCredentialAccessToken).toHaveBeenCalledWith( + 'credential-1', + 'owner-1', + 'selector-execution', + undefined, + undefined, + { privacyMode: 'selector' } + ) + }) + + it('rechecks cancellation before consuming a fulfilled credential bundle', async () => { + mockResolveCredentialAccessToken.mockResolvedValue({ + accessToken: 'fulfilled-access-token', + cloudId: 'cloud-1', + }) + const controller = new AbortController() + const protectedValues = createSelectorProtectedValues() + const recordCredentialUse = vi.fn() + const abortReason = new DOMException('Selector request canceled', 'AbortError') + + const pending = resolveSelectorCredentialBundle({ + credential: { + suppliedId: 'credential-1', + access: { ok: true, credentialOwnerUserId: 'owner-1' }, + signal: controller.signal, + }, + protectedValues, + providerId: 'atlassian', + recordCredentialUse, + }) + queueMicrotask(() => controller.abort(abortReason)) + + await expect(pending).rejects.toBe(abortReason) + expect(protectedValues.contains('fulfilled-access-token')).toBe(false) + expect(protectedValues.contains('cloud-1')).toBe(false) + expect(recordCredentialUse).not.toHaveBeenCalled() + }) +}) diff --git a/apps/sim/lib/selectors/server/providers/credential-bundle.ts b/apps/sim/lib/selectors/server/providers/credential-bundle.ts new file mode 100644 index 00000000000..5ce91a2557e --- /dev/null +++ b/apps/sim/lib/selectors/server/providers/credential-bundle.ts @@ -0,0 +1,65 @@ +import { + resolveCredentialTokenBundle, + type ServiceAccountTokenResult, +} from '@/lib/oauth/credential-service' +import { waitForSelectorCredentialResolution } from '@/lib/selectors/server/credentials' +import { SelectorConnectionUnavailableError } from '@/lib/selectors/server/errors' +import type { + AuthorizedSelectorCredential, + SelectorProtectedValues, +} from '@/lib/selectors/server/types' + +/** + * Resolves credentials whose service-account variants need provider metadata in + * addition to the access token (for example Atlassian's cloud id). + */ +export async function resolveSelectorCredentialBundle(input: { + credential: AuthorizedSelectorCredential | undefined + scopes?: readonly string[] + impersonateEmail?: string + protectedValues: SelectorProtectedValues + recordCredentialUse?: (providerId: string) => void + providerId?: string +}): Promise { + const credential = input.credential + if (!credential) throw new SelectorConnectionUnavailableError() + + credential.signal?.throwIfAborted() + if (credential.fixedToken) { + if (input.providerId) { + input.recordCredentialUse?.(credential.providerId ?? input.providerId) + } + return { accessToken: credential.fixedToken } + } + + const ownerUserId = credential.access?.credentialOwnerUserId + if (!ownerUserId) throw new SelectorConnectionUnavailableError() + + let bundle: ServiceAccountTokenResult | null + try { + bundle = await waitForSelectorCredentialResolution( + resolveCredentialTokenBundle( + credential.suppliedId, + ownerUserId, + 'selector-execution', + input.scopes ? [...input.scopes] : undefined, + input.impersonateEmail, + { privacyMode: 'selector' } + ), + credential.signal + ) + credential.signal?.throwIfAborted() + } catch (error) { + if (credential.signal?.aborted) throw error + throw new SelectorConnectionUnavailableError() + } + if (!bundle?.accessToken) throw new SelectorConnectionUnavailableError() + + input.protectedValues.add(bundle.accessToken) + input.protectedValues.add(bundle.cloudId, 'reference') + input.protectedValues.add(bundle.domain, 'reference') + input.protectedValues.add(bundle.instanceUrl, 'reference') + input.protectedValues.add(bundle.apiDomain, 'reference') + if (input.providerId) input.recordCredentialUse?.(credential.providerId ?? input.providerId) + return bundle +} diff --git a/apps/sim/lib/selectors/server/providers/flat-results.ts b/apps/sim/lib/selectors/server/providers/flat-results.ts new file mode 100644 index 00000000000..1f9db909d7f --- /dev/null +++ b/apps/sim/lib/selectors/server/providers/flat-results.ts @@ -0,0 +1,23 @@ +import { SelectorOptionsUnavailableError } from '@/lib/selectors/server/errors' +import { + detailSelectorResult, + listSelectorResult, + type SelectorServerDiagnostics, + type ServerSelectorExecutionResult, +} from '@/lib/selectors/server/types' +import type { SafeSelectorOption, SelectorRequest } from '@/lib/selectors/types' + +/** Projects a bounded provider list into the selector operation's list/detail result. */ +export function flatSelectorResult( + request: SelectorRequest, + items: SafeSelectorOption[], + supportsDetail = false, + diagnostics?: SelectorServerDiagnostics +): ServerSelectorExecutionResult { + if (request.kind === 'list') return listSelectorResult(items, undefined, diagnostics) + if (!supportsDetail) throw new SelectorOptionsUnavailableError() + return { + ...detailSelectorResult(items.find((item) => item.id === request.id) ?? null), + ...(diagnostics ? { diagnostics } : {}), + } +} diff --git a/apps/sim/lib/selectors/server/providers/google.test.ts b/apps/sim/lib/selectors/server/providers/google.test.ts new file mode 100644 index 00000000000..104551e29eb --- /dev/null +++ b/apps/sim/lib/selectors/server/providers/google.test.ts @@ -0,0 +1,154 @@ +/** + * @vitest-environment node + */ +import { afterAll, beforeEach, describe, expect, it, vi } from 'vitest' + +const { mockFetch, mockResolveSelectorOAuthAccessToken } = vi.hoisted(() => ({ + mockFetch: vi.fn(), + mockResolveSelectorOAuthAccessToken: vi.fn(), +})) + +vi.mock('@/lib/selectors/server/credentials', () => ({ + resolveSelectorOAuthAccessToken: mockResolveSelectorOAuthAccessToken, +})) + +import { createSelectorProtectedValues } from '@/lib/selectors/server/protected-values' +import { googleSelectorAttachments } from '@/lib/selectors/server/providers/google' +import type { ExecuteServerSelectorArgs } from '@/lib/selectors/server/types' + +function driveDetailArgs(signal?: AbortSignal): ExecuteServerSelectorArgs { + return { + selectorKey: 'google.drive', + context: { oauthCredential: 'credential-1' }, + request: { kind: 'detail', id: 'drive-item-1' }, + scope: { kind: 'workspace', workspaceId: 'workspace-1' }, + workspaceId: 'workspace-1', + principal: { kind: 'session', userId: 'user-1', sessionId: 'session-1' }, + requesterUserId: 'user-1', + credential: { suppliedId: 'credential-1' }, + references: new Map(), + protectedValues: createSelectorProtectedValues(), + signal, + } +} + +function listArgs( + selectorKey: 'google.tasks.lists' | 'google.calendar', + cursor?: string +): ExecuteServerSelectorArgs { + return { + selectorKey, + context: { oauthCredential: 'credential-1' }, + request: { kind: 'list', ...(cursor ? { cursor } : {}) }, + scope: { kind: 'workspace', workspaceId: 'workspace-1' }, + workspaceId: 'workspace-1', + principal: { kind: 'session', userId: 'user-1', sessionId: 'session-1' }, + requesterUserId: 'user-1', + credential: { suppliedId: 'credential-1' }, + references: new Map(), + protectedValues: createSelectorProtectedValues(), + } +} + +describe('Google server selector adapters', () => { + beforeEach(() => { + vi.clearAllMocks() + vi.stubGlobal('fetch', mockFetch) + mockResolveSelectorOAuthAccessToken.mockResolvedValue('server-only-token') + }) + + afterAll(() => vi.unstubAllGlobals()) + + it('uses the bounded 404 path before hydrating a shared drive', async () => { + mockFetch + .mockResolvedValueOnce(new Response('not forwarded', { status: 404 })) + .mockResolvedValueOnce( + new Response(JSON.stringify({ id: 'drive-item-1', name: 'Shared drive' }), { + status: 200, + }) + ) + + await expect( + googleSelectorAttachments['google.drive'].execute(driveDetailArgs()) + ).resolves.toEqual({ + kind: 'detail', + item: { id: 'drive-item-1', label: 'Shared drive' }, + }) + + expect(String(mockFetch.mock.calls[0]?.[0])).toContain('/drive/v3/files/drive-item-1') + expect(String(mockFetch.mock.calls[1]?.[0])).toContain('/drive/v3/drives/drive-item-1') + }) + + it('preserves caller cancellation during detail hydration', async () => { + const controller = new AbortController() + const abortError = new DOMException('The operation was aborted', 'AbortError') + controller.abort() + mockFetch.mockRejectedValueOnce(abortError) + + await expect( + googleSelectorAttachments['google.drive'].execute(driveDetailArgs(controller.signal)) + ).rejects.toBe(abortError) + }) + + it('returns one task-list page and preserves the continuation token', async () => { + const items = Array.from({ length: 1_000 }, (_, index) => ({ + id: `task-list-${index}`, + title: `Task list ${index}`, + })) + mockFetch.mockResolvedValueOnce( + new Response(JSON.stringify({ items, nextPageToken: 'page-1' }), { status: 200 }) + ) + + const result = await googleSelectorAttachments['google.tasks.lists'].execute( + listArgs('google.tasks.lists') + ) + + expect(result).toMatchObject({ kind: 'list', nextCursor: 'page-1' }) + expect(result.kind === 'list' ? result.items : []).toHaveLength(1_000) + expect(mockFetch).toHaveBeenCalledTimes(1) + }) + + it('forwards a Google continuation token on demand', async () => { + mockFetch.mockResolvedValueOnce( + new Response( + JSON.stringify({ + items: [{ id: 'calendar-2', summary: 'Calendar 2' }], + nextPageToken: 'page-3', + }), + { status: 200 } + ) + ) + + await expect( + googleSelectorAttachments['google.calendar'].execute(listArgs('google.calendar', 'page-2')) + ).resolves.toEqual({ + kind: 'list', + items: [{ id: 'calendar-2', label: 'Calendar 2' }], + nextCursor: 'page-3', + }) + expect(new URL(String(mockFetch.mock.calls[0]?.[0])).searchParams.get('pageToken')).toBe( + 'page-2' + ) + expect(mockFetch).toHaveBeenCalledTimes(1) + }) + + it('hydrates a selected calendar without traversing the calendar list', async () => { + mockFetch.mockResolvedValueOnce( + new Response(JSON.stringify({ id: 'team@example.com', summary: 'Team calendar' }), { + status: 200, + }) + ) + + await expect( + googleSelectorAttachments['google.calendar'].execute({ + ...listArgs('google.calendar'), + request: { kind: 'detail', id: 'team@example.com' }, + }) + ).resolves.toEqual({ + kind: 'detail', + item: { id: 'team@example.com', label: 'Team calendar' }, + }) + expect(String(mockFetch.mock.calls[0]?.[0])).toContain('/calendars/team%40example.com') + expect(mockFetch).toHaveBeenCalledTimes(1) + }) +}) diff --git a/apps/sim/lib/selectors/server/providers/google.ts b/apps/sim/lib/selectors/server/providers/google.ts new file mode 100644 index 00000000000..418df995295 --- /dev/null +++ b/apps/sim/lib/selectors/server/providers/google.ts @@ -0,0 +1,439 @@ +import { getScopesForService } from '@/lib/oauth/utils' +import type { ServerSelectorKey } from '@/lib/selectors/manifest' +import { resolveSelectorOAuthAccessToken } from '@/lib/selectors/server/credentials' +import { + SelectorConnectionUnavailableError, + SelectorContextUnavailableError, + SelectorOptionsUnavailableError, +} from '@/lib/selectors/server/errors' +import { + fetchProviderJson, + fetchProviderJsonWithStatus, +} from '@/lib/selectors/server/providers/provider-http' +import { + detailSelectorResult, + type ExecuteServerSelectorArgs, + listSelectorResult, + requireListRequest, + type ServerSelectorAttachmentMap, +} from '@/lib/selectors/server/types' + +type GoogleSelectorKey = Extract< + ServerSelectorKey, + 'google.tasks.lists' | 'gmail.labels' | 'google.calendar' | 'google.drive' | 'google.sheets' +> + +interface GoogleTaskList { + id: string + title: string +} + +interface CalendarListItem { + id: string + summary: string + primary?: boolean +} + +function requireGoogleCalendarId(value: string): string { + const id = value.trim() + if (!id || id.length > 1_024 || /[\u0000-\u001F\u007F]/.test(id)) { + throw new SelectorContextUnavailableError() + } + return id +} + +interface DriveFile { + id: string + name: string + mimeType: string + shortcutDetails?: { targetId?: string } +} + +interface GmailLabel { + id: string + name: string + type?: 'system' | 'user' +} + +interface Sheet { + properties: { sheetId: number; title: string; index: number } +} + +interface GooglePage { + items: T[] + nextCursor?: string +} + +async function googleAccessToken(args: ExecuteServerSelectorArgs, serviceId: string) { + if (!args.credential) throw new SelectorConnectionUnavailableError() + try { + return await resolveSelectorOAuthAccessToken({ + credential: args.credential, + serviceId, + scopes: getScopesForService(serviceId), + impersonateEmail: args.context.impersonateUserEmail, + protectedValues: args.protectedValues, + }) + } catch (error) { + if (error instanceof SelectorConnectionUnavailableError) throw error + throw new SelectorConnectionUnavailableError() + } +} + +async function fetchGooglePage(input: { + args: ExecuteServerSelectorArgs + accessToken: string + buildUrl(pageToken: string | undefined): URL + getItems(page: R): T[] | undefined +}): Promise> { + const request = requireListRequest(input.args.selectorKey, input.args.request) + const body = await fetchProviderJson(input.buildUrl(request.cursor), { + headers: { Authorization: `Bearer ${input.accessToken}` }, + signal: input.args.signal, + }) + const nextCursor = body.nextPageToken?.trim() || undefined + return { + items: input.getItems(body) ?? [], + ...(nextCursor ? { nextCursor } : {}), + } +} + +async function listTaskLists(args: ExecuteServerSelectorArgs): Promise> { + const accessToken = await googleAccessToken(args, 'google-tasks') + return fetchGooglePage({ + args, + accessToken, + buildUrl: (pageToken) => { + const url = new URL('https://tasks.googleapis.com/tasks/v1/users/@me/lists') + url.searchParams.set('maxResults', '1000') + if (pageToken) url.searchParams.set('pageToken', pageToken) + return url + }, + getItems: (page) => page.items, + }) +} + +async function executeTaskLists(args: ExecuteServerSelectorArgs) { + if (args.request.kind === 'detail') { + const accessToken = await googleAccessToken(args, 'google-tasks') + const detailId = requireGoogleId(args.request.id) + const list = await fetchProviderJson( + `https://tasks.googleapis.com/tasks/v1/users/@me/lists/${detailId}`, + { headers: { Authorization: `Bearer ${accessToken}` }, signal: args.signal } + ) + return detailSelectorResult(list.id && list.title ? { id: list.id, label: list.title } : null) + } + const result = await listTaskLists(args) + return listSelectorResult( + result.items + .filter((list) => list.id && list.title) + .map((list) => ({ + id: list.id, + label: list.title, + })), + result.nextCursor + ) +} + +function gmailLabelName(label: GmailLabel): string { + if (label.type !== 'system') return label.name + return label.name.charAt(0).toUpperCase() + label.name.slice(1).toLowerCase() +} + +async function executeGmailLabels(args: ExecuteServerSelectorArgs) { + requireListRequest(args.selectorKey, args.request) + const accessToken = await googleAccessToken(args, 'gmail') + const data = await fetchProviderJson<{ labels?: GmailLabel[] }>( + 'https://gmail.googleapis.com/gmail/v1/users/me/labels', + { headers: { Authorization: `Bearer ${accessToken}` }, signal: args.signal } + ) + if (!Array.isArray(data.labels)) throw new SelectorOptionsUnavailableError() + return listSelectorResult( + data.labels + .filter((label) => label.id && label.name) + .map((label) => ({ + id: label.id, + label: gmailLabelName(label), + })) + ) +} + +async function executeCalendars(args: ExecuteServerSelectorArgs) { + const accessToken = await googleAccessToken(args, 'google-calendar') + if (args.request.kind === 'detail') { + const calendar = await fetchProviderJson( + `https://www.googleapis.com/calendar/v3/calendars/${encodeURIComponent(requireGoogleCalendarId(args.request.id))}`, + { headers: { Authorization: `Bearer ${accessToken}` }, signal: args.signal } + ) + return detailSelectorResult({ id: calendar.id, label: calendar.summary }) + } + requireListRequest(args.selectorKey, args.request) + const result = await fetchGooglePage< + CalendarListItem, + { items?: CalendarListItem[]; nextPageToken?: string } + >({ + args, + accessToken, + buildUrl: (pageToken) => { + const url = new URL('https://www.googleapis.com/calendar/v3/users/me/calendarList') + url.searchParams.set('maxResults', '250') + if (pageToken) url.searchParams.set('pageToken', pageToken) + return url + }, + getItems: (page) => page.items, + }) + const calendars = result.items + calendars.sort((a, b) => { + if (a.primary && !b.primary) return -1 + if (!a.primary && b.primary) return 1 + return a.summary.localeCompare(b.summary) + }) + return listSelectorResult( + calendars + .filter((calendar) => calendar.id && calendar.summary) + .map((calendar) => ({ + id: calendar.id, + label: calendar.summary, + })), + result.nextCursor + ) +} + +function escapeDriveQuery(value: string): string { + return value.replace(/\\/g, '\\\\').replace(/'/g, "\\'") +} + +function requireGoogleId(value: string | undefined, maxLength = 255): string { + const trimmed = value?.trim() ?? '' + if (!trimmed || trimmed.length > maxLength || !/^[A-Za-z0-9_-]+$/.test(trimmed)) { + throw new SelectorContextUnavailableError() + } + return trimmed +} + +async function fetchSharedDrivePage( + accessToken: string, + pageToken: string | undefined, + signal?: AbortSignal +): Promise> { + try { + const url = new URL('https://www.googleapis.com/drive/v3/drives') + url.searchParams.set('pageSize', '100') + url.searchParams.set('fields', 'nextPageToken,drives(id,name)') + if (pageToken) url.searchParams.set('pageToken', pageToken) + const data = await fetchProviderJson<{ + drives?: Array<{ id: string; name: string }> + nextPageToken?: string + }>(url, { headers: { Authorization: `Bearer ${accessToken}` }, signal }) + const nextCursor = data.nextPageToken?.trim() + return { + items: (data.drives ?? []).map((drive) => ({ + id: drive.id, + name: drive.name, + mimeType: 'application/vnd.google-apps.folder', + })), + ...(nextCursor ? { nextCursor } : {}), + } + } catch (error) { + if (signal?.aborted) throw error + return { items: [] } + } +} + +type DriveCursor = { source: 'drives' | 'files'; pageToken?: string } + +function parseDriveCursor(cursor: string | undefined): DriveCursor | undefined { + if (!cursor) return undefined + const source = cursor.slice(0, 2) + if (source !== 'd:' && source !== 'f:') throw new SelectorContextUnavailableError() + const pageToken = cursor.slice(2) || undefined + return { source: source === 'd:' ? 'drives' : 'files', pageToken } +} + +function driveCursor(source: DriveCursor['source'], pageToken?: string): string { + return `${source === 'drives' ? 'd' : 'f'}:${pageToken ?? ''}` +} + +async function listDriveFiles( + args: ExecuteServerSelectorArgs, + accessToken: string +): Promise> { + const folderId = args.context.fileId?.trim() + if (folderId) requireGoogleId(folderId, 50) + + const mimeType = args.context.mimeType + const search = args.request.kind === 'list' ? args.request.search : undefined + const clauses = ['trashed = false'] + if (folderId) clauses.push(`'${escapeDriveQuery(folderId)}' in parents`) + if (mimeType) clauses.push(`mimeType = '${escapeDriveQuery(mimeType)}'`) + if (search) clauses.push(`name contains '${escapeDriveQuery(search)}'`) + + const includeSharedDrives = + !folderId && mimeType === 'application/vnd.google-apps.folder' && !search + const request = requireListRequest(args.selectorKey, args.request) + const cursor = parseDriveCursor(request.cursor) + if (cursor?.source === 'drives' && !includeSharedDrives) { + throw new SelectorContextUnavailableError() + } + + if (includeSharedDrives && (!cursor || cursor.source === 'drives')) { + const drives = await fetchSharedDrivePage(accessToken, cursor?.pageToken, args.signal) + if (drives.items.length > 0 || drives.nextCursor) { + return { + items: drives.items, + nextCursor: drives.nextCursor + ? driveCursor('drives', drives.nextCursor) + : driveCursor('files'), + } + } + } + + const pageToken = cursor?.source === 'files' ? cursor.pageToken : undefined + const url = new URL('https://www.googleapis.com/drive/v3/files') + url.searchParams.set('q', clauses.join(' and ')) + url.searchParams.set('corpora', 'allDrives') + url.searchParams.set('supportsAllDrives', 'true') + url.searchParams.set('includeItemsFromAllDrives', 'true') + url.searchParams.set('pageSize', '100') + url.searchParams.set('fields', 'nextPageToken,files(id,name,mimeType)') + if (pageToken) url.searchParams.set('pageToken', pageToken) + const data = await fetchProviderJson<{ files?: DriveFile[]; nextPageToken?: string }>(url, { + headers: { Authorization: `Bearer ${accessToken}` }, + signal: args.signal, + }) + const nextPageToken = data.nextPageToken?.trim() + + return { + items: data.files ?? [], + ...(nextPageToken ? { nextCursor: driveCursor('files', nextPageToken) } : {}), + } +} + +async function fetchDriveDetail( + args: ExecuteServerSelectorArgs, + accessToken: string, + fileId: string +): Promise { + const id = requireGoogleId(fileId) + const headers = { Authorization: `Bearer ${accessToken}` } + const result = await fetchProviderJsonWithStatus( + `https://www.googleapis.com/drive/v3/files/${id}?fields=id,name,mimeType,shortcutDetails&supportsAllDrives=true`, + { headers, redirect: 'error', signal: args.signal }, + { passthroughStatuses: [404] } + ) + + if (!result.ok) { + const drive = await fetchProviderJson<{ id: string; name: string }>( + `https://www.googleapis.com/drive/v3/drives/${id}?fields=id,name`, + { headers, signal: args.signal } + ) + return { id: drive.id, name: drive.name, mimeType: 'application/vnd.google-apps.folder' } + } + const file = result.data + const targetId = + file.mimeType === 'application/vnd.google-apps.shortcut' + ? file.shortcutDetails?.targetId + : undefined + if (!targetId) return file + + let validatedTargetId: string + try { + validatedTargetId = requireGoogleId(targetId) + } catch { + return file + } + try { + const target = await fetchProviderJson( + `https://www.googleapis.com/drive/v3/files/${validatedTargetId}?fields=id,name,mimeType&supportsAllDrives=true`, + { headers, signal: args.signal } + ) + return { ...target, id: file.id } + } catch (error) { + if (args.signal?.aborted) throw error + return file + } +} + +async function executeDrive(args: ExecuteServerSelectorArgs) { + const accessToken = await googleAccessToken(args, 'google-drive') + if (args.request.kind === 'detail') { + const file = await fetchDriveDetail(args, accessToken, args.request.id) + if (!file.id || !file.name) throw new SelectorOptionsUnavailableError() + return detailSelectorResult({ id: file.id, label: file.name }) + } + const result = await listDriveFiles(args, accessToken) + return listSelectorResult( + result.items + .filter((file) => file.id && file.name) + .map((file) => ({ + id: file.id, + label: file.name, + })), + result.nextCursor + ) +} + +async function executeSheets(args: ExecuteServerSelectorArgs) { + requireListRequest(args.selectorKey, args.request) + const spreadsheetId = args.context.spreadsheetId?.trim() + if (!spreadsheetId) throw new SelectorContextUnavailableError() + const validatedSpreadsheetId = requireGoogleId(spreadsheetId) + + const accessToken = await googleAccessToken(args, 'google-sheets') + const data = await fetchProviderJson<{ sheets?: Sheet[] }>( + `https://sheets.googleapis.com/v4/spreadsheets/${validatedSpreadsheetId}?fields=sheets.properties`, + { + headers: { + Authorization: `Bearer ${accessToken}`, + 'Content-Type': 'application/json', + }, + signal: args.signal, + } + ) + const sheets = data.sheets ?? [] + sheets.sort((a, b) => a.properties.index - b.properties.index) + return listSelectorResult( + sheets + .filter((sheet) => sheet.properties?.title) + .map((sheet) => ({ + id: sheet.properties.title, + label: sheet.properties.title, + })) + ) +} + +const storedCredential = (serviceIds: readonly string[]) => + ({ kind: 'stored', field: 'oauthCredential', serviceIds }) as const + +export const googleSelectorAttachments = { + 'google.tasks.lists': { + credential: storedCredential(['google-tasks']), + destination: 'fixed', + execute: executeTaskLists, + }, + 'gmail.labels': { + credential: storedCredential(['gmail']), + destination: 'fixed', + execute: executeGmailLabels, + }, + 'google.calendar': { + credential: storedCredential(['google-calendar']), + destination: 'fixed', + execute: executeCalendars, + }, + 'google.drive': { + credential: { + kind: 'stored', + field: 'oauthCredential', + serviceIds: ['google-drive', 'google-docs', 'google-sheets', 'google-forms'], + resourceServiceId: 'google-drive', + }, + destination: 'fixed', + execute: executeDrive, + }, + 'google.sheets': { + credential: storedCredential(['google-sheets']), + destination: 'fixed', + execute: executeSheets, + }, +} satisfies ServerSelectorAttachmentMap diff --git a/apps/sim/lib/selectors/server/providers/harmonic.test.ts b/apps/sim/lib/selectors/server/providers/harmonic.test.ts new file mode 100644 index 00000000000..3a7a3793d37 --- /dev/null +++ b/apps/sim/lib/selectors/server/providers/harmonic.test.ts @@ -0,0 +1,68 @@ +/** + * @vitest-environment node + */ +import { afterAll, beforeEach, describe, expect, it, vi } from 'vitest' + +const { mockFetch, mockResolveCredentialBundle } = vi.hoisted(() => ({ + mockFetch: vi.fn(), + mockResolveCredentialBundle: vi.fn(), +})) + +vi.mock('@/lib/selectors/server/providers/credential-bundle', () => ({ + resolveSelectorCredentialBundle: mockResolveCredentialBundle, +})) + +import { createSelectorProtectedValues } from '@/lib/selectors/server/protected-values' +import { harmonicSelectorAttachments } from '@/lib/selectors/server/providers/harmonic' +import type { ExecuteServerSelectorArgs } from '@/lib/selectors/server/types' + +function detailArgs(id: string): ExecuteServerSelectorArgs { + return { + selectorKey: 'harmonic.savedSearches', + context: { oauthCredential: 'credential-1' }, + request: { kind: 'detail', id }, + scope: { kind: 'workspace', workspaceId: 'workspace-1' }, + workspaceId: 'workspace-1', + principal: { kind: 'session', userId: 'user-1', sessionId: 'session-1' }, + requesterUserId: 'user-1', + credential: { suppliedId: 'credential-1' }, + references: new Map(), + protectedValues: createSelectorProtectedValues(), + } +} + +describe('Harmonic server selector adapter', () => { + beforeEach(() => { + vi.clearAllMocks() + vi.stubGlobal('fetch', mockFetch) + mockResolveCredentialBundle.mockResolvedValue({ accessToken: 'server-only-token' }) + }) + + afterAll(() => vi.unstubAllGlobals()) + + it('hydrates a selected saved search after the list projection cap', async () => { + const rows = Array.from({ length: 501 }, (_, index) => ({ + id: index + 1, + entity_urn: `urn:harmonic:saved_search:${index + 1}`, + name: `Search ${index + 1}`, + type: 'PERSONS', + })) + mockFetch.mockResolvedValueOnce(new Response(JSON.stringify(rows), { status: 200 })) + + await expect( + harmonicSelectorAttachments['harmonic.savedSearches'].execute(detailArgs('501')) + ).resolves.toMatchObject({ + kind: 'detail', + item: { + id: '501', + label: 'Search 501', + meta: { + id: '501', + urn: 'urn:harmonic:saved_search:501', + name: 'Search 501', + }, + }, + diagnostics: { truncated: { reason: 'provider-cap', limit: 500 } }, + }) + }) +}) diff --git a/apps/sim/lib/selectors/server/providers/harmonic.ts b/apps/sim/lib/selectors/server/providers/harmonic.ts new file mode 100644 index 00000000000..52f4f04eeed --- /dev/null +++ b/apps/sim/lib/selectors/server/providers/harmonic.ts @@ -0,0 +1,191 @@ +import { isPlainRecord } from '@sim/utils/object' +import { z } from 'zod' +import { readResponseJsonWithLimit } from '@/lib/core/utils/stream-limits' +import type { ServerSelectorKey } from '@/lib/selectors/manifest' +import { SelectorOptionsUnavailableError } from '@/lib/selectors/server/errors' +import { resolveSelectorCredentialBundle } from '@/lib/selectors/server/providers/credential-bundle' +import { selectorProviderStatusError } from '@/lib/selectors/server/providers/provider-http' +import { + detailSelectorResult, + type ExecuteServerSelectorArgs, + listSelectorResult, + type ServerSelectorAttachmentMap, +} from '@/lib/selectors/server/types' + +type HarmonicSelectorKey = Extract + +const HARMONIC_URL = 'https://api.harmonic.ai/savedSearches' +const HARMONIC_SAVED_SEARCH_SELECTOR_MAX_OPTIONS = 500 +const MAX_RESPONSE_BYTES = 1024 * 1024 +const MAX_PROVIDER_ROWS = 2_000 +const FETCH_TIMEOUT_MS = 10_000 + +const harmonicSavedSearchUrnSchema = z + .string() + .trim() + .min(1) + .max(512) + .regex(/^urn:harmonic:saved_search:[^\s]+$/, 'Invalid Harmonic saved-search URN') +const harmonicSavedSearchNameSchema = z.string().trim().min(1).max(1_000) + +/** Validates the documented fields consumed from a PERSONS saved-search row. */ +const harmonicPeopleSavedSearchProviderSchema = z + .object({ + id: z.number().int().safe(), + entity_urn: harmonicSavedSearchUrnSchema, + name: harmonicSavedSearchNameSchema, + type: z.literal('PERSONS'), + }) + .passthrough() + +interface SavedSearch { + id: string + urn: string + name: string +} + +function normalizeSavedSearches( + value: unknown, + requestedId?: string +): { items: SavedSearch[]; detailItem?: SavedSearch; truncated: boolean } { + if (!Array.isArray(value) || value.length > MAX_PROVIDER_ROWS) { + throw new SelectorOptionsUnavailableError() + } + + const byUrn = new Map() + const urnById = new Map() + const items: SavedSearch[] = [] + let detailItem: SavedSearch | undefined + let truncated = false + for (const item of value) { + if (!isPlainRecord(item) || item.type !== 'PERSONS') continue + const parsed = harmonicPeopleSavedSearchProviderSchema.safeParse(item) + if (!parsed.success) throw new SelectorOptionsUnavailableError() + const option = { + id: String(parsed.data.id), + urn: parsed.data.entity_urn, + name: parsed.data.name, + } + const existing = byUrn.get(option.urn) + const existingUrn = urnById.get(option.id) + if ( + (existing && (existing.id !== option.id || existing.name !== option.name)) || + (existingUrn && existingUrn !== option.urn) + ) { + throw new SelectorOptionsUnavailableError() + } + if (existing) { + if (requestedId === existing.urn || requestedId === existing.id) detailItem = existing + continue + } + byUrn.set(option.urn, option) + urnById.set(option.id, option.urn) + if (requestedId === option.urn || requestedId === option.id) detailItem = option + if (items.length < HARMONIC_SAVED_SEARCH_SELECTOR_MAX_OPTIONS) items.push(option) + else truncated = true + } + return { + items: items.sort( + (left, right) => left.name.localeCompare(right.name) || left.urn.localeCompare(right.urn) + ), + ...(detailItem ? { detailItem } : {}), + truncated, + } +} + +async function listSavedSearches( + args: ExecuteServerSelectorArgs +): Promise<{ items: SavedSearch[]; detailItem?: SavedSearch; truncated: boolean }> { + const { accessToken } = await resolveSelectorCredentialBundle({ + credential: args.credential, + protectedValues: args.protectedValues, + }) + const timeoutSignal = AbortSignal.timeout(FETCH_TIMEOUT_MS) + const signal = args.signal ? AbortSignal.any([args.signal, timeoutSignal]) : timeoutSignal + + let response: Response + try { + response = await fetch(HARMONIC_URL, { + headers: { Accept: 'application/json', apikey: accessToken }, + redirect: 'error', + signal, + }) + } catch (error) { + if (args.signal?.aborted) throw error + throw new SelectorOptionsUnavailableError() + } + if (!response.ok) { + await response.body?.cancel().catch(() => {}) + throw selectorProviderStatusError(response.status) + } + + try { + const body = await readResponseJsonWithLimit(response, { + label: 'Harmonic saved-search response', + maxBytes: MAX_RESPONSE_BYTES, + signal, + }) + return normalizeSavedSearches( + body, + args.request.kind === 'detail' ? args.request.id.trim() : undefined + ) + } catch (error) { + if (args.signal?.aborted) throw error + if (error instanceof SelectorOptionsUnavailableError) throw error + throw new SelectorOptionsUnavailableError() + } +} + +function toOption(search: SavedSearch, id = search.urn) { + return { + id, + label: search.name, + meta: { id: search.id, urn: search.urn, name: search.name }, + } +} + +async function executeSavedSearches(args: ExecuteServerSelectorArgs) { + const { items: searches, detailItem, truncated } = await listSavedSearches(args) + if (args.request.kind === 'detail') { + return { + ...detailSelectorResult(detailItem ? toOption(detailItem, args.request.id) : null), + ...(truncated + ? { + diagnostics: { + truncated: { + reason: 'provider-cap' as const, + limit: HARMONIC_SAVED_SEARCH_SELECTOR_MAX_OPTIONS, + }, + }, + } + : {}), + } + } + return listSelectorResult( + searches.map((search) => toOption(search)), + undefined, + truncated + ? { + truncated: { + reason: 'provider-cap', + limit: HARMONIC_SAVED_SEARCH_SELECTOR_MAX_OPTIONS, + }, + } + : undefined + ) +} + +/** + * The integration this selector reaches. Declared rather than derived: Harmonic is an API-key integration with no entry in the deployment OAuth + * catalog, so its service id maps to no block type. + */ +const integrationBlockTypes = ['harmonic'] as const + +export const harmonicSelectorAttachments = { + 'harmonic.savedSearches': { + credential: { kind: 'stored', field: 'oauthCredential', serviceIds: ['harmonic'] }, + integrationBlockTypes, + destination: 'fixed', + execute: executeSavedSearches, + }, +} satisfies ServerSelectorAttachmentMap diff --git a/apps/sim/lib/selectors/server/providers/hubspot.test.ts b/apps/sim/lib/selectors/server/providers/hubspot.test.ts new file mode 100644 index 00000000000..c421f7189b6 --- /dev/null +++ b/apps/sim/lib/selectors/server/providers/hubspot.test.ts @@ -0,0 +1,182 @@ +/** + * @vitest-environment node + */ +import { afterAll, beforeEach, describe, expect, it, vi } from 'vitest' + +const { mockFetch, mockResolveSelectorOAuthAccessToken } = vi.hoisted(() => ({ + mockFetch: vi.fn(), + mockResolveSelectorOAuthAccessToken: vi.fn(), +})) + +vi.mock('@/lib/selectors/server/credentials', () => ({ + resolveSelectorOAuthAccessToken: mockResolveSelectorOAuthAccessToken, +})) + +import { createSelectorProtectedValues } from '@/lib/selectors/server/protected-values' +import { hubspotSelectorAttachments } from '@/lib/selectors/server/providers/hubspot' +import type { ExecuteServerSelectorArgs } from '@/lib/selectors/server/types' + +function args( + request: ExecuteServerSelectorArgs['request'], + selectorKey: ExecuteServerSelectorArgs['selectorKey'] = 'hubspot.lists' +): ExecuteServerSelectorArgs { + return { + selectorKey, + context: { oauthCredential: 'credential-1' }, + request, + scope: { kind: 'workspace', workspaceId: 'workspace-1' }, + workspaceId: 'workspace-1', + principal: { kind: 'session', userId: 'user-1', sessionId: 'session-1' }, + requesterUserId: 'user-1', + credential: { suppliedId: 'credential-1' }, + references: new Map(), + protectedValues: createSelectorProtectedValues(), + } +} + +describe('HubSpot server selector adapter', () => { + beforeEach(() => { + vi.clearAllMocks() + vi.stubGlobal('fetch', mockFetch) + mockResolveSelectorOAuthAccessToken.mockResolvedValue('server-only-token') + }) + + afterAll(() => vi.unstubAllGlobals()) + + it('preserves list search and follows the response offset on demand', async () => { + mockFetch + .mockResolvedValueOnce( + new Response( + JSON.stringify({ + hasMore: true, + lists: [{ listId: 'list-1', name: 'Revenue prospects' }], + offset: 500, + total: 501, + }), + { status: 200 } + ) + ) + .mockResolvedValueOnce( + new Response( + JSON.stringify({ + hasMore: false, + lists: [{ listId: 'list-2', name: 'Revenue customers' }], + offset: 501, + total: 501, + }), + { status: 200 } + ) + ) + + const first = await hubspotSelectorAttachments['hubspot.lists'].execute( + args({ kind: 'list', search: ' Revenue ' }) + ) + const second = await hubspotSelectorAttachments['hubspot.lists'].execute( + args({ kind: 'list', search: ' Revenue ', cursor: '500' }) + ) + + expect(first).toEqual({ + kind: 'list', + items: [{ id: 'list-1', label: 'Revenue prospects' }], + nextCursor: '500', + }) + expect(second).toEqual({ + kind: 'list', + items: [{ id: 'list-2', label: 'Revenue customers' }], + }) + expect(String(mockFetch.mock.calls[0]?.[0])).toBe('https://api.hubapi.com/crm/v3/lists/search') + expect(JSON.parse(String(mockFetch.mock.calls[0]?.[1]?.body))).toEqual({ + count: 500, + offset: 0, + query: 'Revenue', + processingTypes: ['MANUAL', 'DYNAMIC', 'SNAPSHOT'], + }) + expect(JSON.parse(String(mockFetch.mock.calls[1]?.[1]?.body))).toEqual({ + count: 500, + offset: 500, + query: 'Revenue', + processingTypes: ['MANUAL', 'DYNAMIC', 'SNAPSHOT'], + }) + expect(mockFetch).toHaveBeenCalledTimes(2) + }) + + it('hydrates a selected list directly by id', async () => { + mockFetch.mockResolvedValueOnce( + new Response(JSON.stringify({ list: { listId: '123', name: 'Revenue prospects' } }), { + status: 200, + }) + ) + + await expect( + hubspotSelectorAttachments['hubspot.lists'].execute(args({ kind: 'detail', id: '123' })) + ).resolves.toEqual({ + kind: 'detail', + item: { id: '123', label: 'Revenue prospects' }, + }) + expect(String(mockFetch.mock.calls[0]?.[0])).toBe('https://api.hubapi.com/crm/v3/lists/123') + expect(mockFetch).toHaveBeenCalledTimes(1) + }) + + it('paginates active owners through the HubSpot continuation cursor on demand', async () => { + mockFetch + .mockResolvedValueOnce( + new Response( + JSON.stringify({ + results: [ + { id: '100', firstName: 'Former', lastName: 'Owner', archived: true }, + { id: '101', firstName: 'Ada', lastName: 'Lovelace', archived: false }, + ], + paging: { next: { after: 'owner-page-2' } }, + }), + { status: 200 } + ) + ) + .mockResolvedValueOnce( + new Response(JSON.stringify({ results: [{ id: '102', email: 'grace@example.com' }] }), { + status: 200, + }) + ) + + const first = await hubspotSelectorAttachments['hubspot.owners'].execute( + args({ kind: 'list' }, 'hubspot.owners') + ) + const second = await hubspotSelectorAttachments['hubspot.owners'].execute( + args({ kind: 'list', cursor: 'owner-page-2' }, 'hubspot.owners') + ) + + expect(first).toEqual({ + kind: 'list', + items: [{ id: '101', label: 'Ada Lovelace' }], + nextCursor: 'owner-page-2', + }) + expect(second).toEqual({ + kind: 'list', + items: [{ id: '102', label: 'grace@example.com' }], + }) + const firstUrl = new URL(String(mockFetch.mock.calls[0]?.[0])) + const secondUrl = new URL(String(mockFetch.mock.calls[1]?.[0])) + expect(firstUrl.searchParams.get('limit')).toBe('100') + expect(firstUrl.searchParams.has('after')).toBe(false) + expect(secondUrl.searchParams.get('after')).toBe('owner-page-2') + expect(mockFetch).toHaveBeenCalledTimes(2) + }) + + it('hydrates a selected owner directly by id', async () => { + mockFetch.mockResolvedValueOnce( + new Response(JSON.stringify({ id: '777', firstName: 'Katherine', lastName: 'Johnson' }), { + status: 200, + }) + ) + + await expect( + hubspotSelectorAttachments['hubspot.owners'].execute( + args({ kind: 'detail', id: '000777' }, 'hubspot.owners') + ) + ).resolves.toEqual({ + kind: 'detail', + item: { id: '000777', label: 'Katherine Johnson' }, + }) + expect(String(mockFetch.mock.calls[0]?.[0])).toBe('https://api.hubapi.com/crm/v3/owners/000777') + expect(mockFetch).toHaveBeenCalledTimes(1) + }) +}) diff --git a/apps/sim/lib/selectors/server/providers/hubspot.ts b/apps/sim/lib/selectors/server/providers/hubspot.ts new file mode 100644 index 00000000000..be1b0d69331 --- /dev/null +++ b/apps/sim/lib/selectors/server/providers/hubspot.ts @@ -0,0 +1,257 @@ +import { z } from 'zod' +import { getScopesForService } from '@/lib/oauth/utils' +import { MAX_SELECTOR_OPTIONS } from '@/lib/selectors/limits' +import type { ServerSelectorKey } from '@/lib/selectors/manifest' +import { resolveSelectorOAuthAccessToken } from '@/lib/selectors/server/credentials' +import { + SelectorConnectionUnavailableError, + SelectorContextUnavailableError, + SelectorOptionsUnavailableError, +} from '@/lib/selectors/server/errors' +import { fetchProviderJson } from '@/lib/selectors/server/providers/provider-http' +import { + detailSelectorResult, + type ExecuteServerSelectorArgs, + listSelectorResult, + requireListRequest, + type ServerSelectorAttachmentMap, +} from '@/lib/selectors/server/types' + +type HubSpotSelectorKey = Extract< + ServerSelectorKey, + | 'hubspot.properties' + | 'hubspot.lists' + | 'hubspot.pipelines' + | 'hubspot.pipelineStages' + | 'hubspot.owners' +> + +const BUILT_IN_PATH: Record = { + contact: 'contacts', + company: 'companies', + deal: 'deals', + ticket: 'tickets', +} + +const HUBSPOT_LISTS_PAGE_SIZE = 500 + +const hubspotListSchema = z.object({ + listId: z.string().min(1).max(100), + name: z.string().min(1).max(1_000), + deletedAt: z.string().nullable().optional(), +}) + +const hubspotListsPageSchema = z.object({ + hasMore: z.boolean(), + lists: z.array(hubspotListSchema).max(HUBSPOT_LISTS_PAGE_SIZE), + offset: z.number().int().nonnegative(), +}) + +const hubspotListDetailSchema = z.object({ + list: hubspotListSchema, +}) + +function resolveObjectType(args: ExecuteServerSelectorArgs): string | null { + const selected = args.context.objectType ?? 'contact' + if (selected !== 'custom') return selected + return args.context.customObjectTypeId?.trim() || null +} + +async function hubspotToken(args: ExecuteServerSelectorArgs): Promise { + if (!args.credential) throw new SelectorConnectionUnavailableError() + try { + return await resolveSelectorOAuthAccessToken({ + credential: args.credential, + serviceId: 'hubspot', + scopes: getScopesForService('hubspot'), + protectedValues: args.protectedValues, + }) + } catch (error) { + if (error instanceof SelectorConnectionUnavailableError) throw error + throw new SelectorConnectionUnavailableError() + } +} + +async function executeProperties(args: ExecuteServerSelectorArgs) { + requireListRequest(args.selectorKey, args.request) + const objectType = resolveObjectType(args) + if (!objectType) return listSelectorResult([]) + const accessToken = await hubspotToken(args) + const path = BUILT_IN_PATH[objectType] ?? objectType + const data = await fetchProviderJson<{ + results?: Array<{ + name: string + label: string + hidden?: boolean + archived?: boolean + }> + }>(`https://api.hubapi.com/crm/v3/properties/${encodeURIComponent(path)}`, { + headers: { Authorization: `Bearer ${accessToken}` }, + signal: args.signal, + }) + if (!Array.isArray(data.results)) throw new SelectorOptionsUnavailableError() + return listSelectorResult( + data.results + .filter((property) => !property.hidden && !property.archived && property.name) + .map((property) => ({ id: property.name, label: property.label || property.name })) + .sort((left, right) => left.label.localeCompare(right.label)) + ) +} + +async function executeLists(args: ExecuteServerSelectorArgs) { + const accessToken = await hubspotToken(args) + if (args.request.kind === 'detail') { + const listId = args.request.id.trim() + if (!listId || listId.length > 100) throw new SelectorContextUnavailableError() + const body = await fetchProviderJson( + `https://api.hubapi.com/crm/v3/lists/${encodeURIComponent(listId)}`, + { + headers: { Authorization: `Bearer ${accessToken}` }, + signal: args.signal, + } + ) + const parsed = hubspotListDetailSchema.safeParse(body) + if (!parsed.success) throw new SelectorOptionsUnavailableError() + const list = parsed.data.list + return detailSelectorResult(list.deletedAt ? null : { id: args.request.id, label: list.name }) + } + + requireListRequest(args.selectorKey, args.request) + const cursor = args.request.cursor + if (cursor && !/^\d{1,10}$/.test(cursor)) throw new SelectorContextUnavailableError() + const offset = cursor ? Number(cursor) : 0 + if (!Number.isSafeInteger(offset) || offset < 0 || offset > MAX_SELECTOR_OPTIONS) { + throw new SelectorContextUnavailableError() + } + const search = args.request.search?.trim() + const body = await fetchProviderJson('https://api.hubapi.com/crm/v3/lists/search', { + method: 'POST', + headers: { + Authorization: `Bearer ${accessToken}`, + 'Content-Type': 'application/json', + }, + body: JSON.stringify({ + count: HUBSPOT_LISTS_PAGE_SIZE, + offset, + ...(search ? { query: search } : {}), + processingTypes: ['MANUAL', 'DYNAMIC', 'SNAPSHOT'], + }), + signal: args.signal, + }) + const parsed = hubspotListsPageSchema.safeParse(body) + if (!parsed.success) throw new SelectorOptionsUnavailableError() + const data = parsed.data + if (data.hasMore && data.offset <= offset) throw new SelectorOptionsUnavailableError() + return listSelectorResult( + data.lists + .filter((list) => !list.deletedAt && list.listId && list.name) + .map((list) => ({ id: list.listId, label: list.name })) + .sort((left, right) => left.label.localeCompare(right.label)), + data.hasMore ? String(data.offset) : undefined + ) +} + +interface HubSpotPipeline { + id: string + label: string + stages?: Array<{ id: string; label: string }> + archived?: boolean +} + +interface HubSpotOwner { + id: string + email?: string + firstName?: string + lastName?: string + archived?: boolean +} + +function hubspotOwnerOption(owner: HubSpotOwner) { + return { + id: owner.id, + label: [owner.firstName, owner.lastName].filter(Boolean).join(' ') || owner.email || owner.id, + } +} + +async function loadPipelines(args: ExecuteServerSelectorArgs): Promise { + const objectType = resolveObjectType(args) + if (!objectType) return [] + const accessToken = await hubspotToken(args) + const path = BUILT_IN_PATH[objectType] ?? objectType + const data = await fetchProviderJson<{ results?: HubSpotPipeline[] }>( + `https://api.hubapi.com/crm/v3/pipelines/${encodeURIComponent(path)}`, + { headers: { Authorization: `Bearer ${accessToken}` }, signal: args.signal } + ) + return (data.results ?? []).filter((pipeline) => !pipeline.archived) +} + +async function executePipelines(args: ExecuteServerSelectorArgs) { + requireListRequest(args.selectorKey, args.request) + const pipelines = await loadPipelines(args) + return listSelectorResult( + pipelines + .filter((pipeline) => pipeline.id && pipeline.label) + .map((pipeline) => ({ id: pipeline.id, label: pipeline.label })) + .sort((left, right) => left.label.localeCompare(right.label)) + ) +} + +async function executePipelineStages(args: ExecuteServerSelectorArgs) { + requireListRequest(args.selectorKey, args.request) + const pipelineId = args.context.pipelineId + if (!pipelineId) throw new SelectorContextUnavailableError() + const pipeline = (await loadPipelines(args)).find((candidate) => candidate.id === pipelineId) + return listSelectorResult( + (pipeline?.stages ?? []) + .filter((stage) => stage.id && stage.label) + .map((stage) => ({ id: stage.id, label: stage.label })) + ) +} + +async function executeOwners(args: ExecuteServerSelectorArgs) { + const accessToken = await hubspotToken(args) + if (args.request.kind === 'detail') { + const ownerId = args.request.id.trim() + if (!ownerId || ownerId.length > 100) throw new SelectorContextUnavailableError() + const owner = await fetchProviderJson( + `https://api.hubapi.com/crm/v3/owners/${encodeURIComponent(ownerId)}`, + { + headers: { Authorization: `Bearer ${accessToken}` }, + signal: args.signal, + } + ) + return detailSelectorResult( + owner.archived || !owner.id ? null : { ...hubspotOwnerOption(owner), id: ownerId } + ) + } + + requireListRequest(args.selectorKey, args.request) + const url = new URL('https://api.hubapi.com/crm/v3/owners') + url.searchParams.set('limit', '100') + if (args.request.cursor) url.searchParams.set('after', args.request.cursor) + const data = await fetchProviderJson<{ + results?: HubSpotOwner[] + paging?: { next?: { after?: string } } + }>(url, { headers: { Authorization: `Bearer ${accessToken}` }, signal: args.signal }) + return listSelectorResult( + (data.results ?? []) + .filter((owner) => !owner.archived && owner.id) + .map(hubspotOwnerOption) + .sort((left, right) => left.label.localeCompare(right.label)), + data.paging?.next?.after + ) +} + +const credential = { kind: 'stored', field: 'oauthCredential', serviceIds: ['hubspot'] } as const + +export const hubspotSelectorAttachments = { + 'hubspot.properties': { credential, destination: 'fixed', execute: executeProperties }, + 'hubspot.lists': { credential, destination: 'fixed', execute: executeLists }, + 'hubspot.pipelines': { credential, destination: 'fixed', execute: executePipelines }, + 'hubspot.pipelineStages': { + credential, + destination: 'fixed', + execute: executePipelineStages, + }, + 'hubspot.owners': { credential, destination: 'fixed', execute: executeOwners }, +} satisfies ServerSelectorAttachmentMap diff --git a/apps/sim/lib/selectors/server/providers/imap.test.ts b/apps/sim/lib/selectors/server/providers/imap.test.ts new file mode 100644 index 00000000000..64ff784d737 --- /dev/null +++ b/apps/sim/lib/selectors/server/providers/imap.test.ts @@ -0,0 +1,153 @@ +/** + * @vitest-environment node + */ +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const { + MockImapConnectionPolicyError, + mockListImapMailboxes, + mockNormalizeResolvedImapConnection, +} = vi.hoisted(() => ({ + MockImapConnectionPolicyError: class extends Error { + constructor(readonly code: 'context' | 'hidden_auth' | 'destination' | 'transport') { + super('IMAP connection is unavailable') + } + }, + mockListImapMailboxes: vi.fn(), + mockNormalizeResolvedImapConnection: vi.fn(), +})) + +vi.mock('@/lib/imap/connection.server', () => ({ + ImapConnectionPolicyError: MockImapConnectionPolicyError, + listImapMailboxes: mockListImapMailboxes, + normalizeResolvedImapConnection: mockNormalizeResolvedImapConnection, +})) + +import { + SelectorConnectionUnavailableError, + SelectorContextUnavailableError, +} from '@/lib/selectors/server/errors' +import { createSelectorProtectedValues } from '@/lib/selectors/server/protected-values' +import { imapSelectorAttachments } from '@/lib/selectors/server/providers/imap' +import type { ExecuteServerSelectorArgs } from '@/lib/selectors/server/types' + +function mailboxArgs( + overrides: Partial = {} +): ExecuteServerSelectorArgs { + return { + selectorKey: 'imap.mailboxes', + context: { + host: 'imap.example.com', + port: '993', + secure: 'true', + username: 'mailbox-user', + password: 'secret{{literal}}value', + }, + request: { kind: 'list' }, + scope: { kind: 'workspace', workspaceId: 'workspace-1' }, + workspaceId: 'workspace-1', + principal: { kind: 'session', userId: 'user-1', sessionId: 'session-1' }, + requesterUserId: 'user-1', + references: new Map(), + protectedValues: createSelectorProtectedValues(), + ...overrides, + } +} + +describe('IMAP server selector adapter', () => { + beforeEach(() => { + vi.clearAllMocks() + mockNormalizeResolvedImapConnection.mockReturnValue({ + host: 'imap.example.com', + port: 993, + secure: true, + username: 'mailbox-user', + password: 'secret{{literal}}value', + }) + mockListImapMailboxes.mockResolvedValue([{ path: 'INBOX', name: 'Inbox', delimiter: '/' }]) + }) + + it('normalizes authorized resolved values without treating their braces as templates', async () => { + await expect(imapSelectorAttachments['imap.mailboxes'].execute(mailboxArgs())).resolves.toEqual( + { + kind: 'list', + items: [{ id: 'INBOX', label: 'Inbox' }], + } + ) + + expect(mockNormalizeResolvedImapConnection).toHaveBeenCalledWith({ + host: 'imap.example.com', + port: '993', + secure: 'true', + username: 'mailbox-user', + password: 'secret{{literal}}value', + }) + expect(mockListImapMailboxes).toHaveBeenCalledOnce() + }) + + it('rejects hidden shared authentication before normalization or network access', async () => { + await expect( + imapSelectorAttachments['imap.mailboxes'].execute( + mailboxArgs({ + references: new Map([ + [ + 'password', + { + field: 'password', + name: 'IMAP_PASSWORD', + scope: 'workspace', + visible: false, + }, + ], + ]), + }) + ) + ).rejects.toBeInstanceOf(SelectorConnectionUnavailableError) + + expect(mockNormalizeResolvedImapConnection).not.toHaveBeenCalled() + expect(mockListImapMailboxes).not.toHaveBeenCalled() + }) + + it('maps an invalid port to context unavailable before mailbox access', async () => { + mockNormalizeResolvedImapConnection.mockImplementationOnce(() => { + throw new MockImapConnectionPolicyError('context') + }) + + await expect( + imapSelectorAttachments['imap.mailboxes'].execute( + mailboxArgs({ context: { ...mailboxArgs().context, port: '-1' } }) + ) + ).rejects.toBeInstanceOf(SelectorContextUnavailableError) + + expect(mockListImapMailboxes).not.toHaveBeenCalled() + }) + + it('conceals destination policy failures as connection unavailable', async () => { + mockListImapMailboxes.mockRejectedValueOnce(new MockImapConnectionPolicyError('destination')) + + await expect( + imapSelectorAttachments['imap.mailboxes'].execute(mailboxArgs()) + ).rejects.toBeInstanceOf(SelectorConnectionUnavailableError) + }) + + it('preserves cancellation before mapping IMAP policy failures', async () => { + const controller = new AbortController() + const abortError = new DOMException('The operation was aborted', 'AbortError') + controller.abort(abortError) + mockListImapMailboxes.mockRejectedValueOnce(new MockImapConnectionPolicyError('destination')) + + await expect( + imapSelectorAttachments['imap.mailboxes'].execute( + mailboxArgs({ signal: controller.signal }), + { + host: 'imap.example.com', + port: 993, + secure: true, + username: 'mailbox-user', + password: 'secret{{literal}}value', + } + ) + ).rejects.toBe(abortError) + expect(mockListImapMailboxes).toHaveBeenCalledOnce() + }) +}) diff --git a/apps/sim/lib/selectors/server/providers/imap.ts b/apps/sim/lib/selectors/server/providers/imap.ts new file mode 100644 index 00000000000..984172f9c5a --- /dev/null +++ b/apps/sim/lib/selectors/server/providers/imap.ts @@ -0,0 +1,69 @@ +import { + ImapConnectionPolicyError, + listImapMailboxes, + normalizeResolvedImapConnection, +} from '@/lib/imap/connection.server' +import { + SelectorConnectionUnavailableError, + SelectorContextUnavailableError, +} from '@/lib/selectors/server/errors' +import { + definePreparedSelectorAttachment, + listSelectorResult, + type ServerSelectorAttachmentMap, +} from '@/lib/selectors/server/types' + +function throwPublicImapError(error: unknown): never { + if (!(error instanceof ImapConnectionPolicyError)) throw error + if (error.code === 'context') throw new SelectorContextUnavailableError() + throw new SelectorConnectionUnavailableError() +} + +/** + * The integration this selector reaches. Declared rather than derived: the selector opens an IMAP connection from raw host and password fields in + * the request context and carries no stored connection, so the OAuth + * credential catalog can identify nothing to gate it on. + */ +const integrationBlockTypes = ['imap'] as const + +export const imapSelectorAttachments = { + 'imap.mailboxes': definePreparedSelectorAttachment({ + integrationBlockTypes, + destination: { + kind: 'user-controlled', + async prepare(args) { + args.signal?.throwIfAborted() + const hiddenSharedAuth = ['username', 'password'].some((field) => { + const reference = args.references.get(field) + return reference !== undefined && !reference.visible + }) + if (hiddenSharedAuth) throw new SelectorConnectionUnavailableError() + + try { + return normalizeResolvedImapConnection({ + host: args.context.host, + port: args.context.port, + secure: args.context.secure, + username: args.context.username, + password: args.context.password, + }) + } catch (error) { + args.signal?.throwIfAborted() + throwPublicImapError(error) + } + }, + }, + async execute(args, connection) { + let mailboxes + try { + mailboxes = await listImapMailboxes(connection, args.signal) + } catch (error) { + args.signal?.throwIfAborted() + throwPublicImapError(error) + } + return listSelectorResult( + mailboxes.map((mailbox) => ({ id: mailbox.path, label: mailbox.name })) + ) + }, + }), +} satisfies ServerSelectorAttachmentMap<'imap.mailboxes'> diff --git a/apps/sim/lib/selectors/server/providers/jira.test.ts b/apps/sim/lib/selectors/server/providers/jira.test.ts new file mode 100644 index 00000000000..faba3525138 --- /dev/null +++ b/apps/sim/lib/selectors/server/providers/jira.test.ts @@ -0,0 +1,95 @@ +/** + * @vitest-environment node + */ +import { afterAll, beforeEach, describe, expect, it, vi } from 'vitest' + +const { mockFetch, mockResolveSelectorAtlassianCloudId, mockResolveSelectorCredentialBundle } = + vi.hoisted(() => ({ + mockFetch: vi.fn(), + mockResolveSelectorAtlassianCloudId: vi.fn(), + mockResolveSelectorCredentialBundle: vi.fn(), + })) + +vi.mock('@/lib/selectors/server/providers/atlassian', () => ({ + resolveSelectorAtlassianCloudId: mockResolveSelectorAtlassianCloudId, +})) + +vi.mock('@/lib/selectors/server/providers/credential-bundle', () => ({ + resolveSelectorCredentialBundle: mockResolveSelectorCredentialBundle, +})) + +import { createSelectorProtectedValues } from '@/lib/selectors/server/protected-values' +import { jiraSelectorAttachments } from '@/lib/selectors/server/providers/jira' +import type { ExecuteServerSelectorArgs } from '@/lib/selectors/server/types' + +function args(): ExecuteServerSelectorArgs { + return { + selectorKey: 'jira.projects', + context: { oauthCredential: 'credential-1', domain: 'acme.atlassian.net' }, + request: { kind: 'list', search: 'payments', cursor: '50' }, + scope: { kind: 'workspace', workspaceId: 'workspace-1' }, + workspaceId: 'workspace-1', + principal: { kind: 'session', userId: 'user-1', sessionId: 'session-1' }, + requesterUserId: 'user-1', + credential: { suppliedId: 'credential-1' }, + references: new Map(), + protectedValues: createSelectorProtectedValues(), + } +} + +describe('Jira server selector adapter', () => { + beforeEach(() => { + vi.clearAllMocks() + vi.stubGlobal('fetch', mockFetch) + mockResolveSelectorCredentialBundle.mockResolvedValue({ accessToken: 'server-only-token' }) + mockResolveSelectorAtlassianCloudId.mockResolvedValue('cloud-1') + }) + + afterAll(() => vi.unstubAllGlobals()) + + it('returns one project page and preserves provider search and continuation', async () => { + mockFetch.mockResolvedValueOnce( + new Response( + JSON.stringify({ + values: Array.from({ length: 50 }, (_, index) => ({ + id: `project-${index + 1}`, + name: `Payments ${index + 1}`, + })), + maxResults: 50, + isLast: false, + }), + { status: 200 } + ) + ) + + await expect(jiraSelectorAttachments['jira.projects'].execute(args())).resolves.toEqual({ + kind: 'list', + items: Array.from({ length: 50 }, (_, index) => ({ + id: `project-${index + 1}`, + label: `Payments ${index + 1}`, + })), + nextCursor: '100', + }) + const url = new URL(String(mockFetch.mock.calls[0]?.[0])) + expect(url.searchParams.get('query')).toBe('payments') + expect(url.searchParams.get('startAt')).toBe('50') + expect(url.searchParams.get('maxResults')).toBe('50') + expect(mockFetch).toHaveBeenCalledTimes(1) + }) + + it('preserves a requested project key when hydrating its label', async () => { + mockFetch.mockResolvedValueOnce( + new Response(JSON.stringify({ id: '10001', name: 'Engineering' }), { status: 200 }) + ) + + await expect( + jiraSelectorAttachments['jira.projects'].execute({ + ...args(), + request: { kind: 'detail', id: 'ENG' }, + }) + ).resolves.toEqual({ + kind: 'detail', + item: { id: 'ENG', label: 'Engineering' }, + }) + }) +}) diff --git a/apps/sim/lib/selectors/server/providers/jira.ts b/apps/sim/lib/selectors/server/providers/jira.ts new file mode 100644 index 00000000000..f88b861e192 --- /dev/null +++ b/apps/sim/lib/selectors/server/providers/jira.ts @@ -0,0 +1,222 @@ +import { z } from 'zod' +import { getScopesForService } from '@/lib/oauth/utils' +import { MAX_SELECTOR_OPTIONS } from '@/lib/selectors/limits' +import type { ServerSelectorKey } from '@/lib/selectors/manifest' +import { + SelectorContextUnavailableError, + SelectorOptionsUnavailableError, +} from '@/lib/selectors/server/errors' +import { resolveSelectorAtlassianCloudId } from '@/lib/selectors/server/providers/atlassian' +import { resolveSelectorCredentialBundle } from '@/lib/selectors/server/providers/credential-bundle' +import { fetchProviderJson } from '@/lib/selectors/server/providers/provider-http' +import { + detailSelectorResult, + type ExecuteServerSelectorArgs, + listSelectorResult, + type ServerSelectorAttachmentMap, +} from '@/lib/selectors/server/types' + +type JiraSelectorKey = Extract + +const JIRA_SCOPES = getScopesForService('jira') +const JIRA_PROJECTS_PAGE_SIZE = 50 +const JIRA_ISSUES_LIMIT = 25 + +const jiraProjectSchema = z.object({ + id: z.string().min(1).max(100), + name: z.string().min(1).max(1_000), +}) + +const jiraProjectPageSchema = z.object({ + values: z.array(jiraProjectSchema).max(JIRA_PROJECTS_PAGE_SIZE).optional(), + isLast: z.boolean().optional(), + maxResults: z.number().int().positive().max(JIRA_PROJECTS_PAGE_SIZE).optional(), +}) + +const jiraIssueSchema = z.object({ + key: z.string().min(1).max(100), + fields: z + .object({ + summary: z.string().max(10_000).nullable().optional(), + }) + .optional(), +}) + +const jiraIssuePageSchema = z.object({ + issues: z.array(jiraIssueSchema).max(100).optional(), +}) + +function requirePathId(value: string | undefined): string { + const trimmed = value?.trim() ?? '' + if (!/^[A-Za-z0-9_-]{1,100}$/.test(trimmed)) { + throw new SelectorContextUnavailableError() + } + return trimmed +} + +function requireIssueKey(value: string): string { + const trimmed = value.trim() + if (!/^[A-Za-z][A-Za-z0-9_]*-\d+$/.test(trimmed)) { + throw new SelectorContextUnavailableError() + } + return trimmed +} + +function escapeJql(value: string): string { + return value.replace(/\\/g, '\\\\').replace(/"/g, '\\"') +} + +async function resolveJiraAuth(args: ExecuteServerSelectorArgs) { + const bundle = await resolveSelectorCredentialBundle({ + credential: args.credential, + scopes: JIRA_SCOPES, + protectedValues: args.protectedValues, + recordCredentialUse: args.recordCredentialUse, + providerId: 'jira', + }) + const cloudId = await resolveSelectorAtlassianCloudId({ + accessToken: bundle.accessToken, + domain: args.context.domain, + providedCloudId: bundle.cloudId, + providedDomain: bundle.domain, + product: 'Jira', + signal: args.signal, + }) + return { accessToken: bundle.accessToken, cloudId } +} + +function jiraHeaders(accessToken: string) { + return { Authorization: `Bearer ${accessToken}`, Accept: 'application/json' } +} + +async function listProjects(args: ExecuteServerSelectorArgs) { + const auth = await resolveJiraAuth(args) + const cursor = args.request.kind === 'list' ? args.request.cursor : undefined + if (cursor && !/^\d{1,10}$/.test(cursor)) { + throw new SelectorContextUnavailableError() + } + const startAt = cursor ? Number(cursor) : 0 + if (!Number.isSafeInteger(startAt) || startAt < 0 || startAt > MAX_SELECTOR_OPTIONS) { + throw new SelectorContextUnavailableError() + } + + const url = new URL(`https://api.atlassian.com/ex/jira/${auth.cloudId}/rest/api/3/project/search`) + if (args.request.kind === 'list' && args.request.search) { + url.searchParams.set('query', args.request.search) + } + url.searchParams.set('orderBy', 'name') + url.searchParams.set('expand', 'description,lead,url,projectKeys') + url.searchParams.set('startAt', String(startAt)) + url.searchParams.set('maxResults', String(JIRA_PROJECTS_PAGE_SIZE)) + + const body = await fetchProviderJson(url, { + headers: jiraHeaders(auth.accessToken), + redirect: 'error', + signal: args.signal, + }) + const parsed = jiraProjectPageSchema.safeParse(body) + if (!parsed.success) throw new SelectorOptionsUnavailableError() + const values = parsed.data.values ?? [] + const pageSize = parsed.data.maxResults ?? JIRA_PROJECTS_PAGE_SIZE + const nextStartAt = startAt + values.length + if (parsed.data.isLast === false && values.length === 0) { + throw new SelectorOptionsUnavailableError() + } + const hasMore = + parsed.data.isLast === false || (parsed.data.isLast === undefined && values.length >= pageSize) + + return { + items: values.map((project) => ({ id: project.id, label: project.name })), + nextCursor: hasMore ? String(nextStartAt) : undefined, + } +} + +async function getProject(args: ExecuteServerSelectorArgs, projectId: string) { + const auth = await resolveJiraAuth(args) + const body = await fetchProviderJson( + `https://api.atlassian.com/ex/jira/${auth.cloudId}/rest/api/3/project/${encodeURIComponent(projectId)}`, + { + headers: jiraHeaders(auth.accessToken), + redirect: 'error', + signal: args.signal, + } + ) + const parsed = jiraProjectSchema.safeParse(body) + if (!parsed.success) throw new SelectorOptionsUnavailableError() + return { id: projectId, label: parsed.data.name } +} + +async function fetchIssues( + args: ExecuteServerSelectorArgs, + issueKey?: string +): Promise> { + const auth = await resolveJiraAuth(args) + const jqlParts: string[] = [] + + if (issueKey) { + jqlParts.push(`issueKey = "${escapeJql(issueKey)}"`) + } else { + const projectId = args.context.projectId + const search = args.request.kind === 'list' ? args.request.search : undefined + if (!projectId && !search) return [] + if (projectId) jqlParts.push(`project = "${escapeJql(requirePathId(projectId))}"`) + if (search) { + const escaped = escapeJql(search) + jqlParts.push(`(key ~ "${escaped}" OR summary ~ "${escaped}")`) + } + } + + const url = new URL(`https://api.atlassian.com/ex/jira/${auth.cloudId}/rest/api/3/search/jql`) + url.searchParams.set( + 'jql', + issueKey ? jqlParts.join(' AND ') : `${jqlParts.join(' AND ')} ORDER BY updated DESC` + ) + url.searchParams.set('fields', 'summary,key,updated') + url.searchParams.set('maxResults', String(issueKey ? 1 : JIRA_ISSUES_LIMIT)) + + const body = await fetchProviderJson(url, { + headers: jiraHeaders(auth.accessToken), + redirect: 'error', + signal: args.signal, + }) + const parsed = jiraIssuePageSchema.safeParse(body) + if (!parsed.success) throw new SelectorOptionsUnavailableError() + return (parsed.data.issues ?? []).map((issue) => ({ + id: issue.key, + label: issue.fields?.summary || issue.key, + })) +} + +async function executeProjects(args: ExecuteServerSelectorArgs) { + if (args.request.kind === 'detail') { + return detailSelectorResult(await getProject(args, requirePathId(args.request.id))) + } + const result = await listProjects(args) + return listSelectorResult(result.items, result.nextCursor) +} + +async function executeIssues(args: ExecuteServerSelectorArgs) { + if (args.request.kind === 'detail') { + const issues = await fetchIssues(args, requireIssueKey(args.request.id)) + const issue = issues[0] + return detailSelectorResult(issue ? { ...issue, id: args.request.id } : null) + } + return listSelectorResult(await fetchIssues(args)) +} + +const credential = { kind: 'stored', field: 'oauthCredential', serviceIds: ['jira'] } as const + +export const jiraSelectorAttachments = { + 'jira.projects': { + credential, + destination: 'fixed', + auditCredentialUse: true, + execute: executeProjects, + }, + 'jira.issues': { + credential, + destination: 'fixed', + auditCredentialUse: true, + execute: executeIssues, + }, +} satisfies ServerSelectorAttachmentMap diff --git a/apps/sim/lib/selectors/server/providers/jsm.test.ts b/apps/sim/lib/selectors/server/providers/jsm.test.ts new file mode 100644 index 00000000000..b8503ca8bef --- /dev/null +++ b/apps/sim/lib/selectors/server/providers/jsm.test.ts @@ -0,0 +1,97 @@ +/** + * @vitest-environment node + */ +import { afterAll, beforeEach, describe, expect, it, vi } from 'vitest' + +const { mockFetch, mockResolveJsmAuth, mockResolveCloudId } = vi.hoisted(() => ({ + mockFetch: vi.fn(), + mockResolveJsmAuth: vi.fn(), + mockResolveCloudId: vi.fn(), +})) + +vi.mock('@/lib/selectors/server/providers/credential-bundle', () => ({ + resolveSelectorCredentialBundle: mockResolveJsmAuth, +})) + +vi.mock('@/lib/selectors/server/providers/atlassian', () => ({ + resolveSelectorAtlassianCloudId: mockResolveCloudId, +})) + +import { createSelectorProtectedValues } from '@/lib/selectors/server/protected-values' +import { jsmSelectorAttachments } from '@/lib/selectors/server/providers/jsm' +import type { ExecuteServerSelectorArgs } from '@/lib/selectors/server/types' + +function serviceDeskArgs(): ExecuteServerSelectorArgs { + return { + selectorKey: 'jsm.serviceDesks', + context: { oauthCredential: 'credential-1', domain: 'example.atlassian.net' }, + request: { kind: 'list' }, + scope: { kind: 'workspace', workspaceId: 'workspace-1' }, + workspaceId: 'workspace-1', + principal: { kind: 'session', userId: 'user-1', sessionId: 'session-1' }, + requesterUserId: 'user-1', + credential: { suppliedId: 'credential-1' }, + references: new Map(), + protectedValues: createSelectorProtectedValues(), + } +} + +function providerResponse(body: unknown): Response { + return new Response(JSON.stringify(body), { + status: 200, + headers: { 'content-type': 'application/json' }, + }) +} + +describe('JSM server selector adapters', () => { + beforeEach(() => { + vi.clearAllMocks() + vi.stubGlobal('fetch', mockFetch) + mockResolveJsmAuth.mockResolvedValue({ + accessToken: 'server-only-token', + cloudId: 'cloud-1', + domain: 'example.atlassian.net', + }) + mockResolveCloudId.mockResolvedValue('cloud-1') + }) + + afterAll(() => vi.unstubAllGlobals()) + + it('advances short pages by their returned row count and normalizes every option', async () => { + mockFetch + .mockResolvedValueOnce( + providerResponse({ + values: [{ id: '1', projectName: 'One' }], + _links: { next: 'next' }, + }) + ) + .mockResolvedValueOnce( + providerResponse({ + values: [{ id: '2', projectName: 'Two' }], + isLastPage: true, + }) + ) + + await expect( + jsmSelectorAttachments['jsm.serviceDesks'].execute(serviceDeskArgs()) + ).resolves.toEqual({ + kind: 'list', + items: [ + { id: '1', label: 'One' }, + { id: '2', label: 'Two' }, + ], + }) + + const firstUrl = new URL(String(mockFetch.mock.calls[0]?.[0])) + const secondUrl = new URL(String(mockFetch.mock.calls[1]?.[0])) + expect(firstUrl.search).toBe('?start=0&limit=100') + expect(secondUrl.search).toBe('?start=1&limit=100') + expect(mockResolveCloudId).toHaveBeenCalledWith( + expect.objectContaining({ + domain: 'example.atlassian.net', + providedCloudId: 'cloud-1', + providedDomain: 'example.atlassian.net', + }) + ) + }) +}) diff --git a/apps/sim/lib/selectors/server/providers/jsm.ts b/apps/sim/lib/selectors/server/providers/jsm.ts new file mode 100644 index 00000000000..e5507f3a6c5 --- /dev/null +++ b/apps/sim/lib/selectors/server/providers/jsm.ts @@ -0,0 +1,169 @@ +import { z } from 'zod' +import { getScopesForService } from '@/lib/oauth/utils' +import type { ServerSelectorKey } from '@/lib/selectors/manifest' +import { + SelectorContextUnavailableError, + SelectorOptionsUnavailableError, +} from '@/lib/selectors/server/errors' +import { resolveSelectorAtlassianCloudId } from '@/lib/selectors/server/providers/atlassian' +import { resolveSelectorCredentialBundle } from '@/lib/selectors/server/providers/credential-bundle' +import { fetchProviderJson } from '@/lib/selectors/server/providers/provider-http' +import { + detailSelectorResult, + type ExecuteServerSelectorArgs, + listSelectorResult, + type ServerSelectorAttachmentMap, +} from '@/lib/selectors/server/types' +import { getJsmApiBaseUrl, getJsmHeaders } from '@/tools/jsm/utils' + +type JsmSelectorKey = Extract + +const JIRA_SCOPES = getScopesForService('jira') +const JSM_PAGE_SIZE = 100 +const MAX_JSM_PAGES = 50 + +const serviceDeskSchema = z.object({ + id: z.string().min(1).max(100), + projectName: z.string().min(1).max(1_000), +}) + +const requestTypeSchema = z.object({ + id: z.string().min(1).max(100), + name: z.string().min(1).max(1_000), +}) + +function pagedSchema(item: T) { + return z.object({ + values: z.array(item).max(JSM_PAGE_SIZE).optional(), + isLastPage: z.boolean().optional(), + _links: z.object({ next: z.string().max(4_096).optional() }).optional(), + }) +} + +function requireServiceDeskId(value: string | undefined): string { + const trimmed = value?.trim() ?? '' + if (!/^[A-Za-z0-9_-]{1,100}$/.test(trimmed)) { + throw new SelectorContextUnavailableError() + } + return trimmed +} + +function requireDetailId(value: string): string { + const trimmed = value.trim() + if (!trimmed || trimmed.length > 100) throw new SelectorContextUnavailableError() + return trimmed +} + +async function resolveJsmAuth(args: ExecuteServerSelectorArgs) { + const bundle = await resolveSelectorCredentialBundle({ + credential: args.credential, + scopes: JIRA_SCOPES, + protectedValues: args.protectedValues, + }) + const cloudId = await resolveSelectorAtlassianCloudId({ + accessToken: bundle.accessToken, + domain: args.context.domain, + providedCloudId: bundle.cloudId, + providedDomain: bundle.domain, + product: 'Jira', + signal: args.signal, + }) + return { accessToken: bundle.accessToken, cloudId } +} + +async function drainJsmPages(input: { + args: ExecuteServerSelectorArgs + accessToken: string + baseUrl: string + schema: z.ZodType +}): Promise<{ rows: T[]; truncated: boolean }> { + const rows: T[] = [] + let start = 0 + let truncated = true + + for (let page = 0; page < MAX_JSM_PAGES; page++) { + const url = new URL(input.baseUrl) + url.searchParams.set('start', String(start)) + url.searchParams.set('limit', String(JSM_PAGE_SIZE)) + const body = await fetchProviderJson(url, { + headers: getJsmHeaders(input.accessToken), + redirect: 'error', + signal: input.args.signal, + }) + const parsed = pagedSchema(input.schema).safeParse(body) + if (!parsed.success) throw new SelectorOptionsUnavailableError() + const values = parsed.data.values ?? [] + rows.push(...values) + if (parsed.data.isLastPage === true || !parsed.data._links?.next || values.length === 0) { + truncated = false + break + } + start += values.length + } + + return { rows, truncated } +} + +async function serviceDeskOptions(args: ExecuteServerSelectorArgs) { + const auth = await resolveJsmAuth(args) + const result = await drainJsmPages({ + args, + ...auth, + baseUrl: `${getJsmApiBaseUrl(auth.cloudId)}/servicedesk`, + schema: serviceDeskSchema, + }) + return { + items: result.rows.map((row) => ({ id: row.id, label: row.projectName })), + truncated: result.truncated, + } +} + +async function requestTypeOptions(args: ExecuteServerSelectorArgs) { + const serviceDeskId = requireServiceDeskId(args.context.serviceDeskId) + const auth = await resolveJsmAuth(args) + const result = await drainJsmPages({ + args, + ...auth, + baseUrl: `${getJsmApiBaseUrl(auth.cloudId)}/servicedesk/${encodeURIComponent(serviceDeskId)}/requesttype`, + schema: requestTypeSchema, + }) + return { + items: result.rows.map((row) => ({ id: row.id, label: row.name })), + truncated: result.truncated, + } +} + +function resultForRequest( + args: ExecuteServerSelectorArgs, + result: { items: Array<{ id: string; label: string }>; truncated: boolean } +) { + if (args.request.kind === 'list') { + return listSelectorResult( + result.items, + undefined, + result.truncated ? { truncated: { reason: 'provider-cap', pages: MAX_JSM_PAGES } } : undefined + ) + } + const id = requireDetailId(args.request.id) + return { + ...detailSelectorResult(result.items.find((item) => item.id === id) ?? null), + ...(result.truncated + ? { diagnostics: { truncated: { reason: 'provider-cap' as const, pages: MAX_JSM_PAGES } } } + : {}), + } +} + +const credential = { kind: 'stored', field: 'oauthCredential', serviceIds: ['jira'] } as const + +export const jsmSelectorAttachments = { + 'jsm.serviceDesks': { + credential, + destination: 'fixed', + execute: async (args) => resultForRequest(args, await serviceDeskOptions(args)), + }, + 'jsm.requestTypes': { + credential, + destination: 'fixed', + execute: async (args) => resultForRequest(args, await requestTypeOptions(args)), + }, +} satisfies ServerSelectorAttachmentMap diff --git a/apps/sim/lib/selectors/server/providers/linear.test.ts b/apps/sim/lib/selectors/server/providers/linear.test.ts new file mode 100644 index 00000000000..90e4fb42f94 --- /dev/null +++ b/apps/sim/lib/selectors/server/providers/linear.test.ts @@ -0,0 +1,224 @@ +/** + * @vitest-environment node + */ +import { LinearError } from '@linear/sdk' +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const { + MockLinearError, + mockLinearClientOptions, + mockResolveSelectorOAuthAccessToken, + mockTeams, + mockTeam, + mockProject, +} = vi.hoisted(() => { + class MockLinearError extends Error { + status?: number + + constructor(error?: { response?: { status?: number } }) { + super('Linear request failed') + this.name = 'LinearError' + this.status = error?.response?.status + } + } + + return { + MockLinearError, + mockLinearClientOptions: vi.fn(), + mockResolveSelectorOAuthAccessToken: vi.fn(), + mockTeams: vi.fn(), + mockTeam: vi.fn(), + mockProject: vi.fn(), + } +}) + +vi.mock('@linear/sdk', () => ({ + LinearError: MockLinearError, + LinearClient: class LinearClient { + constructor(options: unknown) { + mockLinearClientOptions(options) + } + + teams = mockTeams + team = mockTeam + project = mockProject + }, +})) + +vi.mock('@/lib/selectors/server/credentials', () => ({ + resolveSelectorOAuthAccessToken: mockResolveSelectorOAuthAccessToken, +})) + +import { createSelectorProtectedValues } from '@/lib/selectors/server/protected-values' +import { linearSelectorAttachments } from '@/lib/selectors/server/providers/linear' +import type { ExecuteServerSelectorArgs } from '@/lib/selectors/server/types' + +function teamArgs(signal?: AbortSignal): ExecuteServerSelectorArgs { + return { + selectorKey: 'linear.teams', + context: { oauthCredential: 'credential-1' }, + request: { kind: 'list' }, + scope: { kind: 'workspace', workspaceId: 'workspace-1' }, + workspaceId: 'workspace-1', + principal: { kind: 'session', userId: 'user-1', sessionId: 'session-1' }, + requesterUserId: 'user-1', + credential: { suppliedId: 'credential-1' }, + references: new Map(), + protectedValues: createSelectorProtectedValues(), + signal, + } +} + +function projectArgs(teamIds: string, cursor?: string): ExecuteServerSelectorArgs { + return { + ...teamArgs(), + selectorKey: 'linear.projects', + context: { oauthCredential: 'credential-1', teamId: teamIds }, + request: { kind: 'list', ...(cursor ? { cursor } : {}) }, + } +} + +function linearError(status: number): LinearError { + return new LinearError({ response: { status } }) +} + +describe('Linear server selector adapter errors', () => { + beforeEach(() => { + vi.clearAllMocks() + mockResolveSelectorOAuthAccessToken.mockResolvedValue('server-only-token') + }) + + it('constructs the v91 client with OAuth credentials and request cancellation', async () => { + const controller = new AbortController() + mockTeams.mockResolvedValueOnce({ + nodes: [], + pageInfo: { hasNextPage: false, endCursor: undefined }, + }) + + await linearSelectorAttachments['linear.teams'].execute(teamArgs(controller.signal)) + + expect(mockLinearClientOptions).toHaveBeenCalledWith({ + accessToken: 'server-only-token', + redirect: 'error', + signal: controller.signal, + }) + }) + + it('uses Linear personal API keys without exposing them as OAuth tokens', async () => { + mockResolveSelectorOAuthAccessToken.mockResolvedValueOnce('lin_api_personal-token') + mockTeams.mockResolvedValueOnce({ + nodes: [], + pageInfo: { hasNextPage: false, endCursor: undefined }, + }) + + await linearSelectorAttachments['linear.teams'].execute(teamArgs()) + + expect(mockLinearClientOptions).toHaveBeenCalledWith({ + apiKey: 'lin_api_personal-token', + redirect: 'error', + signal: undefined, + }) + }) + + it.each([ + [401, 'SelectorConnectionUnavailableError', 401], + [403, 'SelectorConnectionUnavailableError', 403], + [429, 'SelectorOptionsUnavailableError', 429], + [500, 'SelectorOptionsUnavailableError', 502], + ] as const)( + 'maps trusted Linear status %i to the safe selector taxonomy', + async (status, name, safeStatus) => { + mockTeams.mockRejectedValueOnce(linearError(status)) + + await expect( + linearSelectorAttachments['linear.teams'].execute(teamArgs()) + ).rejects.toMatchObject({ name, status: safeStatus }) + } + ) + + it('does not trust a status-shaped unknown error', async () => { + mockTeams.mockRejectedValueOnce({ status: 401 }) + + await expect( + linearSelectorAttachments['linear.teams'].execute(teamArgs()) + ).rejects.toMatchObject({ name: 'SelectorOptionsUnavailableError', status: 502 }) + }) + + it('preserves caller cancellation', async () => { + const controller = new AbortController() + const abortError = new DOMException('The operation was aborted', 'AbortError') + controller.abort(abortError) + mockTeams.mockRejectedValueOnce(abortError) + + await expect( + linearSelectorAttachments['linear.teams'].execute(teamArgs(controller.signal)) + ).rejects.toBe(abortError) + }) + + it('fetches one selected team page at a time', async () => { + mockTeam.mockImplementation(async (teamId: string) => ({ + projects: async ({ after }: { after?: string }) => { + return { + nodes: [{ id: `project-${teamId}-${after ?? '0'}`, name: `Project ${teamId}` }], + pageInfo: { + hasNextPage: teamId === 'team-1' && !after, + endCursor: teamId === 'team-1' && !after ? 'team-1-page-2' : undefined, + }, + } + }, + })) + const teamIds = 'team-1,team-2' + + const first = await linearSelectorAttachments['linear.projects'].execute(projectArgs(teamIds)) + expect(first).toEqual({ + kind: 'list', + items: [{ id: 'project-team-1-0', label: 'Project team-1' }], + nextCursor: 'team=0&after=team-1-page-2', + }) + expect(mockTeam).toHaveBeenCalledTimes(1) + + const second = await linearSelectorAttachments['linear.projects'].execute( + projectArgs(teamIds, 'team=0&after=team-1-page-2') + ) + expect(second).toEqual({ + kind: 'list', + items: [{ id: 'project-team-1-team-1-page-2', label: 'Project team-1' }], + nextCursor: 'team=1', + }) + expect(mockTeam).toHaveBeenCalledTimes(2) + + const third = await linearSelectorAttachments['linear.projects'].execute( + projectArgs(teamIds, 'team=1') + ) + expect(third).toEqual({ + kind: 'list', + items: [{ id: 'project-team-2-0', label: 'Project team-2' }], + }) + expect(mockTeam).toHaveBeenCalledTimes(3) + }) + + it('rejects malformed multi-team cursors before requesting a team', async () => { + await expect( + linearSelectorAttachments['linear.projects'].execute( + projectArgs('team-1,team-2', 'team=0&operation=teams') + ) + ).rejects.toMatchObject({ name: 'SelectorContextUnavailableError' }) + expect(mockTeam).not.toHaveBeenCalled() + }) + + it('hydrates a selected project without traversing its teams', async () => { + mockProject.mockResolvedValueOnce({ id: 'project-1', name: 'Project One' }) + + await expect( + linearSelectorAttachments['linear.projects'].execute({ + ...projectArgs('team-1,team-2'), + request: { kind: 'detail', id: 'project-1' }, + }) + ).resolves.toEqual({ + kind: 'detail', + item: { id: 'project-1', label: 'Project One' }, + }) + expect(mockProject).toHaveBeenCalledWith('project-1') + expect(mockTeam).not.toHaveBeenCalled() + }) +}) diff --git a/apps/sim/lib/selectors/server/providers/linear.ts b/apps/sim/lib/selectors/server/providers/linear.ts new file mode 100644 index 00000000000..c151650cfed --- /dev/null +++ b/apps/sim/lib/selectors/server/providers/linear.ts @@ -0,0 +1,166 @@ +import { LinearClient, LinearError } from '@linear/sdk' +import { getScopesForService } from '@/lib/oauth/utils' +import type { ServerSelectorKey } from '@/lib/selectors/manifest' +import { resolveSelectorOAuthAccessToken } from '@/lib/selectors/server/credentials' +import { + SelectorConnectionUnavailableError, + SelectorContextUnavailableError, + SelectorOptionsUnavailableError, +} from '@/lib/selectors/server/errors' +import { selectorProviderStatusError } from '@/lib/selectors/server/providers/provider-http' +import { + detailSelectorResult, + type ExecuteServerSelectorArgs, + listSelectorResult, + requireListRequest, + type ServerSelectorAttachmentMap, +} from '@/lib/selectors/server/types' + +type LinearSelectorKey = Extract + +const LINEAR_SCOPES = getScopesForService('linear') +const LINEAR_PAGE_SIZE = 250 +const MAX_SELECTED_TEAMS = 100 +const MAX_LINEAR_CURSOR_LENGTH = 4_096 + +function throwLinearSelectorError(error: unknown, signal?: AbortSignal): never { + if (signal?.aborted) throw error + if (error instanceof SelectorContextUnavailableError) throw error + if (error instanceof LinearError && typeof error.status === 'number') { + throw selectorProviderStatusError(error.status) + } + throw new SelectorOptionsUnavailableError() +} + +async function linearClient(args: ExecuteServerSelectorArgs) { + if (!args.credential) throw new SelectorConnectionUnavailableError() + const token = await resolveSelectorOAuthAccessToken({ + credential: args.credential, + serviceId: 'linear', + scopes: LINEAR_SCOPES, + protectedValues: args.protectedValues, + }) + return token.startsWith('lin_api_') + ? new LinearClient({ apiKey: token, redirect: 'error', signal: args.signal }) + : new LinearClient({ accessToken: token, redirect: 'error', signal: args.signal }) +} + +function selectedTeamIds(raw: string | undefined): string[] { + const ids = (raw ?? '') + .split(',') + .map((id) => id.trim()) + .filter(Boolean) + if (ids.length === 0 || ids.length > MAX_SELECTED_TEAMS || ids.some((id) => id.length > 100)) { + throw new SelectorContextUnavailableError() + } + return ids +} + +function requireLinearId(value: string): string { + const id = value.trim() + if (!/^[A-Za-z0-9_-]{1,100}$/.test(id)) throw new SelectorContextUnavailableError() + return id +} + +function linearCursor(cursor: string | undefined): string | undefined { + if (!cursor) return undefined + if (cursor.length > MAX_LINEAR_CURSOR_LENGTH) throw new SelectorContextUnavailableError() + return cursor +} + +interface LinearProjectsCursor { + teamIndex: number + after?: string +} + +function parseProjectsCursor(cursor: string | undefined): LinearProjectsCursor { + if (!cursor) return { teamIndex: 0 } + if (cursor.length > MAX_LINEAR_CURSOR_LENGTH) throw new SelectorContextUnavailableError() + + const params = new URLSearchParams(cursor) + if ([...params.keys()].some((key) => key !== 'team' && key !== 'after')) { + throw new SelectorContextUnavailableError() + } + const team = params.get('team') + const after = params.get('after') || undefined + if (!team || !/^\d{1,3}$/.test(team) || params.getAll('team').length !== 1) { + throw new SelectorContextUnavailableError() + } + if (params.getAll('after').length > 1) throw new SelectorContextUnavailableError() + + const teamIndex = Number(team) + if (!Number.isSafeInteger(teamIndex) || teamIndex < 0 || teamIndex >= MAX_SELECTED_TEAMS) { + throw new SelectorContextUnavailableError() + } + return { teamIndex, ...(after ? { after } : {}) } +} + +function projectsCursor(cursor: LinearProjectsCursor): string { + const params = new URLSearchParams({ team: String(cursor.teamIndex) }) + if (cursor.after) params.set('after', cursor.after) + return params.toString() +} + +async function executeTeams(args: ExecuteServerSelectorArgs) { + const client = await linearClient(args) + try { + if (args.request.kind === 'detail') { + const team = await client.team(requireLinearId(args.request.id)) + return detailSelectorResult({ id: team.id, label: team.name }) + } + const request = requireListRequest(args.selectorKey, args.request) + const result = await client.teams({ + first: LINEAR_PAGE_SIZE, + after: linearCursor(request.cursor), + }) + const nextCursor = + result.pageInfo.hasNextPage && result.pageInfo.endCursor + ? result.pageInfo.endCursor + : undefined + return listSelectorResult( + result.nodes.map((team) => ({ id: team.id, label: team.name })), + nextCursor + ) + } catch (error) { + throwLinearSelectorError(error, args.signal) + } +} + +async function executeProjects(args: ExecuteServerSelectorArgs) { + const client = await linearClient(args) + try { + if (args.request.kind === 'detail') { + const project = await client.project(requireLinearId(args.request.id)) + return detailSelectorResult({ id: project.id, label: project.name }) + } + const request = requireListRequest(args.selectorKey, args.request) + const teamIds = selectedTeamIds(args.context.teamId) + const cursor = parseProjectsCursor(request.cursor) + if (cursor.teamIndex >= teamIds.length) throw new SelectorContextUnavailableError() + + const team = await client.team(teamIds[cursor.teamIndex]) + const result = await team.projects({ first: LINEAR_PAGE_SIZE, after: cursor.after }) + let nextCursor: string | undefined + if (result.pageInfo.hasNextPage && result.pageInfo.endCursor) { + nextCursor = projectsCursor({ + teamIndex: cursor.teamIndex, + after: result.pageInfo.endCursor, + }) + } else if (cursor.teamIndex + 1 < teamIds.length) { + nextCursor = projectsCursor({ teamIndex: cursor.teamIndex + 1 }) + } + return listSelectorResult( + result.nodes.map((project) => ({ id: project.id, label: project.name })), + nextCursor + ) + } catch (error) { + throwLinearSelectorError(error, args.signal) + } +} + +const credential = { kind: 'stored', field: 'oauthCredential', serviceIds: ['linear'] } as const + +export const linearSelectorAttachments = { + 'linear.teams': { credential, destination: 'fixed', execute: executeTeams }, + 'linear.projects': { credential, destination: 'fixed', execute: executeProjects }, +} satisfies ServerSelectorAttachmentMap diff --git a/apps/sim/lib/selectors/server/providers/managed-agent.test.ts b/apps/sim/lib/selectors/server/providers/managed-agent.test.ts new file mode 100644 index 00000000000..19aa67e794f --- /dev/null +++ b/apps/sim/lib/selectors/server/providers/managed-agent.test.ts @@ -0,0 +1,48 @@ +/** + * @vitest-environment node + */ +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const mocks = vi.hoisted(() => ({ + list: vi.fn(), + resolveBundle: vi.fn(), +})) + +vi.mock('@/lib/managed-agents/session-client', () => ({ + AGENT_MEMORY_BETA: 'agent-memory-test', + managedAgentsList: mocks.list, +})) + +vi.mock('@/lib/selectors/server/providers/credential-bundle', () => ({ + resolveSelectorCredentialBundle: mocks.resolveBundle, +})) + +import { createSelectorProtectedValues } from '@/lib/selectors/server/protected-values' +import { managedAgentSelectorAttachments } from '@/lib/selectors/server/providers/managed-agent' +import type { ExecuteServerSelectorArgs } from '@/lib/selectors/server/types' + +const args: ExecuteServerSelectorArgs = { + selectorKey: 'managedAgent.agents', + context: {}, + request: { kind: 'list' }, + scope: { kind: 'workspace', workspaceId: 'workspace-1' }, + workspaceId: 'workspace-1', + principal: { kind: 'session', userId: 'user-1', sessionId: 'session-1' }, + requesterUserId: 'user-1', + credential: { suppliedId: 'credential-1' }, + references: new Map(), + protectedValues: createSelectorProtectedValues(), +} + +describe('Managed Agent server selector adapter', () => { + beforeEach(() => vi.clearAllMocks()) + + it('preserves the editor empty-list fallback when key resolution fails', async () => { + mocks.resolveBundle.mockRejectedValue(new Error('credential unavailable')) + + await expect( + managedAgentSelectorAttachments['managedAgent.agents'].execute(args) + ).resolves.toEqual({ kind: 'list', items: [] }) + expect(mocks.list).not.toHaveBeenCalled() + }) +}) diff --git a/apps/sim/lib/selectors/server/providers/managed-agent.ts b/apps/sim/lib/selectors/server/providers/managed-agent.ts new file mode 100644 index 00000000000..6400bc0bc75 --- /dev/null +++ b/apps/sim/lib/selectors/server/providers/managed-agent.ts @@ -0,0 +1,137 @@ +import { AGENT_MEMORY_BETA, managedAgentsList } from '@/lib/managed-agents/session-client' +import type { ServerSelectorKey } from '@/lib/selectors/manifest' +import { resolveSelectorCredentialBundle } from '@/lib/selectors/server/providers/credential-bundle' +import { + type ExecuteServerSelectorArgs, + listSelectorResult, + requireListRequest, + type ServerSelectorAttachmentMap, +} from '@/lib/selectors/server/types' +import type { SafeSelectorOption } from '@/lib/selectors/types' + +type ManagedAgentSelectorKey = Extract< + ServerSelectorKey, + | 'managedAgent.agents' + | 'managedAgent.environments' + | 'managedAgent.vaults' + | 'managedAgent.memoryStores' +> + +type ManagedAgentResource = 'agents' | 'environments' | 'vaults' | 'memory-stores' + +interface ManagedAgentRow { + id?: unknown + name?: unknown + config?: { type?: unknown } +} + +const RESOURCE_ENDPOINTS: Record = { + agents: { path: '/v1/agents' }, + environments: { path: '/v1/environments' }, + vaults: { path: '/v1/vaults' }, + 'memory-stores': { path: '/v1/memory_stores', beta: AGENT_MEMORY_BETA }, +} + +function toOption( + resource: ManagedAgentResource, + row: ManagedAgentRow, + environmentType: string | undefined +): SafeSelectorOption | null { + if (typeof row.id !== 'string' || !row.id) return null + const name = typeof row.name === 'string' ? row.name.trim() : '' + + if (resource === 'environments') { + const type = row.config?.type + const validType = type === 'cloud' || type === 'self_hosted' ? type : undefined + if ( + (environmentType === 'cloud' || environmentType === 'self_hosted') && + validType !== undefined && + validType !== environmentType + ) { + return null + } + return { + id: row.id, + label: `${name || row.id}${validType ? ` (${validType})` : ''}`, + ...(validType ? { meta: { type: validType } } : {}), + } + } + + if (resource === 'vaults') return { id: row.id, label: name || row.id } + return { id: row.id, label: name ? `${name} (${row.id})` : row.id } +} + +async function executeResource(args: ExecuteServerSelectorArgs, resource: ManagedAgentResource) { + requireListRequest(args.selectorKey, args.request) + const endpoint = RESOURCE_ENDPOINTS[resource] + + try { + const bundle = await resolveSelectorCredentialBundle({ + credential: args.credential, + protectedValues: args.protectedValues, + recordCredentialUse: args.recordCredentialUse, + providerId: 'claude-platform', + }) + const rows = await managedAgentsList({ + apiKey: bundle.accessToken, + path: endpoint.path, + beta: endpoint.beta, + signal: args.signal, + }) + return listSelectorResult( + rows + .map((row) => toOption(resource, row, args.context.environmentType)) + .filter((option): option is SafeSelectorOption => option !== null) + ) + } catch (error) { + if (args.signal?.aborted) throw error + // Preserve the existing editor behavior for beta resources that are not + // enabled in a Claude workspace: an unavailable collection is an empty list. + return listSelectorResult([]) + } +} + +const credential = { + kind: 'stored', + field: 'oauthCredential', + serviceIds: ['claude-platform'], +} as const + +/** + * The integration this selector reaches. Declared rather than derived: The managed-agent platform is an + * API-key integration with no entry in the deployment OAuth catalog, so its + * service id maps to no block type and the allowlist would have nothing to + * judge it on. + */ +const integrationBlockTypes = ['managed_agent'] as const + +export const managedAgentSelectorAttachments = { + 'managedAgent.agents': { + credential, + integrationBlockTypes, + destination: 'fixed', + auditCredentialUse: true, + execute: (args) => executeResource(args, 'agents'), + }, + 'managedAgent.environments': { + credential, + integrationBlockTypes, + destination: 'fixed', + auditCredentialUse: true, + execute: (args) => executeResource(args, 'environments'), + }, + 'managedAgent.vaults': { + credential, + integrationBlockTypes, + destination: 'fixed', + auditCredentialUse: true, + execute: (args) => executeResource(args, 'vaults'), + }, + 'managedAgent.memoryStores': { + credential, + integrationBlockTypes, + destination: 'fixed', + auditCredentialUse: true, + execute: (args) => executeResource(args, 'memory-stores'), + }, +} satisfies ServerSelectorAttachmentMap diff --git a/apps/sim/lib/selectors/server/providers/microsoft.test.ts b/apps/sim/lib/selectors/server/providers/microsoft.test.ts new file mode 100644 index 00000000000..c408a88ef33 --- /dev/null +++ b/apps/sim/lib/selectors/server/providers/microsoft.test.ts @@ -0,0 +1,220 @@ +/** + * @vitest-environment node + */ +import { afterAll, beforeEach, describe, expect, it, vi } from 'vitest' + +const { mockFetch, mockResolveSelectorOAuthAccessToken } = vi.hoisted(() => ({ + mockFetch: vi.fn(), + mockResolveSelectorOAuthAccessToken: vi.fn(), +})) + +vi.mock('@/lib/selectors/server/credentials', () => ({ + resolveSelectorOAuthAccessToken: mockResolveSelectorOAuthAccessToken, +})) + +import { createSelectorProtectedValues } from '@/lib/selectors/server/protected-values' +import { microsoftSelectorAttachments } from '@/lib/selectors/server/providers/microsoft' +import type { ExecuteServerSelectorArgs } from '@/lib/selectors/server/types' + +function listArgs( + selectorKey: 'microsoft.chats' | 'onedrive.files' | 'microsoft.excel.sheets', + cursor?: string +): ExecuteServerSelectorArgs { + return { + selectorKey, + context: { oauthCredential: 'credential-1' }, + request: { kind: 'list', ...(cursor ? { cursor } : {}) }, + scope: { kind: 'workspace', workspaceId: 'workspace-1' }, + workspaceId: 'workspace-1', + principal: { kind: 'session', userId: 'user-1', sessionId: 'session-1' }, + requesterUserId: 'user-1', + credential: { suppliedId: 'credential-1' }, + references: new Map(), + protectedValues: createSelectorProtectedValues(), + } +} + +function plannerTaskDetailArgs(): ExecuteServerSelectorArgs { + return { + ...listArgs('microsoft.chats'), + selectorKey: 'microsoft.planner', + context: { oauthCredential: 'credential-1', planId: 'plan-1' }, + request: { kind: 'detail', id: 'task-1' }, + } +} + +describe('Microsoft server selector adapters', () => { + beforeEach(() => { + vi.clearAllMocks() + vi.stubGlobal('fetch', mockFetch) + mockResolveSelectorOAuthAccessToken.mockResolvedValue('server-only-token') + }) + + afterAll(() => vi.unstubAllGlobals()) + + it('bounds chat label enrichment concurrency', async () => { + let activeEnrichments = 0 + let maxActiveEnrichments = 0 + mockFetch.mockImplementation(async (input) => { + const url = String(input) + if (url.includes('/me/chats')) { + return new Response( + JSON.stringify({ + value: Array.from({ length: 25 }, (_, index) => ({ id: `chat-${index}` })), + }), + { status: 200 } + ) + } + if (url.includes('/members')) { + activeEnrichments += 1 + maxActiveEnrichments = Math.max(maxActiveEnrichments, activeEnrichments) + await Promise.resolve() + activeEnrichments -= 1 + return new Response(JSON.stringify({ value: [{ displayName: 'Member' }] }), { + status: 200, + }) + } + throw new Error(`Unexpected Microsoft Graph request: ${url}`) + }) + + const result = await microsoftSelectorAttachments['microsoft.chats'].execute( + listArgs('microsoft.chats') + ) + + expect(result.kind === 'list' ? result.items : []).toHaveLength(25) + expect(maxActiveEnrichments).toBeLessThanOrEqual(10) + }) + + it('returns one Graph page and follows the continuation URL only on demand', async () => { + mockFetch.mockImplementation(async (input) => { + const url = new URL(String(input)) + const page = Number(url.searchParams.get('page') ?? '0') + const value = Array.from({ length: 999 }, (_, index) => ({ + id: `file-${page}-${index}`, + name: `File ${page}-${index}`, + file: {}, + })) + return new Response( + JSON.stringify({ + value, + '@odata.nextLink': `https://graph.microsoft.com/v1.0/me/drive/root/children?page=${page + 1}`, + }), + { status: 200 } + ) + }) + + const first = await microsoftSelectorAttachments['onedrive.files'].execute( + listArgs('onedrive.files') + ) + + expect(first).toMatchObject({ + kind: 'list', + nextCursor: 'https://graph.microsoft.com/v1.0/me/drive/root/children?page=1', + }) + expect(first.kind === 'list' ? first.items : []).toHaveLength(999) + expect(mockFetch).toHaveBeenCalledTimes(1) + + const second = await microsoftSelectorAttachments['onedrive.files'].execute( + listArgs('onedrive.files', 'https://graph.microsoft.com/v1.0/me/drive/root/children?page=1') + ) + + expect(second.kind === 'list' ? second.items[0]?.id : undefined).toBe('file-1-0') + expect(mockFetch).toHaveBeenCalledTimes(2) + }) + + it('rejects a Graph cursor for another operation', async () => { + await expect( + microsoftSelectorAttachments['onedrive.files'].execute( + listArgs('onedrive.files', 'https://graph.microsoft.com/v1.0/me/chats?$skiptoken=1') + ) + ).rejects.toMatchObject({ name: 'SelectorContextUnavailableError' }) + expect(mockFetch).not.toHaveBeenCalled() + }) + + it('hydrates a selected drive item without listing its siblings', async () => { + mockFetch.mockResolvedValueOnce( + new Response(JSON.stringify({ id: 'file-1', name: 'Quarterly report.xlsx' }), { + status: 200, + }) + ) + + await expect( + microsoftSelectorAttachments['onedrive.files'].execute({ + ...listArgs('onedrive.files'), + request: { kind: 'detail', id: 'file-1' }, + }) + ).resolves.toEqual({ + kind: 'detail', + item: { id: 'file-1', label: 'Quarterly report.xlsx' }, + }) + expect(String(mockFetch.mock.calls[0]?.[0])).toContain('/me/drive/items/file-1') + expect(mockFetch).toHaveBeenCalledTimes(1) + }) + + it('includes folders unless the selector explicitly requests files only', async () => { + const response = () => + new Response( + JSON.stringify({ + value: [ + { id: 'file-1', name: 'Report.pdf', file: {} }, + { id: 'folder-1', name: 'Reports', folder: {} }, + ], + }), + { status: 200 } + ) + mockFetch.mockResolvedValueOnce(response()).mockResolvedValueOnce(response()) + + const allItems = await microsoftSelectorAttachments['onedrive.files'].execute( + listArgs('onedrive.files') + ) + const filesOnly = await microsoftSelectorAttachments['onedrive.files'].execute({ + ...listArgs('onedrive.files'), + context: { oauthCredential: 'credential-1', mimeType: 'file' }, + }) + + expect(allItems.kind === 'list' ? allItems.items.map((item) => item.id) : []).toEqual([ + 'file-1', + 'folder-1', + ]) + expect(filesOnly.kind === 'list' ? filesOnly.items.map((item) => item.id) : []).toEqual([ + 'file-1', + ]) + }) + + it('rejects a planner task detail from another plan', async () => { + mockFetch.mockResolvedValueOnce( + new Response(JSON.stringify({ id: 'task-1', title: 'Task', planId: 'plan-2' }), { + status: 200, + }) + ) + + await expect( + microsoftSelectorAttachments['microsoft.planner'].execute(plannerTaskDetailArgs()) + ).resolves.toEqual({ kind: 'detail', item: null }) + }) + + it('paginates workbook worksheets through Graph continuation URLs', async () => { + mockFetch.mockResolvedValueOnce( + new Response( + JSON.stringify({ + value: [{ id: 'sheet-1', name: 'Sheet 1', position: 0 }], + '@odata.nextLink': + 'https://graph.microsoft.com/v1.0/me/drive/items/workbook-1/workbook/worksheets?$skiptoken=next', + }), + { status: 200 } + ) + ) + + await expect( + microsoftSelectorAttachments['microsoft.excel.sheets'].execute({ + ...listArgs('microsoft.excel.sheets'), + context: { oauthCredential: 'credential-1', spreadsheetId: 'workbook-1' }, + }) + ).resolves.toEqual({ + kind: 'list', + items: [{ id: 'Sheet 1', label: 'Sheet 1' }], + nextCursor: + 'https://graph.microsoft.com/v1.0/me/drive/items/workbook-1/workbook/worksheets?$skiptoken=next', + }) + }) +}) diff --git a/apps/sim/lib/selectors/server/providers/microsoft.ts b/apps/sim/lib/selectors/server/providers/microsoft.ts new file mode 100644 index 00000000000..b2984dde819 --- /dev/null +++ b/apps/sim/lib/selectors/server/providers/microsoft.ts @@ -0,0 +1,681 @@ +import { + validateMicrosoftGraphId, + validatePathSegment, + validateSharePointSiteId, +} from '@/lib/core/security/input-validation' +import { mapWithConcurrency } from '@/lib/core/utils/concurrency' +import type { ServerSelectorKey } from '@/lib/selectors/manifest' +import { resolveSelectorOAuthAccessToken } from '@/lib/selectors/server/credentials' +import { + SelectorConnectionUnavailableError, + SelectorContextUnavailableError, +} from '@/lib/selectors/server/errors' +import { flatSelectorResult } from '@/lib/selectors/server/providers/flat-results' +import { fetchProviderJson } from '@/lib/selectors/server/providers/provider-http' +import type { + ExecuteServerSelectorArgs, + SelectorCredentialPolicy, + ServerSelectorAttachmentMap, +} from '@/lib/selectors/server/types' +import { + detailSelectorResult, + listSelectorResult, + requireListRequest, +} from '@/lib/selectors/server/types' +import type { SafeSelectorOption } from '@/lib/selectors/types' +import { GRAPH_ID_PATTERN, getItemBasePath } from '@/tools/microsoft_excel/utils' +import { assertGraphNextPageUrl, getGraphNextPageUrl } from '@/tools/sharepoint/utils' + +type MicrosoftSelectorKey = Extract< + ServerSelectorKey, + | 'microsoft.planner.plans' + | 'outlook.folders' + | 'outlook.calendars' + | 'microsoft.teams' + | 'microsoft.chats' + | 'microsoft.channels' + | 'microsoft.planner' + | 'onedrive.files' + | 'onedrive.folders' + | 'microsoft.excel.sheets' + | 'microsoft.excel.drives' + | 'microsoft.excel' + | 'microsoft.word' +> + +const CHAT_LABEL_CONCURRENCY = 10 + +function microsoftCredential(serviceId: string): SelectorCredentialPolicy { + return { kind: 'stored', field: 'oauthCredential', serviceIds: [serviceId] } +} + +async function graphToken(args: ExecuteServerSelectorArgs, serviceId: string): Promise { + if (!args.credential) throw new SelectorConnectionUnavailableError() + return resolveSelectorOAuthAccessToken({ + credential: args.credential, + serviceId, + protectedValues: args.protectedValues, + }) +} + +interface GraphPage { + items: T[] + nextCursor?: string +} + +function graphPageUrl(cursor: string | undefined, initialUrl: string): string { + if (!cursor) return initialUrl + let cursorUrl: string + try { + cursorUrl = assertGraphNextPageUrl(cursor) + } catch { + throw new SelectorContextUnavailableError() + } + if (new URL(cursorUrl).pathname !== new URL(initialUrl).pathname) { + throw new SelectorContextUnavailableError() + } + return cursorUrl +} + +async function fetchGraphPage(input: { + args: ExecuteServerSelectorArgs + serviceId: string + initialUrl: string + token?: string + includeItem?(item: T): boolean +}): Promise> { + const request = requireListRequest(input.args.selectorKey, input.args.request) + const token = input.token ?? (await graphToken(input.args, input.serviceId)) + const data = await fetchProviderJson<{ value?: T[] } & Record>( + graphPageUrl(request.cursor, input.initialUrl), + { + headers: { Authorization: `Bearer ${token}`, 'Content-Type': 'application/json' }, + signal: input.args.signal, + redirect: 'error', + } + ) + const values = Array.isArray(data.value) + ? input.includeItem + ? data.value.filter(input.includeItem) + : data.value + : [] + const nextLink = getGraphNextPageUrl(data) + const nextCursor = nextLink ? graphPageUrl(nextLink, input.initialUrl) : undefined + return { items: values, ...(nextCursor ? { nextCursor } : {}) } +} + +async function fetchGraphDetail(input: { + args: ExecuteServerSelectorArgs + serviceId: string + url: string + token?: string +}): Promise { + const token = input.token ?? (await graphToken(input.args, input.serviceId)) + return fetchProviderJson(input.url, { + headers: { Authorization: `Bearer ${token}`, 'Content-Type': 'application/json' }, + signal: input.args.signal, + redirect: 'error', + }) +} + +function requireGraphId(value: string | undefined, label: string): string { + if (!value) throw new SelectorContextUnavailableError() + const validation = validateMicrosoftGraphId(value, label) + if (!validation.isValid) throw new SelectorContextUnavailableError() + return validation.sanitized ?? value +} + +function requireDriveId(value: string | undefined): string | undefined { + if (!value) return undefined + const validation = validatePathSegment(value, { + paramName: 'driveId', + customPattern: GRAPH_ID_PATTERN, + }) + if (!validation.isValid) throw new SelectorContextUnavailableError() + return validation.sanitized ?? value +} + +function encodeGraphSearch(value: string): string { + return encodeURIComponent(value).replace(/'/g, '%27') +} + +async function listPlannerPlans(args: ExecuteServerSelectorArgs) { + const page = await fetchGraphPage<{ id: string; title: string }>({ + args, + serviceId: 'microsoft-planner', + initialUrl: 'https://graph.microsoft.com/v1.0/me/planner/plans', + }) + return { + items: page.items.map((plan) => ({ id: plan.id, label: plan.title })), + nextCursor: page.nextCursor, + } +} + +async function listPlannerTasks(args: ExecuteServerSelectorArgs) { + const planId = requireGraphId(args.context.planId, 'planId') + const page = await fetchGraphPage<{ id: string; title: string }>({ + args, + serviceId: 'microsoft-planner', + initialUrl: `https://graph.microsoft.com/v1.0/planner/plans/${encodeURIComponent(planId)}/tasks`, + }) + return { + items: page.items.map((task) => ({ id: task.id, label: task.title })), + nextCursor: page.nextCursor, + } +} + +async function listOutlookFolders(args: ExecuteServerSelectorArgs) { + const page = await fetchGraphPage<{ id: string; displayName: string }>({ + args, + serviceId: 'outlook', + initialUrl: 'https://graph.microsoft.com/v1.0/me/mailFolders?$top=999', + }) + return { + items: page.items.map((folder) => ({ id: folder.id, label: folder.displayName })), + nextCursor: page.nextCursor, + } +} + +async function executeOutlookFolders(args: ExecuteServerSelectorArgs) { + if (args.request.kind === 'detail') { + const folderId = requireGraphId(args.request.id, 'folderId') + const folder = await fetchGraphDetail<{ id: string; displayName: string }>({ + args, + serviceId: 'outlook', + url: `https://graph.microsoft.com/v1.0/me/mailFolders/${encodeURIComponent(folderId)}?$select=id,displayName`, + }) + return detailSelectorResult({ id: folder.id, label: folder.displayName }) + } + const page = await listOutlookFolders(args) + return listSelectorResult(page.items, page.nextCursor) +} + +async function listOutlookCalendars(args: ExecuteServerSelectorArgs) { + const page = await fetchGraphPage<{ id: string; name: string }>({ + args, + serviceId: 'outlook', + initialUrl: 'https://graph.microsoft.com/v1.0/me/calendars?$top=100', + }) + return { + items: page.items.map((calendar) => ({ id: calendar.id, label: calendar.name })), + nextCursor: page.nextCursor, + } +} + +async function executeOutlookCalendars(args: ExecuteServerSelectorArgs) { + if (args.request.kind === 'detail') { + const calendarId = requireGraphId(args.request.id, 'calendarId') + const calendar = await fetchGraphDetail<{ id: string; name: string }>({ + args, + serviceId: 'outlook', + url: `https://graph.microsoft.com/v1.0/me/calendars/${encodeURIComponent(calendarId)}?$select=id,name`, + }) + return detailSelectorResult({ id: calendar.id, label: calendar.name }) + } + const page = await listOutlookCalendars(args) + return listSelectorResult(page.items, page.nextCursor) +} + +async function listTeams(args: ExecuteServerSelectorArgs) { + const page = await fetchGraphPage<{ id: string; displayName?: string }>({ + args, + serviceId: 'microsoft-teams', + initialUrl: 'https://graph.microsoft.com/v1.0/me/joinedTeams', + }) + return { + items: page.items.map((team) => ({ id: team.id, label: team.displayName || team.id })), + nextCursor: page.nextCursor, + } +} + +async function executeTeams(args: ExecuteServerSelectorArgs) { + if (args.request.kind === 'detail') { + const teamId = requireGraphId(args.request.id, 'teamId') + const team = await fetchGraphDetail<{ id: string; displayName?: string }>({ + args, + serviceId: 'microsoft-teams', + url: `https://graph.microsoft.com/v1.0/teams/${encodeURIComponent(teamId)}?$select=id,displayName`, + }) + return detailSelectorResult({ id: team.id, label: team.displayName || team.id }) + } + const page = await listTeams(args) + return listSelectorResult(page.items, page.nextCursor) +} + +async function listChannels(args: ExecuteServerSelectorArgs) { + const teamId = requireGraphId(args.context.teamId, 'teamId') + const page = await fetchGraphPage<{ id: string; displayName?: string }>({ + args, + serviceId: 'microsoft-teams', + initialUrl: `https://graph.microsoft.com/v1.0/teams/${encodeURIComponent(teamId)}/channels`, + }) + return { + items: page.items.map((channel) => ({ + id: channel.id, + label: channel.displayName || channel.id, + })), + nextCursor: page.nextCursor, + } +} + +async function executeChannels(args: ExecuteServerSelectorArgs) { + const teamId = requireGraphId(args.context.teamId, 'teamId') + if (args.request.kind === 'detail') { + const channelId = requireGraphId(args.request.id, 'channelId') + const channel = await fetchGraphDetail<{ id: string; displayName?: string }>({ + args, + serviceId: 'microsoft-teams', + url: `https://graph.microsoft.com/v1.0/teams/${encodeURIComponent(teamId)}/channels/${encodeURIComponent(channelId)}?$select=id,displayName`, + }) + return detailSelectorResult({ + id: channel.id, + label: channel.displayName || channel.id, + }) + } + const page = await listChannels(args) + return listSelectorResult(page.items, page.nextCursor) +} + +async function chatDisplayName( + chat: { id: string; topic?: string }, + token: string, + signal?: AbortSignal +): Promise { + if (chat.topic?.trim() && chat.topic !== 'null') return chat.topic + const validation = validateMicrosoftGraphId(chat.id, 'chatId') + if (!validation.isValid) return `Chat ${chat.id.slice(0, 8)}...` + const headers = { Authorization: `Bearer ${token}`, 'Content-Type': 'application/json' } + try { + const members = await fetchProviderJson<{ value?: Array<{ displayName?: string }> }>( + `https://graph.microsoft.com/v1.0/chats/${encodeURIComponent(chat.id)}/members`, + { headers, signal, redirect: 'error' } + ) + const names = (members.value ?? []) + .flatMap((member) => + member.displayName && member.displayName !== 'Unknown' ? [member.displayName] : [] + ) + .slice(0, 3) + if (names.length === 1) return names[0] + if (names.length === 2) return names.join(' & ') + if (names.length > 2) return `${names.slice(0, 2).join(', ')} & ${names.length - 2} more` + } catch { + signal?.throwIfAborted() + // A label enrichment failure must not hide an otherwise selectable chat. + } + try { + const messages = await fetchProviderJson<{ + value?: Array<{ + eventDetail?: { chatDisplayName?: string } + from?: { user?: { displayName?: string } } + }> + }>( + `https://graph.microsoft.com/v1.0/chats/${encodeURIComponent(chat.id)}/messages?$top=10&$orderby=createdDateTime desc`, + { headers, signal, redirect: 'error' } + ) + for (const message of messages.value ?? []) { + if (message.eventDetail?.chatDisplayName) return message.eventDetail.chatDisplayName + } + const names = [ + ...new Set( + (messages.value ?? []).flatMap((message) => { + const name = message.from?.user?.displayName + return name && name !== 'Unknown' ? [name] : [] + }) + ), + ].slice(0, 3) + if (names.length === 1) return names[0] + if (names.length === 2) return names.join(' & ') + if (names.length > 2) return `${names.slice(0, 2).join(', ')} & ${names.length - 2} more` + } catch { + signal?.throwIfAborted() + // Fall through to the stable id-based label. + } + return `Chat ${chat.id.split(':')[0] || chat.id.slice(0, 8)}...` +} + +async function listChats(args: ExecuteServerSelectorArgs) { + const token = await graphToken(args, 'microsoft-teams') + const page = await fetchGraphPage<{ id: string; topic?: string }>({ + args, + serviceId: 'microsoft-teams', + token, + initialUrl: 'https://graph.microsoft.com/v1.0/me/chats?$top=50', + }) + return { + items: await mapWithConcurrency(page.items, CHAT_LABEL_CONCURRENCY, async (chat) => ({ + id: chat.id, + label: await chatDisplayName(chat, token, args.signal), + })), + nextCursor: page.nextCursor, + } +} + +async function executeChats(args: ExecuteServerSelectorArgs) { + if (args.request.kind === 'detail') { + const chatId = requireGraphId(args.request.id, 'chatId') + const token = await graphToken(args, 'microsoft-teams') + const chat = await fetchGraphDetail<{ id: string; topic?: string }>({ + args, + serviceId: 'microsoft-teams', + token, + url: `https://graph.microsoft.com/v1.0/chats/${encodeURIComponent(chatId)}`, + }) + return detailSelectorResult({ + id: chat.id, + label: await chatDisplayName(chat, token, args.signal), + }) + } + const page = await listChats(args) + return listSelectorResult(page.items, page.nextCursor) +} + +interface DriveItem { + id: string + name: string + file?: { mimeType?: string } + folder?: Record + mimeType?: string +} + +async function listOneDriveFiles(args: ExecuteServerSelectorArgs) { + const filesOnly = args.context.mimeType === 'file' + const query = new URLSearchParams() + query.set( + '$select', + 'id,name,file,folder,webUrl,size,createdDateTime,lastModifiedDateTime,createdBy,thumbnails' + ) + query.set('$top', '999') + const page = await fetchGraphPage({ + args, + serviceId: 'onedrive', + initialUrl: `https://graph.microsoft.com/v1.0/me/drive/root/children?${query}`, + includeItem: (item) => (filesOnly ? Boolean(item.file && !item.folder) : true), + }) + return { + items: page.items.map((item) => ({ id: item.id, label: item.name })), + nextCursor: page.nextCursor, + } +} + +async function listOneDriveFolders(args: ExecuteServerSelectorArgs) { + const driveId = requireDriveId(args.context.driveId) + const drivePath = driveId ? `drives/${encodeURIComponent(driveId)}` : 'me/drive' + const page = await fetchGraphPage({ + args, + serviceId: 'onedrive', + initialUrl: `https://graph.microsoft.com/v1.0/${drivePath}/root/children?$filter=folder ne null&$select=id,name,folder,webUrl,createdDateTime,lastModifiedDateTime&$top=999`, + includeItem: (item) => Boolean(item.folder), + }) + return { + items: page.items.map((item) => ({ id: item.id, label: item.name })), + nextCursor: page.nextCursor, + } +} + +async function getDriveItem( + args: ExecuteServerSelectorArgs, + serviceId: string +): Promise { + const itemId = requireGraphId( + args.request.kind === 'detail' ? args.request.id : undefined, + 'itemId' + ) + const driveId = requireDriveId(args.context.driveId) + let basePath: string + try { + basePath = getItemBasePath(itemId, driveId) + } catch { + throw new SelectorContextUnavailableError() + } + const item = await fetchGraphDetail<{ id: string; name: string }>({ + args, + serviceId, + url: `${basePath}?$select=id,name,file,folder`, + }) + return { id: item.id, label: item.name } +} + +async function executeOneDriveFiles(args: ExecuteServerSelectorArgs) { + if (args.request.kind === 'detail') { + return detailSelectorResult(await getDriveItem(args, 'onedrive')) + } + const page = await listOneDriveFiles(args) + return listSelectorResult(page.items, page.nextCursor) +} + +async function executeOneDriveFolders(args: ExecuteServerSelectorArgs) { + if (args.request.kind === 'detail') { + return detailSelectorResult(await getDriveItem(args, 'onedrive')) + } + const page = await listOneDriveFolders(args) + return listSelectorResult(page.items, page.nextCursor) +} + +async function listWorksheets( + args: ExecuteServerSelectorArgs +): Promise> { + const spreadsheetId = requireGraphId(args.context.spreadsheetId, 'spreadsheetId') + const driveId = requireDriveId(args.context.driveId) + let basePath: string + try { + basePath = getItemBasePath(spreadsheetId, driveId) + } catch { + throw new SelectorContextUnavailableError() + } + const page = await fetchGraphPage<{ id: string; name: string; position: number }>({ + args, + serviceId: 'microsoft-excel', + initialUrl: `${basePath}/workbook/worksheets?$select=id,name,position&$orderby=position`, + }) + return { + items: page.items + .sort((left, right) => left.position - right.position) + .map((sheet) => ({ id: sheet.name, label: sheet.name })), + nextCursor: page.nextCursor, + } +} + +function requireSiteId(value: string | undefined): string { + if (!value) throw new SelectorContextUnavailableError() + const validation = validateSharePointSiteId(value, 'siteId') + if (!validation.isValid) throw new SelectorContextUnavailableError() + return validation.sanitized ?? value +} + +async function executeDrives(args: ExecuteServerSelectorArgs) { + const siteId = requireSiteId(args.context.siteId) + const token = await graphToken(args, 'microsoft-excel') + if (args.request.kind === 'detail') { + const driveId = requireDriveId(args.request.id) + if (!driveId) throw new SelectorContextUnavailableError() + const drive = await fetchProviderJson<{ id: string; name: string }>( + `https://graph.microsoft.com/v1.0/sites/${encodeURIComponent(siteId)}/drives/${encodeURIComponent(driveId)}?$select=id,name,driveType,webUrl`, + { headers: { Authorization: `Bearer ${token}` }, signal: args.signal, redirect: 'error' } + ) + return flatSelectorResult(args.request, [{ id: drive.id, label: drive.name }], true) + } + const page = await fetchGraphPage<{ id: string; name: string }>({ + args, + serviceId: 'microsoft-excel', + token, + initialUrl: `https://graph.microsoft.com/v1.0/sites/${encodeURIComponent(siteId)}/drives?$select=id,name,driveType,webUrl&$top=999`, + }) + return listSelectorResult( + page.items.map((drive) => ({ id: drive.id, label: drive.name })), + page.nextCursor + ) +} + +const OFFICE_FILE_TYPES = { + excel: { + extension: '.xlsx', + mimeType: 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet', + serviceId: 'microsoft-excel', + }, + word: { + extension: '.docx', + mimeType: 'application/vnd.openxmlformats-officedocument.wordprocessingml.document', + serviceId: 'microsoft-word', + }, +} as const + +async function listOfficeFiles( + args: ExecuteServerSelectorArgs, + fileType: keyof typeof OFFICE_FILE_TYPES +) { + const config = OFFICE_FILE_TYPES[fileType] + const driveId = requireDriveId(args.context.driveId) + const drivePath = driveId ? `drives/${encodeURIComponent(driveId)}` : 'me/drive' + const search = args.request.kind === 'list' ? (args.request.search ?? '') : '' + const searchQuery = search ? `${search} ${config.extension}` : config.extension + const params = new URLSearchParams() + params.set( + '$select', + 'id,name,mimeType,webUrl,thumbnails,createdDateTime,lastModifiedDateTime,size,createdBy' + ) + params.set('$top', '999') + const page = await fetchGraphPage({ + args, + serviceId: config.serviceId, + initialUrl: `https://graph.microsoft.com/v1.0/${drivePath}/root/search(q='${encodeGraphSearch(searchQuery)}')?${params}`, + includeItem: (file) => + file.name?.toLowerCase().endsWith(config.extension) || file.mimeType === config.mimeType, + }) + return { + items: page.items.map((file) => ({ id: file.id, label: file.name })), + nextCursor: page.nextCursor, + } +} + +async function executeOfficeFiles( + args: ExecuteServerSelectorArgs, + fileType: keyof typeof OFFICE_FILE_TYPES +) { + if (args.request.kind === 'detail') { + return detailSelectorResult(await getDriveItem(args, OFFICE_FILE_TYPES[fileType].serviceId)) + } + const page = await listOfficeFiles(args, fileType) + return listSelectorResult(page.items, page.nextCursor) +} + +async function executePlannerPlans(args: ExecuteServerSelectorArgs) { + if (args.request.kind === 'detail') { + const planId = requireGraphId(args.request.id, 'planId') + const token = await graphToken(args, 'microsoft-planner') + const plan = await fetchProviderJson<{ id: string; title: string }>( + `https://graph.microsoft.com/v1.0/planner/plans/${encodeURIComponent(planId)}`, + { + headers: { Authorization: `Bearer ${token}`, 'Content-Type': 'application/json' }, + signal: args.signal, + redirect: 'error', + } + ) + return detailSelectorResult({ id: plan.id, label: plan.title }) + } + const page = await listPlannerPlans(args) + return listSelectorResult(page.items, page.nextCursor) +} + +async function executePlannerTasks(args: ExecuteServerSelectorArgs) { + if (args.request.kind === 'detail') { + const planId = requireGraphId(args.context.planId, 'planId') + const taskId = requireGraphId(args.request.id, 'taskId') + const token = await graphToken(args, 'microsoft-planner') + const task = await fetchProviderJson<{ id: string; title: string; planId: string }>( + `https://graph.microsoft.com/v1.0/planner/tasks/${encodeURIComponent(taskId)}?$select=id,title,planId`, + { + headers: { Authorization: `Bearer ${token}`, 'Content-Type': 'application/json' }, + signal: args.signal, + redirect: 'error', + } + ) + if (task.planId !== planId) return detailSelectorResult(null) + return detailSelectorResult({ id: task.id, label: task.title }) + } + const page = await listPlannerTasks(args) + return listSelectorResult(page.items, page.nextCursor) +} + +const plannerCredential = microsoftCredential('microsoft-planner') +const outlookCredential = microsoftCredential('outlook') +const teamsCredential = microsoftCredential('microsoft-teams') +const oneDriveCredential = microsoftCredential('onedrive') +const oneDriveFolderCredential: SelectorCredentialPolicy = { + kind: 'stored', + field: 'oauthCredential', + serviceIds: ['onedrive', 'microsoft-word'], + resourceServiceId: 'onedrive', +} +const excelCredential = microsoftCredential('microsoft-excel') +const wordCredential = microsoftCredential('microsoft-word') + +export const microsoftSelectorAttachments = { + 'microsoft.planner.plans': { + credential: plannerCredential, + destination: 'fixed', + execute: executePlannerPlans, + }, + 'microsoft.planner': { + credential: plannerCredential, + destination: 'fixed', + execute: executePlannerTasks, + }, + 'outlook.folders': { + credential: outlookCredential, + destination: 'fixed', + execute: executeOutlookFolders, + }, + 'outlook.calendars': { + credential: outlookCredential, + destination: 'fixed', + execute: executeOutlookCalendars, + }, + 'microsoft.teams': { + credential: teamsCredential, + destination: 'fixed', + execute: executeTeams, + }, + 'microsoft.chats': { + credential: teamsCredential, + destination: 'fixed', + execute: executeChats, + }, + 'microsoft.channels': { + credential: teamsCredential, + destination: 'fixed', + execute: executeChannels, + }, + 'onedrive.files': { + credential: oneDriveCredential, + destination: 'fixed', + execute: executeOneDriveFiles, + }, + 'onedrive.folders': { + credential: oneDriveFolderCredential, + destination: 'fixed', + execute: executeOneDriveFolders, + }, + 'microsoft.excel.sheets': { + credential: excelCredential, + destination: 'fixed', + execute: async (args) => { + const page = await listWorksheets(args) + return listSelectorResult(page.items, page.nextCursor) + }, + }, + 'microsoft.excel.drives': { + credential: excelCredential, + destination: 'fixed', + execute: executeDrives, + }, + 'microsoft.excel': { + credential: excelCredential, + destination: 'fixed', + execute: async (args) => executeOfficeFiles(args, 'excel'), + }, + 'microsoft.word': { + credential: wordCredential, + destination: 'fixed', + execute: async (args) => executeOfficeFiles(args, 'word'), + }, +} satisfies ServerSelectorAttachmentMap diff --git a/apps/sim/lib/selectors/server/providers/monday.test.ts b/apps/sim/lib/selectors/server/providers/monday.test.ts new file mode 100644 index 00000000000..19ac53d0e76 --- /dev/null +++ b/apps/sim/lib/selectors/server/providers/monday.test.ts @@ -0,0 +1,77 @@ +/** + * @vitest-environment node + */ +import { afterAll, beforeEach, describe, expect, it, vi } from 'vitest' + +const { mockFetch, mockResolveSelectorOAuthAccessToken } = vi.hoisted(() => ({ + mockFetch: vi.fn(), + mockResolveSelectorOAuthAccessToken: vi.fn(), +})) + +vi.mock('@/lib/selectors/server/credentials', () => ({ + resolveSelectorOAuthAccessToken: mockResolveSelectorOAuthAccessToken, +})) + +import { createSelectorProtectedValues } from '@/lib/selectors/server/protected-values' +import { mondaySelectorAttachments } from '@/lib/selectors/server/providers/monday' +import type { ExecuteServerSelectorArgs } from '@/lib/selectors/server/types' + +function listArgs(): ExecuteServerSelectorArgs { + return { + selectorKey: 'monday.boards', + context: { oauthCredential: 'credential-1' }, + request: { kind: 'list' }, + scope: { kind: 'workspace', workspaceId: 'workspace-1' }, + workspaceId: 'workspace-1', + principal: { kind: 'session', userId: 'user-1', sessionId: 'session-1' }, + requesterUserId: 'user-1', + credential: { suppliedId: 'credential-1' }, + references: new Map(), + protectedValues: createSelectorProtectedValues(), + } +} + +describe('Monday server selector adapter', () => { + beforeEach(() => { + vi.clearAllMocks() + vi.stubGlobal('fetch', mockFetch) + mockResolveSelectorOAuthAccessToken.mockResolvedValue('server-only-token') + }) + + afterAll(() => vi.unstubAllGlobals()) + + it('falls back to the provider ID when a board name is empty', async () => { + mockFetch.mockResolvedValueOnce( + new Response(JSON.stringify({ data: { boards: [{ id: 'board-1', name: '' }] } }), { + status: 200, + }) + ) + + await expect(mondaySelectorAttachments['monday.boards'].execute(listArgs())).resolves.toEqual({ + kind: 'list', + items: [{ id: 'board-1', label: 'board-1' }], + }) + }) + + it('hydrates a selected board through a direct ID lookup', async () => { + mockFetch.mockResolvedValueOnce( + new Response(JSON.stringify({ data: { boards: [{ id: '9001', name: 'Direct board' }] } }), { + status: 200, + }) + ) + + await expect( + mondaySelectorAttachments['monday.boards'].execute({ + ...listArgs(), + request: { kind: 'detail', id: '9001' }, + }) + ).resolves.toEqual({ + kind: 'detail', + item: { id: '9001', label: 'Direct board' }, + }) + const body = JSON.parse(String(mockFetch.mock.calls[0]?.[1]?.body)) as { query: string } + expect(body.query).toContain('boards(ids: [9001])') + expect(body.query).not.toContain('limit:') + expect(mockFetch).toHaveBeenCalledTimes(1) + }) +}) diff --git a/apps/sim/lib/selectors/server/providers/monday.ts b/apps/sim/lib/selectors/server/providers/monday.ts new file mode 100644 index 00000000000..8a168193dc2 --- /dev/null +++ b/apps/sim/lib/selectors/server/providers/monday.ts @@ -0,0 +1,142 @@ +import { validateMondayNumericId } from '@/lib/core/security/input-validation' +import type { ServerSelectorKey } from '@/lib/selectors/manifest' +import { resolveSelectorOAuthAccessToken } from '@/lib/selectors/server/credentials' +import { + SelectorContextUnavailableError, + SelectorOptionsUnavailableError, +} from '@/lib/selectors/server/errors' +import { flatSelectorResult } from '@/lib/selectors/server/providers/flat-results' +import { fetchProviderJson } from '@/lib/selectors/server/providers/provider-http' +import { + detailSelectorResult, + type ExecuteServerSelectorArgs, + type ServerSelectorAttachmentMap, +} from '@/lib/selectors/server/types' +import type { SafeSelectorOption } from '@/lib/selectors/types' +import { MONDAY_API_URL, mondayHeaders } from '@/tools/monday/utils' + +type MondaySelectorKey = Extract + +const credential = { + kind: 'stored', + field: 'oauthCredential', + serviceIds: ['monday'], +} as const + +const PAGE_SIZE = 100 +const MAX_PAGES = 50 + +interface MondayResponse { + errors?: Array<{ message?: string }> + error_message?: string + data?: T +} + +function requireMondayData(response: MondayResponse): T { + if (response.errors?.length || response.error_message || response.data === undefined) { + throw new SelectorOptionsUnavailableError() + } + return response.data +} + +async function accessToken(args: ExecuteServerSelectorArgs): Promise { + if (!args.credential) throw new SelectorOptionsUnavailableError() + return resolveSelectorOAuthAccessToken({ + credential: args.credential, + serviceId: 'monday', + protectedValues: args.protectedValues, + }) +} + +async function listBoards(args: ExecuteServerSelectorArgs) { + const token = await accessToken(args) + const items: SafeSelectorOption[] = [] + let truncated = false + for (let page = 1; page <= MAX_PAGES; page++) { + const response = await fetchProviderJson< + MondayResponse<{ boards?: Array<{ id: string; name?: string | null }> }> + >(MONDAY_API_URL, { + method: 'POST', + headers: mondayHeaders(token), + body: JSON.stringify({ + query: `{ boards(limit: ${PAGE_SIZE}, page: ${page}, state: active) { id name } }`, + }), + signal: args.signal, + redirect: 'error', + }) + const boards = requireMondayData(response).boards ?? [] + items.push(...boards.map((board) => ({ id: board.id, label: board.name?.trim() || board.id }))) + if (boards.length < PAGE_SIZE) break + if (page === MAX_PAGES) truncated = true + } + return { items, truncated } +} + +async function getBoard( + args: ExecuteServerSelectorArgs, + boardId: string +): Promise { + const validated = validateMondayNumericId(boardId, 'boardId') + if (!validated.isValid) throw new SelectorContextUnavailableError() + const token = await accessToken(args) + const response = await fetchProviderJson< + MondayResponse<{ boards?: Array<{ id: string; name?: string | null }> }> + >(MONDAY_API_URL, { + method: 'POST', + headers: mondayHeaders(token), + body: JSON.stringify({ + query: `{ boards(ids: [${validated.sanitized}]) { id name } }`, + }), + signal: args.signal, + redirect: 'error', + }) + const board = requireMondayData(response).boards?.[0] + return board ? { id: boardId, label: board.name?.trim() || board.id } : null +} + +async function listGroups(args: ExecuteServerSelectorArgs): Promise { + const boardId = args.context.boardId + if (!boardId) throw new SelectorContextUnavailableError() + const validated = validateMondayNumericId(boardId, 'boardId') + if (!validated.isValid) throw new SelectorContextUnavailableError() + const token = await accessToken(args) + const response = await fetchProviderJson< + MondayResponse<{ + boards?: Array<{ groups?: Array<{ id: string; title?: string | null }> }> + }> + >(MONDAY_API_URL, { + method: 'POST', + headers: mondayHeaders(token), + body: JSON.stringify({ + query: `{ boards(ids: [${validated.sanitized}]) { groups { id title } } }`, + }), + signal: args.signal, + redirect: 'error', + }) + const groups = requireMondayData(response).boards?.[0]?.groups ?? [] + return groups.map((group) => ({ id: group.id, label: group.title?.trim() || group.id })) +} + +export const mondaySelectorAttachments = { + 'monday.boards': { + credential, + destination: 'fixed', + execute: async (args) => { + if (args.request.kind === 'detail') { + return detailSelectorResult(await getBoard(args, args.request.id)) + } + const result = await listBoards(args) + return flatSelectorResult( + args.request, + result.items, + false, + result.truncated ? { truncated: { reason: 'provider-cap', pages: MAX_PAGES } } : undefined + ) + }, + }, + 'monday.groups': { + credential, + destination: 'fixed', + execute: async (args) => flatSelectorResult(args.request, await listGroups(args), true), + }, +} satisfies ServerSelectorAttachmentMap diff --git a/apps/sim/lib/selectors/server/providers/netsuite.test.ts b/apps/sim/lib/selectors/server/providers/netsuite.test.ts new file mode 100644 index 00000000000..1ebc2d2f8c2 --- /dev/null +++ b/apps/sim/lib/selectors/server/providers/netsuite.test.ts @@ -0,0 +1,53 @@ +/** + * @vitest-environment node + */ +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const mocks = vi.hoisted(() => ({ + executeListRecordTypes: vi.fn(), +})) + +vi.mock('@/lib/internal/netsuite/operations/list-record-types', () => ({ + executeNetsuiteListRecordTypesOperation: mocks.executeListRecordTypes, +})) + +vi.mock('@/lib/internal/netsuite/operations/get-async-status', () => ({ + executeNetsuiteGetAsyncStatusOperation: vi.fn(), +})) + +import { SelectorConnectionUnavailableError } from '@/lib/selectors/server/errors' +import { createSelectorProtectedValues } from '@/lib/selectors/server/protected-values' +import { netsuiteSelectorAttachments } from '@/lib/selectors/server/providers/netsuite' +import type { ExecuteServerSelectorArgs } from '@/lib/selectors/server/types' + +const args: ExecuteServerSelectorArgs = { + selectorKey: 'netsuite.recordTypes', + context: {}, + request: { kind: 'list' }, + scope: { kind: 'workspace', workspaceId: 'workspace-1' }, + workspaceId: 'workspace-1', + principal: { kind: 'session', userId: 'user-1', sessionId: 'session-1' }, + requesterUserId: 'user-1', + references: new Map(), + protectedValues: createSelectorProtectedValues(), +} + +describe('NetSuite server selector adapter', () => { + beforeEach(() => vi.clearAllMocks()) + + it('preserves a safe provider authentication status without forwarding its body', async () => { + mocks.executeListRecordTypes.mockResolvedValue({ + success: false, + output: { status: 401, data: 'provider-secret-canary' }, + error: 'provider-secret-canary', + }) + + await expect( + netsuiteSelectorAttachments['netsuite.recordTypes'].execute(args, { + oauthCredential: 'credential-1', + accessToken: 'server-token', + instanceUrl: 'https://123.suitetalk.api.netsuite.com', + }) + ).rejects.toEqual(new SelectorConnectionUnavailableError(401)) + }) +}) diff --git a/apps/sim/lib/selectors/server/providers/netsuite.ts b/apps/sim/lib/selectors/server/providers/netsuite.ts new file mode 100644 index 00000000000..1991c6d7396 --- /dev/null +++ b/apps/sim/lib/selectors/server/providers/netsuite.ts @@ -0,0 +1,262 @@ +import { isPlainRecord } from '@sim/utils/object' +import { NETSUITE_SERVICE_ACCOUNT_PROVIDER_ID } from '@/lib/credentials/client-credential-accounts/descriptors' +import { executeNetsuiteGetAsyncStatusOperation } from '@/lib/internal/netsuite/operations/get-async-status' +import { executeNetsuiteListRecordTypesOperation } from '@/lib/internal/netsuite/operations/list-record-types' +import { resolveOAuthAccountId } from '@/lib/oauth/credential-service' +import type { ServerSelectorKey } from '@/lib/selectors/manifest' +import { + SelectorConnectionUnavailableError, + SelectorContextUnavailableError, + SelectorOptionsUnavailableError, +} from '@/lib/selectors/server/errors' +import { resolveSelectorCredentialBundle } from '@/lib/selectors/server/providers/credential-bundle' +import { flatSelectorResult } from '@/lib/selectors/server/providers/flat-results' +import { selectorProviderStatusError } from '@/lib/selectors/server/providers/provider-http' +import type { + ExecuteServerSelectorArgs, + ServerSelectorAttachmentMap, +} from '@/lib/selectors/server/types' +import { definePreparedSelectorAttachment } from '@/lib/selectors/server/types' +import type { SafeSelectorOption } from '@/lib/selectors/types' +import type { NetSuiteAuthParams } from '@/tools/netsuite/types' +import { normalizeSuiteTalkUrl } from '@/tools/netsuite/utils' +import type { ToolResponse } from '@/tools/types' + +type NetSuiteSelectorKey = Extract< + ServerSelectorKey, + 'netsuite.recordTypes' | 'netsuite.asyncTasks' +> + +type NetSuiteSelectorKind = 'record_types' | 'async_tasks' +type PreparedNetSuiteDestination = NetSuiteAuthParams & { instanceUrl: string } + +const NETSUITE_SELECTOR_KIND = { + 'netsuite.recordTypes': 'record_types', + 'netsuite.asyncTasks': 'async_tasks', +} as const satisfies Record + +const MAX_RECORD_TYPES = 1_000 +const MAX_ASYNC_TASKS = 100 +const MAX_ID_LENGTH = 512 + +interface NetSuiteSelectorObject { + id: string + label: string + detail: string | null +} + +function requireString(value: unknown, maxLength: number): string { + if (typeof value !== 'string' || !value.trim()) throw new SelectorOptionsUnavailableError() + const normalized = value.trim() + if (normalized.length > maxLength) throw new SelectorOptionsUnavailableError() + return normalized +} + +function requireItems(data: unknown): Record[] { + if (!isPlainRecord(data) || !Array.isArray(data.items) || !data.items.every(isPlainRecord)) { + throw new SelectorOptionsUnavailableError() + } + return data.items +} + +function dedupeAndSort(objects: NetSuiteSelectorObject[]): NetSuiteSelectorObject[] { + const unique = new Map() + for (const object of objects) { + if (!unique.has(object.id)) unique.set(object.id, object) + } + return [...unique.values()].sort( + (left, right) => left.label.localeCompare(right.label) || left.id.localeCompare(right.id) + ) +} + +function normalizeRecordTypes(data: unknown): NetSuiteSelectorObject[] { + const objects: NetSuiteSelectorObject[] = [] + const names = new Set() + for (const item of requireItems(data)) { + const name = requireString(item.name, MAX_ID_LENGTH) + if (names.has(name)) continue + if (names.size >= MAX_RECORD_TYPES) throw new SelectorOptionsUnavailableError() + names.add(name) + objects.push({ id: name, label: name, detail: null }) + } + return dedupeAndSort(objects) +} + +function taskIdFromHref(href: unknown, origin: string, jobId: string): string { + const hrefValue = requireString(href, 4_096) + let url: URL + try { + url = new URL(hrefValue, origin) + } catch { + throw new SelectorOptionsUnavailableError() + } + if ( + url.protocol !== 'https:' || + url.origin !== origin || + url.username || + url.password || + url.search || + url.hash + ) { + throw new SelectorOptionsUnavailableError() + } + + const match = url.pathname.match(/^\/services\/rest\/async\/v1\/job\/([^/]+)\/task\/([^/]+)$/) + if (!match?.[1] || !match[2]) throw new SelectorOptionsUnavailableError() + + let linkedJobId: string + let taskId: string + try { + linkedJobId = decodeURIComponent(match[1]) + taskId = decodeURIComponent(match[2]) + } catch { + throw new SelectorOptionsUnavailableError() + } + if (linkedJobId !== jobId || !taskId || taskId.length > MAX_ID_LENGTH) { + throw new SelectorOptionsUnavailableError() + } + + const canonicalPath = `/services/rest/async/v1/job/${encodeURIComponent(linkedJobId)}/task/${encodeURIComponent(taskId)}` + if ( + url.pathname !== canonicalPath || + (hrefValue !== canonicalPath && hrefValue !== `${origin}${canonicalPath}`) + ) { + throw new SelectorOptionsUnavailableError() + } + return taskId +} + +function normalizeAsyncTasks( + data: unknown, + instanceUrl: string, + jobId: string +): NetSuiteSelectorObject[] { + const origin = normalizeSuiteTalkUrl(instanceUrl) + const objects = new Map() + for (const item of requireItems(data)) { + if (!Array.isArray(item.links) || item.links.length === 0 || !item.links.every(isPlainRecord)) { + throw new SelectorOptionsUnavailableError() + } + const selfLinks = item.links.filter((link) => link.rel === 'self') + if (selfLinks.length === 0) throw new SelectorOptionsUnavailableError() + for (const link of selfLinks) { + const id = taskIdFromHref(link.href, origin, jobId) + if (objects.has(id)) continue + if (objects.size >= MAX_ASYNC_TASKS) throw new SelectorOptionsUnavailableError() + objects.set(id, { id, label: id, detail: null }) + } + } + return dedupeAndSort([...objects.values()]) +} + +function requireJobId(args: ExecuteServerSelectorArgs): string { + const jobId = args.context.jobId?.trim() + if (!jobId || jobId.length > MAX_ID_LENGTH) throw new SelectorContextUnavailableError() + return jobId +} + +async function requireNetSuiteServiceAccount(args: ExecuteServerSelectorArgs): Promise { + const credential = args.credential + const access = credential?.access + if (!credential || !access?.resolvedCredentialId || access.credentialType !== 'service_account') { + throw new SelectorConnectionUnavailableError() + } + const resolved = await resolveOAuthAccountId(access.resolvedCredentialId) + if ( + resolved?.credentialType !== 'service_account' || + resolved.providerId !== NETSUITE_SERVICE_ACCOUNT_PROVIDER_ID + ) { + throw new SelectorConnectionUnavailableError() + } + return access.resolvedCredentialId +} + +async function prepareNetSuiteDestination( + args: ExecuteServerSelectorArgs +): Promise { + const resolvedCredentialId = await requireNetSuiteServiceAccount(args) + if (!args.credential) throw new SelectorConnectionUnavailableError() + const token = await resolveSelectorCredentialBundle({ + credential: args.credential, + protectedValues: args.protectedValues, + }) + if (!token.instanceUrl) throw new SelectorConnectionUnavailableError() + let instanceUrl: string + try { + instanceUrl = normalizeSuiteTalkUrl(token.instanceUrl) + } catch { + throw new SelectorConnectionUnavailableError() + } + return { + oauthCredential: resolvedCredentialId, + accessToken: token.accessToken, + instanceUrl, + } +} + +async function executeDiscoveryTool( + kind: NetSuiteSelectorKind, + args: ExecuteServerSelectorArgs, + auth: NetSuiteAuthParams, + jobId?: string +): Promise { + if (args.signal?.aborted) throw args.signal.reason + if (kind === 'record_types') { + return executeNetsuiteListRecordTypesOperation(auth, args.signal) + } + if (!jobId) throw new SelectorOptionsUnavailableError() + return executeNetsuiteGetAsyncStatusOperation({ ...auth, jobId, view: 'tasks' }, args.signal) +} + +function toOptions(objects: NetSuiteSelectorObject[]): SafeSelectorOption[] { + return objects.map((object) => ({ + id: object.id, + label: object.label, + ...(object.detail ? { meta: { detail: object.detail } } : {}), + })) +} + +async function executeNetSuite(args: ExecuteServerSelectorArgs, auth: PreparedNetSuiteDestination) { + const kind = NETSUITE_SELECTOR_KIND[args.selectorKey as NetSuiteSelectorKey] + if (!kind) throw new SelectorOptionsUnavailableError() + const jobId = kind === 'async_tasks' ? requireJobId(args) : undefined + const result = await executeDiscoveryTool(kind, args, auth, jobId) + if (!result.success) { + const status = result.output.status + throw selectorProviderStatusError(Number.isInteger(status) ? status : 502) + } + const objects = + kind === 'record_types' + ? normalizeRecordTypes(result.output.data) + : normalizeAsyncTasks(result.output.data, auth.instanceUrl, jobId as string) + return flatSelectorResult(args.request, toOptions(objects), true) +} + +const credential = { + kind: 'stored', + field: 'oauthCredential', + serviceIds: ['netsuite'], +} as const + +/** + * The integration this selector reaches. Declared rather than derived: NetSuite is an + * API-key integration with no entry in the deployment OAuth catalog, so its + * service id maps to no block type and the allowlist would have nothing to + * judge it on. + */ +const integrationBlockTypes = ['netsuite'] as const + +export const netsuiteSelectorAttachments = { + 'netsuite.recordTypes': definePreparedSelectorAttachment({ + credential, + integrationBlockTypes, + destination: { kind: 'credential-bound', prepare: prepareNetSuiteDestination }, + execute: executeNetSuite, + }), + 'netsuite.asyncTasks': definePreparedSelectorAttachment({ + credential, + integrationBlockTypes, + destination: { kind: 'credential-bound', prepare: prepareNetSuiteDestination }, + execute: executeNetSuite, + }), +} satisfies ServerSelectorAttachmentMap diff --git a/apps/sim/lib/selectors/server/providers/notion.test.ts b/apps/sim/lib/selectors/server/providers/notion.test.ts new file mode 100644 index 00000000000..55b955fb5db --- /dev/null +++ b/apps/sim/lib/selectors/server/providers/notion.test.ts @@ -0,0 +1,66 @@ +/** + * @vitest-environment node + */ +import { afterAll, beforeEach, describe, expect, it, vi } from 'vitest' + +const { mockFetch, mockResolveSelectorOAuthAccessToken } = vi.hoisted(() => ({ + mockFetch: vi.fn(), + mockResolveSelectorOAuthAccessToken: vi.fn(), +})) + +vi.mock('@/lib/selectors/server/credentials', () => ({ + resolveSelectorOAuthAccessToken: mockResolveSelectorOAuthAccessToken, +})) + +import { createSelectorProtectedValues } from '@/lib/selectors/server/protected-values' +import { notionSelectorAttachments } from '@/lib/selectors/server/providers/notion' +import type { ExecuteServerSelectorArgs } from '@/lib/selectors/server/types' + +function detailArgs(): ExecuteServerSelectorArgs { + return { + selectorKey: 'notion.pages', + context: { oauthCredential: 'credential-1' }, + request: { kind: 'detail', id: '1234567890abcdef1234567890abcdef' }, + scope: { kind: 'workspace', workspaceId: 'workspace-1' }, + workspaceId: 'workspace-1', + principal: { kind: 'session', userId: 'user-1', sessionId: 'session-1' }, + requesterUserId: 'user-1', + credential: { suppliedId: 'credential-1' }, + references: new Map(), + protectedValues: createSelectorProtectedValues(), + } +} + +describe('Notion server selector adapter', () => { + beforeEach(() => { + vi.clearAllMocks() + vi.stubGlobal('fetch', mockFetch) + mockResolveSelectorOAuthAccessToken.mockResolvedValue('server-only-token') + }) + + afterAll(() => vi.unstubAllGlobals()) + + it('hydrates a selected page directly without scanning the capped search listing', async () => { + mockFetch.mockResolvedValueOnce( + new Response( + JSON.stringify({ + object: 'page', + id: 'page-provider-id', + properties: { + Project: { type: 'title', title: [{ plain_text: 'Planning' }] }, + }, + }), + { status: 200 } + ) + ) + + await expect(notionSelectorAttachments['notion.pages'].execute(detailArgs())).resolves.toEqual({ + kind: 'detail', + item: { id: '1234567890abcdef1234567890abcdef', label: 'Planning' }, + }) + expect(String(mockFetch.mock.calls[0]?.[0])).toContain( + '/v1/pages/1234567890abcdef1234567890abcdef' + ) + expect(mockFetch).toHaveBeenCalledTimes(1) + }) +}) diff --git a/apps/sim/lib/selectors/server/providers/notion.ts b/apps/sim/lib/selectors/server/providers/notion.ts new file mode 100644 index 00000000000..fb8cf419800 --- /dev/null +++ b/apps/sim/lib/selectors/server/providers/notion.ts @@ -0,0 +1,147 @@ +import type { ServerSelectorKey } from '@/lib/selectors/manifest' +import { resolveSelectorOAuthAccessToken } from '@/lib/selectors/server/credentials' +import { + SelectorContextUnavailableError, + SelectorOptionsUnavailableError, +} from '@/lib/selectors/server/errors' +import { flatSelectorResult } from '@/lib/selectors/server/providers/flat-results' +import { fetchProviderJson } from '@/lib/selectors/server/providers/provider-http' +import { + detailSelectorResult, + type ExecuteServerSelectorArgs, + type ServerSelectorAttachmentMap, +} from '@/lib/selectors/server/types' +import type { SafeSelectorOption } from '@/lib/selectors/types' +import { extractTitleFromItem } from '@/tools/notion/utils' + +type NotionSelectorKey = Extract + +const credential = { + kind: 'stored', + field: 'oauthCredential', + serviceIds: ['notion'], +} as const + +const PAGE_SIZE = 100 +const MAX_PAGES = 20 + +interface NotionSearchPage { + results?: unknown[] + has_more?: boolean + next_cursor?: string | null +} + +async function notionToken(args: ExecuteServerSelectorArgs): Promise { + if (!args.credential) throw new SelectorOptionsUnavailableError() + return resolveSelectorOAuthAccessToken({ + credential: args.credential, + serviceId: 'notion', + protectedValues: args.protectedValues, + }) +} + +function notionHeaders(token: string) { + return { + Authorization: `Bearer ${token}`, + 'Content-Type': 'application/json', + 'Notion-Version': '2022-06-28', + } +} + +function toNotionOption(value: unknown, requestedId?: string): SafeSelectorOption | null { + if (!value || typeof value !== 'object' || typeof (value as { id?: unknown }).id !== 'string') { + return null + } + return { + id: requestedId ?? (value as { id: string }).id, + label: extractTitleFromItem(value), + } +} + +async function getNotionObject( + args: ExecuteServerSelectorArgs, + object: 'database' | 'page', + id: string +): Promise { + const token = await notionToken(args) + const value = await fetchProviderJson( + `https://api.notion.com/v1/${object === 'database' ? 'databases' : 'pages'}/${encodeURIComponent(id)}`, + { + headers: notionHeaders(token), + signal: args.signal, + redirect: 'error', + } + ) + const option = toNotionOption(value, id) + if (!option) throw new SelectorOptionsUnavailableError() + return option +} + +async function listNotionObjects( + args: ExecuteServerSelectorArgs, + object: 'database' | 'page' +): Promise<{ items: SafeSelectorOption[]; truncated: boolean }> { + const token = await notionToken(args) + const results: unknown[] = [] + let cursor: string | undefined + let truncated = false + + for (let page = 0; page < MAX_PAGES; page++) { + const data = await fetchProviderJson('https://api.notion.com/v1/search', { + method: 'POST', + headers: notionHeaders(token), + body: JSON.stringify({ + filter: { value: object, property: 'object' }, + page_size: PAGE_SIZE, + ...(cursor ? { start_cursor: cursor } : {}), + }), + signal: args.signal, + redirect: 'error', + }) + if (Array.isArray(data.results)) results.push(...data.results) + if (!data.has_more) break + const nextCursor = data.next_cursor?.trim() + if (!nextCursor) { + truncated = true + break + } + cursor = nextCursor + if (page === MAX_PAGES - 1) truncated = true + } + + return { + items: results.flatMap((value) => { + const option = toNotionOption(value) + return option ? [option] : [] + }), + truncated, + } +} + +async function executeNotionObjects(args: ExecuteServerSelectorArgs, object: 'database' | 'page') { + if (args.request.kind === 'detail') { + const id = args.request.id.trim() + if (!/^[0-9a-f-]{32,36}$/i.test(id)) throw new SelectorContextUnavailableError() + return detailSelectorResult(await getNotionObject(args, object, id)) + } + const { items, truncated } = await listNotionObjects(args, object) + return flatSelectorResult( + args.request, + items, + true, + truncated ? { truncated: { reason: 'provider-cap', pages: MAX_PAGES } } : undefined + ) +} + +export const notionSelectorAttachments = { + 'notion.databases': { + credential, + destination: 'fixed', + execute: async (args) => executeNotionObjects(args, 'database'), + }, + 'notion.pages': { + credential, + destination: 'fixed', + execute: async (args) => executeNotionObjects(args, 'page'), + }, +} satisfies ServerSelectorAttachmentMap diff --git a/apps/sim/lib/selectors/server/providers/pipedrive.test.ts b/apps/sim/lib/selectors/server/providers/pipedrive.test.ts new file mode 100644 index 00000000000..d09256e27c4 --- /dev/null +++ b/apps/sim/lib/selectors/server/providers/pipedrive.test.ts @@ -0,0 +1,61 @@ +/** + * @vitest-environment node + */ +import { afterAll, beforeEach, describe, expect, it, vi } from 'vitest' + +const { mockFetch, mockResolveSelectorCredentialBundle } = vi.hoisted(() => ({ + mockFetch: vi.fn(), + mockResolveSelectorCredentialBundle: vi.fn(), +})) + +vi.mock('@/lib/selectors/server/providers/credential-bundle', () => ({ + resolveSelectorCredentialBundle: mockResolveSelectorCredentialBundle, +})) + +import { createSelectorProtectedValues } from '@/lib/selectors/server/protected-values' +import { pipedriveSelectorAttachments } from '@/lib/selectors/server/providers/pipedrive' +import type { ExecuteServerSelectorArgs } from '@/lib/selectors/server/types' + +function args(): ExecuteServerSelectorArgs { + return { + selectorKey: 'pipedrive.pipelines', + context: { oauthCredential: 'credential-1' }, + request: { kind: 'list' }, + scope: { kind: 'workspace', workspaceId: 'workspace-1' }, + workspaceId: 'workspace-1', + principal: { kind: 'session', userId: 'user-1', sessionId: 'session-1' }, + requesterUserId: 'user-1', + credential: { suppliedId: 'credential-1' }, + references: new Map(), + protectedValues: createSelectorProtectedValues(), + } +} + +describe('Pipedrive server selector adapter', () => { + beforeEach(() => { + vi.clearAllMocks() + vi.stubGlobal('fetch', mockFetch) + mockResolveSelectorCredentialBundle.mockResolvedValue({ accessToken: 'server-only-token' }) + }) + + afterAll(() => vi.unstubAllGlobals()) + + it('rejects a semantic failure instead of returning an empty pipeline list', async () => { + mockFetch.mockResolvedValueOnce( + new Response( + JSON.stringify({ + success: false, + error: 'Requested service is not available', + error_info: 'Please check developers.pipedrive.com', + data: null, + additional_data: null, + }), + { status: 200 } + ) + ) + + await expect( + pipedriveSelectorAttachments['pipedrive.pipelines'].execute(args()) + ).rejects.toMatchObject({ name: 'SelectorOptionsUnavailableError' }) + }) +}) diff --git a/apps/sim/lib/selectors/server/providers/pipedrive.ts b/apps/sim/lib/selectors/server/providers/pipedrive.ts new file mode 100644 index 00000000000..b8512ee84a4 --- /dev/null +++ b/apps/sim/lib/selectors/server/providers/pipedrive.ts @@ -0,0 +1,108 @@ +import { z } from 'zod' +import { MAX_SELECTOR_OPTIONS } from '@/lib/selectors/limits' +import type { ServerSelectorKey } from '@/lib/selectors/manifest' +import { + SelectorConnectionUnavailableError, + SelectorOptionsUnavailableError, +} from '@/lib/selectors/server/errors' +import { appendSelectorOptions } from '@/lib/selectors/server/option-budget' +import { resolveSelectorCredentialBundle } from '@/lib/selectors/server/providers/credential-bundle' +import { flatSelectorResult } from '@/lib/selectors/server/providers/flat-results' +import { fetchProviderJson } from '@/lib/selectors/server/providers/provider-http' +import type { ServerSelectorAttachmentMap } from '@/lib/selectors/server/types' +import type { SafeSelectorOption } from '@/lib/selectors/types' +import { getPipedriveAuthHeaders } from '@/tools/pipedrive/utils' + +type PipedriveSelectorKey = Extract + +const PAGE_SIZE = 500 +const MAX_PAGES = 50 + +const pipedrivePageSchema = z.object({ + success: z.literal(true), + data: z.array( + z.object({ + id: z.union([z.number().finite(), z.string().min(1)]), + name: z.string().min(1), + }) + ), + additional_data: z + .object({ + pagination: z + .object({ + more_items_in_collection: z.boolean().optional(), + next_start: z.number().int().nonnegative().optional(), + }) + .optional(), + }) + .optional(), +}) + +export const pipedriveSelectorAttachments = { + 'pipedrive.pipelines': { + credential: { + kind: 'stored', + field: 'oauthCredential', + serviceIds: ['pipedrive'], + }, + destination: 'fixed', + execute: async (args) => { + if (!args.credential) throw new SelectorConnectionUnavailableError() + const token = await resolveSelectorCredentialBundle({ + credential: args.credential, + protectedValues: args.protectedValues, + }) + const items: SafeSelectorOption[] = [] + let start = 0 + let truncated = false + for (let page = 0; page < MAX_PAGES; page++) { + const url = new URL('https://api.pipedrive.com/v1/pipelines') + url.searchParams.set('start', String(start)) + url.searchParams.set('limit', String(PAGE_SIZE)) + const body = await fetchProviderJson(url, { + headers: getPipedriveAuthHeaders({ + accessToken: token.accessToken, + authStyle: token.authStyle, + }), + signal: args.signal, + redirect: 'error', + }) + const parsed = pipedrivePageSchema.safeParse(body) + if (!parsed.success) throw new SelectorOptionsUnavailableError() + const data = parsed.data + const appended = appendSelectorOptions( + items, + (data.data ?? []).map((pipeline) => ({ + id: String(pipeline.id), + label: pipeline.name, + })) + ) + const pagination = data.additional_data?.pagination + if (!pagination?.more_items_in_collection || typeof pagination.next_start !== 'number') { + if (appended.overflow) truncated = true + break + } + if (appended.full) { + truncated = true + break + } + start = pagination.next_start + if (page === MAX_PAGES - 1) truncated = true + } + return flatSelectorResult( + args.request, + items, + true, + truncated + ? { + truncated: { + reason: 'provider-cap', + limit: MAX_SELECTOR_OPTIONS, + pages: MAX_PAGES, + }, + } + : undefined + ) + }, + }, +} satisfies ServerSelectorAttachmentMap diff --git a/apps/sim/lib/selectors/server/providers/provider-http.test.ts b/apps/sim/lib/selectors/server/providers/provider-http.test.ts new file mode 100644 index 00000000000..df8de2820e6 --- /dev/null +++ b/apps/sim/lib/selectors/server/providers/provider-http.test.ts @@ -0,0 +1,103 @@ +/** + * @vitest-environment node + */ +import { afterAll, beforeEach, describe, expect, it, vi } from 'vitest' +import { + SelectorConnectionUnavailableError, + SelectorOptionsUnavailableError, +} from '@/lib/selectors/server/errors' +import { + fetchProviderJson, + fetchProviderJsonWithStatus, + RetryableProviderNetworkError, +} from '@/lib/selectors/server/providers/provider-http' + +const mockFetch = vi.fn() + +function openBody(onCancel: () => void): ReadableStream { + return new ReadableStream({ + start(controller) { + controller.enqueue(new TextEncoder().encode('{}')) + }, + cancel: onCancel, + }) +} + +describe('provider HTTP selector boundary', () => { + beforeEach(() => { + vi.clearAllMocks() + vi.stubGlobal('fetch', mockFetch) + }) + + afterAll(() => vi.unstubAllGlobals()) + + it('cancels rejected and declared-oversized provider bodies', async () => { + const rejectedCancel = vi.fn() + mockFetch.mockResolvedValueOnce( + new Response(openBody(rejectedCancel), { status: 502, statusText: 'Bad Gateway' }) + ) + + await expect(fetchProviderJson('https://provider.example/items')).rejects.toBeInstanceOf( + SelectorOptionsUnavailableError + ) + expect(rejectedCancel).toHaveBeenCalledOnce() + + const oversizedCancel = vi.fn() + mockFetch.mockResolvedValueOnce( + new Response(openBody(oversizedCancel), { + status: 200, + headers: { 'content-length': String(16 * 1024 * 1024 + 1) }, + }) + ) + + await expect(fetchProviderJson('https://provider.example/items')).rejects.toBeInstanceOf( + SelectorOptionsUnavailableError + ) + expect(oversizedCancel).toHaveBeenCalledOnce() + }) + + it('returns only an allowlisted error status after discarding its body', async () => { + const cancel = vi.fn() + mockFetch.mockResolvedValueOnce(new Response(openBody(cancel), { status: 404 })) + + await expect( + fetchProviderJsonWithStatus('https://provider.example/item', undefined, { + passthroughStatuses: [404], + }) + ).resolves.toEqual({ ok: false, status: 404 }) + expect(cancel).toHaveBeenCalledOnce() + }) + + it('can preserve a generic retry signal without exposing the raw network error', async () => { + mockFetch.mockRejectedValueOnce(new Error('fetch failed with provider-secret-canary')) + + await expect( + fetchProviderJsonWithStatus('https://provider.example/item', undefined, { + passthroughNetworkErrors: true, + }) + ).rejects.toEqual(new RetryableProviderNetworkError()) + }) + + it.each([ + [401, SelectorConnectionUnavailableError, 401], + [403, SelectorConnectionUnavailableError, 403], + [429, SelectorOptionsUnavailableError, 429], + [500, SelectorOptionsUnavailableError, 502], + ])( + 'maps provider status %s without exposing its body', + async (status, ErrorType, expectedStatus) => { + const cancel = vi.fn() + mockFetch.mockResolvedValueOnce( + new Response(openBody(cancel), { status, statusText: 'provider-controlled diagnostic' }) + ) + + await expect(fetchProviderJson('https://provider.example/items')).rejects.toMatchObject({ + name: ErrorType.name, + status: expectedStatus, + message: + status === 401 || status === 403 ? 'Connection unavailable' : 'Options unavailable', + }) + expect(cancel).toHaveBeenCalledOnce() + } + ) +}) diff --git a/apps/sim/lib/selectors/server/providers/provider-http.ts b/apps/sim/lib/selectors/server/providers/provider-http.ts new file mode 100644 index 00000000000..00394622b00 --- /dev/null +++ b/apps/sim/lib/selectors/server/providers/provider-http.ts @@ -0,0 +1,128 @@ +import { parseRetryAfter } from '@sim/utils/retry' +import { + SelectorConnectionUnavailableError, + SelectorOptionsUnavailableError, +} from '@/lib/selectors/server/errors' + +const PROVIDER_TIMEOUT_MS = 30_000 +const MAX_PROVIDER_RESPONSE_BYTES = 16 * 1024 * 1024 + +async function cancelResponseBody(response: Response): Promise { + try { + await response.body?.cancel() + } catch {} +} + +async function readBoundedBody(response: Response): Promise { + const declaredLength = Number(response.headers.get('content-length')) + if (Number.isFinite(declaredLength) && declaredLength > MAX_PROVIDER_RESPONSE_BYTES) { + await cancelResponseBody(response) + throw new SelectorOptionsUnavailableError() + } + if (!response.body) return '' + + const reader = response.body.getReader() + const chunks: Uint8Array[] = [] + let total = 0 + while (true) { + const { done, value } = await reader.read() + if (done) break + total += value.byteLength + if (total > MAX_PROVIDER_RESPONSE_BYTES) { + await reader.cancel().catch(() => undefined) + throw new SelectorOptionsUnavailableError() + } + chunks.push(value) + } + + const body = new Uint8Array(total) + let offset = 0 + for (const chunk of chunks) { + body.set(chunk, offset) + offset += chunk.byteLength + } + return new TextDecoder().decode(body) +} + +export type ProviderJsonStatusResult = + | { ok: true; status: number; data: T } + | { ok: false; status: number; retryAfterMs?: number } + +export interface FetchProviderJsonStatusOptions { + passthroughStatuses?: readonly number[] + passthroughStatus?: (status: number) => boolean + /** Preserve a generic, retryable network signal without exposing the raw fetch error. */ + passthroughNetworkErrors?: boolean +} + +export class RetryableProviderNetworkError extends Error { + constructor() { + super('Provider network unavailable') + this.name = 'RetryableProviderNetworkError' + } +} + +export function selectorProviderStatusError(status: number): Error { + if (status === 401) return new SelectorConnectionUnavailableError(401) + if (status === 403) return new SelectorConnectionUnavailableError(403) + if (status === 429) return new SelectorOptionsUnavailableError(429) + return new SelectorOptionsUnavailableError(502) +} + +/** + * Fetches bounded provider JSON while allowing callers to branch on explicitly + * allowlisted non-success statuses. Passthrough response bodies are discarded; + * provider payloads never escape this boundary on an error status. + */ +export async function fetchProviderJsonWithStatus( + input: RequestInfo | URL, + init?: RequestInit, + options: FetchProviderJsonStatusOptions = {} +): Promise> { + let response: Response + const timeoutSignal = AbortSignal.timeout(PROVIDER_TIMEOUT_MS) + const signal = init?.signal ? AbortSignal.any([init.signal, timeoutSignal]) : timeoutSignal + try { + response = await fetch(input, { ...init, signal, redirect: init?.redirect ?? 'error' }) + } catch (error) { + if (init?.signal?.aborted) throw error + if (options.passthroughNetworkErrors) throw new RetryableProviderNetworkError() + throw new SelectorOptionsUnavailableError() + } + + if (!response.ok) { + const retryAfterMs = parseRetryAfter(response.headers.get('retry-after')) + await cancelResponseBody(response) + if ( + options.passthroughStatuses?.includes(response.status) || + options.passthroughStatus?.(response.status) + ) { + return { + ok: false, + status: response.status, + ...(retryAfterMs !== null && retryAfterMs > 0 ? { retryAfterMs } : {}), + } + } + throw selectorProviderStatusError(response.status) + } + try { + return { + ok: true, + status: response.status, + data: JSON.parse(await readBoundedBody(response)) as T, + } + } catch (error) { + if (init?.signal?.aborted) throw error + if (error instanceof SelectorOptionsUnavailableError) throw error + throw new SelectorOptionsUnavailableError() + } +} + +export async function fetchProviderJson( + input: RequestInfo | URL, + init?: RequestInit +): Promise { + const result = await fetchProviderJsonWithStatus(input, init) + if (!result.ok) throw new SelectorOptionsUnavailableError() + return result.data +} diff --git a/apps/sim/lib/selectors/server/providers/sharepoint.test.ts b/apps/sim/lib/selectors/server/providers/sharepoint.test.ts new file mode 100644 index 00000000000..72327b52785 --- /dev/null +++ b/apps/sim/lib/selectors/server/providers/sharepoint.test.ts @@ -0,0 +1,172 @@ +/** + * @vitest-environment node + */ +import { afterAll, beforeEach, describe, expect, it, vi } from 'vitest' + +const { mockFetch, mockResolveSelectorOAuthAccessToken } = vi.hoisted(() => ({ + mockFetch: vi.fn(), + mockResolveSelectorOAuthAccessToken: vi.fn(), +})) + +vi.mock('@/lib/selectors/server/credentials', () => ({ + resolveSelectorOAuthAccessToken: mockResolveSelectorOAuthAccessToken, +})) + +import { createSelectorProtectedValues } from '@/lib/selectors/server/protected-values' +import { sharepointSelectorAttachments } from '@/lib/selectors/server/providers/sharepoint' +import type { ExecuteServerSelectorArgs } from '@/lib/selectors/server/types' + +function detailArgs( + selectorKey: 'sharepoint.lists' | 'sharepoint.sites', + id: string +): ExecuteServerSelectorArgs { + return { + selectorKey, + context: { + oauthCredential: 'credential-1', + ...(selectorKey === 'sharepoint.lists' ? { siteId: 'contoso.sharepoint.com,site,web' } : {}), + }, + request: { kind: 'detail', id }, + scope: { kind: 'workspace', workspaceId: 'workspace-1' }, + workspaceId: 'workspace-1', + principal: { kind: 'session', userId: 'user-1', sessionId: 'session-1' }, + requesterUserId: 'user-1', + credential: { suppliedId: 'credential-1' }, + references: new Map(), + protectedValues: createSelectorProtectedValues(), + } +} + +function listArgs( + selectorKey: 'sharepoint.lists' | 'sharepoint.sites', + cursor?: string, + search?: string +): ExecuteServerSelectorArgs { + return { + ...detailArgs(selectorKey, ''), + request: { + kind: 'list', + ...(cursor ? { cursor } : {}), + ...(search ? { search } : {}), + }, + } +} + +describe('SharePoint server selector adapter', () => { + beforeEach(() => { + vi.clearAllMocks() + vi.stubGlobal('fetch', mockFetch) + mockResolveSelectorOAuthAccessToken.mockResolvedValue('server-only-token') + }) + + afterAll(() => vi.unstubAllGlobals()) + + it.each([ + { + selectorKey: 'sharepoint.sites' as const, + search: ' Engineering ', + firstValue: { id: 'site-1', name: 'Engineering' }, + firstItem: { id: 'site-1', label: 'Engineering' }, + secondValue: { id: 'site-2', name: 'Operations' }, + secondItem: { id: 'site-2', label: 'Operations' }, + nextCursor: 'https://graph.microsoft.com/v1.0/sites?search=Engineering&$skiptoken=next', + }, + { + selectorKey: 'sharepoint.lists' as const, + search: undefined, + firstValue: { id: 'list-1', displayName: 'Planning', list: { hidden: false } }, + firstItem: { id: 'list-1', label: 'Planning' }, + secondValue: { id: 'list-2', displayName: 'Operations', list: { hidden: false } }, + secondItem: { id: 'list-2', label: 'Operations' }, + nextCursor: + 'https://graph.microsoft.com/v1.0/sites/contoso.sharepoint.com%2Csite%2Cweb/lists?$skiptoken=next', + }, + ])('paginates $selectorKey only when its cursor is requested', async (testCase) => { + mockFetch + .mockResolvedValueOnce( + new Response( + JSON.stringify({ value: [testCase.firstValue], '@odata.nextLink': testCase.nextCursor }), + { status: 200 } + ) + ) + .mockResolvedValueOnce( + new Response(JSON.stringify({ value: [testCase.secondValue] }), { status: 200 }) + ) + + const first = await sharepointSelectorAttachments[testCase.selectorKey].execute( + listArgs(testCase.selectorKey, undefined, testCase.search) + ) + + expect(first).toEqual({ + kind: 'list', + items: [testCase.firstItem], + nextCursor: testCase.nextCursor, + }) + expect(mockFetch).toHaveBeenCalledTimes(1) + if (testCase.search) { + expect(new URL(String(mockFetch.mock.calls[0]?.[0])).searchParams.get('search')).toBe( + 'Engineering' + ) + } + + const second = await sharepointSelectorAttachments[testCase.selectorKey].execute( + listArgs(testCase.selectorKey, testCase.nextCursor, testCase.search) + ) + + expect(second).toEqual({ kind: 'list', items: [testCase.secondItem] }) + expect(String(mockFetch.mock.calls[1]?.[0])).toBe(testCase.nextCursor) + expect(mockFetch).toHaveBeenCalledTimes(2) + }) + + it('rejects a Graph cursor for another SharePoint resource', async () => { + await expect( + sharepointSelectorAttachments['sharepoint.lists'].execute( + listArgs( + 'sharepoint.lists', + 'https://graph.microsoft.com/v1.0/sites/another-site/lists?$skiptoken=next' + ) + ) + ).rejects.toMatchObject({ name: 'SelectorContextUnavailableError' }) + expect(mockResolveSelectorOAuthAccessToken).not.toHaveBeenCalled() + expect(mockFetch).not.toHaveBeenCalled() + }) + + it('hydrates a selected list directly within its site', async () => { + mockFetch.mockResolvedValueOnce( + new Response(JSON.stringify({ id: 'list-1', displayName: 'Planning' }), { status: 200 }) + ) + + await expect( + sharepointSelectorAttachments['sharepoint.lists'].execute( + detailArgs('sharepoint.lists', 'list-1') + ) + ).resolves.toEqual({ + kind: 'detail', + item: { id: 'list-1', label: 'Planning' }, + }) + expect(String(mockFetch.mock.calls[0]?.[0])).toContain( + '/sites/contoso.sharepoint.com%2Csite%2Cweb/lists/list-1' + ) + expect(mockFetch).toHaveBeenCalledTimes(1) + }) + + it('hydrates a selected site directly by its compound ID', async () => { + const siteId = 'contoso.sharepoint.com,site,web' + mockFetch.mockResolvedValueOnce( + new Response(JSON.stringify({ id: siteId, displayName: 'Engineering' }), { status: 200 }) + ) + + await expect( + sharepointSelectorAttachments['sharepoint.sites'].execute( + detailArgs('sharepoint.sites', siteId) + ) + ).resolves.toEqual({ + kind: 'detail', + item: { id: siteId, label: 'Engineering' }, + }) + expect(String(mockFetch.mock.calls[0]?.[0])).toContain( + '/sites/contoso.sharepoint.com%2Csite%2Cweb' + ) + expect(mockFetch).toHaveBeenCalledTimes(1) + }) +}) diff --git a/apps/sim/lib/selectors/server/providers/sharepoint.ts b/apps/sim/lib/selectors/server/providers/sharepoint.ts new file mode 100644 index 00000000000..0c289d7a3d0 --- /dev/null +++ b/apps/sim/lib/selectors/server/providers/sharepoint.ts @@ -0,0 +1,207 @@ +import { + validateMicrosoftGraphId, + validateSharePointSiteId, +} from '@/lib/core/security/input-validation' +import type { ServerSelectorKey } from '@/lib/selectors/manifest' +import { resolveSelectorOAuthAccessToken } from '@/lib/selectors/server/credentials' +import { + SelectorConnectionUnavailableError, + SelectorContextUnavailableError, + SelectorOptionsUnavailableError, +} from '@/lib/selectors/server/errors' +import { fetchProviderJson } from '@/lib/selectors/server/providers/provider-http' +import { + detailSelectorResult, + type ExecuteServerSelectorArgs, + listSelectorResult, + requireListRequest, + type ServerSelectorAttachmentMap, +} from '@/lib/selectors/server/types' +import type { SafeSelectorOption } from '@/lib/selectors/types' +import { assertGraphNextPageUrl, getGraphNextPageUrl } from '@/tools/sharepoint/utils' + +type SharePointSelectorKey = Extract + +const sharepointCredential = { + kind: 'stored', + field: 'oauthCredential', + serviceIds: ['sharepoint'], +} as const + +const siteCredential = { + kind: 'stored', + field: 'oauthCredential', + serviceIds: ['sharepoint', 'microsoft-excel'], + resourceServiceId: 'sharepoint', +} as const + +async function graphToken(args: ExecuteServerSelectorArgs): Promise { + if (!args.credential) throw new SelectorConnectionUnavailableError() + return resolveSelectorOAuthAccessToken({ + credential: args.credential, + serviceId: 'sharepoint', + protectedValues: args.protectedValues, + }) +} + +interface GraphPage { + items: T[] + nextCursor?: string +} + +function graphPageUrl(cursor: string | undefined, initialUrl: string): string { + if (!cursor) return initialUrl + let cursorUrl: string + try { + cursorUrl = assertGraphNextPageUrl(cursor) + } catch { + throw new SelectorContextUnavailableError() + } + if (new URL(cursorUrl).pathname !== new URL(initialUrl).pathname) { + throw new SelectorContextUnavailableError() + } + return cursorUrl +} + +async function fetchGraphPage( + args: ExecuteServerSelectorArgs, + initialUrl: string +): Promise> { + const request = requireListRequest(args.selectorKey, args.request) + const requestUrl = graphPageUrl(request.cursor, initialUrl) + const token = await graphToken(args) + const data = await fetchProviderJson<{ value?: T[] } & Record>(requestUrl, { + headers: { Authorization: `Bearer ${token}` }, + signal: args.signal, + redirect: 'error', + }) + const nextLink = getGraphNextPageUrl(data) + const nextCursor = nextLink ? graphPageUrl(nextLink, initialUrl) : undefined + return { + items: Array.isArray(data.value) ? data.value : [], + ...(nextCursor ? { nextCursor } : {}), + } +} + +function requireSiteId(value: string | undefined): string { + const validation = validateSharePointSiteId(value) + if (!validation.isValid || !validation.sanitized) { + throw new SelectorContextUnavailableError() + } + return validation.sanitized +} + +function requireListId(value: string): string { + const trimmed = value.trim() + const validation = validateMicrosoftGraphId(trimmed, 'listId') + if (!validation.isValid || trimmed.length > 512) { + throw new SelectorContextUnavailableError() + } + return trimmed +} + +async function getGraphDetail(args: ExecuteServerSelectorArgs, url: string): Promise { + const token = await graphToken(args) + return fetchProviderJson(url, { + headers: { Authorization: `Bearer ${token}` }, + signal: args.signal, + redirect: 'error', + }) +} + +async function getList( + args: ExecuteServerSelectorArgs, + listId: string +): Promise { + const siteId = requireSiteId(args.context.siteId) + const requestedId = requireListId(listId) + const list = await getGraphDetail<{ id?: string; displayName?: string | null }>( + args, + `https://graph.microsoft.com/v1.0/sites/${encodeURIComponent(siteId)}/lists/${encodeURIComponent(requestedId)}?$select=id,displayName,list` + ) + const providerId = typeof list.id === 'string' ? list.id.trim() : '' + const displayName = typeof list.displayName === 'string' ? list.displayName.trim() : '' + const label = displayName || providerId + if (!providerId || !label) throw new SelectorOptionsUnavailableError() + return { id: requestedId, label } +} + +async function getSite( + args: ExecuteServerSelectorArgs, + siteId: string +): Promise { + const requestedId = requireSiteId(siteId) + const site = await getGraphDetail<{ + id?: string + name?: string | null + displayName?: string | null + }>( + args, + `https://graph.microsoft.com/v1.0/sites/${encodeURIComponent(requestedId)}?$select=id,name,displayName,webUrl` + ) + const providerId = typeof site.id === 'string' ? site.id.trim() : '' + const displayName = typeof site.displayName === 'string' ? site.displayName.trim() : '' + const name = typeof site.name === 'string' ? site.name.trim() : '' + const label = displayName || name || providerId + if (!providerId || !label) throw new SelectorOptionsUnavailableError() + return { id: requestedId, label } +} + +async function listLists(args: ExecuteServerSelectorArgs) { + const siteId = requireSiteId(args.context.siteId) + const page = await fetchGraphPage<{ + id: string + displayName: string + list?: { hidden?: boolean } + }>( + args, + `https://graph.microsoft.com/v1.0/sites/${encodeURIComponent(siteId)}/lists?$select=id,displayName,description,webUrl,list&$top=999` + ) + return { + items: page.items + .filter((list) => list.list?.hidden !== true) + .map((list) => ({ id: list.id, label: list.displayName })), + nextCursor: page.nextCursor, + } +} + +async function listSites(args: ExecuteServerSelectorArgs) { + const request = requireListRequest(args.selectorKey, args.request) + const url = new URL('https://graph.microsoft.com/v1.0/sites') + url.searchParams.set('search', request.search?.trim() || '*') + url.searchParams.set('$select', 'id,name,displayName,webUrl,createdDateTime,lastModifiedDateTime') + url.searchParams.set('$top', '999') + const page = await fetchGraphPage<{ id: string; name: string; displayName?: string }>( + args, + url.toString() + ) + return { + items: page.items.map((site) => ({ id: site.id, label: site.displayName || site.name })), + nextCursor: page.nextCursor, + } +} + +export const sharepointSelectorAttachments = { + 'sharepoint.lists': { + credential: sharepointCredential, + destination: 'fixed', + execute: async (args) => { + if (args.request.kind === 'detail') { + return detailSelectorResult(await getList(args, args.request.id)) + } + const page = await listLists(args) + return listSelectorResult(page.items, page.nextCursor) + }, + }, + 'sharepoint.sites': { + credential: siteCredential, + destination: 'fixed', + execute: async (args) => { + if (args.request.kind === 'detail') { + return detailSelectorResult(await getSite(args, args.request.id)) + } + const page = await listSites(args) + return listSelectorResult(page.items, page.nextCursor) + }, + }, +} satisfies ServerSelectorAttachmentMap diff --git a/apps/sim/lib/selectors/server/providers/slack.test.ts b/apps/sim/lib/selectors/server/providers/slack.test.ts new file mode 100644 index 00000000000..a7b8a373f07 --- /dev/null +++ b/apps/sim/lib/selectors/server/providers/slack.test.ts @@ -0,0 +1,279 @@ +/** + * @vitest-environment node + */ +import { account } from '@sim/db/schema' +import { queueTableRows, resetDbChainMock } from '@sim/testing' +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const { mockFetchProviderJson, mockResolveSelectorOAuthAccessToken } = vi.hoisted(() => ({ + mockFetchProviderJson: vi.fn(), + mockResolveSelectorOAuthAccessToken: vi.fn(), +})) + +vi.mock('@/lib/selectors/server/providers/provider-http', () => ({ + fetchProviderJson: mockFetchProviderJson, +})) + +vi.mock('@/lib/selectors/server/credentials', () => ({ + resolveSelectorOAuthAccessToken: mockResolveSelectorOAuthAccessToken, +})) + +import { createSelectorProtectedValues } from '@/lib/selectors/server/protected-values' +import { slackSelectorAttachments } from '@/lib/selectors/server/providers/slack' +import type { ExecuteServerSelectorArgs } from '@/lib/selectors/server/types' + +const SCOPED_ACCOUNT_ID = 'slack-usr_U12345678-123e4567-e89b-12d3-a456-426614174000' + +function args( + selectorKey: 'slack.channels' | 'slack.users', + request: ExecuteServerSelectorArgs['request'] = { kind: 'list' }, + authentication: 'bot' | 'oauth' = 'bot', + signal?: AbortSignal +): ExecuteServerSelectorArgs { + return { + selectorKey, + context: { oauthCredential: 'credential-1' }, + request, + scope: { kind: 'workspace', workspaceId: 'workspace-1' }, + workspaceId: 'workspace-1', + principal: { kind: 'session', userId: 'user-1', sessionId: 'session-1' }, + requesterUserId: 'user-1', + credential: + authentication === 'bot' + ? { suppliedId: 'credential-1', fixedToken: 'xoxb-server-only-token' } + : { + suppliedId: 'credential-1', + access: { + ok: true, + credentialOwnerUserId: 'owner-1', + resolvedCredentialId: 'credential-1', + credentialType: 'oauth', + }, + }, + references: new Map(), + protectedValues: createSelectorProtectedValues(), + signal, + } +} + +function queueScopedAccount(): void { + queueTableRows(account, [{ accountId: SCOPED_ACCOUNT_ID }]) +} + +function requestedUrl(call: number): URL { + return new URL(String(mockFetchProviderJson.mock.calls[call]?.[0])) +} + +function channel(id: string, name: string, isPrivate = false, isMember?: boolean) { + return { id, name, is_private: isPrivate, ...(isMember ? { is_member: true } : {}) } +} + +function user(id: string, name: string, realName?: string) { + return { id, name, ...(realName ? { real_name: realName } : {}) } +} + +function slackPage>(body: T, nextCursor?: string) { + return { + ok: true, + ...body, + ...(nextCursor ? { response_metadata: { next_cursor: nextCursor } } : {}), + } +} + +function execute( + selectorKey: 'slack.channels' | 'slack.users', + request: ExecuteServerSelectorArgs['request'] = { kind: 'list' }, + authentication: 'bot' | 'oauth' = 'bot', + signal?: AbortSignal +) { + return slackSelectorAttachments[selectorKey].execute( + args(selectorKey, request, authentication, signal) + ) +} + +describe('Slack server selector adapters', () => { + beforeEach(() => { + vi.clearAllMocks() + resetDbChainMock() + mockResolveSelectorOAuthAccessToken.mockResolvedValue('xoxb-server-only-token') + }) + + it('does not fall back after channel listing is cancelled', async () => { + const controller = new AbortController() + const abortError = new DOMException('The operation was aborted', 'AbortError') + controller.abort(abortError) + mockFetchProviderJson.mockRejectedValue(abortError) + + await expect( + execute('slack.channels', { kind: 'list' }, 'bot', controller.signal) + ).rejects.toBe(abortError) + + expect(mockFetchProviderJson).toHaveBeenCalledOnce() + expect(mockFetchProviderJson.mock.calls[0]?.[0]).toBeInstanceOf(URL) + }) + + it('does not return a public-only fallback when membership lookup is cancelled', async () => { + const controller = new AbortController() + const abortError = new DOMException('The operation was aborted', 'AbortError') + queueScopedAccount() + mockFetchProviderJson + .mockResolvedValueOnce(slackPage({ channels: [channel('C123', 'general')] })) + .mockImplementationOnce(async () => { + controller.abort(abortError) + throw abortError + }) + + await expect( + execute('slack.channels', { kind: 'list' }, 'oauth', controller.signal) + ).rejects.toBe(abortError) + expect(mockFetchProviderJson).toHaveBeenCalledTimes(2) + }) + + it('continues a short users page only when its returned cursor is requested', async () => { + mockFetchProviderJson + .mockResolvedValueOnce( + slackPage({ members: [user('U001', 'first', 'First User')] }, 'users-page-2') + ) + .mockResolvedValueOnce(slackPage({ members: [user('U002', 'second', 'Second User')] })) + + const first = await execute('slack.users') + expect(first).toMatchObject({ + kind: 'list', + items: [{ id: 'U001', label: 'First User' }], + nextCursor: expect.any(String), + }) + if (first.kind !== 'list' || !first.nextCursor) throw new Error('Expected a users cursor') + + await expect( + execute('slack.users', { kind: 'list', cursor: first.nextCursor }) + ).resolves.toEqual({ + kind: 'list', + items: [{ id: 'U002', label: 'Second User' }], + }) + expect(requestedUrl(0).searchParams.has('cursor')).toBe(false) + expect(requestedUrl(1).searchParams.get('cursor')).toBe('users-page-2') + }) + + it('continues public and installing-user private channel streams independently', async () => { + queueScopedAccount() + queueScopedAccount() + mockFetchProviderJson + .mockResolvedValueOnce( + slackPage( + { + channels: [channel('C001', 'general'), channel('G001', 'bot-only', true, true)], + }, + 'conversations-page-2' + ) + ) + .mockResolvedValueOnce(slackPage({ channels: [] }, 'memberships-page-2')) + .mockResolvedValueOnce(slackPage({ channels: [channel('C002', 'announcements')] })) + .mockResolvedValueOnce( + slackPage({ channels: [channel('G002', 'installing-user-private', true)] }) + ) + + const first = await execute('slack.channels', { kind: 'list' }, 'oauth') + expect(first).toMatchObject({ + kind: 'list', + items: [{ id: 'C001', label: '#general' }], + nextCursor: expect.any(String), + }) + if (first.kind !== 'list' || !first.nextCursor) throw new Error('Expected a channels cursor') + + await expect( + execute('slack.channels', { kind: 'list', cursor: first.nextCursor }, 'oauth') + ).resolves.toEqual({ + kind: 'list', + items: [ + { id: 'C002', label: '#announcements' }, + { id: 'G002', label: '#installing-user-private' }, + ], + }) + expect(requestedUrl(2).searchParams.get('cursor')).toBe('conversations-page-2') + expect(requestedUrl(3).searchParams.get('cursor')).toBe('memberships-page-2') + }) + + it('fails closed for private list and detail results when membership cannot be verified', async () => { + queueScopedAccount() + queueScopedAccount() + mockFetchProviderJson + .mockResolvedValueOnce( + slackPage({ + channels: [channel('C001', 'general'), channel('G001', 'private', true, true)], + }) + ) + .mockRejectedValueOnce(new Error('membership lookup failed')) + .mockResolvedValueOnce(slackPage({ channel: channel('G001', 'private', true, true) })) + .mockRejectedValueOnce(new Error('member list failed')) + + await expect(execute('slack.channels', { kind: 'list' }, 'oauth')).resolves.toEqual({ + kind: 'list', + items: [{ id: 'C001', label: '#general' }], + }) + await expect( + execute('slack.channels', { kind: 'detail', id: 'G001' }, 'oauth') + ).resolves.toEqual({ kind: 'detail', item: null }) + }) + + it('preserves bot-only fallback without allowing its cursor under scoped OAuth', async () => { + mockFetchProviderJson + .mockRejectedValueOnce(new Error('private scope unavailable')) + .mockResolvedValueOnce(slackPage({ channels: [channel('C001', 'general')] }, 'public-page-2')) + + const botResult = await execute('slack.channels') + expect(botResult).toMatchObject({ + kind: 'list', + items: [{ id: 'C001', label: '#general' }], + nextCursor: expect.any(String), + }) + expect(requestedUrl(0).searchParams.get('types')).toBe('public_channel,private_channel') + expect(requestedUrl(1).searchParams.get('types')).toBe('public_channel') + if (botResult.kind !== 'list' || !botResult.nextCursor) { + throw new Error('Expected a public-only bot cursor') + } + + queueTableRows(account, []) + mockFetchProviderJson.mockRejectedValueOnce(new Error('OAuth list failed')) + await expect(execute('slack.channels', { kind: 'list' }, 'oauth')).rejects.toMatchObject({ + name: 'SelectorOptionsUnavailableError', + }) + + queueScopedAccount() + await expect( + execute('slack.channels', { kind: 'list', cursor: botResult.nextCursor }, 'oauth') + ).rejects.toMatchObject({ name: 'SelectorContextUnavailableError' }) + expect(mockFetchProviderJson).toHaveBeenCalledTimes(3) + }) + + it('hydrates saved users and installing-user private channels directly by id', async () => { + mockFetchProviderJson.mockResolvedValueOnce( + slackPage({ user: user('U999', 'saved', 'Saved User') }) + ) + await expect(execute('slack.users', { kind: 'detail', id: 'U999' })).resolves.toEqual({ + kind: 'detail', + item: { id: 'U999', label: 'Saved User' }, + }) + + queueScopedAccount() + mockFetchProviderJson + .mockResolvedValueOnce(slackPage({ channel: channel('G999', 'saved-private', true, true) })) + .mockResolvedValueOnce(slackPage({ members: ['UOTHER'] }, 'members-page-2')) + .mockResolvedValueOnce(slackPage({ members: ['U12345678'] })) + await expect( + execute('slack.channels', { kind: 'detail', id: 'G999' }, 'oauth') + ).resolves.toEqual({ + kind: 'detail', + item: { id: 'G999', label: '#saved-private' }, + }) + + expect( + mockFetchProviderJson.mock.calls.map((_, index) => requestedUrl(index).pathname) + ).toEqual([ + '/api/users.info', + '/api/conversations.info', + '/api/conversations.members', + '/api/conversations.members', + ]) + expect(requestedUrl(3).searchParams.get('cursor')).toBe('members-page-2') + }) +}) diff --git a/apps/sim/lib/selectors/server/providers/slack.ts b/apps/sim/lib/selectors/server/providers/slack.ts new file mode 100644 index 00000000000..93f5400b315 --- /dev/null +++ b/apps/sim/lib/selectors/server/providers/slack.ts @@ -0,0 +1,547 @@ +import { db } from '@sim/db' +import { account } from '@sim/db/schema' +import { eq } from 'drizzle-orm' +import { validateAlphanumericId } from '@/lib/core/security/input-validation' +import { MAX_SELECTOR_OPTIONS, MAX_SELECTOR_PAGES } from '@/lib/selectors/limits' +import type { ServerSelectorKey } from '@/lib/selectors/manifest' +import { resolveSelectorOAuthAccessToken } from '@/lib/selectors/server/credentials' +import { + SelectorConnectionUnavailableError, + SelectorContextUnavailableError, + SelectorOptionsUnavailableError, +} from '@/lib/selectors/server/errors' +import { fetchProviderJson } from '@/lib/selectors/server/providers/provider-http' +import { + detailSelectorResult, + type ExecuteServerSelectorArgs, + listSelectorResult, + requireListRequest, + type ServerSelectorAttachmentMap, +} from '@/lib/selectors/server/types' + +type SlackSelectorKey = Extract +type SlackMethod = + | 'conversations.info' + | 'conversations.list' + | 'conversations.members' + | 'users.conversations' + | 'users.info' + | 'users.list' +type SlackCursorMode = 'users' | 'scoped' | 'oauth' | 'bot-all' | 'bot-public' + +const SLACK_PAGE_LIMIT = 200 +const SLACK_CURSOR_VERSION = '1' +const MAX_SLACK_SELECTOR_CURSOR_LENGTH = 16 * 1024 +const SCOPED_USER_ID_PATTERN = + /-usr_([UW][A-Z0-9]+)-[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i + +interface SlackApiResponse { + ok?: boolean + error?: string + channel?: SlackChannel + channels?: SlackChannel[] + user?: SlackUser + members?: Array + response_metadata?: { next_cursor?: string } +} + +interface SlackChannel { + id?: string + name?: string + is_private?: boolean + is_archived?: boolean + is_member?: boolean +} + +interface SlackUser { + id?: string + name?: string + real_name?: string + deleted?: boolean + is_bot?: boolean +} + +interface SlackChannelPage { + channels: SlackChannel[] + nextCursor?: string +} + +type SlackCursorState = + | { mode: 'users'; cursor: string } + | { mode: 'scoped'; conversations?: string; memberships?: string } + | { mode: 'oauth' | 'bot-all' | 'bot-public'; conversations: string } + +interface SlackChannelAuthentication { + accessToken: string + isBotCredential: boolean + scopedUserId: string | null +} + +function parseScopedSlackUserId(accountId: string): string | null { + return SCOPED_USER_ID_PATTERN.exec(accountId)?.[1] ?? null +} + +async function readScopedSlackUserId(args: ExecuteServerSelectorArgs): Promise { + const access = args.credential?.access + if (access?.credentialType !== 'oauth' || !access.resolvedCredentialId) return null + const [row] = await db + .select({ accountId: account.accountId }) + .from(account) + .where(eq(account.id, access.resolvedCredentialId)) + .limit(1) + return row ? parseScopedSlackUserId(row.accountId) : null +} + +async function fetchSlackApi( + args: ExecuteServerSelectorArgs, + method: SlackMethod, + accessToken: string, + params: Record, + acceptedErrors: readonly string[] = [] +): Promise { + const url = new URL(`https://slack.com/api/${method}`) + for (const [key, value] of Object.entries(params)) url.searchParams.set(key, value) + + let data: SlackApiResponse + try { + data = await fetchProviderJson(url, { + method: 'GET', + headers: { + Authorization: `Bearer ${accessToken}`, + 'Content-Type': 'application/json', + }, + signal: args.signal, + redirect: 'error', + }) + } catch (error) { + if (args.signal?.aborted) throw error + if ( + error instanceof SelectorConnectionUnavailableError || + error instanceof SelectorOptionsUnavailableError + ) { + throw error + } + throw new SelectorOptionsUnavailableError() + } + if (!data.ok && !acceptedErrors.includes(data.error ?? '')) { + throw new SelectorOptionsUnavailableError() + } + return data +} + +function readProviderCursor(data: SlackApiResponse): string | undefined { + const cursor = data.response_metadata?.next_cursor?.trim() || undefined + if (cursor && cursor.length > MAX_SLACK_SELECTOR_CURSOR_LENGTH) { + throw new SelectorOptionsUnavailableError() + } + return cursor +} + +function readCursorParam(params: URLSearchParams, key: string): string | undefined { + const values = params.getAll(key) + if (values.length === 0) return undefined + if (values.length !== 1) throw new SelectorContextUnavailableError() + const value = values[0]?.trim() + if (!value || value.length > MAX_SLACK_SELECTOR_CURSOR_LENGTH) { + throw new SelectorContextUnavailableError() + } + return value +} + +function parseSlackCursor(cursor: string | undefined): SlackCursorState | undefined { + if (!cursor) return undefined + if (cursor.length > MAX_SLACK_SELECTOR_CURSOR_LENGTH) { + throw new SelectorContextUnavailableError() + } + + const params = new URLSearchParams(cursor) + const allowedKeys = new Set(['v', 'mode', 'cursor', 'conversations', 'memberships']) + if ([...params.keys()].some((key) => !allowedKeys.has(key))) { + throw new SelectorContextUnavailableError() + } + const version = readCursorParam(params, 'v') + const mode = readCursorParam(params, 'mode') as SlackCursorMode | undefined + if (version !== SLACK_CURSOR_VERSION) throw new SelectorContextUnavailableError() + + if (mode === 'users') { + if (params.has('conversations') || params.has('memberships')) { + throw new SelectorContextUnavailableError() + } + const userCursor = readCursorParam(params, 'cursor') + if (!userCursor) throw new SelectorContextUnavailableError() + return { mode, cursor: userCursor } + } + + if (params.has('cursor')) throw new SelectorContextUnavailableError() + const conversations = readCursorParam(params, 'conversations') + const memberships = readCursorParam(params, 'memberships') + if (mode === 'scoped') { + if (!conversations && !memberships) throw new SelectorContextUnavailableError() + return { + mode, + ...(conversations ? { conversations } : {}), + ...(memberships ? { memberships } : {}), + } + } + if (mode !== 'oauth' && mode !== 'bot-all' && mode !== 'bot-public') { + throw new SelectorContextUnavailableError() + } + if (!conversations || memberships) throw new SelectorContextUnavailableError() + return { mode, conversations } +} + +function encodeSlackCursor(state: SlackCursorState): string { + const params = new URLSearchParams({ v: SLACK_CURSOR_VERSION, mode: state.mode }) + if (state.mode === 'users') { + params.set('cursor', state.cursor) + } else { + if (state.conversations) params.set('conversations', state.conversations) + if (state.mode === 'scoped' && state.memberships) { + params.set('memberships', state.memberships) + } + } + const cursor = params.toString() + if (cursor.length > MAX_SLACK_SELECTOR_CURSOR_LENGTH) { + throw new SelectorOptionsUnavailableError() + } + return cursor +} + +function channelOption(channel: SlackChannel): { id: string; label: string } | null { + if (!channel.id || !channel.name || channel.is_archived) return null + const validation = validateAlphanumericId(channel.id, 'channelId', 50) + if (!validation.isValid || !/^[CDG][A-Z0-9]+$/i.test(channel.id)) return null + return { id: channel.id, label: `#${channel.name}` } +} + +function userOption(user: SlackUser): { id: string; label: string } | null { + if (!user.id || !user.name || user.deleted || user.is_bot) return null + const validation = validateAlphanumericId(user.id, 'userId', 50) + if (!validation.isValid || !/^[UW][A-Z0-9]+$/i.test(user.id)) return null + return { id: user.id, label: user.real_name || user.name } +} + +function uniqueOptions( + items: Array<{ id: string; label: string }> +): Array<{ id: string; label: string }> { + return [...new Map(items.map((item) => [item.id, item])).values()] +} + +async function fetchChannelPage( + args: ExecuteServerSelectorArgs, + method: 'conversations.list' | 'users.conversations', + accessToken: string, + params: Record, + cursor?: string +): Promise { + const data = await fetchSlackApi(args, method, accessToken, { + ...params, + limit: String(SLACK_PAGE_LIMIT), + ...(cursor ? { cursor } : {}), + }) + return { + channels: Array.isArray(data.channels) ? data.channels : [], + nextCursor: readProviderCursor(data), + } +} + +async function resolveChannelAuthentication( + args: ExecuteServerSelectorArgs +): Promise { + if (!args.credential) throw new SelectorConnectionUnavailableError() + const accessToken = await resolveSelectorOAuthAccessToken({ + credential: args.credential, + serviceId: 'slack', + protectedValues: args.protectedValues, + }) + const isBotCredential = + Boolean(args.credential.fixedToken) || args.credential.access?.credentialType !== 'oauth' + return { + accessToken, + isBotCredential, + scopedUserId: await readScopedSlackUserId(args), + } +} + +function assertChannelCursorMode( + cursor: SlackCursorState | undefined, + authentication: SlackChannelAuthentication +): void { + if (!cursor) return + if (cursor.mode === 'users') throw new SelectorContextUnavailableError() + if (authentication.scopedUserId) { + if (cursor.mode !== 'scoped') throw new SelectorContextUnavailableError() + return + } + if (authentication.isBotCredential) { + if (cursor.mode !== 'bot-all' && cursor.mode !== 'bot-public') { + throw new SelectorContextUnavailableError() + } + return + } + if (cursor.mode !== 'oauth') throw new SelectorContextUnavailableError() +} + +async function listScopedSlackChannels( + args: ExecuteServerSelectorArgs, + authentication: SlackChannelAuthentication, + cursor: Extract | undefined +) { + const publicPage = + !cursor || cursor.conversations + ? await fetchChannelPage( + args, + 'conversations.list', + authentication.accessToken, + { + types: 'public_channel,private_channel', + exclude_archived: 'true', + }, + cursor?.conversations + ) + : undefined + + let privatePage: SlackChannelPage | undefined + if (!cursor || cursor.memberships) { + try { + privatePage = await fetchChannelPage( + args, + 'users.conversations', + authentication.accessToken, + { + user: authentication.scopedUserId!, + types: 'private_channel', + exclude_archived: 'true', + }, + cursor?.memberships + ) + } catch (error) { + if (args.signal?.aborted) throw error + privatePage = undefined + } + } + + const items = uniqueOptions([ + ...(publicPage?.channels ?? []).flatMap((channel) => { + if (channel.is_private !== false) return [] + const option = channelOption(channel) + return option ? [option] : [] + }), + ...(privatePage?.channels ?? []).flatMap((channel) => { + const option = channelOption(channel) + return option ? [option] : [] + }), + ]) + const conversations = publicPage?.nextCursor + const memberships = privatePage?.nextCursor + return listSelectorResult( + items, + conversations || memberships + ? encodeSlackCursor({ + mode: 'scoped', + ...(conversations ? { conversations } : {}), + ...(memberships ? { memberships } : {}), + }) + : undefined + ) +} + +async function listUnscopedSlackChannels( + args: ExecuteServerSelectorArgs, + authentication: SlackChannelAuthentication, + cursor: Extract | undefined +) { + let mode: 'oauth' | 'bot-all' | 'bot-public' = authentication.isBotCredential + ? cursor?.mode === 'bot-public' + ? 'bot-public' + : 'bot-all' + : 'oauth' + let page: SlackChannelPage + try { + page = await fetchChannelPage( + args, + 'conversations.list', + authentication.accessToken, + { + types: mode === 'bot-public' ? 'public_channel' : 'public_channel,private_channel', + exclude_archived: 'true', + }, + cursor?.conversations + ) + } catch (error) { + if (args.signal?.aborted) throw error + if (!authentication.isBotCredential || mode === 'bot-public') throw error + mode = 'bot-public' + page = await fetchChannelPage(args, 'conversations.list', authentication.accessToken, { + types: 'public_channel', + exclude_archived: 'true', + }) + } + + const items = page.channels.flatMap((channel) => { + if (channel.is_private && !channel.is_member) return [] + const option = channelOption(channel) + return option ? [option] : [] + }) + return listSelectorResult( + items, + page.nextCursor ? encodeSlackCursor({ mode, conversations: page.nextCursor }) : undefined + ) +} + +async function installingUserIsChannelMember( + args: ExecuteServerSelectorArgs, + accessToken: string, + channelId: string, + scopedUserId: string +): Promise { + let cursor: string | undefined + let examinedMembers = 0 + const seenCursors = new Set() + for (let page = 0; page < MAX_SELECTOR_PAGES; page++) { + const data = await fetchSlackApi(args, 'conversations.members', accessToken, { + channel: channelId, + limit: String(SLACK_PAGE_LIMIT), + ...(cursor ? { cursor } : {}), + }) + const remaining = MAX_SELECTOR_OPTIONS - examinedMembers + const members = Array.isArray(data.members) ? data.members.slice(0, remaining) : [] + if (members.some((member) => member === scopedUserId)) return true + examinedMembers += members.length + if (examinedMembers >= MAX_SELECTOR_OPTIONS) return false + + cursor = readProviderCursor(data) + if (!cursor || seenCursors.has(cursor)) return false + seenCursors.add(cursor) + } + return false +} + +async function hydrateSlackChannel( + args: ExecuteServerSelectorArgs, + authentication: SlackChannelAuthentication, + rawChannelId: string +) { + const channelId = rawChannelId.trim() + const validation = validateAlphanumericId(channelId, 'channelId', 50) + if (!validation.isValid || !/^[CDG][A-Z0-9]+$/i.test(channelId)) { + return detailSelectorResult(null) + } + const data = await fetchSlackApi( + args, + 'conversations.info', + authentication.accessToken, + { channel: channelId }, + ['channel_not_found'] + ) + if (!data.ok || data.channel?.id !== channelId || typeof data.channel.is_private !== 'boolean') { + return detailSelectorResult(null) + } + const option = channelOption(data.channel) + if (!option) return detailSelectorResult(null) + if (data.channel.is_private) { + if (authentication.scopedUserId) { + try { + if ( + !(await installingUserIsChannelMember( + args, + authentication.accessToken, + channelId, + authentication.scopedUserId + )) + ) { + return detailSelectorResult(null) + } + } catch (error) { + if (args.signal?.aborted) throw error + return detailSelectorResult(null) + } + } else if (!data.channel.is_member) { + return detailSelectorResult(null) + } + } + return detailSelectorResult(option) +} + +async function executeSlackChannels(args: ExecuteServerSelectorArgs) { + const cursor = args.request.kind === 'list' ? parseSlackCursor(args.request.cursor) : undefined + const authentication = await resolveChannelAuthentication(args) + if (args.request.kind === 'detail') { + return hydrateSlackChannel(args, authentication, args.request.id) + } + requireListRequest(args.selectorKey, args.request) + assertChannelCursorMode(cursor, authentication) + if (authentication.scopedUserId) { + return listScopedSlackChannels( + args, + authentication, + cursor as Extract | undefined + ) + } + return listUnscopedSlackChannels( + args, + authentication, + cursor as Extract | undefined + ) +} + +async function executeSlackUsers(args: ExecuteServerSelectorArgs) { + const request = + args.request.kind === 'list' ? requireListRequest(args.selectorKey, args.request) : null + const cursor = request ? parseSlackCursor(request.cursor) : undefined + if (cursor && cursor.mode !== 'users') throw new SelectorContextUnavailableError() + if (!args.credential) throw new SelectorConnectionUnavailableError() + const accessToken = await resolveSelectorOAuthAccessToken({ + credential: args.credential, + serviceId: 'slack', + protectedValues: args.protectedValues, + }) + if (args.request.kind === 'detail') { + const userId = args.request.id.trim() + const validation = validateAlphanumericId(userId, 'userId', 50) + if (!validation.isValid || !/^[UW][A-Z0-9]+$/i.test(userId)) { + return detailSelectorResult(null) + } + const data = await fetchSlackApi(args, 'users.info', accessToken, { user: userId }, [ + 'user_not_found', + ]) + if (!data.ok || data.user?.id !== userId) return detailSelectorResult(null) + return detailSelectorResult(userOption(data.user)) + } + + const data = await fetchSlackApi(args, 'users.list', accessToken, { + limit: String(SLACK_PAGE_LIMIT), + ...(cursor?.mode === 'users' ? { cursor: cursor.cursor } : {}), + }) + const users = (data.members ?? []).filter( + (member): member is SlackUser => typeof member === 'object' && member !== null + ) + const nextCursor = readProviderCursor(data) + return listSelectorResult( + users.flatMap((user) => { + const option = userOption(user) + return option ? [option] : [] + }), + nextCursor ? encodeSlackCursor({ mode: 'users', cursor: nextCursor }) : undefined + ) +} + +const credential = { + kind: 'stored-or-fixed-token', + field: 'oauthCredential', + serviceIds: ['slack'], + tokenPrefixes: ['xoxb-'], +} as const + +export const slackSelectorAttachments = { + 'slack.channels': { + credential, + destination: 'fixed', + execute: executeSlackChannels, + }, + 'slack.users': { + credential, + destination: 'fixed', + execute: executeSlackUsers, + }, +} satisfies ServerSelectorAttachmentMap diff --git a/apps/sim/lib/selectors/server/providers/snowflake.test.ts b/apps/sim/lib/selectors/server/providers/snowflake.test.ts new file mode 100644 index 00000000000..b3d9fb08df3 --- /dev/null +++ b/apps/sim/lib/selectors/server/providers/snowflake.test.ts @@ -0,0 +1,217 @@ +/** + * @vitest-environment node + */ +import { afterAll, beforeEach, describe, expect, it, vi } from 'vitest' + +const { mockFetch, mockResolveCredentialBundle } = vi.hoisted(() => ({ + mockFetch: vi.fn(), + mockResolveCredentialBundle: vi.fn(), +})) + +vi.mock('@/lib/selectors/server/providers/credential-bundle', () => ({ + resolveSelectorCredentialBundle: mockResolveCredentialBundle, +})) + +import { createSelectorProtectedValues } from '@/lib/selectors/server/protected-values' +import { snowflakeSelectorAttachments } from '@/lib/selectors/server/providers/snowflake' +import type { ExecuteServerSelectorArgs } from '@/lib/selectors/server/types' + +const STATEMENT_HANDLE = '019c06a4-0000-df4f-0000-00100006589e' + +function jsonResponse(body: unknown, status = 200): Response { + return new Response(JSON.stringify(body), { + status, + headers: { 'Content-Type': 'application/json' }, + }) +} + +function tableArgs(signal?: AbortSignal): ExecuteServerSelectorArgs { + return { + selectorKey: 'snowflake.tables', + context: { + oauthCredential: 'credential-1', + database: 'ANALYTICS', + schema: 'PUBLIC', + }, + request: { kind: 'list' }, + scope: { kind: 'workspace', workspaceId: 'workspace-1' }, + workspaceId: 'workspace-1', + principal: { kind: 'session', userId: 'user-1', sessionId: 'session-1' }, + requesterUserId: 'user-1', + credential: { suppliedId: 'credential-1' }, + references: new Map(), + signal, + protectedValues: createSelectorProtectedValues(), + } +} + +describe('Snowflake server selector adapter', () => { + beforeEach(() => { + vi.clearAllMocks() + vi.stubGlobal('fetch', mockFetch) + mockResolveCredentialBundle.mockResolvedValue({ + accessToken: 'server-only-token', + domain: 'acme.snowflakecomputing.com', + }) + }) + + afterAll(() => vi.unstubAllGlobals()) + + it('returns every advertised result partition in order', async () => { + mockFetch + .mockResolvedValueOnce( + jsonResponse({ + statementHandle: STATEMENT_HANDLE, + data: [['ALPHA', 'first']], + resultSetMetaData: { + numRows: 3, + partitionInfo: [{ rowCount: 1 }, { rowCount: 2 }], + }, + }) + ) + .mockResolvedValueOnce( + jsonResponse({ + data: [ + ['BETA', null], + ['GAMMA', 'third'], + ], + }) + ) + + await expect( + snowflakeSelectorAttachments['snowflake.tables'].execute(tableArgs()) + ).resolves.toEqual({ + kind: 'list', + items: [ + { + id: 'ALPHA', + label: 'ALPHA — first', + meta: { name: 'ALPHA', detail: 'first' }, + }, + { id: 'BETA', label: 'BETA', meta: { name: 'BETA' } }, + { + id: 'GAMMA', + label: 'GAMMA — third', + meta: { name: 'GAMMA', detail: 'third' }, + }, + ], + }) + + expect(mockFetch).toHaveBeenCalledTimes(2) + expect(String(mockFetch.mock.calls[1]?.[0])).toBe( + `https://acme.snowflakecomputing.com/api/v2/statements/${STATEMENT_HANDLE}?partition=1` + ) + expect(mockFetch.mock.calls[1]?.[1]).toMatchObject({ method: 'GET', redirect: 'error' }) + }) + + it('rejects the whole selector when a later partition fails', async () => { + mockFetch + .mockResolvedValueOnce( + jsonResponse({ + statementHandle: STATEMENT_HANDLE, + data: [['ALPHA', null]], + resultSetMetaData: { + numRows: 2, + partitionInfo: [{ rowCount: 1 }, { rowCount: 1 }], + }, + }) + ) + .mockResolvedValueOnce(jsonResponse({ message: 'private provider payload' }, 500)) + + await expect( + snowflakeSelectorAttachments['snowflake.tables'].execute(tableArgs()) + ).rejects.toMatchObject({ + name: 'SelectorOptionsUnavailableError', + message: 'Options unavailable', + status: 502, + }) + expect(mockFetch).toHaveBeenCalledTimes(2) + }) + + it('preserves caller cancellation during a later partition', async () => { + const controller = new AbortController() + const abortError = new DOMException('The operation was aborted', 'AbortError') + let markLaterFetchStarted: (() => void) | undefined + const laterFetchStarted = new Promise((resolve) => { + markLaterFetchStarted = resolve + }) + mockFetch + .mockResolvedValueOnce( + jsonResponse({ + statementHandle: STATEMENT_HANDLE, + data: [['ALPHA', null]], + resultSetMetaData: { + numRows: 3, + partitionInfo: [{ rowCount: 1 }, { rowCount: 1 }, { rowCount: 1 }], + }, + }) + ) + .mockImplementationOnce((_input: RequestInfo | URL, init?: RequestInit) => { + markLaterFetchStarted?.() + return new Promise((_resolve, reject) => { + init?.signal?.addEventListener('abort', () => reject(init.signal?.reason), { once: true }) + }) + }) + + const execution = snowflakeSelectorAttachments['snowflake.tables'].execute( + tableArgs(controller.signal) + ) + await laterFetchStarted + controller.abort(abortError) + + await expect(execution).rejects.toBe(abortError) + expect(mockFetch).toHaveBeenCalledTimes(2) + }) + + it.each([ + { + name: 'missing partition metadata', + body: { + statementHandle: STATEMENT_HANDLE, + data: [['ALPHA', null]], + resultSetMetaData: { numRows: 1 }, + }, + }, + { + name: 'more than 16 partitions', + body: { + statementHandle: STATEMENT_HANDLE, + data: [['ALPHA', null]], + resultSetMetaData: { + numRows: 1, + partitionInfo: Array.from({ length: 17 }, () => ({ rowCount: 0 })), + }, + }, + }, + { + name: 'more than 1,000 rows', + body: { + statementHandle: STATEMENT_HANDLE, + data: [['ALPHA', null]], + resultSetMetaData: { numRows: 1_001, partitionInfo: [{ rowCount: 1 }] }, + }, + }, + { + name: 'an invalid handle for a partitioned result', + body: { + statementHandle: '../untrusted-handle', + data: [['ALPHA', null]], + resultSetMetaData: { + numRows: 2, + partitionInfo: [{ rowCount: 1 }, { rowCount: 1 }], + }, + }, + }, + ])('rejects $name before requesting more data', async ({ body }) => { + mockFetch.mockResolvedValueOnce(jsonResponse(body)) + + await expect( + snowflakeSelectorAttachments['snowflake.tables'].execute(tableArgs()) + ).rejects.toMatchObject({ + name: 'SelectorOptionsUnavailableError', + message: 'Options unavailable', + status: 502, + }) + expect(mockFetch).toHaveBeenCalledTimes(1) + }) +}) diff --git a/apps/sim/lib/selectors/server/providers/snowflake.ts b/apps/sim/lib/selectors/server/providers/snowflake.ts new file mode 100644 index 00000000000..3708f047c1e --- /dev/null +++ b/apps/sim/lib/selectors/server/providers/snowflake.ts @@ -0,0 +1,293 @@ +import { MAX_SELECTOR_OPTIONS } from '@/lib/selectors/limits' +import type { ServerSelectorKey } from '@/lib/selectors/manifest' +import { + SelectorConnectionUnavailableError, + SelectorOptionsUnavailableError, +} from '@/lib/selectors/server/errors' +import { resolveSelectorCredentialBundle } from '@/lib/selectors/server/providers/credential-bundle' +import { flatSelectorResult } from '@/lib/selectors/server/providers/flat-results' +import { selectorProviderStatusError } from '@/lib/selectors/server/providers/provider-http' +import type { + ExecuteServerSelectorArgs, + ServerSelectorAttachmentMap, +} from '@/lib/selectors/server/types' +import { definePreparedSelectorAttachment } from '@/lib/selectors/server/types' +import type { SafeSelectorOption } from '@/lib/selectors/types' +import type { SnowflakeSelectorKind } from '@/tools/snowflake/selector-kinds' +import { buildSelectorStatement } from '@/tools/snowflake/sql' +import { + buildSnowflakeAuthHeaders, + normalizeSnowflakeHost, + readSnowflakeResult, +} from '@/tools/snowflake/utils' + +type SnowflakeSelectorKey = Extract +type SnowflakeScopeLevel = 'account' | 'database' | 'schema' + +interface SnowflakeSelectorSpec { + kind: SnowflakeSelectorKind + scope: SnowflakeScopeLevel +} + +const SNOWFLAKE_SELECTOR_SPECS = { + 'snowflake.databases': { kind: 'databases', scope: 'account' }, + 'snowflake.warehouses': { kind: 'warehouses', scope: 'account' }, + 'snowflake.roles': { kind: 'roles', scope: 'account' }, + 'snowflake.schemas': { kind: 'schemas', scope: 'database' }, + 'snowflake.tables': { kind: 'tables', scope: 'schema' }, + 'snowflake.fileFormats': { kind: 'file_formats', scope: 'schema' }, + 'snowflake.procedures': { kind: 'procedures', scope: 'schema' }, +} as const satisfies Record + +const SELECTOR_ROW_LIMIT = 1_000 +const SELECTOR_TIMEOUT_SECONDS = 20 +const SELECTOR_FETCH_TIMEOUT_MS = (SELECTOR_TIMEOUT_SECONDS + 10) * 1_000 +const SELECTOR_MAX_PARTITIONS = 16 +const SELECTOR_MAX_AGGREGATE_RESPONSE_BYTES = 16 * 1024 * 1024 +const SNOWFLAKE_STATEMENT_HANDLE_PATTERN = + /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i + +interface SnowflakeObject { + name: string + detail: string | null +} + +interface SnowflakeDestination { + accessToken: string + baseUrl: string +} + +function requirePartitionCount(value: number | null): number { + if ( + typeof value !== 'number' || + !Number.isSafeInteger(value) || + value < 1 || + value > SELECTOR_MAX_PARTITIONS + ) { + throw new SelectorOptionsUnavailableError() + } + return value +} + +function requireTotalRows(value: number | null): number { + if ( + typeof value !== 'number' || + !Number.isSafeInteger(value) || + value < 0 || + value > SELECTOR_ROW_LIMIT + ) { + throw new SelectorOptionsUnavailableError() + } + return value +} + +function requireStatementHandle(value: string): string { + if (!SNOWFLAKE_STATEMENT_HANDLE_PATTERN.test(value)) { + throw new SelectorOptionsUnavailableError() + } + return value +} + +async function fetchSnowflakeResponse(url: string, init: RequestInit): Promise { + const response = await fetch(url, { ...init, redirect: 'error' }) + if (!response.ok) { + await response.body?.cancel().catch(() => undefined) + throw selectorProviderStatusError(response.status) + } + return response +} + +function parseAvailableRoles(cellValue: string | null | undefined): SnowflakeObject[] { + if (!cellValue) return [] + let parsed: unknown + try { + parsed = JSON.parse(cellValue) + } catch { + return [] + } + if (!Array.isArray(parsed)) return [] + return parsed + .filter((role): role is string => typeof role === 'string' && role.length > 0) + .sort((left, right) => left.localeCompare(right)) + .map((role) => ({ name: role, detail: null })) +} + +function toOption(object: SnowflakeObject): SafeSelectorOption { + return { + id: object.name, + label: object.detail ? `${object.name} — ${object.detail}` : object.name, + meta: { name: object.name, ...(object.detail ? { detail: object.detail } : {}) }, + } +} + +async function prepareSnowflakeDestination( + args: ExecuteServerSelectorArgs +): Promise { + if (!args.credential) throw new SelectorConnectionUnavailableError() + const token = await resolveSelectorCredentialBundle({ + credential: args.credential, + protectedValues: args.protectedValues, + }) + if (!token.domain) throw new SelectorConnectionUnavailableError() + try { + return { + accessToken: token.accessToken, + baseUrl: normalizeSnowflakeHost(token.domain), + } + } catch { + throw new SelectorConnectionUnavailableError() + } +} + +async function executeSnowflake( + args: ExecuteServerSelectorArgs, + destination: SnowflakeDestination +) { + const spec = SNOWFLAKE_SELECTOR_SPECS[args.selectorKey as SnowflakeSelectorKey] + if (!spec) throw new SelectorOptionsUnavailableError() + + let statement: string + try { + statement = buildSelectorStatement( + spec.kind, + { + ...(spec.scope !== 'account' ? { database: args.context.database } : {}), + ...(spec.scope === 'schema' ? { schema: args.context.schema } : {}), + }, + SELECTOR_ROW_LIMIT + ).statement + } catch { + throw new SelectorOptionsUnavailableError() + } + + const timeoutSignal = AbortSignal.timeout(SELECTOR_FETCH_TIMEOUT_MS) + const signal = args.signal ? AbortSignal.any([args.signal, timeoutSignal]) : timeoutSignal + const headers = buildSnowflakeAuthHeaders(destination.accessToken) + const byteBudget = { remainingBytes: SELECTOR_MAX_AGGREGATE_RESPONSE_BYTES } + try { + const response = await fetchSnowflakeResponse(`${destination.baseUrl}/api/v2/statements`, { + method: 'POST', + headers, + body: JSON.stringify({ + statement, + timeout: SELECTOR_TIMEOUT_SECONDS, + parameters: { rows_per_resultset: SELECTOR_ROW_LIMIT }, + }), + signal, + }) + const output = await readSnowflakeResult(response, { signal, byteBudget }) + if (output.status !== 'SUCCEEDED' || !output.result) { + throw new SelectorOptionsUnavailableError() + } + + const partitionCount = requirePartitionCount(output.result.partitionCount) + const totalRows = requireTotalRows(output.result.totalRows) + const rows = [...output.result.rows] + if (rows.length > SELECTOR_ROW_LIMIT) throw new SelectorOptionsUnavailableError() + + if (partitionCount > 1) { + const statementHandle = requireStatementHandle(output.statementHandle) + for (let partition = 1; partition < partitionCount; partition += 1) { + signal.throwIfAborted() + const partitionResponse = await fetchSnowflakeResponse( + `${destination.baseUrl}/api/v2/statements/${encodeURIComponent(statementHandle)}?partition=${partition}`, + { method: 'GET', headers, signal } + ) + const partitionOutput = await readSnowflakeResult(partitionResponse, { + currentPartition: partition, + partitionCount, + fallbackStatementHandle: statementHandle, + signal, + byteBudget, + }) + if ( + partitionOutput.status !== 'SUCCEEDED' || + partitionOutput.statementHandle !== statementHandle || + !partitionOutput.result || + partitionOutput.result.partitionCount !== partitionCount + ) { + throw new SelectorOptionsUnavailableError() + } + rows.push(...partitionOutput.result.rows) + if (rows.length > SELECTOR_ROW_LIMIT) throw new SelectorOptionsUnavailableError() + } + } + + signal.throwIfAborted() + if (rows.length !== totalRows) throw new SelectorOptionsUnavailableError() + const objects: SnowflakeObject[] = + spec.kind === 'roles' + ? parseAvailableRoles(rows[0]?.[0]) + : rows.flatMap((row) => { + const name = row[0] + if (typeof name !== 'string' || !name) return [] + return [{ name, detail: typeof row[1] === 'string' ? row[1] : null }] + }) + if (objects.length > MAX_SELECTOR_OPTIONS) throw new SelectorOptionsUnavailableError() + return flatSelectorResult(args.request, objects.map(toOption), true) + } catch (error) { + if (args.signal?.aborted) throw error + if (error instanceof SelectorConnectionUnavailableError) throw error + if (error instanceof SelectorOptionsUnavailableError) throw error + throw new SelectorOptionsUnavailableError() + } +} + +const credential = { + kind: 'stored', + field: 'oauthCredential', + serviceIds: ['snowflake'], +} as const + +/** + * The integration this selector reaches. Declared rather than derived: Snowflake is an + * API-key integration with no entry in the deployment OAuth catalog, so its + * service id maps to no block type and the allowlist would have nothing to + * judge it on. + */ +const integrationBlockTypes = ['snowflake'] as const + +export const snowflakeSelectorAttachments = { + 'snowflake.databases': definePreparedSelectorAttachment({ + credential, + integrationBlockTypes, + destination: { kind: 'credential-bound', prepare: prepareSnowflakeDestination }, + execute: executeSnowflake, + }), + 'snowflake.schemas': definePreparedSelectorAttachment({ + credential, + integrationBlockTypes, + destination: { kind: 'credential-bound', prepare: prepareSnowflakeDestination }, + execute: executeSnowflake, + }), + 'snowflake.tables': definePreparedSelectorAttachment({ + credential, + integrationBlockTypes, + destination: { kind: 'credential-bound', prepare: prepareSnowflakeDestination }, + execute: executeSnowflake, + }), + 'snowflake.warehouses': definePreparedSelectorAttachment({ + credential, + integrationBlockTypes, + destination: { kind: 'credential-bound', prepare: prepareSnowflakeDestination }, + execute: executeSnowflake, + }), + 'snowflake.roles': definePreparedSelectorAttachment({ + credential, + integrationBlockTypes, + destination: { kind: 'credential-bound', prepare: prepareSnowflakeDestination }, + execute: executeSnowflake, + }), + 'snowflake.fileFormats': definePreparedSelectorAttachment({ + credential, + integrationBlockTypes, + destination: { kind: 'credential-bound', prepare: prepareSnowflakeDestination }, + execute: executeSnowflake, + }), + 'snowflake.procedures': definePreparedSelectorAttachment({ + credential, + integrationBlockTypes, + destination: { kind: 'credential-bound', prepare: prepareSnowflakeDestination }, + execute: executeSnowflake, + }), +} satisfies ServerSelectorAttachmentMap diff --git a/apps/sim/lib/selectors/server/providers/trello.ts b/apps/sim/lib/selectors/server/providers/trello.ts new file mode 100644 index 00000000000..ebca7959044 --- /dev/null +++ b/apps/sim/lib/selectors/server/providers/trello.ts @@ -0,0 +1,60 @@ +import type { ServerSelectorKey } from '@/lib/selectors/manifest' +import { resolveSelectorOAuthAccessToken } from '@/lib/selectors/server/credentials' +import { + SelectorConnectionUnavailableError, + SelectorOptionsUnavailableError, +} from '@/lib/selectors/server/errors' +import { fetchProviderJson } from '@/lib/selectors/server/providers/provider-http' +import { + detailSelectorResult, + listSelectorResult, + type ServerSelectorAttachmentMap, +} from '@/lib/selectors/server/types' + +type TrelloSelectorKey = Extract + +export const trelloSelectorAttachments = { + 'trello.boards': { + credential: { + kind: 'stored', + field: 'oauthCredential', + serviceIds: ['trello'], + }, + destination: 'fixed', + execute: async (args) => { + if (!args.credential) throw new SelectorConnectionUnavailableError() + const apiKey = process.env.TRELLO_API_KEY + if (!apiKey) throw new SelectorOptionsUnavailableError() + args.protectedValues.add(apiKey) + const token = await resolveSelectorOAuthAccessToken({ + credential: args.credential, + serviceId: 'trello', + protectedValues: args.protectedValues, + }) + const url = new URL('https://api.trello.com/1/members/me/boards') + url.searchParams.set('key', apiKey) + url.searchParams.set('token', token) + url.searchParams.set('fields', 'id,name,closed') + const data = await fetchProviderJson(url, { + headers: { Accept: 'application/json' }, + signal: args.signal, + redirect: 'error', + }) + if (!Array.isArray(data)) throw new SelectorOptionsUnavailableError() + const boards = data.flatMap((value) => { + if (!value || typeof value !== 'object') return [] + const board = value as { id?: unknown; name?: unknown; closed?: unknown } + if (typeof board.id !== 'string' || typeof board.name !== 'string') return [] + return [{ id: board.id, label: board.name, closed: board.closed === true }] + }) + if (args.request.kind === 'detail') { + const detailId = args.request.id + const board = boards.find((item) => item.id === detailId) + return detailSelectorResult(board ? { id: board.id, label: board.label } : null) + } + return listSelectorResult( + boards.filter((board) => !board.closed).map(({ id, label }) => ({ id, label })) + ) + }, + }, +} satisfies ServerSelectorAttachmentMap diff --git a/apps/sim/lib/selectors/server/providers/wealthbox.ts b/apps/sim/lib/selectors/server/providers/wealthbox.ts new file mode 100644 index 00000000000..bd23662639f --- /dev/null +++ b/apps/sim/lib/selectors/server/providers/wealthbox.ts @@ -0,0 +1,83 @@ +import type { ServerSelectorKey } from '@/lib/selectors/manifest' +import { resolveSelectorOAuthAccessToken } from '@/lib/selectors/server/credentials' +import { SelectorConnectionUnavailableError } from '@/lib/selectors/server/errors' +import { flatSelectorResult } from '@/lib/selectors/server/providers/flat-results' +import { fetchProviderJson } from '@/lib/selectors/server/providers/provider-http' +import type { ServerSelectorAttachmentMap } from '@/lib/selectors/server/types' + +type WealthboxSelectorKey = Extract + +const PAGE_SIZE = 50 +const MAX_PAGES = 50 + +interface WealthboxContactsPage { + contacts?: Array> + meta?: { total_pages?: number; current_page?: number } +} + +export const wealthboxSelectorAttachments = { + 'wealthbox.contacts': { + credential: { + kind: 'stored', + field: 'oauthCredential', + serviceIds: ['wealthbox'], + }, + destination: 'fixed', + execute: async (args) => { + if (!args.credential) throw new SelectorConnectionUnavailableError() + const token = await resolveSelectorOAuthAccessToken({ + credential: args.credential, + serviceId: 'wealthbox', + protectedValues: args.protectedValues, + }) + const contacts: Array> = [] + let truncated = false + for (let page = 1; page <= MAX_PAGES; page++) { + const url = new URL('https://api.crmworkspace.com/v1/contacts') + url.searchParams.set('per_page', String(PAGE_SIZE)) + url.searchParams.set('page', String(page)) + const data = await fetchProviderJson(url, { + headers: { Authorization: `Bearer ${token}`, 'Content-Type': 'application/json' }, + signal: args.signal, + redirect: 'error', + }) + const pageContacts = Array.isArray(data.contacts) ? data.contacts : [] + contacts.push(...pageContacts) + const totalPages = data.meta?.total_pages + const currentPage = data.meta?.current_page ?? page + if ( + (typeof totalPages === 'number' && totalPages > 0 && currentPage >= totalPages) || + pageContacts.length < PAGE_SIZE + ) { + break + } + if (page === MAX_PAGES) truncated = true + } + const search = + args.request.kind === 'list' ? args.request.search?.trim().toLowerCase() : undefined + const items = contacts.flatMap((contact) => { + const id = contact.id === undefined || contact.id === null ? '' : String(contact.id) + if (!id) return [] + const firstName = typeof contact.first_name === 'string' ? contact.first_name : '' + const lastName = typeof contact.last_name === 'string' ? contact.last_name : '' + const label = `${firstName} ${lastName}`.trim() || `Contact ${id}` + const content = + typeof contact.background_information === 'string' ? contact.background_information : '' + if ( + search && + !label.toLowerCase().includes(search) && + !content.toLowerCase().includes(search) + ) { + return [] + } + return [{ id, label }] + }) + return flatSelectorResult( + args.request, + items, + false, + truncated ? { truncated: { reason: 'provider-cap', pages: MAX_PAGES } } : undefined + ) + }, + }, +} satisfies ServerSelectorAttachmentMap diff --git a/apps/sim/lib/selectors/server/providers/webflow.test.ts b/apps/sim/lib/selectors/server/providers/webflow.test.ts new file mode 100644 index 00000000000..2b5cff89f2d --- /dev/null +++ b/apps/sim/lib/selectors/server/providers/webflow.test.ts @@ -0,0 +1,126 @@ +/** + * @vitest-environment node + */ +import { afterAll, beforeEach, describe, expect, it, vi } from 'vitest' + +const { mockFetch, mockResolveSelectorOAuthAccessToken } = vi.hoisted(() => ({ + mockFetch: vi.fn(), + mockResolveSelectorOAuthAccessToken: vi.fn(), +})) + +vi.mock('@/lib/selectors/server/credentials', () => ({ + resolveSelectorOAuthAccessToken: mockResolveSelectorOAuthAccessToken, +})) + +import { createSelectorProtectedValues } from '@/lib/selectors/server/protected-values' +import { webflowSelectorAttachments } from '@/lib/selectors/server/providers/webflow' +import type { ExecuteServerSelectorArgs } from '@/lib/selectors/server/types' + +const collectionId = '680000000000000000000001' + +function args(request: ExecuteServerSelectorArgs['request']): ExecuteServerSelectorArgs { + return { + selectorKey: 'webflow.items', + context: { oauthCredential: 'credential-1', collectionId }, + request, + scope: { kind: 'workspace', workspaceId: 'workspace-1' }, + workspaceId: 'workspace-1', + principal: { kind: 'session', userId: 'user-1', sessionId: 'session-1' }, + requesterUserId: 'user-1', + credential: { suppliedId: 'credential-1' }, + references: new Map(), + protectedValues: createSelectorProtectedValues(), + } +} + +describe('Webflow server selector adapter', () => { + beforeEach(() => { + vi.clearAllMocks() + vi.stubGlobal('fetch', mockFetch) + mockResolveSelectorOAuthAccessToken.mockResolvedValue('server-only-token') + }) + + afterAll(() => vi.unstubAllGlobals()) + + it('fetches one searched item page beyond the old 50-page boundary', async () => { + const itemId = '680000000000000000001389' + mockFetch.mockResolvedValueOnce( + new Response( + JSON.stringify({ + items: [{ id: itemId, fieldData: { name: 'Needle beyond fifty' } }], + pagination: { limit: 100, offset: 5000, total: 5002 }, + }), + { status: 200 } + ) + ) + + await expect( + webflowSelectorAttachments['webflow.items'].execute( + args({ kind: 'list', search: ' Needle ', cursor: '5000' }) + ) + ).resolves.toEqual({ + kind: 'list', + items: [{ id: itemId, label: 'Needle beyond fifty' }], + nextCursor: '5001', + }) + + const url = new URL(String(mockFetch.mock.calls[0]?.[0])) + expect(url.pathname).toBe(`/v2/collections/${collectionId}/items`) + expect(url.searchParams.get('limit')).toBe('100') + expect(url.searchParams.get('offset')).toBe('5000') + expect(url.searchParams.get('filter[name][contains]')).toBe('Needle') + expect(mockFetch).toHaveBeenCalledTimes(1) + }) + + it('continues after a full page when optional pagination fields are omitted', async () => { + const items = Array.from({ length: 100 }, (_, index) => ({ + id: index.toString(16).padStart(24, '0'), + fieldData: { name: `Item ${index}` }, + })) + mockFetch.mockResolvedValueOnce( + new Response(JSON.stringify({ items, pagination: {} }), { status: 200 }) + ) + + const result = await webflowSelectorAttachments['webflow.items'].execute( + args({ kind: 'list', cursor: '5000' }) + ) + + expect(result.kind).toBe('list') + if (result.kind !== 'list') throw new Error('Expected a list selector result') + expect(result.items).toHaveLength(100) + expect(result.nextCursor).toBe('5100') + expect(mockFetch).toHaveBeenCalledTimes(1) + }) + + it('hydrates saved items directly and treats a missing item as absent', async () => { + const itemId = '680000000000000000001389' + const missingItemId = '680000000000000000001390' + mockFetch + .mockResolvedValueOnce( + new Response(JSON.stringify({ id: itemId, fieldData: { title: 'Saved item title' } }), { + status: 200, + }) + ) + .mockResolvedValueOnce(new Response(null, { status: 404 })) + + await expect( + webflowSelectorAttachments['webflow.items'].execute(args({ kind: 'detail', id: itemId })) + ).resolves.toEqual({ + kind: 'detail', + item: { id: itemId, label: 'Saved item title' }, + }) + await expect( + webflowSelectorAttachments['webflow.items'].execute( + args({ kind: 'detail', id: missingItemId }) + ) + ).resolves.toEqual({ kind: 'detail', item: null }) + + expect(String(mockFetch.mock.calls[0]?.[0])).toBe( + `https://api.webflow.com/v2/collections/${collectionId}/items/${itemId}` + ) + expect(String(mockFetch.mock.calls[1]?.[0])).toBe( + `https://api.webflow.com/v2/collections/${collectionId}/items/${missingItemId}` + ) + expect(mockFetch).toHaveBeenCalledTimes(2) + }) +}) diff --git a/apps/sim/lib/selectors/server/providers/webflow.ts b/apps/sim/lib/selectors/server/providers/webflow.ts new file mode 100644 index 00000000000..cbd11b7db59 --- /dev/null +++ b/apps/sim/lib/selectors/server/providers/webflow.ts @@ -0,0 +1,241 @@ +import { validateAlphanumericId } from '@/lib/core/security/input-validation' +import type { ServerSelectorKey } from '@/lib/selectors/manifest' +import { resolveSelectorOAuthAccessToken } from '@/lib/selectors/server/credentials' +import { + SelectorConnectionUnavailableError, + SelectorContextUnavailableError, + SelectorOptionsUnavailableError, +} from '@/lib/selectors/server/errors' +import { flatSelectorResult } from '@/lib/selectors/server/providers/flat-results' +import { + fetchProviderJson, + fetchProviderJsonWithStatus, +} from '@/lib/selectors/server/providers/provider-http' +import type { + ExecuteServerSelectorArgs, + ServerSelectorAttachmentMap, +} from '@/lib/selectors/server/types' +import { detailSelectorResult, listSelectorResult } from '@/lib/selectors/server/types' +import type { SafeSelectorOption } from '@/lib/selectors/types' + +type WebflowSelectorKey = Extract< + ServerSelectorKey, + 'webflow.sites' | 'webflow.collections' | 'webflow.items' +> + +const WEBFLOW_ITEM_PAGE_SIZE = 100 + +interface WebflowItem { + id?: unknown + fieldData?: { + name?: unknown + title?: unknown + slug?: unknown + } +} + +interface WebflowItemPage { + items?: unknown + pagination?: { + limit?: unknown + offset?: unknown + total?: unknown + } +} + +const credential = { + kind: 'stored', + field: 'oauthCredential', + serviceIds: ['webflow'], +} as const + +async function tokenFor(args: ExecuteServerSelectorArgs): Promise { + if (!args.credential) throw new SelectorConnectionUnavailableError() + return resolveSelectorOAuthAccessToken({ + credential: args.credential, + serviceId: 'webflow', + protectedValues: args.protectedValues, + }) +} + +function requireWebflowId(value: string | undefined, name: string): string { + if (!value) throw new SelectorContextUnavailableError() + const validation = validateAlphanumericId(value, name) + if (!validation.isValid) throw new SelectorContextUnavailableError() + return validation.sanitized ?? value +} + +function parseItemOffset(cursor: string | undefined): number { + if (cursor === undefined) return 0 + if (!/^(0|[1-9]\d*)$/.test(cursor)) throw new SelectorContextUnavailableError() + const offset = Number(cursor) + if (!Number.isSafeInteger(offset)) throw new SelectorContextUnavailableError() + return offset +} + +function projectItem(item: WebflowItem): SafeSelectorOption | null { + if (typeof item.id !== 'string' || !item.id) return null + const { name, title, slug } = item.fieldData ?? {} + const label = [name, title, slug].find( + (candidate): candidate is string => typeof candidate === 'string' && candidate.length > 0 + ) + return { id: item.id, label: label ?? item.id } +} + +function requirePaginationNumber(value: unknown, options: { positive?: boolean } = {}): number { + if ( + !Number.isSafeInteger(value) || + (value as number) < 0 || + (options.positive && (value as number) === 0) + ) { + throw new SelectorOptionsUnavailableError() + } + return value as number +} + +async function listSites(args: ExecuteServerSelectorArgs): Promise { + const token = await tokenFor(args) + const data = await fetchProviderJson<{ + sites?: Array<{ id: string; displayName?: string; shortName?: string }> + }>('https://api.webflow.com/v2/sites', { + headers: { Authorization: `Bearer ${token}`, accept: 'application/json' }, + signal: args.signal, + redirect: 'error', + }) + return (data.sites ?? []).map((site) => ({ + id: site.id, + label: site.displayName || site.shortName || site.id, + })) +} + +async function listCollections(args: ExecuteServerSelectorArgs): Promise { + const siteId = requireWebflowId(args.context.siteId, 'siteId') + const token = await tokenFor(args) + const data = await fetchProviderJson<{ + collections?: Array<{ id: string; displayName?: string; slug?: string }> + }>(`https://api.webflow.com/v2/sites/${encodeURIComponent(siteId)}/collections`, { + headers: { Authorization: `Bearer ${token}`, accept: 'application/json' }, + signal: args.signal, + redirect: 'error', + }) + return (data.collections ?? []).map((collection) => ({ + id: collection.id, + label: collection.displayName || collection.slug || collection.id, + })) +} + +async function listItems(args: ExecuteServerSelectorArgs) { + if (args.request.kind !== 'list') throw new SelectorContextUnavailableError() + const offset = parseItemOffset(args.request.cursor) + const collectionId = requireWebflowId(args.context.collectionId, 'collectionId') + const token = await tokenFor(args) + const url = new URL( + `https://api.webflow.com/v2/collections/${encodeURIComponent(collectionId)}/items` + ) + url.searchParams.set('limit', String(WEBFLOW_ITEM_PAGE_SIZE)) + url.searchParams.set('offset', String(offset)) + const search = args.request.search?.trim() + if (search) url.searchParams.set('filter[name][contains]', search) + + const data = await fetchProviderJson(url, { + headers: { Authorization: `Bearer ${token}`, accept: 'application/json' }, + signal: args.signal, + redirect: 'error', + }) + if ( + !data || + typeof data !== 'object' || + !Array.isArray(data.items) || + !data.pagination || + typeof data.pagination !== 'object' || + Array.isArray(data.pagination) + ) { + throw new SelectorOptionsUnavailableError() + } + + const reportedLimit = + data.pagination.limit === undefined + ? WEBFLOW_ITEM_PAGE_SIZE + : requirePaginationNumber(data.pagination.limit, { positive: true }) + const reportedOffset = + data.pagination.offset === undefined ? offset : requirePaginationNumber(data.pagination.offset) + const reportedTotal = + data.pagination.total === undefined ? undefined : requirePaginationNumber(data.pagination.total) + if ( + reportedLimit > WEBFLOW_ITEM_PAGE_SIZE || + reportedOffset !== offset || + data.items.length > reportedLimit + ) { + throw new SelectorOptionsUnavailableError() + } + if (data.items.length === 0 && reportedTotal !== undefined && reportedOffset < reportedTotal) { + throw new SelectorOptionsUnavailableError() + } + + const nextOffset = reportedOffset + data.items.length + if ( + !Number.isSafeInteger(nextOffset) || + (nextOffset <= reportedOffset && data.items.length > 0) + ) { + throw new SelectorOptionsUnavailableError() + } + const items = data.items.flatMap((item) => { + if (!item || typeof item !== 'object') return [] + const option = projectItem(item as WebflowItem) + return option ? [option] : [] + }) + return listSelectorResult( + items, + data.items.length > 0 && + (reportedTotal === undefined + ? data.items.length === reportedLimit + : nextOffset < reportedTotal) + ? String(nextOffset) + : undefined + ) +} + +async function getItem(args: ExecuteServerSelectorArgs) { + if (args.request.kind !== 'detail') throw new SelectorContextUnavailableError() + const collectionId = requireWebflowId(args.context.collectionId, 'collectionId') + const itemId = requireWebflowId(args.request.id, 'itemId') + const token = await tokenFor(args) + const result = await fetchProviderJsonWithStatus( + `https://api.webflow.com/v2/collections/${encodeURIComponent(collectionId)}/items/${encodeURIComponent(itemId)}`, + { + headers: { Authorization: `Bearer ${token}`, accept: 'application/json' }, + signal: args.signal, + redirect: 'error', + }, + { passthroughStatuses: [404] } + ) + if (!result.ok) return detailSelectorResult(null) + if (!result.data || typeof result.data !== 'object') { + throw new SelectorOptionsUnavailableError() + } + const item = projectItem(result.data) + if (!item || item.id !== itemId) throw new SelectorOptionsUnavailableError() + return detailSelectorResult(item) +} + +function executeItems(args: ExecuteServerSelectorArgs) { + return args.request.kind === 'detail' ? getItem(args) : listItems(args) +} + +export const webflowSelectorAttachments = { + 'webflow.sites': { + credential, + destination: 'fixed', + execute: async (args) => flatSelectorResult(args.request, await listSites(args)), + }, + 'webflow.collections': { + credential, + destination: 'fixed', + execute: async (args) => flatSelectorResult(args.request, await listCollections(args)), + }, + 'webflow.items': { + credential, + destination: 'fixed', + execute: executeItems, + }, +} satisfies ServerSelectorAttachmentMap diff --git a/apps/sim/lib/selectors/server/providers/zoho-desk.test.ts b/apps/sim/lib/selectors/server/providers/zoho-desk.test.ts new file mode 100644 index 00000000000..6b6fda70dd9 --- /dev/null +++ b/apps/sim/lib/selectors/server/providers/zoho-desk.test.ts @@ -0,0 +1,86 @@ +/** + * @vitest-environment node + */ +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const { mockResolveSelectorCredentialBundle, mockSecureFetchWithValidation } = vi.hoisted(() => ({ + mockResolveSelectorCredentialBundle: vi.fn(), + mockSecureFetchWithValidation: vi.fn(), +})) + +vi.mock('@/lib/core/security/input-validation.server', () => ({ + secureFetchWithValidation: mockSecureFetchWithValidation, +})) + +vi.mock('@/lib/selectors/server/providers/credential-bundle', () => ({ + resolveSelectorCredentialBundle: mockResolveSelectorCredentialBundle, +})) + +import { SelectorConnectionUnavailableError } from '@/lib/selectors/server/errors' +import { createSelectorProtectedValues } from '@/lib/selectors/server/protected-values' +import { zohoDeskSelectorAttachments } from '@/lib/selectors/server/providers/zoho-desk' +import type { ExecuteServerSelectorArgs } from '@/lib/selectors/server/types' + +function organizationArgs(signal: AbortSignal): ExecuteServerSelectorArgs { + return { + selectorKey: 'zoho_desk.organizations', + context: { oauthCredential: 'credential-1' }, + request: { kind: 'list' }, + scope: { kind: 'workspace', workspaceId: 'workspace-1' }, + workspaceId: 'workspace-1', + principal: { kind: 'session', userId: 'user-1', sessionId: 'session-1' }, + requesterUserId: 'user-1', + credential: { suppliedId: 'credential-1' }, + references: new Map(), + protectedValues: createSelectorProtectedValues(), + signal, + } +} + +describe('Zoho Desk server selector adapters', () => { + beforeEach(() => { + vi.clearAllMocks() + mockResolveSelectorCredentialBundle.mockResolvedValue({ + accessToken: 'server-only-token', + apiDomain: 'https://desk.zoho.com', + }) + }) + + it('preserves caller cancellation from the provider boundary', async () => { + const controller = new AbortController() + const abortError = new DOMException('The operation was aborted', 'AbortError') + controller.abort() + mockSecureFetchWithValidation.mockRejectedValueOnce(abortError) + + await expect( + zohoDeskSelectorAttachments['zoho_desk.organizations'].execute( + organizationArgs(controller.signal) + ) + ).rejects.toBe(abortError) + }) + + it('conceals and cancels a rejected provider response while preserving its safe category', async () => { + const cancel = vi.fn() + const body = new ReadableStream({ + start(controller) { + controller.enqueue(new TextEncoder().encode('provider-controlled-secret')) + }, + cancel, + }) + mockSecureFetchWithValidation.mockResolvedValueOnce(new Response(body, { status: 401 })) + + await expect( + zohoDeskSelectorAttachments['zoho_desk.organizations'].execute( + organizationArgs(new AbortController().signal) + ) + ).rejects.toEqual(new SelectorConnectionUnavailableError(401)) + expect(cancel).toHaveBeenCalledOnce() + expect(mockSecureFetchWithValidation).toHaveBeenCalledWith( + expect.any(String), + expect.objectContaining({ + profile: 'configuredEndpoint', + logUrlValidationDetails: false, + }) + ) + }) +}) diff --git a/apps/sim/lib/selectors/server/providers/zoho-desk.ts b/apps/sim/lib/selectors/server/providers/zoho-desk.ts new file mode 100644 index 00000000000..f6d0e57b20b --- /dev/null +++ b/apps/sim/lib/selectors/server/providers/zoho-desk.ts @@ -0,0 +1,224 @@ +import { db } from '@sim/db' +import { account } from '@sim/db/schema' +import { eq } from 'drizzle-orm' +import { secureFetchWithValidation } from '@/lib/core/security/input-validation.server' +import { resolveOAuthAccountId } from '@/lib/oauth/credential-service' +import type { ServerSelectorKey } from '@/lib/selectors/manifest' +import { + SelectorConnectionUnavailableError, + SelectorContextUnavailableError, + SelectorOptionsUnavailableError, +} from '@/lib/selectors/server/errors' +import { resolveSelectorCredentialBundle } from '@/lib/selectors/server/providers/credential-bundle' +import { flatSelectorResult } from '@/lib/selectors/server/providers/flat-results' +import { selectorProviderStatusError } from '@/lib/selectors/server/providers/provider-http' +import type { + ExecuteServerSelectorArgs, + ServerSelectorAttachmentMap, +} from '@/lib/selectors/server/types' +import { definePreparedSelectorAttachment } from '@/lib/selectors/server/types' +import type { SafeSelectorOption } from '@/lib/selectors/types' +import { assertZohoUrl, extractZohoDeskBaseFromScope } from '@/tools/zoho_desk/host-allowlist' +import { buildZohoDeskHeaders, getZohoDeskApiBase } from '@/tools/zoho_desk/utils' + +type ZohoDeskSelectorKey = Extract< + ServerSelectorKey, + 'zoho_desk.organizations' | 'zoho_desk.departments' | 'zoho_desk.agents' +> + +const credential = { + kind: 'stored', + field: 'oauthCredential', + serviceIds: ['zoho-desk'], +} as const + +async function readOAuthApiDomain(credentialId: string): Promise { + try { + const resolved = await resolveOAuthAccountId(credentialId) + if (!resolved?.accountId) return undefined + const [row] = await db + .select({ scope: account.scope }) + .from(account) + .where(eq(account.id, resolved.accountId)) + .limit(1) + return extractZohoDeskBaseFromScope(row?.scope) + } catch { + return undefined + } +} + +interface ZohoDeskDestination { + accessToken: string + apiBase: string +} + +async function prepareZohoDestination( + args: ExecuteServerSelectorArgs +): Promise { + if (!args.credential) throw new SelectorConnectionUnavailableError() + const bundle = await resolveSelectorCredentialBundle({ + credential: args.credential, + protectedValues: args.protectedValues, + }) + const apiDomain = bundle.apiDomain ?? (await readOAuthApiDomain(args.credential.suppliedId)) + args.protectedValues.add(apiDomain, 'reference') + const apiBase = getZohoDeskApiBase({ apiDomain }) + return { accessToken: bundle.accessToken, apiBase } +} + +async function fetchZoho( + args: ExecuteServerSelectorArgs, + url: URL, + headers: Record +): Promise<{ status: number; data: unknown[] }> { + let response + try { + response = await secureFetchWithValidation(url.toString(), { + profile: 'configuredEndpoint', + method: 'GET', + headers, + timeout: 15_000, + maxResponseBytes: 2 * 1024 * 1024, + stripAuthOnRedirect: true, + signal: args.signal, + logUrlValidationDetails: false, + }) + } catch (error) { + if (args.signal?.aborted) throw error + throw new SelectorOptionsUnavailableError() + } + if (response.status === 204) return { status: 204, data: [] } + if (!response.ok) { + await response.body?.cancel().catch(() => undefined) + throw selectorProviderStatusError(response.status) + } + let body: unknown + try { + body = await response.json() + } catch { + throw new SelectorOptionsUnavailableError() + } + if (!body || typeof body !== 'object' || !Array.isArray((body as { data?: unknown }).data)) { + throw new SelectorOptionsUnavailableError() + } + return { status: response.status, data: (body as { data: unknown[] }).data } +} + +async function listOrganizations( + args: ExecuteServerSelectorArgs, + destination: ZohoDeskDestination +): Promise { + const { accessToken, apiBase } = destination + let url: URL + try { + url = assertZohoUrl(`${apiBase}/organizations`) + } catch { + throw new SelectorConnectionUnavailableError() + } + const { data } = await fetchZoho(args, url, { + Authorization: `Zoho-oauthtoken ${accessToken}`, + 'Content-Type': 'application/json', + }) + return data.flatMap((value) => { + if (!value || typeof value !== 'object') return [] + const organization = value as { + id?: string | number + companyName?: string + portalName?: string + } + if (organization.id === undefined || organization.id === null) return [] + const id = String(organization.id) + return [{ id, label: organization.companyName || organization.portalName || id }] + }) +} + +async function listOrgResources( + args: ExecuteServerSelectorArgs, + kind: 'departments' | 'agents', + destination: ZohoDeskDestination +) { + const orgId = args.context.orgId + if (!orgId) throw new SelectorContextUnavailableError() + const { accessToken, apiBase } = destination + const headers = buildZohoDeskHeaders({ accessToken, orgId }) + const items: SafeSelectorOption[] = [] + const seen = new Set() + let truncated = false + + for (let page = 0; page < 20; page++) { + let url: URL + try { + url = assertZohoUrl(`${apiBase}/${kind}`) + } catch { + throw new SelectorConnectionUnavailableError() + } + url.searchParams.set('from', String(page * 200)) + url.searchParams.set('limit', '200') + if (kind === 'agents') url.searchParams.set('status', 'ACTIVE') + const result = await fetchZoho(args, url, headers) + if (result.status === 204) break + + for (const value of result.data) { + if (!value || typeof value !== 'object') continue + const record = value as Record + if (record.id === undefined || record.id === null) continue + const id = String(record.id) + if (seen.has(id)) continue + seen.add(id) + let label: string + if (kind === 'departments') { + label = + (typeof record.name === 'string' && record.name) || + (typeof record.nameInCustomerPortal === 'string' && record.nameInCustomerPortal) || + id + } else { + const name = typeof record.name === 'string' ? record.name.trim() : '' + const fullName = [record.firstName, record.lastName] + .filter((part): part is string => typeof part === 'string' && Boolean(part.trim())) + .map((part) => part.trim()) + .join(' ') + label = + name || fullName || (typeof record.emailId === 'string' && record.emailId.trim()) || id + } + items.push({ id, label }) + } + if (result.data.length < 200) break + if (page === 19) truncated = true + } + return { items, truncated } +} + +export const zohoDeskSelectorAttachments = { + 'zoho_desk.organizations': definePreparedSelectorAttachment({ + credential, + destination: { kind: 'credential-bound', prepare: prepareZohoDestination }, + execute: async (args, destination) => + flatSelectorResult(args.request, await listOrganizations(args, destination)), + }), + 'zoho_desk.departments': definePreparedSelectorAttachment({ + credential, + destination: { kind: 'credential-bound', prepare: prepareZohoDestination }, + execute: async (args, destination) => { + const result = await listOrgResources(args, 'departments', destination) + return flatSelectorResult( + args.request, + result.items, + false, + result.truncated ? { truncated: { reason: 'provider-cap', pages: 20 } } : undefined + ) + }, + }), + 'zoho_desk.agents': definePreparedSelectorAttachment({ + credential, + destination: { kind: 'credential-bound', prepare: prepareZohoDestination }, + execute: async (args, destination) => { + const result = await listOrgResources(args, 'agents', destination) + return flatSelectorResult( + args.request, + result.items, + false, + result.truncated ? { truncated: { reason: 'provider-cap', pages: 20 } } : undefined + ) + }, + }), +} satisfies ServerSelectorAttachmentMap diff --git a/apps/sim/lib/selectors/server/providers/zoom.test.ts b/apps/sim/lib/selectors/server/providers/zoom.test.ts new file mode 100644 index 00000000000..f2a274816eb --- /dev/null +++ b/apps/sim/lib/selectors/server/providers/zoom.test.ts @@ -0,0 +1,79 @@ +/** + * @vitest-environment node + */ +import { afterAll, beforeEach, describe, expect, it, vi } from 'vitest' + +const { mockFetch, mockResolveSelectorOAuthAccessToken } = vi.hoisted(() => ({ + mockFetch: vi.fn(), + mockResolveSelectorOAuthAccessToken: vi.fn(), +})) + +vi.mock('@/lib/selectors/server/credentials', () => ({ + resolveSelectorOAuthAccessToken: mockResolveSelectorOAuthAccessToken, +})) + +import { createSelectorProtectedValues } from '@/lib/selectors/server/protected-values' +import { zoomSelectorAttachments } from '@/lib/selectors/server/providers/zoom' +import type { ExecuteServerSelectorArgs } from '@/lib/selectors/server/types' + +function args( + request: ExecuteServerSelectorArgs['request'] = { kind: 'list' } +): ExecuteServerSelectorArgs { + return { + selectorKey: 'zoom.meetings', + context: { oauthCredential: 'credential-1' }, + request, + scope: { kind: 'workspace', workspaceId: 'workspace-1' }, + workspaceId: 'workspace-1', + principal: { kind: 'session', userId: 'user-1', sessionId: 'session-1' }, + requesterUserId: 'user-1', + credential: { suppliedId: 'credential-1' }, + references: new Map(), + protectedValues: createSelectorProtectedValues(), + } +} + +describe('Zoom server selector adapter', () => { + beforeEach(() => { + vi.clearAllMocks() + vi.stubGlobal('fetch', mockFetch) + mockResolveSelectorOAuthAccessToken.mockResolvedValue('server-only-token') + }) + + afterAll(() => vi.unstubAllGlobals()) + + it('returns one meeting page and forwards its continuation token on demand', async () => { + mockFetch.mockResolvedValueOnce( + new Response( + JSON.stringify({ meetings: [{ id: 123, topic: 'Planning' }], next_page_token: 'page-2' }), + { status: 200 } + ) + ) + + await expect( + zoomSelectorAttachments['zoom.meetings'].execute(args({ kind: 'list', cursor: 'page-1' })) + ).resolves.toEqual({ + kind: 'list', + items: [{ id: '123', label: 'Planning' }], + nextCursor: 'page-2', + }) + const url = new URL(String(mockFetch.mock.calls[0]?.[0])) + expect(url.searchParams.get('next_page_token')).toBe('page-1') + expect(mockFetch).toHaveBeenCalledTimes(1) + }) + + it('hydrates a selected meeting without draining the list', async () => { + mockFetch.mockResolvedValueOnce( + new Response(JSON.stringify({ id: 123, topic: 'Planning' }), { status: 200 }) + ) + + await expect( + zoomSelectorAttachments['zoom.meetings'].execute(args({ kind: 'detail', id: '123' })) + ).resolves.toEqual({ + kind: 'detail', + item: { id: '123', label: 'Planning' }, + }) + expect(String(mockFetch.mock.calls[0]?.[0]).endsWith('/v2/meetings/123')).toBe(true) + expect(mockFetch).toHaveBeenCalledTimes(1) + }) +}) diff --git a/apps/sim/lib/selectors/server/providers/zoom.ts b/apps/sim/lib/selectors/server/providers/zoom.ts new file mode 100644 index 00000000000..232114188a5 --- /dev/null +++ b/apps/sim/lib/selectors/server/providers/zoom.ts @@ -0,0 +1,82 @@ +import type { ServerSelectorKey } from '@/lib/selectors/manifest' +import { resolveSelectorOAuthAccessToken } from '@/lib/selectors/server/credentials' +import { + SelectorConnectionUnavailableError, + SelectorContextUnavailableError, +} from '@/lib/selectors/server/errors' +import { fetchProviderJson } from '@/lib/selectors/server/providers/provider-http' +import { + detailSelectorResult, + listSelectorResult, + requireListRequest, + type ServerSelectorAttachmentMap, +} from '@/lib/selectors/server/types' + +type ZoomSelectorKey = Extract + +const PAGE_SIZE = 300 + +interface ZoomMeetingsPage { + meetings?: Array<{ id: number | string; topic?: string }> + next_page_token?: string +} + +interface ZoomMeeting { + id: number | string + topic?: string +} + +function encodeZoomMeetingId(value: string): string { + const id = value.trim() + if (!id || id.length > 256 || /[\u0000-\u001F\u007F]/.test(id)) { + throw new SelectorContextUnavailableError() + } + const encoded = encodeURIComponent(id) + return id.startsWith('/') || id.includes('//') ? encodeURIComponent(encoded) : encoded +} + +export const zoomSelectorAttachments = { + 'zoom.meetings': { + credential: { + kind: 'stored', + field: 'oauthCredential', + serviceIds: ['zoom'], + }, + destination: 'fixed', + execute: async (args) => { + if (!args.credential) throw new SelectorConnectionUnavailableError() + const token = await resolveSelectorOAuthAccessToken({ + credential: args.credential, + serviceId: 'zoom', + protectedValues: args.protectedValues, + }) + const headers = { Authorization: `Bearer ${token}`, 'Content-Type': 'application/json' } + if (args.request.kind === 'detail') { + const meeting = await fetchProviderJson( + `https://api.zoom.us/v2/meetings/${encodeZoomMeetingId(args.request.id)}`, + { headers, signal: args.signal, redirect: 'error' } + ) + const id = String(meeting.id) + return detailSelectorResult({ id, label: meeting.topic || `Meeting ${id}` }) + } + + const request = requireListRequest(args.selectorKey, args.request) + const url = new URL('https://api.zoom.us/v2/users/me/meetings') + url.searchParams.set('page_size', String(PAGE_SIZE)) + url.searchParams.set('type', 'scheduled') + if (request.cursor) url.searchParams.set('next_page_token', request.cursor) + const data = await fetchProviderJson(url, { + headers, + signal: args.signal, + redirect: 'error', + }) + return listSelectorResult( + (data.meetings ?? []).map((meeting) => { + const id = String(meeting.id) + return { id, label: meeting.topic || `Meeting ${id}` } + }), + data.next_page_token?.trim() || undefined + ) + }, + }, +} satisfies ServerSelectorAttachmentMap diff --git a/apps/sim/lib/selectors/server/references.test.ts b/apps/sim/lib/selectors/server/references.test.ts new file mode 100644 index 00000000000..b19f14b86d7 --- /dev/null +++ b/apps/sim/lib/selectors/server/references.test.ts @@ -0,0 +1,171 @@ +/** + * @vitest-environment node + */ +import { environmentUtilsMockFns, resetEnvironmentUtilsMock } from '@sim/testing' +import { beforeEach, describe, expect, it } from 'vitest' +import { SelectorContextUnavailableError } from '@/lib/selectors/server/errors' +import { createSelectorProtectedValues } from '@/lib/selectors/server/protected-values' +import { resolveSelectorReferences } from '@/lib/selectors/server/references' + +const baseInput = { + selectorKey: 'imap.mailboxes' as const, + requesterUserId: 'user-1', + workspaceId: 'workspace-1', +} + +describe('resolveSelectorReferences', () => { + beforeEach(() => { + resetEnvironmentUtilsMock() + }) + + it('keeps browser-known literals local without treating them as server-only secrets', async () => { + const protectedValues = createSelectorProtectedValues() + + const result = await resolveSelectorReferences({ + ...baseInput, + context: { + host: 'imap.example.com', + port: '993', + secure: 'true', + username: 'mailbox-user', + password: 'literal-password', + }, + request: { kind: 'list' }, + protectedValues, + }) + + expect(result.context).toEqual({ + host: 'imap.example.com', + port: '993', + secure: 'true', + username: 'mailbox-user', + password: 'literal-password', + }) + expect(result.references.size).toBe(0) + expect(protectedValues.contains('prefix-literal-password-suffix')).toBe(false) + expect(environmentUtilsMockFns.mockResolveEffectiveEnvironmentVariables).not.toHaveBeenCalled() + }) + + it('resolves personal, visible shared, and hidden use-only references with workspace precedence', async () => { + environmentUtilsMockFns.mockResolveEffectiveEnvironmentVariables.mockResolvedValue({ + PERSONAL_HOST: { + value: 'personal.example.com', + scope: 'personal', + visible: true, + }, + SHARED_USERNAME: { + value: 'shared-user', + scope: 'workspace', + visible: true, + }, + SHARED_PASSWORD: { + value: 'hidden-password', + scope: 'workspace', + visible: false, + }, + }) + const protectedValues = createSelectorProtectedValues() + + const result = await resolveSelectorReferences({ + ...baseInput, + context: { + host: '{{PERSONAL_HOST}}', + username: '{{SHARED_USERNAME}}', + password: '{{SHARED_PASSWORD}}', + }, + request: { kind: 'list' }, + protectedValues, + }) + + expect(result.context).toEqual({ + host: 'personal.example.com', + username: 'shared-user', + password: 'hidden-password', + }) + expect([...result.references.values()]).toEqual([ + { + field: 'host', + name: 'PERSONAL_HOST', + scope: 'personal', + visible: true, + }, + { + field: 'username', + name: 'SHARED_USERNAME', + scope: 'workspace', + visible: true, + }, + { + field: 'password', + name: 'SHARED_PASSWORD', + scope: 'workspace', + visible: false, + }, + ]) + expect(protectedValues.contains('hidden-password')).toBe(true) + expect(protectedValues.contains('prefix-personal.example.com-suffix')).toBe(false) + expect(protectedValues.contains('prefix-shared-user-suffix')).toBe(false) + expect(environmentUtilsMockFns.mockResolveEffectiveEnvironmentVariables).toHaveBeenCalledWith( + 'user-1', + 'workspace-1', + ['PERSONAL_HOST', 'SHARED_USERNAME', 'SHARED_PASSWORD'] + ) + }) + + it('projects missing, inaccessible, embedded, and runtime references to one context error', async () => { + environmentUtilsMockFns.mockResolveEffectiveEnvironmentVariables.mockResolvedValue({}) + + const contexts = [ + { host: '{{MISSING}}', username: 'user', password: 'password' }, + { host: '{{INACCESSIBLE_SHARED}}', username: 'user', password: 'password' }, + { host: 'imap.{{HOST}}', username: 'user', password: 'password' }, + { host: '', username: 'user', password: 'password' }, + ] + + for (const context of contexts) { + await expect( + resolveSelectorReferences({ + ...baseInput, + context, + request: { kind: 'list' }, + protectedValues: createSelectorProtectedValues(), + }) + ).rejects.toEqual(new SelectorContextUnavailableError()) + } + }) + + it('loads duplicate references once while retaining field-level provenance', async () => { + environmentUtilsMockFns.mockResolveEffectiveEnvironmentVariables.mockResolvedValue({ + REPEATED: { + value: 'resolved-value', + scope: 'workspace', + visible: false, + }, + }) + + const result = await resolveSelectorReferences({ + ...baseInput, + context: { + host: '{{REPEATED}}', + port: '993', + secure: 'true', + username: '{{REPEATED}}', + password: 'literal-password', + }, + request: { kind: 'detail', id: '{{REPEATED}}' }, + protectedValues: createSelectorProtectedValues(), + }) + + expect(environmentUtilsMockFns.mockResolveEffectiveEnvironmentVariables).toHaveBeenCalledWith( + 'user-1', + 'workspace-1', + ['REPEATED'] + ) + expect(result.context).toMatchObject({ + host: 'resolved-value', + username: 'resolved-value', + }) + expect(result.request).toEqual({ kind: 'detail', id: 'resolved-value' }) + expect([...result.references.keys()]).toEqual(['host', 'username', 'request.id']) + }) +}) diff --git a/apps/sim/lib/selectors/server/references.ts b/apps/sim/lib/selectors/server/references.ts new file mode 100644 index 00000000000..07a7be0e823 --- /dev/null +++ b/apps/sim/lib/selectors/server/references.ts @@ -0,0 +1,103 @@ +import { resolveEffectiveEnvironmentVariables } from '@/lib/environment/utils' +import { getSelectorManifestEntry, type ServerSelectorKey } from '@/lib/selectors/manifest' +import { SelectorContextUnavailableError } from '@/lib/selectors/server/errors' +import type { + ResolvedSelectorReference, + SelectorProtectedValues, +} from '@/lib/selectors/server/types' +import type { SelectorContext, SelectorRequest } from '@/lib/selectors/types' + +const EXACT_ENVIRONMENT_REFERENCE = /^\{\{\s*([A-Za-z_][A-Za-z0-9_]*)\s*\}\}$/ + +export interface ResolvedSelectorInputs { + context: SelectorContext + request: SelectorRequest + references: ReadonlyMap +} + +function rejectsEmbeddedReference(value: string): boolean { + return (value.includes('{{') || value.includes('}}')) && !EXACT_ENVIRONMENT_REFERENCE.test(value) +} + +function containsRuntimeReference(value: string): boolean { + return /<[^<>]+>/.test(value) +} + +export async function resolveSelectorReferences(input: { + selectorKey: ServerSelectorKey + context: SelectorContext + request: SelectorRequest + requesterUserId: string + workspaceId: string + protectedValues: SelectorProtectedValues +}): Promise { + const contextEntries = Object.entries(input.context).filter( + (entry): entry is [string, string] => entry[1] !== undefined + ) + const resolvableValues = [ + ...contextEntries.map(([, value]) => value), + ...(input.request.kind === 'detail' ? [input.request.id] : []), + ] + if ( + resolvableValues.some( + (value) => rejectsEmbeddedReference(value) || containsRuntimeReference(value) + ) + ) { + throw new SelectorContextUnavailableError() + } + + const manifest = getSelectorManifestEntry(input.selectorKey) + if (!resolvableValues.some((value) => EXACT_ENVIRONMENT_REFERENCE.test(value))) { + const context = Object.fromEntries(contextEntries) as SelectorContext + return { context, request: input.request, references: new Map() } + } + + const sensitiveFields = new Set(manifest.context.sensitive ?? []) + + const referenceNames = [ + ...new Set( + resolvableValues.flatMap((value) => { + const match = EXACT_ENVIRONMENT_REFERENCE.exec(value) + return match ? [match[1]] : [] + }) + ), + ] + const resolvedVariables = await resolveEffectiveEnvironmentVariables( + input.requesterUserId, + input.workspaceId, + referenceNames + ) + const references = new Map() + + const resolve = (field: string, value: string): string => { + const match = EXACT_ENVIRONMENT_REFERENCE.exec(value) + if (!match) return value + + const name = match[1] + const variable = Object.hasOwn(resolvedVariables, name) ? resolvedVariables[name] : undefined + if (!variable) throw new SelectorContextUnavailableError() + + if (!variable.visible) { + input.protectedValues.add(variable.value, sensitiveFields.has(field) ? 'secret' : 'reference') + } + references.set(field, { + field, + name, + scope: variable.scope, + visible: variable.visible, + }) + return variable.value + } + + const context: SelectorContext = {} + for (const [field, value] of contextEntries) { + context[field as keyof SelectorContext] = resolve(field, value) + } + + const request = + input.request.kind === 'detail' + ? { ...input.request, id: resolve('request.id', input.request.id) } + : input.request + + return { context, request, references } +} diff --git a/apps/sim/lib/selectors/server/registry.ts b/apps/sim/lib/selectors/server/registry.ts new file mode 100644 index 00000000000..ef9a815f1af --- /dev/null +++ b/apps/sim/lib/selectors/server/registry.ts @@ -0,0 +1,71 @@ +import type { ServerSelectorKey } from '@/lib/selectors/manifest' +import { internalSelectorAttachments } from '@/lib/selectors/server/internal' +import { airtableSelectorAttachments } from '@/lib/selectors/server/providers/airtable' +import { asanaSelectorAttachments } from '@/lib/selectors/server/providers/asana' +import { attioSelectorAttachments } from '@/lib/selectors/server/providers/attio' +import { bigQuerySelectorAttachments } from '@/lib/selectors/server/providers/bigquery' +import { bitbucketSelectorAttachments } from '@/lib/selectors/server/providers/bitbucket' +import { calcomSelectorAttachments } from '@/lib/selectors/server/providers/calcom' +import { clickupSelectorAttachments } from '@/lib/selectors/server/providers/clickup' +import { cloudWatchSelectorAttachments } from '@/lib/selectors/server/providers/cloudwatch' +import { confluenceSelectorAttachments } from '@/lib/selectors/server/providers/confluence' +import { googleSelectorAttachments } from '@/lib/selectors/server/providers/google' +import { harmonicSelectorAttachments } from '@/lib/selectors/server/providers/harmonic' +import { hubspotSelectorAttachments } from '@/lib/selectors/server/providers/hubspot' +import { imapSelectorAttachments } from '@/lib/selectors/server/providers/imap' +import { jiraSelectorAttachments } from '@/lib/selectors/server/providers/jira' +import { jsmSelectorAttachments } from '@/lib/selectors/server/providers/jsm' +import { linearSelectorAttachments } from '@/lib/selectors/server/providers/linear' +import { managedAgentSelectorAttachments } from '@/lib/selectors/server/providers/managed-agent' +import { microsoftSelectorAttachments } from '@/lib/selectors/server/providers/microsoft' +import { mondaySelectorAttachments } from '@/lib/selectors/server/providers/monday' +import { netsuiteSelectorAttachments } from '@/lib/selectors/server/providers/netsuite' +import { notionSelectorAttachments } from '@/lib/selectors/server/providers/notion' +import { pipedriveSelectorAttachments } from '@/lib/selectors/server/providers/pipedrive' +import { sharepointSelectorAttachments } from '@/lib/selectors/server/providers/sharepoint' +import { slackSelectorAttachments } from '@/lib/selectors/server/providers/slack' +import { snowflakeSelectorAttachments } from '@/lib/selectors/server/providers/snowflake' +import { trelloSelectorAttachments } from '@/lib/selectors/server/providers/trello' +import { wealthboxSelectorAttachments } from '@/lib/selectors/server/providers/wealthbox' +import { webflowSelectorAttachments } from '@/lib/selectors/server/providers/webflow' +import { zohoDeskSelectorAttachments } from '@/lib/selectors/server/providers/zoho-desk' +import { zoomSelectorAttachments } from '@/lib/selectors/server/providers/zoom' +import type { ServerSelectorAttachment } from '@/lib/selectors/server/types' + +export const serverSelectorRegistry = { + ...internalSelectorAttachments, + ...airtableSelectorAttachments, + ...asanaSelectorAttachments, + ...attioSelectorAttachments, + ...bigQuerySelectorAttachments, + ...bitbucketSelectorAttachments, + ...calcomSelectorAttachments, + ...clickupSelectorAttachments, + ...cloudWatchSelectorAttachments, + ...confluenceSelectorAttachments, + ...googleSelectorAttachments, + ...harmonicSelectorAttachments, + ...hubspotSelectorAttachments, + ...imapSelectorAttachments, + ...jiraSelectorAttachments, + ...jsmSelectorAttachments, + ...linearSelectorAttachments, + ...managedAgentSelectorAttachments, + ...microsoftSelectorAttachments, + ...mondaySelectorAttachments, + ...netsuiteSelectorAttachments, + ...notionSelectorAttachments, + ...pipedriveSelectorAttachments, + ...sharepointSelectorAttachments, + ...slackSelectorAttachments, + ...snowflakeSelectorAttachments, + ...trelloSelectorAttachments, + ...wealthboxSelectorAttachments, + ...webflowSelectorAttachments, + ...zohoDeskSelectorAttachments, + ...zoomSelectorAttachments, +} satisfies Record + +export function getServerSelectorAttachment(key: ServerSelectorKey): ServerSelectorAttachment { + return serverSelectorRegistry[key] +} diff --git a/apps/sim/lib/selectors/server/sanitize.test.ts b/apps/sim/lib/selectors/server/sanitize.test.ts new file mode 100644 index 00000000000..00f7a9e349d --- /dev/null +++ b/apps/sim/lib/selectors/server/sanitize.test.ts @@ -0,0 +1,299 @@ +/** + * @vitest-environment node + */ +import { describe, expect, it } from 'vitest' +import { SelectorOptionsUnavailableError } from '@/lib/selectors/server/errors' +import { createSelectorProtectedValues } from '@/lib/selectors/server/protected-values' +import { sanitizeSelectorResult } from '@/lib/selectors/server/sanitize' +import type { SelectorExecutionResult } from '@/lib/selectors/types' + +describe('sanitizeSelectorResult', () => { + it('fails closed when protected plaintext appears in any response field', () => { + const protectedValues = createSelectorProtectedValues() + protectedValues.add('selector-secret-canary') + const results: SelectorExecutionResult[] = [ + { + kind: 'list', + items: [{ id: 'selector-secret-canary', label: 'Safe label' }], + }, + { + kind: 'list', + items: [{ id: 'safe-id', label: 'prefix-selector-secret-canary-suffix' }], + }, + { + kind: 'detail', + item: { id: 'safe-id', label: 'Safe label', meta: { value: 'selector-secret-canary' } }, + }, + { + kind: 'list', + items: [{ id: 'safe-id', label: 'Safe label' }], + nextCursor: 'cursor-selector-secret-canary', + }, + ] + + for (const result of results) { + expect(() => sanitizeSelectorResult(result, protectedValues)).toThrow( + SelectorOptionsUnavailableError + ) + } + }) + + it('fails closed when protected plaintext is percent encoded', () => { + const protectedValues = createSelectorProtectedValues() + protectedValues.add('selector secret/canary') + + for (const value of [ + 'selector%20secret%2Fcanary', + 'selector%2520secret%252Fcanary', + 'prefix-selector%20secret%2Fcanary-suffix', + ]) { + expect(() => + sanitizeSelectorResult( + { kind: 'list', items: [{ id: 'safe-id', label: value }] }, + protectedValues + ) + ).toThrow(SelectorOptionsUnavailableError) + } + }) + + it('rejects malformed encoded output without treating literal percentages as encoding', () => { + const protectedValues = createSelectorProtectedValues() + protectedValues.add('secret') + + expect(() => + sanitizeSelectorResult( + { kind: 'list', items: [{ id: 'safe-id', label: '%73ecret%ZZ' }] }, + protectedValues + ) + ).toThrow(SelectorOptionsUnavailableError) + expect( + sanitizeSelectorResult( + { kind: 'list', items: [{ id: 'safe-id', label: 'Save 50% today' }] }, + protectedValues + ) + ).toEqual({ kind: 'list', items: [{ id: 'safe-id', label: 'Save 50% today' }] }) + }) + + it('distinguishes short identifiers from raw secrets when checking substrings', () => { + const referenceValues = createSelectorProtectedValues() + referenceValues.add('a', 'reference') + + expect( + sanitizeSelectorResult( + { kind: 'list', items: [{ id: 'INBOX', label: 'Drafts' }] }, + referenceValues + ) + ).toEqual({ kind: 'list', items: [{ id: 'INBOX', label: 'Drafts' }] }) + expect(() => + sanitizeSelectorResult( + { kind: 'list', items: [{ id: 'a', label: 'Exact identifier' }] }, + referenceValues + ) + ).toThrow(SelectorOptionsUnavailableError) + + const secretValues = createSelectorProtectedValues() + secretValues.add('a', 'secret') + expect(() => + sanitizeSelectorResult( + { kind: 'list', items: [{ id: 'INBOX', label: 'Drafts' }] }, + secretValues + ) + ).toThrow(SelectorOptionsUnavailableError) + }) + + it('returns only the normalized selector option envelope', () => { + const result = sanitizeSelectorResult( + { + kind: 'list', + items: [ + { + id: 'resource-1', + label: 'Resource one', + meta: { count: 3, active: true, parentId: null }, + }, + ], + nextCursor: 'next-page', + }, + createSelectorProtectedValues() + ) + + expect(result).toEqual({ + kind: 'list', + items: [ + { + id: 'resource-1', + label: 'Resource one', + meta: { count: 3, active: true, parentId: null }, + }, + ], + nextCursor: 'next-page', + }) + }) + + it('allows only exact protected detail-id repeats for later reference restoration', () => { + const protectedValues = createSelectorProtectedValues() + protectedValues.add('ID') + + expect( + sanitizeSelectorResult( + { + kind: 'detail', + item: { id: 'ID', label: 'ID', meta: { resourceId: 'ID' } }, + }, + protectedValues, + { allowedDetailExactProtectedValue: 'ID' } + ) + ).toEqual({ + kind: 'detail', + item: { id: 'ID', label: 'ID', meta: { resourceId: 'ID' } }, + }) + + const rejectedResults: SelectorExecutionResult[] = [ + { + kind: 'detail', + item: { id: 'ID', label: 'prefix-ID-suffix' }, + }, + { + kind: 'detail', + item: { id: 'ID', label: 'ID', meta: { resourceId: 'prefix-ID-suffix' } }, + }, + { + kind: 'list', + items: [{ id: 'ID', label: 'ID' }], + }, + { + kind: 'list', + items: [{ id: 'safe-id', label: 'Safe label' }], + nextCursor: 'ID', + }, + ] + + for (const result of rejectedResults) { + expect(() => + sanitizeSelectorResult(result, protectedValues, { + allowedDetailExactProtectedValue: 'ID', + }) + ).toThrow(SelectorOptionsUnavailableError) + } + + expect( + sanitizeSelectorResult({ kind: 'detail', item: null }, protectedValues, { + allowedDetailExactProtectedValue: 'ID', + }) + ).toEqual({ kind: 'detail', item: null }) + }) + + it('still rejects other protected values when allowing an exact detail ID', () => { + const protectedValues = createSelectorProtectedValues() + protectedValues.add('resolved-id') + protectedValues.add('another-secret') + + expect(() => + sanitizeSelectorResult( + { + kind: 'detail', + item: { + id: 'resolved-id', + label: 'resolved-id', + meta: { resourceId: 'another-secret' }, + }, + }, + protectedValues, + { allowedDetailExactProtectedValue: 'resolved-id' } + ) + ).toThrow(SelectorOptionsUnavailableError) + }) + + it('rejects an allowed detail ID that embeds another protected value', () => { + const protectedValues = createSelectorProtectedValues() + protectedValues.add('resolved-another-secret-id') + protectedValues.add('another-secret') + + expect(() => + sanitizeSelectorResult( + { + kind: 'detail', + item: { + id: 'resolved-another-secret-id', + label: 'Resolved item', + }, + }, + protectedValues, + { allowedDetailExactProtectedValue: 'resolved-another-secret-id' } + ) + ).toThrow(SelectorOptionsUnavailableError) + }) + + it('rejects protected plaintext in metadata keys without applying the detail exemption', () => { + const protectedValues = createSelectorProtectedValues() + protectedValues.add('resolved-id') + + expect(() => + sanitizeSelectorResult( + { + kind: 'detail', + item: { + id: 'resolved-id', + label: 'resolved-id', + meta: { 'prefix-resolved-id-suffix': null }, + }, + }, + protectedValues, + { allowedDetailExactProtectedValue: 'resolved-id' } + ) + ).toThrow(SelectorOptionsUnavailableError) + }) + + it('rejects protected numeric metadata without applying the detail exemption', () => { + const protectedValues = createSelectorProtectedValues() + protectedValues.add('1234') + + expect(() => + sanitizeSelectorResult( + { + kind: 'detail', + item: { id: '1234', label: '1234', meta: { resourceId: 1234 } }, + }, + protectedValues, + { allowedDetailExactProtectedValue: '1234' } + ) + ).toThrow(SelectorOptionsUnavailableError) + }) + + it('preserves allowed metadata keys that shadow object prototype properties', () => { + const meta = Object.create(null) as Record + meta.__proto__ = null + + const result = sanitizeSelectorResult( + { + kind: 'list', + items: [{ id: 'resource-1', label: 'Resource one', meta }], + }, + createSelectorProtectedValues() + ) + + expect(result.kind).toBe('list') + if (result.kind !== 'list') throw new Error('Expected list selector result') + expect(Object.hasOwn(result.items[0].meta ?? {}, '__proto__')).toBe(true) + expect(result.items[0].meta?.__proto__).toBeNull() + expect(JSON.stringify(result.items[0].meta)).toBe('{"__proto__":null}') + }) + + it('rejects metadata strings larger than the response contract permits', () => { + expect(() => + sanitizeSelectorResult( + { + kind: 'list', + items: [ + { + id: 'resource-1', + label: 'Resource one', + meta: { description: 'x'.repeat(16 * 1024 + 1) }, + }, + ], + }, + createSelectorProtectedValues() + ) + ).toThrow(SelectorOptionsUnavailableError) + }) +}) diff --git a/apps/sim/lib/selectors/server/sanitize.ts b/apps/sim/lib/selectors/server/sanitize.ts new file mode 100644 index 00000000000..e12def7cfde --- /dev/null +++ b/apps/sim/lib/selectors/server/sanitize.ts @@ -0,0 +1,151 @@ +import { MAX_SELECTOR_OPTIONS } from '@/lib/selectors/limits' +import { SelectorOptionsUnavailableError } from '@/lib/selectors/server/errors' +import type { + SelectorProtectedValues, + ServerSelectorExecutionResult, +} from '@/lib/selectors/server/types' +import type { + SafeOptionMeta, + SafeOptionMetaValue, + SafeSelectorOption, + SelectorExecutionResult, +} from '@/lib/selectors/types' + +const MAX_OPTION_TEXT = 16 * 1024 +const MAX_META_FIELDS = 32 +const MAX_PERCENT_DECODE_ROUNDS = 3 +const PERCENT_ESCAPE_PATTERN = /%[0-9a-f]{2}/i + +export interface SanitizeSelectorResultOptions { + allowedDetailExactProtectedValue?: string +} + +function containsProtectedValue( + value: string, + protectedValues: SelectorProtectedValues, + allowedExactValue?: string +): boolean { + if (protectedValues.contains(value)) { + return !allowedExactValue || protectedValues.containsExceptExact(value, allowedExactValue) + } + + let decoded = value + for (let round = 0; round < MAX_PERCENT_DECODE_ROUNDS; round += 1) { + if (!PERCENT_ESCAPE_PATTERN.test(decoded)) return false + let next: string + try { + next = decodeURIComponent(decoded) + } catch { + return true + } + if (next === decoded) return false + if (protectedValues.contains(next)) return true + decoded = next + } + return PERCENT_ESCAPE_PATTERN.test(decoded) +} + +function requireSafeString( + value: unknown, + protectedValues: SelectorProtectedValues, + allowedExactValue?: string +): string { + if (typeof value !== 'string' || value.length === 0 || value.length > MAX_OPTION_TEXT) { + throw new SelectorOptionsUnavailableError() + } + if (containsProtectedValue(value, protectedValues, allowedExactValue)) { + throw new SelectorOptionsUnavailableError() + } + return value +} + +function sanitizeMeta( + value: unknown, + protectedValues: SelectorProtectedValues, + allowedExactValue?: string +): SafeOptionMeta | undefined { + if (value === undefined) return undefined + if (!value || typeof value !== 'object' || Array.isArray(value)) { + throw new SelectorOptionsUnavailableError() + } + + const entries = Object.entries(value) + if (entries.length > MAX_META_FIELDS) throw new SelectorOptionsUnavailableError() + const meta: SafeOptionMeta = {} + for (const [key, entry] of entries) { + if (!key || key.length > 128) throw new SelectorOptionsUnavailableError() + if (containsProtectedValue(key, protectedValues)) { + throw new SelectorOptionsUnavailableError() + } + if ( + entry !== null && + typeof entry !== 'string' && + typeof entry !== 'number' && + typeof entry !== 'boolean' + ) { + throw new SelectorOptionsUnavailableError() + } + if ( + typeof entry === 'number' && + (!Number.isFinite(entry) || protectedValues.contains(String(entry))) + ) { + throw new SelectorOptionsUnavailableError() + } + if (typeof entry === 'string') { + if (entry.length > MAX_OPTION_TEXT) throw new SelectorOptionsUnavailableError() + if (containsProtectedValue(entry, protectedValues, allowedExactValue)) { + throw new SelectorOptionsUnavailableError() + } + } + Object.defineProperty(meta, key, { + value: entry as SafeOptionMetaValue, + enumerable: true, + configurable: true, + writable: true, + }) + } + return meta +} + +function sanitizeOption( + value: unknown, + protectedValues: SelectorProtectedValues, + allowedExactValue?: string +): SafeSelectorOption { + if (!value || typeof value !== 'object' || Array.isArray(value)) { + throw new SelectorOptionsUnavailableError() + } + const option = value as { id?: unknown; label?: unknown; meta?: unknown } + const meta = sanitizeMeta(option.meta, protectedValues, allowedExactValue) + return { + id: requireSafeString(option.id, protectedValues, allowedExactValue), + label: requireSafeString(option.label, protectedValues, allowedExactValue), + ...(meta ? { meta } : {}), + } +} + +export function sanitizeSelectorResult( + result: ServerSelectorExecutionResult, + protectedValues: SelectorProtectedValues, + options?: SanitizeSelectorResultOptions +): SelectorExecutionResult { + if (result.kind === 'detail') { + return { + kind: 'detail', + item: result.item + ? sanitizeOption(result.item, protectedValues, options?.allowedDetailExactProtectedValue) + : null, + } + } + + if (result.items.length > MAX_SELECTOR_OPTIONS) throw new SelectorOptionsUnavailableError() + if (result.nextCursor !== undefined) { + requireSafeString(result.nextCursor, protectedValues) + } + return { + kind: 'list', + items: result.items.map((item) => sanitizeOption(item, protectedValues)), + ...(result.nextCursor ? { nextCursor: result.nextCursor } : {}), + ...(result.diagnostics?.truncated ? { truncated: true } : {}), + } +} diff --git a/apps/sim/lib/selectors/server/types.ts b/apps/sim/lib/selectors/server/types.ts new file mode 100644 index 00000000000..4249782a7e9 --- /dev/null +++ b/apps/sim/lib/selectors/server/types.ts @@ -0,0 +1,208 @@ +import type { SessionPrincipal } from '@sim/auth/principal' +import type { CredentialAccessResult } from '@/lib/auth/credential-access' +import { MAX_SELECTOR_OPTIONS } from '@/lib/selectors/limits' +import type { SelectorKey, ServerSelectorKey } from '@/lib/selectors/manifest' +import type { + SafeSelectorOption, + SelectorContext, + SelectorExecutionResult, + SelectorRequest, + SelectorScope, +} from '@/lib/selectors/types' + +export type SelectorDestinationPolicy = 'fixed' | 'credential-bound' | 'user-controlled' + +export type SelectorProtectedValueKind = 'secret' | 'reference' + +/** + * The service whose API a selector actually reaches. + * + * `serviceIds` names which *credentials* a selector accepts, which is not the + * same question as which *resource* it reads. `google.drive` accepts a Drive, + * Docs, Sheets or Forms connection because all four carry Drive scope, but it + * only ever calls the Drive API. Judging the integration allowlist against the + * accepted set let a group that permits `google_sheets_v2` and excludes + * `google_drive` read Drive through it. + * + * Required whenever `serviceIds` names more than one service, and must be one + * of them; `lib/selectors/manifest.test.ts` pins both. A single-service + * declaration is its own resource and omits it. + */ +export type SelectorCredentialPolicy = + | { + kind: 'stored' + field: 'oauthCredential' + serviceIds: readonly string[] + resourceServiceId?: string + } + | { + kind: 'stored-or-fixed-token' + field: 'oauthCredential' + serviceIds: readonly string[] + tokenPrefixes: readonly string[] + resourceServiceId?: string + } + +export interface AuthorizedSelectorCredential { + suppliedId: string + access?: CredentialAccessResult + fixedToken?: string + /** Trusted provider id loaded during server-side credential binding. */ + providerId?: string + /** Cancels only this selector's wait for shared credential resolution. */ + signal?: AbortSignal +} + +export interface SelectorProtectedValues { + add(value: string | null | undefined, kind?: SelectorProtectedValueKind): void + contains(value: string): boolean + containsExceptExact(value: string, allowedExactValue: string): boolean +} + +export interface ResolvedSelectorReference { + field: string + name: string + scope: 'personal' | 'workspace' + visible: boolean +} + +export interface ExecuteServerSelectorArgs { + selectorKey: ServerSelectorKey + context: SelectorContext + request: SelectorRequest + scope: SelectorScope + workspaceId: string + principal: SessionPrincipal + requesterUserId: string + credential?: AuthorizedSelectorCredential + references: ReadonlyMap + signal?: AbortSignal + protectedValues: SelectorProtectedValues + recordCredentialUse?: (providerId: string) => void +} + +export interface SelectorServerDiagnostics { + truncated?: { + reason: 'provider-cap' + limit?: number + pages?: number + } +} + +export type ServerSelectorExecutionResult = SelectorExecutionResult & { + diagnostics?: SelectorServerDiagnostics +} + +export interface PreparedSelectorDestination { + kind: Exclude + prepare(args: ExecuteServerSelectorArgs): Promise +} + +export interface ServerSelectorAttachment { + credential?: SelectorCredentialPolicy + /** + * The block type(s) whose integration this selector's API belongs to, for a + * selector the OAuth credential catalog cannot identify. + * + * The integration gate normally derives the block type from the credential + * policy's service ids. Two shapes defeat that: a selector authenticated from + * raw context fields rather than a stored connection (CloudWatch's AWS keys, + * IMAP's host and password) declares no policy at all, and an API-key + * integration (Snowflake, NetSuite, Harmonic) owns no OAuth catalog entry, so + * its service id maps to nothing. Both still reach a third-party API with the + * caller's credentials, so both must name their integration here. Internal + * selectors — the ones reading only Sim's own workspace data — name none, and + * that is what leaves them ungated. + */ + integrationBlockTypes?: readonly string[] + destination: 'fixed' | PreparedSelectorDestination + auditCredentialUse?: boolean + execute( + args: ExecuteServerSelectorArgs, + preparedDestination?: unknown + ): Promise +} + +export type ServerSelectorAttachmentMap = { + [P in K]: ServerSelectorAttachment +} + +export function listSelectorResult( + items: SafeSelectorOption[], + nextCursor?: string, + diagnostics?: SelectorServerDiagnostics +): ServerSelectorExecutionResult { + const overBudget = items.length > MAX_SELECTOR_OPTIONS + const boundedItems = overBudget ? items.slice(0, MAX_SELECTOR_OPTIONS) : items + const boundedDiagnostics = overBudget + ? { + ...diagnostics, + truncated: { + ...diagnostics?.truncated, + reason: 'provider-cap' as const, + limit: MAX_SELECTOR_OPTIONS, + }, + } + : diagnostics + return { + kind: 'list', + items: boundedItems, + ...(nextCursor ? { nextCursor } : {}), + ...(boundedDiagnostics ? { diagnostics: boundedDiagnostics } : {}), + } +} + +export function detailSelectorResult(item: SafeSelectorOption | null): SelectorExecutionResult { + return { kind: 'detail', item } +} + +export function definePreparedSelectorAttachment(input: { + credential?: SelectorCredentialPolicy + integrationBlockTypes?: readonly string[] + destination: { + kind: Exclude + prepare(args: ExecuteServerSelectorArgs): Promise + } + auditCredentialUse?: boolean + execute( + args: ExecuteServerSelectorArgs, + preparedDestination: TPrepared + ): Promise +}): ServerSelectorAttachment { + return { + ...(input.credential ? { credential: input.credential } : {}), + ...(input.integrationBlockTypes ? { integrationBlockTypes: input.integrationBlockTypes } : {}), + destination: { + kind: input.destination.kind, + prepare: input.destination.prepare, + }, + ...(input.auditCredentialUse ? { auditCredentialUse: true } : {}), + execute: async (args, preparedDestination) => + input.execute( + args, + preparedDestination === undefined + ? await input.destination.prepare(args) + : (preparedDestination as TPrepared) + ), + } +} + +export function requireListRequest( + selectorKey: SelectorKey, + request: SelectorRequest +): Extract { + if (request.kind !== 'list') { + throw new Error(`Selector ${selectorKey} received an unsupported detail request`) + } + return request +} + +export function requireDetailRequest( + selectorKey: SelectorKey, + request: SelectorRequest +): Extract { + if (request.kind !== 'detail') { + throw new Error(`Selector ${selectorKey} received an unsupported list request`) + } + return request +} diff --git a/apps/sim/lib/selectors/types.ts b/apps/sim/lib/selectors/types.ts new file mode 100644 index 00000000000..ad644b8946b --- /dev/null +++ b/apps/sim/lib/selectors/types.ts @@ -0,0 +1,127 @@ +import type { ComponentType } from 'react' + +export const selectorContextKeys = [ + 'oauthCredential', + 'domain', + 'teamId', + 'projectId', + 'knowledgeBaseId', + 'planId', + 'mimeType', + 'fileId', + 'siteId', + 'collectionId', + 'spreadsheetId', + 'driveId', + 'excludeWorkflowId', + 'baseId', + 'datasetId', + 'serviceDeskId', + 'impersonateUserEmail', + 'boardId', + 'spaceId', + 'listSpaceId', + 'folderId', + 'awsAccessKeyId', + 'awsSecretAccessKey', + 'awsRegion', + 'logGroupName', + 'tableId', + 'jobId', + 'database', + 'schema', + 'orgId', + 'workspaceSlug', + 'objectType', + 'customObjectTypeId', + 'pipelineId', + 'environmentType', + 'credentialGroupId', + 'language', + 'host', + 'port', + 'secure', + 'username', + 'password', +] as const + +export type SelectorContextKey = (typeof selectorContextKeys)[number] +export type SelectorContext = Partial> + +export type SelectorClassification = 'local' | 'internal-server' | 'provider-server' +export type SelectorScopeKind = 'workflow' | 'workspace' +export type SelectorListMode = 'flat' | 'paginated' + +export interface SelectorReadiness { + all?: readonly SelectorContextKey[] + any?: readonly SelectorContextKey[] +} + +export interface SelectorManifestEntry { + classification: SelectorClassification + context: { + allowed: readonly SelectorContextKey[] + readiness?: SelectorReadiness + sensitive?: readonly SelectorContextKey[] + /** Active input aliases that project into a canonical wire-context field. */ + sourceFields?: Partial> + } + scopeKinds: readonly SelectorScopeKind[] + listMode: SelectorListMode + supportsSearch: boolean + supportsDetail: boolean + resolvesUnknownIds: boolean + staleTime: number +} + +export type SafeOptionMetaValue = string | number | boolean | null +export type SafeOptionMeta = Record + +export interface SafeSelectorOption { + id: string + label: string + meta?: SafeOptionMeta +} + +export interface SelectorOption extends SafeSelectorOption { + icon?: ComponentType<{ className?: string }> +} + +export interface SelectorPage { + items: SelectorOption[] + nextCursor?: string +} + +export type SelectorScope = + | { + kind: 'workflow' + workflowId: string + workspaceId?: string + } + | { + kind: 'workspace' + workspaceId: string + } + +export type SelectorRequest = + | { + kind: 'list' + search?: string + cursor?: string + } + | { + kind: 'detail' + id: string + } + +export type SelectorExecutionResult = + | { + kind: 'list' + items: SafeSelectorOption[] + nextCursor?: string + truncated?: boolean + } + | { + kind: 'detail' + item: SafeSelectorOption | null + } diff --git a/apps/sim/lib/settings/application/workspace-section-access.test.ts b/apps/sim/lib/settings/application/workspace-section-access.test.ts new file mode 100644 index 00000000000..14d53c9fa57 --- /dev/null +++ b/apps/sim/lib/settings/application/workspace-section-access.test.ts @@ -0,0 +1,243 @@ +/** + * @vitest-environment node + */ +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const mocks = vi.hoisted(() => ({ + canOpenOrganizationSettingsSection: vi.fn(), + checkWorkspaceAccess: vi.fn(), + getWorkspaceOwnerSubscriptionAccess: vi.fn(), + isCredentialGroupsAvailable: vi.fn(), + isCustomBlocksEligibleForOrganization: vi.fn(), + isForkingAvailableForWorkspace: vi.fn(), + isOrganizationOnEnterprisePlan: vi.fn(), + isOrganizationSettingsSectionAvailable: vi.fn(), + isPlatformAdmin: vi.fn(), + resolveVerifiedUserAccessControlContext: vi.fn(), + resolveWorkspaceNavigation: vi.fn(), +})) + +vi.mock('@/components/settings/navigation', () => ({ + getOrganizationSettingsFeatures: vi.fn((hasEnterprisePlan: boolean) => ({ hasEnterprisePlan })), + isOrganizationSettingsSectionAvailable: mocks.isOrganizationSettingsSectionAvailable, + resolveWorkspaceNavigation: mocks.resolveWorkspaceNavigation, + UNIFIED_TO_ORGANIZATION_SECTION: { + organization: 'members', + billing: 'billing', + 'access-control': 'access-control', + }, + UNIFIED_TO_WORKSPACE_SECTION: { + secrets: 'secrets', + 'credential-groups': 'credential-groups', + forks: 'forks', + 'custom-blocks': 'custom-blocks', + }, + workspaceSectionUsesPermissionConfig: vi.fn((section: string) => + ['secrets', 'api-keys', 'inbox', 'mcp', 'custom-tools'].includes(section) + ), +})) +vi.mock('@/lib/billing/core/workspace-access', () => ({ + getWorkspaceOwnerSubscriptionAccess: mocks.getWorkspaceOwnerSubscriptionAccess, +})) +vi.mock('@/lib/billing/core/subscription', () => ({ + isOrganizationOnEnterprisePlan: mocks.isOrganizationOnEnterprisePlan, +})) +vi.mock('@/lib/credential-groups/availability', () => ({ + isCredentialGroupsAvailable: mocks.isCredentialGroupsAvailable, +})) +vi.mock('@/lib/core/config/env-flags', () => ({ isBillingEnabled: true, isHosted: true })) +vi.mock('@/lib/organizations/settings-access', () => ({ + canOpenOrganizationSettingsSection: mocks.canOpenOrganizationSettingsSection, +})) +vi.mock('@/lib/permissions/super-user', () => ({ isPlatformAdmin: mocks.isPlatformAdmin })) +vi.mock('@/lib/workflows/custom-blocks/operations', () => ({ + isCustomBlocksEligibleForOrganization: mocks.isCustomBlocksEligibleForOrganization, +})) +vi.mock('@/lib/workspaces/permissions/utils', () => ({ + checkWorkspaceAccess: mocks.checkWorkspaceAccess, +})) +vi.mock('@/ee/access-control/utils/permission-check', () => ({ + resolveVerifiedUserAccessControlContext: mocks.resolveVerifiedUserAccessControlContext, +})) +vi.mock('@/ee/workspace-forking/lib/lineage/authz', () => ({ + isForkingAvailableForWorkspace: mocks.isForkingAvailableForWorkspace, +})) + +import { authorizeWorkspaceSettingsSection } from '@/lib/settings/application/workspace-section-access' + +const PERSONAL_ACCESS = { + exists: true, + hasAccess: true, + permission: 'admin', + workspace: { + id: 'workspace-1', + organizationId: null, + billedAccountUserId: 'owner-1', + }, +} + +const ORGANIZATION_ACCESS = { + ...PERSONAL_ACCESS, + workspace: { + ...PERSONAL_ACCESS.workspace, + organizationId: 'organization-1', + }, +} + +function authorize(section: Parameters[0]['section']) { + return authorizeWorkspaceSettingsSection({ + workspaceId: 'workspace-1', + userId: 'viewer-1', + section, + }) +} + +describe('authorizeWorkspaceSettingsSection', () => { + beforeEach(() => { + vi.clearAllMocks() + mocks.checkWorkspaceAccess.mockResolvedValue(PERSONAL_ACCESS) + mocks.getWorkspaceOwnerSubscriptionAccess.mockResolvedValue({ isEnterprise: true }) + mocks.isCredentialGroupsAvailable.mockResolvedValue(true) + mocks.isCustomBlocksEligibleForOrganization.mockResolvedValue(true) + mocks.isForkingAvailableForWorkspace.mockResolvedValue(true) + mocks.isOrganizationOnEnterprisePlan.mockResolvedValue(true) + mocks.isOrganizationSettingsSectionAvailable.mockReturnValue(true) + mocks.isPlatformAdmin.mockResolvedValue(true) + mocks.canOpenOrganizationSettingsSection.mockResolvedValue(true) + mocks.resolveVerifiedUserAccessControlContext.mockResolvedValue({ config: {} }) + mocks.resolveWorkspaceNavigation.mockReturnValue([{ id: 'secrets' }]) + }) + + it('conceals missing and inaccessible workspaces before section-specific reads', async () => { + mocks.checkWorkspaceAccess.mockResolvedValue({ + exists: true, + hasAccess: false, + permission: null, + workspace: PERSONAL_ACCESS.workspace, + }) + + await expect(authorize('billing')).resolves.toEqual({ + allowed: false, + disposition: 'not-found', + }) + expect(mocks.canOpenOrganizationSettingsSection).not.toHaveBeenCalled() + expect(mocks.getWorkspaceOwnerSubscriptionAccess).not.toHaveBeenCalled() + }) + + it('opens ordinary sections from workspace access alone', async () => { + await expect(authorize('general')).resolves.toEqual({ allowed: true }) + + expect(mocks.getWorkspaceOwnerSubscriptionAccess).not.toHaveBeenCalled() + expect(mocks.canOpenOrganizationSettingsSection).not.toHaveBeenCalled() + expect(mocks.resolveVerifiedUserAccessControlContext).not.toHaveBeenCalled() + expect(mocks.isPlatformAdmin).not.toHaveBeenCalled() + }) + + it('conceals platform sections from non-platform admins', async () => { + mocks.isPlatformAdmin.mockResolvedValue(false) + + await expect(authorize('admin')).resolves.toEqual({ + allowed: false, + disposition: 'not-found', + }) + expect(mocks.isPlatformAdmin).toHaveBeenCalledWith('viewer-1') + }) + + it('loads canonical access-control policy for affected organization sections', async () => { + mocks.checkWorkspaceAccess.mockResolvedValue(ORGANIZATION_ACCESS) + mocks.resolveVerifiedUserAccessControlContext.mockResolvedValue({ + config: { hideSecretsTab: true }, + }) + mocks.resolveWorkspaceNavigation.mockReturnValue([]) + + await expect(authorize('secrets')).resolves.toEqual({ + allowed: false, + disposition: 'redirect-general', + }) + expect(mocks.getWorkspaceOwnerSubscriptionAccess).not.toHaveBeenCalled() + expect(mocks.resolveVerifiedUserAccessControlContext).toHaveBeenCalledWith( + 'viewer-1', + 'workspace-1', + 'organization-1' + ) + expect(mocks.resolveWorkspaceNavigation).toHaveBeenCalledWith( + expect.objectContaining({ permissionConfig: { hideSecretsTab: true } }) + ) + }) + + it('resolves environment access-control policy for the same section in a personal workspace', async () => { + await authorize('secrets') + + expect(mocks.getWorkspaceOwnerSubscriptionAccess).not.toHaveBeenCalled() + expect(mocks.resolveVerifiedUserAccessControlContext).toHaveBeenCalledWith( + 'viewer-1', + 'workspace-1', + null + ) + }) + + it('enforces canonical permission config independently of billing subscription state', async () => { + mocks.checkWorkspaceAccess.mockResolvedValue(ORGANIZATION_ACCESS) + mocks.getWorkspaceOwnerSubscriptionAccess.mockResolvedValue({ isEnterprise: false }) + mocks.resolveVerifiedUserAccessControlContext.mockResolvedValue({ + entitled: true, + config: { hideSecretsTab: true }, + }) + mocks.resolveWorkspaceNavigation.mockReturnValue([]) + + await expect(authorize('secrets')).resolves.toEqual({ + allowed: false, + disposition: 'redirect-general', + }) + expect(mocks.getWorkspaceOwnerSubscriptionAccess).not.toHaveBeenCalled() + }) + + it('resolves the exact entitlement source only for gated workspace sections', async () => { + mocks.resolveWorkspaceNavigation.mockReturnValue([{ id: 'credential-groups' }]) + await authorize('credential-groups') + expect(mocks.isCredentialGroupsAvailable).toHaveBeenCalledWith({ + workspaceId: 'workspace-1', + ownerBilling: { isEnterprise: true }, + }) + expect(mocks.isForkingAvailableForWorkspace).not.toHaveBeenCalled() + + mocks.resolveWorkspaceNavigation.mockReturnValue([{ id: 'forks' }]) + await authorize('forks') + expect(mocks.isForkingAvailableForWorkspace).toHaveBeenCalledWith(null, 'viewer-1') + + mocks.checkWorkspaceAccess.mockResolvedValue(ORGANIZATION_ACCESS) + mocks.resolveWorkspaceNavigation.mockReturnValue([{ id: 'custom-blocks' }]) + await authorize('custom-blocks') + expect(mocks.isCustomBlocksEligibleForOrganization).toHaveBeenCalledWith('organization-1') + }) + + it('allows personal billing only to the billed account owner', async () => { + await expect(authorize('billing')).resolves.toEqual({ + allowed: false, + disposition: 'redirect-general', + }) + + mocks.checkWorkspaceAccess.mockResolvedValue({ + ...PERSONAL_ACCESS, + workspace: { ...PERSONAL_ACCESS.workspace, billedAccountUserId: 'viewer-1' }, + }) + await expect(authorize('billing')).resolves.toEqual({ allowed: true }) + expect(mocks.canOpenOrganizationSettingsSection).not.toHaveBeenCalled() + }) + + it('requires current organization access and plan availability for enterprise sections', async () => { + mocks.checkWorkspaceAccess.mockResolvedValue(ORGANIZATION_ACCESS) + mocks.canOpenOrganizationSettingsSection.mockResolvedValue(false) + + await expect(authorize('access-control')).resolves.toEqual({ + allowed: false, + disposition: 'redirect-general', + }) + expect(mocks.canOpenOrganizationSettingsSection).toHaveBeenCalledWith( + 'organization-1', + 'viewer-1', + 'access-control' + ) + expect(mocks.isOrganizationOnEnterprisePlan).toHaveBeenCalledWith('organization-1') + }) +}) diff --git a/apps/sim/lib/settings/application/workspace-section-access.ts b/apps/sim/lib/settings/application/workspace-section-access.ts new file mode 100644 index 00000000000..28227608393 --- /dev/null +++ b/apps/sim/lib/settings/application/workspace-section-access.ts @@ -0,0 +1,138 @@ +import { + getOrganizationSettingsFeatures, + isOrganizationSettingsSectionAvailable, + resolveWorkspaceNavigation, + UNIFIED_TO_ORGANIZATION_SECTION, + UNIFIED_TO_WORKSPACE_SECTION, + type UnifiedSettingsSection, + type WorkspaceSettingsSection, + workspaceSectionUsesPermissionConfig, +} from '@/components/settings/navigation' +import { isOrganizationOnEnterprisePlan } from '@/lib/billing/core/subscription' +import { getWorkspaceOwnerSubscriptionAccess } from '@/lib/billing/core/workspace-access' +import { isBillingEnabled, isHosted } from '@/lib/core/config/env-flags' +import { isCredentialGroupsAvailable } from '@/lib/credential-groups/availability' +import { canOpenOrganizationSettingsSection } from '@/lib/organizations/settings-access' +import { isPlatformAdmin } from '@/lib/permissions/super-user' +import { isCustomBlocksEligibleForOrganization } from '@/lib/workflows/custom-blocks/operations' +import { checkWorkspaceAccess } from '@/lib/workspaces/permissions/utils' +import { resolveVerifiedUserAccessControlContext } from '@/ee/access-control/utils/permission-check' +import { isForkingAvailableForWorkspace } from '@/ee/workspace-forking/lib/lineage/authz' + +export type WorkspaceSettingsSectionAccess = + | { allowed: true } + | { allowed: false; disposition: 'not-found' | 'redirect-general' } + +interface AuthorizeWorkspaceSettingsSectionInput { + workspaceId: string + userId: string + section: UnifiedSettingsSection +} + +async function canOpenWorkspaceSection( + section: WorkspaceSettingsSection, + input: AuthorizeWorkspaceSettingsSectionInput, + workspace: { + organizationId: string | null + }, + permission: NonNullable>['permission']> +): Promise { + const needsOwnerBilling = section === 'credential-groups' + const ownerBilling = needsOwnerBilling + ? await getWorkspaceOwnerSubscriptionAccess(input.workspaceId) + : null + + const [accessControl, credentialGroupsAvailable, forksAvailable, customBlocksAvailable] = + await Promise.all([ + workspaceSectionUsesPermissionConfig(section) + ? resolveVerifiedUserAccessControlContext( + input.userId, + input.workspaceId, + workspace.organizationId + ) + : null, + section === 'credential-groups' && ownerBilling + ? isCredentialGroupsAvailable({ workspaceId: input.workspaceId, ownerBilling }) + : false, + section === 'forks' + ? isForkingAvailableForWorkspace(workspace.organizationId, input.userId) + : false, + section === 'custom-blocks' && workspace.organizationId + ? isCustomBlocksEligibleForOrganization(workspace.organizationId) + : false, + ]) + + const navigation = resolveWorkspaceNavigation({ + permission, + permissionConfig: accessControl?.config ?? {}, + entitlements: { + byok: isHosted, + credentialGroups: credentialGroupsAvailable, + inbox: true, + customBlocks: customBlocksAvailable, + forks: forksAvailable, + sandboxes: true, + }, + }) + return navigation.some((item) => item.id === section) +} + +async function canOpenOrganizationSection( + input: AuthorizeWorkspaceSettingsSectionInput, + workspace: { + organizationId: string | null + billedAccountUserId: string + } +): Promise { + const organizationSection = UNIFIED_TO_ORGANIZATION_SECTION[input.section] + if (!organizationSection) return true + if (!isBillingEnabled && (input.section === 'billing' || input.section === 'organization')) { + return false + } + if (!workspace.organizationId) { + return input.section === 'billing' && workspace.billedAccountUserId === input.userId + } + + const needsEnterprisePlan = organizationSection !== 'members' && organizationSection !== 'billing' + const [canOpenSection, isEnterpriseOrganization] = await Promise.all([ + canOpenOrganizationSettingsSection(workspace.organizationId, input.userId, organizationSection), + needsEnterprisePlan + ? isOrganizationOnEnterprisePlan(workspace.organizationId) + : Promise.resolve(false), + ]) + return ( + canOpenSection && + isOrganizationSettingsSectionAvailable( + organizationSection, + getOrganizationSettingsFeatures(needsEnterprisePlan && isEnterpriseOrganization) + ) + ) +} + +export async function authorizeWorkspaceSettingsSection( + input: AuthorizeWorkspaceSettingsSectionInput +): Promise { + const requiresPlatformAdmin = input.section === 'admin' || input.section === 'mothership' + const [access, viewerIsPlatformAdmin] = await Promise.all([ + checkWorkspaceAccess(input.workspaceId, input.userId), + requiresPlatformAdmin ? isPlatformAdmin(input.userId) : Promise.resolve(false), + ]) + if (!access.exists || !access.hasAccess || !access.workspace || !access.permission) { + return { allowed: false, disposition: 'not-found' } + } + if (requiresPlatformAdmin && !viewerIsPlatformAdmin) { + return { allowed: false, disposition: 'not-found' } + } + + const workspaceSection = UNIFIED_TO_WORKSPACE_SECTION[input.section] + if ( + workspaceSection && + !(await canOpenWorkspaceSection(workspaceSection, input, access.workspace, access.permission)) + ) { + return { allowed: false, disposition: 'redirect-general' } + } + if (!(await canOpenOrganizationSection(input, access.workspace))) { + return { allowed: false, disposition: 'redirect-general' } + } + return { allowed: true } +} diff --git a/apps/sim/lib/settings/prefetch-current-user-settings.ts b/apps/sim/lib/settings/prefetch-current-user-settings.ts new file mode 100644 index 00000000000..139f714a227 --- /dev/null +++ b/apps/sim/lib/settings/prefetch-current-user-settings.ts @@ -0,0 +1,29 @@ +import type { QueryClient } from '@tanstack/react-query' +import { getUserSettingsContract } from '@/lib/api/contracts/user' +import { internalSessionAuth } from '@/lib/api/server/routes/internal-json-route' +import { getCurrentUserSettingsUseCase } from '@/lib/users/application/read-current-user' +import { + GENERAL_SETTINGS_STALE_TIME, + generalSettingsKeys, + mapGeneralSettingsResponse, +} from '@/hooks/queries/current-user-data' + +type GetPrincipal = () => ReturnType + +export function prefetchCurrentUserSettings( + queryClient: QueryClient, + getPrincipal: GetPrincipal = () => internalSessionAuth.authenticate() +) { + return queryClient.prefetchQuery({ + queryKey: generalSettingsKeys.settings(), + queryFn: async () => { + const settings = await getCurrentUserSettingsUseCase.execute({ + principal: await getPrincipal(), + input: {}, + }) + const response = getUserSettingsContract.response.schema.parse({ data: settings }) + return mapGeneralSettingsResponse(response.data) + }, + staleTime: GENERAL_SETTINGS_STALE_TIME, + }) +} diff --git a/apps/sim/lib/skills/application/operations.test.ts b/apps/sim/lib/skills/application/operations.test.ts index 5669aadb134..a9152130d80 100644 --- a/apps/sim/lib/skills/application/operations.test.ts +++ b/apps/sim/lib/skills/application/operations.test.ts @@ -2,7 +2,25 @@ * @vitest-environment node */ import { requirePrincipalSubjectUserId } from '@sim/auth/principal' -import { describe, expect, it } from 'vitest' +import { permissionGroupScopeMock, permissionGroupScopeMockFns } from '@sim/testing' +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const mocks = vi.hoisted(() => ({ + resolvePermission: vi.fn(), +})) + +const resolveGroupConfigMock = permissionGroupScopeMockFns.mockResolvePermissionGroupConfig + +vi.mock('@sim/platform-authz/workspace', () => ({ + permissionSatisfies: () => true, + resolveEffectiveWorkspacePermission: mocks.resolvePermission, +})) + +vi.mock('@/lib/permission-groups/config-scope.server', () => permissionGroupScopeMock) + +import type { WorkspaceOperation } from '@/lib/core/application' +import { authorizeWorkspaceOperation, PermissionGroupCapabilityError } from '@/lib/core/application' +import { DEFAULT_PERMISSION_GROUP_CONFIG } from '@/lib/permission-groups/fields' import { skillOperations } from '@/lib/skills/application/operations' /** @@ -100,3 +118,46 @@ describe('skill operation registry', () => { expect(new Set(ids).size).toBe(ids.length) }) }) + +const sessionPrincipal = { kind: 'session', userId: 'user-1', sessionId: 'session-1' } as const +const context = { + workspaceId: 'workspace-1', + workspaceOrganizationId: 'organization-1', + allowPersonalApiKeys: true, +} + +/** + * The declaration is only half the gate. These call the funnel so a capability + * cannot be declared on the operations and then read by nothing. + */ +describe('skill operations under a group that blocks skills', () => { + beforeEach(() => { + vi.clearAllMocks() + mocks.resolvePermission.mockResolvedValue('admin') + }) + + it('refuses authoring and editor grants, not only loading', async () => { + resolveGroupConfigMock.mockResolvedValue({ + ...DEFAULT_PERMISSION_GROUP_CONFIG, + disableSkills: true, + }) + + for (const operation of Object.values(skillOperations)) { + await expect( + authorizeWorkspaceOperation(sessionPrincipal, operation as WorkspaceOperation, context), + operation.id + ).rejects.toBeInstanceOf(PermissionGroupCapabilityError) + } + }) + + it('allows them all when the group withholds nothing', async () => { + resolveGroupConfigMock.mockResolvedValue(DEFAULT_PERMISSION_GROUP_CONFIG) + + for (const operation of Object.values(skillOperations)) { + await expect( + authorizeWorkspaceOperation(sessionPrincipal, operation as WorkspaceOperation, context), + operation.id + ).resolves.toBeUndefined() + } + }) +}) diff --git a/apps/sim/lib/skills/application/operations.ts b/apps/sim/lib/skills/application/operations.ts index 32ed520ccad..06cb71eb7fd 100644 --- a/apps/sim/lib/skills/application/operations.ts +++ b/apps/sim/lib/skills/application/operations.ts @@ -36,35 +36,47 @@ const HUMAN_HTTP_SKILL_EDITOR_POLICY = { * act. Denying it keeps the whole lifecycle under one authorization model. * Pinned in `operations.test.ts`. */ +/** + * Every operation declares `skills.use`. The key reads "block agents from + * loading skills", and the skills a group's members author are exactly the ones + * their agents would load — so authoring, sharing, and editor grants are gated + * with execution rather than left as a side door that fills the workspace with + * skills the group may not run. + */ export const skillOperations = { list: defineWorkspaceOperation({ id: 'skills.list', minimumRole: 'read', workspaceApiKey: 'allow', + capability: 'skills.use', ...ALL_PRINCIPAL_POLICY, }), listAvailable: defineWorkspaceOperation({ id: 'skills.list_available', minimumRole: 'read', workspaceApiKey: 'deny', + capability: 'skills.use', ...HUMAN_PRINCIPAL_POLICY, }), read: defineWorkspaceOperation({ id: 'skills.read', minimumRole: 'read', workspaceApiKey: 'allow', + capability: 'skills.use', ...ALL_PRINCIPAL_POLICY, }), create: defineWorkspaceOperation({ id: 'skills.create', minimumRole: 'write', workspaceApiKey: 'deny', + capability: 'skills.use', ...HUMAN_PRINCIPAL_POLICY, }), update: defineWorkspaceOperation({ id: 'skills.update', minimumRole: 'read', workspaceApiKey: 'deny', + capability: 'skills.use', ...HUMAN_PRINCIPAL_POLICY, }), /** @@ -81,30 +93,35 @@ export const skillOperations = { id: 'skills.upsert', minimumRole: 'read', workspaceApiKey: 'deny', + capability: 'skills.use', ...HUMAN_PRINCIPAL_POLICY, }), delete: defineWorkspaceOperation({ id: 'skills.delete', minimumRole: 'read', workspaceApiKey: 'deny', + capability: 'skills.use', ...HUMAN_PRINCIPAL_POLICY, }), listEditors: defineWorkspaceOperation({ id: 'skills.editors.list', minimumRole: 'read', workspaceApiKey: 'allow', + capability: 'skills.use', ...HTTP_SKILL_EDITOR_READ_POLICY, }), grantEditor: defineWorkspaceOperation({ id: 'skills.editors.grant', minimumRole: 'read', workspaceApiKey: 'deny', + capability: 'skills.use', ...HUMAN_HTTP_SKILL_EDITOR_POLICY, }), revokeEditor: defineWorkspaceOperation({ id: 'skills.editors.revoke', minimumRole: 'read', workspaceApiKey: 'deny', + capability: 'skills.use', ...HUMAN_HTTP_SKILL_EDITOR_POLICY, }), } as const diff --git a/apps/sim/lib/table/__tests__/column-type-registry.test.ts b/apps/sim/lib/table/__tests__/column-type-registry.test.ts index 91ec369e31e..a63886a4a5c 100644 --- a/apps/sim/lib/table/__tests__/column-type-registry.test.ts +++ b/apps/sim/lib/table/__tests__/column-type-registry.test.ts @@ -10,6 +10,7 @@ * here. */ import { describe, expect, it } from 'vitest' +import { zonedWallClockToUtc } from '@/lib/core/utils/timezone' import type { ColumnType } from '@/lib/table/column-types' import { ALL_COLUMN_TYPES, @@ -121,6 +122,143 @@ describe('conversion write-back', () => { }) }) +describe('ttl columns', () => { + const column = { name: 'expires_at', type: 'ttl' } as ColumnDefinition + + it('stores integer epoch seconds while accepting date-shaped input', () => { + expect(COLUMN_TYPE_REGISTRY.ttl.coerce('2023-11-14T22:13:20Z', column)).toEqual({ + ok: true, + value: 1_700_000_000, + }) + expect(COLUMN_TYPE_REGISTRY.ttl.coerce(1_700_000_000, column)).toEqual({ + ok: true, + value: 1_700_000_000, + }) + expect(COLUMN_TYPE_REGISTRY.ttl.coerce('1700000000', column)).toEqual({ + ok: true, + value: 1_700_000_000, + }) + expect(COLUMN_TYPE_REGISTRY.ttl.coerce('2023-11-14T22:13:20.123Z', column)).toEqual({ + ok: true, + value: 1_700_000_001, + }) + expect(COLUMN_TYPE_REGISTRY.ttl.coerce('not-a-date', column)).toEqual({ ok: false }) + expect(COLUMN_TYPE_REGISTRY.ttl.coerce(1_700_000_000.5, column)).toEqual({ ok: false }) + }) + + it.each(['2023-02-29', '2023-02-29T12:00:00', '2023-02-29T12:00:00-05:00'])( + 'rejects a nonexistent ISO calendar input: %s', + (value) => { + expect(COLUMN_TYPE_REGISTRY.ttl.coerce(value, column, { timezone: 'UTC' })).toEqual({ + ok: false, + }) + } + ) + + it.each([ + ['2024-02-29', '2024-02-29T00:00:00Z'], + ['2024-02-29T12:00:00', '2024-02-29T12:00:00Z'], + ['2024-02-29T12:00:00-05:00', '2024-02-29T17:00:00Z'], + ])('accepts a valid leap-day ISO calendar input: %s', (value, expectedInstant) => { + expect(COLUMN_TYPE_REGISTRY.ttl.coerce(value, column, { timezone: 'UTC' })).toEqual({ + ok: true, + value: Math.floor(Date.parse(expectedInstant) / 1000), + }) + }) + + it('renders and edits epoch seconds as a date', () => { + expect(COLUMN_TYPE_REGISTRY.ttl.formatForDisplay(1_700_000_000, column)).toBe( + '11/14/2023 10:13:20 PM' + ) + expect(COLUMN_TYPE_REGISTRY.ttl.formatForInput(1_700_000_000, column)).toBe( + '2023-11-14T22:13:20Z' + ) + expect( + COLUMN_TYPE_REGISTRY.ttl.formatForInput(1_700_000_000, column, { + timezone: 'America/New_York', + }) + ).toBe('2023-11-14T17:13:20-05:00') + }) + + it('preserves the exact instant across both sides of a daylight-saving fold', () => { + expect( + COLUMN_TYPE_REGISTRY.ttl.formatForInput(1_699_162_200, column, { + timezone: 'America/New_York', + }) + ).toBe('2023-11-05T01:30:00-04:00') + expect( + COLUMN_TYPE_REGISTRY.ttl.formatForInput(1_699_165_800, column, { + timezone: 'America/New_York', + }) + ).toBe('2023-11-05T01:30:00-05:00') + }) + + it('matches the shared wall-clock resolver in every effective timezone', () => { + const wallClock = '2026-06-15T09:00:30' + for (const timezone of [ + 'UTC', + 'America/Los_Angeles', + 'America/New_York', + 'Asia/Kathmandu', + 'Australia/Lord_Howe', + ]) { + const expected = Math.floor(zonedWallClockToUtc(wallClock, timezone).getTime() / 1000) + expect(COLUMN_TYPE_REGISTRY.ttl.coerce(wallClock, column, { timezone })).toEqual({ + ok: true, + value: expected, + }) + } + }) + + it.each([ + ['Europe/Berlin', '2026-03-29T02:30'], + ['Australia/Lord_Howe', '2026-10-04T02:15'], + ])('coerces a %s spring-forward gap wall clock to the compatible epoch', (timezone, input) => { + const expected = Math.floor(zonedWallClockToUtc(input, timezone).getTime() / 1000) + expect(COLUMN_TYPE_REGISTRY.ttl.coerce(input, column, { timezone })).toEqual({ + ok: true, + value: expected, + }) + }) + + it('coerces a localized month-name gap input in the explicit workspace timezone', () => { + const timezone = 'America/New_York' + const expected = Math.floor(zonedWallClockToUtc('2026-03-08T02:30', timezone).getTime() / 1000) + expect(COLUMN_TYPE_REGISTRY.ttl.coerce('March 8, 2026 2:30 AM', column, { timezone })).toEqual({ + ok: true, + value: expected, + }) + }) + + it('rejects an impossible ISO expiration date', () => { + expect( + COLUMN_TYPE_REGISTRY.ttl.coerce('2026-02-30T12:00:00', column, { timezone: 'UTC' }) + ).toEqual({ ok: false }) + }) + + it('round-trips epoch seconds after the editor timezone changes', () => { + for (const seconds of [1_700_000_000, 1_699_162_200, 1_699_165_800]) { + for (const timezone of [ + 'UTC', + 'America/Los_Angeles', + 'America/New_York', + 'Asia/Kathmandu', + 'Australia/Lord_Howe', + ]) { + const editable = COLUMN_TYPE_REGISTRY.ttl.formatForInput(seconds, column, { timezone }) + expect(COLUMN_TYPE_REGISTRY.ttl.coerce(editable, column, { timezone })).toEqual({ + ok: true, + value: seconds, + }) + } + } + }) + + it('limits a table to one ttl column', () => { + expect(COLUMN_TYPE_REGISTRY.ttl.maxPerTable).toBe(1) + }) +}) + describe('intentional divergences from the pre-registry behavior', () => { // A differential run of the registry against the pre-refactor implementations // (55 values x 7 column shapes) found ZERO coercion differences and exactly diff --git a/apps/sim/lib/table/__tests__/service-filter-threading.test.ts b/apps/sim/lib/table/__tests__/service-filter-threading.test.ts index 52ae732e805..a7089a000f4 100644 --- a/apps/sim/lib/table/__tests__/service-filter-threading.test.ts +++ b/apps/sim/lib/table/__tests__/service-filter-threading.test.ts @@ -15,6 +15,8 @@ import { decodeCursor } from '@/lib/table/rows/cursor' import { buildFilterClause, buildSortClause } from '@/lib/table/sql' import type { ColumnDefinition, TableDefinition } from '@/lib/table/types' +const { mockFireTableTrigger } = vi.hoisted(() => ({ mockFireTableTrigger: vi.fn() })) + vi.mock('@/lib/table/sql', () => ({ buildFilterClause: vi.fn(() => sql`true`), buildSortClause: vi.fn(() => sql`true`), @@ -23,7 +25,7 @@ vi.mock('@/lib/table/sql', () => ({ })) vi.mock('@/lib/table/trigger', () => ({ - fireTableTrigger: vi.fn(), + fireTableTrigger: mockFireTableTrigger, })) vi.mock('@/lib/table/workflow-group-deps', () => ({ @@ -64,7 +66,9 @@ vi.mock('@/lib/table/validation', () => ({ })) import { + deleteRow, deleteRowsByFilter, + deleteRowsByIds, queryRows, requireTableRowIds, updateRowsByFilter, @@ -183,6 +187,104 @@ describe('service filter threading', () => { }) }) +describe('delete trigger dispatch', () => { + beforeEach(() => { + vi.clearAllMocks() + resetDbChainMock() + }) + + it('fires with the committed snapshot after deleting one row', async () => { + dbChainMockFns.returning.mockResolvedValueOnce([{ id: 'row-1', data: { name: 'Ada' } }]) + + await deleteRow(TABLE, 'row-1', 'req-delete-one') + + expect(mockFireTableTrigger).toHaveBeenCalledWith( + TABLE.id, + TABLE.workspaceId, + TABLE.name, + 'delete', + [{ id: 'row-1', data: { name: 'Ada' } }], + null, + TABLE.schema, + 'req-delete-one' + ) + }) + + it('returns after deleting one row without waiting for trigger dispatch', async () => { + dbChainMockFns.returning.mockResolvedValueOnce([{ id: 'row-1', data: { name: 'Ada' } }]) + let releaseTrigger: (() => void) | undefined + const triggerPending = new Promise((resolve) => { + releaseTrigger = resolve + }) + mockFireTableTrigger.mockReturnValueOnce(triggerPending) + const deletion = deleteRow(TABLE, 'row-1', 'req-delete-one') + const onDeleteSettled = vi.fn() + void deletion.then(onDeleteSettled) + + await vi.waitFor(() => expect(mockFireTableTrigger).toHaveBeenCalledTimes(1)) + await Promise.resolve() + + try { + expect(onDeleteSettled).toHaveBeenCalledTimes(1) + } finally { + releaseTrigger?.() + await deletion + } + }) + + it('fires once with every committed snapshot in an ID batch', async () => { + dbChainMockFns.returning.mockResolvedValueOnce([ + { id: 'row-1', data: { name: 'Ada' } }, + { id: 'row-2', data: { name: 'Grace' } }, + ]) + + await deleteRowsByIds( + TABLE, + { tableId: TABLE.id, workspaceId: TABLE.workspaceId, rowIds: ['row-1', 'row-2'] }, + 'req-delete-many' + ) + + expect(mockFireTableTrigger).toHaveBeenCalledWith( + TABLE.id, + TABLE.workspaceId, + TABLE.name, + 'delete', + [ + { id: 'row-1', data: { name: 'Ada' } }, + { id: 'row-2', data: { name: 'Grace' } }, + ], + null, + TABLE.schema, + 'req-delete-many' + ) + }) + + it('dispatches byte-bounded ID-delete snapshots before loading the next batch', async () => { + setEnv({ + TABLE_MAX_ROW_SIZE_BYTES: TABLE_LIMITS.DELETE_SNAPSHOT_BATCH_MAX_BYTES * 2, + }) + dbChainMockFns.returning + .mockResolvedValueOnce([{ id: 'row-1', data: { name: 'Ada' } }]) + .mockResolvedValueOnce([{ id: 'row-2', data: { name: 'Grace' } }]) + + try { + await deleteRowsByIds( + TABLE, + { tableId: TABLE.id, workspaceId: TABLE.workspaceId, rowIds: ['row-1', 'row-2'] }, + 'req-delete-bounded' + ) + } finally { + setEnv({ TABLE_MAX_ROW_SIZE_BYTES: undefined }) + } + + expect(mockFireTableTrigger).toHaveBeenCalledTimes(2) + expect(mockFireTableTrigger.mock.calls[0][4]).toEqual([{ id: 'row-1', data: { name: 'Ada' } }]) + expect(mockFireTableTrigger.mock.calls[1][4]).toEqual([ + { id: 'row-2', data: { name: 'Grace' } }, + ]) + }) +}) + describe('bulk update/delete limited-subset ordering', () => { beforeEach(() => { vi.clearAllMocks() diff --git a/apps/sim/lib/table/__tests__/trigger.test.ts b/apps/sim/lib/table/__tests__/trigger.test.ts index 64b5c4afc5e..fc04df054f1 100644 --- a/apps/sim/lib/table/__tests__/trigger.test.ts +++ b/apps/sim/lib/table/__tests__/trigger.test.ts @@ -60,7 +60,7 @@ interface Payload { function webhookEntry(config: Record = {}) { return { webhook: { id: 'wh_1', providerConfig: { tableId: 'tbl_1', eventType: 'insert', ...config } }, - workflow: { id: 'wf_1' }, + workflow: { id: 'wf_1', workspaceId: 'ws_1' }, } } @@ -69,12 +69,13 @@ function firedPayloads(): Payload[] { } async function fire( - eventType: 'insert' | 'update', + eventType: 'insert' | 'update' | 'delete', data: RowData, oldRows: Map | null = null ) { await fireTableTrigger( 'tbl_1', + 'ws_1', 'Issues', eventType, [{ id: 'row_1', data } as never], @@ -166,6 +167,19 @@ describe('fireTableTrigger — payload shape', () => { const [payload] = firedPayloads() expect(payload.changedColumns).toEqual(['Status']) }) + + it('emits the deleted row snapshot as the event row and previous row', async () => { + mockFetchActiveWebhooks.mockResolvedValue([webhookEntry({ eventType: 'delete' })]) + + await fire('delete', { col_title: 'Removed issue', col_status: 'opt_closed' }) + + const [payload] = firedPayloads() + const deletedRow = { Title: 'Removed issue', Status: 'Closed' } + expect(payload.rawRow).toEqual(deletedRow) + expect(payload.row).toEqual({ ...deletedRow, Tags: null }) + expect(payload.previousRow).toEqual(deletedRow) + expect(payload.changedColumns).toEqual([]) + }) }) describe('fireTableTrigger — gating', () => { @@ -180,6 +194,14 @@ describe('fireTableTrigger — gating', () => { expect(mockProcessPolledWebhookEvent).not.toHaveBeenCalled() }) + it('fires nothing for a workflow in a different workspace', async () => { + mockFetchActiveWebhooks.mockResolvedValue([ + { ...webhookEntry(), workflow: { id: 'wf_other', workspaceId: 'ws_other' } }, + ]) + await fire('insert', { col_title: 'x' }) + expect(mockProcessPolledWebhookEvent).not.toHaveBeenCalled() + }) + it('fires nothing when the event type does not match', async () => { mockFetchActiveWebhooks.mockResolvedValue([webhookEntry({ eventType: 'update' })]) await fire('insert', { col_title: 'x' }) diff --git a/apps/sim/lib/table/__tests__/validation.test.ts b/apps/sim/lib/table/__tests__/validation.test.ts index 9d698c9d96b..04b70ce4af9 100644 --- a/apps/sim/lib/table/__tests__/validation.test.ts +++ b/apps/sim/lib/table/__tests__/validation.test.ts @@ -195,6 +195,18 @@ describe('Validation', () => { expect(result.errors).toContain('Duplicate column names found') }) + it('rejects more than one TTL column', () => { + const result = validateTableSchema({ + columns: [ + { name: 'expires_at', type: 'ttl' }, + { name: 'delete_at', type: 'ttl' }, + ], + } as TableSchema) + + expect(result.valid).toBe(false) + expect(result.errors).toContain('A table can have at most 1 Expiration column') + }) + it('should reject null schema', () => { const result = validateTableSchema(null as unknown as TableSchema) expect(result.valid).toBe(false) diff --git a/apps/sim/lib/table/application/copilot-bulk-rows.ts b/apps/sim/lib/table/application/copilot-bulk-rows.ts index 76c55db7350..0c29ed56b06 100644 --- a/apps/sim/lib/table/application/copilot-bulk-rows.ts +++ b/apps/sim/lib/table/application/copilot-bulk-rows.ts @@ -3,6 +3,7 @@ import { resolvePrincipalAttribution } from '@sim/auth/principal' import { createLogger } from '@sim/logger' import { getErrorMessage } from '@sim/utils/errors' import { generateId } from '@sim/utils/id' +import { capabilityGovernedPrincipalUserId } from '@/lib/core/application' import { isTriggerDevEnabled } from '@/lib/core/config/env-flags' import { OrchestrationError } from '@/lib/core/orchestration/types' import { runDetached } from '@/lib/core/utils/background' @@ -255,6 +256,7 @@ export const copilotUpdateRowsByFilter = defineAuthorizedTableUseCase({ actorUserId: resolvePrincipalAttribution(principal, { workspaceBillingOwnerUserId: context.billedAccountUserId, }).attributedUserId, + capabilityGovernedUserId: capabilityGovernedPrincipalUserId(principal), secretProvenance: createExactEmptyTableRowSecretProvenance(idData), }, requestId() diff --git a/apps/sim/lib/table/application/exports.test.ts b/apps/sim/lib/table/application/exports.test.ts index 736ecd7c0c2..10d48cbf25c 100644 --- a/apps/sim/lib/table/application/exports.test.ts +++ b/apps/sim/lib/table/application/exports.test.ts @@ -3,17 +3,21 @@ */ import type { WorkflowExecutionDelegatedPrincipal } from '@sim/auth/principal' +import { permissionGroupScopeMock, permissionGroupScopeMockFns } from '@sim/testing' import { beforeEach, describe, expect, it, vi } from 'vitest' import type { TableDefinition } from '@/lib/table/types' const mocks = vi.hoisted(() => ({ cancel: vi.fn(), + resolveWorkspaceContext: vi.fn(), create: vi.fn(), getTable: vi.fn(), require: vi.fn(), resolveContext: vi.fn(), })) +const resolveGroupConfigMock = permissionGroupScopeMockFns.mockResolvePermissionGroupConfig + vi.mock('@sim/audit', () => ({ AuditAction: { TABLE_EXPORTED: 'table.exported' }, AuditResourceType: { TABLE: 'table' }, @@ -23,15 +27,11 @@ vi.mock('@sim/platform-authz/workspace', () => ({ permissionSatisfies: () => true, resolveEffectiveWorkspacePermission: vi.fn(), })) +vi.mock('@/lib/permission-groups/config-scope.server', () => permissionGroupScopeMock) vi.mock('@/lib/table', () => ({ getTableById: mocks.getTable })) vi.mock('@/lib/table/application/context', () => ({ resolveActiveTableContext: mocks.resolveContext, - resolveTableWorkspaceContext: vi.fn(async (workspaceId: string) => ({ - workspaceId, - workspaceOrganizationId: null, - allowPersonalApiKeys: true, - billedAccountUserId: 'billing-owner-1', - })), + resolveTableWorkspaceContext: mocks.resolveWorkspaceContext, })) vi.mock('@/lib/table/orchestration/export-resource', () => ({ cancelTableExportResource: mocks.cancel, @@ -43,6 +43,7 @@ vi.mock('@/lib/uploads/core/storage-service', () => ({ generatePresignedDownloadUrl: vi.fn(), })) +import { DEFAULT_PERMISSION_GROUP_CONFIG } from '@/lib/permission-groups/fields' import { cancelTableExportUseCase, createTableExportUseCase, @@ -110,6 +111,13 @@ describe('table export application use cases', () => { mocks.create.mockResolvedValue(record) mocks.require.mockResolvedValue(record) mocks.cancel.mockResolvedValue({ ...record, status: 'canceled' }) + resolveGroupConfigMock.mockResolvedValue(null) + mocks.resolveWorkspaceContext.mockImplementation(async (workspaceId: string) => ({ + workspaceId, + workspaceOrganizationId: null, + allowPersonalApiKeys: true, + billedAccountUserId: 'billing-owner-1', + })) }) it('returns domain records for create and read operations', async () => { @@ -164,4 +172,61 @@ describe('table export application use cases', () => { expect(mocks.create).not.toHaveBeenCalled() }) + + describe('permission-group capability', () => { + const member = { kind: 'session' as const, userId: 'user-1' } + const governedContext = { + tableId: table.id, + table, + workspaceId: table.workspaceId, + workspaceOrganizationId: 'org-1', + allowPersonalApiKeys: true, + billedAccountUserId: 'billing-owner-1', + } + + beforeEach(() => { + mocks.resolveContext.mockResolvedValue(governedContext) + mocks.resolveWorkspaceContext.mockImplementation(async (workspaceId: string) => ({ + workspaceId, + workspaceOrganizationId: 'org-1', + allowPersonalApiKeys: true, + billedAccountUserId: 'billing-owner-1', + })) + resolveGroupConfigMock.mockResolvedValue({ + ...DEFAULT_PERMISSION_GROUP_CONFIG, + disableTableExport: true, + }) + }) + + it('refuses to generate an export when the group withholds tables.export', async () => { + await expect( + createTableExportUseCase.execute({ + principal: member, + input: { tableId: 'table-1', workspaceId: 'workspace-1', format: 'csv' }, + }) + ).rejects.toMatchObject({ capability: 'tables.export' }) + + expect(mocks.create).not.toHaveBeenCalled() + }) + + it('still allows cancelling an export, which stops extraction rather than performing it', async () => { + await expect( + cancelTableExportUseCase.execute({ + principal: member, + input: { exportId: 'export-1', workspaceId: 'workspace-1' }, + }) + ).resolves.toMatchObject({ export: { status: 'canceled' } }) + }) + + it('generates an export when the group withholds nothing', async () => { + resolveGroupConfigMock.mockResolvedValue(DEFAULT_PERMISSION_GROUP_CONFIG) + + await expect( + createTableExportUseCase.execute({ + principal: member, + input: { tableId: 'table-1', workspaceId: 'workspace-1', format: 'csv' }, + }) + ).resolves.toEqual({ export: record }) + }) + }) }) diff --git a/apps/sim/lib/table/application/groups.test.ts b/apps/sim/lib/table/application/groups.test.ts index 6eb36a21c5c..db29e6c125f 100644 --- a/apps/sim/lib/table/application/groups.test.ts +++ b/apps/sim/lib/table/application/groups.test.ts @@ -72,7 +72,7 @@ vi.mock('@/lib/workflows/application/context', () => ({ resolveActiveWorkflowApplicationContext: mocks.resolveWorkflowContext, })) vi.mock('@/lib/workflows/application/resolve-workflow-outputs', () => ({ - loadResolvedWorkflowOutputs: mocks.loadWorkflowOutputs, + loadResolvedDeployedWorkflowOutputs: mocks.loadWorkflowOutputs, })) import { v2WorkflowGroupSchema } from '@/lib/api/contracts/v2/tables' @@ -290,6 +290,29 @@ describe('workflow and enrichment Table application commands', () => { expect(mocks.signal).toHaveBeenCalledWith(table.id) }) + /** + * Adding an output backfills it from saved runs, and a backfilled cell can + * satisfy a downstream group's deps and start it. That cascade is gated on + * the acting person, which is not the billing attribution beside it. + */ + it('names the acting person, not the billing actor, as the backfill cascade subject', async () => { + await addWorkflowTableGroupOutput.execute({ + principal, + input: { + tableId: table.id, + workspaceId: table.workspaceId, + groupId: group.id, + blockId: 'block-2', + path: 'score', + }, + }) + + expect(mocks.addOutput).toHaveBeenCalledWith( + expect.objectContaining({ capabilityGovernedUserId: 'user-1' }), + 'request-1' + ) + }) + it('persists disabled auto-run on a newly created workflow group', async () => { const result = await createWorkflowTableGroup.execute({ principal, @@ -333,6 +356,7 @@ describe('workflow and enrichment Table application commands', () => { isManualRun: false, requestId: 'request-1', triggeredByUserId: 'user-1', + capabilityGovernedUserId: 'user-1', }) }) @@ -1063,6 +1087,9 @@ describe('workflow and enrichment Table application commands', () => { expect(mocks.addOutput).toHaveBeenCalledWith( expect.objectContaining({ + // A copilot delegation stays governed, so the backfill's downstream + // cells run under the delegating person rather than ungated. + capabilityGovernedUserId: 'user-1', resolvedOutput: expect.objectContaining({ workflowId: 'workflow-1', columnType: 'number', diff --git a/apps/sim/lib/table/application/groups.ts b/apps/sim/lib/table/application/groups.ts index f8eef13eaca..bbbb1750f8e 100644 --- a/apps/sim/lib/table/application/groups.ts +++ b/apps/sim/lib/table/application/groups.ts @@ -3,6 +3,7 @@ import { resolvePrincipalAttribution } from '@sim/auth/principal' import { createLogger } from '@sim/logger' import { generateId } from '@sim/utils/id' import type { V2AddWorkflowGroupBody } from '@/lib/api/contracts/v2/tables' +import { capabilityGovernedPrincipalUserId } from '@/lib/core/application' import { asOrchestrationError, OrchestrationError } from '@/lib/core/orchestration/types' import { runDetached } from '@/lib/core/utils/background' import { generateRequestId } from '@/lib/core/utils/request' @@ -35,7 +36,7 @@ import { } from '@/lib/table/workflow-groups/service' import { resolveActiveWorkflowApplicationContext } from '@/lib/workflows/application/context' import type { ResolveWorkflowOutputsResult } from '@/lib/workflows/application/resolve-workflow-outputs' -import { loadResolvedWorkflowOutputs } from '@/lib/workflows/application/resolve-workflow-outputs' +import { loadResolvedDeployedWorkflowOutputs } from '@/lib/workflows/application/resolve-workflow-outputs' import { getEnrichment } from '@/enrichments/registry' import type { EnrichmentConfig } from '@/enrichments/types' @@ -64,7 +65,7 @@ async function resolveWorkflowForAuthorizedTableCommand( workflowId, assertedWorkspaceId: workspaceId, }) - return loadResolvedWorkflowOutputs(workflowContext) + return loadResolvedDeployedWorkflowOutputs(workflowContext) } async function resolveRelatedWorkflowForTableRoute( @@ -182,6 +183,11 @@ function dispatchGroupAutoRun(params: { workspaceId: string groupId: string actorUserId: string + /** + * The gate's subject, which is not the meter's `actorUserId`; `null` means no + * acting person. See {@link InsertRowData.capabilityGovernedUserId} in `@/lib/table/types`. + */ + capabilityGovernedUserId: string | null label: string }): void { runDetached(params.label, async () => { @@ -193,6 +199,7 @@ function dispatchGroupAutoRun(params: { isManualRun: false, requestId: generateRequestId(), triggeredByUserId: params.actorUserId, + capabilityGovernedUserId: params.capabilityGovernedUserId, }) logger.info('Started table group auto-run', { tableId: params.tableId, @@ -274,6 +281,7 @@ export const createTableGroupUseCase = defineAuthorizedTableUseCase({ } const actorUserId = attributedUserId(principal, context.billedAccountUserId) + const capabilityGovernedUserId = capabilityGovernedPrincipalUserId(principal) const groupId = input.group.id ?? generateId() /** * The public surface lets an `enrichment` group omit `workflowId`, so the @@ -302,10 +310,16 @@ export const createTableGroupUseCase = defineAuthorizedTableUseCase({ autoRun: input.autoRun ?? false, suppressAutoRunDispatch: true, actorUserId, + capabilityGovernedUserId, }, generateRequestId() ) - return { table, group: groupFromTable(table, groupId), actorUserId } + return { + table, + group: groupFromTable(table, groupId), + actorUserId, + capabilityGovernedUserId, + } }, projectAudit({ result }) { return { @@ -325,6 +339,7 @@ export const createTableGroupUseCase = defineAuthorizedTableUseCase({ workspaceId: context.workspaceId, groupId: result.group.id, actorUserId: result.actorUserId, + capabilityGovernedUserId: result.capabilityGovernedUserId, label: 'table-group-create-auto-run', }) } @@ -411,6 +426,7 @@ export const createWorkflowTableGroup = defineAuthorizedTableUseCase({ outputs, } const actorUserId = attributedUserId(principal, context.billedAccountUserId) + const capabilityGovernedUserId = capabilityGovernedPrincipalUserId(principal) const table = await addWorkflowGroup( { tableId: context.tableId, @@ -420,10 +436,16 @@ export const createWorkflowTableGroup = defineAuthorizedTableUseCase({ autoRun: input.autoRun ?? false, suppressAutoRunDispatch: true, actorUserId, + capabilityGovernedUserId, }, generateRequestId() ) - return { table, group: groupFromTable(table, groupId), actorUserId } + return { + table, + group: groupFromTable(table, groupId), + actorUserId, + capabilityGovernedUserId, + } }, projectAudit({ result }) { return { @@ -443,6 +465,7 @@ export const createWorkflowTableGroup = defineAuthorizedTableUseCase({ workspaceId: context.workspaceId, groupId: result.group.id, actorUserId: result.actorUserId, + capabilityGovernedUserId: result.capabilityGovernedUserId, label: 'table-workflow-group-create-auto-run', }) } @@ -550,6 +573,7 @@ export const createTableEnrichmentGroup = defineAuthorizedTableUseCase({ autoRun: input.autoRun ?? false, } const actorUserId = attributedUserId(principal, context.billedAccountUserId) + const capabilityGovernedUserId = capabilityGovernedPrincipalUserId(principal) const table = await addWorkflowGroup( { tableId: context.tableId, @@ -559,10 +583,16 @@ export const createTableEnrichmentGroup = defineAuthorizedTableUseCase({ autoRun: input.autoRun ?? false, suppressAutoRunDispatch: true, actorUserId, + capabilityGovernedUserId, }, generateRequestId() ) - return { table, group: groupFromTable(table, groupId), actorUserId } + return { + table, + group: groupFromTable(table, groupId), + actorUserId, + capabilityGovernedUserId, + } }, projectAudit({ result }) { return { @@ -586,6 +616,7 @@ export const createTableEnrichmentGroup = defineAuthorizedTableUseCase({ workspaceId: context.workspaceId, groupId: result.group.id, actorUserId: result.actorUserId, + capabilityGovernedUserId: result.capabilityGovernedUserId, label: 'table-enrichment-group-create-auto-run', }) } @@ -596,7 +627,11 @@ export interface UpdateTableGroupInput extends TableGroupInput, Omit< UpdateWorkflowGroupData, - 'tableId' | 'workspaceId' | 'actorUserId' | 'suppressAutoRunDispatch' + | 'tableId' + | 'workspaceId' + | 'actorUserId' + | 'capabilityGovernedUserId' + | 'suppressAutoRunDispatch' > {} export const updateTableGroupUseCase = defineAuthorizedTableUseCase({ @@ -735,6 +770,7 @@ export const updateTableGroupUseCase = defineAuthorizedTableUseCase({ } } const actorUserId = attributedUserId(principal, context.billedAccountUserId) + const capabilityGovernedUserId = capabilityGovernedPrincipalUserId(principal) const hasMappingUpdates = Boolean(input.mappingUpdates && input.mappingUpdates.length > 0) if (hasMappingUpdates && !resolvedWorkflow) { throw new Error('Workflow metadata is required for workflow group mapping updates') @@ -764,6 +800,7 @@ export const updateTableGroupUseCase = defineAuthorizedTableUseCase({ workspaceId: context.workspaceId, groupId: input.groupId, actorUserId, + capabilityGovernedUserId, suppressAutoRunDispatch: true, ...(input.workflowId !== undefined ? { workflowId: input.workflowId } : {}), ...(input.name !== undefined ? { name: input.name } : {}), @@ -795,6 +832,7 @@ export const updateTableGroupUseCase = defineAuthorizedTableUseCase({ JSON.stringify(context.table.metadata) !== JSON.stringify(table.metadata), startAutoRun: previousGroup?.autoRun === false && input.autoRun === true, actorUserId, + capabilityGovernedUserId, } }, projectAudit({ result }) { @@ -816,6 +854,7 @@ export const updateTableGroupUseCase = defineAuthorizedTableUseCase({ workspaceId: context.workspaceId, groupId: result.group.id, actorUserId: result.actorUserId, + capabilityGovernedUserId: result.capabilityGovernedUserId, label: 'table-group-update-auto-run', }) } @@ -956,12 +995,14 @@ export const updateWorkflowTableGroup = defineAuthorizedTableUseCase({ } const actorUserId = attributedUserId(principal, context.billedAccountUserId) + const capabilityGovernedUserId = capabilityGovernedPrincipalUserId(principal) const table = await updateWorkflowGroup( { tableId: context.tableId, workspaceId: context.workspaceId, groupId: input.groupId, actorUserId, + capabilityGovernedUserId, suppressAutoRunDispatch: true, ...(input.workflowId !== undefined ? { workflowId: input.workflowId } : {}), ...(input.name !== undefined ? { name: input.name } : {}), @@ -984,6 +1025,7 @@ export const updateWorkflowTableGroup = defineAuthorizedTableUseCase({ JSON.stringify(context.table.metadata) !== JSON.stringify(table.metadata), startAutoRun: previousGroup.autoRun === false && input.autoRun === true, actorUserId, + capabilityGovernedUserId, } }, projectAudit({ result }) { @@ -1005,6 +1047,7 @@ export const updateWorkflowTableGroup = defineAuthorizedTableUseCase({ workspaceId: context.workspaceId, groupId: result.group.id, actorUserId: result.actorUserId, + capabilityGovernedUserId: result.capabilityGovernedUserId, label: 'table-workflow-group-update-auto-run', }) } @@ -1079,6 +1122,11 @@ export const addWorkflowTableGroupOutput = defineAuthorizedTableUseCase({ context.workspaceId ) const outputs = requireWorkflowOutputs(resolvedWorkflow, group.workflowId) + validateRequestedOutputs( + [...group.outputs, { blockId: input.blockId, path: input.path }], + resolvedWorkflow, + group.workflowId + ) const output = outputs.find( (candidate) => candidate.blockId === input.blockId && candidate.path === input.path ) @@ -1099,6 +1147,7 @@ export const addWorkflowTableGroupOutput = defineAuthorizedTableUseCase({ actorUserId: resolvePrincipalAttribution(principal, { workspaceBillingOwnerUserId: context.billedAccountUserId, }).attributedUserId, + capabilityGovernedUserId: capabilityGovernedPrincipalUserId(principal), resolvedOutput: { workflowId: resolvedWorkflow.workflowId, columnType: columnTypeForLeaf(output.leafType), diff --git a/apps/sim/lib/table/application/imports.test.ts b/apps/sim/lib/table/application/imports.test.ts index c5226c24e5d..162f762fe2f 100644 --- a/apps/sim/lib/table/application/imports.test.ts +++ b/apps/sim/lib/table/application/imports.test.ts @@ -22,6 +22,11 @@ const mocks = vi.hoisted(() => ({ startUploadedImport: vi.fn(), tableImportBodyFromUpload: vi.fn(), resourceFromUpload: vi.fn(), + getUserPermissionConfig: vi.fn(), +})) + +vi.mock('@/lib/permission-groups/resolve.server', () => ({ + getUserPermissionConfig: mocks.getUserPermissionConfig, })) vi.mock('@sim/platform-authz/workspace', () => ({ @@ -71,6 +76,7 @@ vi.mock('@/lib/uploads/contexts/workspace/workspace-file-manager', () => ({ getWorkspaceFile: mocks.getWorkspaceFile, })) +import { DEFAULT_PERMISSION_GROUP_CONFIG } from '@/lib/permission-groups/fields' import { cancelTableImportUseCase, completeTableImportUseCase, @@ -167,6 +173,7 @@ describe('table import application use cases', () => { mocks.createResource.mockResolvedValue({ record, upload: null }) mocks.getWorkspaceFile.mockResolvedValue(workspaceFile) mocks.resourceFromUpload.mockReturnValue(record) + mocks.getUserPermissionConfig.mockResolvedValue(null) }) it('creates an import through the domain resource boundary without presenting a v2 DTO', async () => { @@ -456,4 +463,152 @@ describe('table import application use cases', () => { expect(mocks.resolveTableContext).not.toHaveBeenCalled() expect(mocks.createResource).not.toHaveBeenCalled() }) + + describe('permission-group capability', () => { + beforeEach(() => { + mocks.getUserPermissionConfig.mockResolvedValue({ + ...DEFAULT_PERMISSION_GROUP_CONFIG, + disableTableCreation: true, + }) + mocks.resolveTableContext.mockResolvedValue({ + tableId: 'table-1', + ...workspaceContext, + }) + }) + + it('refuses an import that would create a table when the group withholds tables.create', async () => { + await expect( + createTableImportUseCase.execute({ + principal: reader, + input: { + body: { + workspaceId: 'workspace-1', + source: record.source, + target: { type: 'new', name: 'People' }, + }, + }, + request: new Request('http://localhost:3000/api/table/imports', { method: 'POST' }), + }) + ).rejects.toMatchObject({ capability: 'tables.create' }) + + expect(mocks.createResource).not.toHaveBeenCalled() + }) + + /** + * A run carries the role of whoever triggered it but not their + * capabilities — the same exemption `authorizeWorkspaceOperation` applies. + * Keying this check on the raw subject instead would re-apply a capability + * the funnel deliberately passed, failing an executor import under a group + * that withholds table creation from the person who started the workflow. + */ + it('exempts an executor delegation carrying a subject, as the funnel does', async () => { + await expect( + createTableImportUseCase.execute({ + principal: executor, + input: { + body: { + workspaceId: 'workspace-1', + source: record.source, + target: { type: 'new', name: 'People' }, + }, + }, + request: new Request('http://localhost:3000/api/table/imports', { method: 'POST' }), + }) + ).resolves.toBeDefined() + + expect(mocks.createResource).toHaveBeenCalled() + }) + + it('exempts an executor delegation at completion too', async () => { + await expect( + completeTableImportUseCase.execute({ + principal: executor, + input: { + importId: 'import-1', + workspaceId: 'workspace-1', + uploadToken: 'signed-token', + }, + }) + ).resolves.toBeDefined() + + expect(mocks.startUploadedImport).toHaveBeenCalled() + }) + + it('still allows importing into an existing table, which creates nothing', async () => { + await expect( + createTableImportUseCase.execute({ + principal: reader, + input: { + body: { + workspaceId: 'workspace-1', + source: record.source, + target: { type: 'existing', tableId: 'table-1' }, + }, + }, + request: new Request('http://localhost:3000/api/table/imports', { method: 'POST' }), + }) + ).resolves.toEqual({ import: { record, upload: null } }) + }) + + /** + * Creation and completion are separate requests, so a group that withholds + * creation between them has to be read again at completion — otherwise the + * upload started while it was allowed still lands a table. + */ + it('refuses to complete an upload that would create a table, and never starts the import', async () => { + await expect( + completeTableImportUseCase.execute({ + principal: reader, + input: { + importId: 'import-1', + workspaceId: 'workspace-1', + uploadToken: 'signed-token', + }, + }) + ).rejects.toMatchObject({ capability: 'tables.create' }) + + expect(mocks.startUploadedImport).not.toHaveBeenCalled() + }) + + it('still completes an upload targeting an existing table', async () => { + mocks.tableImportBodyFromUpload.mockReturnValue({ + workspaceId: 'workspace-1', + source: record.source, + target: { type: 'existing', tableId: 'table-1' }, + }) + + await expect( + completeTableImportUseCase.execute({ + principal: reader, + input: { + importId: 'import-1', + workspaceId: 'workspace-1', + uploadToken: 'signed-token', + }, + }) + ).resolves.toEqual({ import: { ...record, status: 'ready' } }) + + expect(mocks.startUploadedImport).toHaveBeenCalledTimes(1) + }) + + /** + * A workspace API key has no acting person, so there is no group to read — + * the same exemption `createTableImportUseCase` makes, and the reason the + * re-check must not become a blanket refusal on the completion leg. + */ + it('still completes an upload driven by a workspace key, which has no subject', async () => { + await expect( + completeTableImportUseCase.execute({ + principal: workspaceKey, + input: { + importId: 'import-1', + workspaceId: 'workspace-1', + uploadToken: 'signed-token', + }, + }) + ).resolves.toEqual({ import: { ...record, status: 'ready' } }) + + expect(mocks.startUploadedImport).toHaveBeenCalledTimes(1) + }) + }) }) diff --git a/apps/sim/lib/table/application/imports.ts b/apps/sim/lib/table/application/imports.ts index 0b14531c299..ba596f1dfb1 100644 --- a/apps/sim/lib/table/application/imports.ts +++ b/apps/sim/lib/table/application/imports.ts @@ -1,11 +1,15 @@ import { type Principal, resolvePrincipalAttribution } from '@sim/auth/principal' import { createLogger } from '@sim/logger' -import { authorizeWorkspaceOperation } from '@/lib/core/application' +import { + authorizeWorkspaceOperation, + capabilityGovernedPrincipalUserId, +} from '@/lib/core/application' import { OrchestrationError } from '@/lib/core/orchestration/types' import { MAX_FOLDERS_PER_WORKSPACE } from '@/lib/folders/constants' import { withFolderTreeLock } from '@/lib/folders/locks' import { ROOT_FOLDER_PATH } from '@/lib/folders/paths' import { loadActiveFolderPathIndex, resolveFolderPathFromIndex } from '@/lib/folders/queries' +import { assertWorkspaceCapability } from '@/lib/permission-groups/capability-assertions' import { type TableAuthorizationContext, tableDelegationPolicy, @@ -168,6 +172,21 @@ export const createTableImportUseCase = defineAuthorizedTableUseCase({ resolveContext: ({ input }: { input: CreateTableImportInput }) => resolveCreateTableImportContext(input), async execute({ principal, input, context, request }): Promise { + /** + * permission-group-enforced: tables.create — an import targeting `new` + * creates a table, but one targeting `existing` only fills one, and the + * operation cannot tell them apart: the target is request input the + * authorization funnel never sees. Keyed to the governed subject, which + * names nobody for an actorless run and nobody for an executor delegation — + * the funnel exempts a run from capabilities even when it carries the + * subject of whoever triggered it. A copilot delegation stays governed. + */ + if (input.body.target.type === 'new') { + const actingUserId = capabilityGovernedPrincipalUserId(principal) + if (actingUserId) { + await assertWorkspaceCapability(actingUserId, context.workspaceId, 'tables.create') + } + } const attribution = resolvePrincipalAttribution(principal, { workspaceBillingOwnerUserId: context.billedAccountUserId, }) @@ -273,6 +292,22 @@ export const completeTableImportUseCase = defineAuthorizedTableUseCase({ await authorizeWorkspaceOperation(principal, tableOperations.completeImport, context, { delegation: tableDelegationPolicy, }) + /** + * permission-group-enforced: tables.create — the same assertion + * `createTableImportUseCase` makes, repeated here because the two are + * separate requests: an upload started before the group withheld + * creation would otherwise still land a table when it completed. Read + * from the claimed session so the target is the one the upload was + * created for, and keyed to the governed subject for the reason the + * create path is: an actorless run and an executor delegation are both + * ungoverned, a copilot delegation is not. + */ + if (tableImportBodyFromUpload(claimed).target.type === 'new') { + const actingUserId = capabilityGovernedPrincipalUserId(principal) + if (actingUserId) { + await assertWorkspaceCapability(actingUserId, context.workspaceId, 'tables.create') + } + } return { value: null } }, }) diff --git a/apps/sim/lib/table/application/operations.ts b/apps/sim/lib/table/application/operations.ts index 209d11979a7..2f9aaa110aa 100644 --- a/apps/sim/lib/table/application/operations.ts +++ b/apps/sim/lib/table/application/operations.ts @@ -1,4 +1,5 @@ import { defineWorkspaceOperation } from '@/lib/core/application' +import type { OperationDeclarableCapability } from '@/lib/core/application/operation' const ALL_PRINCIPAL_POLICY = { principalKinds: ['session', 'personal_api_key', 'workspace_api_key', 'delegated'], @@ -24,6 +25,7 @@ function readOperation(id: Id) { id, minimumRole: 'read', workspaceApiKey: 'allow', + capability: 'tables.use', ...ALL_PRINCIPAL_POLICY, }) } @@ -33,15 +35,29 @@ function writeOperation(id: Id) { id, minimumRole: 'write', workspaceApiKey: 'allow', + capability: 'tables.use', ...ALL_PRINCIPAL_POLICY, }) } -function toolWriteOperation(id: Id) { +/** + * Not every table operation needs the same capability — creating a table and + * exporting one are each withheld separately from ordinary table use — so the + * factories that mint more than one kind take the capability as an argument. + * + * No default, deliberately: a default would let a new operation inherit + * `tables.use` without anyone deciding it should, which is exactly the + * unreviewed omission this gate exists to prevent. + */ +function toolWriteOperation( + id: Id, + capability: OperationDeclarableCapability +) { return defineWorkspaceOperation({ id, minimumRole: 'write', workspaceApiKey: 'allow', + capability, ...ALL_TABLE_TOOL_PRINCIPAL_POLICY, }) } @@ -51,15 +67,20 @@ function toolReadOperation(id: Id) { id, minimumRole: 'read', workspaceApiKey: 'allow', + capability: 'tables.use', ...ALL_TABLE_TOOL_PRINCIPAL_POLICY, }) } -function internalExecutorReadOperation(id: Id) { +function internalExecutorReadOperation( + id: Id, + capability: OperationDeclarableCapability +) { return defineWorkspaceOperation({ id, minimumRole: 'read', workspaceApiKey: 'allow', + capability, ...INTERNAL_EXECUTOR_PRINCIPAL_POLICY, }) } @@ -69,15 +90,20 @@ function internalExecutorWriteOperation(id: Id) { id, minimumRole: 'write', workspaceApiKey: 'allow', + capability: 'tables.use', ...INTERNAL_EXECUTOR_PRINCIPAL_POLICY, }) } -function delegatedWriteOperation(id: Id) { +function delegatedWriteOperation( + id: Id, + capability: OperationDeclarableCapability +) { return defineWorkspaceOperation({ id, minimumRole: 'write', workspaceApiKey: 'deny', + capability, principalKinds: ['delegated'], delegatedServices: ['copilot'], }) @@ -86,7 +112,7 @@ function delegatedWriteOperation(id: Id) { export const tableOperations = { list: toolReadOperation('tables.list'), read: toolReadOperation('tables.read'), - create: toolWriteOperation('tables.create'), + create: toolWriteOperation('tables.create', 'tables.create'), update: writeOperation('tables.update'), delete: writeOperation('tables.delete'), restore: writeOperation('tables.restore'), @@ -96,18 +122,21 @@ export const tableOperations = { id: 'tables.vfs.rename', minimumRole: 'write', workspaceApiKey: 'deny', + capability: 'tables.use', ...COPILOT_PRINCIPAL_POLICY, }), moveByVfsPath: defineWorkspaceOperation({ id: 'tables.vfs.move', minimumRole: 'write', workspaceApiKey: 'deny', + capability: 'tables.use', ...COPILOT_PRINCIPAL_POLICY, }), deleteByVfsPath: defineWorkspaceOperation({ id: 'tables.vfs.delete', minimumRole: 'write', workspaceApiKey: 'deny', + capability: 'tables.use', ...COPILOT_PRINCIPAL_POLICY, }), listFolders: readOperation('tables.folders.list'), @@ -122,37 +151,46 @@ export const tableOperations = { queryRows: toolReadOperation('tables.rows.query'), searchRows: readOperation('tables.rows.search'), readRow: toolReadOperation('tables.rows.read'), - createRows: toolWriteOperation('tables.rows.create'), + createRows: toolWriteOperation('tables.rows.create', 'tables.use'), replaceRows: writeOperation('tables.rows.replace'), - updateRow: toolWriteOperation('tables.rows.update'), - updateRows: toolWriteOperation('tables.rows.update_many'), - deleteRow: toolWriteOperation('tables.rows.delete'), - deleteRows: toolWriteOperation('tables.rows.delete_many'), - upsertRow: toolWriteOperation('tables.rows.upsert'), + updateRow: toolWriteOperation('tables.rows.update', 'tables.use'), + updateRows: toolWriteOperation('tables.rows.update_many', 'tables.use'), + deleteRow: toolWriteOperation('tables.rows.delete', 'tables.use'), + deleteRows: toolWriteOperation('tables.rows.delete_many', 'tables.use'), + upsertRow: toolWriteOperation('tables.rows.upsert', 'tables.use'), listViews: readOperation('tables.views.list'), readView: readOperation('tables.views.read'), createView: writeOperation('tables.views.create'), updateView: writeOperation('tables.views.update'), deleteView: writeOperation('tables.views.delete'), listGroups: readOperation('tables.groups.list'), - createGroup: toolWriteOperation('tables.groups.create'), - updateGroup: toolWriteOperation('tables.groups.update'), - deleteGroup: toolWriteOperation('tables.groups.delete'), + createGroup: toolWriteOperation('tables.groups.create', 'tables.use'), + updateGroup: toolWriteOperation('tables.groups.update', 'tables.use'), + deleteGroup: toolWriteOperation('tables.groups.delete', 'tables.use'), startRun: writeOperation('tables.runs.start'), /** Reading the state of a run — including one you started — is a read. */ readRun: readOperation('tables.runs.read'), cancelRuns: writeOperation('tables.runs.cancel'), createImport: internalExecutorWriteOperation('tables.imports.create'), - createFromWorkspaceFile: delegatedWriteOperation('tables.imports.create_from_workspace_file'), - importWorkspaceFile: delegatedWriteOperation('tables.imports.workspace_file'), - readImport: internalExecutorReadOperation('tables.imports.read'), + createFromWorkspaceFile: delegatedWriteOperation( + 'tables.imports.create_from_workspace_file', + 'tables.create' + ), + importWorkspaceFile: delegatedWriteOperation('tables.imports.workspace_file', 'tables.use'), + readImport: internalExecutorReadOperation('tables.imports.read', 'tables.use'), createImportParts: internalExecutorWriteOperation('tables.imports.create_parts'), completeImport: internalExecutorWriteOperation('tables.imports.complete'), cancelImport: internalExecutorWriteOperation('tables.imports.cancel'), - createExport: internalExecutorReadOperation('tables.exports.create'), - readExport: internalExecutorReadOperation('tables.exports.read'), - cancelExport: internalExecutorReadOperation('tables.exports.cancel'), - downloadExport: internalExecutorReadOperation('tables.exports.download'), + /** + * Only generating the file and fetching it are extraction. Reading an + * export's status carries no rows, and cancelling one stops an extraction + * rather than performing it — gating either would strand a member with an + * export they can neither watch nor stop after the group changed. + */ + createExport: internalExecutorReadOperation('tables.exports.create', 'tables.export'), + readExport: internalExecutorReadOperation('tables.exports.read', 'tables.use'), + cancelExport: internalExecutorReadOperation('tables.exports.cancel', 'tables.use'), + downloadExport: internalExecutorReadOperation('tables.exports.download', 'tables.export'), } as const export type TableOperation = (typeof tableOperations)[keyof typeof tableOperations] diff --git a/apps/sim/lib/table/application/rows.ts b/apps/sim/lib/table/application/rows.ts index 4ca671cdc4b..4bce8ce9ad5 100644 --- a/apps/sim/lib/table/application/rows.ts +++ b/apps/sim/lib/table/application/rows.ts @@ -9,6 +9,7 @@ import { db } from '@sim/db' import { getRequestContext } from '@sim/logger' import { generateId } from '@sim/utils/id' import { isPlainRecord } from '@sim/utils/object' +import { capabilityGovernedPrincipalUserId } from '@/lib/core/application' import { isFeatureEnabled } from '@/lib/core/config/feature-flags' import { OrchestrationError } from '@/lib/core/orchestration/types' import { isPrivateSecretProvenanceScopeCompatible } from '@/lib/execution/durable-secret-provenance' @@ -788,6 +789,7 @@ export const createTableRows = defineAuthorizedTableUseCase({ workspaceId: context.workspaceId, data, userId, + capabilityGovernedUserId: capabilityGovernedPrincipalUserId(principal), position: input.position, afterRowId: input.afterRowId, beforeRowId: input.beforeRowId, @@ -847,6 +849,7 @@ export const createTableRows = defineAuthorizedTableUseCase({ workspaceId: context.workspaceId, rows, userId, + capabilityGovernedUserId: capabilityGovernedPrincipalUserId(principal), orderKeys: input.orderKeys, secretProvenance, }, @@ -1151,6 +1154,7 @@ export const updateTableRow = defineAuthorizedTableUseCase({ rowId: input.rowId, data, actorUserId: actorUserId(principal, context.billedAccountUserId), + capabilityGovernedUserId: capabilityGovernedPrincipalUserId(principal), secretProvenance, }, context.table, @@ -1213,6 +1217,7 @@ export const updateTableRows = defineAuthorizedTableUseCase({ data, limit: input.limit, actorUserId: actorUserId(principal, context.billedAccountUserId), + capabilityGovernedUserId: capabilityGovernedPrincipalUserId(principal), secretProvenance, }, requestId(input), @@ -1305,6 +1310,7 @@ export const batchUpdateTableRows = defineAuthorizedTableUseCase({ updates, workspaceId: context.workspaceId, actorUserId: actorUserId(principal, context.billedAccountUserId), + capabilityGovernedUserId: capabilityGovernedPrincipalUserId(principal), secretProvenanceByRowId: Object.fromEntries( updates.flatMap((update, index) => { const stamp = secretProvenance[index] @@ -1472,6 +1478,7 @@ export const upsertTableRow = defineAuthorizedTableUseCase({ data, conflictTarget, userId: actorUserId(principal, context.billedAccountUserId), + capabilityGovernedUserId: capabilityGovernedPrincipalUserId(principal), secretProvenance, }, context.table, diff --git a/apps/sim/lib/table/application/runs.test.ts b/apps/sim/lib/table/application/runs.test.ts index 93a1675e4ef..e2735125059 100644 --- a/apps/sim/lib/table/application/runs.test.ts +++ b/apps/sim/lib/table/application/runs.test.ts @@ -159,11 +159,39 @@ describe('table run application use cases', () => { mode: 'all', requestId: 'request-1', triggeredByUserId: PRINCIPAL.userId, + capabilityGovernedUserId: PRINCIPAL.userId, }) expect(result.dispatchId).toBe('dispatch-1') expect(mockSignalRowsChanged).toHaveBeenCalledWith(TABLE.id) }) + /** + * A workspace key names no human, so its run is ungoverned. The meter still + * needs someone, and attribution answers with the workspace billed account — + * a bystander whose tool denylist must not reach the run's cells. The two + * subjects are carried separately precisely so this case can differ. + */ + it('carries the billed account as the meter but nobody as the gate for a workspace key', async () => { + await startTableRun.execute({ + principal: { kind: 'workspace_api_key', workspaceId: TABLE.workspaceId, keyId: 'key-1' }, + input: { + kind: 'row_enrichment', + tableId: TABLE.id, + assertedWorkspaceId: TABLE.workspaceId, + rowId: 'row-1', + groupId: 'group-1', + requestId: 'request-1', + }, + }) + + expect(mockRunWorkflowColumn).toHaveBeenCalledWith( + expect.objectContaining({ + triggeredByUserId: 'billing-owner-1', + capabilityGovernedUserId: null, + }) + ) + }) + it('rejects missing canonical groups and rows without dispatching', async () => { await expect( startTableRun.execute({ diff --git a/apps/sim/lib/table/application/runs.ts b/apps/sim/lib/table/application/runs.ts index 7b9725f9414..6480304b46c 100644 --- a/apps/sim/lib/table/application/runs.ts +++ b/apps/sim/lib/table/application/runs.ts @@ -1,6 +1,7 @@ import { resolvePrincipalAttribution } from '@sim/auth/principal' import { getRequestContext } from '@sim/logger' import { generateId } from '@sim/utils/id' +import { capabilityGovernedPrincipalUserId } from '@/lib/core/application' import { OrchestrationError } from '@/lib/core/orchestration/types' import { DEFAULT_TABLE_PLAN_LIMITS, @@ -96,6 +97,14 @@ export const startTableRun = defineAuthorizedTableUseCase({ resolveContext: ({ input }: { input: StartTableRunInput }) => resolveActiveTableContext(input), async execute({ principal, input, context }): Promise { const triggeredByUserId = actorUserId(principal, context.billedAccountUserId) + /** + * The gate's subject, which is not the meter's. `actorUserId` substitutes + * the workspace billed account when the credential names no human, so a + * workspace-API-key run would otherwise carry that bystander into the + * cells' tool denylist. Null here means no acting person and no per-tool + * gate — the same answer an executor delegation gets from the funnel. + */ + const capabilityGovernedUserId = capabilityGovernedPrincipalUserId(principal) if (input.kind === 'row_enrichment') { requireCanonicalGroups(context.table, [input.groupId]) const row = await getRowById(context.tableId, input.rowId, context.workspaceId) @@ -108,6 +117,7 @@ export const startTableRun = defineAuthorizedTableUseCase({ mode: 'all', requestId: requestId(input), triggeredByUserId, + capabilityGovernedUserId, }) return { table: context.table, @@ -165,6 +175,7 @@ export const startTableRun = defineAuthorizedTableUseCase({ limit: input.limit, requestId: requestId(input), triggeredByUserId, + capabilityGovernedUserId, }) return { table: context.table, diff --git a/apps/sim/lib/table/application/workspace-file-imports.ts b/apps/sim/lib/table/application/workspace-file-imports.ts index 25917059fc5..22febc596f5 100644 --- a/apps/sim/lib/table/application/workspace-file-imports.ts +++ b/apps/sim/lib/table/application/workspace-file-imports.ts @@ -3,6 +3,7 @@ import { resolvePrincipalAttribution } from '@sim/auth/principal' import { createLogger } from '@sim/logger' import { getErrorMessage } from '@sim/utils/errors' import { generateId } from '@sim/utils/id' +import { capabilityGovernedPrincipalUserId } from '@/lib/core/application' import { isTriggerDevEnabled } from '@/lib/core/config/env-flags' import { OrchestrationError } from '@/lib/core/orchestration/types' import { runDetached } from '@/lib/core/utils/background' @@ -226,6 +227,9 @@ async function batchInsertAll(params: { rows: RowData[] workspaceId: string userId: string + /** The gate's subject for enrichment the landed rows auto-fire; see + * {@link BatchInsertData.capabilityGovernedUserId}. */ + capabilityGovernedUserId: string | null assertNotAborted?: () => void }): Promise { let inserted = 0 @@ -238,6 +242,7 @@ async function batchInsertAll(params: { rows: batch, workspaceId: params.workspaceId, userId: params.userId, + capabilityGovernedUserId: params.capabilityGovernedUserId, secretProvenance: batch.map(createExactEmptyTableRowSecretProvenance), }, { ...params.table, rowCount: params.table.rowCount + inserted }, @@ -401,6 +406,7 @@ export const createTableFromWorkspaceFile = defineAuthorizedTableUseCase({ }), workspaceId: context.workspaceId, userId, + capabilityGovernedUserId: capabilityGovernedPrincipalUserId(principal), assertNotAborted: input.assertNotAborted, }) const summary = summarizeRejections(rejections, cellsRejected, sourceFile) @@ -563,6 +569,7 @@ export const importWorkspaceFileIntoTable = defineAuthorizedTableUseCase({ rows, workspaceId: context.workspaceId, userId, + capabilityGovernedUserId: capabilityGovernedPrincipalUserId(principal), assertNotAborted: input.assertNotAborted, }) return { diff --git a/apps/sim/lib/table/backfill-governed-subject.test.ts b/apps/sim/lib/table/backfill-governed-subject.test.ts new file mode 100644 index 00000000000..40f03203deb --- /dev/null +++ b/apps/sim/lib/table/backfill-governed-subject.test.ts @@ -0,0 +1,120 @@ +/** + * @vitest-environment node + */ + +import { tableRowExecutions, userTableRows, workflowExecutionLogs } from '@sim/db/schema' +import { queueTableRows, resetDbChainMock } from '@sim/testing' +import { beforeEach, describe, expect, it, vi } from 'vitest' +import type { TableDefinition } from '@/lib/table/types' + +const { mockBatchUpdateRows, mockMaterializeExecutionData, mockGetFunctionalBlockOutput } = + vi.hoisted(() => ({ + mockBatchUpdateRows: vi.fn(), + mockMaterializeExecutionData: vi.fn(), + mockGetFunctionalBlockOutput: vi.fn(), + })) + +vi.mock('@/lib/table/rows/service', () => ({ + batchUpdateRows: mockBatchUpdateRows, +})) +vi.mock('@/lib/logs/execution/trace-store', () => ({ + materializeExecutionData: mockMaterializeExecutionData, +})) +vi.mock('@/lib/logs/execution/functional-outputs', () => ({ + getFunctionalBlockOutput: mockGetFunctionalBlockOutput, +})) +vi.mock('@/lib/table/rows/secret-provenance', () => ({ + createTableRowSecretProvenanceFromRegistry: () => ({ complete: true, columns: {} }), +})) + +import { maybeBackfillGroupOutputs } from '@/lib/table/backfill-runner' + +const TABLE = { + id: 'table-1', + workspaceId: 'workspace-1', + schema: { columns: [], workflowGroups: [] }, +} as unknown as TableDefinition + +/** Queues the four reads one inline backfill page makes, in the order it makes them. */ +function queueOnePage(): void { + queueTableRows(tableRowExecutions, [{ count: 1 }]) + queueTableRows(tableRowExecutions, [{ rowId: 'row-1', executionId: 'execution-1' }]) + queueTableRows(userTableRows, [{ id: 'row-1', data: {} }]) + queueTableRows(workflowExecutionLogs, [ + { + executionId: 'execution-1', + workflowId: 'workflow-1', + workspaceId: 'workspace-1', + executionData: {}, + }, + ]) + queueTableRows(tableRowExecutions, []) +} + +describe('backfill cascade governance', () => { + beforeEach(() => { + vi.clearAllMocks() + resetDbChainMock() + mockMaterializeExecutionData.mockResolvedValue({}) + mockGetFunctionalBlockOutput.mockReturnValue({ value: 'filled' }) + mockBatchUpdateRows.mockResolvedValue({ affectedCount: 1, affectedRowIds: ['row-1'] }) + }) + + /** + * A backfilled cell is a dependency: `batchUpdateRows` starts every downstream + * group whose deps it just satisfied. Passing no subject there ran those + * cells with no per-tool gate, which is what `null` means on this field. + */ + it('cascades under the person who made the schema change', async () => { + queueOnePage() + + await maybeBackfillGroupOutputs({ + table: TABLE, + groupId: 'group-1', + outputs: [{ blockId: 'block-1', path: 'value', columnName: 'value' }], + overwrite: true, + requestId: 'request-1', + actorUserId: 'billed-owner', + capabilityGovernedUserId: 'member-1', + }) + + expect(mockBatchUpdateRows).toHaveBeenCalledWith( + expect.objectContaining({ + actorUserId: 'billed-owner', + capabilityGovernedUserId: 'member-1', + }), + expect.anything(), + expect.anything(), + expect.anything() + ) + }) + + /** + * The one payload that can omit the field is a large backfill enqueued before + * it existed and still running after the deploy. `actorUserId` is not a + * recovery: `attributedUserId` yields the workspace's billed account for a + * change made by a workspace API key, and nothing here tells that apart from + * a human — so borrowing it would apply a bystander's denylist, the exact + * substitution this field removes. Null for one deploy's worth of in-flight + * jobs is the least wrong of the available answers. + */ + it('keeps an absent subject null rather than borrowing the billing actor', async () => { + queueOnePage() + + await maybeBackfillGroupOutputs({ + table: TABLE, + groupId: 'group-1', + outputs: [{ blockId: 'block-1', path: 'value', columnName: 'value' }], + overwrite: true, + requestId: 'request-1', + actorUserId: 'billed-owner', + }) + + expect(mockBatchUpdateRows).toHaveBeenCalledWith( + expect.objectContaining({ capabilityGovernedUserId: null }), + expect.anything(), + expect.anything(), + expect.anything() + ) + }) +}) diff --git a/apps/sim/lib/table/backfill-runner.ts b/apps/sim/lib/table/backfill-runner.ts index 685bd43740b..582add45b7c 100644 --- a/apps/sim/lib/table/backfill-runner.ts +++ b/apps/sim/lib/table/backfill-runner.ts @@ -56,6 +56,31 @@ export interface TableBackfillPayload { overwrite: boolean /** User who triggered the schema change, for usage attribution on the row writes. */ actorUserId?: string | null + /** + * Person whose permission group gates any cell the backfill's writes cascade + * into. Separate from `actorUserId`, which is a billing attribution and names + * the workspace billed account when the schema change carried no human. Null + * when the change had no acting person. + * + * Absent only on a payload enqueued before this field existed and still + * running after the deploy that added it — a backfill over more rows than + * `BACKFILL_ASYNC_THRESHOLD_ROWS`, mid-flight at the cutover. Such a payload + * reads as null, and that is a deliberate choice between two wrong answers + * rather than the status quo: before this field, the cascaded cells gated on + * `actorUserId`, so for a session-made change the window loosens the gate for + * as long as that one job runs. + * + * Falling back to `actorUserId` would close that and open a worse one. + * `attributedUserId` yields the workspace's billed account for a change made + * by a workspace API key, and nothing on the payload distinguishes that id + * from a human actor — so the fallback would apply a bystander's denylist, + * which is the substitution this field exists to remove. Failing closed + * instead would abandon the backfill's writes entirely, turning a bounded + * governance edge into visible data loss on runs the schema change promised + * to fill. Null is the least wrong of the three, and the window is one + * deploy long. + */ + capabilityGovernedUserId?: string | null } /** @@ -136,8 +161,11 @@ async function processBackfillPage(opts: { execs: Array<{ rowId: string; executionId: string | null }> requestId: string actorUserId?: string | null + /** See {@link TableBackfillPayload.capabilityGovernedUserId}. */ + capabilityGovernedUserId?: string | null }): Promise { - const { table, outputs, overwrite, execs, requestId, actorUserId } = opts + const { table, outputs, overwrite, execs, requestId, actorUserId, capabilityGovernedUserId } = + opts const executionIdsByRow = new Map() for (const e of execs) { @@ -224,6 +252,15 @@ async function processBackfillPage(opts: { updates, workspaceId: table.workspaceId, actorUserId, + /** + * A backfill replays values already produced by earlier runs, but the + * cells it fills are dependencies: `batchUpdateRows` starts every + * downstream group whose deps just became satisfied. Those cells are + * governed by whoever made the schema change, carried separately from + * `actorUserId` — an attribution that names the workspace billed account + * when the change carried no human, whose denylist is nobody's to run. + */ + capabilityGovernedUserId: capabilityGovernedUserId ?? null, secretProvenanceByRowId, }, table, @@ -242,7 +279,8 @@ async function processBackfillPage(opts: { * passes skip already-filled cells). */ export async function runTableBackfill(payload: TableBackfillPayload): Promise { - const { jobId, tableId, groupId, outputs, overwrite, actorUserId } = payload + const { jobId, tableId, groupId, outputs, overwrite, actorUserId, capabilityGovernedUserId } = + payload const requestId = generateId().slice(0, 8) try { @@ -268,6 +306,7 @@ export async function runTableBackfill(payload: TableBackfillPayload): Promise { - const { table, groupId, outputs, overwrite, requestId, actorUserId } = opts + const { table, groupId, outputs, overwrite, requestId, actorUserId, capabilityGovernedUserId } = + opts if (outputs.length === 0) return const [{ count: completedCount }] = await db @@ -347,7 +389,15 @@ export async function maybeBackfillGroupOutputs(opts: { const execs = await selectCompletedExecPage(table.id, groupId, afterRowId, BACKFILL_PAGE_SIZE) if (execs.length === 0) break afterRowId = execs[execs.length - 1].rowId - await processBackfillPage({ table, outputs, overwrite, execs, requestId, actorUserId }) + await processBackfillPage({ + table, + outputs, + overwrite, + execs, + requestId, + actorUserId, + capabilityGovernedUserId, + }) } return } @@ -370,6 +420,7 @@ export async function maybeBackfillGroupOutputs(opts: { outputs, overwrite, actorUserId, + capabilityGovernedUserId, } if (isTriggerDevEnabled) { try { diff --git a/apps/sim/lib/table/cell-write.test.ts b/apps/sim/lib/table/cell-write.test.ts index 61fbbaee7b7..c49bf6675eb 100644 --- a/apps/sim/lib/table/cell-write.test.ts +++ b/apps/sim/lib/table/cell-write.test.ts @@ -157,6 +157,8 @@ describe('writeWorkflowGroupState', () => { workspaceId: TABLE.workspaceId, executionsPatch: { [GROUP.id]: RUNNING_STATE }, cancellationGuard: { groupId: GROUP.id, executionId: CONTEXT.executionId }, + /** A cell result carries no acting person down to the write layer. */ + capabilityGovernedUserId: null, secretProvenance, }, TABLE, diff --git a/apps/sim/lib/table/cell-write.ts b/apps/sim/lib/table/cell-write.ts index 789d8702b56..72b5f7968bd 100644 --- a/apps/sim/lib/table/cell-write.ts +++ b/apps/sim/lib/table/cell-write.ts @@ -93,6 +93,12 @@ export async function writeWorkflowGroupState( executionsPatch, cancellationGuard, secretProvenance: payload.secretProvenance, + /** + * A cell result carries no acting person down to this layer — the + * write has no `actorUserId` either, so any cascade it fires is + * already actorless on both the meter and the gate. + */ + capabilityGovernedUserId: null, }, table, requestId, diff --git a/apps/sim/lib/table/column-types/extension-points.test.ts b/apps/sim/lib/table/column-types/extension-points.test.ts new file mode 100644 index 00000000000..84484a2ae6d --- /dev/null +++ b/apps/sim/lib/table/column-types/extension-points.test.ts @@ -0,0 +1,79 @@ +/** + * @vitest-environment node + */ +import { afterEach, describe, expect, it } from 'vitest' +import { + COLUMN_TYPE_REGISTRY, + validateColumnTypeLimits, + valueForTypeConversion, + wouldExceedColumnTypeLimit, +} from '@/lib/table/column-types' +import type { ColumnDefinition } from '@/lib/table/types' + +const definition = COLUMN_TYPE_REGISTRY.string +const originalMaxPerTable = definition.maxPerTable +const originalValueForConversion = definition.valueForConversion + +function restoreOptionalProperty(key: 'maxPerTable' | 'valueForConversion', value: unknown) { + if (value === undefined) { + Reflect.deleteProperty(definition, key) + return + } + Object.assign(definition, { [key]: value }) +} + +afterEach(() => { + restoreOptionalProperty('maxPerTable', originalMaxPerTable) + restoreOptionalProperty('valueForConversion', originalValueForConversion) +}) + +describe('column type extension points', () => { + it('enforces registry-declared per-table limits', () => { + Object.assign(definition, { maxPerTable: 1 }) + const columns: ColumnDefinition[] = [ + { name: 'first', type: 'string' }, + { name: 'second', type: 'string' }, + ] + + expect(wouldExceedColumnTypeLimit(columns.slice(0, 1), 'string', 1)).toBe(true) + expect(validateColumnTypeLimits(columns)).toEqual([ + `A table can have at most 1 ${definition.label} column`, + ]) + }) + + it('lets the source type normalize a value before conversion', () => { + Object.assign(definition, { + valueForConversion: (_value: unknown, target: ColumnDefinition) => + target.type === 'number' ? 42 : 'unchanged', + }) + + expect( + valueForTypeConversion( + 'stored-value', + { name: 'source', type: 'string' }, + { name: 'target', type: 'number' } + ) + ).toBe(42) + expect( + valueForTypeConversion( + 'stored-value', + { name: 'source', type: 'number' }, + { name: 'target', type: 'string' } + ) + ).toBe('stored-value') + }) + + it('preserves an intentional null from source normalization', () => { + Object.assign(definition, { + valueForConversion: () => null, + }) + + expect( + valueForTypeConversion( + 'stored-value', + { name: 'source', type: 'string' }, + { name: 'target', type: 'number' } + ) + ).toBeNull() + }) +}) diff --git a/apps/sim/lib/table/column-types/import-coercion.ts b/apps/sim/lib/table/column-types/import-coercion.ts new file mode 100644 index 00000000000..9bc0768c61e --- /dev/null +++ b/apps/sim/lib/table/column-types/import-coercion.ts @@ -0,0 +1,20 @@ +import { parseTtlEpochSeconds } from '@/lib/table/column-types/ttl' +import type { ColumnType } from '@/lib/table/column-types/types' +import type { NormalizeDateCellOptions } from '@/lib/table/dates' +import type { JsonValue } from '@/lib/table/types' + +type ImportValue = Exclude +type ImportCoercer = (value: unknown, options?: NormalizeDateCellOptions) => ImportValue + +const IMPORT_COERCERS: Partial> = { + ttl: (value, options) => parseTtlEpochSeconds(value, options), +} + +/** Applies lightweight type-specific CSV coercion without loading the full column registry. */ +export function coerceColumnTypeImportValue( + type: ColumnType, + value: unknown, + options?: NormalizeDateCellOptions +): ImportValue | undefined { + return IMPORT_COERCERS[type]?.(value, options) +} diff --git a/apps/sim/lib/table/column-types/registry.server.ts b/apps/sim/lib/table/column-types/registry.server.ts index 6ba27f5c616..5a6e23791ea 100644 --- a/apps/sim/lib/table/column-types/registry.server.ts +++ b/apps/sim/lib/table/column-types/registry.server.ts @@ -273,6 +273,7 @@ export const COLUMN_TYPE_SERVER_REGISTRY: Record = { number: numberColumnType, boolean: booleanColumnType, date: dateColumnType, + ttl: ttlColumnType, json: jsonColumnType, select: selectColumnType, currency: currencyColumnType, @@ -90,6 +94,16 @@ export function isValueCompatible(value: unknown, target: ColumnDefinition): boo return definition.coerce(value as JsonValue, target).ok } +/** Applies source-owned normalization before a value is converted to another type. */ +export function valueForTypeConversion( + value: JsonValue, + source: ColumnDefinition, + target: ColumnDefinition +): JsonValue { + const normalized = columnTypeOf(source).valueForConversion?.(value, target) + return normalized === undefined ? value : normalized +} + /** This type's own metadata errors; types carrying no metadata report none. */ export function validateTypeMetadata(column: ColumnDefinition): string[] { return columnTypeOf(column).validateDefinition?.(column) ?? [] @@ -115,3 +129,31 @@ export function typeMetadataOf(column: ColumnDefinition): Partial | null { return columnTypeOf(column).filterOperatorsFor?.(column) ?? null } + +/** Schema-level cardinality errors declared by column type definitions. */ +export function validateColumnTypeLimits(columns: readonly ColumnDefinition[]): string[] { + const errors: string[] = [] + for (const definition of ALL_COLUMN_TYPES) { + if (definition.maxPerTable === undefined) continue + if (wouldExceedColumnTypeLimit(columns, definition.id)) { + errors.push(`A table can have at most ${definition.maxPerTable} ${definition.label} column`) + } + } + return errors +} + +/** Whether adding columns of a type would exceed its registry-declared table limit. */ +export function wouldExceedColumnTypeLimit( + columns: readonly ColumnDefinition[], + type: ColumnType, + additionalColumns = 0 +): boolean { + const definition = COLUMN_TYPE_REGISTRY[type] + if (definition.maxPerTable === undefined) return false + + const count = columns.reduce( + (total, column) => total + (column.type === type ? 1 : 0), + additionalColumns + ) + return count > definition.maxPerTable +} diff --git a/apps/sim/lib/table/column-types/select.ts b/apps/sim/lib/table/column-types/select.ts index c282d681638..017a470b534 100644 --- a/apps/sim/lib/table/column-types/select.ts +++ b/apps/sim/lib/table/column-types/select.ts @@ -8,7 +8,7 @@ import { splitMultiSelectInput, } from '@/lib/table/select-options' import { selectValueToNames } from '@/lib/table/select-values' -import type { JsonValue } from '@/lib/table/types' +import type { FilterOp, JsonValue } from '@/lib/table/types' /** * Operators that make sense on a `select` column (whose values are opaque option @@ -31,6 +31,37 @@ export const MULTI_SELECT_OPERATORS: ReadonlySet = new Set([ '$empty', ]) +/** + * The same allowlists in the v2 bare-operator grammar, applied inside + * `fieldPredicate` so both wire formats gate identically. Not derived from the + * `$` sets above by string surgery because the mapping is not 1:1 — `$empty` + * splits into `isEmpty`/`isNotEmpty`. `isNull`/`isNotNull` have no `$` + * equivalent and are allowed on both: a strict null check is meaningful on any + * column, select included. + * + * Exported so LLM enrichment can DERIVE the operator guidance it gives the + * model rather than restating it. A hand-copied list drifts, and the cost of + * drift here is a predicate the validator rejects at run time. + */ +export const SINGLE_SELECT_OPS: ReadonlySet = new Set([ + 'eq', + 'ne', + 'in', + 'nin', + 'isEmpty', + 'isNotEmpty', + 'isNull', + 'isNotNull', +]) +export const MULTI_SELECT_OPS: ReadonlySet = new Set([ + 'contains', + 'ncontains', + 'isEmpty', + 'isNotEmpty', + 'isNull', + 'isNotNull', +]) + export const selectColumnType: ColumnTypeDefinition = { id: 'select', label: 'Select', diff --git a/apps/sim/lib/table/column-types/ttl.test.ts b/apps/sim/lib/table/column-types/ttl.test.ts new file mode 100644 index 00000000000..717e96ebb8a --- /dev/null +++ b/apps/sim/lib/table/column-types/ttl.test.ts @@ -0,0 +1,189 @@ +/** + * @vitest-environment node + */ + +import { describe, expect, it } from 'vitest' +import { + formatInstantInTimeZone, + getSupportedTimezones, + zonedWallClockToUtc, +} from '@/lib/core/utils/timezone' +import { parseTtlEpochSeconds, ttlColumnType } from '@/lib/table/column-types/ttl' +import { retypeCellRewrite } from '@/lib/table/columns/service' +import type { ColumnDefinition, JsonValue } from '@/lib/table/types' + +const column = (over: Partial): ColumnDefinition => + ({ name: 'col', type: 'string', ...over }) as ColumnDefinition + +describe('TTL column type', () => { + it('converts epoch seconds to an ISO date before retyping', () => { + expect( + retypeCellRewrite(1_700_000_000, column({ type: 'date' }), column({ type: 'ttl' })) + ).toEqual({ value: '2023-11-14T22:13:20Z' }) + }) + + it('keeps blank and malformed TTL values out of the epoch-zero formatter', () => { + const cases: Array<[unknown, string]> = [ + [null, ''], + [undefined, ''], + ['', ''], + [' ', ' '], + [false, 'false'], + [[], ''], + ] + for (const [value, fallback] of cases) { + expect(ttlColumnType.formatForDisplay(value, column({ type: 'ttl' }))).toBe(fallback) + expect(ttlColumnType.formatForInput(value, column({ type: 'ttl' }))).toBe(fallback) + } + }) + + it('preserves blank and malformed TTL values when converting to a date', () => { + const target = column({ type: 'date' }) + const values: JsonValue[] = [null, '', ' ', false, []] + + for (const value of values) { + expect(ttlColumnType.valueForConversion?.(value, target)).toEqual(value) + } + }) + + it.each([ + ['UTC', '2026-06-15T09:00:30', '2026-06-15T09:00:30.000Z'], + ['America/New_York', '2026-06-15T09:00:30', '2026-06-15T13:00:30.000Z'], + ['America/New_York', '2026-01-15T09:00:30', '2026-01-15T14:00:30.000Z'], + ['Asia/Kathmandu', '2026-06-15T09:00:30', '2026-06-15T03:15:30.000Z'], + ['Australia/Lord_Howe', '2026-06-15T09:00:30', '2026-06-14T22:30:30.000Z'], + ])('stores %s wall-clock input as the expected epoch second', (timezone, input, iso) => { + expect(parseTtlEpochSeconds(input, { timezone })).toBe(Date.parse(iso) / 1000) + }) + + it.each([ + ['America/New_York', '2026-11-01T01:30', '2026-11-01T06:30:00.000Z'], + ['Europe/Berlin', '2026-10-25T02:30', '2026-10-25T01:30:00.000Z'], + ['Australia/Lord_Howe', '2026-04-05T01:45', '2026-04-04T15:15:00.000Z'], + ])( + 'chooses the later expiration when %s repeats a wall-clock time', + (timezone, input, laterInstant) => { + expect(parseTtlEpochSeconds(input, { timezone })).toBe(Date.parse(laterInstant) / 1000) + } + ) + + it.each([ + ['America/New_York', '2026-03-08T02:30', '2026-03-08T07:30:00.000Z'], + ['Europe/Berlin', '2026-03-29T02:30', '2026-03-29T01:30:00.000Z'], + ['Australia/Lord_Howe', '2026-10-04T02:15', '2026-10-03T15:45:00.000Z'], + ])( + 'moves a nonexistent %s wall-clock expiration forward across the gap', + (timezone, input, compatibleInstant) => { + expect(parseTtlEpochSeconds(input, { timezone })).toBe(Date.parse(compatibleInstant) / 1000) + } + ) + + it('rounds fractional instants up so expiration is never stored early', () => { + expect(parseTtlEpochSeconds('2023-11-14T22:13:20.001Z')).toBe(1_700_000_001) + expect(parseTtlEpochSeconds('2023-11-14T22:13:20.999Z')).toBe(1_700_000_001) + expect(parseTtlEpochSeconds('2023-11-14T22:13:20.0001Z')).toBe(1_700_000_001) + expect(parseTtlEpochSeconds('2023-11-14t22:13:20.001Z')).toBe(1_700_000_001) + expect(parseTtlEpochSeconds('2023-11-14t17:13:20.001', { timezone: 'America/New_York' })).toBe( + 1_700_000_001 + ) + expect(parseTtlEpochSeconds(new Date('2023-11-14T22:13:20.001Z'))).toBe(1_700_000_001) + expect(parseTtlEpochSeconds('2023-11-14T22:13:20.000Z')).toBe(1_700_000_000) + }) + + it('rounds historical sub-minute timezone offsets toward a later expiration', () => { + const timezone = 'Africa/Monrovia' + const exactInstant = Date.parse('1970-01-01T00:44:30Z') / 1000 + + expect(parseTtlEpochSeconds('1970-01-01T00:00:00', { timezone })).toBeGreaterThanOrEqual( + exactInstant + ) + + const editable = ttlColumnType.formatForInput(exactInstant, column({ type: 'ttl' }), { + timezone, + }) + expect(editable).toBe('1970-01-01T00:00:00-00:45') + expect(parseTtlEpochSeconds(editable, { timezone })).toBeGreaterThanOrEqual(exactInstant) + }) + + it('never resolves representative wall clocks early in any supported timezone', () => { + for (const timezone of getSupportedTimezones()) { + for (const wallClock of ['1970-01-01T00:00:00', '2026-06-15T09:00:30']) { + const exactSecond = Math.ceil( + zonedWallClockToUtc(wallClock, timezone, { ambiguousTime: 'later' }).getTime() / 1000 + ) + expect( + parseTtlEpochSeconds(wallClock, { timezone }), + `${timezone} ${wallClock}` + ).toBeGreaterThanOrEqual(exactSecond) + } + } + }) + + it('never moves stored epoch seconds earlier when formatted in any supported timezone', () => { + for (const timezone of getSupportedTimezones()) { + for (const seconds of [0, Date.parse('2026-11-01T06:30:00Z') / 1000]) { + const editable = ttlColumnType.formatForInput(seconds, column({ type: 'ttl' }), { + timezone, + }) + expect( + parseTtlEpochSeconds(editable, { timezone }), + `${timezone} ${editable}` + ).toBeGreaterThanOrEqual(seconds) + } + } + }) + + it('uses the timezone supplied for each call rather than a previous setting', () => { + const input = '2026-06-15T09:00:30' + + expect(parseTtlEpochSeconds(input, { timezone: 'America/New_York' })).toBe( + Date.parse('2026-06-15T13:00:30Z') / 1000 + ) + expect(parseTtlEpochSeconds(input, { timezone: 'Asia/Kathmandu' })).toBe( + Date.parse('2026-06-15T03:15:30Z') / 1000 + ) + expect(parseTtlEpochSeconds(input, { timezone: 'America/New_York' })).toBe( + Date.parse('2026-06-15T13:00:30Z') / 1000 + ) + }) + + it('round-trips the same epoch after the editor timezone changes', () => { + const seconds = Date.parse('2026-11-01T06:30:00Z') / 1000 + + for (const timezone of [ + 'UTC', + 'America/Los_Angeles', + 'America/New_York', + 'Asia/Kathmandu', + 'Australia/Lord_Howe', + ]) { + const editable = ttlColumnType.formatForInput(seconds, column({ type: 'ttl' }), { timezone }) + expect(editable).toBe(formatInstantInTimeZone(new Date(seconds * 1000), timezone)) + expect(parseTtlEpochSeconds(editable, { timezone })).toBe(seconds) + } + }) + + it('round-trips a low-year expiration through the editor', () => { + const input = '0050-01-15T12:00:00' + const seconds = parseTtlEpochSeconds(input, { timezone: 'UTC' }) + + expect(seconds).toBe(Date.parse(`${input}Z`) / 1000) + const editable = ttlColumnType.formatForInput(seconds, column({ type: 'ttl' }), { + timezone: 'UTC', + }) + expect(editable).toBe(`${input}Z`) + expect(parseTtlEpochSeconds(editable, { timezone: 'UTC' })).toBe(seconds) + }) + + it('keeps the TTL repeated-hour policy separate from ordinary date behavior', () => { + const input = '2026-11-01T01:30' + const timezone = 'America/New_York' + + expect(zonedWallClockToUtc(input, timezone, { ambiguousTime: 'earlier' }).toISOString()).toBe( + '2026-11-01T05:30:00.000Z' + ) + expect(parseTtlEpochSeconds(input, { timezone })).toBe( + Date.parse('2026-11-01T06:30:00Z') / 1000 + ) + }) +}) diff --git a/apps/sim/lib/table/column-types/ttl.ts b/apps/sim/lib/table/column-types/ttl.ts new file mode 100644 index 00000000000..5b04023e05c --- /dev/null +++ b/apps/sim/lib/table/column-types/ttl.ts @@ -0,0 +1,128 @@ +import { TypeTtl } from '@sim/emcn/icons' +import { formatInstantInTimeZone } from '@/lib/core/utils/timezone' +import type { ColumnTypeDefinition } from '@/lib/table/column-types/types' +import { + formatDateCellDisplay, + type NormalizeDateCellOptions, + normalizeDateCellValue, +} from '@/lib/table/dates' +import type { ColumnDefinition } from '@/lib/table/types' + +const NUMERIC_VALUE_PATTERN = /^-?\d+(?:\.\d+)?$/ +const ISO_DATE_PREFIX_PATTERN = /^(\d{4}-\d{2}-\d{2})(?:$|[T ])/i +const FRACTIONAL_SECONDS_PATTERN = /[T ]\d{1,2}:\d{2}:\d{2}\.(\d+)/i + +function isRepresentableEpochSeconds(value: number): boolean { + return Number.isSafeInteger(value) && !Number.isNaN(new Date(value * 1000).getTime()) +} + +/** Rounds toward the future so integer-second storage can never expire an instant early. */ +function epochSecondAtOrAfter(milliseconds: number): number { + return Math.ceil(milliseconds / 1000) +} + +/** Whether an ISO-shaped input names any instant after its whole second. */ +function hasFractionalSecond(value: string): boolean { + const digits = value.match(FRACTIONAL_SECONDS_PATTERN)?.[1] + return digits ? /[1-9]/.test(digits) : false +} + +/** Converts a TTL cell input to integer Unix epoch seconds. */ +export function parseTtlEpochSeconds( + value: unknown, + options?: NormalizeDateCellOptions +): number | null { + if (typeof value === 'number') return isRepresentableEpochSeconds(value) ? value : null + + if (value instanceof Date) { + const milliseconds = value.getTime() + return Number.isNaN(milliseconds) ? null : epochSecondAtOrAfter(milliseconds) + } + + if (typeof value !== 'string') return null + const trimmed = value.trim() + if (!trimmed) return null + + if (NUMERIC_VALUE_PATTERN.test(trimmed)) { + const numeric = Number(trimmed) + return isRepresentableEpochSeconds(numeric) ? numeric : null + } + + const ttlOptions: NormalizeDateCellOptions = { + ...options, + ambiguousTime: 'later', + offsetMinuteRounding: 'floor', + } + const normalized = normalizeDateCellValue(trimmed, ttlOptions) + if (normalized === null) return null + const instant = /^\d{4}-\d{2}-\d{2}$/.test(normalized) + ? normalizeDateCellValue(`${normalized}T00:00:00`, ttlOptions) + : normalized + if (instant === null) return null + const inputIsoDate = trimmed.match(ISO_DATE_PREFIX_PATTERN)?.[1] + if (inputIsoDate && instant.slice(0, 10) !== inputIsoDate) return null + const milliseconds = Date.parse(instant) + (hasFractionalSecond(trimmed) ? 1 : 0) + if (Number.isNaN(milliseconds)) return null + const seconds = epochSecondAtOrAfter(milliseconds) + return isRepresentableEpochSeconds(seconds) ? seconds : null +} + +function epochSecondsToIso(value: unknown): string | null { + if ( + typeof value !== 'number' && + (typeof value !== 'string' || !NUMERIC_VALUE_PATTERN.test(value.trim())) + ) { + return null + } + const seconds = typeof value === 'number' ? value : Number(value) + if (!isRepresentableEpochSeconds(seconds)) return null + return new Date(seconds * 1000).toISOString().replace('.000Z', 'Z') +} + +function epochSecondsToEditable(value: unknown, timeZone?: string): string | null { + const iso = epochSecondsToIso(value) + if (!iso || !timeZone) return iso + return formatInstantInTimeZone(new Date(iso), timeZone, { offsetMinuteRounding: 'floor' }) +} + +export const ttlColumnType: ColumnTypeDefinition = { + id: 'ttl', + label: 'Expiration', + maxPerTable: 1, + icon: TypeTtl, + jsonbCast: 'numeric', + storesOpaqueIds: false, + supportsUnique: true, + sampleValue: 1_706_659_200, + ownedMetadata: [], + workflowInputType: 'number', + editor: 'date', + expandable: false, + typeaheadPattern: /[\d\-/]/, + parseErrorMessage: 'Invalid expiration date', + + coerce(value, _column, context) { + const seconds = parseTtlEpochSeconds(value, context) + return seconds === null ? { ok: false } : { ok: true, value: seconds } + }, + + valueForConversion(value, target: ColumnDefinition) { + if (target.type !== 'date') return value + return epochSecondsToIso(value) ?? value + }, + + validateCell(value, column) { + return typeof value === 'number' && isRepresentableEpochSeconds(value) + ? null + : `${column.name} must be valid epoch seconds` + }, + + formatForDisplay(value) { + const iso = epochSecondsToIso(value) + return iso === null ? String(value ?? '') : formatDateCellDisplay(iso, { seconds: true }) + }, + + formatForInput(value, _column, context) { + return epochSecondsToEditable(value, context?.timezone) ?? String(value ?? '') + }, +} diff --git a/apps/sim/lib/table/column-types/types.ts b/apps/sim/lib/table/column-types/types.ts index f481cea0a28..72edeead0d0 100644 --- a/apps/sim/lib/table/column-types/types.ts +++ b/apps/sim/lib/table/column-types/types.ts @@ -20,6 +20,7 @@ */ import type React from 'react' +import type { NormalizeDateCellOptions } from '@/lib/table/dates' import type { ColumnDefinition, JsonValue } from '@/lib/table/types' /** @@ -36,6 +37,7 @@ export const COLUMN_TYPES = [ 'currency', 'boolean', 'date', + 'ttl', 'json', 'select', ] as const @@ -72,6 +74,8 @@ export interface ColumnTypeDefinition { /** Human label in the type picker, column header menu, and docs. */ readonly label: string + /** Maximum columns of this type a table may contain. Omitted when unlimited. */ + readonly maxPerTable?: number /** Type icon. A component reference only — never invoked server-side. */ readonly icon: React.ComponentType<{ className?: string }> /** @@ -157,7 +161,14 @@ export interface ColumnTypeDefinition { * implementation — the server calls it before persisting and the grid calls * it to fill the optimistic cache, so the two can no longer disagree. */ - coerce(value: JsonValue, column: ColumnDefinition): CoerceResult + coerce( + value: JsonValue, + column: ColumnDefinition, + context?: NormalizeDateCellOptions + ): CoerceResult + + /** Source-owned normalization applied before checking or rewriting a type conversion. */ + valueForConversion?(value: JsonValue, target: ColumnDefinition): JsonValue /** Validates a stored cell's shape. Returns an error message, or null when valid. */ validateCell(value: JsonValue, column: ColumnDefinition): string | null @@ -203,7 +214,11 @@ export interface ColumnTypeDefinition { formatForDisplay(value: unknown, column: ColumnDefinition): string /** Stored value → the text an editor input starts with. */ - formatForInput(value: unknown, column: ColumnDefinition): string + formatForInput( + value: unknown, + column: ColumnDefinition, + context?: NormalizeDateCellOptions + ): string /** * Metadata stamped onto a newly created column of this type, so the schema diff --git a/apps/sim/lib/table/columns/retype-cell.test.ts b/apps/sim/lib/table/columns/retype-cell.test.ts index 479563fc74b..b1b6a74d888 100644 --- a/apps/sim/lib/table/columns/retype-cell.test.ts +++ b/apps/sim/lib/table/columns/retype-cell.test.ts @@ -2,13 +2,25 @@ * @vitest-environment node */ -import { describe, expect, it } from 'vitest' +import { afterEach, describe, expect, it } from 'vitest' +import { COLUMN_TYPE_REGISTRY } from '@/lib/table/column-types' import { retypeCellRewrite } from '@/lib/table/columns/service' import type { ColumnDefinition } from '@/lib/table/types' const column = (over: Partial): ColumnDefinition => ({ name: 'col', type: 'string', ...over }) as ColumnDefinition +const sourceDefinition = COLUMN_TYPE_REGISTRY.string +const originalValueForConversion = sourceDefinition.valueForConversion + +afterEach(() => { + if (originalValueForConversion === undefined) { + Reflect.deleteProperty(sourceDefinition, 'valueForConversion') + return + } + Object.assign(sourceDefinition, { valueForConversion: originalValueForConversion }) +}) + describe('retypeCellRewrite', () => { it('preserves an empty string the target type can hold', () => { // `''` is a real stored value: `coerceRowValues` keeps it for `string`, and @@ -32,6 +44,29 @@ describe('retypeCellRewrite', () => { expect(retypeCellRewrite('true', column({ type: 'boolean' }))).toEqual({ value: true }) }) + it('writes back null produced by source normalization', () => { + Object.assign(sourceDefinition, { valueForConversion: () => null }) + + expect( + retypeCellRewrite('stored-value', column({ type: 'number' }), column({ type: 'string' })) + ).toEqual({ value: null }) + }) + + it('coerces source-normalized values into select storage', () => { + Object.assign(sourceDefinition, { valueForConversion: () => 'Choice' }) + + expect( + retypeCellRewrite( + 'stored-value', + column({ + type: 'select', + options: [{ id: 'opt_choice', name: 'Choice' }], + }), + column({ type: 'string' }) + ) + ).toEqual({ value: 'opt_choice' }) + }) + it('skips a cell whose stored value already matches the coercion', () => { expect(retypeCellRewrite('kept', column({ type: 'json' }))).toBeNull() expect(retypeCellRewrite(3, column({ type: 'json' }))).toBeNull() diff --git a/apps/sim/lib/table/columns/service.ts b/apps/sim/lib/table/columns/service.ts index b68ce8393ac..fa0a274d146 100644 --- a/apps/sim/lib/table/columns/service.ts +++ b/apps/sim/lib/table/columns/service.ts @@ -27,6 +27,7 @@ import { columnTypeOf, isValueCompatible, TYPE_SPECIFIC_COLUMN_KEYS, + valueForTypeConversion, } from '@/lib/table/column-types' import { migrationFrom, @@ -42,6 +43,7 @@ import { updateTableRowsWithDerivedSecretProvenance } from '@/lib/table/rows/sec import { assertValidSchema } from '@/lib/table/schema-invariants' import { selectValueToNames } from '@/lib/table/select-values' import { withLockedTable } from '@/lib/table/service' +import { assertTableRowTtlEnabled } from '@/lib/table/ttl-availability' import { scaledStatementTimeoutMs, setTableTxTimeouts } from '@/lib/table/tx' import type { ColumnDefinition, @@ -130,6 +132,8 @@ export async function addTableColumn( requestId: string, options?: ColumnMutationOptions ): Promise { + if (column.type === 'ttl') await assertTableRowTtlEnabled() + return withLockedTable( tableId, async (table, trx) => { @@ -767,17 +771,24 @@ export function applyPendingRename( */ export function retypeCellRewrite( value: unknown, - target: ColumnDefinition + target: ColumnDefinition, + source?: ColumnDefinition ): { value: JsonValue } | null { if (value === null || value === undefined) return null - if (!isValueCompatibleWithColumn(value, target)) { + const effective = source + ? valueForTypeConversion(value as JsonValue, source, target) + : (value as JsonValue) + + if (effective === null) return { value: null } + + if (!isValueCompatibleWithColumn(effective, target)) { // Incompatible non-blanks never reach here: the compatibility scan already // refused the whole conversion for them. - return value === '' ? { value: null } : null + return effective === '' ? { value: null } : null } - const coerced = columnTypeById(target.type).coerce(value as JsonValue, target) + const coerced = columnTypeById(target.type).coerce(effective, target) if (coerced.ok && !Object.is(coerced.value, value)) return { value: coerced.value } return null } @@ -849,6 +860,8 @@ export async function updateColumnType( requestId: string, options?: ColumnMutationOptions ): Promise { + if (data.newType === 'ttl') await assertTableRowTtlEnabled() + return withLockedTable( data.tableId, async (table, trx) => { @@ -913,6 +926,7 @@ export async function updateColumnType( const isSelectType = data.newType === 'select' const targetOptions = data.options ?? column.options ?? [] const targetMultiple = data.multiple ?? column.multiple + const sourceNormalizesConversion = columnTypeOf(column).valueForConversion !== undefined // Leaving `select` behind: stored cells hold option ids, which mean nothing // once the column is text/number/etc. Check compatibility against the option // NAME — that's what the cell will actually become (migrated below). @@ -944,6 +958,12 @@ export async function updateColumnType( isSelectType, targetMultiple: !!targetMultiple, }) + const renamedColumns = schema.columns.map((c, i) => (i === columnIndex ? convertedColumn : c)) + const updatedColumns = renamedColumns.map((c, i) => + i === columnIndex ? applyPendingRename(renamedColumns, columnIndex, data.newName) : c + ) + const updatedSchema: TableSchema = { ...schema, columns: updatedColumns } + assertValidSchema(updatedSchema, table.metadata?.columnOrder) let incompatibleCount = 0 let blankCount = 0 @@ -972,7 +992,7 @@ export async function updateColumnType( const effective = convertingAwayFromSelect ? selectValueForConversion(column, value) - : value + : valueForTypeConversion(value as JsonValue, column, convertedColumn) if (!isValueCompatibleWithColumn(effective, convertedColumn)) { if (effective === null || effective === '') { @@ -1000,11 +1020,6 @@ export async function updateColumnType( ) } - const renamedColumns = schema.columns.map((c, i) => (i === columnIndex ? convertedColumn : c)) - const updatedColumns = renamedColumns.map((c, i) => - i === columnIndex ? applyPendingRename(renamedColumns, columnIndex, data.newName) : c - ) - const columnValidation = validateColumnDefinition(updatedColumns[columnIndex]) if (!columnValidation.valid) { throw new OrchestrationError( @@ -1013,7 +1028,6 @@ export async function updateColumnType( ) } - const updatedSchema: TableSchema = { ...schema, columns: updatedColumns } const now = new Date() // Cell rewrites are owned by the column-type registry, keyed by direction. @@ -1029,9 +1043,7 @@ export async function updateColumnType( resolved: new Map(), } await migrationFrom(column.type)?.(migrationContext) - if (isSelectType) { - await migrationTo(data.newType)?.(migrationContext) - } else { + if (!isSelectType || sourceNormalizesConversion) { let rewriteAfterId: string | undefined while (true) { const rows = await readColumnRetypePage( @@ -1045,7 +1057,7 @@ export async function updateColumnType( if (rows.length === 0) break const coercedByRowId = new Map() for (const row of rows) { - const rewrite = retypeCellRewrite(row.value, convertedColumn) + const rewrite = retypeCellRewrite(row.value, convertedColumn, column) if (rewrite) coercedByRowId.set(row.id, rewrite.value) } await writeBackCoercedCells( @@ -1059,6 +1071,9 @@ export async function updateColumnType( if (rows.length < retypeScanBatchSize) break } } + if (isSelectType) { + await migrationTo(data.newType)?.(migrationContext) + } // A `unique` arriving with this retype is validated HERE, against the values // the conversion just wrote — not by the separate constraint write that diff --git a/apps/sim/lib/table/columns/ttl-limit.test.ts b/apps/sim/lib/table/columns/ttl-limit.test.ts new file mode 100644 index 00000000000..8738e7eee49 --- /dev/null +++ b/apps/sim/lib/table/columns/ttl-limit.test.ts @@ -0,0 +1,99 @@ +/** + * @vitest-environment node + */ +import { beforeEach, describe, expect, it, vi } from 'vitest' +import type { TableDefinition, TableLocks } from '@/lib/table/types' + +const { mockAssertTableRowTtlEnabled, mockTimeoutExecute, mockWithLockedTable } = vi.hoisted( + () => ({ + mockAssertTableRowTtlEnabled: vi.fn(), + mockTimeoutExecute: vi.fn(), + mockWithLockedTable: vi.fn(), + }) +) + +vi.mock('@/lib/table/service', () => ({ withLockedTable: mockWithLockedTable })) +vi.mock('@/lib/table/ttl-availability', () => ({ + assertTableRowTtlEnabled: mockAssertTableRowTtlEnabled, +})) + +import { addTableColumn, updateColumnType } from '@/lib/table/columns/service' + +const UNLOCKED: TableLocks = { + schemaLocked: false, + insertLocked: false, + updateLocked: false, + deleteLocked: false, +} + +function makeTable(): TableDefinition { + return { + id: 'table-1', + name: 'Tasks', + schema: { + columns: [ + { id: 'col-name', name: 'name', type: 'string' }, + { id: 'col-ttl', name: 'expires_at', type: 'ttl' }, + ], + }, + rowCount: 0, + maxRows: 100, + workspaceId: 'workspace-1', + createdBy: 'user-1', + locks: UNLOCKED, + createdAt: new Date(), + updatedAt: new Date(), + } +} + +const transaction = new Proxy( + { execute: mockTimeoutExecute }, + { + get(target, property) { + if (property in target) return target[property as keyof typeof target] + throw new Error(`Unexpected transaction method: ${String(property)}`) + }, + } +) + +describe('TTL column mutation limit', () => { + beforeEach(() => { + vi.clearAllMocks() + mockAssertTableRowTtlEnabled.mockResolvedValue(undefined) + mockTimeoutExecute.mockResolvedValue([]) + mockWithLockedTable.mockImplementation(async (_tableId, mutate) => + mutate(makeTable(), transaction) + ) + }) + + it('rejects adding a TTL column before locking when the feature is disabled', async () => { + mockAssertTableRowTtlEnabled.mockRejectedValue(new Error('Expiration columns are not enabled')) + + await expect( + addTableColumn('table-1', { name: 'expiry', type: 'ttl' }, 'request-1') + ).rejects.toThrow('Expiration columns are not enabled') + expect(mockWithLockedTable).not.toHaveBeenCalled() + }) + + it('rejects retyping to TTL before locking when the feature is disabled', async () => { + mockAssertTableRowTtlEnabled.mockRejectedValue(new Error('Expiration columns are not enabled')) + + await expect( + updateColumnType({ tableId: 'table-1', columnName: 'name', newType: 'ttl' }, 'request-1') + ).rejects.toThrow('Expiration columns are not enabled') + expect(mockWithLockedTable).not.toHaveBeenCalled() + }) + + it('rejects adding a second TTL column before persistence', async () => { + await expect( + addTableColumn('table-1', { name: 'another_expiry', type: 'ttl' }, 'request-1') + ).rejects.toThrow('A table can have at most 1 Expiration column') + }) + + it('rejects retyping another column to TTL before scanning cells', async () => { + await expect( + updateColumnType({ tableId: 'table-1', columnName: 'name', newType: 'ttl' }, 'request-1') + ).rejects.toThrow('A table can have at most 1 Expiration column') + expect(mockTimeoutExecute).toHaveBeenCalled() + }) +}) diff --git a/apps/sim/lib/table/constants.test.ts b/apps/sim/lib/table/constants.test.ts index d3bd1ef4848..050246dddb2 100644 --- a/apps/sim/lib/table/constants.test.ts +++ b/apps/sim/lib/table/constants.test.ts @@ -39,7 +39,9 @@ declare module '@/lib/table/constants?constants-test' { import { getBillingDisabledTableLimits, + getDeleteSnapshotBatchSize, getMaxPageBytes, + getMaxRowSizeBytes, TABLE_LIMITS, } from '@/lib/table/constants?constants-test' @@ -86,3 +88,35 @@ describe('getMaxPageBytes', () => { expect(getMaxPageBytes()).toBe(2 * 1024 * 1024) }) }) + +describe('getMaxRowSizeBytes', () => { + beforeEach(() => { + for (const key of Object.keys(mockEnv)) delete mockEnv[key] + }) + + it('caps overrides at the delete snapshot byte budget', () => { + mockEnv.TABLE_MAX_ROW_SIZE_BYTES = String(TABLE_LIMITS.DELETE_SNAPSHOT_BATCH_MAX_BYTES * 2) + + expect(getMaxRowSizeBytes()).toBe(TABLE_LIMITS.DELETE_SNAPSHOT_BATCH_MAX_BYTES) + }) +}) + +describe('getDeleteSnapshotBatchSize', () => { + beforeEach(() => { + for (const key of Object.keys(mockEnv)) delete mockEnv[key] + }) + + it('derives a worst-case row cap from the delete snapshot byte budget', () => { + expect(getDeleteSnapshotBatchSize()).toBe( + Math.floor(TABLE_LIMITS.DELETE_SNAPSHOT_BATCH_MAX_BYTES / TABLE_LIMITS.MAX_ROW_SIZE_BYTES) + ) + }) + + it('always processes one row and never exceeds the delete row-count cap', () => { + mockEnv.TABLE_MAX_ROW_SIZE_BYTES = String(TABLE_LIMITS.DELETE_SNAPSHOT_BATCH_MAX_BYTES * 2) + expect(getDeleteSnapshotBatchSize()).toBe(1) + + mockEnv.TABLE_MAX_ROW_SIZE_BYTES = '1' + expect(getDeleteSnapshotBatchSize()).toBe(TABLE_LIMITS.DELETE_BATCH_SIZE) + }) +}) diff --git a/apps/sim/lib/table/constants.ts b/apps/sim/lib/table/constants.ts index 2da9bf6b58a..56e1f25c05b 100644 --- a/apps/sim/lib/table/constants.ts +++ b/apps/sim/lib/table/constants.ts @@ -39,6 +39,14 @@ export const TABLE_LIMITS = { UPDATE_BATCH_SIZE: 100, /** Batch size for bulk delete operations */ DELETE_BATCH_SIZE: 1000, + /** + * Serialized row-data budget for one committed delete snapshot batch. Batch + * deletes measure stored JSONB bytes while holding row locks and stop at this + * budget. A historical row already larger than the budget is deleted alone + * and logged; current writes cannot create another because row admission is + * capped at the same value. + */ + DELETE_SNAPSHOT_BATCH_MAX_BYTES: 32 * 1024 * 1024, /** Maximum rows per batch insert */ MAX_BATCH_INSERT_SIZE: 1000, /** Maximum rows per bulk update/delete operation */ @@ -140,13 +148,33 @@ export function getMaxPageBytes(): number { /** * Maximum serialized size in bytes of a single row. Defaults to * `TABLE_LIMITS.MAX_ROW_SIZE_BYTES`; overridable via the - * `TABLE_MAX_ROW_SIZE_BYTES` env var (server-only, read at call time). + * `TABLE_MAX_ROW_SIZE_BYTES` env var (server-only, read at call time), capped + * at the delete snapshot budget so every accepted row fits in one batch. */ export function getMaxRowSizeBytes(): number { - return envNumber(env.TABLE_MAX_ROW_SIZE_BYTES, TABLE_LIMITS.MAX_ROW_SIZE_BYTES, { - min: 1, - integer: true, - }) + return Math.min( + envNumber(env.TABLE_MAX_ROW_SIZE_BYTES, TABLE_LIMITS.MAX_ROW_SIZE_BYTES, { + min: 1, + integer: true, + }), + TABLE_LIMITS.DELETE_SNAPSHOT_BATCH_MAX_BYTES + ) +} + +/** + * Initial row-count cap for a delete snapshot batch. Delete paths additionally + * measure the selected rows as stored and shorten each transaction to the byte + * budget; this count avoids scanning more candidate ids than current writes can + * possibly fit. + */ +export function getDeleteSnapshotBatchSize(): number { + return Math.max( + 1, + Math.min( + TABLE_LIMITS.DELETE_BATCH_SIZE, + Math.floor(TABLE_LIMITS.DELETE_SNAPSHOT_BATCH_MAX_BYTES / getMaxRowSizeBytes()) + ) + ) } export type PlanName = keyof typeof DEFAULT_TABLE_PLAN_LIMITS diff --git a/apps/sim/lib/table/dates.test.ts b/apps/sim/lib/table/dates.test.ts index 3ff51410e15..36df539d5c3 100644 --- a/apps/sim/lib/table/dates.test.ts +++ b/apps/sim/lib/table/dates.test.ts @@ -22,6 +22,8 @@ function localOffsetSuffix(local: Date): string { describe('isCalendarDateString', () => { it('accepts YYYY-MM-DD and rejects everything else', () => { expect(isCalendarDateString('2026-07-06')).toBe(true) + expect(isCalendarDateString('2024-02-29')).toBe(true) + expect(isCalendarDateString('2026-02-30')).toBe(false) expect(isCalendarDateString('2026-13-45')).toBe(false) expect(isCalendarDateString('2026-07-06T00:00:00Z')).toBe(false) expect(isCalendarDateString('07/06/2026')).toBe(false) @@ -80,6 +82,71 @@ describe('normalizeDateCellValue', () => { ) }) + it('uses the requested low year when applying IANA timezone rules', () => { + const normalized = normalizeDateCellValue('0050-01-15T12:00:00', { + timezone: 'America/New_York', + }) + + expect(normalized).toBe('0050-01-15T12:00:00-04:56') + expect(storedDateToEditable(normalized ?? '')).toBe('0050-01-15T12:00:00-04:56') + }) + + it('reads localized numeric wall clocks before applying the provided IANA zone', () => { + expect(normalizeDateCellValue('3/8/2026 2:30 AM', { timezone: 'America/New_York' })).toBe( + '2026-03-08T02:30:00-05:00' + ) + expect(normalizeDateCellValue('7/6/2026, 16:04:55', { timezone: 'Asia/Tokyo' })).toBe( + '2026-07-06T16:04:55+09:00' + ) + }) + + it('reads month-name wall clocks independently of the runtime timezone', () => { + expect(normalizeDateCellValue('March 8, 2026 2:30 AM', { timezone: 'America/New_York' })).toBe( + '2026-03-08T02:30:00-05:00' + ) + }) + + it('rejects impossible month-name calendar dates', () => { + expect( + normalizeDateCellValue('February 29, 2025 2:30 AM', { timezone: 'America/New_York' }) + ).toBeNull() + expect( + normalizeDateCellValue('April 31, 2026 4:04 PM', { timezone: 'America/New_York' }) + ).toBeNull() + }) + + it('accepts valid leap-day month-name wall clocks in either date order', () => { + expect( + normalizeDateCellValue('February 29, 2024 4:04 PM', { timezone: 'America/New_York' }) + ).toBe('2024-02-29T16:04:00-05:00') + expect(normalizeDateCellValue('29 Feb 2024 4:04 PM', { timezone: 'America/New_York' })).toBe( + '2024-02-29T16:04:00-05:00' + ) + }) + + it.each([ + ['America/New_York', '2026-11-01 01:30:00', '2026-11-01T01:30:00-04:00'], + ['America/New_York', '2026-03-08 02:30:00', '2026-03-08T02:30:00-05:00'], + ['Asia/Kathmandu', '2026-06-15 09:00:00', '2026-06-15T09:00:00+05:45'], + ['Australia/Lord_Howe', '2026-06-15 09:00:00', '2026-06-15T09:00:00+10:30'], + ])('uses the shared timezone rules for %s', (timezone, input, expected) => { + expect(normalizeDateCellValue(input, { timezone })).toBe(expected) + }) + + it('uses each provided timezone independently when the setting changes', () => { + const input = '2026-06-15 09:00:30' + + expect(normalizeDateCellValue(input, { timezone: 'America/New_York' })).toBe( + '2026-06-15T09:00:30-04:00' + ) + expect(normalizeDateCellValue(input, { timezone: 'Asia/Kathmandu' })).toBe( + '2026-06-15T09:00:30+05:45' + ) + expect(normalizeDateCellValue(input, { timezone: 'America/New_York' })).toBe( + '2026-06-15T09:00:30-04:00' + ) + }) + it('ignores the zone option when the input carries an explicit offset', () => { expect( normalizeDateCellValue('2026-07-06T23:04:55.000Z', { timezone: 'America/New_York' }) @@ -107,6 +174,28 @@ describe('normalizeDateCellValue', () => { expect(normalizeDateCellValue('2026-13-45')).toBeNull() expect(normalizeDateCellValue('13/06/2026')).toBeNull() }) + + it('rejects impossible ISO calendar and time fields', () => { + expect(normalizeDateCellValue('2026-02-30')).toBeNull() + expect(normalizeDateCellValue('2025-02-29T12:00:00Z')).toBeNull() + expect(normalizeDateCellValue('2026-02-30 12:00', { timezone: 'UTC' })).toBeNull() + expect(normalizeDateCellValue('2026-02-30 12:00 PDT')).toBeNull() + expect(normalizeDateCellValue('2026-07-06T24:00', { timezone: 'UTC' })).toBeNull() + expect(normalizeDateCellValue('2026-07-06 24:00+00')).toBeNull() + expect(normalizeDateCellValue('2026-07-06T12:60:00-04:00')).toBeNull() + expect(normalizeDateCellValue('02/30/2026')).toBeNull() + expect(normalizeDateCellValue('February 29, 2025')).toBeNull() + expect(normalizeDateCellValue('February 29, 2025 12:00')).toBeNull() + expect(normalizeDateCellValue('February 29, 2025 12:00', { timezone: 'UTC' })).toBeNull() + }) + + it('accepts leap days and valid daylight-saving gap wall clocks', () => { + expect(normalizeDateCellValue('2024-02-29')).toBe('2024-02-29') + expect(normalizeDateCellValue('2024-02-29T12:00:00Z')).toBe('2024-02-29T12:00:00Z') + expect(normalizeDateCellValue('2026-03-08T02:30:00', { timezone: 'America/New_York' })).toBe( + '2026-03-08T02:30:00-05:00' + ) + }) }) describe('formatDateCellDisplay', () => { diff --git a/apps/sim/lib/table/dates.ts b/apps/sim/lib/table/dates.ts index a38eca7f21e..f112ad2f242 100644 --- a/apps/sim/lib/table/dates.ts +++ b/apps/sim/lib/table/dates.ts @@ -23,7 +23,15 @@ * barrel (the barrel is server-tainted). */ -const CALENDAR_DATE_PATTERN = /^\d{4}-\d{2}-\d{2}$/ +import { + formatIsoYear, + formatUtcOffsetSuffix, + type ZonedWallClockOptions, + zonedWallClockWithOffset, +} from '@/lib/core/utils/timezone' + +const CALENDAR_DATE_PATTERN = /^(\d{4})-(\d{2})-(\d{2})$/ +const LOCALIZED_CALENDAR_DATE_PATTERN = /^(\d{1,2})\/(\d{1,2})\/(\d{4})$/ /** * Canonical (or canonical-enough legacy) instant: a literal wall time with an @@ -31,7 +39,35 @@ const CALENDAR_DATE_PATTERN = /^\d{4}-\d{2}-\d{2}$/ * groups are the wall-time fields display renders verbatim. */ const WALL_INSTANT_PATTERN = - /^(\d{4})-(\d{2})-(\d{2})T(\d{2}):(\d{2})(?::(\d{2}))?(?:\.\d+)?(?:Z|[+-]\d{2}:?\d{2})?$/ + /^(\d{4})-(\d{2})-(\d{2})[T ](\d{2}):(\d{2})(?::(\d{2}))?(?:\.\d+)?(?:\s*(?:Z|UTC?|GMT|[ECMP][SD]T)|[+-]\d{1,2}(?::?\d{2})?)?$/i + +const LOCALIZED_WALL_CLOCK_PATTERN = + /^(\d{1,2})\/(\d{1,2})\/(\d{4})[ ,]+(\d{1,2}):(\d{2})(?::(\d{2}))?(?:\s*(AM|PM))?$/i + +const MONTH_NAME_PATTERN = + 'Jan(?:uary)?|Feb(?:ruary)?|Mar(?:ch)?|Apr(?:il)?|May|Jun(?:e)?|Jul(?:y)?|Aug(?:ust)?|Sep(?:t(?:ember)?)?|Oct(?:ober)?|Nov(?:ember)?|Dec(?:ember)?' +const MONTH_FIRST_DATE_PATTERN = new RegExp( + `\\b(${MONTH_NAME_PATTERN})\\s+(\\d{1,2})(?:,)?\\s+(\\d{4})\\b`, + 'i' +) +const DAY_FIRST_DATE_PATTERN = new RegExp( + `\\b(\\d{1,2})\\s+(${MONTH_NAME_PATTERN})(?:,)?\\s+(\\d{4})\\b`, + 'i' +) +const MONTH_BY_ABBREVIATION: Record = { + JAN: 1, + FEB: 2, + MAR: 3, + APR: 4, + MAY: 5, + JUN: 6, + JUL: 7, + AUG: 8, + SEP: 9, + OCT: 10, + NOV: 11, + DEC: 12, +} /** * Legacy shape: old CSV imports stored date-only columns as UTC-midnight @@ -67,81 +103,10 @@ const US_ABBREVIATION_OFFSET_MINUTES: Record = { /** True when `value` is a canonical timezone-free calendar date. */ export function isCalendarDateString(value: string): boolean { - return CALENDAR_DATE_PATTERN.test(value) && !Number.isNaN(Date.parse(value)) -} - -/** A wall-clock reading of an instant in some timezone. */ -export interface WallClockParts { - year: number - /** 1-based month. */ - month: number - day: number - hour: number - minute: number - second: number -} - -/** - * The wall-clock reading of `date` in `timeZone` — or in the runtime's local - * zone when omitted. Throws a RangeError on an invalid IANA zone — callers - * validate at the boundary. - */ -export function getWallClockParts(date: Date, timeZone?: string): WallClockParts { - if (!timeZone) { - return { - year: date.getFullYear(), - month: date.getMonth() + 1, - day: date.getDate(), - hour: date.getHours(), - minute: date.getMinutes(), - second: date.getSeconds(), - } - } - const parts = new Intl.DateTimeFormat('en-US', { - timeZone, - hourCycle: 'h23', - year: 'numeric', - month: '2-digit', - day: '2-digit', - hour: '2-digit', - minute: '2-digit', - second: '2-digit', - }).formatToParts(date) - const get = (type: string) => Number(parts.find((p) => p.type === type)?.value) - return { - year: get('year'), - month: get('month'), - day: get('day'), - hour: get('hour'), - minute: get('minute'), - second: get('second'), - } -} - -/** Offset of `timeZone` from UTC (ms east) at the moment `at`. */ -function zoneOffsetMs(timeZone: string, at: Date): number { - const wall = getWallClockParts(at, timeZone) - const asUtc = Date.UTC(wall.year, wall.month - 1, wall.day, wall.hour, wall.minute, wall.second) - return asUtc - at.getTime() -} - -/** - * Converts a wall-clock reading in `timeZone` to the UTC instant it denotes. - * Two-pass so readings near a DST transition resolve with the offset in - * force at that wall time. - */ -function wallTimeInZoneToUtc(wall: Date, timeZone: string): Date { - const guess = Date.UTC( - wall.getFullYear(), - wall.getMonth(), - wall.getDate(), - wall.getHours(), - wall.getMinutes(), - wall.getSeconds(), - wall.getMilliseconds() + const calendar = value.match(CALENDAR_DATE_PATTERN) + return Boolean( + calendar && isValidCalendarDay(Number(calendar[1]), Number(calendar[2]), Number(calendar[3])) ) - const adjusted = guess - zoneOffsetMs(timeZone, new Date(guess)) - return new Date(guess - zoneOffsetMs(timeZone, new Date(adjusted))) } function pad(n: number): string { @@ -149,19 +114,11 @@ function pad(n: number): string { } function toLocalCalendarDate(date: Date): string { - return `${date.getFullYear()}-${pad(date.getMonth() + 1)}-${pad(date.getDate())}` + return `${formatIsoYear(date.getFullYear())}-${pad(date.getMonth() + 1)}-${pad(date.getDate())}` } function toUtcCalendarDate(date: Date): string { - return `${date.getUTCFullYear()}-${pad(date.getUTCMonth() + 1)}-${pad(date.getUTCDate())}` -} - -/** `Z` for zero, else `±HH:MM`. */ -function formatOffsetSuffix(offsetMinutes: number): string { - if (offsetMinutes === 0) return 'Z' - const sign = offsetMinutes > 0 ? '+' : '-' - const abs = Math.abs(offsetMinutes) - return `${sign}${pad(Math.floor(abs / 60))}:${pad(abs % 60)}` + return `${formatIsoYear(date.getUTCFullYear())}-${pad(date.getUTCMonth() + 1)}-${pad(date.getUTCDate())}` } /** @@ -186,14 +143,118 @@ function extractExplicitOffsetMinutes(value: string): number | null { function formatUtcFieldsAsWall(shifted: Date, offsetMinutes: number): string { return `${toUtcCalendarDate(shifted)}T${pad(shifted.getUTCHours())}:${pad( shifted.getUTCMinutes() - )}:${pad(shifted.getUTCSeconds())}${formatOffsetSuffix(offsetMinutes)}` + )}:${pad(shifted.getUTCSeconds())}${formatUtcOffsetSuffix(offsetMinutes)}` } /** Serializes local-read fields of `parsed` as a wall time with `offset`. */ function formatLocalFieldsAsWall(parsed: Date, offsetMinutes: number): string { return `${toLocalCalendarDate(parsed)}T${pad(parsed.getHours())}:${pad( parsed.getMinutes() - )}:${pad(parsed.getSeconds())}${formatOffsetSuffix(offsetMinutes)}` + )}:${pad(parsed.getSeconds())}${formatUtcOffsetSuffix(offsetMinutes)}` +} + +/** True when numeric year, month, and day fields describe a real calendar day. */ +function isValidCalendarDay(year: number, month: number, day: number): boolean { + if (month < 1 || month > 12 || day < 1) return false + const leapYear = year % 4 === 0 && (year % 100 !== 0 || year % 400 === 0) + const daysInMonth = [31, leapYear ? 29 : 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31] + return day <= daysInMonth[month - 1] +} + +/** Validates and formats numeric wall-clock fields as naive ISO. */ +function formatValidatedWallClock( + year: number, + month: number, + day: number, + hour: number, + minute: number, + second: number +): string | null { + if ( + !isValidCalendarDay(year, month, day) || + hour < 0 || + hour > 23 || + minute < 0 || + minute > 59 || + second < 0 || + second > 59 + ) { + return null + } + return `${String(year).padStart(4, '0')}-${pad(month)}-${pad(day)}T${pad(hour)}:${pad(minute)}:${pad(second)}` +} + +/** Reads an ISO-shaped wall clock literally, before runtime timezone normalization. */ +function parseIsoWallClock(match: RegExpMatchArray): string | null { + return formatValidatedWallClock( + Number(match[1]), + Number(match[2]), + Number(match[3]), + Number(match[4]), + Number(match[5]), + Number(match[6] ?? 0) + ) +} + +/** Reads a supported US numeric wall clock literally, including 12-hour input. */ +function parseLocalizedWallClock(match: RegExpMatchArray): string | null { + const meridiem = match[7]?.toUpperCase() + let hour = Number(match[4]) + if (meridiem) { + if (hour < 1 || hour > 12) return null + hour = (hour % 12) + (meridiem === 'PM' ? 12 : 0) + } + return formatValidatedWallClock( + Number(match[3]), + Number(match[1]), + Number(match[2]), + hour, + Number(match[5]), + Number(match[6] ?? 0) + ) +} + +interface CalendarFields { + year: number + month: number + day: number +} + +/** Extracts literal calendar fields from supported month-name date forms. */ +function extractMonthNameCalendar(value: string): CalendarFields | null { + const monthFirst = value.match(MONTH_FIRST_DATE_PATTERN) + if (monthFirst) { + return { + year: Number(monthFirst[3]), + month: MONTH_BY_ABBREVIATION[monthFirst[1].slice(0, 3).toUpperCase()], + day: Number(monthFirst[2]), + } + } + const dayFirst = value.match(DAY_FIRST_DATE_PATTERN) + if (!dayFirst) return null + return { + year: Number(dayFirst[3]), + month: MONTH_BY_ABBREVIATION[dayFirst[2].slice(0, 3).toUpperCase()], + day: Number(dayFirst[1]), + } +} + +/** Recovers broader naive `Date.parse` inputs without consulting the runtime timezone. */ +function parseNaiveWallClockAsUtc(value: string): string | null { + const calendar = extractMonthNameCalendar(value) + if (calendar && !isValidCalendarDay(calendar.year, calendar.month, calendar.day)) return null + const ms = Date.parse(`${value} UTC`) + if (Number.isNaN(ms)) return null + const parsed = new Date(ms) + if ( + calendar && + (parsed.getUTCFullYear() !== calendar.year || + parsed.getUTCMonth() + 1 !== calendar.month || + parsed.getUTCDate() !== calendar.day) + ) { + return null + } + return `${toUtcCalendarDate(parsed)}T${pad(parsed.getUTCHours())}:${pad(parsed.getUTCMinutes())}:${pad(parsed.getUTCSeconds())}` } export interface NormalizeDateCellOptions { @@ -205,6 +266,14 @@ export interface NormalizeDateCellOptions { * zone. */ timezone?: string + /** + * Which instant to use when a naive wall time occurs twice during a DST + * fall-back. Ordinary date cells preserve their historical earlier-instant + * behavior; instant-like callers may explicitly choose `later`. + */ + ambiguousTime?: ZonedWallClockOptions['ambiguousTime'] + /** How sub-minute historical offsets are serialized to RFC 3339 minutes. */ + offsetMinuteRounding?: ZonedWallClockOptions['offsetMinuteRounding'] } /** @@ -220,12 +289,37 @@ export function normalizeDateCellValue( ): string | null { const trimmed = raw.trim() if (!trimmed) return null - if (CALENDAR_DATE_PATTERN.test(trimmed)) { - return Number.isNaN(Date.parse(trimmed)) ? null : trimmed + const calendar = trimmed.match(CALENDAR_DATE_PATTERN) + if (calendar) { + return isValidCalendarDay(Number(calendar[1]), Number(calendar[2]), Number(calendar[3])) + ? trimmed + : null + } + const localizedCalendar = trimmed.match(LOCALIZED_CALENDAR_DATE_PATTERN) + if (localizedCalendar) { + const month = Number(localizedCalendar[1]) + const day = Number(localizedCalendar[2]) + const year = Number(localizedCalendar[3]) + return isValidCalendarDay(year, month, day) + ? `${String(year).padStart(4, '0')}-${pad(month)}-${pad(day)}` + : null } + const isoMatch = trimmed.match(WALL_INSTANT_PATTERN) + const isoWallClock = isoMatch ? parseIsoWallClock(isoMatch) : undefined + if (isoWallClock === null) return null + const localizedMatch = trimmed.match(LOCALIZED_WALL_CLOCK_PATTERN) + const localizedWallClock = localizedMatch ? parseLocalizedWallClock(localizedMatch) : undefined + if (localizedWallClock === null) return null const ms = Date.parse(trimmed) if (Number.isNaN(ms)) return null const parsed = new Date(ms) + const monthNameCalendar = extractMonthNameCalendar(trimmed) + if ( + monthNameCalendar && + !isValidCalendarDay(monthNameCalendar.year, monthNameCalendar.month, monthNameCalendar.day) + ) { + return null + } if (!TIME_COMPONENT_PATTERN.test(trimmed)) { return ISO_REDUCED_DATE_PATTERN.test(trimmed) ? toUtcCalendarDate(parsed) @@ -238,11 +332,12 @@ export function normalizeDateCellValue( return formatUtcFieldsAsWall(new Date(ms + explicitOffset * 60_000), explicitOffset) } if (options?.timezone) { - // `parsed`'s local getters recover the wall-clock fields V8 read from the - // naive string; stamp them with the requested zone's offset at that time. - const instant = wallTimeInZoneToUtc(parsed, options.timezone) - const offsetMinutes = Math.round(zoneOffsetMs(options.timezone, instant) / 60_000) - return formatLocalFieldsAsWall(parsed, offsetMinutes) + const wallClock = isoWallClock ?? localizedWallClock ?? parseNaiveWallClockAsUtc(trimmed) + if (!wallClock) return null + return zonedWallClockWithOffset(wallClock, options.timezone, { + ambiguousTime: options.ambiguousTime ?? 'earlier', + offsetMinuteRounding: options.offsetMinuteRounding, + }) } return formatLocalFieldsAsWall(parsed, -parsed.getTimezoneOffset()) } diff --git a/apps/sim/lib/table/delete-runner.test.ts b/apps/sim/lib/table/delete-runner.test.ts index aa19a0faac4..8eb163aeed0 100644 --- a/apps/sim/lib/table/delete-runner.test.ts +++ b/apps/sim/lib/table/delete-runner.test.ts @@ -16,6 +16,7 @@ const { mockAppendTableEvent, mockSignalTableRowsChanged, mockBuildFilterClause, + mockFireTableTrigger, } = vi.hoisted(() => ({ mockGetTableById: vi.fn(), mockGetJobProgress: vi.fn(), @@ -28,6 +29,7 @@ const { mockAppendTableEvent: vi.fn(), mockSignalTableRowsChanged: vi.fn(), mockBuildFilterClause: vi.fn(), + mockFireTableTrigger: vi.fn(), })) vi.mock('@/lib/table/service', () => ({ @@ -49,6 +51,7 @@ vi.mock('@/lib/table/events', () => ({ signalTableRowsChanged: mockSignalTableRowsChanged, })) vi.mock('@/lib/table/sql', () => ({ buildFilterClause: mockBuildFilterClause })) +vi.mock('@/lib/table/trigger', () => ({ fireTableTrigger: mockFireTableTrigger })) vi.mock('@/lib/table/constants', () => ({ TABLE_LIMITS: { DELETE_PAGE_SIZE: 2 }, USER_TABLE_ROWS_SQL_NAME: 'user_table_rows', @@ -62,7 +65,13 @@ const UNLOCKED = { updateLocked: false, deleteLocked: false, } -const table = { id: 'tbl_1', workspaceId: 'ws_1', schema: { columns: [] }, locks: UNLOCKED } +const table = { + id: 'tbl_1', + name: 'Issues', + workspaceId: 'ws_1', + schema: { columns: [] }, + locks: UNLOCKED, +} const cutoff = new Date('2026-06-05T00:00:00Z') function basePayload(overrides = {}) { @@ -77,7 +86,23 @@ describe('runTableDelete', () => { mockUpdateJobProgress.mockResolvedValue(true) mockMarkJobReady.mockResolvedValue(true) mockMarkJobFailed.mockResolvedValue(undefined) - mockDeletePageByIds.mockImplementation((_t, _w, ids: string[]) => Promise.resolve(ids.length)) + mockDeletePageByIds.mockImplementation( + async ( + _t, + _w, + ids: string[], + _proof, + _revalidate, + onDeleted?: ( + rows: Array<{ id: string; data: Record }>, + table?: typeof table + ) => void | Promise + ) => { + const rows = ids.map((id) => ({ id, data: { title: id } })) + await onDeleted?.(rows) + return rows.length + } + ) mockBuildFilterClause.mockReturnValue({}) }) @@ -115,6 +140,7 @@ describe('runTableDelete', () => { 'ws_1', ['a', 'b'], expect.anything(), + expect.any(Function), expect.any(Function) ) expect(mockMarkJobCanceled).toHaveBeenCalledWith('tbl_1', 'job_1') @@ -150,6 +176,7 @@ describe('runTableDelete', () => { 'ws_1', ['a', 'b'], expect.anything(), + expect.any(Function), expect.any(Function) ) expect(mockDeletePageByIds).toHaveBeenNthCalledWith( @@ -158,17 +185,62 @@ describe('runTableDelete', () => { 'ws_1', ['c'], expect.anything(), + expect.any(Function), expect.any(Function) ) expect(mockMarkJobReady).toHaveBeenCalledWith('tbl_1', 'job_1') expect(mockAppendTableEvent).toHaveBeenCalledWith( expect.objectContaining({ kind: 'job', type: 'delete', status: 'ready', progress: 3 }) ) + expect(mockFireTableTrigger).toHaveBeenCalledTimes(2) + expect(mockFireTableTrigger).toHaveBeenNthCalledWith( + 1, + 'tbl_1', + 'ws_1', + 'Issues', + 'delete', + [ + { id: 'a', data: { title: 'a' } }, + { id: 'b', data: { title: 'b' } }, + ], + null, + table.schema, + expect.any(String) + ) // The live grid must be told rows changed so deleted rows drop out of every open editor — // the `job` progress event only drives the delete meter, not the rows query. expect(mockSignalTableRowsChanged).toHaveBeenCalledWith('tbl_1') }) + it('uses the table definition revalidated with each committed delete batch', async () => { + const renamedTable = { + ...table, + name: 'Renamed issues', + schema: { columns: [{ id: 'col-title', name: 'Renamed title', type: 'string' }] }, + } + mockSelectRowIdPage.mockResolvedValueOnce(['a']).mockResolvedValueOnce([]) + mockDeletePageByIds.mockImplementationOnce( + async (_t, _w, ids: string[], _proof, _revalidate, onDeleted) => { + const rows = ids.map((id) => ({ id, data: { 'col-title': id } })) + await onDeleted?.(rows, renamedTable) + return rows.length + } + ) + + await runTableDelete(basePayload()) + + expect(mockFireTableTrigger).toHaveBeenCalledWith( + renamedTable.id, + renamedTable.workspaceId, + renamedTable.name, + 'delete', + [{ id: 'a', data: { 'col-title': 'a' } }], + null, + renamedTable.schema, + expect.any(String) + ) + }) + it('stops once maxRows is reached and caps the final page fetch to the remaining budget', async () => { // budget 3 with page size 2: first page fills 2, the second is capped to the remaining 1. mockSelectRowIdPage.mockResolvedValueOnce(['a', 'b']).mockResolvedValueOnce(['c']) @@ -196,6 +268,7 @@ describe('runTableDelete', () => { 'ws_1', ['x'], expect.anything(), + expect.any(Function), expect.any(Function) ) // Second page is queried after the last id of the first page (cursor advanced past 'keep'). diff --git a/apps/sim/lib/table/delete-runner.ts b/apps/sim/lib/table/delete-runner.ts index a1c14302bed..6b010f02518 100644 --- a/apps/sim/lib/table/delete-runner.ts +++ b/apps/sim/lib/table/delete-runner.ts @@ -14,9 +14,10 @@ import { } from '@/lib/table/jobs/service' import { assertRowDelete, type MutationProof, TableLockedError } from '@/lib/table/mutation-locks' import type { DbTransaction } from '@/lib/table/planner' -import { deletePageByIds, selectRowIdPage } from '@/lib/table/rows/ordering' +import { type DeletedTableRow, deletePageByIds, selectRowIdPage } from '@/lib/table/rows/ordering' import { getTableById } from '@/lib/table/service' import { buildFilterClause } from '@/lib/table/sql' +import { fireTableTrigger } from '@/lib/table/trigger' const logger = createLogger('TableDeleteRunner') @@ -122,6 +123,22 @@ export async function runTableDelete(payload: TableDeletePayload): Promise // an absent filter is still legitimate (delete-all is an explicit caller mode). if (filter && !filterClause) throw new Error('Filter is required for bulk delete') const excluded = new Set(excludeRowIds ?? []) + const dispatchDeleteTriggers = async ( + rows: DeletedTableRow[], + committedTable?: TableDefinition + ) => { + const triggerTable = committedTable ?? table + await fireTableTrigger( + triggerTable.id, + triggerTable.workspaceId, + triggerTable.name, + 'delete', + rows, + null, + triggerTable.schema, + requestId + ) + } // Resume the persisted count: a retried attempt's earlier batches are already committed, // so starting at zero would overwrite cumulative progress with this attempt's smaller @@ -170,7 +187,14 @@ export async function runTableDelete(payload: TableDeletePayload): Promise // returns or throws. (An attempt that ends up committing nothing only over-refetches — harmless.) deletedAny = true try { - processed += await deletePageByIds(tableId, workspaceId, toDelete, pageProof, revalidate) + processed += await deletePageByIds( + tableId, + workspaceId, + toDelete, + pageProof, + revalidate, + dispatchDeleteTriggers + ) } catch (err) { if (!(err instanceof TableLockedError)) throw err // A lock landed between batches. Batches already committed stay diff --git a/apps/sim/lib/table/dispatch-governed-subject.test.ts b/apps/sim/lib/table/dispatch-governed-subject.test.ts new file mode 100644 index 00000000000..b1d4937cef3 --- /dev/null +++ b/apps/sim/lib/table/dispatch-governed-subject.test.ts @@ -0,0 +1,94 @@ +/** + * @vitest-environment node + */ +import { dbChainMockFns, resetDbChainMock } from '@sim/testing' +import { beforeEach, describe, expect, it, vi } from 'vitest' + +vi.mock('@/lib/table/events', () => ({ + appendTableEvent: vi.fn(), +})) +vi.mock('@/lib/table/service', () => ({ + getTableById: vi.fn(), +})) + +import { insertDispatch } from '@/lib/table/dispatcher' + +const BASE = { + tableId: 'table-1', + workspaceId: 'workspace-1', + requestId: 'req-1', + mode: 'all' as const, + scope: { groupIds: ['group-1'] }, + isManualRun: true, +} + +/** The values `insertDispatch` handed to the single `db.insert(...).values(...)`. */ +function insertedRow(): Record { + expect(dbChainMockFns.values).toHaveBeenCalledTimes(1) + return dbChainMockFns.values.mock.calls[0][0] as Record +} + +describe('insertDispatch governed subject', () => { + beforeEach(() => { + vi.clearAllMocks() + resetDbChainMock() + }) + + /** + * The bug this replaces: an optional field defaulting to `triggeredByUserId` + * meant a workspace-key auto-dispatch stored the workspace billed account as + * its gate subject — a bystander whose tool denylist would then run against + * a request nobody meant to govern. + */ + it('stores null for an actorless run even when the attribution names a user', async () => { + await insertDispatch({ + ...BASE, + triggeredByUserId: 'billing-owner', + capabilityGovernedUserId: null, + }) + const row = insertedRow() + expect(row.triggeredByUserId).toBe('billing-owner') + expect(row.capabilityGovernedUserId).toBeNull() + }) + + it('stores the acting person for a session-triggered run', async () => { + await insertDispatch({ + ...BASE, + triggeredByUserId: 'user-1', + capabilityGovernedUserId: 'user-1', + }) + const row = insertedRow() + expect(row.capabilityGovernedUserId).toBe('user-1') + }) + + /** + * The two fields are independent: a delegated run can be metered to the payer + * while staying governed by the person who asked for it. + */ + it('keeps the gate subject independent of the meter subject', async () => { + await insertDispatch({ + ...BASE, + triggeredByUserId: 'billing-owner', + capabilityGovernedUserId: 'requesting-user', + }) + const row = insertedRow() + expect(row.triggeredByUserId).toBe('billing-owner') + expect(row.capabilityGovernedUserId).toBe('requesting-user') + }) + + /** + * A row written before the column existed reads `capability_governed_user_id` + * as NULL with `triggered_by_user_id` still set. Under the new semantics that + * shape means "actorless, ungated" — which is why the 0315 migration + * backfills the legacy subject onto non-terminal pre-migration rows rather + * than letting them fall through to it. + */ + it('never reconstructs the gate subject from the attribution', async () => { + await insertDispatch({ + ...BASE, + triggeredByUserId: 'user-1', + capabilityGovernedUserId: null, + }) + expect(insertedRow().capabilityGovernedUserId).toBeNull() + }) +}) diff --git a/apps/sim/lib/table/dispatcher.ts b/apps/sim/lib/table/dispatcher.ts index c20d1d837a5..d05e905819d 100644 --- a/apps/sim/lib/table/dispatcher.ts +++ b/apps/sim/lib/table/dispatcher.ts @@ -98,6 +98,10 @@ export interface DispatchRow { isManualRun: boolean /** User who triggered the run (for usage attribution); null for auto-fire. */ triggeredByUserId: string | null + /** Person whose permission group gates this run's cells; null when the run + * has no acting person. Deliberately not `triggeredByUserId` — see the + * column comment on `table_run_dispatches`. */ + capabilityGovernedUserId: string | null requestedAt: Date /** Set when the dispatch reached `complete`; null while it is still active. */ completedAt: Date | null @@ -248,6 +252,14 @@ export async function insertDispatch(input: { limit?: DispatchLimit | null isManualRun: boolean triggeredByUserId?: string | null + /** + * The person whose permission group gates this run's cells, or `null` when + * the run has no acting person (workspace key, schedule, auto-fire). + * + * Never defaulted from `triggeredByUserId`, and required with an explicit + * `null`; see {@link InsertRowData.capabilityGovernedUserId} in `@/lib/table/types`. + */ + capabilityGovernedUserId: string | null }): Promise { const id = `tdsp_${generateId().replace(/-/g, '')}` await db.insert(tableRunDispatches).values({ @@ -265,6 +277,7 @@ export async function insertDispatch(input: { cursor: -1, isManualRun: input.isManualRun, triggeredByUserId: input.triggeredByUserId ?? null, + capabilityGovernedUserId: input.capabilityGovernedUserId, }) return id } @@ -349,6 +362,7 @@ export async function listActiveDispatches(tableId: string): Promise ({ ...p, dispatchId, triggeredByUserId: dispatch.triggeredByUserId ?? undefined })) + capabilityGovernedUserId: dispatch.capabilityGovernedUserId, + }).map((p) => ({ + ...p, + dispatchId, + triggeredByUserId: dispatch.triggeredByUserId ?? undefined, + })) // Cursor advances to the last position in this chunk regardless of // eligibility — otherwise a window full of skipped cells loops forever. @@ -790,6 +810,15 @@ async function stampQueuedForBatch( jobId: null, workflowId: runOpts.workflowId, error: null, + /** + * The marker outlives this dispatch's own worker: a cell task that + * finds the row's cascade lock held bails, and whoever owns the lock + * drains this marker instead. Persisting the subject is what makes + * that drain run under the person who requested THIS cell rather + * than under the owner's — a different dispatch, and often an + * actorless auto-fire with no gate at all. + */ + capabilityGovernedUserId: runOpts.capabilityGovernedUserId, }, } ) @@ -1026,6 +1055,7 @@ export async function cancelStaleDispatches( processedCount: row.processedCount, isManualRun: row.isManualRun, triggeredByUserId: row.triggeredByUserId, + capabilityGovernedUserId: row.capabilityGovernedUserId, requestedAt: row.requestedAt, completedAt: row.completedAt, cancelledAt: row.cancelledAt, @@ -1109,6 +1139,7 @@ export async function markActiveDispatchesCancelled( processedCount: row.processedCount, isManualRun: row.isManualRun, triggeredByUserId: row.triggeredByUserId, + capabilityGovernedUserId: row.capabilityGovernedUserId, requestedAt: row.requestedAt, completedAt: row.completedAt, cancelledAt: row.cancelledAt, diff --git a/apps/sim/lib/table/events.attribution.test.ts b/apps/sim/lib/table/events.attribution.test.ts index 3ed09ce7135..eb47880f4cc 100644 --- a/apps/sim/lib/table/events.attribution.test.ts +++ b/apps/sim/lib/table/events.attribution.test.ts @@ -3,12 +3,7 @@ */ import { readdir, readFile } from 'node:fs/promises' import { join } from 'node:path' -import { describe, expect, it, vi } from 'vitest' - -// Structurally slow — it scans call sites across the repo — so under a fully-parallel local run this file -// blows the default timeout while passing in isolation and on CI. Give it a -// real budget instead of letting machine load decide the verdict. -vi.setConfig({ testTimeout: 30_000 }) +import { beforeAll, describe, expect, it } from 'vitest' /** * `signalTableRowsChangedByActor` lets the acting tab skip its own refetch, which is only sound @@ -51,24 +46,49 @@ const FORWARDING_MODULE = 'lib/table/application/rows.ts' */ const SUPPLIER_PATTERNS = [/actorClientId:/, /signalTableRowsChangedByActor\([^)]*,/] as const -async function* walk(dir: string): AsyncGenerator { - for (const entry of await readdir(dir, { withFileTypes: true })) { - if (entry.name === 'node_modules' || entry.name === '.next') continue - const full = join(dir, entry.name) - if (entry.isDirectory()) yield* walk(full) - else if (entry.name.endsWith('.ts') && !entry.name.includes('.test.')) yield full +/** Files read per batch; bounds open descriptors while keeping the disk busy. */ +const READ_BATCH_SIZE = 64 + +async function walk(dir: string): Promise { + const entries = await readdir(dir, { withFileTypes: true }) + const nested = await Promise.all( + entries.map(async (entry) => { + if (entry.name === 'node_modules' || entry.name === '.next') return [] + const full = join(dir, entry.name) + if (entry.isDirectory()) return walk(full) + return entry.name.endsWith('.ts') && !entry.name.includes('.test.') ? [full] : [] + }) + ) + return nested.flat() +} + +/** + * Every source file under the app root, keyed by its relative path. Read once + * for the file: both sweeps scan the same tree, and walking it per test was the + * whole cost of this file. + */ +let sources: Map + +async function readSources(): Promise> { + const files = await walk(APP_ROOT) + const found = new Map() + for (let start = 0; start < files.length; start += READ_BATCH_SIZE) { + const batch = files.slice(start, start + READ_BATCH_SIZE) + const contents = await Promise.all(batch.map((file) => readFile(file, 'utf8'))) + batch.forEach((file, index) => { + found.set(file.slice(APP_ROOT.length + 1), contents[index]) + }) } + return found } -async function filesMatching( +function filesMatching( matches: (source: string) => boolean, skip: (relative: string) => boolean = () => false -) { +): string[] { const found: string[] = [] - for await (const file of walk(APP_ROOT)) { - const source = await readFile(file, 'utf8') + for (const [relative, source] of sources) { if (!matches(source)) continue - const relative = file.slice(APP_ROOT.length + 1) if (skip(relative)) continue found.push(relative) } @@ -76,8 +96,17 @@ async function filesMatching( } describe('signalTableRowsChangedByActor call sites', () => { - it('is called only where the acting tab reconciles the write locally', async () => { - const callers = await filesMatching( + /** + * Structurally slow — it reads every source file in the app — so under a + * fully-parallel local run the scan blows the default budget while passing in + * isolation and on CI. Give it a real budget of its own, outside any test's. + */ + beforeAll(async () => { + sources = await readSources() + }, 30_000) + + it('is called only where the acting tab reconciles the write locally', () => { + const callers = filesMatching( (source) => source.includes('signalTableRowsChangedByActor('), (relative) => relative === DECLARING_MODULE ) @@ -85,8 +114,8 @@ describe('signalTableRowsChangedByActor call sites', () => { expect(callers).toEqual([...ATTRIBUTED_CALL_SITES].sort()) }) - it('is given an actor only by surfaces whose client hook reconciles locally', async () => { - const suppliers = await filesMatching( + it('is given an actor only by surfaces whose client hook reconciles locally', () => { + const suppliers = filesMatching( (source) => SUPPLIER_PATTERNS.some((pattern) => pattern.test(source)), (relative) => relative === DECLARING_MODULE || relative === FORWARDING_MODULE ) diff --git a/apps/sim/lib/table/import-data.ts b/apps/sim/lib/table/import-data.ts index 808a4108f5d..e1da840c599 100644 --- a/apps/sim/lib/table/import-data.ts +++ b/apps/sim/lib/table/import-data.ts @@ -265,7 +265,14 @@ export async function importAppendRows( table: TableDefinition, additions: { id?: string; name: string; type: string; required?: boolean; unique?: boolean }[], rows: RowData[], - ctx: { workspaceId: string; userId?: string; requestId: string } + ctx: { + workspaceId: string + userId?: string + requestId: string + /** Gate subject for cells the appended rows auto-fire — the subject the + * importing surface resolved from its principal, or `null` for none. */ + capabilityGovernedUserId: string | null + } ): Promise<{ inserted: TableRow[]; table: TableDefinition }> { // Gate capacity before opening the tx — the lookup is a separate pool read. const rowLimit = await assertRowCapacity({ @@ -294,6 +301,7 @@ export async function importAppendRows( rows: batch, workspaceId: ctx.workspaceId, userId: ctx.userId, + capabilityGovernedUserId: ctx.capabilityGovernedUserId, secretProvenance: batch.map(createExactEmptyTableRowSecretProvenance), }, working, diff --git a/apps/sim/lib/table/import.test.ts b/apps/sim/lib/table/import.test.ts index d0aa04500e1..f8259213a45 100644 --- a/apps/sim/lib/table/import.test.ts +++ b/apps/sim/lib/table/import.test.ts @@ -172,6 +172,27 @@ describe('import', () => { ) expect(coerceValue('not-a-date', 'date')).toBe('not-a-date') }) + + it('coerces TTL imports to epoch seconds and rejects invalid input', () => { + expect(coerceValue('2023-11-14T22:13:20Z', 'ttl')).toBe(1_700_000_000) + expect(coerceValue('1700000000', 'ttl')).toBe(1_700_000_000) + expect(coerceValue('2023-11-14 17:13:20', 'ttl', { timezone: 'America/New_York' })).toBe( + 1_700_000_000 + ) + expect(coerceValue('not-a-date', 'ttl')).toBeNull() + }) + + it('applies the timezone supplied to each TTL import independently', () => { + const input = '2026-06-15 09:00:30' + + expect(coerceValue(input, 'ttl', { timezone: 'America/New_York' })).toBe( + Date.parse('2026-06-15T13:00:30Z') / 1000 + ) + expect(coerceValue(input, 'ttl', { timezone: 'Asia/Kathmandu' })).toBe( + Date.parse('2026-06-15T03:15:30Z') / 1000 + ) + expect(coerceValue('2023-11-14T22:13:20.001Z', 'ttl')).toBe(1_700_000_001) + }) }) describe('buildAutoMapping', () => { diff --git a/apps/sim/lib/table/import.ts b/apps/sim/lib/table/import.ts index 2ad05fa5fa5..50ea7cfc3b2 100644 --- a/apps/sim/lib/table/import.ts +++ b/apps/sim/lib/table/import.ts @@ -15,6 +15,7 @@ import type { Options as CsvParseOptions } from 'csv-parse' import { OrchestrationError } from '@/lib/core/orchestration/types' import { getColumnId } from '@/lib/table/column-keys' import type { ColumnType } from '@/lib/table/column-types' +import { coerceColumnTypeImportValue } from '@/lib/table/column-types/import-coercion' import { parseCurrencyInput } from '@/lib/table/currency' import { type NormalizeDateCellOptions, normalizeDateCellValue } from '@/lib/table/dates' import type { ColumnDefinition, RowData, TableSchema } from '@/lib/table/types' @@ -468,12 +469,10 @@ export function inferSchemaFromCsv( * back to the original string when unparseable so that schema validation can * reject it with context rather than silently inserting `null`. * - * Deliberately NOT routed through the column-type registry's `coerce`, despite - * covering the same types. The registry's contract is "coerced or rejected", - * which the write path turns into `null`; an import instead wants an - * unparseable date or JSON blob to survive as its raw string so the row-level - * validation error names the offending value. Unifying the two would silently - * swap a descriptive import error for a blanked cell. + * Deliberately not routed through the column-type registry: its contract is + * "coerced or rejected", while an import needs invalid raw text to survive so + * row-level validation can name it. Type-specific import behavior uses a + * lightweight capability map so CSV clients do not load the full registry. */ export function coerceValue( value: unknown, @@ -481,6 +480,10 @@ export function coerceValue( options?: NormalizeDateCellOptions & { currencyCode?: string } ): string | number | boolean | null | Record | unknown[] { if (value === null || value === undefined || value === '') return null + + const typeSpecificValue = coerceColumnTypeImportValue(colType, value, options) + if (typeSpecificValue !== undefined) return typeSpecificValue + switch (colType) { case 'number': { const n = Number(value) diff --git a/apps/sim/lib/table/llm/enrichment.test.ts b/apps/sim/lib/table/llm/enrichment.test.ts new file mode 100644 index 00000000000..4950f0fe624 --- /dev/null +++ b/apps/sim/lib/table/llm/enrichment.test.ts @@ -0,0 +1,258 @@ +/** + * @vitest-environment node + */ +import { describe, expect, it } from 'vitest' +import { MULTI_SELECT_OPS, SINGLE_SELECT_OPS } from '@/lib/table/column-types' +import { enrichTableToolDescription, enrichTableToolParameters } from '@/lib/table/llm/enrichment' +import type { TableSummary } from '@/lib/table/types' + +const TABLE: TableSummary = { + name: 'Players', + columns: [ + { name: 'status', type: 'string' }, + { name: 'wins', type: 'number' }, + ], +} + +const V2_SCHEMA = { + properties: { + filter: { type: 'object' }, + order: { type: 'array' }, + columns: { type: 'array' }, + limit: { type: 'number' }, + cursor: { type: 'string' }, + }, + required: [] as string[], +} + +describe('enrichTableToolDescription for table_query_rows_v2', () => { + const enriched = enrichTableToolDescription('Query rows.', TABLE, 'table_query_rows_v2') + + it('names the real columns', () => { + expect(enriched).toContain('status (string)') + expect(enriched).toContain('wins (number)') + }) + + it('teaches the predicate grammar built from a real column', () => { + expect(enriched).toContain('{"field":"wins","op":"gte","value":10}') + expect(enriched).toContain('"all"') + }) + + it('never teaches the v1 MongoDB grammar or offset paging', () => { + expect(enriched).not.toContain('$eq') + expect(enriched).not.toContain('offset') + }) + + it('describes order rather than sort', () => { + expect(enriched).toContain('Example order: [{"field":"wins","direction":"desc"}]') + }) + + /** + * A metrics table is all-numeric and a lookup table is all-text; both are + * common, and each picks a different arm of the example builder. + */ + it('builds a numeric example when the table has no string column', () => { + const numeric = enrichTableToolDescription( + 'Query rows.', + { name: 'Scores', columns: [{ name: 'wins', type: 'number' }] }, + 'table_query_rows_v2' + ) + expect(numeric).toContain('{"field":"wins","op":"gte","value":10}') + expect(numeric).not.toContain('AND group') + }) + + it('builds a string example when the table has no numeric column', () => { + const textual = enrichTableToolDescription( + 'Query rows.', + { name: 'Statuses', columns: [{ name: 'status', type: 'string' }] }, + 'table_query_rows_v2' + ) + expect(textual).toContain('{"field":"status","op":"eq","value":"active"}') + expect(textual).not.toContain('"op":"gte"') + }) + + /** + * An unknown field is rejected outright, so every field the instructions name + * has to be a column the table actually has. + */ + it('names a real text column in the wildcard example', () => { + expect(enriched).toContain('{"field":"status","op":"ilike","value":"*jo*"}') + expect(enriched).not.toContain('{"field":"name"') + }) + + /** + * `buildPatternClause` ESCAPES `%` before translating `*`, so a model that + * sends `%` gets a literal-percent match and silently wrong rows rather than + * an error. All four pattern operators share that translation. + */ + it('covers every pattern operator in the wildcard rule and warns off %', () => { + expect(enriched).toContain('like, ilike, nlike and nilike all use * as the wildcard - never %') + }) + + /** + * A question with no condition ("the 5 most recent rows") is answered with + * order and limit; the model must not invent a predicate to satisfy it. + */ + it('tells the model a filter is optional when no condition was asked for', () => { + expect(enriched).toContain('omit filter entirely and use order and limit') + expect(enriched).toContain('omit it whenever the question carries no condition') + }) + + it('drops the wildcard example when the table has no text column', () => { + const numeric = enrichTableToolDescription( + 'Query rows.', + { name: 'Scores', columns: [{ name: 'wins', type: 'number' }] }, + 'table_query_rows_v2' + ) + expect(numeric).toContain('matching anywhere in a text value') + expect(numeric).not.toContain('"op":"ilike"') + }) + + it('omits the example rather than naming a placeholder column', () => { + const bare = enrichTableToolDescription( + 'Query rows.', + { name: 'Blobs', columns: [{ name: 'payload', type: 'json' }] }, + 'table_query_rows_v2' + ) + expect(bare).toContain('payload (json)') + expect(bare).not.toContain('Example filter') + expect(bare).not.toContain('Example order') + }) +}) + +/** + * A select column rejects any operator outside its subset — the query layer + * throws rather than returning no rows — so the description has to name the + * subset per column instead of advertising the full operator list. + */ +describe('select columns in the v2 description', () => { + const SELECT_TABLE: TableSummary = { + name: 'Transactions', + columns: [ + { name: 'category', type: 'select', multiple: false }, + { name: 'tags', type: 'select', multiple: true }, + { name: 'description', type: 'string' }, + ], + } + + const enriched = enrichTableToolDescription('Query rows.', SELECT_TABLE, 'table_query_rows_v2') + + it('names the allowed operators on a single-select column', () => { + expect(enriched).toContain( + 'category (single-select; only eq, ne, in, nin, isEmpty, isNotEmpty, isNull, isNotNull)' + ) + }) + + it('names the allowed operators on a multi-select column', () => { + expect(enriched).toContain( + 'tags (multi-select; only contains, ncontains, isEmpty, isNotEmpty, isNull, isNotNull)' + ) + }) + + /** + * The list is derived from the sets `fieldPredicate` gates on, so it cannot + * drift from the validator the way a hand-copied list did. + */ + it('derives the operator list from the validator sets', () => { + for (const op of SINGLE_SELECT_OPS) expect(enriched).toContain(op) + for (const op of MULTI_SELECT_OPS) expect(enriched).toContain(op) + }) + + it('leaves non-select columns unannotated', () => { + expect(enriched).toContain('description (string)') + }) + + it('never claims the table has no array columns', () => { + expect(enriched).not.toContain('no array columns') + }) + + it('steers multi-select matching to contains rather than ilike', () => { + expect(enriched).toContain('match it by option name with contains, never ilike') + }) +}) + +describe('enrichTableToolParameters for table_query_rows_v2', () => { + const { properties, required } = enrichTableToolParameters( + V2_SCHEMA, + TABLE, + 'table_query_rows_v2' + ) + + /** + * The v1 branch force-pushes `filter` into `required` because a v1 query + * without one fails. A v2 query without a filter is valid and returns every + * row, so forcing it would make the model invent a filter for "list all". + */ + it('leaves filter optional', () => { + expect(required).not.toContain('filter') + }) + + it('describes filter with the predicate grammar and real columns', () => { + expect(properties.filter.description).toContain('status, wins') + expect(properties.filter.description).toContain('"op"') + expect(properties.filter.description).not.toContain('$eq') + }) + + it('enriches order, columns, limit, and cursor', () => { + expect(properties.order.description).toContain('direction') + expect(properties.columns.description).toContain('status, wins') + expect(properties.limit.description).toContain('5MB') + expect(properties.cursor.description).toContain('nextCursor') + }) + + it('does not enrich a sort property that v2 does not have', () => { + expect(properties.sort).toBeUndefined() + }) + + /** + * The parameter schema is what the model reads when deciding the filter's + * shape, so the select restriction has to appear there and not only in the + * tool description. + */ + it('carries the select restriction into the filter parameter description', () => { + const withSelect = enrichTableToolParameters( + V2_SCHEMA, + { + name: 'Transactions', + columns: [ + { name: 'category', type: 'select', multiple: false }, + { name: 'tags', type: 'select', multiple: true }, + ], + }, + 'table_query_rows_v2' + ) + expect(withSelect.properties.filter.description).toContain('rejected outright') + expect(withSelect.properties.filter.description).toContain('category accept only') + expect(withSelect.properties.filter.description).toContain('tags hold a list') + }) + + it('omits the restriction note when no column restricts operators', () => { + expect(properties.filter.description).not.toContain('rejected outright') + }) +}) + +describe('v1 enrichment is unchanged', () => { + it('still forces filter required and teaches $eq', () => { + const { properties, required } = enrichTableToolParameters( + { properties: { filter: { type: 'object' }, sort: { type: 'object' } }, required: [] }, + TABLE, + 'table_query_rows' + ) + expect(required).toContain('filter') + expect(properties.filter.description).toContain('$eq') + }) + + /** + * Both Table blocks expose the bulk tools and the rows route accepts either + * grammar, so these keep teaching $eq — enrichment cannot tell which block + * called it. + */ + it('keeps the shared bulk tools on the v1 grammar', () => { + const { properties } = enrichTableToolParameters( + { properties: { filter: { type: 'object' } }, required: [] }, + TABLE, + 'table_update_rows_by_filter' + ) + expect(properties.filter.description).toContain('$eq') + }) +}) diff --git a/apps/sim/lib/table/llm/enrichment.ts b/apps/sim/lib/table/llm/enrichment.ts index 007f5ca4c26..1626224c3f3 100644 --- a/apps/sim/lib/table/llm/enrichment.ts +++ b/apps/sim/lib/table/llm/enrichment.ts @@ -1,15 +1,16 @@ -/** - * LLM tool enrichment utilities for table operations. - * - * Provides functions to enrich tool descriptions and parameter schemas - * with table-specific information so LLMs can construct proper queries. - */ - -import { columnTypeById } from '@/lib/table/column-types' +import { columnTypeById, MULTI_SELECT_OPS, SINGLE_SELECT_OPS } from '@/lib/table/column-types' import type { TableSummary } from '@/lib/table/types' /** - * Operations that use filters and need filter-specific enrichment. + * Operations that take a v1 MongoDB-style filter (`{"col": {"$eq": v}}`) and + * need filter-specific enrichment. + * + * The two bulk operations stay here even though the v2 Table block also exposes + * them: `resolveBulkFilter` in the rows route accepts either grammar, and + * enrichment is keyed on tool id alone — it cannot see which block invoked it — + * so one grammar has to be taught, and `$eq` is the one that works for both. + * Only the query tool has a v2-exclusive id, which is why it is the only one + * that gets a predicate-grammar branch. */ export const FILTER_OPERATIONS = new Set([ 'table_query_rows', @@ -17,6 +18,89 @@ export const FILTER_OPERATIONS = new Set([ 'table_delete_rows_by_filter', ]) +/** + * The v2 row query. Its filter is a typed predicate + * (`{"field":"wins","op":"gte","value":10}`) with `all`/`any` groups, it orders + * via `order` rather than `sort`, and it pages by opaque `cursor` rather than + * offset — so none of the v1 enrichment above applies to it. + */ +const TABLE_QUERY_ROWS_V2 = 'table_query_rows_v2' + +/** The operators a select column actually accepts, read from the validator's own sets. */ +function selectOperatorList(multiple: boolean | undefined): string { + return Array.from(multiple ? MULTI_SELECT_OPS : SINGLE_SELECT_OPS).join(', ') +} + +/** + * Renders one column line for the v2 description. + * + * A select column accepts only a subset of the operators — `eq`/`ne`/`in`/`nin` + * when single, `contains`/`ncontains` when multi — and the query layer THROWS on + * anything else (`buildFilterConditions` in `lib/table/sql.ts`). Naming the + * subset inline is what stops the model from reaching for `ilike` on a select + * column and turning a valid question into a validation error. + */ +function v2ColumnLine(column: TableSummary['columns'][number]): string { + if (column.type !== 'select') return ` - ${column.name} (${column.type})` + const kind = column.multiple ? 'multi-select' : 'single-select' + return ` - ${column.name} (${kind}; only ${selectOperatorList(column.multiple)})` +} + +/** Whether any column restricts its operators, i.e. whether the note is worth emitting. */ +function hasSelectColumn(table: TableSummary): boolean { + return table.columns.some((column) => column.type === 'select') +} + +/** + * A one-line summary of the per-column restrictions, for the `filter` parameter + * description. The parameter schema is what a model reads when it is deciding + * the SHAPE of the filter, so the restriction has to appear there too and not + * only in the tool description. + */ +function selectRestrictionNote(table: TableSummary): string { + const single = table.columns.filter((c) => c.type === 'select' && !c.multiple).map((c) => c.name) + const multi = table.columns.filter((c) => c.type === 'select' && c.multiple).map((c) => c.name) + const parts: string[] = [] + if (single.length > 0) { + parts.push(`${single.join(', ')} accept only ${selectOperatorList(false)}`) + } + if (multi.length > 0) { + parts.push( + `${multi.join(', ')} hold a list and accept only ${selectOperatorList(true)} (match by option name)` + ) + } + return ` Restricted columns - a predicate using any other operator on one is rejected outright: ${parts.join('; ')}.` +} + +/** + * Builds a predicate example from real columns, preferring a numeric `gte` over + * a string `eq` because ranking and threshold questions are what the grammar + * most often gets wrong. Returns an empty string when the table has no column + * to name, so the model is never shown a placeholder it might copy literally. + */ +function v2PredicateExample(table: TableSummary): string { + const stringCol = table.columns.find((c) => c.type === 'string') + const numberCol = table.columns.find((c) => c.type === 'number') + + if (numberCol && stringCol) { + return ` + +Example filter (one condition): {"field":"${numberCol.name}","op":"gte","value":10} +Example filter (AND group): {"all":[{"field":"${numberCol.name}","op":"gte","value":10},{"field":"${stringCol.name}","op":"eq","value":"active"}]}` + } + if (numberCol) { + return ` + +Example filter: {"field":"${numberCol.name}","op":"gte","value":10}` + } + if (stringCol) { + return ` + +Example filter: {"field":"${stringCol.name}","op":"eq","value":"active"}` + } + return '' +} + /** * Operations that need column info for data construction. */ @@ -41,6 +125,43 @@ export function enrichTableToolDescription( const columnList = table.columns.map((col) => ` - ${col.name} (${col.type})`).join('\n') + if (toolId === TABLE_QUERY_ROWS_V2) { + const v2ColumnList = table.columns.map(v2ColumnLine).join('\n') + /* + * An unknown field is rejected outright (`Unknown filter column`), so the + * wildcard example names a real text column or is dropped entirely rather + * than inviting the model to copy a placeholder. + */ + const textCol = table.columns.find((c) => c.type === 'string') + const wildcardRule = textCol + ? `5. like, ilike, nlike and nilike all use * as the wildcard - never % - e.g. {"field":"${textCol.name}","op":"ilike","value":"*jo*"}` + : '5. like, ilike, nlike and nilike all use * as the wildcard - never % - matching anywhere in a text value' + const numberCol = table.columns.find((c) => c.type === 'number') + const orderExample = numberCol + ? ` +Example order: [{"field":"${numberCol.name}","direction":"desc"}] for highest first, "asc" for lowest first` + : '' + + return `${originalDescription} + +INSTRUCTIONS: +1. Build the filter yourself from the user's question - do NOT ask for confirmation. If the question names no condition at all ("the 5 most recent rows"), omit filter entirely and use order and limit instead of inventing one +2. A single condition is a plain object: {"field":"","op":"","value":}; use an array value for in/nin and omit value for isNull, isNotNull, isEmpty, and isNotEmpty +3. For multiple conditions wrap them in {"all":[...]} for AND or {"any":[...]} for OR; groups nest +4. Operators: eq, ne, gt, gte, lt, lte, in, nin, like, ilike, nlike, nilike, contains, ncontains, startsWith, endsWith, isNull, isNotNull, isEmpty, isNotEmpty +${wildcardRule} +6. Any column listed below with a restricted operator set accepts ONLY those operators - the query is rejected outright otherwise. JSON columns reject eq, ne, gt, gte, lt, lte, in, and nin; a multi-select cell holds a list, so match it by option name with contains, never ilike +7. For substring matching on a text column use ilike with *x* +8. For ranking queries (highest, lowest, Nth, top N) set order and a small limit, e.g. limit 1 for the highest, 2 for the second highest +9. Omit limit to return every matching row; the query fails if the result exceeds 5MB, so narrow with a filter instead of guessing a limit +10. With a limit, a page can end early at the byte budget - a non-null nextCursor means more rows remain, so pass it back as cursor and loop until it is null. Never infer completion from page size +11. A filter is optional: omit it whenever the question carries no condition, not only when the user wants every row + +Table "${table.name}" columns: +${v2ColumnList} +${v2PredicateExample(table)}${orderExample}` + } + if (FILTER_OPERATIONS.has(toolId)) { const stringCols = table.columns.filter((c) => c.type === 'string') const numberCols = table.columns.filter((c) => c.type === 'number') @@ -140,6 +261,50 @@ export function enrichTableToolParameters( const enrichedProperties = { ...llmSchema.properties } const enrichedRequired = llmSchema.required ? [...llmSchema.required] : [] + if (toolId === TABLE_QUERY_ROWS_V2) { + if (enrichedProperties.filter) { + enrichedProperties.filter = { + ...enrichedProperties.filter, + description: `Predicate built from the user's question using columns: ${columnNames}. One condition is {"field":"","op":"","value":}; combine with {"all":[...]} for AND or {"any":[...]} for OR. Operators: eq, ne, gt, gte, lt, lte, in, nin, like, ilike, nlike, nilike, contains, ncontains, startsWith, endsWith, isNull, isNotNull, isEmpty, isNotEmpty.${hasSelectColumn(table) ? selectRestrictionNote(table) : ''} Omit only to match every row.`, + } + } + + if (enrichedProperties.order) { + enrichedProperties.order = { + ...enrichedProperties.order, + description: `Sort spec as [{"field":"","direction":"asc"|"desc"}] over columns: ${columnNames}. REQUIRED for ranking queries (highest, lowest, Nth).`, + } + } + + if (enrichedProperties.columns) { + enrichedProperties.columns = { + ...enrichedProperties.columns, + description: `Column names to include in each row. Available: ${columnNames}. Omit to return all columns.`, + } + } + + if (enrichedProperties.limit) { + enrichedProperties.limit = { + ...enrichedProperties.limit, + description: `Maximum rows per page (min: 1). Omit to return every matching row; the query fails if the result exceeds 5MB, so narrow with a filter rather than guessing. For ranking queries: 1 for the highest/lowest, 2 for the second highest.`, + } + } + + if (enrichedProperties.cursor) { + enrichedProperties.cursor = { + ...enrichedProperties.cursor, + description: `Opaque cursor from a prior page's nextCursor. Omit for the first page. A non-null nextCursor means more rows remain even if the page came back short - loop until it is null.`, + } + } + + /* + * Deliberately NOT pushed into `required`: unlike v1, a v2 query with no + * filter is valid and returns every row. Forcing it would make the model + * invent a filter for "list everything". + */ + return { properties: enrichedProperties, required: enrichedRequired } + } + if (enrichedProperties.filter && FILTER_OPERATIONS.has(toolId)) { enrichedProperties.filter = { ...enrichedProperties.filter, diff --git a/apps/sim/lib/table/orchestration/import.test.ts b/apps/sim/lib/table/orchestration/import.test.ts index 405613b0de1..1212f5795e6 100644 --- a/apps/sim/lib/table/orchestration/import.test.ts +++ b/apps/sim/lib/table/orchestration/import.test.ts @@ -91,6 +91,7 @@ function importParams(overrides: Record = {}) { mode: 'append' as const, timezone: 'UTC', requestId: 'req-1', + capabilityGovernedUserId: 'user-1' as string | null, ...overrides, } } @@ -133,6 +134,43 @@ describe('performTableCsvImport', () => { expect(mockSignalSchemaChanged).toHaveBeenCalledWith('table-1') }) + /** + * The rows an import lands start the table's workflow columns, and those + * cells gate their tools on the governed subject. Dropping it here would run + * the importing member's cells with no per-tool gate at all — the one thing + * `null` means on this field. + */ + it('dispatches the auto-fired cells under the importing person', async () => { + await performTableCsvImport(importParams({ capabilityGovernedUserId: 'user-9' })) + + expect(mockDispatchAfterBatchInsert).toHaveBeenCalledWith( + expect.anything(), + expect.anything(), + 'req-1', + 'user-1', + 'user-9' + ) + expect(mockImportAppendRows).toHaveBeenCalledWith( + expect.anything(), + expect.anything(), + expect.anything(), + expect.objectContaining({ capabilityGovernedUserId: 'user-9' }) + ) + }) + + /** An actorless import still says so explicitly rather than by omission. */ + it('carries a null subject through unchanged', async () => { + await performTableCsvImport(importParams({ capabilityGovernedUserId: null })) + + expect(mockDispatchAfterBatchInsert).toHaveBeenCalledWith( + expect.anything(), + expect.anything(), + 'req-1', + 'user-1', + null + ) + }) + it('reports the deleted count on a replace', async () => { const result = await performTableCsvImport(importParams({ mode: 'replace' })) @@ -271,6 +309,39 @@ describe('performTableCsvImport', () => { }) }) + it('counts invalid TTL cells that the import blanks', async () => { + const result = await performTableCsvImport( + importParams({ + table: { + ...TABLE, + schema: { + columns: [ + { + id: 'col_expires_at', + name: 'expires_at', + type: 'ttl', + required: false, + unique: false, + }, + ], + }, + }, + fileStream: csvStream('expires_at\n2023-11-14T22:13:20Z\nnot-a-date\n'), + }) + ) + + expect(result.success).toBe(true) + expect(result.data?.rejections).toEqual({ + rowsRejected: 0, + cellsRejected: 1, + rejectedSamples: [], + }) + expect(mockImportAppendRows.mock.calls[0][2]).toEqual([ + { col_expires_at: 1_700_000_000 }, + { col_expires_at: null }, + ]) + }) + it('omits the accounting entirely from a clean import', async () => { const result = await performTableCsvImport(importParams()) @@ -311,6 +382,7 @@ describe('performCreateTableFromCsv', () => { folderId: null, timezone: 'UTC', requestId: 'req-1', + capabilityGovernedUserId: 'user-1', } } diff --git a/apps/sim/lib/table/orchestration/import.ts b/apps/sim/lib/table/orchestration/import.ts index bc9c0d69a5a..5b2dc97549b 100644 --- a/apps/sim/lib/table/orchestration/import.ts +++ b/apps/sim/lib/table/orchestration/import.ts @@ -234,6 +234,14 @@ export interface PerformTableCsvImportParams { /** IANA zone used to read naive datetimes (Excel/Sheets exports carry no offset). */ timezone: string requestId?: string + /** + * The person whose permission group gates any cell this import auto-fires, + * or `null` when no person is behind it. An import lands rows, and landing + * rows starts the table's workflow and enrichment cells. Threaded from the + * surface that holds the principal — the route has already gated the same + * subject. Required; see {@link InsertRowData.capabilityGovernedUserId} in `@/lib/table/types`. + */ + capabilityGovernedUserId: string | null } export interface TableCsvImportData extends ImportRejectionFields { @@ -271,8 +279,17 @@ export interface PerformTableCsvImportResult { export async function performTableCsvImport( params: PerformTableCsvImportParams ): Promise { - const { table, workspaceId, userId, fileStream, fileName, fallbackDelimiter, mode, timezone } = - params + const { + table, + workspaceId, + userId, + fileStream, + fileName, + fallbackDelimiter, + mode, + timezone, + capabilityGovernedUserId, + } = params const requestId = params.requestId ?? generateRequestId() if (table.archivedAt) return fail('Cannot import into an archived table', 'validation') @@ -367,10 +384,11 @@ export async function performTableCsvImport( workspaceId, userId, requestId, + capabilityGovernedUserId, }) // Fire trigger + scheduler AFTER the tx commits — both read through the // global db connection and would otherwise see no rows. - dispatchAfterBatchInsert(finalTable, inserted, requestId, userId) + dispatchAfterBatchInsert(finalTable, inserted, requestId, userId, capabilityGovernedUserId) logger.info(`[${requestId}] Append CSV imported`, { tableId: table.id, @@ -418,6 +436,15 @@ export async function performTableCsvImport( export interface PerformCreateTableFromCsvParams { workspaceId: string userId: string + /** + * The person whose permission group gates any cell this import auto-fires, + * or `null` when no person is behind it. An import lands rows, and landing + * rows starts the table's workflow and enrichment cells. Threaded from the + * surface that holds the principal — the route has already gated the same + * subject. Required; see {@link InsertRowData.capabilityGovernedUserId} in `@/lib/table/types`. + */ + capabilityGovernedUserId: string | null + /** Multipart file stream. The caller still owns destroying it. */ fileStream: Readable fileName: string @@ -461,8 +488,16 @@ export interface PerformCreateTableFromCsvResult { export async function performCreateTableFromCsv( params: PerformCreateTableFromCsvParams ): Promise { - const { workspaceId, userId, fileStream, fileName, fallbackDelimiter, folderId, timezone } = - params + const { + workspaceId, + userId, + fileStream, + fileName, + fallbackDelimiter, + folderId, + timezone, + capabilityGovernedUserId, + } = params const requestId = params.requestId ?? generateRequestId() const { delimiter, stream } = await sniffCsvDelimiterFromStream(fileStream, fallbackDelimiter) @@ -507,6 +542,7 @@ export async function performCreateTableFromCsv( rows: coerced as RowData[], workspaceId, userId, + capabilityGovernedUserId, secretProvenance: coerced.map(createExactEmptyTableRowSecretProvenance), }, // The created table's rowCount is frozen at 0; pass the running total so the diff --git a/apps/sim/lib/table/prestamp-governed-subject.test.ts b/apps/sim/lib/table/prestamp-governed-subject.test.ts new file mode 100644 index 00000000000..70bdf7e0bdb --- /dev/null +++ b/apps/sim/lib/table/prestamp-governed-subject.test.ts @@ -0,0 +1,101 @@ +/** + * @vitest-environment node + */ +import { dbChainMockFns, resetDbChainMock } from '@sim/testing' +import { beforeAll, beforeEach, describe, expect, it, vi } from 'vitest' + +const mocks = vi.hoisted(() => ({ + getTableById: vi.fn(), + writeWorkflowGroupState: vi.fn(), + batchEnqueueAndWait: vi.fn(), +})) + +vi.mock('@/lib/table/events', () => ({ appendTableEvent: vi.fn() })) +vi.mock('@/lib/table/service', () => ({ getTableById: mocks.getTableById })) +vi.mock('@/lib/table/cell-write', () => ({ + writeWorkflowGroupState: mocks.writeWorkflowGroupState, +})) +vi.mock('@/lib/billing/core/billing-attribution', () => ({ + assertBillingAttributionSnapshot: (snapshot: unknown) => snapshot, + resolveBillingAttribution: async () => ({ actorUserId: 'billing-owner' }), + resolveSystemBillingAttribution: async () => ({ actorUserId: null }), +})) +vi.mock('@/lib/core/async-jobs/config', () => ({ + getJobQueue: async () => ({ batchEnqueueAndWait: mocks.batchEnqueueAndWait }), +})) + +import { dispatcherStep } from '@/lib/table/dispatcher' + +const GROUP = { id: 'group-1', workflowId: 'workflow-1', outputs: [] } + +const DISPATCH = { + id: 'tdsp_1', + tableId: 'table-1', + workspaceId: 'workspace-1', + requestId: 'req-1', + mode: 'incomplete', + scope: { groupIds: ['group-1'] }, + status: 'dispatching', + cursor: -1, + limit: null, + processedCount: 0, + isManualRun: true, + triggeredByUserId: 'billing-owner', + capabilityGovernedUserId: 'requesting-member', + requestedAt: new Date('2026-08-21T15:00:00.000Z'), + completedAt: null, + cancelledAt: null, +} + +describe('the dispatcher pre-stamp', () => { + /** + * `buildEnqueueItems` resolves the cell task with a dynamic import of + * `@/background/workflow-column-execution` — the largest graph this step + * touches, and one none of this file's mocks intercept. Under a loaded + * parallel run that first resolution costs whole seconds, which is why the + * only test here needed a 20s budget to hold. Warm it once, outside any + * per-test budget, so the test measures the pre-stamp rather than a module + * load. + */ + beforeAll(async () => { + await Promise.all([ + import('@/background/workflow-column-execution'), + import('@/lib/table/workflow-columns'), + ]) + }, 60_000) + + beforeEach(() => { + vi.clearAllMocks() + resetDbChainMock() + mocks.getTableById.mockResolvedValue({ + id: 'table-1', + workspaceId: 'workspace-1', + schema: { columns: [], workflowGroups: [GROUP] }, + }) + mocks.writeWorkflowGroupState.mockResolvedValue('wrote') + dbChainMockFns.limit + .mockResolvedValueOnce([DISPATCH]) + .mockResolvedValueOnce([{ id: 'row-1', tableId: 'table-1', position: 0, data: {} }]) + .mockResolvedValueOnce([DISPATCH]) + }) + + /** + * The marker outlives its own worker: a cell task that finds the row's + * cascade lock held bails, and the lock owner drains the marker. Without the + * subject on the stamp, that drain runs the request under the owner's + * subject — a different dispatch, often an ungated auto-fire. + */ + it('stamps the dispatch’s governed subject onto every cell it queues', async () => { + await dispatcherStep('tdsp_1') + + expect(mocks.writeWorkflowGroupState).toHaveBeenCalledWith( + expect.anything(), + expect.objectContaining({ + executionState: expect.objectContaining({ + status: 'pending', + capabilityGovernedUserId: 'requesting-member', + }), + }) + ) + }) +}) diff --git a/apps/sim/lib/table/resume-context-governed-subject.test.ts b/apps/sim/lib/table/resume-context-governed-subject.test.ts new file mode 100644 index 00000000000..aa9f977502e --- /dev/null +++ b/apps/sim/lib/table/resume-context-governed-subject.test.ts @@ -0,0 +1,78 @@ +/** + * @vitest-environment node + */ +import { dbChainMockFns, resetDbChainMock } from '@sim/testing' +import { beforeEach, describe, expect, it, vi } from 'vitest' +import { + findCellContextByExecutionId, + stashCellContextForResume, +} from '@/lib/table/workflow-columns' + +const CONTEXT = { + executionId: 'execution-1', + tableId: 'table-1', + tableName: 'Table', + rowId: 'row-1', + groupId: 'group-1', + workspaceId: 'workspace-1', + workflowId: 'workflow-1', + capabilityGovernedUserId: 'requesting-member', +} + +/** + * The pause snapshot is `paused_executions.metadata`, a jsonb document, so the + * subject rides it without a schema change. What this pins is that it is + * actually written and read back — the resume worker has no other source for + * it once the row's marker has been claimed. + */ +describe('the governed subject across a pause', () => { + beforeEach(() => { + vi.clearAllMocks() + resetDbChainMock() + }) + + it('writes the subject into the stashed cell context', async () => { + await stashCellContextForResume(CONTEXT) + + const [{ metadata }] = dbChainMockFns.set.mock.calls[0] + /** The jsonb literal the `||` merge appends, as bound to the SQL template. */ + const [, serializedPatch] = (metadata as { values: string[] }).values + expect(JSON.parse(serializedPatch).cellContext).toMatchObject({ + capabilityGovernedUserId: 'requesting-member', + }) + }) + + it('reads the stashed subject back', async () => { + dbChainMockFns.limit.mockResolvedValueOnce([ + { metadata: { cellContext: { ...CONTEXT, executionId: undefined } } }, + ]) + + const context = await findCellContextByExecutionId('execution-1') + + expect(context?.capabilityGovernedUserId).toBe('requesting-member') + }) + + /** A pause stashed before the subject was carried must read as ungated, not + * as `undefined` leaking into the payload the compiler now requires. */ + it('normalizes a legacy stash with no subject to null', async () => { + dbChainMockFns.limit.mockResolvedValueOnce([ + { + metadata: { + cellContext: { + tableId: 'table-1', + tableName: 'Table', + rowId: 'row-1', + groupId: 'group-1', + workspaceId: 'workspace-1', + workflowId: 'workflow-1', + }, + }, + }, + ]) + + const context = await findCellContextByExecutionId('execution-1') + + expect(context).not.toBeNull() + expect(context?.capabilityGovernedUserId).toBeNull() + }) +}) diff --git a/apps/sim/lib/table/rows/__tests__/ordering-delete.test.ts b/apps/sim/lib/table/rows/__tests__/ordering-delete.test.ts new file mode 100644 index 00000000000..c16b325af9d --- /dev/null +++ b/apps/sim/lib/table/rows/__tests__/ordering-delete.test.ts @@ -0,0 +1,209 @@ +/** + * @vitest-environment node + */ +import { databaseMock } from '@sim/testing' +import { beforeEach, describe, expect, it, vi } from 'vitest' +import type { MutationProof } from '@/lib/table/mutation-locks' +import type { DbTransaction } from '@/lib/table/planner' + +const { mockGetDeleteSnapshotBatchSize } = vi.hoisted(() => ({ + mockGetDeleteSnapshotBatchSize: vi.fn(() => 1), +})) + +vi.mock('@/lib/table/constants', () => ({ + getDeleteSnapshotBatchSize: mockGetDeleteSnapshotBatchSize, + TABLE_LIMITS: { DELETE_SNAPSHOT_BATCH_MAX_BYTES: 100, UPDATE_BATCH_SIZE: 100 }, +})) +vi.mock('@/lib/table/tx', () => ({ setTableTxTimeouts: vi.fn() })) + +import { + type DeletedRowsHandler, + deleteOrderedRowsByIds, + deletePageByIds, + planDeleteSnapshotBatch, +} from '@/lib/table/rows/ordering' + +const mockTransaction = databaseMock.db.transaction as ReturnType +const proof = {} as MutationProof<'delete'> + +type DeleteRunner = (onDeleted: DeletedRowsHandler) => Promise + +describe('ordered row delete trigger handoff', () => { + beforeEach(() => { + vi.clearAllMocks() + mockGetDeleteSnapshotBatchSize.mockReturnValue(1) + }) + + it.each([ + [ + 'direct deletes', + (onDeleted: DeletedRowsHandler) => + deleteOrderedRowsByIds({ + tableId: 'table-1', + workspaceId: 'workspace-1', + rowIds: ['row-1', 'row-2'], + proof, + onDeleted, + }), + ], + [ + 'background delete pages', + (onDeleted: DeletedRowsHandler) => + deletePageByIds('table-1', 'workspace-1', ['row-1', 'row-2'], proof, undefined, onDeleted), + ], + ])( + 'runs %s handlers after commit and before the next batch', + async (_label, run: DeleteRunner) => { + const events: string[] = [] + let batchIndex = 0 + let releaseFirstHandler: (() => void) | undefined + const firstHandlerGate = new Promise((resolve) => { + releaseFirstHandler = resolve + }) + const trx = { + select: () => ({ + from: () => ({ + where: () => ({ + orderBy: () => ({ + for: async () => [{ id: `row-${batchIndex + 1}`, snapshotBytes: 20 }], + }), + }), + }), + }), + delete: () => ({ + where: () => ({ + returning: async () => { + const id = `row-${batchIndex + 1}` + batchIndex++ + return [{ id, data: { title: id } }] + }, + }), + }), + } as unknown as DbTransaction + + mockTransaction.mockImplementation( + async (callback: (transaction: DbTransaction) => Promise) => { + const result = await callback(trx) + events.push(`commit-${mockTransaction.mock.calls.length}`) + return result + } + ) + + const onDeleted = vi.fn(async (rows: Array<{ id: string }>) => { + events.push(`trigger-${rows[0]?.id}`) + if (rows[0]?.id === 'row-1') await firstHandlerGate + }) + const pending = run(onDeleted) + + await vi.waitFor(() => { + expect(events).toEqual(['commit-1', 'trigger-row-1']) + }) + expect(mockTransaction).toHaveBeenCalledTimes(1) + + releaseFirstHandler?.() + await pending + + expect(events).toEqual(['commit-1', 'trigger-row-1', 'commit-2', 'trigger-row-2']) + expect(onDeleted.mock.calls.map(([rows]) => rows)).toEqual([ + [{ id: 'row-1', data: { title: 'row-1' } }], + [{ id: 'row-2', data: { title: 'row-2' } }], + ]) + } + ) + + it('splits one count-sized candidate batch at the snapshot byte budget', async () => { + mockGetDeleteSnapshotBatchSize.mockReturnValue(3) + const snapshots = [ + [ + { id: 'row-1', snapshotBytes: 60 }, + { id: 'row-2', snapshotBytes: 60 }, + { id: 'row-3', snapshotBytes: 10 }, + ], + [ + { id: 'row-2', snapshotBytes: 60 }, + { id: 'row-3', snapshotBytes: 10 }, + ], + ] + const deletedBatches = [ + [{ id: 'row-1', data: { title: 'row-1' } }], + [ + { id: 'row-2', data: { title: 'row-2' } }, + { id: 'row-3', data: { title: 'row-3' } }, + ], + ] + let transactionIndex = 0 + + mockTransaction.mockImplementation( + async (callback: (transaction: DbTransaction) => Promise) => { + const currentIndex = transactionIndex++ + const trx = { + select: () => ({ + from: () => ({ + where: () => ({ + orderBy: () => ({ + for: async () => snapshots[currentIndex], + }), + }), + }), + }), + delete: () => ({ + where: () => ({ + returning: async () => deletedBatches[currentIndex], + }), + }), + } as unknown as DbTransaction + return callback(trx) + } + ) + const onDeleted = vi.fn() + + await expect( + deleteOrderedRowsByIds({ + tableId: 'table-1', + workspaceId: 'workspace-1', + rowIds: ['row-1', 'row-2', 'row-3'], + proof, + onDeleted, + }) + ).resolves.toEqual(['row-1', 'row-2', 'row-3']) + + expect(mockTransaction).toHaveBeenCalledTimes(2) + expect(onDeleted.mock.calls.map(([rows]) => rows)).toEqual(deletedBatches) + }) +}) + +describe('delete snapshot byte planning', () => { + it('stops before an existing row would exceed the byte budget', () => { + expect( + planDeleteSnapshotBatch( + ['missing-row', 'row-1', 'row-2'], + [ + { id: 'row-1', snapshotBytes: 60 }, + { id: 'row-2', snapshotBytes: 60 }, + ], + 100 + ) + ).toEqual({ + rowIds: ['missing-row', 'row-1'], + consumedCount: 2, + oversizedRow: undefined, + }) + }) + + it('isolates an oversized legacy row so no other snapshot joins it', () => { + expect( + planDeleteSnapshotBatch( + ['legacy-row', 'row-2'], + [ + { id: 'legacy-row', snapshotBytes: 150 }, + { id: 'row-2', snapshotBytes: 10 }, + ], + 100 + ) + ).toEqual({ + rowIds: ['legacy-row'], + consumedCount: 1, + oversizedRow: { id: 'legacy-row', snapshotBytes: 150 }, + }) + }) +}) diff --git a/apps/sim/lib/table/rows/executions.test.ts b/apps/sim/lib/table/rows/executions.test.ts index dff9b36ae57..01ad6b1c56d 100644 --- a/apps/sim/lib/table/rows/executions.test.ts +++ b/apps/sim/lib/table/rows/executions.test.ts @@ -33,6 +33,48 @@ describe('writeExecutionsPatch guards', () => { resetDbChainMock() }) + /** + * The dispatcher's `pending` marker is drained by whichever worker owns the + * row's cascade lock, which may belong to another dispatch entirely. Storing + * the requesting subject with the marker is what lets that drain run under + * the person who asked rather than under the owner's own subject. + */ + it('persists the pre-stamp’s governed subject on both the insert and the upsert', async () => { + await writeExecutionsPatch( + dbChainMock.db as unknown as Parameters[0], + 'table-1', + 'row-1', + { + 'group-1': { + ...EXECUTION_STATE, + status: 'pending', + executionId: null, + capabilityGovernedUserId: 'requesting-member', + }, + } + ) + + const values = dbChainMockFns.values.mock.calls[0]?.[0] as Record + expect(values.capabilityGovernedUserId).toBe('requesting-member') + const conflict = dbChainMockFns.onConflictDoUpdate.mock.calls[0]?.[0] as { + set: Record + } + expect(conflict.set.capabilityGovernedUserId).toBe('requesting-member') + }) + + /** A write that names no subject clears it — only an unclaimed marker is read. */ + it('writes null for a state that carries no subject', async () => { + await writeExecutionsPatch( + dbChainMock.db as unknown as Parameters[0], + 'table-1', + 'row-1', + { 'group-1': EXECUTION_STATE } + ) + + const values = dbChainMockFns.values.mock.calls[0]?.[0] as Record + expect(values.capabilityGovernedUserId).toBeNull() + }) + it('rejects a worker write when the atomic stale-or-cancel predicate returns no row', async () => { dbChainMockFns.returning.mockResolvedValueOnce([]) diff --git a/apps/sim/lib/table/rows/executions.ts b/apps/sim/lib/table/rows/executions.ts index e1cae632391..3ed355573c6 100644 --- a/apps/sim/lib/table/rows/executions.ts +++ b/apps/sim/lib/table/rows/executions.ts @@ -5,6 +5,7 @@ * directly from `@/lib/table/rows/executions`. */ +import { db } from '@sim/db' import { tableRowExecutions, userTableRows } from '@sim/db/schema' import { and, eq, inArray, type SQL, sql } from 'drizzle-orm' import type { DbOrTx } from '@/lib/db/types' @@ -309,6 +310,12 @@ export async function writeExecutionsPatch( runningBlockIds: value.runningBlockIds ?? [], blockErrors: value.blockErrors ?? {}, cancelledAt: value.cancelledAt ? new Date(value.cancelledAt) : null, + /** + * Written verbatim rather than made sticky like `enrichmentDetails`: only + * an unclaimed pre-stamp is ever read for it, and a re-stamp by a + * different dispatch must not inherit the previous run's subject. + */ + capabilityGovernedUserId: value.capabilityGovernedUserId ?? null, enrichmentDetails: value.enrichmentDetails ?? null, updatedAt: new Date(), } as const @@ -346,6 +353,7 @@ export async function writeExecutionsPatch( runningBlockIds: insertValues.runningBlockIds, blockErrors: insertValues.blockErrors, cancelledAt: insertValues.cancelledAt, + capabilityGovernedUserId: insertValues.capabilityGovernedUserId, // Sticky: preserve a prior cascade breakdown when this write omits // it (e.g. the running pickup stamp) so only an explicit detail // overwrites it. Re-runs delete the row first, so this never serves @@ -374,6 +382,7 @@ export async function writeExecutionsPatch( runningBlockIds: insertValues.runningBlockIds, blockErrors: insertValues.blockErrors, cancelledAt: insertValues.cancelledAt, + capabilityGovernedUserId: insertValues.capabilityGovernedUserId, // Sticky: preserve a prior cascade breakdown when this write omits it // (e.g. the running pickup stamp) so only an explicit detail overwrites // it. Re-runs delete the row first, so this never serves stale detail. @@ -386,6 +395,87 @@ export async function writeExecutionsPatch( return 'wrote' } +/** + * The governed subject persisted with a cell's dispatcher pre-stamp. + * + * Read on the drain path only — a worker taking over a `pending` marker it did + * not stamp — so the column stays off the hot grid read (`loadExecutionsByRow`) + * and never reaches a client. Returns `null` for a marker written before the + * column existed and for a genuinely actorless request; both mean the same + * thing to the gate. + */ +export async function readStampedCapabilitySubject( + rowId: string, + groupId: string +): Promise { + const [stamped] = await db + .select({ capabilityGovernedUserId: tableRowExecutions.capabilityGovernedUserId }) + .from(tableRowExecutions) + .where(and(eq(tableRowExecutions.rowId, rowId), eq(tableRowExecutions.groupId, groupId))) + .limit(1) + return stamped?.capabilityGovernedUserId ?? null +} + +/** One cell whose unclaimed marker {@link cancelPendingMarkersForGovernedSubject} stopped. */ +export interface CancelledCellMarker { + tableId: string + rowId: string + groupId: string +} + +/** + * Terminalizes every still-unstarted cell marker stamped with `userId`, in the + * caller's transaction. + * + * Cancelling the departing account's `table_run_dispatches` rows is not enough + * on its own. A pre-stamp on `table_row_executions` is drained by whichever + * worker holds the row's cascade lock, and that worker's dispatch-cancel guard + * consults ITS OWN dispatch — so an unrelated, still-active sibling dispatch + * happily drains the deleted person's marker. The subject reference is + * `ON DELETE SET NULL`, which by then makes the marker indistinguishable from a + * legitimately actorless request: the drain runs it with no per-tool gate at + * all. Going terminal here is the same honest reading the dispatch cancel takes + * — a deleted person's runs stop rather than silently lose their gate. + * + * Scoped to `pending`/`queued` because those are the states a marker sits in + * before a worker claims it; a claimed or terminal row carries no subject to + * match anyway. The written state is the canonical cancel + * (`buildCancelledExecution`), which every drain path's `isExecCancelled` check + * already refuses to run. + * + * Returns what it stopped so the caller can announce it: this write is not the + * cancel path the UI listens to, and a collaborator watching the table would + * otherwise keep the cells on their in-flight pill until something else touched + * the row. + */ +export async function cancelPendingMarkersForGovernedSubject( + trx: DbOrTx, + userId: string +): Promise { + const now = new Date() + return trx + .update(tableRowExecutions) + .set({ + status: 'cancelled', + jobId: null, + error: 'Cancelled', + runningBlockIds: [], + cancelledAt: now, + updatedAt: now, + }) + .where( + and( + eq(tableRowExecutions.capabilityGovernedUserId, userId), + inArray(tableRowExecutions.status, ['pending', 'queued']) + ) + ) + .returning({ + tableId: tableRowExecutions.tableId, + rowId: tableRowExecutions.rowId, + groupId: tableRowExecutions.groupId, + }) +} + /** * Strips the given workflow group ids from every row's executions on a table — * used by the column / group delete paths so stale running/queued exec records diff --git a/apps/sim/lib/table/rows/ordering.ts b/apps/sim/lib/table/rows/ordering.ts index c776cc6e075..f20ebee0ef0 100644 --- a/apps/sim/lib/table/rows/ordering.ts +++ b/apps/sim/lib/table/rows/ordering.ts @@ -8,9 +8,10 @@ import { db } from '@sim/db' import { userTableRows } from '@sim/db/schema' +import { createLogger } from '@sim/logger' import { and, asc, desc, eq, gt, inArray, lt, lte, type SQL, sql } from 'drizzle-orm' import type { DbOrTx } from '@/lib/db/types' -import { TABLE_LIMITS } from '@/lib/table/constants' +import { getDeleteSnapshotBatchSize, TABLE_LIMITS } from '@/lib/table/constants' import type { MutationProof } from '@/lib/table/mutation-locks' import { keyBetween, nKeysBetween } from '@/lib/table/order-key' import { type DbExecutor, type DbTransaction, withSeqscanOff } from '@/lib/table/planner' @@ -19,6 +20,106 @@ import { mutateTableRowsWithSecretProvenance } from '@/lib/table/rows/secret-pro import { setTableTxTimeouts } from '@/lib/table/tx' import type { RowData, TableDefinition, TableRowSecretProvenanceWrite } from '@/lib/table/types' +const logger = createLogger('TableRowOrdering') + +export interface DeletedTableRow { + id: string + data: RowData +} + +export type DeletedRowsHandler = ( + rows: DeletedTableRow[], + table?: TableDefinition +) => void | Promise + +interface DeleteSnapshotSize { + id: string + snapshotBytes: number +} + +interface DeleteSnapshotBatchPlan { + rowIds: string[] + consumedCount: number + oversizedRow?: DeleteSnapshotSize +} + +/** + * Selects the largest input-order prefix whose existing rows fit the snapshot + * byte budget. Missing ids are consumed without cost. A legacy row that already + * exceeds the budget is isolated as the only existing row in its transaction so + * deleting historical data remains possible without combining it with another + * snapshot. + */ +export function planDeleteSnapshotBatch( + candidateRowIds: readonly string[], + snapshotSizes: readonly DeleteSnapshotSize[], + maxBytes = TABLE_LIMITS.DELETE_SNAPSHOT_BATCH_MAX_BYTES +): DeleteSnapshotBatchPlan { + const bytesById = new Map(snapshotSizes.map((row) => [row.id, row.snapshotBytes])) + let consumedCount = 0 + let batchBytes = 0 + let existingRows = 0 + let oversizedRow: DeleteSnapshotSize | undefined + + for (const id of candidateRowIds) { + const measuredBytes = bytesById.get(id) + if (measuredBytes === undefined) { + consumedCount++ + continue + } + const snapshotBytes = + Number.isFinite(measuredBytes) && measuredBytes >= 0 ? measuredBytes : maxBytes + 1 + if (existingRows > 0 && batchBytes + snapshotBytes > maxBytes) break + + consumedCount++ + existingRows++ + batchBytes += snapshotBytes + if (snapshotBytes > maxBytes) { + oversizedRow = { id, snapshotBytes } + break + } + } + + return { + rowIds: candidateRowIds.slice(0, consumedCount), + consumedCount, + oversizedRow, + } +} + +async function planLockedDeleteSnapshotBatch( + trx: DbTransaction, + tableId: string, + workspaceId: string, + candidateRowIds: readonly string[] +): Promise { + const snapshotSizes = await trx + .select({ + id: userTableRows.id, + snapshotBytes: sql`octet_length(${userTableRows.data}::text)`.mapWith(Number), + }) + .from(userTableRows) + .where( + and( + eq(userTableRows.tableId, tableId), + eq(userTableRows.workspaceId, workspaceId), + inArray(userTableRows.id, [...candidateRowIds]) + ) + ) + .orderBy(asc(userTableRows.id)) + .for('update') + return planDeleteSnapshotBatch(candidateRowIds, snapshotSizes) +} + +function warnForOversizedLegacySnapshot(oversizedRow: DeleteSnapshotSize | undefined): void { + if (!oversizedRow) return + logger.warn('Deleting oversized legacy row in an isolated snapshot batch', { + rowId: oversizedRow.id, + snapshotBytes: oversizedRow.snapshotBytes, + maxBytes: TABLE_LIMITS.DELETE_SNAPSHOT_BATCH_MAX_BYTES, + }) +} + /** * Starting `position` for an append import — `max(position) + 1`, or 0 when empty. Read once, * unlocked, before streaming: the import worker is the table's sole writer, so it can assign @@ -274,8 +375,8 @@ export async function insertOrderedRow(params: { /** * Deletes a single row by id in its own transaction. Deleting a row never changes - * another row's `order_key`, so no positional reshift is needed. Returns `false` - * when no row matched. + * another row's `order_key`, so no positional reshift is needed. Returns the + * deleted row snapshot, or `null` when no row matched. */ export async function deleteOrderedRow(params: { tableId: string @@ -283,9 +384,9 @@ export async function deleteOrderedRow(params: { workspaceId: string /** Proof the caller asserted the delete lock (see `mutation-locks.ts`). */ proof: MutationProof<'delete'> -}): Promise { +}): Promise { const { tableId, rowId, workspaceId } = params - return db.transaction(async (trx) => { + const deletedRow = await db.transaction(async (trx) => { await setTableTxTimeouts(trx) const [deleted] = await trx .delete(userTableRows) @@ -296,16 +397,27 @@ export async function deleteOrderedRow(params: { eq(userTableRows.workspaceId, workspaceId) ) ) - .returning({ id: userTableRows.id }) - return Boolean(deleted) + .returning({ id: userTableRows.id, data: userTableRows.data }) + return deleted ? { id: deleted.id, data: deleted.data as RowData } : null }) + if (deletedRow) { + const snapshotBytes = Buffer.byteLength(JSON.stringify(deletedRow.data), 'utf8') + warnForOversizedLegacySnapshot( + snapshotBytes > TABLE_LIMITS.DELETE_SNAPSHOT_BATCH_MAX_BYTES + ? { id: deletedRow.id, snapshotBytes } + : undefined + ) + } + return deletedRow } /** - * Deletes the given row ids in batches within one transaction. Deletes leave - * `order_key` untouched, so no positional recompaction is needed. Returns the - * deleted row ids. The caller resolves which ids to delete (used by both - * delete-by-ids and delete-by-filter). + * Deletes the given row ids in byte-bounded, independently committed batches. + * Deletes leave `order_key` untouched, so no positional recompaction is needed. + * The post-commit handler is awaited before the next batch so deleted JSON + * snapshots cannot accumulate in memory. Returns only the compact deleted ids; + * the caller resolves which ids to delete (used by both delete-by-ids and + * delete-by-filter). */ export async function deleteOrderedRowsByIds(params: { tableId: string @@ -313,28 +425,38 @@ export async function deleteOrderedRowsByIds(params: { rowIds: string[] /** Proof the caller asserted the delete lock (see `mutation-locks.ts`). */ proof: MutationProof<'delete'> -}): Promise<{ id: string }[]> { - const { tableId, workspaceId, rowIds } = params + /** Handles each bounded snapshot batch after its transaction commits. */ + onDeleted?: DeletedRowsHandler +}): Promise { + const { tableId, workspaceId, rowIds, onDeleted } = params if (rowIds.length === 0) return [] - return db.transaction(async (trx) => { - await setTableTxTimeouts(trx, { statementMs: 60_000 }) - const deleted: { id: string }[] = [] - for (let i = 0; i < rowIds.length; i += TABLE_LIMITS.DELETE_BATCH_SIZE) { - const batch = rowIds.slice(i, i + TABLE_LIMITS.DELETE_BATCH_SIZE) + const batchSize = getDeleteSnapshotBatchSize() + const deletedIds: string[] = [] + let index = 0 + while (index < rowIds.length) { + const candidates = rowIds.slice(index, index + batchSize) + const { rows, plan } = await db.transaction(async (trx) => { + await setTableTxTimeouts(trx, { statementMs: 60_000 }) + const plan = await planLockedDeleteSnapshotBatch(trx, tableId, workspaceId, candidates) const rows = await trx .delete(userTableRows) .where( and( eq(userTableRows.tableId, tableId), eq(userTableRows.workspaceId, workspaceId), - inArray(userTableRows.id, batch) + inArray(userTableRows.id, plan.rowIds) ) ) - .returning({ id: userTableRows.id }) - deleted.push(...rows) - } - return deleted - }) + .returning({ id: userTableRows.id, data: userTableRows.data }) + return { rows, plan } + }) + index += plan.consumedCount + warnForOversizedLegacySnapshot(plan.oversizedRow) + const deletedRows = rows.map((row) => ({ id: row.id, data: row.data as RowData })) + deletedIds.push(...deletedRows.map((row) => row.id)) + await onDeleted?.(deletedRows) + } + return deletedIds } /** @@ -467,26 +589,36 @@ export async function deletePageByIds( /** Proof the caller asserted the delete lock (see `mutation-locks.ts`). */ _proof: MutationProof<'delete'>, /** Re-asserts the lock inside each batch transaction. See {@link guardBatch}. */ - revalidate?: MutationRevalidator + revalidate?: MutationRevalidator, + /** Called after each batch commits, with snapshots suitable for delete triggers. */ + onDeleted?: DeletedRowsHandler ): Promise { let deleted = 0 - for (let i = 0; i < rowIds.length; i += TABLE_LIMITS.DELETE_BATCH_SIZE) { - const batch = rowIds.slice(i, i + TABLE_LIMITS.DELETE_BATCH_SIZE) - const rows = await db.transaction(async (trx) => { + const batchSize = getDeleteSnapshotBatchSize() + let index = 0 + while (index < rowIds.length) { + const candidates = rowIds.slice(index, index + batchSize) + const { rows, table, plan } = await db.transaction(async (trx) => { await setTableTxTimeouts(trx, { statementMs: 60_000 }) - await guardBatch(trx, tableId, revalidate) - return trx + const table = await guardBatch(trx, tableId, revalidate) + const plan = await planLockedDeleteSnapshotBatch(trx, tableId, workspaceId, candidates) + const rows = await trx .delete(userTableRows) .where( and( eq(userTableRows.tableId, tableId), eq(userTableRows.workspaceId, workspaceId), - inArray(userTableRows.id, batch) + inArray(userTableRows.id, plan.rowIds) ) ) - .returning({ id: userTableRows.id }) + .returning({ id: userTableRows.id, data: userTableRows.data }) + return { rows, table, plan } }) - deleted += rows.length + index += plan.consumedCount + warnForOversizedLegacySnapshot(plan.oversizedRow) + const deletedRows = rows.map((row) => ({ id: row.id, data: row.data as RowData })) + deleted += deletedRows.length + await onDeleted?.(deletedRows, table) } return deleted } diff --git a/apps/sim/lib/table/rows/service.ts b/apps/sim/lib/table/rows/service.ts index f8933c05cdf..ebed42b9c74 100644 --- a/apps/sim/lib/table/rows/service.ts +++ b/apps/sim/lib/table/rows/service.ts @@ -51,6 +51,7 @@ import { } from '@/lib/table/rows/executions' import { acquireRowOrderLock, + type DeletedTableRow, deleteOrderedRow, deleteOrderedRowsByIds, insertOrderedRow, @@ -110,6 +111,24 @@ import { cancelWorkflowGroupRuns, runWorkflowColumn } from '@/lib/table/workflow const logger = createLogger('TableRowsService') +async function dispatchDeleteTriggers( + table: TableDefinition, + deletedRows: DeletedTableRow[], + requestId: string +): Promise { + if (deletedRows.length === 0) return + await fireTableTrigger( + table.id, + table.workspaceId, + table.name, + 'delete', + deletedRows, + null, + table.schema, + requestId + ) +} + /** * Inserts a single row into a table. * @@ -207,6 +226,7 @@ export async function insertRow( void fireTableTrigger( data.tableId, + table.workspaceId, table.name, 'insert', [insertedRow], @@ -222,6 +242,7 @@ export async function insertRow( isManualRun: false, requestId, triggeredByUserId: data.userId, + capabilityGovernedUserId: data.capabilityGovernedUserId, }).catch((err) => logger.error(`[${requestId}] auto-dispatch (insertRow) failed:`, err)) return insertedRow @@ -259,7 +280,7 @@ export async function batchInsertRows( addedRows: result.length, limit: rowLimit, }) - dispatchAfterBatchInsert(table, result, requestId, data.userId) + dispatchAfterBatchInsert(table, result, requestId, data.userId, data.capabilityGovernedUserId) return result } @@ -381,9 +402,20 @@ export function dispatchAfterBatchInsert( table: TableDefinition, result: TableRow[], requestId: string, - actorUserId?: string | null + actorUserId: string | null | undefined, + /** The gate's subject for the auto-fire pass; see {@link InsertRowData.capabilityGovernedUserId}. */ + capabilityGovernedUserId: string | null ): void { - void fireTableTrigger(table.id, table.name, 'insert', result, null, table.schema, requestId) + void fireTableTrigger( + table.id, + table.workspaceId, + table.name, + 'insert', + result, + null, + table.schema, + requestId + ) // Scope to the newly-inserted row ids so the dispatcher doesn't walk every // row in the table. After the sidecar migration, all existing rows have // zero entries → `mode:'new'`'s `NOT EXISTS` filter would otherwise include @@ -396,6 +428,7 @@ export function dispatchAfterBatchInsert( isManualRun: false, requestId, triggeredByUserId: actorUserId, + capabilityGovernedUserId, }).catch((err) => logger.error(`[${requestId}] auto-dispatch (batchInsertRows) failed:`, err)) } @@ -865,6 +898,7 @@ export async function upsertRow( }) void fireTableTrigger( data.tableId, + table.workspaceId, table.name, 'insert', [result.row], @@ -876,6 +910,7 @@ export async function upsertRow( const oldRows = new Map([[result.row.id, result.previousData]]) void fireTableTrigger( data.tableId, + table.workspaceId, table.name, 'update', [result.row], @@ -892,6 +927,7 @@ export async function upsertRow( isManualRun: false, requestId, triggeredByUserId: data.userId, + capabilityGovernedUserId: data.capabilityGovernedUserId, }).catch((err) => logger.error(`[${requestId}] auto-dispatch (upsertRow) failed:`, err)) return result @@ -1801,6 +1837,7 @@ export async function updateRow( const oldRows = new Map([[data.rowId, existingRow.data as RowData]]) void fireTableTrigger( data.tableId, + table.workspaceId, table.name, 'update', [updatedRow], @@ -1843,6 +1880,7 @@ export async function updateRow( groupIds: inFlightDownstreamGroups, requestId, triggeredByUserId: data.actorUserId, + capabilityGovernedUserId: data.capabilityGovernedUserId, }) } catch (err) { logger.error(`[${requestId}] cancel+rerun for in-flight downstream groups failed:`, err) @@ -1857,6 +1895,7 @@ export async function updateRow( isManualRun: false, requestId, triggeredByUserId: data.actorUserId, + capabilityGovernedUserId: data.capabilityGovernedUserId, }).catch((err) => logger.error(`[${requestId}] auto-dispatch (updateRow) failed:`, err)) return updatedRow @@ -1886,6 +1925,7 @@ export async function deleteRow( if (!deleted) throw new OrchestrationError('not_found', 'Row not found') logger.info(`[${requestId}] Deleted row ${rowId} from table ${table.id}`) + void dispatchDeleteTriggers(table, [deleted], requestId) } type BulkUpdateMatch = { id: string; data: RowData } @@ -2044,7 +2084,9 @@ function dispatchBulkUpdateEffects( patch: RowData, now: Date, requestId: string, - actorUserId: BulkUpdateData['actorUserId'] + actorUserId: BulkUpdateData['actorUserId'], + /** The gate's subject for the auto-fire pass; see {@link BulkUpdateData.capabilityGovernedUserId}. */ + capabilityGovernedUserId: string | null ): void { const affectedRowIdSet = new Set(affectedRowIds) const affectedRows = rows.filter((row) => affectedRowIdSet.has(row.id)) @@ -2061,6 +2103,7 @@ function dispatchBulkUpdateEffects( })) void fireTableTrigger( table.id, + table.workspaceId, table.name, 'update', updatedRows, @@ -2076,6 +2119,7 @@ function dispatchBulkUpdateEffects( isManualRun: false, requestId, triggeredByUserId: actorUserId, + capabilityGovernedUserId, }).catch((error) => logger.error(`[${requestId}] auto-dispatch (updateRowsByFilter) failed:`, error) ) @@ -2207,7 +2251,8 @@ export async function updateRowsByFilter( data.data, now, requestId, - data.actorUserId + data.actorUserId, + data.capabilityGovernedUserId ) afterId = nextAfterId if (batchRows.length < TABLE_LIMITS.UPDATE_BATCH_SIZE) break @@ -2277,7 +2322,8 @@ export async function updateRowsByFilter( data.data, now, requestId, - data.actorUserId + data.actorUserId, + data.capabilityGovernedUserId ) return { @@ -2483,6 +2529,7 @@ export async function batchUpdateRows( if (updatedRowsForTrigger.length > 0) { void fireTableTrigger( data.tableId, + table.workspaceId, table.name, 'update', updatedRowsForTrigger, @@ -2515,6 +2562,7 @@ export async function batchUpdateRows( groupIds: inFlightDownstreamGroups, requestId, triggeredByUserId: data.actorUserId, + capabilityGovernedUserId: data.capabilityGovernedUserId, }) } } catch (err) { @@ -2534,6 +2582,7 @@ export async function batchUpdateRows( isManualRun: false, requestId, triggeredByUserId: data.actorUserId, + capabilityGovernedUserId: data.capabilityGovernedUserId, }).catch((err) => logger.error(`[${requestId}] auto-dispatch (batchUpdateRows) failed:`, err)) } @@ -2573,7 +2622,7 @@ export async function deleteRowsByFilter( ) const limit = data.limit - const deletedRows: { id: string }[] = [] + const deletedRowIds: string[] = [] if (limit === undefined) { const cutoff = new Date() let afterId: string | undefined @@ -2589,14 +2638,14 @@ export async function deleteRowsByFilter( if (page.length === 0) break const nextAfterId = page[page.length - 1] for (let index = 0; index < page.length; index += TABLE_LIMITS.DELETE_BATCH_SIZE) { - deletedRows.push( - ...(await deleteOrderedRowsByIds({ - tableId: table.id, - workspaceId: table.workspaceId, - rowIds: page.slice(index, index + TABLE_LIMITS.DELETE_BATCH_SIZE), - proof, - })) - ) + const deletedIds = await deleteOrderedRowsByIds({ + tableId: table.id, + workspaceId: table.workspaceId, + rowIds: page.slice(index, index + TABLE_LIMITS.DELETE_BATCH_SIZE), + proof, + onDeleted: (rows) => dispatchDeleteTriggers(table, rows, requestId), + }) + deletedRowIds.push(...deletedIds) } afterId = nextAfterId if (page.length < TABLE_LIMITS.DELETE_PAGE_SIZE) break @@ -2612,19 +2661,18 @@ export async function deleteRowsByFilter( ) const rowIds = matchingRows.map((row) => row.id) if (rowIds.length > 0) { - deletedRows.push( - ...(await deleteOrderedRowsByIds({ - tableId: table.id, - workspaceId: table.workspaceId, - rowIds, - proof, - })) - ) + const deletedIds = await deleteOrderedRowsByIds({ + tableId: table.id, + workspaceId: table.workspaceId, + rowIds, + proof, + onDeleted: (rows) => dispatchDeleteTriggers(table, rows, requestId), + }) + deletedRowIds.push(...deletedIds) } } - if (deletedRows.length === 0) return { affectedCount: 0, affectedRowIds: [] } - const deletedRowIds = deletedRows.map((row) => row.id) + if (deletedRowIds.length === 0) return { affectedCount: 0, affectedRowIds: [] } logger.info(`[${requestId}] Deleted ${deletedRowIds.length} rows from table ${table.id}`) @@ -2650,19 +2698,18 @@ export async function deleteRowsByIds( const uniqueRequestedRowIds = Array.from(new Set(data.rowIds)) - const deletedRows = await deleteOrderedRowsByIds({ + const deletedIds = await deleteOrderedRowsByIds({ tableId: data.tableId, workspaceId: data.workspaceId, rowIds: uniqueRequestedRowIds, proof, + onDeleted: (rows) => dispatchDeleteTriggers(table, rows, requestId), }) - const deletedIds = deletedRows.map((r) => r.id) const deletedIdSet = new Set(deletedIds) const missingRowIds = uniqueRequestedRowIds.filter((id) => !deletedIdSet.has(id)) logger.info(`[${requestId}] Deleted ${deletedIds.length} rows by ID from table ${data.tableId}`) - return { deletedCount: deletedIds.length, deletedRowIds: deletedIds, diff --git a/apps/sim/lib/table/run-column-governed-subject.test.ts b/apps/sim/lib/table/run-column-governed-subject.test.ts new file mode 100644 index 00000000000..86c3ca3be52 --- /dev/null +++ b/apps/sim/lib/table/run-column-governed-subject.test.ts @@ -0,0 +1,86 @@ +/** + * @vitest-environment node + */ +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const mocks = vi.hoisted(() => ({ + getTableById: vi.fn(), + insertDispatch: vi.fn(async () => 'tdsp_1'), + readDispatch: vi.fn(async () => null), + cancelDispatchById: vi.fn(), + bulkClearWorkflowGroupCells: vi.fn(async () => false), + runDispatcherToCompletion: vi.fn(), + resolveTableDispatchConcurrency: vi.fn(async () => 5), +})) + +vi.mock('@/lib/table/service', () => ({ getTableById: mocks.getTableById })) +vi.mock('@/lib/table/dispatcher', () => ({ + bulkClearWorkflowGroupCells: mocks.bulkClearWorkflowGroupCells, + cancelDispatchById: mocks.cancelDispatchById, + insertDispatch: mocks.insertDispatch, + readDispatch: mocks.readDispatch, + runDispatcherToCompletion: mocks.runDispatcherToCompletion, +})) +vi.mock('@/lib/table/dispatch-concurrency', () => ({ + resolveTableDispatchConcurrency: mocks.resolveTableDispatchConcurrency, +})) + +import { runWorkflowColumn } from '@/lib/table/workflow-columns' + +const TABLE = { + id: 'table-1', + workspaceId: 'workspace-1', + schema: { columns: [], workflowGroups: [{ id: 'group-1', outputs: [] }] }, +} + +const BASE = { + tableId: 'table-1', + workspaceId: 'workspace-1', + groupIds: ['group-1'], + mode: 'new' as const, + isManualRun: false, + requestId: 'req-1', +} + +/** The dispatch row `runWorkflowColumn` asked the dispatcher to insert. */ +function inserted(): Record { + expect(mocks.insertDispatch).toHaveBeenCalledTimes(1) + return mocks.insertDispatch.mock.calls[0][0] as Record +} + +describe('runWorkflowColumn governed subject', () => { + beforeEach(() => { + vi.clearAllMocks() + mocks.getTableById.mockResolvedValue(TABLE) + mocks.insertDispatch.mockResolvedValue('tdsp_1') + mocks.readDispatch.mockResolvedValue(null) + mocks.bulkClearWorkflowGroupCells.mockResolvedValue(false) + mocks.resolveTableDispatchConcurrency.mockResolvedValue(5) + }) + + /** + * The row-write auto-fire case: a workspace API key wrote the row, so the + * attribution names the workspace billed account. Forwarding that as the gate + * subject — which an optional field with a fallback did — puts a bystander's + * tool denylist on a run nobody governs. + */ + it('forwards an explicit null past a non-null attribution', async () => { + await runWorkflowColumn({ + ...BASE, + triggeredByUserId: 'billing-owner', + capabilityGovernedUserId: null, + }) + const row = inserted() + expect(row.triggeredByUserId).toBe('billing-owner') + expect(row.capabilityGovernedUserId).toBeNull() + }) + + it('forwards the acting person for a session-initiated run', async () => { + await runWorkflowColumn({ + ...BASE, + triggeredByUserId: 'user-1', + capabilityGovernedUserId: 'user-1', + }) + expect(inserted().capabilityGovernedUserId).toBe('user-1') + }) +}) diff --git a/apps/sim/lib/table/schema-invariants.ts b/apps/sim/lib/table/schema-invariants.ts index 132b343dddc..ffb7407f1b5 100644 --- a/apps/sim/lib/table/schema-invariants.ts +++ b/apps/sim/lib/table/schema-invariants.ts @@ -11,6 +11,7 @@ import { OrchestrationError } from '@/lib/core/orchestration/types' import { getColumnId } from '@/lib/table/column-keys' +import { validateColumnTypeLimits } from '@/lib/table/column-types' import type { TableSchema, WorkflowGroup } from '@/lib/table/types' /** @@ -19,7 +20,7 @@ import type { TableSchema, WorkflowGroup } from '@/lib/table/types' * etc. Returns a list of human-readable errors (empty if valid). */ export function validateSchema(schema: TableSchema, columnOrder: string[] | undefined): string[] { - const errors: string[] = [] + const errors = validateColumnTypeLimits(schema.columns) // Group refs and columnOrder hold stable column ids (not display names). const columnsById = new Map(schema.columns.map((c) => [getColumnId(c), c])) const groups = schema.workflowGroups ?? [] diff --git a/apps/sim/lib/table/service.test.ts b/apps/sim/lib/table/service.test.ts index 3bde50fa497..d3141b6394b 100644 --- a/apps/sim/lib/table/service.test.ts +++ b/apps/sim/lib/table/service.test.ts @@ -12,6 +12,10 @@ import { beforeEach, describe, expect, it, vi } from 'vitest' import type { DbOrTx } from '@/lib/db/types' import type { TableSchema } from '@/lib/table/types' +const { mockAssertTableRowTtlEnabled } = vi.hoisted(() => ({ + mockAssertTableRowTtlEnabled: vi.fn(), +})) + vi.mock('@/lib/realtime/notify', () => ({ notifyWorkspaceTablesChanged: vi.fn().mockResolvedValue(undefined), })) @@ -21,6 +25,10 @@ vi.mock('@/lib/table/billing', () => ({ notifyTableRowUsage: vi.fn(), })) +vi.mock('@/lib/table/ttl-availability', () => ({ + assertTableRowTtlEnabled: mockAssertTableRowTtlEnabled, +})) + import { createTable, getTableById } from '@/lib/table/service' const WORKSPACE_ID = '6fc7631d-88cd-46f8-9f0a-d4764daef7f8' @@ -58,6 +66,16 @@ describe('createTable schema invariants', () => { beforeEach(() => { vi.clearAllMocks() resetDbChainMock() + mockAssertTableRowTtlEnabled.mockResolvedValue(undefined) + }) + + it('rejects a TTL schema before persistence when the feature is disabled', async () => { + mockAssertTableRowTtlEnabled.mockRejectedValue(new Error('Expiration columns are not enabled')) + + await expect( + create({ columns: [{ name: 'expires_at', type: 'ttl' }] } as TableSchema) + ).rejects.toThrow('Expiration columns are not enabled') + expect(dbChainMockFns.insert).not.toHaveBeenCalled() }) /** diff --git a/apps/sim/lib/table/service.ts b/apps/sim/lib/table/service.ts index cc35089085c..36328223e2a 100644 --- a/apps/sim/lib/table/service.ts +++ b/apps/sim/lib/table/service.ts @@ -57,6 +57,7 @@ import { mutateTableRowsWithSecretProvenance, } from '@/lib/table/rows/secret-provenance' import { assertValidSchema } from '@/lib/table/schema-invariants' +import { assertTableRowTtlEnabled } from '@/lib/table/ttl-availability' import { setTableTxTimeouts } from '@/lib/table/tx' import { type CreateTableData, @@ -560,6 +561,10 @@ export async function createTable( ) } + if (data.schema.columns.some((column) => column.type === 'ttl')) { + await assertTableRowTtlEnabled() + } + const tableId = `tbl_${generateId().replace(/-/g, '')}` const now = new Date() @@ -828,6 +833,7 @@ export async function addTableColumnsWithTx( ...table.schema, columns: [...table.schema.columns, ...additions], } + assertValidSchema(updatedSchema, table.metadata?.columnOrder) const now = new Date() await trx diff --git a/apps/sim/lib/table/sql.ts b/apps/sim/lib/table/sql.ts index 4d5c6c9ab04..a189588a134 100644 --- a/apps/sim/lib/table/sql.ts +++ b/apps/sim/lib/table/sql.ts @@ -15,7 +15,9 @@ import { columnTypeOf, filterOperatorsFor, MULTI_SELECT_OPERATORS, + MULTI_SELECT_OPS, SINGLE_SELECT_OPERATORS, + SINGLE_SELECT_OPS, } from '@/lib/table/column-types' import { NAME_PATTERN } from '@/lib/table/constants' import { normalizeDateCellValue } from '@/lib/table/dates' @@ -36,38 +38,12 @@ import type { * Re-exported: the `$`-prefixed wire whitelists now live with the `select` type * definition, but this module is where callers and tests already look for them. */ -export { MULTI_SELECT_OPERATORS, SINGLE_SELECT_OPERATORS } +export { MULTI_SELECT_OPERATORS, MULTI_SELECT_OPS, SINGLE_SELECT_OPERATORS, SINGLE_SELECT_OPS } type ColumnType = ColumnDefinition['type'] type ColumnMap = ReadonlyMap /** -/** - * The same allowlists in the v2 bare-operator grammar, applied inside - * `fieldPredicate` so both wire formats gate identically. Not derived from the - * `$` sets above by string surgery because the mapping is not 1:1 — `$empty` - * splits into `isEmpty`/`isNotEmpty`. `isNull`/`isNotNull` have no `$` - * equivalent and are allowed on both: a strict null check is meaningful on any - * column, select included. - */ -const SINGLE_SELECT_OPS = new Set([ - 'eq', - 'ne', - 'in', - 'nin', - 'isEmpty', - 'isNotEmpty', - 'isNull', - 'isNotNull', -]) -const MULTI_SELECT_OPS = new Set([ - 'contains', - 'ncontains', - 'isEmpty', - 'isNotEmpty', - 'isNull', - 'isNotNull', -]) /** * Returns the Postgres cast needed to compare a JSONB text value of the given diff --git a/apps/sim/lib/table/trigger.ts b/apps/sim/lib/table/trigger.ts index a08ebc093a0..53623598ac0 100644 --- a/apps/sim/lib/table/trigger.ts +++ b/apps/sim/lib/table/trigger.ts @@ -1,7 +1,7 @@ /** * Direct trigger firing for table row events. * - * When rows are inserted or updated in a table, this module looks up any + * When rows are inserted, updated, or deleted in a table, this module looks up any * active webhook triggers watching that table and fires workflow executions * immediately - no polling or cron involved. */ @@ -15,7 +15,8 @@ import { readCanonicalTriggerValue } from '@/lib/webhooks/polling/canonical' const logger = createLogger('TableTrigger') -type EventType = 'insert' | 'update' +type EventType = 'insert' | 'update' | 'delete' +type TableTriggerRow = Pick interface TableTriggerPayload { row: Record | null @@ -44,16 +45,17 @@ interface WebhookConfig { * This is fire-and-forget - errors are logged but never thrown. * Call with `void fireTableTrigger(...)` to avoid blocking the caller. * - * @param eventType - 'insert' for new rows, 'update' for changed rows - * @param oldRows - Map of row ID to previous data. Pass null for inserts. + * @param workspaceId - Canonical workspace that owns the mutated table. + * @param eventType - The committed row mutation that should trigger workflows. + * @param rows - Committed row snapshots; only the ID and data are needed, including for deletes. + * @param oldRows - Map of row ID to previous data. Pass null for inserts and deletes. */ export async function fireTableTrigger( tableId: string, + workspaceId: string, tableName: string, eventType: EventType, - // Accepts a row without its executions sidecar: the payload projects id and - // data only, and the upsert path deliberately does not load one. - rows: Array>, + rows: TableTriggerRow[], oldRows: Map | null, schema: TableSchema, requestId: string @@ -75,6 +77,7 @@ export async function fireTableTrigger( // Filter to webhooks watching this table with a matching event type const matching = webhooks.filter((entry) => { + if (entry.workflow.workspaceId !== workspaceId) return false const config = entry.webhook.providerConfig as WebhookConfig | null // Canonical key `tableId` first; `tableSelector`/`manualTableId` are a transitional // basic-first fallback for configs deployed before the canonical key was written. @@ -103,7 +106,7 @@ export async function fireTableTrigger( const includeHeaders = config?.includeHeaders !== false for (const row of rows) { - const previousIdData = oldRows?.get(row.id) ?? null + const previousIdData = eventType === 'delete' ? row.data : (oldRows?.get(row.id) ?? null) const rawRow = toNamedRow(row.data) const previousRow = previousIdData ? toNamedRow(previousIdData) : null const changedColumns = previousIdData diff --git a/apps/sim/lib/table/ttl-availability.test.ts b/apps/sim/lib/table/ttl-availability.test.ts new file mode 100644 index 00000000000..4710ce7d0b9 --- /dev/null +++ b/apps/sim/lib/table/ttl-availability.test.ts @@ -0,0 +1,34 @@ +/** + * @vitest-environment node + */ +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const { mockIsFeatureEnabled } = vi.hoisted(() => ({ mockIsFeatureEnabled: vi.fn() })) + +vi.mock('@/lib/core/config/feature-flags', () => ({ + isFeatureEnabled: mockIsFeatureEnabled, +})) + +import { assertTableRowTtlEnabled, isTableRowTtlEnabled } from '@/lib/table/ttl-availability' + +describe('table row TTL availability', () => { + beforeEach(() => { + vi.clearAllMocks() + }) + + it('resolves the global table-row-ttl flag without rollout context', async () => { + mockIsFeatureEnabled.mockResolvedValue(true) + + await expect(isTableRowTtlEnabled()).resolves.toBe(true) + expect(mockIsFeatureEnabled).toHaveBeenCalledWith('table-row-ttl') + }) + + it('rejects TTL column creation while the flag is disabled', async () => { + mockIsFeatureEnabled.mockResolvedValue(false) + + await expect(assertTableRowTtlEnabled()).rejects.toMatchObject({ + code: 'validation', + message: 'Expiration columns are not enabled', + }) + }) +}) diff --git a/apps/sim/lib/table/ttl-availability.ts b/apps/sim/lib/table/ttl-availability.ts new file mode 100644 index 00000000000..f5442b35975 --- /dev/null +++ b/apps/sim/lib/table/ttl-availability.ts @@ -0,0 +1,13 @@ +import { isFeatureEnabled } from '@/lib/core/config/feature-flags' +import { OrchestrationError } from '@/lib/core/orchestration/types' + +/** Whether TTL columns and their cleanup behavior are enabled globally. */ +export function isTableRowTtlEnabled(): Promise { + return isFeatureEnabled('table-row-ttl') +} + +/** Rejects attempts to introduce a TTL column while the feature is disabled. */ +export async function assertTableRowTtlEnabled(): Promise { + if (await isTableRowTtlEnabled()) return + throw new OrchestrationError('validation', 'Expiration columns are not enabled') +} diff --git a/apps/sim/lib/table/types.ts b/apps/sim/lib/table/types.ts index 9319e794c10..98747c75ed6 100644 --- a/apps/sim/lib/table/types.ts +++ b/apps/sim/lib/table/types.ts @@ -248,6 +248,14 @@ export interface RowExecutionMetadata { * re-runs whose `cancelledAt > dispatch.requestedAt` — a user cancel * mid-dispatch must not be overridden by `isManualRun`. */ cancelledAt?: string + /** + * Person whose permission group gates this cell's tools, written with the + * dispatcher's `pending` pre-stamp so the worker that eventually drains the + * marker runs it under the subject that requested it rather than its own. + * Persisted on `tableRowExecutions` but NOT hydrated by `loadExecutionsByRow` + * — it is read on demand, only while the marker is still unclaimed. + */ + capabilityGovernedUserId?: string | null /** * Enrichment cascade breakdown for `enrichment`-type groups, written on the * terminal cell write. Persisted on `tableRowExecutions` but NOT hydrated by @@ -470,7 +478,12 @@ export type TableInfo = Pick /** Simplified table summary for LLM enrichment and display contexts. */ export interface TableSummary { name: string - columns: Array> + /** + * `multiple` is carried because a select column's allowed filter operators + * depend on it — LLM enrichment has to name the right subset or the model + * writes a predicate the query layer rejects. + */ + columns: Array> } export interface TableRow { @@ -703,6 +716,26 @@ export interface InsertRowData { * unstamped write. */ secretProvenance: TableRowSecretProvenanceWrite | undefined + /** + * The person whose permission group gates any enrichment this write + * auto-fires; `null` when the write has no acting person (workspace API key, + * schedule, internal state patch). + * + * THE statement of the rule for every table payload that carries this field. + * It is deliberately not the attribution field beside it, which names the + * workspace billed account when the credential names no human and would run + * that bystander's tool denylist against an actorless run. Which principals + * a group governs at all is `capabilityGovernedPrincipalUserId` in + * `@/lib/core/application`; every surface resolves the subject there and + * threads it down rather than re-deriving it. + * + * Required with an explicit `null` rather than optional: the only way to get + * this wrong is to not think about it, and an optional field with a fallback + * let every producer that had not been taught the distinction silently + * inherit the attribution. Making omission a compile error is what stops the + * next producer from re-introducing that bystander substitution. + */ + capabilityGovernedUserId: string | null } export interface BatchInsertData { @@ -717,6 +750,9 @@ export interface BatchInsertData { orderKeys?: string[] /** Encrypted provenance for the values in `rows`, positionally aligned. Required; see {@link InsertRowData.secretProvenance}. */ secretProvenance: Array | undefined + /** The person whose permission group gates any enrichment this write + * auto-fires. Required; see {@link InsertRowData.capabilityGovernedUserId}. */ + capabilityGovernedUserId: string | null } export interface UpsertRowData { @@ -728,6 +764,9 @@ export interface UpsertRowData { conflictTarget?: string /** Encrypted provenance for the values in `data`. Required; see {@link InsertRowData.secretProvenance}. */ secretProvenance: TableRowSecretProvenanceWrite | undefined + /** The person whose permission group gates any enrichment this write + * auto-fires. Required; see {@link InsertRowData.capabilityGovernedUserId}. */ + capabilityGovernedUserId: string | null } export interface UpsertResult { @@ -776,6 +815,9 @@ export interface UpdateRowData { actorUserId?: string | null /** Encrypted provenance for the values in this partial patch. Required; see {@link InsertRowData.secretProvenance}. */ secretProvenance: TableRowSecretProvenanceWrite | undefined + /** The person whose permission group gates any enrichment this write + * auto-fires. Required; see {@link InsertRowData.capabilityGovernedUserId}. */ + capabilityGovernedUserId: string | null } export interface BulkUpdateData { @@ -786,6 +828,9 @@ export interface BulkUpdateData { actorUserId?: string | null /** Encrypted provenance for the values in this partial patch. Required; see {@link InsertRowData.secretProvenance}. */ secretProvenance: TableRowSecretProvenanceWrite | undefined + /** The person whose permission group gates any enrichment this write + * auto-fires. Required; see {@link InsertRowData.capabilityGovernedUserId}. */ + capabilityGovernedUserId: string | null } export interface BatchUpdateByIdData { @@ -800,6 +845,9 @@ export interface BatchUpdateByIdData { actorUserId?: string | null /** Encrypted provenance for the values in all partial patches; omitted by legacy callers. */ secretProvenanceByRowId?: Record + /** The person whose permission group gates any enrichment this write + * auto-fires. Required; see {@link InsertRowData.capabilityGovernedUserId}. */ + capabilityGovernedUserId: string | null } export interface BulkDeleteData { @@ -937,8 +985,14 @@ export interface AddWorkflowGroupData { autoRun?: boolean /** Persist auto-run state without dispatching through the primitive. */ suppressAutoRunDispatch?: boolean - /** The member adding the group — billed/gated for the auto-run enrichment pass. */ + /** The member adding the group — billed for the auto-run enrichment pass. */ actorUserId?: string | null + /** The person whose permission group gates the auto-run pass this write can + * start; `null` when the write has no acting person (workspace key, system). + * Required with an explicit `null` — deliberately not `actorUserId`, which + * is an attribution and names the workspace billed account when the + * credential names no human. */ + capabilityGovernedUserId: string | null } /** Payload for `updateWorkflowGroup` — diffs outputs and writes columns. */ @@ -976,8 +1030,11 @@ export interface UpdateWorkflowGroupData { autoRun?: boolean /** Skip primitive dispatch when an authorized caller will start the run itself. */ suppressAutoRunDispatch?: boolean - /** The member updating the group — billed/gated for any triggered re-run. */ + /** The member updating the group — billed for any triggered re-run. */ actorUserId?: string | null + /** The person whose permission group gates the auto-run pass this write can + * start. Required; see {@link InsertRowData.capabilityGovernedUserId}. */ + capabilityGovernedUserId: string | null } export interface DeleteWorkflowGroupData { diff --git a/apps/sim/lib/table/validation.ts b/apps/sim/lib/table/validation.ts index aa9a918beb8..e5fd89c3d2d 100644 --- a/apps/sim/lib/table/validation.ts +++ b/apps/sim/lib/table/validation.ts @@ -14,6 +14,7 @@ import { columnTypeOf, isColumnType, TYPE_SPECIFIC_COLUMN_KEYS, + validateColumnTypeLimits, validateTypeMetadata, } from '@/lib/table/column-types' import { @@ -244,6 +245,8 @@ export function validateTableSchema(schema: TableSchema): ValidationResult { errors.push('Duplicate column names found') } + errors.push(...validateColumnTypeLimits(schema.columns)) + return { valid: errors.length === 0, errors } } diff --git a/apps/sim/lib/table/workflow-columns.ts b/apps/sim/lib/table/workflow-columns.ts index 76d9bbafa50..7fd9c6a640f 100644 --- a/apps/sim/lib/table/workflow-columns.ts +++ b/apps/sim/lib/table/workflow-columns.ts @@ -192,6 +192,11 @@ export interface ScheduleOpts { groupIds?: string[] isManualRun?: boolean mode?: DispatchMode + /** Person whose permission group gates every cell this batch emits, or `null` + * for an actorless run. Required so a new call site cannot emit a payload + * with no gate by simply not thinking about one; see + * {@link InsertRowData.capabilityGovernedUserId} in `@/lib/table/types`. */ + capabilityGovernedUserId: string | null } /** Pure eligibility filter + payload building. Shared by the auto-fire path @@ -199,15 +204,15 @@ export interface ScheduleOpts { export function buildPendingRuns( table: TableDefinition, rows: TableRow[], - opts?: ScheduleOpts + opts: ScheduleOpts ): WorkflowGroupCellPayload[] { const allGroups = table.schema.workflowGroups ?? [] if (allGroups.length === 0) return [] if (rows.length === 0) return [] - const groupIdFilter = opts?.groupIds + const groupIdFilter = opts.groupIds ? new Set(opts.groupIds) - : opts?.groupId + : opts.groupId ? new Set([opts.groupId]) : null const groups = groupIdFilter ? allGroups.filter((g) => groupIdFilter.has(g.id)) : allGroups @@ -221,8 +226,8 @@ export function buildPendingRuns( for (const row of orderedRows) { for (const group of groups) { const reason = classifyEligibility(group, row, { - isManualRun: opts?.isManualRun, - mode: opts?.mode, + isManualRun: opts.isManualRun, + mode: opts.mode, }) reasonCounts[reason] = (reasonCounts[reason] ?? 0) + 1 if (reason !== 'eligible' && reason !== 'manual-bypass') continue @@ -235,6 +240,7 @@ export function buildPendingRuns( ...(group.enrichmentId ? { enrichmentId: group.enrichmentId } : {}), workspaceId: table.workspaceId, executionId: generateId(), + capabilityGovernedUserId: opts.capabilityGovernedUserId, }) } } @@ -449,6 +455,13 @@ export interface WorkflowGroupCellPayload { * auto-fire (row writes, CSV import) → billing falls back to the workspace * billed account. */ triggeredByUserId?: string + /** Person whose permission group gates this cell's tools. Null/absent means + * no acting person, so no per-tool gate applies. Not `triggeredByUserId`; + * see {@link InsertRowData.capabilityGovernedUserId} in `@/lib/table/types`. + * Required like every sibling in `@/lib/table/types`: an omitted key and a + * deliberate `null` both read as "ungated", so the compiler is what makes a + * caller state which one it means. */ + capabilityGovernedUserId: string | null } export type QueuedWorkflowGroupCellPayload = Omit< @@ -730,6 +743,8 @@ export async function cancelWorkflowGroupRuns( secretProvenance: undefined, workspaceId: table.workspaceId, executionsPatch: mutation.executionsPatch, + /** A cancellation stamp writes no cell values and fires no enrichment. */ + capabilityGovernedUserId: null, }, table, `wfgrp-cancel-${mutation.rowId}` @@ -865,6 +880,10 @@ export async function runWorkflowColumn(opts: { * callers (row writes, CSV import) → falls back to the workspace billed * account at billing time. */ triggeredByUserId?: string | null + /** Person whose permission group gates the run's cells; `null` when the run + * has no acting person (workspace key, schedule, auto-fire). Required, and + * never defaulted from `triggeredByUserId`; see {@link InsertRowData.capabilityGovernedUserId} in `@/lib/table/types`. */ + capabilityGovernedUserId: string | null }): Promise<{ dispatchId: string | null; shouldSignalRowsChanged: boolean }> { const { tableId, @@ -877,6 +896,7 @@ export async function runWorkflowColumn(opts: { excludeRowIds, limit, triggeredByUserId, + capabilityGovernedUserId, } = opts const isManualRun = opts.isManualRun ?? true // Empty `rowIds` array means "scope explicitly empty" — auto-fire callers @@ -945,6 +965,7 @@ export async function runWorkflowColumn(opts: { limit, isManualRun, triggeredByUserId, + capabilityGovernedUserId, }) try { @@ -1083,10 +1104,24 @@ export interface CellResumeContext { groupId: string workspaceId: string workflowId: string + /** + * Person whose permission group gates the tools of everything this cell's + * run still has to do. Required, because a pause is the one boundary where + * the subject would otherwise be reconstructed from scratch: the resumed + * cascade is driven by the resume worker, whose payload carries no dispatch + * and no row marker to re-read it from. `null` is the actorless run — no + * per-tool gate — and has to be written, not inferred from an absent key. + * + * Lives in `paused_executions.metadata`, a jsonb document, so carrying it + * needs no schema change: a pause row written before this field existed + * reads back `undefined`, which the resume worker normalizes to `null`. + */ + capabilityGovernedUserId: string | null } interface PausedMetadataPatch { - cellContext?: CellResumeContext + /** Read back from jsonb, so a pause written before a field existed lacks it. */ + cellContext?: Partial & Omit [key: string]: unknown } @@ -1132,7 +1167,13 @@ export async function findCellContextByExecutionId( .where(eq(pausedExecutions.executionId, executionId)) .limit(1) const meta = row?.metadata as PausedMetadataPatch | null - return meta?.cellContext ?? null + const stored = meta?.cellContext + if (!stored) return null + return { + ...stored, + /** A pause stashed before the subject was carried is an ungated resume. */ + capabilityGovernedUserId: stored.capabilityGovernedUserId ?? null, + } } catch (err) { logger.error(`Failed to read cell context for executionId=${executionId}:`, err) return null diff --git a/apps/sim/lib/table/workflow-groups/service.test.ts b/apps/sim/lib/table/workflow-groups/service.test.ts index 5d8f164409f..d36ec47276d 100644 --- a/apps/sim/lib/table/workflow-groups/service.test.ts +++ b/apps/sim/lib/table/workflow-groups/service.test.ts @@ -4,9 +4,10 @@ import { beforeEach, describe, expect, it, vi } from 'vitest' import type { TableDefinition, WorkflowGroup } from '@/lib/table/types' -const { mockWithLockedTable, mockGetTableById } = vi.hoisted(() => ({ +const { mockWithLockedTable, mockGetTableById, mockAssertTableRowTtlEnabled } = vi.hoisted(() => ({ mockWithLockedTable: vi.fn(), mockGetTableById: vi.fn(), + mockAssertTableRowTtlEnabled: vi.fn(), })) vi.mock('@/lib/table/service', () => ({ @@ -20,6 +21,9 @@ vi.mock('@/lib/table/mutation-locks', () => ({ vi.mock('@/lib/table/rows/secret-provenance', () => ({ updateTableRowsWithDerivedSecretProvenance: vi.fn(), })) +vi.mock('@/lib/table/ttl-availability', () => ({ + assertTableRowTtlEnabled: mockAssertTableRowTtlEnabled, +})) vi.mock('@/lib/table/workflow-columns', () => ({ runWorkflowColumn: vi.fn().mockResolvedValue(undefined), stripGroupDeps: (schema: unknown) => schema, @@ -34,7 +38,11 @@ vi.mock('@/lib/table/schema-invariants', () => ({ })) import { TABLE_LIMITS } from '@/lib/table/constants' -import { addWorkflowGroup } from '@/lib/table/workflow-groups/service' +import { + addWorkflowGroup, + addWorkflowGroupOutput, + updateWorkflowGroup, +} from '@/lib/table/workflow-groups/service' function groupAt(index: number): WorkflowGroup { return { @@ -73,6 +81,7 @@ function tableWithGroups(count: number): TableDefinition { describe('addWorkflowGroup group ceiling', () => { beforeEach(() => { vi.clearAllMocks() + mockAssertTableRowTtlEnabled.mockResolvedValue(undefined) }) function add(existingGroups: number) { @@ -108,3 +117,58 @@ describe('addWorkflowGroup group ceiling', () => { await expect(add(TABLE_LIMITS.MAX_WORKFLOW_GROUPS_PER_TABLE - 1)).resolves.toBeDefined() }) }) + +describe('workflow group TTL availability', () => { + beforeEach(() => { + vi.clearAllMocks() + mockAssertTableRowTtlEnabled.mockRejectedValue(new Error('Expiration columns are not enabled')) + }) + + it.each([ + [ + 'group creation', + () => + addWorkflowGroup( + { + tableId: 'table-1', + workspaceId: 'workspace-1', + group: groupAt(1), + outputColumns: [{ name: 'expires_at', type: 'ttl' }], + } as Parameters[0], + 'request-1' + ), + ], + [ + 'group update', + () => + updateWorkflowGroup( + { + tableId: 'table-1', + workspaceId: 'workspace-1', + groupId: 'group-1', + newOutputColumns: [{ name: 'expires_at', type: 'ttl' }], + } as Parameters[0], + 'request-1' + ), + ], + [ + 'single output addition', + () => + addWorkflowGroupOutput( + { + tableId: 'table-1', + workspaceId: 'workspace-1', + groupId: 'group-1', + blockId: 'block-1', + path: 'expiresAt', + capabilityGovernedUserId: null, + resolvedOutput: { workflowId: 'workflow-1', columnType: 'ttl', order: [] }, + }, + 'request-1' + ), + ], + ])('rejects TTL introduction through %s while disabled', async (_label, introduceTtl) => { + await expect(introduceTtl()).rejects.toThrow('Expiration columns are not enabled') + expect(mockWithLockedTable).not.toHaveBeenCalled() + }) +}) diff --git a/apps/sim/lib/table/workflow-groups/service.ts b/apps/sim/lib/table/workflow-groups/service.ts index 7f526593b5a..0796cb81b91 100644 --- a/apps/sim/lib/table/workflow-groups/service.ts +++ b/apps/sim/lib/table/workflow-groups/service.ts @@ -26,6 +26,7 @@ import { stripGroupExecutions } from '@/lib/table/rows/executions' import { updateTableRowsWithDerivedSecretProvenance } from '@/lib/table/rows/secret-provenance' import { assertValidSchema } from '@/lib/table/schema-invariants' import { getTableById, withLockedTable } from '@/lib/table/service' +import { assertTableRowTtlEnabled } from '@/lib/table/ttl-availability' import { setTableTxTimeouts } from '@/lib/table/tx' import type { AddWorkflowGroupData, @@ -132,6 +133,10 @@ export async function addWorkflowGroup( data: AddWorkflowGroupData, requestId: string ): Promise { + if (data.outputColumns.some((column) => column.type === 'ttl')) { + await assertTableRowTtlEnabled() + } + const updatedTable = await withLockedTable( data.tableId, async (table, trx) => { @@ -241,6 +246,7 @@ export async function addWorkflowGroup( groupIds: [data.group.id], requestId, triggeredByUserId: data.actorUserId, + capabilityGovernedUserId: data.capabilityGovernedUserId, }).catch((err) => logger.error(`[${requestId}] auto-dispatch (addWorkflowGroup) failed:`, err)) } @@ -258,6 +264,10 @@ export async function updateWorkflowGroup( requestId: string ): Promise { const mappingUpdates = data.mappingUpdates ?? [] + const introducesTtl = + data.newOutputColumns?.some((column) => column.type === 'ttl') === true || + data.resolvedMappingTypes?.columns.some((column) => column.type === 'ttl') === true + if (introducesTtl) await assertTableRowTtlEnabled() // Phase 1 (no lock): consume the output types resolved and authorized by the // application command. Resolution stays outside the advisory-lock critical @@ -561,6 +571,7 @@ export async function updateWorkflowGroup( overwrite: false, requestId, actorUserId: data.actorUserId, + capabilityGovernedUserId: data.capabilityGovernedUserId, }) } catch (err) { logger.warn( @@ -579,6 +590,7 @@ export async function updateWorkflowGroup( overwrite: true, requestId, actorUserId: data.actorUserId, + capabilityGovernedUserId: data.capabilityGovernedUserId, }) } catch (err) { logger.warn( @@ -600,6 +612,7 @@ export async function updateWorkflowGroup( groupIds: [data.groupId], requestId, triggeredByUserId: data.actorUserId, + capabilityGovernedUserId: data.capabilityGovernedUserId, }).catch((err) => logger.error(`[${requestId}] auto-dispatch (updateWorkflowGroup autoRun=true) failed:`, err) ) @@ -625,8 +638,13 @@ export async function addWorkflowGroupOutput( path: string /** Optional override; defaults to a slug derived from `path`. */ columnName?: string - /** The member adding the output — billed/gated for any backfill-triggered re-run. */ + /** The member adding the output — the billing attribution for the backfill's + * row writes. Not the gate: see `capabilityGovernedUserId`. */ actorUserId?: string | null + /** Person whose permission group gates any cell the backfill's writes + * cascade into; `null` when the change has no acting person. Required; see + * {@link InsertRowData.capabilityGovernedUserId} in `@/lib/table/types`. */ + capabilityGovernedUserId: string | null resolvedOutput: { workflowId: string columnType: ColumnDefinition['type'] @@ -640,6 +658,8 @@ export async function addWorkflowGroupOutput( }, requestId: string ): Promise { + if (data.resolvedOutput.columnType === 'ttl') await assertTableRowTtlEnabled() + // Phase 1 (no lock): validate the authorized workflow metadata against the // group's current workflow. Phase 2 re-validates the same binding under the // table lock before applying the mutation. @@ -741,6 +761,15 @@ export async function addWorkflowGroupOutput( const [db, ib] = orderKey(b) return da !== db ? da - db : ia - ib }) + const invalidOutput = allGroupOutputs.find( + (output) => !resolvedOrder.has(`${output.blockId}::${output.path}`) + ) + if (invalidOutput) { + throw new OrchestrationError( + 'conflict', + `Workflow group "${data.groupId}" mappings changed concurrently; retry the add.` + ) + } const orderedGroupColIds = allGroupOutputs.map((o) => o.columnName) const updatedGroup: WorkflowGroup = { ...group, @@ -847,6 +876,7 @@ export async function addWorkflowGroupOutput( overwrite: false, requestId, actorUserId: data.actorUserId, + capabilityGovernedUserId: data.capabilityGovernedUserId, }) } catch (err) { logger.warn( diff --git a/apps/sim/lib/tool-execution/application/execute-tool.test.ts b/apps/sim/lib/tool-execution/application/execute-tool.test.ts new file mode 100644 index 00000000000..e12f018fdcf --- /dev/null +++ b/apps/sim/lib/tool-execution/application/execute-tool.test.ts @@ -0,0 +1,673 @@ +/** + * @vitest-environment node + */ +import type { PersonalApiKeyPrincipal, WorkspaceApiKeyPrincipal } from '@sim/auth/principal' +import { resetEnvFlagsMock, setEnvFlags } from '@sim/testing/mocks' +import { afterAll, beforeEach, describe, expect, it, vi } from 'vitest' + +const mocks = vi.hoisted(() => ({ + loadWorkspace: vi.fn(), + resolvePermission: vi.fn(), + allowedIntegrationTypes: vi.fn(), + getBlockVisibility: vi.fn(), + listCustomBlocks: vi.fn(), + isDeploymentAvailable: vi.fn(), + recordAudit: vi.fn(), + getAllBlocks: vi.fn(), + executeRegistryTool: vi.fn(), + resolveBillingAttribution: vi.fn(), + recordUsage: vi.fn(), +})) + +vi.mock('@/lib/workspaces/application/workspace-context', () => ({ + loadActiveWorkspaceApplicationContext: mocks.loadWorkspace, +})) + +vi.mock('@sim/platform-authz/workspace', () => ({ + permissionSatisfies: (permission: string | null, required: string) => + permission === 'admin' || permission === 'write' || permission === required, + resolveEffectiveWorkspacePermission: mocks.resolvePermission, +})) + +vi.mock('@sim/audit', () => ({ + recordAudit: mocks.recordAudit, + AuditAction: {}, + AuditResourceType: {}, +})) + +vi.mock('@/lib/integrations/principal-scope.server', () => ({ + allowedIntegrationTypes: mocks.allowedIntegrationTypes, + principalUserId: (principal: { kind: string; userId?: string }) => + principal.kind === 'session' || principal.kind === 'personal_api_key' + ? principal.userId + : undefined, +})) + +vi.mock('@/lib/core/config/block-visibility', () => ({ + getBlockVisibility: mocks.getBlockVisibility, +})) + +vi.mock('@/lib/workflows/custom-blocks/operations', () => ({ + listCustomBlocksWithInputsForWorkspace: mocks.listCustomBlocks, +})) + +vi.mock('@/lib/integrations/availability.server', () => ({ + isIntegrationDeploymentAvailableForVisibility: mocks.isDeploymentAvailable, +})) + +vi.mock('@/blocks/custom/server-overlay', () => ({ + withCustomBlockOverlay: (_rows: unknown, run: () => Promise) => run(), +})) + +vi.mock('@/blocks/visibility/server-context', () => ({ + withBlockVisibility: (_state: unknown, run: () => Promise) => run(), +})) + +vi.mock('@/blocks/registry', () => ({ + getAllBlocks: mocks.getAllBlocks, + getBlock: vi.fn(), + getLatestBlockForViewer: vi.fn(), + getBlockMeta: vi.fn(() => ({ tags: [] })), +})) + +vi.mock('@/tools/utils', () => ({ + getTool: (toolId: string) => + Object.hasOwn(TOOL_METADATA, toolId) ? TOOL_METADATA[toolId] : undefined, +})) + +vi.mock('@/tools/tool-ids', () => ({ + getToolIds: () => Object.freeze(Object.keys(TOOL_METADATA)), + resolveToolId: (toolId: string) => toolId, +})) + +vi.mock('@/tools', () => ({ executeTool: mocks.executeRegistryTool })) + +vi.mock('@/lib/billing/core/billing-attribution', () => ({ + resolveBillingAttribution: mocks.resolveBillingAttribution, + toBillingContext: () => ({ + billingEntity: { type: 'workspace', id: WORKSPACE_ID }, + billingPeriod: { start: new Date('2026-01-01'), end: new Date('2026-02-01') }, + }), +})) + +vi.mock('@/lib/billing/core/usage-log', () => ({ recordUsage: mocks.recordUsage })) + +import { executeToolForCaller } from '@/lib/tool-execution/application/execute-tool' +import type { BlockConfig } from '@/blocks/types' + +const TOOL_METADATA: Record> = { + slack_message: { + id: 'slack_message', + name: 'Slack Send Message', + params: { + accessToken: { type: 'string', required: true, visibility: 'hidden' }, + text: { type: 'string', required: true, visibility: 'user-or-llm' }, + }, + oauth: { required: true, provider: 'slack' }, + }, + firecrawl_scrape: { + id: 'firecrawl_scrape', + name: 'Firecrawl Scrape', + params: { + url: { type: 'string', required: true, visibility: 'user-or-llm' }, + apiKey: { type: 'string', required: true, visibility: 'user-only' }, + }, + hosting: { apiKeyParam: 'apiKey' }, + }, + snowflake_execute_sql: { + id: 'snowflake_execute_sql', + name: 'Snowflake Execute SQL', + params: { + oauthCredential: { type: 'string', required: true, visibility: 'user-only' }, + statement: { type: 'string', required: true, visibility: 'user-or-llm' }, + }, + }, + thinking_tool: { + id: 'thinking_tool', + name: 'Thinking', + params: { thought: { type: 'string', required: true, visibility: 'llm-only' } }, + }, + zendesk_get_ticket: { + id: 'zendesk_get_ticket', + name: 'Zendesk Get Ticket', + params: { + subdomain: { type: 'string', required: true, visibility: 'user-only' }, + apiToken: { type: 'string', required: true, visibility: 'user-only' }, + ticketId: { type: 'string', required: true, visibility: 'user-or-llm' }, + }, + }, + preview_call: { id: 'preview_call', name: 'Preview Call', params: {} }, + confluence_read_v2: { id: 'confluence_read_v2', name: 'Confluence Read', params: {} }, +} + +const WORKSPACE_ID = 'workspace-1' + +const workspaceContext = { + workspaceId: WORKSPACE_ID, + workspaceOrganizationId: 'org-1', + allowPersonalApiKeys: true, + billedAccountUserId: 'billing-owner-1', +} + +const principal: PersonalApiKeyPrincipal = { + kind: 'personal_api_key', + userId: 'user-1', + keyId: 'key-1', +} +const workspaceKey: WorkspaceApiKeyPrincipal = { + kind: 'workspace_api_key', + workspaceId: WORKSPACE_ID, + keyId: 'key-1', +} + +function block(overrides: Partial & { type: string }): BlockConfig { + return { + name: overrides.type, + description: `${overrides.type} block`, + category: 'tools', + bgColor: '#000000', + icon: (() => null) as unknown as BlockConfig['icon'], + subBlocks: [], + tools: { access: [] }, + inputs: {}, + outputs: {}, + ...overrides, + } as BlockConfig +} + +const slackBlock = block({ type: 'slack', tools: { access: ['slack_message'] } }) +const firecrawlBlock = block({ type: 'firecrawl', tools: { access: ['firecrawl_scrape'] } }) +const previewBlock = block({ + type: 'preview_thing', + preview: true, + tools: { access: ['preview_call'] }, +}) +const zendeskBlock = block({ type: 'zendesk', tools: { access: ['zendesk_get_ticket'] } }) +const thinkingBlock = block({ type: 'thinking', tools: { access: ['thinking_tool'] } }) +const snowflakeBlock = block({ type: 'snowflake', tools: { access: ['snowflake_execute_sql'] } }) +const confluenceBlock = block({ + type: 'confluence_v2', + tools: { access: ['confluence_read_v2'] }, +}) + +function run(input: Partial[0]['input']> = {}) { + return executeToolForCaller.execute({ + principal, + input: { + workspaceId: WORKSPACE_ID, + toolId: 'firecrawl_scrape', + input: { url: 'https://example.com' }, + ...input, + }, + }) +} + +describe('executeToolForCaller', () => { + afterAll(resetEnvFlagsMock) + + beforeEach(() => { + vi.clearAllMocks() + // Hosted-key injection only happens where Sim hosts keys. + setEnvFlags({ isHosted: true }) + mocks.loadWorkspace.mockResolvedValue(workspaceContext) + mocks.resolvePermission.mockResolvedValue('write') + mocks.allowedIntegrationTypes.mockResolvedValue(null) + mocks.getBlockVisibility.mockResolvedValue({ revealed: new Set(), disabled: new Set() }) + mocks.listCustomBlocks.mockResolvedValue([]) + mocks.isDeploymentAvailable.mockReturnValue(true) + mocks.getAllBlocks.mockReturnValue([ + slackBlock, + firecrawlBlock, + previewBlock, + confluenceBlock, + zendeskBlock, + thinkingBlock, + snowflakeBlock, + ]) + mocks.executeRegistryTool.mockResolvedValue({ success: true, output: { markdown: '# Hi' } }) + mocks.resolveBillingAttribution.mockResolvedValue({ workspaceId: WORKSPACE_ID }) + }) + + it('runs a visible, permitted tool and reports what it produced', async () => { + await expect(run({ input: { url: 'https://example.com' } })).resolves.toEqual({ + toolId: 'firecrawl_scrape', + status: 'succeeded', + output: { markdown: '# Hi' }, + error: null, + }) + }) + + it('acts as the authenticated caller and enforces credential access', async () => { + await run({ input: { url: 'https://example.com' } }) + + const [, params] = mocks.executeRegistryTool.mock.calls[0] + expect(params._context).toMatchObject({ + userId: 'user-1', + workspaceId: WORKSPACE_ID, + enforceCredentialAccess: true, + }) + }) + + /** + * The bare-name form Copilot also accepts would read an identifier-shaped + * literal secret as a variable lookup. A caller that types the value gets the + * explicit form only. + */ + it('resolves only explicit environment-variable references', async () => { + await run({ input: { url: 'https://example.com' } }) + + const [, params] = mocks.executeRegistryTool.mock.calls[0] + expect(params._context.envReferenceMode).toBe('explicit') + }) + + it('conceals a tool no visible block exposes as absent', async () => { + await expect(run({ toolId: 'preview_call' })).rejects.toMatchObject({ + code: 'not_found', + message: 'Tool not found', + }) + expect(mocks.executeRegistryTool).not.toHaveBeenCalled() + }) + + it('conceals a tool that is in no block at all', async () => { + await expect(run({ toolId: 'not_a_tool' })).rejects.toMatchObject({ code: 'not_found' }) + }) + + /** + * A denied integration is a decision an admin made and can reverse, and the + * built-in catalog is public — so it is named rather than concealed, unlike + * the unrevealed preview above. + */ + it('refuses an integration the workspace does not permit, naming the cause', async () => { + mocks.allowedIntegrationTypes.mockResolvedValue(new Set(['slack'])) + + await expect(run({ toolId: 'firecrawl_scrape' })).rejects.toMatchObject({ + code: 'forbidden', + detailCode: 'INTEGRATION_NOT_ALLOWED', + }) + expect(mocks.executeRegistryTool).not.toHaveBeenCalled() + }) + + it('still runs a permitted integration when an allowlist is set', async () => { + mocks.allowedIntegrationTypes.mockResolvedValue(new Set(['firecrawl'])) + + await expect(run({ toolId: 'firecrawl_scrape' })).resolves.toMatchObject({ + status: 'succeeded', + }) + }) + + it('resolves an unversioned name to the newest visible version', async () => { + await expect(run({ toolId: 'confluence_read', input: {} })).resolves.toMatchObject({ + toolId: 'confluence_read_v2', + }) + }) + + /** + * The workflow path validates `user-only` parameters during serialization. + * This path has no serialization step, so without an explicit check a missing + * credential reached the provider as `undefined`. + */ + it('refuses a missing required user-only input, naming every one of them', async () => { + await expect( + run({ toolId: 'zendesk_get_ticket', input: { ticketId: '42' } }) + ).rejects.toMatchObject({ + code: 'validation', + message: expect.stringContaining('input.subdomain'), + }) + expect(mocks.executeRegistryTool).not.toHaveBeenCalled() + }) + + it('names the missing inputs together rather than one per round trip', async () => { + await expect( + run({ toolId: 'zendesk_get_ticket', input: { ticketId: '42' } }) + ).rejects.toMatchObject({ message: expect.stringContaining('input.apiToken') }) + }) + + it('treats a blank string as missing, the way the merge validator does', async () => { + await expect( + run({ toolId: 'zendesk_get_ticket', input: { ticketId: '4', subdomain: '', apiToken: 't' } }) + ).rejects.toMatchObject({ code: 'validation' }) + }) + + it('runs once every required user-only input is supplied', async () => { + await expect( + run({ + toolId: 'zendesk_get_ticket', + input: { ticketId: '42', subdomain: 'acme', apiToken: 'tok' }, + }) + ).resolves.toMatchObject({ status: 'succeeded' }) + }) + + /** + * `firecrawl_scrape` declares `apiKey` required and `user-only`, and Sim + * supplies it. Rejecting the omission would make every hosted-key tool + * uncallable without a key the caller does not need to have. + */ + it('does not require a key the deployment hosts', async () => { + await expect(run({ input: { url: 'https://example.com' } })).resolves.toMatchObject({ + status: 'succeeded', + }) + }) + + /** + * Self-hosted supplies no hosted keys — `injectHostedKeyIfNeeded` short-circuits + * on `isHosted` — so the exemption must lift with it, or the caller is told a + * key is optional and the provider disagrees. + */ + it('does require that key on a deployment that hosts none', async () => { + setEnvFlags({ isHosted: false }) + + await expect(run({ input: { url: 'https://example.com' } })).rejects.toMatchObject({ + code: 'validation', + message: expect.stringContaining('input.apiKey'), + }) + }) + + /** + * `visibility` describes editor roles, and a direct call has no editor: the + * caller is the only source, so an `llm-only` parameter is as much theirs to + * send as a `user-only` one. Gating the check on `user-only` alone left + * `thinking_tool.thought` dispatching as `undefined`. + */ + it('refuses a missing llm-only input too — the caller is the only source here', async () => { + await expect(run({ toolId: 'thinking_tool', input: {} })).rejects.toMatchObject({ + code: 'validation', + message: expect.stringContaining('input.thought'), + }) + expect(mocks.executeRegistryTool).not.toHaveBeenCalled() + }) + + it('refuses a missing user-or-llm input before dispatch rather than mid-execution', async () => { + await expect(run({ input: {} })).rejects.toMatchObject({ + code: 'validation', + message: expect.stringContaining('input.url'), + }) + }) + + it('accepts a {{VAR}} reference as a present value, leaving resolution to the executor', async () => { + await run({ + toolId: 'zendesk_get_ticket', + input: { ticketId: '4', subdomain: 'acme', apiToken: '{{ZENDESK_TOKEN}}' }, + }) + + const [, params] = mocks.executeRegistryTool.mock.calls[0] + expect(params.apiToken).toBe('{{ZENDESK_TOKEN}}') + }) + + it('requires a credential for an OAuth tool before it dispatches', async () => { + await expect(run({ toolId: 'slack_message', input: { text: 'hi' } })).rejects.toMatchObject({ + code: 'validation', + message: expect.stringContaining('credentialId is required'), + }) + expect(mocks.executeRegistryTool).not.toHaveBeenCalled() + }) + + /** + * Sixty-eight tools declare the selector as a required `user-only` parameter + * (`oauthCredential` or `credential`) with no `oauth` block — Snowflake among + * them. Validating required inputs against the raw body rejected a valid + * top-level `credentialId` as a missing `oauthCredential`. + */ + it('satisfies a declared credential selector with the top-level credentialId', async () => { + await expect( + run({ + toolId: 'snowflake_execute_sql', + credentialId: 'cred-sf', + input: { statement: 'select 1' }, + }) + ).resolves.toMatchObject({ status: 'succeeded' }) + + const [, params] = mocks.executeRegistryTool.mock.calls[0] + expect(params.oauthCredential).toBe('cred-sf') + expect(params.credential).toBeUndefined() + }) + + it('demands credentialId for a declared required selector even without an oauth block', async () => { + await expect( + run({ toolId: 'snowflake_execute_sql', input: { statement: 'select 1' } }) + ).rejects.toMatchObject({ + code: 'validation', + message: expect.stringContaining('credentialId is required'), + }) + expect(mocks.executeRegistryTool).not.toHaveBeenCalled() + }) + + /** + * The alias check has to run before the declared-key check, or a tool that + * declares `oauthCredential` lets a caller bypass the top-level field and + * credential precedence starts differing per tool. + */ + it('refuses input.oauthCredential even where the tool declares it', async () => { + await expect( + run({ + toolId: 'snowflake_execute_sql', + input: { statement: 'select 1', oauthCredential: 'cred-sf' }, + }) + ).rejects.toMatchObject({ + code: 'validation', + message: expect.stringContaining('top-level credentialId'), + }) + expect(mocks.executeRegistryTool).not.toHaveBeenCalled() + }) + + it('passes the named credential through as the tool credential', async () => { + await run({ toolId: 'slack_message', input: { text: 'hi' }, credentialId: 'cred-1' }) + + const [, params] = mocks.executeRegistryTool.mock.calls[0] + expect(params.credential).toBe('cred-1') + }) + + it('refuses a reserved argument rather than dropping it', async () => { + await expect( + run({ input: { url: 'https://a.co', _context: { userId: 'someone-else' } } }) + ).rejects.toMatchObject({ + code: 'validation', + message: expect.stringContaining('_context'), + }) + expect(mocks.executeRegistryTool).not.toHaveBeenCalled() + }) + + it('refuses a hosted-key flag smuggled in as an argument', async () => { + await expect( + run({ input: { url: 'https://a.co', __usingHostedKey: true } }) + ).rejects.toMatchObject({ + code: 'validation', + }) + }) + + /** + * The executor reads this straight out of params and forwards it to + * credential-token resolution as an impersonation request. No tool declares + * it, which is why the check is a declared-parameter allowlist rather than a + * list of names someone remembered. + */ + it('refuses an undeclared impersonation field', async () => { + await expect( + run({ input: { url: 'https://a.co', impersonateUserEmail: 'someone@example.com' } }) + ).rejects.toMatchObject({ + code: 'validation', + message: expect.stringContaining('impersonateUserEmail'), + }) + expect(mocks.executeRegistryTool).not.toHaveBeenCalled() + }) + + /** + * Declared is not accepted. `accessToken` is in the tool's params, but as + * `hidden` — the resolved credential fills it. Letting a caller send it either + * pre-empts the executor's value or is silently overwritten; either way the + * published schema (which omits hidden params) made no such promise. + */ + it('refuses a declared-but-hidden input, saying whose it is', async () => { + await expect( + run({ + toolId: 'slack_message', + credentialId: 'cred-1', + input: { text: 'hi', accessToken: 'xoxb-forged' }, + }) + ).rejects.toMatchObject({ + code: 'validation', + message: expect.stringContaining('input.accessToken is supplied by Sim'), + }) + expect(mocks.executeRegistryTool).not.toHaveBeenCalled() + }) + + it('refuses any other undeclared input, naming it', async () => { + await expect(run({ input: { url: 'https://a.co', nope: 1 } })).rejects.toMatchObject({ + code: 'validation', + message: expect.stringContaining('input.nope'), + }) + }) + + it('refuses a credential named inline instead of at the top level', async () => { + await expect( + run({ input: { url: 'https://a.co', credential: 'cred-1' } }) + ).rejects.toMatchObject({ + code: 'validation', + message: expect.stringContaining('credentialId'), + }) + }) + + /** + * The ledger de-duplicates on `eventKey`, and the derived key is a hash of + * actor, workspace, source and description — identical for every call to the + * same tool. Without a per-call id, `onConflictDoNothing` billed the first + * hosted-key call and silently dropped every one after it. + */ + it('gives each call its own ledger event so repeat calls all bill', async () => { + mocks.executeRegistryTool.mockResolvedValue({ + success: true, + output: { cost: { total: 0.004 } }, + }) + + await run() + await run() + + const keys = mocks.recordUsage.mock.calls.map((call) => call[0].entries[0].eventKey) + expect(keys).toHaveLength(2) + expect(keys[0]).toBeTruthy() + expect(keys[0]).not.toBe(keys[1]) + }) + + it('refuses a workspace API key: the call runs under a person or not at all', async () => { + await expect( + executeToolForCaller.execute({ + principal: workspaceKey, + input: { workspaceId: WORKSPACE_ID, toolId: 'firecrawl_scrape', input: {} }, + }) + ).rejects.toMatchObject({ code: 'forbidden' }) + }) + + it('reports a tool that ran and refused without failing the call', async () => { + mocks.executeRegistryTool.mockResolvedValue({ + success: false, + output: {}, + error: 'Firecrawl returned 402', + }) + + await expect(run()).resolves.toMatchObject({ + status: 'failed', + error: { message: 'Firecrawl returned 402' }, + }) + }) + + it('bills hosted-key spend to the workspace', async () => { + mocks.executeRegistryTool.mockResolvedValue({ + success: true, + output: { markdown: '# Hi', cost: { total: 0.004 } }, + }) + + await run() + + expect(mocks.recordUsage).toHaveBeenCalledWith( + expect.objectContaining({ + userId: 'user-1', + workspaceId: WORKSPACE_ID, + entries: [expect.objectContaining({ category: 'tool', source: 'api-tool', cost: 0.004 })], + }) + ) + }) + + /** + * `output.cost` is not a hosted-key marker. `knowledge_upload_chunk` and the + * enrichment runner report their own cost there, and the registry only writes + * hosted-key cost when it actually injected a key. Billing on the field alone + * charged a second time for spend already metered elsewhere. + */ + it('does not bill a tool that reports its own cost without a hosted key', async () => { + mocks.executeRegistryTool.mockResolvedValue({ + success: true, + output: { chunk: 'ok', cost: { total: 0.002 } }, + }) + + await run({ + toolId: 'zendesk_get_ticket', + input: { ticketId: '4', subdomain: 'a', apiToken: 't' }, + }) + + expect(mocks.recordUsage).not.toHaveBeenCalled() + }) + + /** + * Mirrors the real registry contract: a caller's own key means + * `isUsingHostedKey` is false, so `applyHostedKeyCostToResult` never runs and + * no `output.cost` is written. The meter reads that absence as the verdict. + */ + it('does not bill when the caller brought their own key', async () => { + mocks.executeRegistryTool.mockResolvedValue({ + success: true, + output: { markdown: '# Hi' }, + }) + + await run({ input: { url: 'https://a.co', apiKey: 'sk-mine' } }) + + expect(mocks.recordUsage).not.toHaveBeenCalled() + }) + + /** + * The BYOK shape. The registry injected the org's own key and returned + * `isUsingHostedKey: false`, so it wrote no `output.cost` — and the caller + * omitted the key, which a pre-dispatch derivation reads as "Sim's". Only the + * registry's verdict, carried by the presence of the cost it alone writes, + * gets this right. + */ + it('does not bill a BYOK call, where the key was omitted but Sim did not pay', async () => { + mocks.executeRegistryTool.mockResolvedValue({ + success: true, + output: { markdown: '# Hi' }, + }) + + await run({ input: { url: 'https://a.co' } }) + + expect(mocks.recordUsage).not.toHaveBeenCalled() + }) + + it('does not bill a failed call', async () => { + mocks.executeRegistryTool.mockResolvedValue({ + success: false, + output: { cost: { total: 0.004 } }, + error: 'upstream refused', + }) + + await run() + + expect(mocks.recordUsage).not.toHaveBeenCalled() + }) + + it('records nothing when the call incurred no hosted-key spend', async () => { + await run() + expect(mocks.recordUsage).not.toHaveBeenCalled() + }) + + /** + * The provider already ran and already charged Sim's key, so losing the + * ledger row must not also lose the caller's result. + */ + it('still answers when metering fails', async () => { + mocks.executeRegistryTool.mockResolvedValue({ + success: true, + output: { cost: { total: 0.004 } }, + }) + mocks.recordUsage.mockRejectedValue(new Error('ledger unavailable')) + + await expect(run()).resolves.toMatchObject({ status: 'succeeded' }) + }) +}) diff --git a/apps/sim/lib/tool-execution/application/execute-tool.ts b/apps/sim/lib/tool-execution/application/execute-tool.ts new file mode 100644 index 00000000000..5aad7390ac2 --- /dev/null +++ b/apps/sim/lib/tool-execution/application/execute-tool.ts @@ -0,0 +1,435 @@ +import { createLogger } from '@sim/logger' +import { getErrorMessage } from '@sim/utils/errors' +import { generateId } from '@sim/utils/id' +import { resolveBillingAttribution, toBillingContext } from '@/lib/billing/core/billing-attribution' +import { recordUsage } from '@/lib/billing/core/usage-log' +import { + isBlockTypeAllowed, + loadCatalogWorkspaceContext, + resolveCatalogGate, +} from '@/lib/catalog/application/catalog-context' +import { + resolveVisibleToolId, + resolveVisibleToolOwners, +} from '@/lib/catalog/application/tool-scope' +import { defineAuthorizedWorkspaceUseCase } from '@/lib/core/application' +import { ForbiddenOperationError } from '@/lib/core/application/forbidden' +import { isHosted } from '@/lib/core/config/env-flags' +import { OrchestrationError } from '@/lib/core/orchestration/types' +import { principalUserId } from '@/lib/integrations/principal-scope.server' +import { toolExecutionOperations } from '@/lib/tool-execution/application/operations' +import { executeTool as executeRegistryTool } from '@/tools' +import type { ExecutableToolConfig } from '@/tools/types' +import { getTool } from '@/tools/utils' + +const logger = createLogger('ExecuteToolUseCase') + +const DEFAULT_TIMEOUT_SECONDS = 120 + +export interface ExecuteToolInput { + workspaceId: string + toolId: string + input: Record + credentialId?: string + timeoutSeconds?: number +} + +export interface ExecuteToolResult { + toolId: string + status: 'succeeded' | 'failed' + output: Record | undefined + error: { message: string } | null +} + +/** + * The parameter Sim's hosted key would fill for this call, or `undefined`. + * + * Mirrors `injectHostedKeyIfNeeded`'s tests, in its order, so the two cannot + * disagree about whether a value is coming: the tool declares `hosting`, the + * deployment hosts keys, any `enabled` predicate accepts these params, and the + * caller has not brought a key of their own — which wins where present. + * + * Pre-dispatch only, for the required-input exemption: a parameter Sim will + * fill is not missing. It is deliberately NOT the metering gate — it cannot see + * a BYOK key, which the registry injects while reporting the call as *not* + * hosted, so after dispatch the registry's own verdict is read instead. + */ +function hostedKeyParamFor( + tool: ExecutableToolConfig, + params: Record +): string | undefined { + if (!isHosted || !tool.hosting) return undefined + if (tool.hosting.enabled && !tool.hosting.enabled(params)) return undefined + const supplied = params[tool.hosting.apiKeyParam] + if (typeof supplied === 'string' && supplied.trim().length > 0) return undefined + return tool.hosting.apiKeyParam +} + +/** + * The three spellings the executor accepts for "which credential". + * + * Inside the executor they are interchangeable: `normalizeCopilotCredentialParams` + * folds `credentialId` into `credential`, and `oauthCredential` is copied onto + * `credential` before resolution. On a public contract the credential is named + * once, at the top level, and mapped onto whichever of these the tool declares. + */ +const CREDENTIAL_SELECTORS = ['credential', 'credentialId', 'oauthCredential'] as const + +/** + * The credential-selector parameter a tool declares, if it declares one. + * + * Two shapes exist. A tool with an `oauth` block hides `accessToken` and lets + * resolution fill it, declaring no selector at all. Sixty-eight others — + * Snowflake among them — declare the selector itself as a required `user-only` + * parameter (`oauthCredential` or `credential`) that their block fills from an + * `oauth-input` field. Both are the same contract to a caller: a top-level + * `credentialId`, placed where the tool expects it. + */ +function declaredCredentialSelector(tool: ExecutableToolConfig): string | undefined { + return CREDENTIAL_SELECTORS.find((name) => tool.params?.[name] !== undefined) +} + +/** + * Refuses an input key the tool does not declare. + * + * Strict rather than a denylist, because the denylist was already wrong twice + * over. `_context` carries the acting identity and `enforceCredentialAccess`; + * the `__`-prefixed fields are the reserved transient channel, `__usingHostedKey` + * among them, which decides whether a call bills as hosted spend; and + * `impersonateUserEmail` is read straight out of params by the executor and + * forwarded to credential-token resolution as an impersonation request. Naming + * those three is guesswork about a surface that keeps growing — a declared + * parameter list is the actual boundary, and it is what + * `GET /api/v2/tools/{toolId}` already publishes. + * + * Together with the `hidden` refusal above, the accept-set is exactly the + * publish-set: a key is taken if and only if `GET /api/v2/tools/{toolId}` + * lists it as something the caller may send. + * + * Also collapses the credential spellings. The executor accepts `credential`, + * `credentialId` and `oauthCredential` interchangeably, which is fine where one + * caller writes one of them and wrong on a public contract: three spellings with + * undefined precedence is a shape no client can reason about. The credential is + * named once, at the top level. + */ +function assertNoUndeclaredInputs( + tool: ExecutableToolConfig, + toolId: string, + args: Record +): void { + const params = tool.params ?? {} + + /** + * Unconditional, and first: a tool may *declare* `oauthCredential` as a + * parameter, and it would otherwise pass the declared-key check below and + * bypass the top-level `credentialId` — giving credential precedence that + * differs from one tool to the next. + */ + const credentialAlias = Object.keys(args).find((key) => + (CREDENTIAL_SELECTORS as readonly string[]).includes(key) + ) + if (credentialAlias) { + throw new OrchestrationError( + 'validation', + `input.${credentialAlias} is not accepted; pass the credential as the top-level credentialId field` + ) + } + + /** + * Declared is not the same as accepted. A `hidden` parameter is Sim's to fill + * — a resolved credential's `accessToken`, a hosted key, a block-composed + * shape — and `createUserToolSchema` omits it from what this endpoint and + * Copilot publish. Accepting it anyway either lets a caller pre-empt the + * executor's value or silently discards theirs when the executor overwrites + * it, and both are a contract the published schema does not make. + */ + const hidden = Object.keys(args).filter((key) => params[key]?.visibility === 'hidden') + if (hidden.length > 0) { + throw new OrchestrationError( + 'validation', + `${hidden.map((key) => `input.${key}`).join(', ')} ${hidden.length === 1 ? 'is' : 'are'} supplied by Sim, not by the caller` + ) + } + + const undeclared = Object.keys(args).filter((key) => !Object.hasOwn(params, key)) + if (undeclared.length === 0) return + + throw new OrchestrationError( + 'validation', + `${toolId} does not accept ${undeclared.map((key) => `input.${key}`).join(', ')}` + ) +} + +/** + * Refuses a call missing a required parameter the caller was supposed to send. + * + * `visibility` is an editor-role concept: it says whether a value comes from a + * human filling a block field (`user-only`), the agent block's model choosing an + * argument (`llm-only`), either (`user-or-llm`), or neither (`hidden`). A direct + * call has no editor and no agent block, so those roles collapse — the caller is + * the only source there is. `createUserToolSchema`, which is what both this + * endpoint and Copilot's `call_integration_tool` publish, already says as much + * by omitting `hidden` and nothing else. + * + * So the rule here is not about roles: **Sim supplies it, or the caller must.** + * `hidden` is skipped because Sim fills it from a resolved credential or a + * hosted key — and `check-tool-param-reachability` is what makes that safe to + * assume, since it fails any required `hidden` parameter without a declared + * filler. + * + * Nothing else had checked these. `validateRequiredParametersAfterMerge` covers + * `user-or-llm` alone, because on the workflow path the rest were validated + * during serialization against the block fields holding them, and this path has + * no serialization step. Omitting `zendesk_get_ticket`'s `subdomain` reached + * Zendesk as `undefined` and came back a provider authentication failure — the + * same undiagnosable shape this branch set out to remove. + */ +function assertRequiredCallerInputsPresent( + tool: ExecutableToolConfig, + toolId: string, + params: Record +): void { + const hostedKeyParam = hostedKeyParamFor(tool, params) + + const missing = Object.entries(tool.params ?? {}) + .filter(([name, declaration]) => { + if (!declaration?.required) return false + if (declaration.visibility === 'hidden') return false + if (name === hostedKeyParam) return false + const value = params[name] + return value === undefined || value === null || value === '' + }) + .map(([name]) => name) + + if (missing.length > 0) { + throw new OrchestrationError( + 'validation', + `${toolId} requires ${missing.map((name) => `input.${name}`).join(', ')}` + ) + } +} + +/** + * Runs one code-defined tool for an authenticated caller. + * + * The public counterpart of what Copilot does through `call_integration_tool`, + * and it cannot simply reuse that path's authorization. Copilot's gate is + * applied when the tool *schemas* are built — `projectIntegrationToolsForViewer` + * decides what the model is even told exists — so by the time a call reaches the + * executor the id has already been vouched for. Here the caller types the id, so + * the same two decisions have to be made against it, in this order: + * + * 1. Does a block this caller can see expose the tool at all? A tool behind an + * unrevealed preview or a kill-switched block answers `404`, never `403`, + * because a `403` would confirm it exists. This is the same predicate the + * catalog list and detail reads use, so a tool the catalog will not name + * cannot be run by naming it anyway. + * 2. Does the workspace permit its integration? This one is `403` with + * `INTEGRATION_NOT_ALLOWED`: the built-in catalog is public, so the denial + * leaks nothing, and it is a decision an organization admin made and can + * reverse — which a `404` would hide. + * + * Everything after that is the executor's own: `@/tools` resolves the credential + * under `enforceCredentialAccess`, injects a hosted API key where Sim supplies + * one, applies the `deniedTools` denylist against the resolved id, and projects + * secrets out of the result. + */ +export const executeToolForCaller = defineAuthorizedWorkspaceUseCase({ + operation: toolExecutionOperations.execute, + resolveContext: ({ input }: { input: ExecuteToolInput }) => + loadCatalogWorkspaceContext(input.workspaceId), + authorizationOptions: {}, + execute: async ({ principal, input, context }): Promise => { + const gate = await resolveCatalogGate(principal, context) + + /** + * Resolved against an unrestricted gate so "no such tool" and "not permitted + * here" stay separable. Reusing the caller's own gate would collapse a + * denied integration into a 404 — the same answer an unrevealed preview + * gets — and a member whose admin denied Slack would be told Slack does not + * exist. + */ + const owners = await resolveVisibleToolOwners({ ...gate, allowedIntegrations: null }) + const toolId = resolveVisibleToolId(input.toolId, owners) + const owningBlockTypes = owners.get(toolId) + if (!owningBlockTypes) { + throw new OrchestrationError('not_found', 'Tool not found') + } + + if (!owningBlockTypes.some((blockType) => isBlockTypeAllowed(blockType, gate))) { + throw new ForbiddenOperationError( + 'INTEGRATION_NOT_ALLOWED', + `${toolId} belongs to an integration this workspace does not permit` + ) + } + + const tool = getTool(toolId) + if (!tool) throw new OrchestrationError('not_found', 'Tool not found') + assertNoUndeclaredInputs(tool, toolId, input.input) + + const selector = declaredCredentialSelector(tool) + const requiresCredential = + tool.oauth?.required === true || (selector !== undefined && tool.params[selector]?.required) + if (requiresCredential && !input.credentialId) { + throw new OrchestrationError( + 'validation', + `credentialId is required: ${toolId} authenticates with a ${tool.oauth?.provider ?? 'connected'} credential` + ) + } + + /** + * What the executor will receive, minus `_context`. The credential lands + * under the selector the tool declares, so a declared required + * `oauthCredential` is satisfied by the top-level `credentialId` rather than + * rejected as missing; a tool that declares none gets `credential`, which the + * executor reads for OAuth resolution. + */ + const callerParams: Record = { + ...input.input, + ...(input.credentialId ? { [selector ?? 'credential']: input.credentialId } : {}), + } + assertRequiredCallerInputsPresent(tool, toolId, callerParams) + + const userId = principalUserId(principal) + if (!userId) { + throw new OrchestrationError('forbidden', 'Tool execution requires an acting user') + } + + const billingAttribution = await resolveBillingAttribution({ + actorUserId: userId, + workspaceId: context.workspaceId, + }) + + const params: Record = { + ...callerParams, + _context: { + userId, + workspaceId: context.workspaceId, + enforceCredentialAccess: true, + /** + * Explicit `{{VAR}}` only. The bare-name form the Copilot surface also + * accepts reads any identifier-shaped value as a variable lookup, which + * would silently swap a caller's literal secret for a different one. + */ + envReferenceMode: 'explicit' as const, + billingAttribution, + }, + } + + /** + * The ledger de-duplicates on `eventKey`, and the derived key is a hash of + * actor, workspace, source and description — identical for every call to the + * same tool. Without a per-call id `onConflictDoNothing` silently billed the + * first hosted-key call and nothing after it. A workflow run has an + * `executionId` to distinguish its rows; a direct call has nothing, so it + * mints one. + */ + const callId = generateId() + + const result = await executeRegistryTool(toolId, params, { + signal: AbortSignal.timeout((input.timeoutSeconds ?? DEFAULT_TIMEOUT_SECONDS) * 1000), + operationContext: { + /** + * No workflow owns this call. The empty string is what the Copilot + * in-band route already passes for the same reason — the field is + * required because most in-process tool operations run inside a run, + * and a direct API call is one of the few that does not. + */ + workflowId: '', + userId, + workspaceId: context.workspaceId, + billingAttribution, + }, + }) + + /** + * Only a successful call that spent Sim's key — and that verdict is the + * registry's, not re-derived here. + * + * The registry decides whether Sim's key was used inside + * `injectHostedKeyIfNeeded`, and a workspace or organization BYOK key is + * one of the ways it decides *no*: the org's own key is injected and + * `isUsingHostedKey` is false. A pre-dispatch derivation cannot see that + * (it would need the BYOK lookup), so an earlier version of this gate + * treated every omitted key as Sim's and was wrong for BYOK. + * + * The verdict does propagate, by one path: on a tool with `hosting`, + * `output.cost` has a single writer, `applyHostedKeyCostToResult`, and it + * runs only under `hostedKeyInfo.isUsingHostedKey && finalResult.success`. + * So `hosting` present + success + cost present *is* "Sim's key paid". + * `hosting` is checked because tools without it — `knowledge_upload_chunk`, + * the enrichment runner — report their own cost in that field and are + * metered elsewhere. That no hosted tool does the same is what + * `check-tool-param-reachability` now pins. + */ + if (result.success && tool.hosting) { + await meterHostedKeySpend({ + callId, + toolId, + userId, + workspaceId: context.workspaceId, + billingAttribution, + output: result.output, + }) + } + + return { + toolId, + status: result.success ? 'succeeded' : 'failed', + output: result.output, + error: result.success ? null : { message: result.error ?? `${toolId} did not succeed` }, + } + }, +}) + +/** + * Charges hosted-key spend this call incurred. + * + * `@/tools` computes the cost and hands it back on `output.cost.total`, but it + * writes no ledger row: a workflow run bills through the execution ledger and + * Copilot bills through Go's `_serviceCost`, and this surface is neither. Its + * own doc comment says so — "any new caller of executeTool that is not Copilot + * must arrange its own metering" — and this is that arrangement. + * + * The provider already ran and already charged Sim's key by the time this runs, + * so a metering failure must not destroy the caller's result: it is logged for + * reconciliation and the call still answers, the same choice + * `applyHostedKeyCostToResult` makes one layer down. + */ +async function meterHostedKeySpend(args: { + callId: string + toolId: string + userId: string + workspaceId: string + billingAttribution: Awaited> + output: Record | undefined +}): Promise { + const cost = (args.output?.cost as { total?: unknown } | undefined)?.total + if (typeof cost !== 'number' || !(cost > 0)) return + + const { billingEntity, billingPeriod } = toBillingContext(args.billingAttribution) + try { + await recordUsage({ + userId: args.userId, + workspaceId: args.workspaceId, + billingEntity, + billingPeriod, + entries: [ + { + category: 'tool', + source: 'api-tool', + description: `Tool call: ${args.toolId}`, + cost, + eventKey: args.callId, + }, + ], + }) + } catch (error) { + logger.error('Hosted-key metering failed; tool call succeeded unbilled', { + toolId: args.toolId, + workspaceId: args.workspaceId, + cost, + error: getErrorMessage(error), + }) + } +} diff --git a/apps/sim/lib/tool-execution/application/operations.ts b/apps/sim/lib/tool-execution/application/operations.ts new file mode 100644 index 00000000000..e657c89abaa --- /dev/null +++ b/apps/sim/lib/tool-execution/application/operations.ts @@ -0,0 +1,39 @@ +import { defineWorkspaceOperation } from '@/lib/core/application' + +/** + * Semantic operation for running one code-defined tool directly. + * + * The verb the catalog deliberately does not carry. `catalog.tools.read` + * describes which tools exist; its own policy comment says whether a member may + * call one "is decided on that tool's own operation", and this is it. Kept in + * its own domain rather than beside the catalog reads because execution reaches + * the executable tool registry, which every catalog module is guarded against. + * + * `write` rather than `read`: a tool call sends the mail, opens the issue, posts + * the message. Nothing about it is a read, and `read` is the floor of the role + * ordering, so declaring it there would mean the operation could never refuse a + * member for their role at all. + * + * `workspaceApiKey: 'deny'`, matching `selectors.execute` and + * `mcp_servers.tools.execute`, the two shipped operations that reach a third + * party on the caller's behalf. The call resolves this principal's credentials + * and this principal's secrets, and a workspace key stands for no person — so + * there would be no one to hold accountable for what was sent, and the + * per-integration gate below would have no subject to judge. + * + * `session` is declared alongside the personal key even though v2 authenticates + * only API keys today. It is the kind an internal route or the Copilot adapter + * arrives as, and the role and key policy are already the ones those surfaces + * need — the same reason the workflow-MCP operations admit the human principal + * kinds ahead of a surface that uses them. + */ +export const toolExecutionOperations = { + // permission-group-exempt: declares capability: 'none' because no static capability names running one built-in tool — the per-tool denial is the deniedTools key, applied inside @/tools against the resolved id, and the per-integration denial is the parameterized allowedIntegrations key, which the funnel cannot apply because it never sees which integration a tool id reaches. That decision is enforced from the use case by the owning-block-type check in executeToolForCaller, ahead of dispatch. + execute: defineWorkspaceOperation({ + id: 'tools.execute', + minimumRole: 'write', + workspaceApiKey: 'deny', + principalKinds: ['session', 'personal_api_key'], + capability: 'none', + }), +} as const diff --git a/apps/sim/lib/uploads/contexts/copilot/copilot-file-manager.ts b/apps/sim/lib/uploads/contexts/copilot/copilot-file-manager.ts index ceb7eacc49b..9f182f09eb3 100644 --- a/apps/sim/lib/uploads/contexts/copilot/copilot-file-manager.ts +++ b/apps/sim/lib/uploads/contexts/copilot/copilot-file-manager.ts @@ -91,15 +91,24 @@ export async function uploadCopilotFile(options: { * Uses the unified storage service with explicit copilot context. * Handles S3, Azure Blob, and local storage automatically. * + * `maxBytes` is required for the same reason it is on `fetchWorkspaceFileBuffer`: + * the stored object is admitted far above what one request may hold resident, so a + * caller that omits a ceiling inherits "unbounded" inside the shared app process. + * * @param key File storage key + * @param options.maxBytes Hard ceiling; throws `PayloadSizeLimitError` when exceeded * @returns File buffer * @throws Error if file not found or download fails */ -export async function downloadCopilotFile(key: string): Promise { +export async function downloadCopilotFile( + key: string, + options: { maxBytes: number } +): Promise { try { const fileBuffer = await downloadFile({ key, context: 'copilot', + maxBytes: options.maxBytes, }) logger.info(`Successfully downloaded copilot file: ${key}`, { diff --git a/apps/sim/lib/uploads/contexts/workspace/fetch-external-url.ts b/apps/sim/lib/uploads/contexts/workspace/fetch-external-url.ts index 3a0c495f59d..23b04edbd3c 100644 --- a/apps/sim/lib/uploads/contexts/workspace/fetch-external-url.ts +++ b/apps/sim/lib/uploads/contexts/workspace/fetch-external-url.ts @@ -87,15 +87,16 @@ export async function fetchExternalUrlToWorkspace( timeoutMs = DEFAULT_TIMEOUT_MS, } = options - const urlValidation = await validateUrlWithDNS(url, 'fileUrl') - if (!urlValidation.isValid || !urlValidation.resolvedIP) { - throw new ExternalUrlValidationError(urlValidation.error || 'Invalid external URL') + const urlValidation = await validateUrlWithDNS(url, 'fileUrl', 'contentFetch') + if (!urlValidation.isValid) { + throw new ExternalUrlValidationError(urlValidation.error) } const filename = new URL(url).pathname.split('/').pop() || 'download' const extension = path.extname(filename).toLowerCase().substring(1) const response = await secureFetchWithPinnedIP(url, urlValidation.resolvedIP, { + profile: 'contentFetch', timeout: timeoutMs, maxResponseBytes: maxDownloadBytes, signal, diff --git a/apps/sim/lib/uploads/contexts/workspace/workspace-file-manager-errors.test.ts b/apps/sim/lib/uploads/contexts/workspace/workspace-file-manager-errors.test.ts index 04300f1765a..5fd1cfddf48 100644 --- a/apps/sim/lib/uploads/contexts/workspace/workspace-file-manager-errors.test.ts +++ b/apps/sim/lib/uploads/contexts/workspace/workspace-file-manager-errors.test.ts @@ -1,7 +1,7 @@ /** * @vitest-environment node */ -import { dbChainMockFns, resetDbChainMock } from '@sim/testing' +import { dbChainMockFns, queueTableRows, resetDbChainMock, schemaMock } from '@sim/testing' import { afterAll, beforeEach, describe, expect, it, vi } from 'vitest' import { listWorkspaceFiles, loadActiveWorkspaceFileContext } from './workspace-file-manager' @@ -23,6 +23,56 @@ describe('listWorkspaceFiles error handling', () => { 'database unavailable' ) }) + + it('contains asynchronous record-mapping failures by default', async () => { + dbChainMockFns.orderBy.mockReset() + queueTableRows(schemaMock.workspaceFiles, [ + { + id: 'file-1', + key: 'workspace/workspace-1/file-1.md', + userId: 'user-1', + workspaceId: 'workspace-1', + folderId: null, + originalName: 'file-1.md', + contentType: 'text/markdown', + sizeBytes: null, + width: null, + height: null, + deletedAt: null, + uploadedAt: new Date('2026-01-01T00:00:00Z'), + updatedAt: new Date('2026-01-01T00:00:00Z'), + contentUpdatedAt: new Date('2026-01-01T00:00:00Z'), + }, + ]) + + await expect(listWorkspaceFiles('workspace-1')).resolves.toEqual([]) + }) + + it('propagates asynchronous record-mapping failures for authoritative callers', async () => { + dbChainMockFns.orderBy.mockReset() + queueTableRows(schemaMock.workspaceFiles, [ + { + id: 'file-1', + key: 'workspace/workspace-1/file-1.md', + userId: 'user-1', + workspaceId: 'workspace-1', + folderId: null, + originalName: 'file-1.md', + contentType: 'text/markdown', + sizeBytes: null, + width: null, + height: null, + deletedAt: null, + uploadedAt: new Date('2026-01-01T00:00:00Z'), + updatedAt: new Date('2026-01-01T00:00:00Z'), + contentUpdatedAt: new Date('2026-01-01T00:00:00Z'), + }, + ]) + + await expect(listWorkspaceFiles('workspace-1', { throwOnError: true })).rejects.toThrow( + 'Workspace file is missing canonical size_bytes metadata' + ) + }) }) describe('loadActiveWorkspaceFileContext', () => { diff --git a/apps/sim/lib/uploads/contexts/workspace/workspace-file-manager.ts b/apps/sim/lib/uploads/contexts/workspace/workspace-file-manager.ts index 61e45f59c1c..8abdc85a040 100644 --- a/apps/sim/lib/uploads/contexts/workspace/workspace-file-manager.ts +++ b/apps/sim/lib/uploads/contexts/workspace/workspace-file-manager.ts @@ -1296,7 +1296,7 @@ export async function listWorkspaceFiles( .orderBy(workspaceFiles.uploadedAt) const files = await (limit === undefined ? query : query.limit(limit)) - return hydrateWorkspaceFilePaths(files, workspaceId, options) + return await hydrateWorkspaceFilePaths(files, workspaceId, options) } catch (error) { logger.error(`Failed to list workspace files for ${workspaceId}:`, error) if (options?.throwOnError) throw error diff --git a/apps/sim/lib/uploads/core/storage-service.local-download.test.ts b/apps/sim/lib/uploads/core/storage-service.local-download.test.ts new file mode 100644 index 00000000000..c9f46f3869d --- /dev/null +++ b/apps/sim/lib/uploads/core/storage-service.local-download.test.ts @@ -0,0 +1,85 @@ +/** + * @vitest-environment node + */ +import { Readable } from 'node:stream' +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const { mockCreateReadStream, mockReadFile, mockStat } = vi.hoisted(() => ({ + mockCreateReadStream: vi.fn(), + mockReadFile: vi.fn(), + mockStat: vi.fn(), +})) + +vi.mock('fs', () => ({ createReadStream: mockCreateReadStream })) +vi.mock('fs/promises', () => ({ readFile: mockReadFile, stat: mockStat })) + +vi.mock('@/lib/uploads/config', () => ({ + USE_S3_STORAGE: false, + USE_BLOB_STORAGE: false, + USE_GCS_STORAGE: false, + getStorageConfig: () => ({ bucket: 'b', region: 'r' }), +})) + +vi.mock('@/lib/uploads/core/setup.server', () => ({ UPLOAD_DIR_SERVER: '/uploads' })) + +vi.mock('@/lib/uploads/server/metadata', () => ({ insertFileMetadata: vi.fn() })) + +import { isPayloadSizeLimitError } from '@/lib/core/utils/stream-limits' +import { downloadFile } from '@/lib/uploads/core/storage-service' + +/** A stream that delivers `bytes`, whatever a prior `stat` would have claimed. */ +function streamOf(bytes: number) { + const stream = Readable.from([Buffer.alloc(bytes)]) as Readable & { destroy: () => void } + vi.spyOn(stream, 'destroy') + return stream +} + +describe('downloadFile on local storage', () => { + beforeEach(() => { + vi.clearAllMocks() + mockReadFile.mockResolvedValue(Buffer.alloc(10)) + }) + + it('reads without a ceiling when the caller asks for none', async () => { + const buffer = await downloadFile({ key: 'workspace/ws/file.bin', context: 'workspace' }) + + expect(buffer.length).toBe(10) + expect(mockReadFile).toHaveBeenCalled() + expect(mockCreateReadStream).not.toHaveBeenCalled() + }) + + it('enforces the ceiling on the bytes as they arrive, not on a prior stat', async () => { + // The file grew (or was replaced) after any size a caller could have measured: + // the stream delivers more than the ceiling allows, and a stat-then-read + // implementation would have admitted it. + mockCreateReadStream.mockReturnValue(streamOf(500)) + + await expect( + downloadFile({ key: 'workspace/ws/file.bin', context: 'workspace', maxBytes: 100 }) + ).rejects.toSatisfy(isPayloadSizeLimitError) + + expect(mockStat).not.toHaveBeenCalled() + expect(mockReadFile).not.toHaveBeenCalled() + }) + + it('returns the bytes when they fit the ceiling', async () => { + mockCreateReadStream.mockReturnValue(streamOf(50)) + + const buffer = await downloadFile({ + key: 'workspace/ws/file.bin', + context: 'workspace', + maxBytes: 100, + }) + + expect(buffer.length).toBe(50) + }) + + it('destroys the stream once the read settles', async () => { + const stream = streamOf(50) + mockCreateReadStream.mockReturnValue(stream) + + await downloadFile({ key: 'workspace/ws/file.bin', context: 'workspace', maxBytes: 100 }) + + expect(stream.destroy).toHaveBeenCalled() + }) +}) diff --git a/apps/sim/lib/uploads/core/storage-service.ts b/apps/sim/lib/uploads/core/storage-service.ts index 6a035a0ec4a..f1409a620ad 100644 --- a/apps/sim/lib/uploads/core/storage-service.ts +++ b/apps/sim/lib/uploads/core/storage-service.ts @@ -1,7 +1,7 @@ import type { Readable } from 'node:stream' import { createLogger } from '@sim/logger' import { getErrorMessage } from '@sim/utils/errors' -import { assertKnownSizeWithinLimit } from '@/lib/core/utils/stream-limits' +import { readNodeStreamToBufferWithLimit } from '@/lib/core/utils/stream-limits' import { getStorageConfig, USE_BLOB_STORAGE, @@ -507,7 +507,7 @@ export async function downloadFile(options: DownloadFileOptions): Promise { try { - return await downloadFile({ key: derivativeKey(storageKey), context: 'copilot' }) + return await downloadFile({ + key: derivativeKey(storageKey), + context: 'copilot', + maxBytes: MAX_BUFFERED_TRANSFER_BYTES, + }) } catch { return null } diff --git a/apps/sim/lib/uploads/utils/context-prefix.test.ts b/apps/sim/lib/uploads/utils/context-prefix.test.ts new file mode 100644 index 00000000000..c8cc8a10747 --- /dev/null +++ b/apps/sim/lib/uploads/utils/context-prefix.test.ts @@ -0,0 +1,17 @@ +import { describe, expect, it } from 'vitest' +import { inferContextFromKey, tryInferContextFromKey } from '@/lib/uploads/utils/file-utils' + +describe('tryInferContextFromKey', () => { + it('classifies a known prefix the same way the throwing form does', () => { + for (const key of ['workspace/a/b.txt', 'execution/a/b/c/d.bin', 'kb/x', 'logs/y']) { + expect(tryInferContextFromKey(key)).toBe(inferContextFromKey(key)) + } + }) + + it('answers null where the throwing form raises, so caller input cannot 500', () => { + for (const key of ['', 'garbage', 'not-a-prefix/x.txt', '../escape']) { + expect(tryInferContextFromKey(key)).toBeNull() + expect(() => inferContextFromKey(key)).toThrow() + } + }) +}) diff --git a/apps/sim/lib/uploads/utils/file-utils.server.ts b/apps/sim/lib/uploads/utils/file-utils.server.ts index faa41bf5bfb..d3cb23db6c9 100644 --- a/apps/sim/lib/uploads/utils/file-utils.server.ts +++ b/apps/sim/lib/uploads/utils/file-utils.server.ts @@ -199,9 +199,9 @@ export async function resolveFileInputToUrl( }, } } else { - const urlValidation = await validateUrlWithDNS(fileUrl, 'filePath') + const urlValidation = await validateUrlWithDNS(fileUrl, 'filePath', 'contentFetch') if (!urlValidation.isValid) { - return { error: { status: 400, message: urlValidation.error || 'Invalid URL' } } + return { error: { status: 400, message: urlValidation.error } } } } @@ -276,12 +276,13 @@ export async function downloadFileFromUrl( return downloadFile({ key, context, maxBytes, signal }) } - const urlValidation = await validateUrlWithDNS(fileUrl, 'fileUrl') + const urlValidation = await validateUrlWithDNS(fileUrl, 'fileUrl', 'contentFetch') if (!urlValidation.isValid) { throw new Error(`Invalid file URL: ${urlValidation.error}`) } - const response = await secureFetchWithPinnedIP(fileUrl, urlValidation.resolvedIP!, { + const response = await secureFetchWithPinnedIP(fileUrl, urlValidation.resolvedIP, { + profile: 'contentFetch', timeout: timeoutMs, maxResponseBytes: maxBytes, signal, diff --git a/apps/sim/lib/uploads/utils/file-utils.ts b/apps/sim/lib/uploads/utils/file-utils.ts index 39cfdc2a6aa..3f0df1c73c8 100644 --- a/apps/sim/lib/uploads/utils/file-utils.ts +++ b/apps/sim/lib/uploads/utils/file-utils.ts @@ -757,9 +757,30 @@ export function isInternalFileUrl(fileUrl: string): boolean { * row — see `resolveStoredFileContext` — never this prefix. */ export function inferContextFromKey(key: string): StorageContext { - if (!key) { - throw new Error('Cannot infer context from empty key') + const context = tryInferContextFromKey(key) + if (!context) { + throw new Error( + key + ? `File key must start with a context prefix (kb/, knowledge-base/, chat/, copilot/, execution/, workspace/, profile-pictures/, og-images/, workspace-logos/, or logs/). Got: ${key}` + : 'Cannot infer context from empty key' + ) } + return context +} + +/** + * {@link inferContextFromKey} for a key that came from a caller rather than from + * our own storage, answering `null` instead of throwing. + * + * The throwing form is right where an unclassifiable key means the platform + * built one wrong — that is a bug and should be loud. It is wrong where the key + * is request input being normalized, because there an unrecognized prefix just + * means "this is not a file we can use", and a throw turns a malformed request + * into a 500. Both share this one list so a new context cannot be added to only + * half of them. + */ +export function tryInferContextFromKey(key: string): StorageContext | null { + if (!key) return null if (key.startsWith('kb/') || key.startsWith('knowledge-base/')) return 'knowledge-base' if (key.startsWith('chat/')) return 'chat' @@ -771,9 +792,7 @@ export function inferContextFromKey(key: string): StorageContext { if (key.startsWith('workspace-logos/')) return 'workspace-logos' if (key.startsWith('logs/')) return 'logs' - throw new Error( - `File key must start with a context prefix (kb/, knowledge-base/, chat/, copilot/, execution/, workspace/, profile-pictures/, og-images/, workspace-logos/, or logs/). Got: ${key}` - ) + return null } /** diff --git a/apps/sim/lib/users/account-deletion-cancel-announcement.test.ts b/apps/sim/lib/users/account-deletion-cancel-announcement.test.ts new file mode 100644 index 00000000000..f2498b1180d --- /dev/null +++ b/apps/sim/lib/users/account-deletion-cancel-announcement.test.ts @@ -0,0 +1,123 @@ +/** + * @vitest-environment node + */ +import { dbChainMockFns, resetDbChainMock, schemaMock } from '@sim/testing' +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const mocks = vi.hoisted(() => ({ + isSoleOwnerOfPaidOrganization: vi.fn(), + getPersonalSubscription: vi.fn(), + isUsingCloudStorage: vi.fn(), + appendTableEvent: vi.fn(), +})) + +vi.mock('@/lib/billing/organizations/membership', () => ({ + isSoleOwnerOfPaidOrganization: mocks.isSoleOwnerOfPaidOrganization, +})) +vi.mock('@/lib/billing/core/plan', () => ({ + getHighestPriorityPersonalSubscription: mocks.getPersonalSubscription, +})) +vi.mock('@/lib/uploads', () => ({ + isUsingCloudStorage: mocks.isUsingCloudStorage, + StorageService: { deleteFiles: vi.fn(async () => ({ failed: [] })) }, +})) +vi.mock('@/lib/workspaces/utils', () => ({ + reassignBilledAccountForUser: vi.fn(async () => ({ unresolved: [] })), + reassignOwnedWorkspacesForUser: vi.fn(async () => ({ unresolved: [] })), +})) +vi.mock('@/lib/table/events', () => ({ appendTableEvent: mocks.appendTableEvent })) + +import { deleteUserAccount } from '@/lib/users/account-deletion' + +const DISPATCH_ROWS = [ + { + id: 'tdsp_1', + tableId: 'table-1', + scope: { groupIds: ['group-1'] }, + cursor: 4, + mode: 'all', + isManualRun: true, + }, +] +const MARKER_ROWS = [{ tableId: 'table-1', rowId: 'row-1', groupId: 'group-2' }] + +describe('announcing the work a deleted account’s cancels stopped', () => { + beforeEach(() => { + vi.clearAllMocks() + resetDbChainMock() + mocks.isSoleOwnerOfPaidOrganization.mockResolvedValue({ isSoleOwner: false, name: null }) + mocks.getPersonalSubscription.mockResolvedValue(null) + mocks.isUsingCloudStorage.mockReturnValue(false) + mocks.appendTableEvent.mockResolvedValue(null) + // The two cancels are the only `.returning()` reads this teardown makes: no + // workspace is doomed, so the workspace-delete block never runs. + dbChainMockFns.returning.mockResolvedValueOnce(DISPATCH_ROWS).mockResolvedValueOnce(MARKER_ROWS) + }) + + /** + * These writes bypass the ordinary cancel path, which is what publishes the + * terminal events. Without them a collaborator in a surviving workspace keeps + * watching a dispatch that will never advance, and cells stay on their + * in-flight pill until something unrelated touches the row. + */ + it('publishes the same terminal dispatch and cell events a Stop would', async () => { + await deleteUserAccount('user-1') + + expect(mocks.appendTableEvent).toHaveBeenCalledWith({ + kind: 'dispatch', + tableId: 'table-1', + dispatchId: 'tdsp_1', + status: 'cancelled', + scope: { groupIds: ['group-1'] }, + cursor: 4, + mode: 'all', + isManualRun: true, + }) + expect(mocks.appendTableEvent).toHaveBeenCalledWith({ + kind: 'cell', + tableId: 'table-1', + rowId: 'row-1', + groupId: 'group-2', + status: 'cancelled', + executionId: null, + jobId: null, + error: 'Cancelled', + }) + }) + + /** + * The event log is not transactional, so an event published inside the + * transaction would announce a cancellation a rollback then undoes. + */ + it('publishes nothing when the teardown is rolled back', async () => { + dbChainMockFns.transaction.mockImplementationOnce(async () => { + throw new Error('rolled back') + }) + + await expect(deleteUserAccount('user-1')).rejects.toThrow('rolled back') + expect(mocks.appendTableEvent).not.toHaveBeenCalled() + }) + + /** + * A dispatcher that read its status as active a moment ago can still stamp a + * marker. Its insert's foreign key needs a `FOR KEY SHARE` on the departing + * user's row, which this `FOR UPDATE` conflicts with — so every concurrent + * stamp either commits before the marker cancel can miss it, or blocks until + * the `user` delete has landed and is refused outright. Without it a stamp + * landing between the cancel and the delete is nulled by `ON DELETE SET NULL` + * and drained by a sibling worker as actorless. + */ + it('locks the departing user’s row before cancelling anything it governs', async () => { + await deleteUserAccount('user-1') + + expect(dbChainMockFns.for).toHaveBeenCalledWith('update') + const lockedAt = Math.min(...dbChainMockFns.for.mock.invocationCallOrder) + const cancelledAt = Math.min( + ...dbChainMockFns.update.mock.calls + .map((call, index) => ({ call, index })) + .filter(({ call }) => call[0] === schemaMock.tableRunDispatches) + .map(({ index }) => dbChainMockFns.update.mock.invocationCallOrder[index]) + ) + expect(lockedAt).toBeLessThan(cancelledAt) + }) +}) diff --git a/apps/sim/lib/users/account-deletion-dispatch-cancel.test.ts b/apps/sim/lib/users/account-deletion-dispatch-cancel.test.ts new file mode 100644 index 00000000000..ecd9fb429e5 --- /dev/null +++ b/apps/sim/lib/users/account-deletion-dispatch-cancel.test.ts @@ -0,0 +1,103 @@ +/** + * @vitest-environment node + */ +import { dbChainMockFns, hasMockCondition, resetDbChainMock, schemaMock } from '@sim/testing' +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const { mockIsSoleOwnerOfPaidOrganization, mockGetPersonalSubscription, mockIsUsingCloudStorage } = + vi.hoisted(() => ({ + mockIsSoleOwnerOfPaidOrganization: vi.fn(), + mockGetPersonalSubscription: vi.fn(), + mockIsUsingCloudStorage: vi.fn(), + })) + +vi.mock('@/lib/billing/organizations/membership', () => ({ + isSoleOwnerOfPaidOrganization: mockIsSoleOwnerOfPaidOrganization, +})) +vi.mock('@/lib/billing/core/plan', () => ({ + getHighestPriorityPersonalSubscription: mockGetPersonalSubscription, +})) +vi.mock('@/lib/uploads', () => ({ + isUsingCloudStorage: mockIsUsingCloudStorage, + StorageService: { deleteFiles: vi.fn(async () => ({ failed: [] })) }, +})) +vi.mock('@/lib/workspaces/utils', () => ({ + reassignBilledAccountForUser: vi.fn(async () => ({ unresolved: [] })), + reassignOwnedWorkspacesForUser: vi.fn(async () => ({ unresolved: [] })), +})) + +import { deleteUserAccount } from '@/lib/users/account-deletion' + +describe('deleteUserAccount and the governed-subject foreign key', () => { + beforeEach(() => { + vi.clearAllMocks() + resetDbChainMock() + mockIsSoleOwnerOfPaidOrganization.mockResolvedValue({ isSoleOwner: false, name: null }) + mockGetPersonalSubscription.mockResolvedValue(null) + mockIsUsingCloudStorage.mockReturnValue(false) + }) + + /** + * `capability_governed_user_id` is `ON DELETE SET NULL`, and a subject the + * database erased reads exactly like a run that never had one — so a dispatch + * that outlived its governor would keep executing its remaining windows with + * no per-tool gate at all. The in-process dispatcher has no time ceiling, so + * that is not a short window. Going terminal first is what keeps the nulled + * row unreachable. + */ + it('cancels the account’s still-queued dispatches before deleting the user row', async () => { + await deleteUserAccount('user-1') + + expect(dbChainMockFns.update).toHaveBeenCalledWith(schemaMock.tableRunDispatches) + const cancelled = dbChainMockFns.set.mock.calls.find( + ([patch]) => (patch as { status?: string }).status === 'cancelled' + ) + expect(cancelled).toBeDefined() + expect((cancelled?.[0] as { cancelledAt?: Date }).cancelledAt).toBeInstanceOf(Date) + expect(dbChainMockFns.delete).toHaveBeenCalledWith(schemaMock.user) + }) + + /** + * The subject, not the attribution: `triggered_by_user_id` names the workspace + * billed account when the run's credential named no human, so cancelling on it + * would stop a workspace-key run this account never governed and leave the + * account's own actorless-looking rows alive. + */ + it('cancels on the governed subject and only the still-active statuses', async () => { + await deleteUserAccount('user-1') + + const filter = dbChainMockFns.where.mock.calls + .map(([condition]) => condition) + .find((condition) => + hasMockCondition( + condition, + (node) => node.left === schemaMock.tableRunDispatches.capabilityGovernedUserId + ) + ) + expect(filter).toBeDefined() + expect(hasMockCondition(filter, (node) => node.type === 'eq' && node.right === 'user-1')).toBe( + true + ) + expect( + hasMockCondition( + filter, + (node) => + node.type === 'inArray' && + node.column === schemaMock.tableRunDispatches.status && + Array.isArray(node.values) && + node.values.join(',') === 'pending,dispatching' + ) + ).toBe(true) + }) + + /** The cancel must precede the delete, or the FK has already nulled the subject. */ + it('orders the cancel ahead of the user delete', async () => { + await deleteUserAccount('user-1') + + const cancelOrder = dbChainMockFns.update.mock.invocationCallOrder.at(-1) + const deleteOrder = dbChainMockFns.delete.mock.invocationCallOrder.at(-1) + expect(cancelOrder).toBeDefined() + expect(deleteOrder).toBeDefined() + expect(cancelOrder as number).toBeLessThan(deleteOrder as number) + }) +}) diff --git a/apps/sim/lib/users/account-deletion-marker-cancel.test.ts b/apps/sim/lib/users/account-deletion-marker-cancel.test.ts new file mode 100644 index 00000000000..11c206c0ae0 --- /dev/null +++ b/apps/sim/lib/users/account-deletion-marker-cancel.test.ts @@ -0,0 +1,117 @@ +/** + * @vitest-environment node + */ +import { dbChainMockFns, hasMockCondition, resetDbChainMock, schemaMock } from '@sim/testing' +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const { mockIsSoleOwnerOfPaidOrganization, mockGetPersonalSubscription, mockIsUsingCloudStorage } = + vi.hoisted(() => ({ + mockIsSoleOwnerOfPaidOrganization: vi.fn(), + mockGetPersonalSubscription: vi.fn(), + mockIsUsingCloudStorage: vi.fn(), + })) + +vi.mock('@/lib/billing/organizations/membership', () => ({ + isSoleOwnerOfPaidOrganization: mockIsSoleOwnerOfPaidOrganization, +})) +vi.mock('@/lib/billing/core/plan', () => ({ + getHighestPriorityPersonalSubscription: mockGetPersonalSubscription, +})) +vi.mock('@/lib/uploads', () => ({ + isUsingCloudStorage: mockIsUsingCloudStorage, + StorageService: { deleteFiles: vi.fn(async () => ({ failed: [] })) }, +})) +vi.mock('@/lib/workspaces/utils', () => ({ + reassignBilledAccountForUser: vi.fn(async () => ({ unresolved: [] })), + reassignOwnedWorkspacesForUser: vi.fn(async () => ({ unresolved: [] })), +})) + +import { deleteUserAccount } from '@/lib/users/account-deletion' + +/** The `where` filter of the update that targets `table_row_executions`. */ +function markerCancelFilter() { + return dbChainMockFns.where.mock.calls + .map(([condition]) => condition) + .find((condition) => + hasMockCondition( + condition, + (node) => node.left === schemaMock.tableRowExecutions.capabilityGovernedUserId + ) + ) +} + +describe('deleteUserAccount and the account’s pre-stamped cell markers', () => { + beforeEach(() => { + vi.clearAllMocks() + resetDbChainMock() + mockIsSoleOwnerOfPaidOrganization.mockResolvedValue({ isSoleOwner: false, name: null }) + mockGetPersonalSubscription.mockResolvedValue(null) + mockIsUsingCloudStorage.mockReturnValue(false) + }) + + /** + * Cancelling only the dispatches leaves the markers those dispatches already + * stamped. A marker is drained by whichever worker holds the row's cascade + * lock, and that worker's guard consults its OWN dispatch — so an unrelated + * active dispatch drains the departing account's marker, whose `SET NULL` + * subject then reads as an actorless run with no per-tool gate. + */ + it('terminalizes the account’s still-unstarted markers', async () => { + await deleteUserAccount('user-1') + + expect(dbChainMockFns.update).toHaveBeenCalledWith(schemaMock.tableRowExecutions) + const filter = markerCancelFilter() + expect(filter).toBeDefined() + expect(hasMockCondition(filter, (node) => node.type === 'eq' && node.right === 'user-1')).toBe( + true + ) + }) + + /** The same terminal state a cancel writes, so every `isExecCancelled` drain + * guard already refuses to run it. */ + it('writes the canonical cancelled cell state', async () => { + await deleteUserAccount('user-1') + + const cancelled = dbChainMockFns.set.mock.calls + .map(([patch]) => patch as { status?: string; cancelledAt?: Date; error?: string }) + .filter((patch) => patch.status === 'cancelled') + expect(cancelled.some((patch) => patch.error === 'Cancelled')).toBe(true) + expect( + cancelled.every( + (patch) => patch.cancelledAt === undefined || patch.cancelledAt instanceof Date + ) + ).toBe(true) + }) + + /** Only the states a marker sits in before a worker claims it. */ + it('leaves running and terminal cells alone', async () => { + await deleteUserAccount('user-1') + + expect( + hasMockCondition( + markerCancelFilter(), + (node) => + node.type === 'inArray' && + node.column === schemaMock.tableRowExecutions.status && + Array.isArray(node.values) && + node.values.join(',') === 'pending,queued' + ) + ).toBe(true) + }) + + /** After the FK nulls the subject there is nothing left to match on. */ + it('runs before the user row is deleted', async () => { + await deleteUserAccount('user-1') + + const markerCancelOrder = dbChainMockFns.update.mock.calls.reduce( + (found, call, index) => + call[0] === schemaMock.tableRowExecutions + ? dbChainMockFns.update.mock.invocationCallOrder[index] + : found, + undefined as number | undefined + ) + const deleteOrder = dbChainMockFns.delete.mock.invocationCallOrder.at(-1) + expect(markerCancelOrder).toBeDefined() + expect(markerCancelOrder as number).toBeLessThan(deleteOrder as number) + }) +}) diff --git a/apps/sim/lib/users/account-deletion.ts b/apps/sim/lib/users/account-deletion.ts index 98c063edc0f..447beba1b30 100644 --- a/apps/sim/lib/users/account-deletion.ts +++ b/apps/sim/lib/users/account-deletion.ts @@ -6,6 +6,7 @@ import { member, organization, permissions, + tableRunDispatches, user, workspaceFile, workspaceFiles, @@ -22,6 +23,11 @@ import type { import { getHighestPriorityPersonalSubscription } from '@/lib/billing/core/plan' import { isSoleOwnerOfPaidOrganization } from '@/lib/billing/organizations/membership' import { OrchestrationError } from '@/lib/core/orchestration/types' +import { appendTableEvent, type TableEvent } from '@/lib/table/events' +import { + type CancelledCellMarker, + cancelPendingMarkersForGovernedSubject, +} from '@/lib/table/rows/executions' import type { StorageContext } from '@/lib/uploads' import { isUsingCloudStorage, StorageService } from '@/lib/uploads' import { @@ -509,6 +515,66 @@ async function purgeStorageObjects(batches: StorageKeyBatch[]): Promise { } } +/** One dispatch the deletion stopped, in the shape its terminal event needs. */ +interface CancelledDispatch { + id: string + tableId: string + scope: unknown + cursor: number + mode: string + isManualRun: boolean +} + +/** + * Publishes the terminal events for work the deletion cancelled. + * + * The cancels above are direct writes rather than the ordinary cancel path, so + * nothing had announced them: a collaborator in a surviving workspace kept + * watching a dispatch that will never advance and cells stuck on their in-flight + * pill. These are the same two events `markActiveDispatchesCancelled` and the + * cell writers publish, so the client reconciles exactly as it does for a Stop. + * + * After the commit, never inside it: an event announcing a rollback would be a + * lie, and the SSE log is not transactional. Failures are logged rather than + * raised — the account is already gone, and the periodic refetch is the backstop. + */ +async function announceCancelledTableWork( + dispatches: CancelledDispatch[], + markers: CancelledCellMarker[] +): Promise { + const events = [ + ...dispatches.map((dispatch) => + appendTableEvent({ + kind: 'dispatch' as const, + tableId: dispatch.tableId, + dispatchId: dispatch.id, + status: 'cancelled' as const, + scope: (dispatch.scope ?? undefined) as Extract['scope'], + cursor: dispatch.cursor, + mode: dispatch.mode as 'all' | 'incomplete' | 'new', + isManualRun: dispatch.isManualRun, + }) + ), + ...markers.map((marker) => + appendTableEvent({ + kind: 'cell' as const, + tableId: marker.tableId, + rowId: marker.rowId, + groupId: marker.groupId, + status: 'cancelled' as const, + executionId: null, + jobId: null, + error: 'Cancelled', + }) + ), + ] + const results = await Promise.allSettled(events) + const failed = results.filter((result) => result.status === 'rejected').length + if (failed > 0) { + logger.warn('Some cancellation events were not published during account deletion', { failed }) + } +} + /** * Erases an account and everything only it can reach. * @@ -545,6 +611,9 @@ export async function deleteUserAccount(userId: string): Promise { if (doomedWorkspaceIds.length > 0) { /** @@ -604,9 +673,76 @@ export async function deleteUserAccount(userId: string): Promise kind === principal.kind)) { + throw new ForbiddenOperationError( + 'PRINCIPAL_KIND_NOT_PERMITTED', + `Principal kind ${principal.kind} cannot perform operation ${operation.id}; a first-party session is required` + ) + } +} diff --git a/apps/sim/lib/users/application/delete-account.ts b/apps/sim/lib/users/application/delete-account.ts index 54c4ccd0730..64cbed3ac89 100644 --- a/apps/sim/lib/users/application/delete-account.ts +++ b/apps/sim/lib/users/application/delete-account.ts @@ -1,10 +1,10 @@ import { AuditAction, AuditResourceType, recordAuditBatch } from '@sim/audit' -import type { Principal, SessionPrincipal } from '@sim/auth/principal' import { normalizeEmail } from '@sim/utils/string' import type { AccountDeletionPlan } from '@/lib/api/contracts/user' import type { OperationUseCase } from '@/lib/core/application' import { OrchestrationError } from '@/lib/core/orchestration/types' import { deleteUserAccount, getAccountDeletionPlan } from '@/lib/users/account-deletion' +import { requireUserAccountPrincipal } from '@/lib/users/application/authorization' import { userAccountOperations } from '@/lib/users/application/operations' import { getUserProfile } from '@/lib/users/queries' @@ -14,12 +14,6 @@ import { getUserProfile } from '@/lib/users/queries' * key or a delegated service must never be able to erase the human behind it. * Defence in depth: the route's `internalSessionAuth` already returns nothing else. */ -function requireSelf(principal: Principal): asserts principal is SessionPrincipal { - if (principal.kind !== 'session') { - throw new OrchestrationError('forbidden', 'Session authentication required') - } -} - export const previewAccountDeletionUseCase: OperationUseCase< typeof userAccountOperations.previewDeletion, Record, @@ -27,7 +21,7 @@ export const previewAccountDeletionUseCase: OperationUseCase< > = { operation: userAccountOperations.previewDeletion, async execute({ principal }) { - requireSelf(principal) + requireUserAccountPrincipal(principal, userAccountOperations.previewDeletion) return getAccountDeletionPlan(principal.userId) }, } @@ -44,7 +38,7 @@ export const deleteAccountUseCase: OperationUseCase< > = { operation: userAccountOperations.delete, async execute({ principal, input }) { - requireSelf(principal) + requireUserAccountPrincipal(principal, userAccountOperations.delete) const profile = await getUserProfile(principal.userId) if (!profile) throw new OrchestrationError('not_found', 'Account not found') diff --git a/apps/sim/lib/users/application/operations.ts b/apps/sim/lib/users/application/operations.ts index 145f2c9bfa3..9277501e969 100644 --- a/apps/sim/lib/users/application/operations.ts +++ b/apps/sim/lib/users/application/operations.ts @@ -1,13 +1,43 @@ import type { ApplicationOperation } from '@/lib/core/application' +import { assertOperationCapability } from '@/lib/core/application' + +export interface UserAccountOperation extends ApplicationOperation { + readonly principalKinds: readonly ['session'] +} + +/** + * Bakes the session-only principal policy into the operation — + * `requireUserAccountPrincipal` reads `principalKinds` off it at authorization + * time — and refuses a missing capability at definition time, the same guard + * every other operation factory carries. + */ +function defineUserAccountOperation( + operation: ApplicationOperation +): UserAccountOperation { + assertOperationCapability(operation) + return Object.freeze({ ...operation, principalKinds: Object.freeze(['session'] as const) }) +} /** - * Operations an account performs on itself. They carry no workspace scope and no - * role: the resource *is* the authenticated principal, so a session is both the - * only acceptable credential and the whole authorization story. That policy is - * enforced where it can actually hold — `internalSessionAuth` on the route and - * the principal guard in each use case — rather than restated as inert data here. + * Operations an account performs on itself. They carry no workspace scope and + * no role: the resource *is* the authenticated principal, so a session is both + * the only acceptable credential and the whole authorization story, enforced by + * `internalSessionAuth` on the route and `requireUserAccountPrincipal` in each + * use case. */ export const userAccountOperations = { - previewDeletion: { id: 'users.account.deletion_preview' }, - delete: { id: 'users.account.delete' }, -} as const satisfies Record + // permission-group-exempt: reading your own profile is not a workspace act, so no group key names it + readProfile: defineUserAccountOperation({ id: 'users.account.profile.read', capability: 'none' }), + // permission-group-exempt: reading your own account settings is not a workspace act, so no group key names it + readSettings: defineUserAccountOperation({ + id: 'users.account.settings.read', + capability: 'none', + }), + // permission-group-exempt: the resource is the account itself, and a permission group scopes a workspace the account may leave rather than the account + previewDeletion: defineUserAccountOperation({ + id: 'users.account.deletion_preview', + capability: 'none', + }), + // permission-group-exempt: deleting your own account is not a workspace act, so no group key names it + delete: defineUserAccountOperation({ id: 'users.account.delete', capability: 'none' }), +} as const satisfies Record diff --git a/apps/sim/lib/users/application/read-current-user.test.ts b/apps/sim/lib/users/application/read-current-user.test.ts new file mode 100644 index 00000000000..5c8779dce40 --- /dev/null +++ b/apps/sim/lib/users/application/read-current-user.test.ts @@ -0,0 +1,74 @@ +/** + * @vitest-environment node + */ +import type { PersonalApiKeyPrincipal, SessionPrincipal } from '@sim/auth/principal' +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const mocks = vi.hoisted(() => ({ + getUserProfile: vi.fn(), + getUserSettings: vi.fn(), +})) + +vi.mock('@/lib/users/queries', () => ({ + getUserProfile: mocks.getUserProfile, + getUserSettings: mocks.getUserSettings, +})) + +import { ForbiddenOperationError } from '@/lib/core/application' +import type { OrchestrationError } from '@/lib/core/orchestration/types' +import { + getCurrentUserProfileUseCase, + getCurrentUserSettingsUseCase, +} from '@/lib/users/application/read-current-user' + +const session: SessionPrincipal = { + kind: 'session', + userId: 'user-1', + sessionId: 'session-1', +} +const personalKey: PersonalApiKeyPrincipal = { + kind: 'personal_api_key', + userId: 'user-1', + keyId: 'key-1', +} + +describe('current-user reads', () => { + beforeEach(() => { + vi.clearAllMocks() + }) + + it('rejects non-session principals before loading account data', async () => { + await expect( + getCurrentUserProfileUseCase.execute({ principal: personalKey, input: {} }) + ).rejects.toBeInstanceOf(ForbiddenOperationError) + await expect( + getCurrentUserSettingsUseCase.execute({ principal: personalKey, input: {} }) + ).rejects.toBeInstanceOf(ForbiddenOperationError) + expect(mocks.getUserProfile).not.toHaveBeenCalled() + expect(mocks.getUserSettings).not.toHaveBeenCalled() + }) + + it('reads both resources for the authenticated account identity', async () => { + const profile = { id: 'user-1', name: 'User', email: 'user@example.com', image: null } + const settings = { theme: 'dark' } + mocks.getUserProfile.mockResolvedValue(profile) + mocks.getUserSettings.mockResolvedValue(settings) + + await expect( + getCurrentUserProfileUseCase.execute({ principal: session, input: {} }) + ).resolves.toEqual(profile) + await expect( + getCurrentUserSettingsUseCase.execute({ principal: session, input: {} }) + ).resolves.toEqual(settings) + expect(mocks.getUserProfile).toHaveBeenCalledWith('user-1') + expect(mocks.getUserSettings).toHaveBeenCalledWith('user-1') + }) + + it('classifies a missing current-user profile as not found', async () => { + mocks.getUserProfile.mockResolvedValue(null) + + await expect( + getCurrentUserProfileUseCase.execute({ principal: session, input: {} }) + ).rejects.toMatchObject>({ code: 'not_found' }) + }) +}) diff --git a/apps/sim/lib/users/application/read-current-user.ts b/apps/sim/lib/users/application/read-current-user.ts new file mode 100644 index 00000000000..f05d75dcce0 --- /dev/null +++ b/apps/sim/lib/users/application/read-current-user.ts @@ -0,0 +1,32 @@ +import type { UserProfileApiUser, UserSettingsApi } from '@/lib/api/contracts/user' +import type { OperationUseCase } from '@/lib/core/application' +import { OrchestrationError } from '@/lib/core/orchestration/types' +import { requireUserAccountPrincipal } from '@/lib/users/application/authorization' +import { userAccountOperations } from '@/lib/users/application/operations' +import { getUserProfile, getUserSettings } from '@/lib/users/queries' + +export const getCurrentUserProfileUseCase: OperationUseCase< + typeof userAccountOperations.readProfile, + Record, + UserProfileApiUser +> = { + operation: userAccountOperations.readProfile, + async execute({ principal }) { + requireUserAccountPrincipal(principal, userAccountOperations.readProfile) + const profile = await getUserProfile(principal.userId) + if (!profile) throw new OrchestrationError('not_found', 'User not found') + return profile + }, +} + +export const getCurrentUserSettingsUseCase: OperationUseCase< + typeof userAccountOperations.readSettings, + Record, + UserSettingsApi +> = { + operation: userAccountOperations.readSettings, + async execute({ principal }) { + requireUserAccountPrincipal(principal, userAccountOperations.readSettings) + return getUserSettings(principal.userId) + }, +} diff --git a/apps/sim/lib/webhooks/deploy.test.ts b/apps/sim/lib/webhooks/deploy.test.ts index c9a1e8ac06d..9732e9d0308 100644 --- a/apps/sim/lib/webhooks/deploy.test.ts +++ b/apps/sim/lib/webhooks/deploy.test.ts @@ -255,6 +255,7 @@ describe('resolveWebhookConfigForBlock — slack_oauth routing', () => { canonicalParamId: 'botCredential', required: true, }, + { id: 'commandFilter', mode: 'trigger', required: false }, ], } @@ -266,6 +267,7 @@ describe('resolveWebhookConfigForBlock — slack_oauth routing', () => { ;(getTrigger as unknown as Mock).mockReturnValue(slackTriggerDef) return resolveWebhookConfigForBlock({ block: makeBlock('slack_oauth', values), + blocks: {}, workflow, userId: 'deployer-1', requestId: 'req-1', @@ -293,6 +295,31 @@ describe('resolveWebhookConfigForBlock — slack_oauth routing', () => { expect(mockFetchSlackTeamId).not.toHaveBeenCalled() }) + it('deploys a slash command trigger and preserves its command filter', async () => { + mockGetSlackBotCredential.mockResolvedValue({ + workspaceId: 'ws-1', + botToken: 'xoxb-token', + teamId: 'T123', + botUserId: 'BUSER', + signingSecret: 'secret', + }) + + const result = await resolveSlack({ + eventType: 'slash_command', + commandFilter: '/ask-sim', + customBotCredential: 'cred_bot_1', + }) + + expect(result?.success).toBe(true) + if (!result?.success) throw new Error('expected success') + expect(result.config.provider).toBe('slack') + expect(result.config.routingKey).toBe('cred_bot_1') + expect(result.config.providerConfig).toMatchObject({ + eventType: 'slash_command', + commandFilter: '/ask-sim', + }) + }) + it('does not validate an identity-less migrated bot for ordinary triggers', async () => { mockGetSlackBotCredential.mockResolvedValue({ workspaceId: 'ws-1', @@ -428,6 +455,26 @@ describe('resolveWebhookConfigForBlock — slack_oauth routing', () => { expect(mockRefreshAccessTokenIfNeeded).not.toHaveBeenCalled() }) + it('rejects Agent Sessions events on the native Sim app', async () => { + mockGetSlackBotCredential.mockResolvedValue(null) + mockResolveOAuthAccountId.mockResolvedValue({ accountId: 'acct-1' }) + queueTableRows(credential, [{ id: 'cred_oauth_1' }]) + + const result = await resolveSlack({ + eventType: 'agent_session_stopped', + customBotCredential: 'cred_oauth_1', + }) + + expect(result?.success).toBe(false) + if (result?.success) throw new Error('expected failure') + expect(result?.error).toEqual({ + message: + 'This event is not available on the Sim Slack app. Use a custom bot or choose a supported event.', + status: 400, + }) + expect(mockRefreshAccessTokenIfNeeded).not.toHaveBeenCalled() + }) + it('routes an OAuth account by team_id on the slack_app provider', async () => { mockGetSlackBotCredential.mockResolvedValue(null) mockResolveOAuthAccountId.mockResolvedValue({ accountId: 'acct-1' }) @@ -482,6 +529,7 @@ describe('resolveWebhookConfigForBlock — migrated slack_webhook routing', () = ;(getTrigger as unknown as Mock).mockReturnValue(legacySlackTriggerDef) return resolveWebhookConfigForBlock({ block: makeBlock('slack_webhook', values), + blocks: {}, workflow: { workspaceId: 'ws-1' }, userId: 'deployer-1', requestId: 'req-1', @@ -546,6 +594,7 @@ describe('resolveWebhookConfigForBlock — TikTok routing', () => { ;(getTrigger as unknown as Mock).mockReturnValue(tiktokTriggerDef) return resolveWebhookConfigForBlock({ block: makeBlock('tiktok', { triggerCredentials: credentialReference }), + blocks: {}, workflow, userId: 'deployer-1', requestId: 'req-1', diff --git a/apps/sim/lib/webhooks/deploy.ts b/apps/sim/lib/webhooks/deploy.ts index 2425135c91c..136720edcc1 100644 --- a/apps/sim/lib/webhooks/deploy.ts +++ b/apps/sim/lib/webhooks/deploy.ts @@ -27,6 +27,11 @@ import { type StableDesiredWebhookRegistration, } from '@/lib/webhooks/registration-service' import { LEGACY_SLACK_CUSTOM_BOT_INGRESS_MODE } from '@/lib/webhooks/slack-custom-ingress-constants' +import { + isSlackStreamResponseRequested, + normalizeSlackStreamResponseConfig, + replaceSlackStreamAuthoringConfig, +} from '@/lib/webhooks/slack-stream-config' import { findConflictingWebhookPathOwner } from '@/lib/webhooks/utils.server' import { buildCanonicalIndex, @@ -335,6 +340,7 @@ export async function resolveTriggerCredentialId( */ export async function resolveWebhookConfigForBlock(input: { block: BlockState + blocks: Record workflow: Record userId: string requestId: string @@ -440,6 +446,20 @@ export async function resolveWebhookConfigForBlock(input: { }, } } + try { + replaceSlackStreamAuthoringConfig( + providerConfig, + normalizeSlackStreamResponseConfig(providerConfig, input.blocks) + ) + } catch (error) { + return { + success: false, + error: { + message: getErrorMessage(error, 'Invalid Slack stream configuration.'), + status: 400, + }, + } + } effectiveProvider = 'slack' effectivePath = null routingKey = slackCredentialId @@ -492,6 +512,15 @@ export async function resolveWebhookConfigForBlock(input: { }, } } + if (isSlackStreamResponseRequested(providerConfig)) { + return { + success: false, + error: { + message: 'Streaming Slack trigger responses require a custom bot.', + status: 400, + }, + } + } // Native Sim app: a workspace OAuth Slack credential. Resolve it through the // same workspace/provider-scoped lookup the generic credential path uses, so // a pasted foreign or other-tenant credential id can't bind here and the @@ -660,14 +689,15 @@ export async function resolveWebhookConfigForBlock(input: { async function configurePollingIfNeeded( provider: string, savedWebhook: Record, - requestId: string + requestId: string, + actor: { userId: string; workspaceId: string | null; deploymentVersionId?: string | null } ): Promise { const handler = getProviderHandler(provider) if (!handler.configurePolling) { return null } - const success = await handler.configurePolling({ webhook: savedWebhook, requestId }) + const success = await handler.configurePolling({ webhook: savedWebhook, requestId, ...actor }) if (!success) { await db.delete(webhook).where(eq(webhook.id, savedWebhook.id as string)) return { @@ -720,6 +750,7 @@ export async function prepareStableTriggerWebhooksForDeploy({ signal?.throwIfAborted() const resolved = await resolveWebhookConfigForBlock({ block, + blocks, workflow, userId, requestId, @@ -829,6 +860,7 @@ export async function saveTriggerWebhooksForDeploy({ for (const block of triggerBlocks) { const resolved = await resolveWebhookConfigForBlock({ block, + blocks, workflow, userId, requestId, @@ -1058,7 +1090,12 @@ export async function saveTriggerWebhooksForDeploy({ const pollingError = await configurePollingIfNeeded( sub.provider, { id: sub.webhookId, path: sub.triggerPath, providerConfig: sub.updatedProviderConfig }, - requestId + requestId, + { + userId, + workspaceId: typeof workflow.workspaceId === 'string' ? workflow.workspaceId : null, + deploymentVersionId, + } ) if (pollingError) { logger.error( diff --git a/apps/sim/lib/webhooks/env-resolver.test.ts b/apps/sim/lib/webhooks/env-resolver.test.ts index 0f802ed5947..507f8a7592f 100644 --- a/apps/sim/lib/webhooks/env-resolver.test.ts +++ b/apps/sim/lib/webhooks/env-resolver.test.ts @@ -5,11 +5,20 @@ import { environmentUtilsMockFns, resetEnvironmentUtilsMock } from '@sim/testing' import { afterAll, beforeEach, describe, expect, it, vi } from 'vitest' -const { mockGetEffectiveDecryptedEnv } = environmentUtilsMockFns +const { mockGetEffectiveDecryptedEnv, mockGetExecutionEnvironment } = environmentUtilsMockFns + +const { mockGetWorkspaceBilledAccountUserId } = vi.hoisted(() => ({ + mockGetWorkspaceBilledAccountUserId: vi.fn(), +})) + +vi.mock('@/lib/billing/core/billing-attribution', () => ({ + getWorkspaceBilledAccountUserId: mockGetWorkspaceBilledAccountUserId, +})) afterAll(resetEnvironmentUtilsMock) import { + resolveBackgroundWebhookEnv, resolveWebhookProviderConfig, resolveWebhookRecordProviderConfig, } from '@/lib/webhooks/env-resolver' @@ -103,3 +112,85 @@ describe('webhook env resolver', () => { expect(mockGetEffectiveDecryptedEnv).not.toHaveBeenCalled() }) }) + +/** + * An inbound delivery or a provider URL-validation challenge has no caller, so + * it must resolve the two identities the executor resolves — otherwise the + * workflow owner leaving the workspace silently stops the webhook's own signing + * secret from resolving, and every caller here reads that as a rejected request + * rather than an error. + */ +describe('resolveBackgroundWebhookEnv', () => { + beforeEach(() => { + vi.clearAllMocks() + mockGetEffectiveDecryptedEnv.mockResolvedValue({ FROM_SINGLE_IDENTITY: 'single' }) + mockGetExecutionEnvironment.mockResolvedValue({ + personalDecrypted: { OWNER_KEY: 'owner-value' }, + workspaceDecrypted: { WORKSPACE_KEY: 'workspace-value' }, + }) + }) + + it('splits the workflow owner from the workspace billing account', async () => { + mockGetWorkspaceBilledAccountUserId.mockResolvedValue('billing-1') + + const env = await resolveBackgroundWebhookEnv('owner-1', 'workspace-1') + + expect(mockGetExecutionEnvironment).toHaveBeenCalledWith('owner-1', 'billing-1', 'workspace-1') + expect(env).toEqual({ OWNER_KEY: 'owner-value', WORKSPACE_KEY: 'workspace-value' }) + expect(mockGetEffectiveDecryptedEnv).not.toHaveBeenCalled() + }) + + /** Workspace variables win a name collision, matching every other execution path. */ + it('lets the workspace slice shadow the owner personal slice', async () => { + mockGetWorkspaceBilledAccountUserId.mockResolvedValue('billing-1') + mockGetExecutionEnvironment.mockResolvedValue({ + personalDecrypted: { SHARED: 'personal' }, + workspaceDecrypted: { SHARED: 'workspace' }, + }) + + const env = await resolveBackgroundWebhookEnv('owner-1', 'workspace-1') + + expect(env).toEqual({ SHARED: 'workspace' }) + }) + + /** + * Both degenerate cases still go through the resolver, naming the owner as + * both identities. Short-circuiting them to `getEffectiveDecryptedEnv` read the + * owner's variables without passing the resolver's suspension check. + */ + it('names the owner as both identities when the workspace has no billing account', async () => { + mockGetWorkspaceBilledAccountUserId.mockResolvedValue(null) + + const env = await resolveBackgroundWebhookEnv('owner-1', 'workspace-1') + + expect(mockGetExecutionEnvironment).toHaveBeenCalledWith('owner-1', 'owner-1', 'workspace-1') + expect(mockGetEffectiveDecryptedEnv).not.toHaveBeenCalled() + expect(env).toEqual({ OWNER_KEY: 'owner-value', WORKSPACE_KEY: 'workspace-value' }) + }) + + it('routes a workspaceless webhook through the resolver too', async () => { + const env = await resolveBackgroundWebhookEnv('owner-1') + + expect(mockGetWorkspaceBilledAccountUserId).not.toHaveBeenCalled() + expect(mockGetExecutionEnvironment).toHaveBeenCalledWith('owner-1', 'owner-1', undefined) + expect(mockGetEffectiveDecryptedEnv).not.toHaveBeenCalled() + expect(env).toEqual({ OWNER_KEY: 'owner-value', WORKSPACE_KEY: 'workspace-value' }) + }) + + /** A suspended owner contributes nothing, including on the workspaceless path. */ + it('yields no personal variables for a suspended owner with no workspace', async () => { + mockGetExecutionEnvironment.mockResolvedValue({ + personalDecrypted: {}, + workspaceDecrypted: {}, + }) + + const env = await resolveBackgroundWebhookEnv('suspended-owner') + + expect(mockGetExecutionEnvironment).toHaveBeenCalledWith( + 'suspended-owner', + 'suspended-owner', + undefined + ) + expect(env).toEqual({}) + }) +}) diff --git a/apps/sim/lib/webhooks/env-resolver.ts b/apps/sim/lib/webhooks/env-resolver.ts index 13975c2c956..3220d71d1ec 100644 --- a/apps/sim/lib/webhooks/env-resolver.ts +++ b/apps/sim/lib/webhooks/env-resolver.ts @@ -1,5 +1,6 @@ import { isRecordLike } from '@sim/utils/object' -import { getEffectiveDecryptedEnv } from '@/lib/environment/utils' +import { getWorkspaceBilledAccountUserId } from '@/lib/billing/core/billing-attribution' +import { getEffectiveDecryptedEnv, getExecutionEnvironment } from '@/lib/environment/utils' import { resolveEnvVarReferences } from '@/executor/utils/reference-validation' export interface WebhookEnvResolutionOptions { @@ -7,6 +8,43 @@ export interface WebhookEnvResolutionOptions { onResolved?: (name: string, value: string) => void } +/** + * Resolves the env a webhook config is read against when there is no caller to + * speak of — an inbound delivery or a provider's URL-validation challenge. + * + * Splits the two identities the same way the executor does: personal variables + * stay with the workflow owner who authored the config, and workspace variables + * authorize against the workspace's billing account, which is the identity such + * a run acts as. Reading both slices as the owner made a webhook stop resolving + * its own signing secret the moment that person left the workspace — silently, + * because every caller here treats an unresolvable secret as a rejected request + * rather than an error. + * + * Every case goes through {@link getExecutionEnvironment}, including the two + * that have no second identity to split against — a legacy workspaceless + * webhook, and a workspace with no billing account. Returning + * `getEffectiveDecryptedEnv` directly for those read the owner's variables + * without passing the resolver's suspension check, so the one arrangement that + * still lent a suspended account's secrets was the one with the least going on. + * Naming the owner as both identities keeps the resolution identical to what + * those cases produced before while putting them behind the same gate. + */ +export async function resolveBackgroundWebhookEnv( + workflowOwnerUserId: string, + workspaceId?: string +): Promise> { + const billedAccountUserId = workspaceId + ? await getWorkspaceBilledAccountUserId(workspaceId) + : null + + const snapshot = await getExecutionEnvironment( + workflowOwnerUserId, + billedAccountUserId ?? workflowOwnerUserId, + workspaceId + ) + return { ...snapshot.personalDecrypted, ...snapshot.workspaceDecrypted } +} + /** * Recursively resolves all environment variable references in a configuration object. * Supports both exact matches (`{{VAR_NAME}}`) and embedded patterns (`https://{{HOST}}/path`). diff --git a/apps/sim/lib/webhooks/polling/imap.test.ts b/apps/sim/lib/webhooks/polling/imap.test.ts new file mode 100644 index 00000000000..90bd8151eeb --- /dev/null +++ b/apps/sim/lib/webhooks/polling/imap.test.ts @@ -0,0 +1,124 @@ +/** + * @vitest-environment node + */ +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const { + mockCreateSecureImapClient, + mockDbSelect, + mockHasImapEnvironmentReferences, + mockLogger, + mockMarkWebhookFailed, + mockResolveImapConnectionForActor, +} = vi.hoisted(() => ({ + mockCreateSecureImapClient: vi.fn(), + mockDbSelect: vi.fn(), + mockHasImapEnvironmentReferences: vi.fn(), + mockLogger: { info: vi.fn(), warn: vi.fn(), error: vi.fn() }, + mockMarkWebhookFailed: vi.fn(), + mockResolveImapConnectionForActor: vi.fn(), +})) + +vi.mock('@sim/db', () => ({ + db: { select: mockDbSelect }, +})) + +vi.mock('@/lib/core/idempotency/service', () => ({ + pollingIdempotency: { executeWithIdempotency: vi.fn() }, +})) + +vi.mock('@/lib/imap/connection.server', () => ({ + createSecureImapClient: mockCreateSecureImapClient, + hasImapEnvironmentReferences: mockHasImapEnvironmentReferences, + normalizeLiteralImapConnection: vi.fn(), + resolveImapConnectionForActor: mockResolveImapConnectionForActor, +})) + +vi.mock('@/lib/webhooks/polling/utils', () => ({ + markWebhookFailed: mockMarkWebhookFailed, + markWebhookSuccess: vi.fn(), + updateWebhookProviderConfig: vi.fn(), +})) + +vi.mock('@/lib/webhooks/processor', () => ({ + processPolledWebhookEvent: vi.fn(), +})) + +import { imapPollingHandler } from '@/lib/webhooks/polling/imap' + +describe('IMAP runtime polling policy', () => { + beforeEach(() => { + vi.clearAllMocks() + mockHasImapEnvironmentReferences.mockReturnValue(true) + mockMarkWebhookFailed.mockResolvedValue(undefined) + }) + + it('fails closed before resolution, DNS, or ImapFlow when referenced auth has no deployment actor', async () => { + const result = await imapPollingHandler.pollWebhook({ + webhookData: { + id: 'webhook-1', + deploymentVersionId: null, + providerConfig: { + host: '{{IMAP_HOST}}', + username: 'literal-user', + password: 'literal-password', + }, + } as never, + workflowData: { id: 'workflow-1', workspaceId: 'workspace-1' } as never, + requestId: 'request-1', + logger: mockLogger as never, + }) + + expect(result).toBe('failure') + expect(mockMarkWebhookFailed).toHaveBeenCalledWith('webhook-1', mockLogger) + expect(mockDbSelect).not.toHaveBeenCalled() + expect(mockResolveImapConnectionForActor).not.toHaveBeenCalled() + expect(mockCreateSecureImapClient).not.toHaveBeenCalled() + + const logged = JSON.stringify(mockLogger.error.mock.calls) + expect(logged).not.toContain('literal-user') + expect(logged).not.toContain('literal-password') + expect(logged).not.toContain('Referenced IMAP authentication requires redeployment') + }) + + it('uses the deployment actor and canonical workflow workspace for each referenced poll', async () => { + const mockLimit = vi.fn().mockResolvedValue([{ createdBy: 'deployment-actor' }]) + const mockWhere = vi.fn().mockReturnValue({ limit: mockLimit }) + const mockFrom = vi.fn().mockReturnValue({ where: mockWhere }) + mockDbSelect.mockReturnValue({ from: mockFrom }) + mockResolveImapConnectionForActor.mockResolvedValue({ + host: 'imap.example.com', + port: 993, + secure: true, + username: 'resolved-user', + password: 'resolved-password', + }) + mockCreateSecureImapClient.mockRejectedValue(new Error('connection unavailable')) + + const result = await imapPollingHandler.pollWebhook({ + webhookData: { + id: 'webhook-1', + deploymentVersionId: 'deployment-1', + providerConfig: { + host: '{{IMAP_HOST}}', + username: '{{IMAP_USERNAME}}', + password: '{{IMAP_PASSWORD}}', + }, + } as never, + workflowData: { id: 'workflow-1', workspaceId: 'canonical-workspace' } as never, + requestId: 'request-1', + logger: mockLogger as never, + }) + + expect(result).toBe('failure') + expect(mockResolveImapConnectionForActor).toHaveBeenCalledWith({ + connection: expect.objectContaining({ password: '{{IMAP_PASSWORD}}' }), + actorUserId: 'deployment-actor', + workspaceId: 'canonical-workspace', + }) + expect(mockCreateSecureImapClient).toHaveBeenCalledWith( + expect.objectContaining({ username: 'resolved-user', password: 'resolved-password' }) + ) + expect(mockMarkWebhookFailed).toHaveBeenCalledWith('webhook-1', mockLogger) + }) +}) diff --git a/apps/sim/lib/webhooks/polling/imap.ts b/apps/sim/lib/webhooks/polling/imap.ts index 596f81f3a59..55f7f1b3b1d 100644 --- a/apps/sim/lib/webhooks/polling/imap.ts +++ b/apps/sim/lib/webhooks/polling/imap.ts @@ -1,9 +1,15 @@ +import { db } from '@sim/db' +import { workflowDeploymentVersion } from '@sim/db/schema' import type { Logger } from '@sim/logger' -import { getErrorMessage } from '@sim/utils/errors' -import type { FetchMessageObject, MailboxLockObject } from 'imapflow' -import { ImapFlow } from 'imapflow' +import { and, eq } from 'drizzle-orm' +import type { FetchMessageObject, ImapFlow, MailboxLockObject } from 'imapflow' import { pollingIdempotency } from '@/lib/core/idempotency/service' -import { validateDatabaseHost } from '@/lib/core/security/input-validation.server' +import { + createSecureImapClient, + hasImapEnvironmentReferences, + normalizeLiteralImapConnection, + resolveImapConnectionForActor, +} from '@/lib/imap/connection.server' import { getProviderConfig, type PollingProviderHandler, @@ -71,6 +77,38 @@ interface ImapWebhookPayload { timestamp: string } +async function resolvePollingConnection( + webhookData: PollWebhookContext['webhookData'], + workflowData: PollWebhookContext['workflowData'], + config: ImapWebhookConfig +) { + if (!hasImapEnvironmentReferences(config)) { + return normalizeLiteralImapConnection(config) + } + + if (!webhookData.deploymentVersionId) { + throw new Error('Referenced IMAP authentication requires redeployment') + } + const [deployment] = await db + .select({ createdBy: workflowDeploymentVersion.createdBy }) + .from(workflowDeploymentVersion) + .where( + and( + eq(workflowDeploymentVersion.id, webhookData.deploymentVersionId), + eq(workflowDeploymentVersion.workflowId, workflowData.id) + ) + ) + .limit(1) + if (!deployment?.createdBy) { + throw new Error('Referenced IMAP authentication requires redeployment') + } + return resolveImapConnectionForActor({ + connection: config, + actorUserId: deployment.createdBy, + workspaceId: workflowData.workspaceId, + }) +} + export const imapPollingHandler: PollingProviderHandler = { provider: 'imap', label: 'IMAP', @@ -88,27 +126,8 @@ export const imapPollingHandler: PollingProviderHandler = { return 'failure' } - const hostValidation = await validateDatabaseHost(config.host, 'host') - if (!hostValidation.isValid) { - logger.error( - `[${requestId}] IMAP host validation failed for webhook ${webhookId}: ${hostValidation.error}` - ) - await markWebhookFailed(webhookId, logger) - return 'failure' - } - - const client = new ImapFlow({ - host: hostValidation.resolvedIP!, - servername: config.host, - port: config.port || 993, - secure: config.secure ?? true, - auth: { - user: config.username, - pass: config.password, - }, - tls: { rejectUnauthorized: true }, - logger: false, - }) + const resolvedConnection = await resolvePollingConnection(webhookData, workflowData, config) + const client = await createSecureImapClient(resolvedConnection) let emails: Awaited>['emails'] = [] let latestUidByMailbox: Record = {} @@ -182,7 +201,7 @@ export const imapPollingHandler: PollingProviderHandler = { throw innerError } } catch (error) { - logger.error(`[${requestId}] Error processing IMAP webhook ${webhookId}:`, error) + logger.error(`[${requestId}] Error processing IMAP webhook ${webhookId}`) await markWebhookFailed(webhookId, logger) return 'failure' } @@ -333,8 +352,8 @@ async function fetchNewEmails( } totalEmailsCollected++ } - } catch (mailboxError) { - logger.warn(`[${requestId}] Error processing mailbox ${mailboxPath}:`, mailboxError) + } catch { + logger.warn(`[${requestId}] Error processing mailbox ${mailboxPath}`) } } @@ -574,11 +593,8 @@ async function processEmails( currentOpenMailbox = email.mailboxPath } await client.messageFlagsAdd(email.uid, ['\\Seen'], { uid: true }) - } catch (flagError) { - logger.warn( - `[${requestId}] Failed to mark message ${email.uid} as read:`, - flagError - ) + } catch { + logger.warn(`[${requestId}] Failed to mark message ${email.uid} as read`) } } @@ -590,9 +606,8 @@ async function processEmails( `[${requestId}] Successfully processed email ${email.uid} from ${email.mailboxPath} for webhook ${webhookData.id}` ) processedCount++ - } catch (error) { - const errorMessage = getErrorMessage(error, 'Unknown error') - logger.error(`[${requestId}] Error processing email ${email.uid}:`, errorMessage) + } catch { + logger.error(`[${requestId}] Error processing email ${email.uid}`) failedCount++ } } diff --git a/apps/sim/lib/webhooks/polling/rss.ts b/apps/sim/lib/webhooks/polling/rss.ts index 662eaf6e9a9..1fd4bb0affb 100644 --- a/apps/sim/lib/webhooks/polling/rss.ts +++ b/apps/sim/lib/webhooks/polling/rss.ts @@ -199,7 +199,7 @@ async function fetchNewRssItems( logger: Logger ): Promise<{ feed: RssFeed; items: RssItem[]; etag?: string; lastModified?: string }> { try { - const urlValidation = await validateUrlWithDNS(config.feedUrl, 'feedUrl') + const urlValidation = await validateUrlWithDNS(config.feedUrl, 'feedUrl', 'requestTarget') if (!urlValidation.isValid) { logger.error(`[${requestId}] Invalid RSS feed URL: ${urlValidation.error}`) throw new Error(`Invalid RSS feed URL: ${urlValidation.error}`) @@ -216,7 +216,8 @@ async function fetchNewRssItems( headers['If-Modified-Since'] = config.lastModified } - const response = await secureFetchWithPinnedIP(config.feedUrl, urlValidation.resolvedIP!, { + const response = await secureFetchWithPinnedIP(config.feedUrl, urlValidation.resolvedIP, { + profile: 'requestTarget', headers, timeout: 30000, maxResponseBytes: MAX_RSS_FEED_BYTES, diff --git a/apps/sim/lib/webhooks/processor.ts b/apps/sim/lib/webhooks/processor.ts index da5c2d2c6c7..e618d917621 100644 --- a/apps/sim/lib/webhooks/processor.ts +++ b/apps/sim/lib/webhooks/processor.ts @@ -611,6 +611,8 @@ export async function checkWebhookPreprocessing( const preprocessResult = await preprocessExecution({ workflowId: foundWorkflow.id, userId: foundWorkflow.userId, + // The workflow owner, not whoever sent this delivery — nobody sent it. + userIdIsStoredReference: true, triggerType: 'webhook', executionId, requestId, diff --git a/apps/sim/lib/webhooks/provider-subscriptions.test.ts b/apps/sim/lib/webhooks/provider-subscriptions.test.ts index cdccc03b4a7..26ec78e9d86 100644 --- a/apps/sim/lib/webhooks/provider-subscriptions.test.ts +++ b/apps/sim/lib/webhooks/provider-subscriptions.test.ts @@ -6,18 +6,23 @@ import { environmentUtilsMockFns, resetEnvironmentUtilsMock } from '@sim/testing import type { NextRequest } from 'next/server' import { afterAll, beforeEach, describe, expect, it, vi } from 'vitest' -const { mockGetEffectiveDecryptedEnv } = environmentUtilsMockFns +const { mockGetEffectiveDecryptedEnv, mockGetExecutionEnvironment } = environmentUtilsMockFns afterAll(resetEnvironmentUtilsMock) -const { mockGetProviderHandler } = vi.hoisted(() => ({ +const { mockGetProviderHandler, mockGetWorkspaceBilledAccountUserId } = vi.hoisted(() => ({ mockGetProviderHandler: vi.fn(), + mockGetWorkspaceBilledAccountUserId: vi.fn(), })) vi.mock('@/lib/webhooks/providers', () => ({ getProviderHandler: mockGetProviderHandler, })) +vi.mock('@/lib/billing/core/billing-attribution', () => ({ + getWorkspaceBilledAccountUserId: mockGetWorkspaceBilledAccountUserId, +})) + import { cleanupExternalWebhook, createExternalWebhookSubscription, @@ -128,8 +133,20 @@ describe('cleanupExternalWebhook', () => { beforeEach(() => { vi.clearAllMocks() mockGetEffectiveDecryptedEnv.mockResolvedValue({ CALENDLY_API_KEY: 'real-secret-key' }) + mockGetWorkspaceBilledAccountUserId.mockResolvedValue('billing-1') + mockGetExecutionEnvironment.mockResolvedValue({ + personalDecrypted: {}, + workspaceDecrypted: { CALENDLY_API_KEY: 'real-secret-key' }, + }) }) + /** + * Cleanup resolves through the same two-identity reader as the delivery that + * created the subscription — owner for personal variables, the workspace + * billing account for workspace ones. Reading both slices as the owner let a + * non-admin owner without a credential grant leave `{{VAR}}` unresolved, and + * the provider was handed the literal reference as its credential. + */ it('resolves {{ENV_VAR}} references before deleting the provider subscription', async () => { const deleteSubscription = vi.fn().mockResolvedValue(undefined) mockGetProviderHandler.mockReturnValue({ deleteSubscription }) @@ -150,7 +167,7 @@ describe('cleanupExternalWebhook', () => { await cleanupExternalWebhook(webhook, workflow, 'request-1') - expect(mockGetEffectiveDecryptedEnv).toHaveBeenCalledWith('user-1', 'workspace-1') + expect(mockGetExecutionEnvironment).toHaveBeenCalledWith('user-1', 'billing-1', 'workspace-1') expect(deleteSubscription).toHaveBeenCalledWith( expect.objectContaining({ webhook: expect.objectContaining({ diff --git a/apps/sim/lib/webhooks/provider-subscriptions.ts b/apps/sim/lib/webhooks/provider-subscriptions.ts index 615a5ea7722..8cd221622b9 100644 --- a/apps/sim/lib/webhooks/provider-subscriptions.ts +++ b/apps/sim/lib/webhooks/provider-subscriptions.ts @@ -3,6 +3,7 @@ import { toError } from '@sim/utils/errors' import { omit } from '@sim/utils/object' import type { NextRequest } from 'next/server' import { + resolveBackgroundWebhookEnv, resolveWebhookProviderConfig, resolveWebhookRecordProviderConfig, } from '@/lib/webhooks/env-resolver' @@ -172,8 +173,15 @@ export async function createExternalWebhookSubscription( /** * Clean up external webhook subscriptions for a webhook. - * Resolves persisted `{{ENV_VAR}}` references with the workflow owner's - * effective environment before invoking the provider. + * + * Resolves persisted `{{ENV_VAR}}` references the same way the delivery that + * created the subscription resolved them — owner for personal variables, the + * workspace billing account for workspace ones. Reading both slices as the owner + * meant cleanup could see a narrower selection than execution did: a non-admin + * owner without a credential grant for the referenced key left `{{VAR}}` + * unresolved (`onMissing` defaults to `keep`), and the provider was then handed + * the literal reference as its credential. Since the failure below is non-fatal + * by default, that silently orphaned the subscription at the provider. * * By default, cleanup failure is logged but non-fatal for legacy best-effort callers. * Deployment outbox cleanup passes `throwOnError` so provider failures stay retryable. @@ -197,10 +205,12 @@ export async function cleanupExternalWebhook( } const workspaceId = typeof workflow.workspaceId === 'string' ? workflow.workspaceId : undefined + const envVars = await resolveBackgroundWebhookEnv(workflow.userId, workspaceId) const resolvedWebhook = await resolveWebhookRecordProviderConfig( webhook, workflow.userId, - workspaceId + workspaceId, + { envVars } ) await handler.deleteSubscription({ diff --git a/apps/sim/lib/webhooks/providers/ashby.test.ts b/apps/sim/lib/webhooks/providers/ashby.test.ts index 9c6a81aadf0..a636730fcef 100644 --- a/apps/sim/lib/webhooks/providers/ashby.test.ts +++ b/apps/sim/lib/webhooks/providers/ashby.test.ts @@ -5,6 +5,49 @@ import crypto from 'crypto' import { createMockRequest } from '@sim/testing' import { afterEach, describe, expect, it, vi } from 'vitest' import { ashbyHandler } from '@/lib/webhooks/providers/ashby' +import type { + AuthContext, + EventMatchContext, + FormatInputContext, +} from '@/lib/webhooks/providers/types' + +function authContext( + request: AuthContext['request'], + rawBody: string, + providerConfig: Record +): AuthContext { + return { + request, + rawBody, + requestId: 'r1', + providerConfig, + webhook: {}, + workflow: {}, + } +} + +function eventMatchContext(body: unknown, triggerId: string): EventMatchContext { + return { + webhook: { id: 'w1' }, + workflow: {}, + body, + request: createMockRequest('POST', body), + requestId: 'r1', + providerConfig: { triggerId }, + } +} + +function formatInputContext(body: unknown): FormatInputContext { + return { + webhook: { id: 'w1' }, + workflow: { id: 'workflow-1', userId: 'user-1' }, + body, + headers: {}, + query: {}, + method: 'POST', + requestId: 'r1', + } +} describe('ashbyHandler', () => { describe('verifyAuth', () => { @@ -16,27 +59,13 @@ describe('ashbyHandler', () => { const request = createMockRequest('POST', JSON.parse(rawBody), { 'ashby-signature': signature, }) - const res = ashbyHandler.verifyAuth!({ - request: request as any, - rawBody, - requestId: 'r1', - providerConfig: {}, - webhook: {}, - workflow: {}, - }) + const res = ashbyHandler.verifyAuth!(authContext(request, rawBody, {})) expect(res?.status).toBe(401) }) it('returns 401 when signature header is missing', () => { const request = createMockRequest('POST', JSON.parse(rawBody), {}) - const res = ashbyHandler.verifyAuth!({ - request: request as any, - rawBody, - requestId: 'r1', - providerConfig: { secretToken: secret }, - webhook: {}, - workflow: {}, - }) + const res = ashbyHandler.verifyAuth!(authContext(request, rawBody, { secretToken: secret })) expect(res?.status).toBe(401) }) @@ -44,14 +73,7 @@ describe('ashbyHandler', () => { const request = createMockRequest('POST', JSON.parse(rawBody), { 'ashby-signature': 'sha256=deadbeef', }) - const res = ashbyHandler.verifyAuth!({ - request: request as any, - rawBody, - requestId: 'r1', - providerConfig: { secretToken: secret }, - webhook: {}, - workflow: {}, - }) + const res = ashbyHandler.verifyAuth!(authContext(request, rawBody, { secretToken: secret })) expect(res?.status).toBe(401) }) @@ -59,67 +81,78 @@ describe('ashbyHandler', () => { const request = createMockRequest('POST', JSON.parse(rawBody), { 'ashby-signature': signature, }) - const res = ashbyHandler.verifyAuth!({ - request: request as any, - rawBody, - requestId: 'r1', - providerConfig: { secretToken: secret }, - webhook: {}, - workflow: {}, - }) + const res = ashbyHandler.verifyAuth!(authContext(request, rawBody, { secretToken: secret })) expect(res).toBeNull() }) }) describe('matchEvent', () => { it('rejects ping events', async () => { - const matched = await ashbyHandler.matchEvent!({ - webhook: { id: 'w1' } as any, - body: { action: 'ping', data: { webhookActionType: 'ping' } }, - requestId: 'r1', - providerConfig: { triggerId: 'ashby_application_submit' }, - } as any) + const matched = await ashbyHandler.matchEvent!( + eventMatchContext( + { action: 'ping', data: { webhookActionType: 'ping' } }, + 'ashby_application_submit' + ) + ) expect(matched).toBe(false) }) it('matches when action equals the configured trigger event', async () => { - const matched = await ashbyHandler.matchEvent!({ - webhook: { id: 'w1' } as any, - body: { action: 'applicationSubmit', data: {} }, - requestId: 'r1', - providerConfig: { triggerId: 'ashby_application_submit' }, - } as any) + const matched = await ashbyHandler.matchEvent!( + eventMatchContext({ action: 'applicationSubmit', data: {} }, 'ashby_application_submit') + ) expect(matched).toBe(true) }) it('rejects when action does not match the configured trigger event', async () => { - const matched = await ashbyHandler.matchEvent!({ - webhook: { id: 'w1' } as any, - body: { action: 'jobCreate', data: {} }, - requestId: 'r1', - providerConfig: { triggerId: 'ashby_application_submit' }, - } as any) + const matched = await ashbyHandler.matchEvent!( + eventMatchContext({ action: 'jobCreate', data: {} }, 'ashby_application_submit') + ) expect(matched).toBe(false) }) + + it('matches newly supported Ashby events', async () => { + const matched = await ashbyHandler.matchEvent!( + eventMatchContext( + { action: 'signatureRequestUpdate', data: {} }, + 'ashby_signature_request_update' + ) + ) + expect(matched).toBe(true) + }) + }) + + describe('extractIdempotencyId', () => { + it('uses Ashby webhookActionId across retries and related event deliveries', () => { + expect( + ashbyHandler.extractIdempotencyId!({ + action: 'applicationUpdate', + webhookActionId: 'action-1', + data: { application: { id: 'app-1' } }, + }) + ).toBe('ashby:webhook-action:action-1') + }) }) describe('formatInput', () => { it('spreads data fields to the top level alongside action', async () => { - const result = await ashbyHandler.formatInput!({ - body: { + const result = await ashbyHandler.formatInput!( + formatInputContext({ action: 'applicationSubmit', + webhookActionId: 'action-1', data: { application: { id: 'app-1', status: 'Active' } }, - }, - } as any) + }) + ) expect(result.input).toEqual({ action: 'applicationSubmit', + webhookActionId: 'action-1', application: { id: 'app-1', status: 'Active' }, }) }) it('renames currentInterviewStage.type to stageType, matching the trigger output schema', async () => { - const result = await ashbyHandler.formatInput!({ - body: { + const result = await ashbyHandler.formatInput!( + formatInputContext({ action: 'candidateStageChange', data: { application: { @@ -127,8 +160,8 @@ describe('ashbyHandler', () => { currentInterviewStage: { id: 'stage-1', title: 'Offer', type: 'Offer' }, }, }, - }, - } as any) + }) + ) expect(result.input.application).toEqual({ id: 'app-1', currentInterviewStage: { id: 'stage-1', title: 'Offer', stageType: 'Offer' }, @@ -263,6 +296,21 @@ describe('ashbyHandler', () => { respondWith({ success: true, results: { webhookId: 'ext-1' } }) await expect(ashbyHandler.deleteSubscription?.(ctx(true))).resolves.toBeUndefined() }) + + it('rejects an oversized provider response before buffering it', async () => { + globalThis.fetch = vi.fn().mockResolvedValue( + new Response('{}', { + status: 200, + headers: { + 'content-type': 'application/json', + 'content-length': String(Number.MAX_SAFE_INTEGER), + }, + }) + ) as never + await expect(ashbyHandler.deleteSubscription?.(ctx(true))).rejects.toThrow( + /exceeds maximum size/ + ) + }) }) describe('extractIdempotencyId', () => { diff --git a/apps/sim/lib/webhooks/providers/ashby.ts b/apps/sim/lib/webhooks/providers/ashby.ts index 93925a7d6b9..ac0ab6e1ac5 100644 --- a/apps/sim/lib/webhooks/providers/ashby.ts +++ b/apps/sim/lib/webhooks/providers/ashby.ts @@ -2,8 +2,9 @@ import { createLogger } from '@sim/logger' import { safeCompare } from '@sim/security/compare' import { hmacSha256Hex } from '@sim/security/hmac' import { generateId } from '@sim/utils/id' -import { omit } from '@sim/utils/object' +import { isRecordLike, omit } from '@sim/utils/object' import { NextResponse } from 'next/server' +import { isPayloadSizeLimitError, readResponseJsonWithLimit } from '@/lib/core/utils/stream-limits' import { getNotificationUrl, getProviderConfig } from '@/lib/webhooks/provider-subscription-utils' import type { AuthContext, @@ -92,6 +93,23 @@ function isAshbyWebhookNotFound(data: Record, message: string): } const logger = createLogger('WebhookProvider:Ashby') +const MAX_ASHBY_WEBHOOK_RESPONSE_BYTES = 2 * 1024 * 1024 + +async function readAshbyManagementResponse( + response: Response, + label: string +): Promise> { + try { + const body = await readResponseJsonWithLimit(response, { + maxBytes: MAX_ASHBY_WEBHOOK_RESPONSE_BYTES, + label, + }) + return isRecordLike(body) ? body : {} + } catch (error) { + if (isPayloadSizeLimitError(error)) throw error + return {} + } +} function validateAshbySignature(secretToken: string, signature: string, body: string): boolean { try { @@ -112,15 +130,25 @@ function validateAshbySignature(secretToken: string, signature: string, body: st export const ashbyHandler: WebhookProviderHandler = { extractIdempotencyId(body: unknown): string | null { - const obj = body as Record - const action = typeof obj.action === 'string' ? obj.action : undefined - const data = obj.data as Record | undefined - if (!action || !data) return null + if (!isRecordLike(body)) return null + const action = typeof body.action === 'string' ? body.action : undefined + if (!action) return null + + if (typeof body.webhookActionId === 'string' && body.webhookActionId) { + return `ashby:webhook-action:${body.webhookActionId}` + } + + const data = isRecordLike(body.data) ? body.data : undefined + if (!data) return null const application = data.application as Record | undefined const candidate = data.candidate as Record | undefined const job = data.job as Record | undefined const offer = data.offer as Record | undefined + const interviewSchedule = data.interviewSchedule as Record | undefined + const jobPosting = data.jobPosting as Record | undefined + const opening = data.opening as Record | undefined + const mergedCandidate = data.mergedCandidate as Record | undefined if (application?.id) { const discriminator = application.updatedAt ?? buildFallbackDeliveryFingerprint(data) @@ -136,6 +164,16 @@ export const ashbyHandler: WebhookProviderHandler = { if (job?.id) { return `ashby:${action}:${job.id}` } + if (interviewSchedule?.id) + return `ashby:${action}:${interviewSchedule.id}:${interviewSchedule.updatedAt ?? buildFallbackDeliveryFingerprint(data)}` + if (jobPosting?.id) + return `ashby:${action}:${jobPosting.id}:${jobPosting.updatedAt ?? buildFallbackDeliveryFingerprint(data)}` + if (opening?.id) return `ashby:${action}:${opening.id}` + if (mergedCandidate?.id) return `ashby:${action}:${mergedCandidate.id}` + if (typeof data.applicationId === 'string') + return `ashby:${action}:${data.applicationId}:${data.eventType ?? buildFallbackDeliveryFingerprint(data)}` + if (typeof data.offerId === 'string') + return `ashby:${action}:${data.offerId}:${data.eventType ?? buildFallbackDeliveryFingerprint(data)}` return null }, @@ -162,6 +200,7 @@ export const ashbyHandler: WebhookProviderHandler = { } : {}), action: b.action, + ...(typeof b.webhookActionId === 'string' ? { webhookActionId: b.webhookActionId } : {}), }, } }, @@ -283,7 +322,10 @@ export const ashbyHandler: WebhookProviderHandler = { body: JSON.stringify(requestBody), }) - const responseBody = (await ashbyResponse.json().catch(() => ({}))) as Record + const responseBody = await readAshbyManagementResponse( + ashbyResponse, + 'Ashby webhook creation response' + ) if (!ashbyResponse.ok || !responseBody.success) { // Ashby documents two error shapes and uses both. Reading only @@ -369,7 +411,10 @@ export const ashbyHandler: WebhookProviderHandler = { body: JSON.stringify({ webhookId: externalId }), }) - const responseBody = (await ashbyResponse.json().catch(() => ({}))) as Record + const responseBody = await readAshbyManagementResponse( + ashbyResponse, + 'Ashby webhook deletion response' + ) /** * Ashby returns what would be a 4XX elsewhere as HTTP 200 with diff --git a/apps/sim/lib/webhooks/providers/credential-group.ts b/apps/sim/lib/webhooks/providers/credential-group.ts new file mode 100644 index 00000000000..4fd7d64c18c --- /dev/null +++ b/apps/sim/lib/webhooks/providers/credential-group.ts @@ -0,0 +1,12 @@ +import type { + FormatInputContext, + FormatInputResult, + WebhookProviderHandler, +} from '@/lib/webhooks/providers/types' + +export const credentialGroupProviderHandler: WebhookProviderHandler = { + executionMode: 'queue', + async formatInput({ body }: FormatInputContext): Promise { + return { input: body } + }, +} diff --git a/apps/sim/lib/webhooks/providers/emailbison.ts b/apps/sim/lib/webhooks/providers/emailbison.ts index ee4d1ba1b31..da1839eeb15 100644 --- a/apps/sim/lib/webhooks/providers/emailbison.ts +++ b/apps/sim/lib/webhooks/providers/emailbison.ts @@ -153,13 +153,14 @@ export const emailBisonHandler: WebhookProviderHandler = { }) const targetUrl = emailBisonUrl('/api/webhook-url', {}, apiBaseUrl) - const urlValidation = await validateUrlWithDNS(targetUrl, 'apiBaseUrl') + const urlValidation = await validateUrlWithDNS(targetUrl, 'apiBaseUrl', 'configuredEndpoint') if (!urlValidation.isValid) { logger.warn(`[${requestId}] Invalid Email Bison Instance URL: ${urlValidation.error}`) throw new Error('Email Bison Instance URL could not be validated.') } - const response = await secureFetchWithPinnedIP(targetUrl, urlValidation.resolvedIP!, { + const response = await secureFetchWithPinnedIP(targetUrl, urlValidation.resolvedIP, { + profile: 'configuredEndpoint', method: 'POST', headers: emailBisonHeaders({ apiKey, apiBaseUrl }), body: JSON.stringify({ @@ -229,7 +230,7 @@ export const emailBisonHandler: WebhookProviderHandler = { {}, apiBaseUrl ) - const urlValidation = await validateUrlWithDNS(targetUrl, 'apiBaseUrl') + const urlValidation = await validateUrlWithDNS(targetUrl, 'apiBaseUrl', 'configuredEndpoint') if (!urlValidation.isValid) { logger.warn(`[${requestId}] Invalid Email Bison Instance URL: ${urlValidation.error}`, { webhookId: webhook.id, @@ -239,7 +240,8 @@ export const emailBisonHandler: WebhookProviderHandler = { return } - const response = await secureFetchWithPinnedIP(targetUrl, urlValidation.resolvedIP!, { + const response = await secureFetchWithPinnedIP(targetUrl, urlValidation.resolvedIP, { + profile: 'configuredEndpoint', method: 'DELETE', headers: emailBisonHeaders({ apiKey, apiBaseUrl }), }) diff --git a/apps/sim/lib/webhooks/providers/gitlab.ts b/apps/sim/lib/webhooks/providers/gitlab.ts index 6b5f4cd8d1b..41d71d758e1 100644 --- a/apps/sim/lib/webhooks/providers/gitlab.ts +++ b/apps/sim/lib/webhooks/providers/gitlab.ts @@ -34,6 +34,7 @@ async function cleanupGitLabHookByUrl( host: unknown ): Promise { const res = await secureFetchWithValidation(gitlabProjectHooksUrl(projectId, host), { + profile: 'configuredEndpoint', headers: { 'PRIVATE-TOKEN': accessToken }, }).catch(() => null) if (!res || !res.ok) return @@ -46,6 +47,7 @@ async function cleanupGitLabHookByUrl( .filter((hook) => hook.url === url && hook.id != null) .map((hook) => secureFetchWithValidation(`${gitlabProjectHooksUrl(projectId, host)}/${hook.id}`, { + profile: 'configuredEndpoint', method: 'DELETE', headers: { 'PRIVATE-TOKEN': accessToken }, }).catch(() => null) @@ -198,6 +200,7 @@ export const gitlabHandler: WebhookProviderHandler = { const { getGitLabEventFlags } = await import('@/triggers/gitlab/utils') const secretToken = generateId() const res = await secureFetchWithValidation(gitlabProjectHooksUrl(projectId, host), { + profile: 'configuredEndpoint', method: 'POST', headers: { 'PRIVATE-TOKEN': accessToken, 'Content-Type': 'application/json' }, body: JSON.stringify({ @@ -269,10 +272,7 @@ export const gitlabHandler: WebhookProviderHandler = { const res = await secureFetchWithValidation( `${gitlabProjectHooksUrl(projectId, host)}/${externalId}`, - { - method: 'DELETE', - headers: { 'PRIVATE-TOKEN': accessToken }, - } + { profile: 'configuredEndpoint', method: 'DELETE', headers: { 'PRIVATE-TOKEN': accessToken } } ) if (!res.ok && res.status !== 404) { diff --git a/apps/sim/lib/webhooks/providers/imap.test.ts b/apps/sim/lib/webhooks/providers/imap.test.ts new file mode 100644 index 00000000000..d4637169334 --- /dev/null +++ b/apps/sim/lib/webhooks/providers/imap.test.ts @@ -0,0 +1,188 @@ +/** + * @vitest-environment node + */ +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const { + mockClose, + mockCreateSecureImapClient, + mockDbLimit, + mockDbSelect, + mockDbUpdate, + mockHasImapEnvironmentReferences, + mockLogger, + mockNormalizeLiteralImapConnection, + mockResolveImapConnectionForActor, +} = vi.hoisted(() => ({ + mockClose: vi.fn(), + mockCreateSecureImapClient: vi.fn(), + mockDbLimit: vi.fn(), + mockDbSelect: vi.fn(), + mockDbUpdate: vi.fn(), + mockHasImapEnvironmentReferences: vi.fn(), + mockLogger: { info: vi.fn(), warn: vi.fn(), error: vi.fn() }, + mockNormalizeLiteralImapConnection: vi.fn(), + mockResolveImapConnectionForActor: vi.fn(), +})) + +vi.mock('@sim/db', () => ({ + db: { select: mockDbSelect, update: mockDbUpdate }, +})) + +vi.mock('@sim/logger', () => ({ + createLogger: () => mockLogger, +})) + +vi.mock('@/lib/imap/connection.server', () => ({ + createSecureImapClient: mockCreateSecureImapClient, + hasImapEnvironmentReferences: mockHasImapEnvironmentReferences, + normalizeLiteralImapConnection: mockNormalizeLiteralImapConnection, + resolveImapConnectionForActor: mockResolveImapConnectionForActor, +})) + +import { imapHandler } from '@/lib/webhooks/providers/imap' + +const referenceConfig = { + host: '{{IMAP_HOST}}', + port: '{{IMAP_PORT}}', + secure: '{{IMAP_SECURE}}', + username: '{{IMAP_USERNAME}}', + password: '{{IMAP_PASSWORD}}', + mailbox: 'INBOX', +} + +describe('IMAP polling deployment policy', () => { + beforeEach(() => { + vi.clearAllMocks() + mockHasImapEnvironmentReferences.mockImplementation((connection: object) => + Object.values(connection).some( + (value) => typeof value === 'string' && /^\{\{[^{}]+\}\}$/.test(value) + ) + ) + mockNormalizeLiteralImapConnection.mockImplementation((connection) => ({ + ...connection, + port: 993, + secure: true, + })) + mockResolveImapConnectionForActor.mockResolvedValue({ + host: 'imap.example.com', + port: 143, + secure: false, + username: 'resolved-user', + password: 'resolved-password', + }) + mockCreateSecureImapClient.mockResolvedValue({ close: mockClose }) + mockDbLimit.mockResolvedValue([{ createdBy: 'deployment-actor' }]) + mockDbSelect.mockReturnValue({ + from: vi.fn().mockReturnValue({ + where: vi.fn().mockReturnValue({ limit: mockDbLimit }), + }), + }) + }) + + it('validates references JIT but persists the unresolved reference expressions', async () => { + const persistProviderConfig = vi.fn().mockResolvedValue(true) + + await expect( + imapHandler.configurePolling!({ + webhook: { id: 'webhook-1', providerConfig: referenceConfig }, + requestId: 'request-1', + userId: 'actor-1', + workspaceId: 'workspace-1', + deploymentVersionId: 'deployment-1', + persistProviderConfig, + }) + ).resolves.toBe(true) + + expect(mockResolveImapConnectionForActor).toHaveBeenCalledWith({ + connection: referenceConfig, + actorUserId: 'deployment-actor', + workspaceId: 'workspace-1', + }) + expect(mockCreateSecureImapClient).toHaveBeenCalledWith( + expect.objectContaining({ username: 'resolved-user', password: 'resolved-password' }) + ) + expect(mockClose).toHaveBeenCalledOnce() + + const persisted = persistProviderConfig.mock.calls[0]?.[0] + expect(persisted).toMatchObject(referenceConfig) + expect(persisted.secure).toBe('{{IMAP_SECURE}}') + expect(persisted.port).toBe('{{IMAP_PORT}}') + expect(JSON.stringify(persisted)).not.toContain('resolved-password') + expect(mockDbUpdate).not.toHaveBeenCalled() + }) + + it('rejects reference-backed legacy setup without a deployment actor', async () => { + await expect( + imapHandler.configurePolling!({ + webhook: { id: 'webhook-1', providerConfig: referenceConfig }, + requestId: 'request-1', + userId: 'actor-1', + workspaceId: 'workspace-1', + deploymentVersionId: null, + }) + ).resolves.toBe(false) + + expect(mockDbSelect).not.toHaveBeenCalled() + expect(mockResolveImapConnectionForActor).not.toHaveBeenCalled() + expect(mockCreateSecureImapClient).not.toHaveBeenCalled() + }) + + it('persists legacy defaults when nullable port and secure values are supplied', async () => { + const persistProviderConfig = vi.fn().mockResolvedValue(true) + + await expect( + imapHandler.configurePolling!({ + webhook: { + id: 'webhook-1', + providerConfig: { + host: 'imap.example.com', + port: null, + secure: null, + username: 'literal-user', + password: 'literal-password', + }, + }, + requestId: 'request-1', + userId: 'actor-1', + workspaceId: 'workspace-1', + persistProviderConfig, + }) + ).resolves.toBe(true) + + expect(persistProviderConfig).toHaveBeenCalledWith( + expect.objectContaining({ port: '993', secure: true }) + ) + expect(mockDbSelect).not.toHaveBeenCalled() + }) + + it('fails closed without logging raw connection errors or authentication values', async () => { + mockCreateSecureImapClient.mockRejectedValue( + new Error('provider echoed literal-user and literal-password') + ) + + await expect( + imapHandler.configurePolling!({ + webhook: { + id: 'webhook-1', + providerConfig: { + host: 'imap.example.com', + username: 'literal-user', + password: 'literal-password', + }, + }, + requestId: 'request-1', + userId: 'actor-1', + workspaceId: 'workspace-1', + }) + ).resolves.toBe(false) + + const logged = JSON.stringify(mockLogger.error.mock.calls) + expect(logged).not.toContain('literal-user') + expect(logged).not.toContain('literal-password') + expect(logged).not.toContain('provider echoed') + expect(mockNormalizeLiteralImapConnection).toHaveBeenCalledOnce() + expect(mockCreateSecureImapClient).toHaveBeenCalledOnce() + expect(mockDbUpdate).not.toHaveBeenCalled() + }) +}) diff --git a/apps/sim/lib/webhooks/providers/imap.ts b/apps/sim/lib/webhooks/providers/imap.ts index 201af195116..c4f3e12a158 100644 --- a/apps/sim/lib/webhooks/providers/imap.ts +++ b/apps/sim/lib/webhooks/providers/imap.ts @@ -1,7 +1,13 @@ import { db } from '@sim/db' -import { webhook } from '@sim/db/schema' +import { webhook, workflowDeploymentVersion } from '@sim/db/schema' import { createLogger } from '@sim/logger' import { eq } from 'drizzle-orm' +import { + createSecureImapClient, + hasImapEnvironmentReferences, + normalizeLiteralImapConnection, + resolveImapConnectionForActor, +} from '@/lib/imap/connection.server' import type { FormatInputContext, FormatInputResult, @@ -39,6 +45,9 @@ export const imapHandler: WebhookProviderHandler = { async configurePolling({ webhook: webhookData, requestId, + userId, + workspaceId, + deploymentVersionId, persistProviderConfig, }: PollingConfigContext) { logger.info(`[${requestId}] Setting up IMAP polling for webhook ${webhookData.id}`) @@ -54,10 +63,53 @@ export const imapHandler: WebhookProviderHandler = { return false } + const connection = providerConfig as { + host: string + username: string + password: string + port?: string | number + secure?: boolean + } + const hasReferences = hasImapEnvironmentReferences(connection) + let deploymentActorUserId = userId + if (hasReferences) { + if (!deploymentVersionId) { + throw new Error('Referenced IMAP authentication requires redeployment') + } + const [deployment] = await db + .select({ createdBy: workflowDeploymentVersion.createdBy }) + .from(workflowDeploymentVersion) + .where(eq(workflowDeploymentVersion.id, deploymentVersionId)) + .limit(1) + if (!deployment?.createdBy) { + throw new Error('Referenced IMAP authentication requires redeployment') + } + deploymentActorUserId = deployment.createdBy + } + const resolved = hasReferences + ? await resolveImapConnectionForActor({ + connection, + actorUserId: deploymentActorUserId, + workspaceId, + }) + : normalizeLiteralImapConnection(connection) + const client = await createSecureImapClient(resolved) + client.close() + const configuredProviderConfig = { ...providerConfig, - port: providerConfig.port || '993', - secure: providerConfig.secure !== false, + port: + providerConfig.port === null || + providerConfig.port === undefined || + providerConfig.port === '' + ? '993' + : providerConfig.port, + secure: + providerConfig.secure === null || + providerConfig.secure === undefined || + providerConfig.secure === '' + ? true + : providerConfig.secure, mailbox: providerConfig.mailbox || 'INBOX', searchCriteria: providerConfig.searchCriteria || 'UNSEEN', markAsRead: providerConfig.markAsRead || false, @@ -78,11 +130,9 @@ export const imapHandler: WebhookProviderHandler = { `[${requestId}] Successfully configured IMAP polling for webhook ${webhookData.id}` ) return true - } catch (error: unknown) { - const err = error as Error + } catch { logger.error(`[${requestId}] Failed to configure IMAP polling`, { webhookId: webhookData.id, - error: err.message, }) return false } diff --git a/apps/sim/lib/webhooks/providers/microsoft-teams.ts b/apps/sim/lib/webhooks/providers/microsoft-teams.ts index 4e35dba8467..98dfe9830a2 100644 --- a/apps/sim/lib/webhooks/providers/microsoft-teams.ts +++ b/apps/sim/lib/webhooks/providers/microsoft-teams.ts @@ -94,7 +94,7 @@ async function fetchWithDNSPinning( requestId: string ): Promise { try { - const urlValidation = await validateUrlWithDNS(url, 'contentUrl') + const urlValidation = await validateUrlWithDNS(url, 'contentUrl', 'contentFetch') if (!urlValidation.isValid) { logger.warn(`[${requestId}] Invalid content URL: ${urlValidation.error}`, { url }) return null @@ -103,7 +103,10 @@ async function fetchWithDNSPinning( if (accessToken) { headers.Authorization = `Bearer ${accessToken}` } - const response = await secureFetchWithPinnedIP(url, urlValidation.resolvedIP!, { headers }) + const response = await secureFetchWithPinnedIP(url, urlValidation.resolvedIP, { + profile: 'contentFetch', + headers, + }) return response } catch (error) { logger.error(`[${requestId}] Error fetching URL with DNS pinning`, { diff --git a/apps/sim/lib/webhooks/providers/registry.ts b/apps/sim/lib/webhooks/providers/registry.ts index 0cad20b8d12..a9531f5c4ea 100644 --- a/apps/sim/lib/webhooks/providers/registry.ts +++ b/apps/sim/lib/webhooks/providers/registry.ts @@ -11,6 +11,7 @@ import { circlebackHandler } from '@/lib/webhooks/providers/circleback' import { clerkHandler } from '@/lib/webhooks/providers/clerk' import { clickupHandler } from '@/lib/webhooks/providers/clickup' import { confluenceHandler } from '@/lib/webhooks/providers/confluence' +import { credentialGroupProviderHandler } from '@/lib/webhooks/providers/credential-group' import { emailBisonHandler } from '@/lib/webhooks/providers/emailbison' import { fathomHandler } from '@/lib/webhooks/providers/fathom' import { firefliesHandler } from '@/lib/webhooks/providers/fireflies' @@ -78,6 +79,7 @@ const PROVIDER_HANDLERS: Record = { clerk: clerkHandler, clickup: clickupHandler, confluence: confluenceHandler, + 'credential-group': credentialGroupProviderHandler, emailbison: emailBisonHandler, fireflies: firefliesHandler, generic: genericHandler, diff --git a/apps/sim/lib/webhooks/providers/slack.test.ts b/apps/sim/lib/webhooks/providers/slack.test.ts index 9afe0841769..26d57933fa9 100644 --- a/apps/sim/lib/webhooks/providers/slack.test.ts +++ b/apps/sim/lib/webhooks/providers/slack.test.ts @@ -52,6 +52,122 @@ describe('slackHandler formatInput - Events API', () => { expect(event.action_value).toBe('') expect(event.actions).toEqual([]) }) + + it('maps an agent_session_stopped event', async () => { + const { input } = await slackHandler.formatInput!( + ctx({ + team_id: 'T1', + event_id: 'Ev2', + event: { + type: 'agent_session_stopped', + channel: 'D1', + user: 'U1', + thread_ts: '111.000', + event_ts: '112.000', + streaming_message_ts: ['111.001', '111.002'], + }, + }) + ) + expect(eventOf(input)).toMatchObject({ + event_type: 'agent_session_stopped', + channel: 'D1', + user: 'U1', + thread_ts: '111.000', + timestamp: '112.000', + streaming_message_ts: ['111.001', '111.002'], + team_id: 'T1', + }) + }) + + it('maps the nested assistant_thread_started reply target', async () => { + const { input } = await slackHandler.formatInput!( + ctx({ + team_id: 'T-install', + event: { + type: 'assistant_thread_started', + assistant_thread: { + channel_id: 'C1', + user_id: 'U1', + thread_ts: '111.000', + context: { team_id: 'T-user' }, + }, + }, + }) + ) + expect(eventOf(input)).toMatchObject({ + event_type: 'assistant_thread_started', + channel: 'C1', + user: 'U1', + thread_ts: '111.000', + team_id: 'T-install', + user_team_id: 'T-user', + }) + }) + + it('maps an agent_session_title_changed event and the Agent View tab', async () => { + const titleChanged = await slackHandler.formatInput!( + ctx({ + event: { + type: 'agent_session_title_changed', + channel: 'D1', + user: 'U1', + thread_ts: '111.000', + event_ts: '112.000', + team_id: 'T1', + enterprise_id: 'E1', + title: 'New title', + previous_title: 'Old title', + }, + }) + ) + expect(eventOf(titleChanged.input)).toMatchObject({ + event_type: 'agent_session_title_changed', + title: 'New title', + previous_title: 'Old title', + team_id: 'T1', + enterprise_id: 'E1', + }) + + const appHome = await slackHandler.formatInput!( + ctx({ event: { type: 'app_home_opened', user: 'U1', tab: 'messages' } }) + ) + expect(eventOf(appHome.input).tab).toBe('messages') + }) + + it('maps app_context_changed and normalizes message.im app_context', async () => { + const contextChanged = await slackHandler.formatInput!( + ctx({ + event: { + type: 'app_context_changed', + user: 'U1', + context: { + entities: [{ type: 'slack#/types/channel_id', value: 'C1', team_id: 'T1' }], + }, + }, + }) + ) + expect(eventOf(contextChanged.input)).toMatchObject({ + event_type: 'app_context_changed', + context: { + entities: [{ type: 'slack#/types/channel_id', value: 'C1', team_id: 'T1' }], + }, + }) + expect(resolveSlackEventKey({ event: { type: 'app_context_changed', context: {} } })).toBe( + 'app_context_changed' + ) + + const directMessage = await slackHandler.formatInput!( + ctx({ + event: { + type: 'message', + channel: 'D1', + channel_type: 'im', + app_context: { entities: [] }, + }, + }) + ) + expect(eventOf(directMessage.input).context).toEqual({ entities: [] }) + }) }) describe('slackHandler formatInput - interactivity (block_actions)', () => { @@ -511,6 +627,39 @@ describe('resolveSlackEventKey - interactions', () => { }) }) +describe('shouldSkipSlackTriggerEvent - slash commands', () => { + const slashCommand = { + command: '/ask-sim', + text: 'Summarize this channel', + team_id: 'T1', + channel_id: 'C1', + user_id: 'U1', + } + + it('maps slash command payloads to the selectable trigger event', () => { + expect(resolveSlackEventKey(slashCommand)).toBe('slash_command') + }) + + it('fires for any command when no command filter is set', () => { + expect(shouldSkipSlackTriggerEvent(slashCommand, { eventType: 'slash_command' })).toBe(false) + }) + + it('matches the exact configured command', () => { + expect( + shouldSkipSlackTriggerEvent(slashCommand, { + eventType: 'slash_command', + commandFilter: '/ask-sim', + }) + ).toBe(false) + expect( + shouldSkipSlackTriggerEvent(slashCommand, { + eventType: 'slash_command', + commandFilter: '/deploy', + }) + ).toBe(true) + }) +}) + /** True when an interaction (top-level payload, no event envelope) fires. */ function interactionFires(config: Record, body: Record): boolean { return !shouldSkipSlackTriggerEvent( diff --git a/apps/sim/lib/webhooks/providers/slack.ts b/apps/sim/lib/webhooks/providers/slack.ts index ba9e20b93ef..406181794b7 100644 --- a/apps/sim/lib/webhooks/providers/slack.ts +++ b/apps/sim/lib/webhooks/providers/slack.ts @@ -84,7 +84,14 @@ interface SlackTriggerEvent { text: string timestamp: string thread_ts: string + streaming_message_ts: string[] + title: string + previous_title: string + tab: string + context: Record | null team_id: string + user_team_id: string + enterprise_id: string event_id: string reaction: string item_user: string @@ -134,7 +141,14 @@ function createSlackEvent(): SlackTriggerEvent { text: '', timestamp: '', thread_ts: '', + streaming_message_ts: [], + title: '', + previous_title: '', + tab: '', + context: null, team_id: '', + user_team_id: '', + enterprise_id: '', event_id: '', reaction: '', item_user: '', @@ -341,7 +355,7 @@ async function downloadSlackFiles( } try { - const urlValidation = await validateUrlWithDNS(urlPrivate, 'url_private') + const urlValidation = await validateUrlWithDNS(urlPrivate, 'url_private', 'contentFetch') if (!urlValidation.isValid) { logger.warn('Slack file url_private failed DNS validation, skipping', { fileId: f.id, @@ -350,7 +364,8 @@ async function downloadSlackFiles( continue } - const response = await secureFetchWithPinnedIP(urlPrivate, urlValidation.resolvedIP!, { + const response = await secureFetchWithPinnedIP(urlPrivate, urlValidation.resolvedIP, { + profile: 'contentFetch', headers: { Authorization: `Bearer ${botToken}` }, }) @@ -578,6 +593,10 @@ const CONTENT_MESSAGE_SUBTYPES = new Set([ * fans out to `message` / `message_edited` / `message_deleted` by subtype. */ export function resolveSlackEventKey(body: Record): string | null { + if (typeof body.command === 'string' && body.command.startsWith('/')) { + return 'slash_command' + } + const event = body.event as Record | undefined if (!event) { // Interactivity payloads (button clicks, modal submits) have no `event` @@ -605,6 +624,9 @@ export function resolveSlackEventKey(body: Record): string | nu case 'pin_removed': case 'team_join': case 'app_home_opened': + case 'agent_session_stopped': + case 'agent_session_title_changed': + case 'app_context_changed': case 'assistant_thread_started': case 'assistant_thread_context_changed': return type @@ -765,6 +787,12 @@ export function shouldSkipSlackTriggerEvent( } } + if (supports('command')) { + const expectedCommand = + typeof providerConfig.commandFilter === 'string' ? providerConfig.commandFilter.trim() : '' + if (expectedCommand && body.command !== expectedCommand) return true + } + // Channels — picker or manual IDs, the basic/advanced sides of one canonical // field. DMs always skip it: a DM's channel can't be picked, so a DM allowed // by Source must not be dropped by a channel filter meant for real channels. @@ -936,7 +964,14 @@ export const slackHandler: WebhookProviderHandler = { const isReactionEvent = SLACK_REACTION_EVENTS.has(eventType) const item = rawEvent?.item as Record | undefined - const channel: string = resolveSlackEventChannel(rawEvent) || '' + const assistantThread = isRecordLike(rawEvent?.assistant_thread) + ? rawEvent.assistant_thread + : undefined + const assistantContext = isRecordLike(assistantThread?.context) + ? assistantThread.context + : undefined + const channel: string = + resolveSlackEventChannel(rawEvent) || asString(assistantThread?.channel_id) const messageTs: string = isReactionEvent ? (item?.ts as string) || '' : (rawEvent?.ts as string) || (rawEvent?.event_ts as string) || '' @@ -961,12 +996,30 @@ export const slackHandler: WebhookProviderHandler = { event.subtype = asString(rawEvent?.subtype) event.channel = channel event.channel_type = asString(rawEvent?.channel_type) - event.user = asString(rawEvent?.user) + event.user = asString(rawEvent?.user) || asString(assistantThread?.user_id) event.bot_id = asString(rawEvent?.bot_id) event.text = text event.timestamp = messageTs - event.thread_ts = asString(rawEvent?.thread_ts) - event.team_id = asString(b?.team_id) || asString(rawEvent?.team) + event.thread_ts = asString(rawEvent?.thread_ts) || asString(assistantThread?.thread_ts) + event.streaming_message_ts = Array.isArray(rawEvent?.streaming_message_ts) + ? rawEvent.streaming_message_ts.filter((value): value is string => typeof value === 'string') + : [] + event.title = asString(rawEvent?.title) + event.previous_title = asString(rawEvent?.previous_title) + event.tab = asString(rawEvent?.tab) + event.context = isRecordLike(rawEvent?.context) + ? rawEvent.context + : isRecordLike(rawEvent?.app_context) + ? rawEvent.app_context + : null + event.team_id = + asString(b?.team_id) || + asString(rawEvent?.team_id) || + asString(rawEvent?.team) || + asString(assistantContext?.team_id) + event.user_team_id = + asString(rawEvent?.user_team) || asString(assistantContext?.team_id) || event.team_id + event.enterprise_id = asString(rawEvent?.enterprise_id) || asString(b?.enterprise_id) event.event_id = asString(b?.event_id) event.api_app_id = asString(b?.api_app_id) event.app_id = diff --git a/apps/sim/lib/webhooks/providers/types.ts b/apps/sim/lib/webhooks/providers/types.ts index 64da7506f3c..bc239f3ce3e 100644 --- a/apps/sim/lib/webhooks/providers/types.ts +++ b/apps/sim/lib/webhooks/providers/types.ts @@ -85,6 +85,10 @@ export interface DeleteSubscriptionContext { export interface PollingConfigContext { webhook: Record requestId: string + /** Deployment actor used to validate reference-backed polling configuration. */ + userId: string + workspaceId: string | null + deploymentVersionId?: string | null /** * Stable registration preparation supplies a generation-fenced persistence callback. * Legacy callers omit it and retain the existing provider-owned write behavior. diff --git a/apps/sim/lib/webhooks/providers/zoom.ts b/apps/sim/lib/webhooks/providers/zoom.ts index 677b39aa9cb..0052dbd39e5 100644 --- a/apps/sim/lib/webhooks/providers/zoom.ts +++ b/apps/sim/lib/webhooks/providers/zoom.ts @@ -7,7 +7,7 @@ import { isRecordLike } from '@sim/utils/object' import { and, eq } from 'drizzle-orm' import type { NextRequest } from 'next/server' import { NextResponse } from 'next/server' -import { resolveEnvVarsInObject } from '@/lib/webhooks/env-resolver' +import { resolveBackgroundWebhookEnv, resolveEnvVarsInObject } from '@/lib/webhooks/env-resolver' import type { AuthContext, EventMatchContext, @@ -73,10 +73,18 @@ async function resolveZoomChallengeSecrets( : {} try { + /** + * Two identities, because a failed challenge is not a failed delivery: + * Zoom deactivates the endpoint outright when URL validation does not + * answer, so an owner who left the workspace would take the webhook down + * at the provider rather than drop one request. + */ + const envVars = await resolveBackgroundWebhookEnv(row.userId, row.workspaceId ?? undefined) const config = await resolveEnvVarsInObject( rawConfig, row.userId, - row.workspaceId ?? undefined + row.workspaceId ?? undefined, + { envVars } ) const secretToken = typeof config.secretToken === 'string' ? config.secretToken : '' return { secretToken } diff --git a/apps/sim/lib/webhooks/registration-service.ts b/apps/sim/lib/webhooks/registration-service.ts index 4a650143049..a32ab8932dd 100644 --- a/apps/sim/lib/webhooks/registration-service.ts +++ b/apps/sim/lib/webhooks/registration-service.ts @@ -163,6 +163,10 @@ async function createCandidateProviderState( const configured = await handler.configurePolling({ webhook: { ...webhookData, providerConfig }, requestId: input.requestId, + userId: input.userId, + workspaceId: + typeof input.workflow.workspaceId === 'string' ? input.workflow.workspaceId : null, + deploymentVersionId: input.fence.deploymentVersionId, persistProviderConfig: async (configuredProviderConfig) => { persistedProviderConfig = configuredProviderConfig await dependencies.checkpointCandidate({ diff --git a/apps/sim/lib/webhooks/slack-agent-api.test.ts b/apps/sim/lib/webhooks/slack-agent-api.test.ts new file mode 100644 index 00000000000..833816bc9e5 --- /dev/null +++ b/apps/sim/lib/webhooks/slack-agent-api.test.ts @@ -0,0 +1,182 @@ +/** + * @vitest-environment node + */ +import { afterEach, describe, expect, it, vi } from 'vitest' +import { + appendSlackAgentStream, + setSlackAgentSessionStatus, + startSlackAgentStream, + stopSlackAgentStream, +} from '@/lib/webhooks/slack-agent-api' + +afterEach(() => { + vi.unstubAllGlobals() +}) + +describe('Slack agent API transport', () => { + it('starts a structured stream with the reply recipient and task display mode', async () => { + const fetchMock = vi.fn().mockResolvedValue( + new Response(JSON.stringify({ ok: true, channel: 'C1', ts: '101.2' }), { + status: 200, + }) + ) + vi.stubGlobal('fetch', fetchMock) + + await expect( + startSlackAgentStream( + 'xoxb-test', + { + channel: 'C1', + threadTs: '100.1', + initiatorUserId: 'U1', + recipientUserId: 'U1', + recipientTeamId: 'T1', + }, + [{ type: 'markdown_text', text: 'Hello' }], + 'plan' + ) + ).resolves.toEqual({ channel: 'C1', ts: '101.2' }) + + expect(fetchMock).toHaveBeenCalledWith( + 'https://slack.com/api/chat.startStream', + expect.objectContaining({ + method: 'POST', + body: JSON.stringify({ + channel: 'C1', + thread_ts: '100.1', + chunks: [{ type: 'markdown_text', text: 'Hello' }], + task_display_mode: 'plan', + recipient_user_id: 'U1', + recipient_team_id: 'T1', + }), + }) + ) + }) + + it('uses the documented append, stop, and session status bodies', async () => { + const fetchMock = vi.fn().mockImplementation(async (url: string) => { + const body = url.endsWith('agents.sessions.setStatus') + ? { ok: true, status: 'processing', agent_status: 'active' } + : { ok: true, channel: 'D1', ts: '101.2' } + return new Response(JSON.stringify(body), { status: 200 }) + }) + vi.stubGlobal('fetch', fetchMock) + + await appendSlackAgentStream('xoxb-test', 'D1', '101.2', [ + { type: 'markdown_text', text: 'More' }, + ]) + await stopSlackAgentStream('xoxb-test', 'D1', '101.2', 'processing') + await setSlackAgentSessionStatus('xoxb-test', { channel: 'D1', threadTs: '100.1' }, 'active') + + expect(JSON.parse(fetchMock.mock.calls[0][1].body)).toEqual({ + channel: 'D1', + ts: '101.2', + chunks: [{ type: 'markdown_text', text: 'More' }], + }) + expect(JSON.parse(fetchMock.mock.calls[1][1].body)).toEqual({ + channel: 'D1', + ts: '101.2', + session_status: 'processing', + }) + expect(JSON.parse(fetchMock.mock.calls[2][1].body)).toEqual({ + channel_id: 'D1', + thread_ts: '100.1', + status: 'active', + }) + }) + + it('sets the human initiator when creating a processing session', async () => { + const fetchMock = vi + .fn() + .mockResolvedValue( + new Response( + JSON.stringify({ ok: true, status: 'processing', agent_status: 'processing' }), + { status: 200 } + ) + ) + vi.stubGlobal('fetch', fetchMock) + + await setSlackAgentSessionStatus( + 'xoxb-test', + { channel: 'D1', threadTs: '100.1', initiatorUserId: 'U1' }, + 'processing' + ) + + expect(JSON.parse(fetchMock.mock.calls[0][1].body)).toEqual({ + channel_id: 'D1', + thread_ts: '100.1', + status: 'processing', + initiator_user_id: 'U1', + }) + }) + + it('fails fast on Slack logical errors', async () => { + vi.stubGlobal( + 'fetch', + vi.fn().mockResolvedValue( + new Response(JSON.stringify({ ok: false, error: 'missing_scope' }), { + status: 200, + }) + ) + ) + + await expect( + setSlackAgentSessionStatus( + 'xoxb-test', + { channel: 'D1', threadTs: '100.1', initiatorUserId: 'U1' }, + 'processing' + ) + ).rejects.toThrow('missing_scope') + }) + + it('fails fast when Slack does not recognize the stop-event subscription', async () => { + vi.stubGlobal( + 'fetch', + vi.fn().mockResolvedValue( + new Response( + JSON.stringify({ + ok: true, + status: 'processing', + agent_status: 'processing', + response_metadata: { + warnings: ['missing_agent_session_stopped_event_subscription'], + }, + }), + { status: 200 } + ) + ) + ) + + await expect( + setSlackAgentSessionStatus( + 'xoxb-test', + { channel: 'D1', threadTs: '100.1', initiatorUserId: 'U1' }, + 'processing' + ) + ).rejects.toThrow('missing_agent_session_stopped_event_subscription') + }) + + it('fails fast when Slack does not confirm the requested session status', async () => { + vi.stubGlobal( + 'fetch', + vi.fn().mockResolvedValue( + new Response( + JSON.stringify({ + ok: true, + status: 'active', + agent_status: 'active', + }), + { status: 200 } + ) + ) + ) + + await expect( + setSlackAgentSessionStatus( + 'xoxb-test', + { channel: 'D1', threadTs: '100.1', initiatorUserId: 'U1' }, + 'processing' + ) + ).rejects.toThrow('expected processing') + }) +}) diff --git a/apps/sim/lib/webhooks/slack-agent-api.ts b/apps/sim/lib/webhooks/slack-agent-api.ts new file mode 100644 index 00000000000..a9a3a31d487 --- /dev/null +++ b/apps/sim/lib/webhooks/slack-agent-api.ts @@ -0,0 +1,158 @@ +import { getErrorMessage } from '@sim/utils/errors' +import { isRecordLike } from '@sim/utils/object' + +export type SlackStreamChunk = + | { type: 'markdown_text'; text: string } + | { + type: 'task_update' + id: string + title: string + status: 'in_progress' | 'complete' | 'error' + details?: string + output?: string + } + +interface SlackApiResponse { + ok?: boolean + error?: string + channel?: string + ts?: string + status?: string + agent_status?: string + response_metadata?: unknown +} + +interface SlackStreamTarget { + channel: string + threadTs: string + initiatorUserId?: string + recipientUserId?: string + recipientTeamId?: string +} + +async function callSlackAgentApi( + method: string, + token: string, + body: Record, + signal?: AbortSignal +): Promise { + const response = await fetch(`https://slack.com/api/${method}`, { + method: 'POST', + headers: { + Authorization: `Bearer ${token}`, + 'Content-Type': 'application/json; charset=utf-8', + }, + body: JSON.stringify(body), + signal, + }) + const value = (await response.json()) as unknown + if (!isRecordLike(value)) { + throw new Error(`Slack ${method} returned an invalid response`) + } + const data = value as SlackApiResponse + if (!response.ok || data.ok !== true) { + throw new Error( + data.error + ? `Slack ${method} failed: ${data.error}` + : `Slack ${method} failed with status ${response.status}` + ) + } + return data +} + +export async function setSlackAgentSessionStatus( + token: string, + target: Pick, + status: 'active' | 'processing' | 'suspended', + signal?: AbortSignal +): Promise { + const data = await callSlackAgentApi( + 'agents.sessions.setStatus', + token, + { + channel_id: target.channel, + thread_ts: target.threadTs, + status, + ...(target.initiatorUserId ? { initiator_user_id: target.initiatorUserId } : {}), + }, + signal + ) + + const responseMetadata = data.response_metadata + if (responseMetadata !== undefined && !isRecordLike(responseMetadata)) { + throw new Error('Slack agents.sessions.setStatus returned invalid response metadata') + } + const warnings = responseMetadata?.warnings + if ( + warnings !== undefined && + (!Array.isArray(warnings) || warnings.some((warning) => typeof warning !== 'string')) + ) { + throw new Error('Slack agents.sessions.setStatus returned invalid warnings') + } + if (warnings?.includes('missing_agent_session_stopped_event_subscription')) { + throw new Error( + 'Slack agents.sessions.setStatus warning: missing_agent_session_stopped_event_subscription' + ) + } + if (data.agent_status !== status) { + throw new Error( + `Slack agents.sessions.setStatus returned agent_status=${String(data.agent_status)}; expected ${status}` + ) + } +} + +export async function startSlackAgentStream( + token: string, + target: SlackStreamTarget, + chunks: SlackStreamChunk[], + taskDisplayMode: 'timeline' | 'plan', + signal?: AbortSignal +): Promise<{ channel: string; ts: string }> { + const data = await callSlackAgentApi( + 'chat.startStream', + token, + { + channel: target.channel, + thread_ts: target.threadTs, + chunks, + task_display_mode: taskDisplayMode, + ...(target.recipientUserId ? { recipient_user_id: target.recipientUserId } : {}), + ...(target.recipientTeamId ? { recipient_team_id: target.recipientTeamId } : {}), + }, + signal + ) + if (typeof data.channel !== 'string' || typeof data.ts !== 'string') { + throw new Error('Slack chat.startStream response is missing channel or timestamp') + } + return { channel: data.channel, ts: data.ts } +} + +export async function appendSlackAgentStream( + token: string, + channel: string, + ts: string, + chunks: SlackStreamChunk[], + signal?: AbortSignal +): Promise { + if (chunks.length === 0) return + await callSlackAgentApi('chat.appendStream', token, { channel, ts, chunks }, signal) +} + +export async function stopSlackAgentStream( + token: string, + channel: string, + ts: string, + sessionStatus: 'active' | 'processing' | 'suspended', + signal?: AbortSignal +): Promise { + await callSlackAgentApi( + 'chat.stopStream', + token, + { channel, ts, session_status: sessionStatus }, + signal + ) +} + +export function formatSlackApiFailure(error: unknown): Error { + return new Error(getErrorMessage(error, 'Slack agent response delivery failed')) +} diff --git a/apps/sim/lib/webhooks/slack-custom-ingress.test.ts b/apps/sim/lib/webhooks/slack-custom-ingress.test.ts index 23a11c59b6f..d0e1d826a3d 100644 --- a/apps/sim/lib/webhooks/slack-custom-ingress.test.ts +++ b/apps/sim/lib/webhooks/slack-custom-ingress.test.ts @@ -1,8 +1,36 @@ /** * @vitest-environment node */ -import { describe, expect, it } from 'vitest' -import { getLegacySlackCustomBotCredentialId } from '@/lib/webhooks/slack-custom-ingress' +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const mocks = vi.hoisted(() => ({ + cancel: vi.fn(), + getCredential: vi.fn(), + listSessions: vi.fn(), + unregister: vi.fn(), + setStatus: vi.fn(), +})) + +vi.mock('@/lib/execution/cancel-workflow-execution', () => ({ + cancelWorkflowExecution: mocks.cancel, +})) +vi.mock('@/lib/oauth/credential-service', () => ({ + getSlackBotCredential: mocks.getCredential, +})) +vi.mock('@/lib/webhooks/slack-agent-api', () => ({ + setSlackAgentSessionStatus: mocks.setStatus, +})) +vi.mock('@/lib/webhooks/slack-stream-sessions', () => ({ + resolveStoppedSlackSession: (body: Record) => + body.kind === 'stop' ? { channel: 'D1', threadTs: '100.1' } : null, + listSlackStreamSessions: mocks.listSessions, + unregisterSlackStreamSession: mocks.unregister, +})) + +import { + getLegacySlackCustomBotCredentialId, + handleSlackAgentSessionStopped, +} from '@/lib/webhooks/slack-custom-ingress' function webhook(overrides: Record = {}) { return { @@ -37,3 +65,68 @@ describe('getLegacySlackCustomBotCredentialId', () => { ).toThrow(/routing key does not match/) }) }) + +describe('handleSlackAgentSessionStopped', () => { + beforeEach(() => { + vi.clearAllMocks() + mocks.listSessions.mockResolvedValue([]) + mocks.getCredential.mockResolvedValue({ botToken: 'xoxb-test' }) + }) + + it('cancels every active execution and returns the session to active', async () => { + mocks.listSessions.mockResolvedValue([ + { + executionId: 'execution-1', + workflowId: 'workflow-1', + userId: 'user-1', + workspaceId: 'workspace-1', + }, + { + executionId: 'execution-2', + workflowId: 'workflow-2', + userId: 'user-2', + workspaceId: 'workspace-1', + }, + ]) + + await handleSlackAgentSessionStopped('credential-1', { kind: 'stop' }) + + expect(mocks.cancel).toHaveBeenCalledTimes(2) + expect(mocks.cancel).toHaveBeenNthCalledWith(1, { + executionId: 'execution-1', + workflowId: 'workflow-1', + attributedUserId: 'user-1', + workspaceId: 'workspace-1', + }) + expect(mocks.cancel).toHaveBeenNthCalledWith(2, { + executionId: 'execution-2', + workflowId: 'workflow-2', + attributedUserId: 'user-2', + workspaceId: 'workspace-1', + }) + expect(mocks.unregister).toHaveBeenCalledTimes(2) + expect(mocks.setStatus).toHaveBeenCalledWith( + 'xoxb-test', + { channel: 'D1', threadTs: '100.1' }, + 'active' + ) + }) + + it('keeps the session mapping when cancellation fails so Slack can retry', async () => { + mocks.listSessions.mockResolvedValue([ + { + executionId: 'execution-1', + workflowId: 'workflow-1', + userId: 'user-1', + workspaceId: 'workspace-1', + }, + ]) + mocks.cancel.mockRejectedValue(new Error('cancel failed')) + + await expect(handleSlackAgentSessionStopped('credential-1', { kind: 'stop' })).rejects.toThrow( + 'cancel failed' + ) + expect(mocks.unregister).not.toHaveBeenCalled() + expect(mocks.setStatus).not.toHaveBeenCalled() + }) +}) diff --git a/apps/sim/lib/webhooks/slack-custom-ingress.ts b/apps/sim/lib/webhooks/slack-custom-ingress.ts index b2ba47723a6..77fe062895e 100644 --- a/apps/sim/lib/webhooks/slack-custom-ingress.ts +++ b/apps/sim/lib/webhooks/slack-custom-ingress.ts @@ -1,11 +1,18 @@ import { createLogger } from '@sim/logger' import type { NextRequest } from 'next/server' import { NextResponse } from 'next/server' +import { cancelWorkflowExecution } from '@/lib/execution/cancel-workflow-execution' import { getSlackBotCredential } from '@/lib/oauth/credential-service' import { findWebhooksByRoutingKey, type WebhookDispatchResult } from '@/lib/webhooks/processor' import { verifySlackRequestSignature } from '@/lib/webhooks/providers/slack' +import { setSlackAgentSessionStatus } from '@/lib/webhooks/slack-agent-api' import { LEGACY_SLACK_CUSTOM_BOT_INGRESS_MODE } from '@/lib/webhooks/slack-custom-ingress-constants' import { dispatchSlackWebhooks } from '@/lib/webhooks/slack-dispatch' +import { + listSlackStreamSessions, + resolveStoppedSlackSession, + unregisterSlackStreamSession, +} from '@/lib/webhooks/slack-stream-sessions' const logger = createLogger('SlackCustomBotIngress') @@ -110,3 +117,32 @@ export async function dispatchSlackCustomBotCredential({ return dispatchSlackWebhooks(webhooks, { body, request, requestId, receivedAt }) } + +/** Cancels every workflow currently associated with Slack's stopped agent session. */ +export async function handleSlackAgentSessionStopped( + credentialId: string, + body: unknown +): Promise { + const target = resolveStoppedSlackSession(body) + if (!target) return + + const executions = await listSlackStreamSessions(credentialId, target) + if (executions.length === 0) return + await Promise.all( + executions.map(async (execution) => { + await cancelWorkflowExecution({ + executionId: execution.executionId, + workflowId: execution.workflowId, + attributedUserId: execution.userId, + workspaceId: execution.workspaceId, + }) + await unregisterSlackStreamSession(credentialId, target, execution.executionId) + }) + ) + + const credential = await getSlackBotCredential(credentialId) + if (!credential) { + throw new Error('Slack agent session stop credential is unavailable') + } + await setSlackAgentSessionStatus(credential.botToken, target, 'active') +} diff --git a/apps/sim/lib/webhooks/slack-execution-stream.test.ts b/apps/sim/lib/webhooks/slack-execution-stream.test.ts new file mode 100644 index 00000000000..8f0c5cc812f --- /dev/null +++ b/apps/sim/lib/webhooks/slack-execution-stream.test.ts @@ -0,0 +1,442 @@ +/** + * @vitest-environment node + */ +import { toError } from '@sim/utils/errors' +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const { + mockAppendSlackAgentStream, + mockGetSlackBotCredential, + mockRegisterSlackStreamSession, + mockSetSlackAgentSessionStatus, + mockStartSlackAgentStream, + mockStopSlackAgentStream, + mockUnregisterSlackStreamSession, +} = vi.hoisted(() => ({ + mockAppendSlackAgentStream: vi.fn(), + mockGetSlackBotCredential: vi.fn(), + mockRegisterSlackStreamSession: vi.fn(), + mockSetSlackAgentSessionStatus: vi.fn(), + mockStartSlackAgentStream: vi.fn(), + mockStopSlackAgentStream: vi.fn(), + mockUnregisterSlackStreamSession: vi.fn(), +})) + +vi.mock('@/lib/oauth/credential-service', () => ({ + getSlackBotCredential: mockGetSlackBotCredential, +})) + +vi.mock('@/lib/webhooks/slack-agent-api', () => ({ + appendSlackAgentStream: mockAppendSlackAgentStream, + formatSlackApiFailure: (error: unknown) => toError(error), + setSlackAgentSessionStatus: mockSetSlackAgentSessionStatus, + startSlackAgentStream: mockStartSlackAgentStream, + stopSlackAgentStream: mockStopSlackAgentStream, +})) + +vi.mock('@/lib/webhooks/slack-stream-sessions', () => ({ + registerSlackStreamSession: mockRegisterSlackStreamSession, + unregisterSlackStreamSession: mockUnregisterSlackStreamSession, +})) + +import { SlackExecutionStreamController } from '@/lib/webhooks/slack-execution-stream' +import type { SlackStreamResponseConfig } from '@/lib/webhooks/slack-stream-config' +import type { AgentStreamEvent } from '@/providers/stream-events' + +const BASE_CONFIG: SlackStreamResponseConfig = { + enabled: true, + outputConfigs: [{ blockId: 'agent', path: 'content' }], + includeThinking: true, + includeToolCalls: true, + taskTitle: 'Running', + taskDisplayMode: 'plan', +} + +function createLoggingSession() { + return { + projectLiveDisplayText: vi.fn(async (_key: string, text: string) => ({ + chunk: text, + })), + projectDisplayContent: vi.fn(async (content: Record) => content), + } +} + +function createByteStream(text = ''): ReadableStream { + return new ReadableStream({ + start(controller) { + if (text) controller.enqueue(new TextEncoder().encode(text)) + controller.close() + }, + }) +} + +function createOpenByteStream(): { + stream: ReadableStream + close: () => void +} { + let closeStream: (() => void) | undefined + const stream = new ReadableStream({ + start(controller) { + closeStream = () => controller.close() + }, + }) + return { + stream, + close: () => { + if (!closeStream) throw new Error('Test stream was not initialized') + closeStream() + }, + } +} + +async function createController( + config: SlackStreamResponseConfig = BASE_CONFIG, + triggerInput: Record = { + event: { + channel: 'C123', + thread_ts: '1700000000.000001', + user: 'U123', + user_team_id: 'T123', + }, + } +) { + const loggingSession = createLoggingSession() + const controller = await SlackExecutionStreamController.create({ + credentialId: 'cred-1', + workspaceId: 'workspace-1', + workflowId: 'workflow-1', + executionId: 'execution-1', + userId: 'user-1', + triggerInput, + config, + loggingSession: loggingSession as never, + }) + return { controller, loggingSession } +} + +describe('SlackExecutionStreamController', () => { + beforeEach(() => { + vi.clearAllMocks() + mockGetSlackBotCredential.mockResolvedValue({ + botToken: 'xoxb-token', + workspaceId: 'workspace-1', + }) + mockStartSlackAgentStream.mockResolvedValue({ + channel: 'C123', + ts: '1700000001.000002', + }) + }) + + it('streams agent text and task events for each selected invocation', async () => { + const { controller } = await createController() + const events: AgentStreamEvent[] = [ + { type: 'thinking_delta', text: 'Checking context' }, + { type: 'tool_call_start', id: 'tool-1', name: 'slack_send_message' }, + { + type: 'tool_call_end', + id: 'tool-1', + name: 'slack_send_message', + status: 'success', + }, + { type: 'tool_call_start', id: 'tool-2', name: 'mcp-6da535c1-ask_question' }, + { + type: 'tool_call_end', + id: 'tool-2', + name: 'mcp-6da535c1-ask_question', + status: 'success', + }, + { type: 'text_delta', text: 'Hello ', turn: 'pending' }, + { type: 'text_delta', text: 'world', turn: 'pending' }, + { type: 'turn_end', turn: 'final' }, + ] + + await controller.callbacks.onStream?.({ + blockId: 'agent', + executionOrder: 4, + stream: createByteStream(), + streamFormat: 'text', + clientStreamTransformed: false, + subscribe: ({ onEvent }) => { + for (const event of events) void onEvent(event) + return vi.fn() + }, + }) + + expect(controller.selectedOutputs).toEqual(['agent_content']) + expect(mockRegisterSlackStreamSession).toHaveBeenCalledWith( + 'cred-1', + { + channel: 'C123', + threadTs: '1700000000.000001', + initiatorUserId: 'U123', + recipientUserId: 'U123', + recipientTeamId: 'T123', + }, + { + executionId: 'execution-1', + workflowId: 'workflow-1', + userId: 'user-1', + workspaceId: 'workspace-1', + } + ) + expect(mockStartSlackAgentStream).toHaveBeenCalledWith( + 'xoxb-token', + { + channel: 'C123', + threadTs: '1700000000.000001', + initiatorUserId: 'U123', + recipientUserId: 'U123', + recipientTeamId: 'T123', + }, + [ + { + type: 'task_update', + id: 'sim-execution-1-4', + title: 'Running', + status: 'in_progress', + }, + ], + 'plan', + undefined + ) + const appendedChunks = mockAppendSlackAgentStream.mock.calls.flatMap((call) => call[3]) + expect(appendedChunks).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + type: 'task_update', + title: 'Thinking', + status: 'complete', + }), + expect.objectContaining({ + type: 'task_update', + title: 'Slack Send Message', + status: 'in_progress', + }), + expect.objectContaining({ + type: 'task_update', + title: 'Slack Send Message', + status: 'complete', + }), + expect.objectContaining({ + type: 'task_update', + title: 'Ask Question', + status: 'in_progress', + }), + expect.objectContaining({ + type: 'task_update', + title: 'Ask Question', + status: 'complete', + }), + { type: 'markdown_text', text: 'Hello ' }, + { type: 'markdown_text', text: 'world' }, + expect.objectContaining({ + type: 'task_update', + id: 'sim-execution-1-4', + status: 'complete', + }), + ]) + ) + expect(mockStopSlackAgentStream).toHaveBeenCalledWith( + 'xoxb-token', + 'C123', + '1700000001.000002', + 'processing', + undefined + ) + + await controller.finalize({ + success: true, + output: {}, + status: 'completed', + }) + controller.assertSucceeded() + + expect(mockSetSlackAgentSessionStatus).toHaveBeenLastCalledWith( + 'xoxb-token', + { + channel: 'C123', + threadTs: '1700000000.000001', + initiatorUserId: 'U123', + recipientUserId: 'U123', + recipientTeamId: 'T123', + }, + 'active' + ) + expect(mockUnregisterSlackStreamSession).toHaveBeenCalledWith( + 'cred-1', + { + channel: 'C123', + threadTs: '1700000000.000001', + initiatorUserId: 'U123', + recipientUserId: 'U123', + recipientTeamId: 'T123', + }, + 'execution-1' + ) + }) + + it('appends pending answer text before the model turn is classified', async () => { + const { controller } = await createController() + const { stream, close } = createOpenByteStream() + const streaming = controller.callbacks.onStream?.({ + blockId: 'agent', + executionOrder: 5, + stream, + streamFormat: 'text', + clientStreamTransformed: false, + subscribe: ({ onEvent }) => { + void onEvent({ + type: 'text_delta', + text: 'Once upon a time', + turn: 'pending', + }) + return vi.fn() + }, + }) + + await vi.waitFor(() => { + expect(mockAppendSlackAgentStream).toHaveBeenCalledWith( + 'xoxb-token', + 'C123', + '1700000001.000002', + [{ type: 'markdown_text', text: 'Once upon a time' }], + undefined + ) + }) + expect(mockStopSlackAgentStream).not.toHaveBeenCalled() + + close() + await streaming + }) + + it('streams transformed answer text with tool and thinking events from the event sink', async () => { + const { controller } = await createController() + const events: AgentStreamEvent[] = [ + { type: 'thinking_delta', text: 'Checking Gmail' }, + { type: 'tool_call_start', id: 'tool-1', name: 'gmail_send_email' }, + { + type: 'tool_call_end', + id: 'tool-1', + name: 'gmail_send_email', + status: 'success', + }, + { type: 'text_delta', text: 'Unselected structured response', turn: 'pending' }, + { type: 'turn_end', turn: 'final' }, + ] + + const subscribe = vi.fn(({ onEvent }) => { + for (const event of events) void onEvent(event) + return vi.fn() + }) + + await controller.callbacks.onStream?.({ + blockId: 'agent', + executionOrder: 6, + stream: createByteStream('Selected answer'), + streamFormat: 'text', + clientStreamTransformed: true, + subscribe, + }) + + expect(subscribe).toHaveBeenCalledOnce() + const appendedChunks = mockAppendSlackAgentStream.mock.calls.flatMap((call) => call[3]) + expect(appendedChunks).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + type: 'task_update', + title: 'Thinking', + status: 'complete', + }), + expect.objectContaining({ + type: 'task_update', + title: 'Gmail Send Email', + status: 'in_progress', + }), + expect.objectContaining({ + type: 'task_update', + title: 'Gmail Send Email', + status: 'complete', + }), + { type: 'markdown_text', text: 'Selected answer' }, + ]) + ) + expect(appendedChunks).not.toContainEqual({ + type: 'markdown_text', + text: 'Unselected structured response', + }) + }) + + it('sends a selected nested non-streaming output after block completion', async () => { + const config: SlackStreamResponseConfig = { + ...BASE_CONFIG, + outputConfigs: [{ workflowId: 'child-workflow', blockId: 'lookup', path: 'result.name' }], + } + const { controller } = await createController(config, { + event: { channel: 'D123', timestamp: '1700000000.000001', user: 'U123' }, + }) + + await controller.callbacks.onBlockComplete?.('lookup', 'Lookup', 'generic', { + output: { result: { name: 'Ada' } }, + executionTime: 10, + startedAt: '2026-08-31T00:00:00.000Z', + executionOrder: 7, + endedAt: '2026-08-31T00:00:00.010Z', + outputBlockId: 'child-workflow.lookup', + childWorkflowInstanceId: 'child-instance-1', + }) + + expect(mockStartSlackAgentStream).toHaveBeenCalledWith( + 'xoxb-token', + { + channel: 'D123', + threadTs: '1700000000.000001', + initiatorUserId: 'U123', + }, + expect.any(Array), + 'plan', + undefined + ) + expect(mockAppendSlackAgentStream).toHaveBeenCalledWith( + 'xoxb-token', + 'C123', + '1700000001.000002', + [{ type: 'markdown_text', text: 'Ada' }], + undefined + ) + }) + + it('keeps repeated invocations of the same child workflow distinct', async () => { + const config: SlackStreamResponseConfig = { + ...BASE_CONFIG, + outputConfigs: [{ workflowId: 'child-workflow', blockId: 'agent', path: 'content' }], + } + const { controller } = await createController(config, { + event: { channel: 'D123', timestamp: '1700000000.000001', user: 'U123' }, + }) + + for (const childWorkflowInstanceId of ['child-instance-1', 'child-instance-2']) { + await controller.callbacks.onStream?.({ + blockId: 'child-workflow.agent', + childWorkflowInstanceId, + executionOrder: 1, + stream: createByteStream(childWorkflowInstanceId), + execution: { success: true, output: {} }, + }) + } + + expect(mockStartSlackAgentStream).toHaveBeenCalledTimes(2) + expect(() => controller.assertSucceeded()).not.toThrow() + }) + + it('rejects credentials that do not belong to the workflow workspace', async () => { + mockGetSlackBotCredential.mockResolvedValue({ + botToken: 'xoxb-token', + workspaceId: 'workspace-2', + }) + + await expect(createController()).rejects.toThrow( + 'Slack streaming credential is unavailable in this workspace' + ) + expect(mockRegisterSlackStreamSession).not.toHaveBeenCalled() + }) +}) diff --git a/apps/sim/lib/webhooks/slack-execution-stream.ts b/apps/sim/lib/webhooks/slack-execution-stream.ts new file mode 100644 index 00000000000..3a9f43b3dbe --- /dev/null +++ b/apps/sim/lib/webhooks/slack-execution-stream.ts @@ -0,0 +1,511 @@ +import { getErrorMessage } from '@sim/utils/errors' +import { isRecordLike } from '@sim/utils/object' +import { truncate } from '@sim/utils/string' +import { getToolDisplayTitle } from '@/lib/copilot/tools/tool-display' +import type { LoggingSession } from '@/lib/logs/execution/logging-session' +import { getSlackBotCredential } from '@/lib/oauth/credential-service' +import { pluckByPath } from '@/lib/table/pluck' +import { + appendSlackAgentStream, + formatSlackApiFailure, + type SlackStreamChunk, + setSlackAgentSessionStatus, + startSlackAgentStream, + stopSlackAgentStream, +} from '@/lib/webhooks/slack-agent-api' +import type { + SlackStreamOutputConfig, + SlackStreamResponseConfig, +} from '@/lib/webhooks/slack-stream-config' +import { + registerSlackStreamSession, + type SlackStreamSessionTarget, + unregisterSlackStreamSession, +} from '@/lib/webhooks/slack-stream-sessions' +import { formatOutputSelector, scopeOutputBlockId } from '@/lib/workflows/streaming/output-selector' +import type { BlockCompletionCallbackData, ExecutionCallbacks } from '@/executor/execution/types' +import type { ExecutionResult, StreamingExecution } from '@/executor/types' +import type { AgentStreamEvent } from '@/providers/stream-events' + +const TEXT_FLUSH_SIZE = 128 +const SLACK_MARKDOWN_LIMIT = 12_000 +const TASK_TEXT_LIMIT = 256 + +interface SlackReplyTarget extends SlackStreamSessionTarget { + initiatorUserId: string + recipientUserId?: string + recipientTeamId?: string +} + +interface SlackExecutionStreamControllerOptions { + credentialId: string + workspaceId: string + workflowId: string + executionId: string + userId: string + triggerInput: Record + config: SlackStreamResponseConfig + loggingSession: LoggingSession + abortSignal?: AbortSignal +} + +function requireEvent(triggerInput: Record): Record { + if (!isRecordLike(triggerInput.event)) { + throw new Error('Slack streaming trigger input is missing its normalized event') + } + return triggerInput.event +} + +export function resolveSlackReplyTarget(triggerInput: Record): SlackReplyTarget { + const event = requireEvent(triggerInput) + if (typeof event.channel !== 'string' || !event.channel) { + throw new Error('Slack streaming trigger event is missing a channel') + } + const threadTs = + typeof event.thread_ts === 'string' && event.thread_ts + ? event.thread_ts + : typeof event.timestamp === 'string' + ? event.timestamp + : '' + if (!threadTs) { + throw new Error('Slack streaming trigger event is missing a thread timestamp') + } + if (typeof event.user !== 'string' || !event.user) { + throw new Error('Slack streaming trigger event is missing the initiator user ID') + } + + if (event.channel.startsWith('D')) { + return { channel: event.channel, threadTs, initiatorUserId: event.user } + } + if (typeof event.user_team_id !== 'string' || !event.user_team_id) { + throw new Error('Slack channel streaming requires the recipient team ID') + } + return { + channel: event.channel, + threadTs, + initiatorUserId: event.user, + recipientUserId: event.user, + recipientTeamId: event.user_team_id, + } +} + +function splitMarkdown(text: string): SlackStreamChunk[] { + const chunks: SlackStreamChunk[] = [] + for (let offset = 0; offset < text.length; offset += SLACK_MARKDOWN_LIMIT) { + chunks.push({ + type: 'markdown_text', + text: text.slice(offset, offset + SLACK_MARKDOWN_LIMIT), + }) + } + return chunks +} + +function formatOutput(value: unknown): string { + if (typeof value === 'string') return value + const serialized = JSON.stringify(value, null, 2) + if (serialized === undefined) throw new Error('Selected Slack stream output is not serializable') + return serialized +} + +class SlackInvocationStream { + private channel?: string + private ts?: string + private answerBuffer = '' + private fullAnswer = '' + private thinking = '' + private emittedAnswer = false + private chain: Promise = Promise.resolve() + + constructor( + private readonly token: string, + private readonly target: SlackReplyTarget, + private readonly config: SlackStreamResponseConfig, + private readonly taskId: string, + private readonly title: string, + private readonly projectLiveText: (text: string) => Promise, + private readonly projectFinalText: (text: string) => Promise, + private readonly signal?: AbortSignal + ) {} + + private enqueue(operation: () => Promise): Promise { + this.chain = this.chain.then(operation) + return this.chain + } + + private async ensureStarted(): Promise { + if (this.channel && this.ts) return + const started = await startSlackAgentStream( + this.token, + this.target, + [ + { + type: 'task_update', + id: this.taskId, + title: this.title, + status: 'in_progress', + }, + ], + this.config.taskDisplayMode, + this.signal + ) + this.channel = started.channel + this.ts = started.ts + } + + private async append(chunks: SlackStreamChunk[]): Promise { + await this.ensureStarted() + await appendSlackAgentStream(this.token, this.channel!, this.ts!, chunks, this.signal) + } + + private async flushAnswer(force: boolean): Promise { + if (!force && this.answerBuffer.length < TEXT_FLUSH_SIZE) return + if (!this.answerBuffer) return + const value = this.answerBuffer + this.answerBuffer = '' + const projected = await this.projectLiveText(value) + if (!projected) return + await this.append(splitMarkdown(projected)) + this.emittedAnswer = true + } + + private async appendAnswer(text: string, force = false): Promise { + if (!text) return + this.fullAnswer += text + this.answerBuffer += text + await this.flushAnswer(force || !this.emittedAnswer) + } + + private async flushThinking(): Promise { + if (!this.config.includeThinking || !this.thinking) return + const value = this.thinking + this.thinking = '' + const projected = await this.projectLiveText(value) + if (!projected) return + await this.append([ + { + type: 'task_update', + id: `${this.taskId}-thinking`, + title: 'Thinking', + status: 'complete', + details: truncate(projected, TASK_TEXT_LIMIT), + }, + ]) + } + + onEvent(event: AgentStreamEvent): Promise { + return this.enqueue(async () => { + switch (event.type) { + case 'text_delta': + if (event.turn !== 'intermediate') { + await this.appendAnswer(event.text) + } + return + case 'turn_end': + await this.flushThinking() + await this.flushAnswer(true) + return + case 'thinking_delta': + if (this.config.includeThinking) this.thinking += event.text + return + case 'tool_call_start': + await this.flushThinking() + if (this.config.includeToolCalls) { + await this.append([ + { + type: 'task_update', + id: `${this.taskId}-tool-${event.id}`, + title: truncate(getToolDisplayTitle(event.name), TASK_TEXT_LIMIT), + status: 'in_progress', + }, + ]) + } + return + case 'tool_call_end': + if (this.config.includeToolCalls) { + await this.append([ + { + type: 'task_update', + id: `${this.taskId}-tool-${event.id}`, + title: truncate(getToolDisplayTitle(event.name), TASK_TEXT_LIMIT), + status: event.status === 'success' ? 'complete' : 'error', + }, + ]) + } + } + }) + } + + appendProjectedBytes(text: string): Promise { + return this.enqueue(() => this.appendAnswer(text)) + } + + complete(): Promise { + return this.enqueue(async () => { + await this.flushThinking() + await this.flushAnswer(true) + if (!this.emittedAnswer && this.fullAnswer) { + const projected = await this.projectFinalText(this.fullAnswer) + if (projected) { + await this.append(splitMarkdown(projected)) + this.emittedAnswer = true + } + } + await this.append([ + { + type: 'task_update', + id: this.taskId, + title: this.title, + status: 'complete', + }, + ]) + await stopSlackAgentStream(this.token, this.channel!, this.ts!, 'processing', this.signal) + }) + } + + sendSettledOutput(text: string): Promise { + return this.enqueue(async () => { + await this.ensureStarted() + await this.append(splitMarkdown(text)) + await this.append([ + { + type: 'task_update', + id: this.taskId, + title: this.title, + status: 'complete', + }, + ]) + await stopSlackAgentStream(this.token, this.channel!, this.ts!, 'processing', this.signal) + }) + } +} + +export class SlackExecutionStreamController { + readonly selectedOutputs: string[] + readonly callbacks: ExecutionCallbacks + + private failure?: Error + private readonly target: SlackReplyTarget + private readonly token: string + private readonly invocations = new Map() + + private constructor( + private readonly options: SlackExecutionStreamControllerOptions, + token: string, + target: SlackReplyTarget + ) { + this.token = token + this.target = target + this.selectedOutputs = options.config.outputConfigs.map((output) => + formatOutputSelector(output.blockId, output.path, output.workflowId) + ) + this.callbacks = { + onStream: (stream) => this.onStream(stream), + onBlockComplete: (blockId, _blockName, _blockType, data) => + this.onBlockComplete(blockId, data), + } + } + + static async create( + options: SlackExecutionStreamControllerOptions + ): Promise { + const credential = await getSlackBotCredential(options.credentialId) + if (!credential || credential.workspaceId !== options.workspaceId) { + throw new Error('Slack streaming credential is unavailable in this workspace') + } + const target = resolveSlackReplyTarget(options.triggerInput) + const controller = new SlackExecutionStreamController(options, credential.botToken, target) + await registerSlackStreamSession(options.credentialId, target, { + executionId: options.executionId, + workflowId: options.workflowId, + userId: options.userId, + workspaceId: options.workspaceId, + }) + try { + await setSlackAgentSessionStatus( + credential.botToken, + target, + 'processing', + options.abortSignal + ) + } catch (error) { + await unregisterSlackStreamSession(options.credentialId, target, options.executionId) + throw error + } + return controller + } + + private selectedForBlock(blockId: string): SlackStreamOutputConfig[] { + return this.options.config.outputConfigs.filter((output) => { + const selectedBlockId = output.workflowId + ? scopeOutputBlockId(output.workflowId, output.blockId) + : output.blockId + return selectedBlockId === blockId + }) + } + + private invocationKey( + blockId: string, + executionOrder: number, + childWorkflowInstanceId?: string + ): string { + return `${blockId}:${childWorkflowInstanceId ?? executionOrder}` + } + + private taskId(executionOrder: number, childWorkflowInstanceId?: string): string { + return `sim-${this.options.executionId}-${childWorkflowInstanceId ?? executionOrder}` + } + + private recordFailure(error: unknown): Error { + const failure = formatSlackApiFailure(error) + this.failure ??= failure + return failure + } + + private async projectLiveText( + text: string, + provenance: StreamingExecution['displayResolvedSecretTraceProvenance'] + ): Promise { + const display = await this.options.loggingSession.projectLiveDisplayText( + 'chunk', + text, + provenance + ) + return typeof display.chunk === 'string' ? display.chunk : null + } + + private async projectFinalText( + text: string, + provenance: StreamingExecution['displayResolvedSecretTraceProvenance'] + ): Promise { + const display = await this.options.loggingSession.projectDisplayContent({ text }, provenance) + return typeof display.text === 'string' ? display.text : null + } + + private async onStream(stream: StreamingExecution): Promise { + try { + if (!stream.blockId || stream.executionOrder === undefined) { + throw new Error('Slack streaming received a stream without invocation metadata') + } + if (this.selectedForBlock(stream.blockId).length === 0) { + throw new Error(`Slack streaming received an unselected block: ${stream.blockId}`) + } + const key = this.invocationKey( + stream.blockId, + stream.executionOrder, + stream.childWorkflowInstanceId + ) + if (this.invocations.has(key)) { + throw new Error(`Duplicate Slack stream invocation: ${key}`) + } + const invocation = new SlackInvocationStream( + this.token, + this.target, + this.options.config, + this.taskId(stream.executionOrder, stream.childWorkflowInstanceId), + this.options.config.taskTitle, + (text) => this.projectLiveText(text, stream.displayResolvedSecretTraceProvenance), + (text) => this.projectFinalText(text, stream.displayResolvedSecretTraceProvenance), + this.options.abortSignal + ) + this.invocations.set(key, invocation) + + const answerFromEventSink = Boolean(stream.subscribe) && !stream.clientStreamTransformed + const unsubscribe = stream.subscribe?.({ + onEvent: async (event) => { + if (!answerFromEventSink && event.type === 'text_delta') return + await invocation.onEvent(event) + }, + }) + const reader = stream.stream.getReader() + const decoder = new TextDecoder() + try { + while (true) { + const { done, value } = await reader.read() + if (done) break + if (!answerFromEventSink) { + await invocation.appendProjectedBytes(decoder.decode(value, { stream: true })) + } + } + if (!answerFromEventSink) { + const remainder = decoder.decode() + if (remainder) await invocation.appendProjectedBytes(remainder) + } + await invocation.complete() + } finally { + unsubscribe?.() + } + } catch (error) { + throw this.recordFailure(error) + } + } + + private async onBlockComplete(blockId: string, data: BlockCompletionCallbackData): Promise { + try { + const selectedOutputBlockId = data.outputBlockId ?? blockId + const selected = this.selectedForBlock(selectedOutputBlockId) + if (selected.length === 0) return + const key = this.invocationKey( + selectedOutputBlockId, + data.executionOrder, + data.childWorkflowInstanceId + ) + if (this.invocations.has(key)) return + + const display = await this.options.loggingSession.projectDisplayContent( + { output: data.output }, + data.displayResolvedSecretTraceProvenance + ) + if (!Object.hasOwn(display, 'output')) return + const values = selected.flatMap((selection) => { + const value = pluckByPath(display.output, selection.path) + return value === undefined ? [] : [{ path: selection.path, value }] + }) + if (values.length === 0) return + const text = + values.length === 1 + ? formatOutput(values[0].value) + : values.map(({ path, value }) => `*${path}*\n${formatOutput(value)}`).join('\n\n') + const invocation = new SlackInvocationStream( + this.token, + this.target, + this.options.config, + this.taskId(data.executionOrder, data.childWorkflowInstanceId), + this.options.config.taskTitle, + async (value) => value, + async (value) => value, + this.options.abortSignal + ) + this.invocations.set(key, invocation) + await invocation.sendSettledOutput(text) + } catch (error) { + this.recordFailure(error) + } + } + + async finalize(result: ExecutionResult): Promise { + const status = + result.status === 'cancelled' || (result.success && result.status !== 'paused') + ? 'active' + : 'suspended' + try { + await setSlackAgentSessionStatus(this.token, this.target, status) + } catch (error) { + this.recordFailure(error) + } + try { + await unregisterSlackStreamSession( + this.options.credentialId, + this.target, + this.options.executionId + ) + } catch (error) { + this.recordFailure(error) + } + } + + assertSucceeded(): void { + if (this.failure) { + throw new Error(getErrorMessage(this.failure, 'Slack response streaming failed')) + } + } +} diff --git a/apps/sim/lib/webhooks/slack-stream-config.test.ts b/apps/sim/lib/webhooks/slack-stream-config.test.ts new file mode 100644 index 00000000000..5a8a0a8e441 --- /dev/null +++ b/apps/sim/lib/webhooks/slack-stream-config.test.ts @@ -0,0 +1,130 @@ +/** + * @vitest-environment node + */ +import { describe, expect, it } from 'vitest' +import { + normalizeSlackStreamResponseConfig, + readSlackStreamResponseConfig, + replaceSlackStreamAuthoringConfig, +} from '@/lib/webhooks/slack-stream-config' + +const CHILD_WORKFLOW_ID = '11111111-1111-4111-8111-111111111111' + +describe('Slack stream response config', () => { + it('normalizes selected outputs and replaces authoring fields', () => { + const providerConfig: Record = { + eventType: 'app_mention', + streamResponse: true, + streamOutputs: ['rootagent.content', `${CHILD_WORKFLOW_ID}.writer.result.value`], + streamIncludeThinking: true, + streamIncludeToolCalls: false, + streamTaskTitle: ' Working ', + streamTaskDisplayMode: 'plan', + } + const normalized = normalizeSlackStreamResponseConfig(providerConfig, { + 'block-1': { id: 'block-1', name: 'Root Agent' }, + }) + replaceSlackStreamAuthoringConfig(providerConfig, normalized) + + expect(normalized).toEqual({ + enabled: true, + outputConfigs: [ + { blockId: 'block-1', path: 'content' }, + { workflowId: CHILD_WORKFLOW_ID, blockId: 'writer', path: 'result.value' }, + ], + includeThinking: true, + includeToolCalls: false, + taskTitle: 'Working', + taskDisplayMode: 'plan', + }) + expect(readSlackStreamResponseConfig(providerConfig)).toEqual(normalized) + expect(providerConfig.streamResponse).toBeUndefined() + expect(providerConfig.streamOutputs).toBeUndefined() + expect(providerConfig.streamTaskTitle).toBeUndefined() + }) + + it('defaults omitted or blank response status labels to Running', () => { + expect( + normalizeSlackStreamResponseConfig( + { + eventType: 'message', + streamResponse: true, + streamOutputs: ['block.content'], + }, + { block: { id: 'block', name: 'Block' } } + )?.taskTitle + ).toBe('Running') + expect( + normalizeSlackStreamResponseConfig( + { + eventType: 'message', + streamResponse: true, + streamOutputs: ['block.content'], + streamTaskTitle: ' ', + }, + { block: { id: 'block', name: 'Block' } } + )?.taskTitle + ).toBe('Running') + }) + + it('upgrades persisted configs with omitted or blank response status labels', () => { + expect( + readSlackStreamResponseConfig({ + streamResponseConfig: { + enabled: true, + outputConfigs: [{ blockId: 'block', path: 'content' }], + includeThinking: false, + includeToolCalls: true, + taskDisplayMode: 'timeline', + }, + })?.taskTitle + ).toBe('Running') + expect( + readSlackStreamResponseConfig({ + streamResponseConfig: { + enabled: true, + outputConfigs: [{ blockId: 'block', path: 'content' }], + includeThinking: false, + includeToolCalls: true, + taskTitle: ' ', + taskDisplayMode: 'timeline', + }, + })?.taskTitle + ).toBe('Running') + }) + + it('rejects non-reply events and malformed output selectors', () => { + expect(() => + normalizeSlackStreamResponseConfig( + { + eventType: 'reaction_added', + streamResponse: true, + streamOutputs: ['block.content'], + }, + {} + ) + ).toThrow('reply-capable') + expect(() => + normalizeSlackStreamResponseConfig( + { + eventType: 'message', + streamResponse: true, + streamOutputs: ['block_content'], + }, + {} + ) + ).toThrow('Invalid Slack stream output selector') + }) + + it('clears stale normalized config when streaming is disabled', () => { + const providerConfig: Record = { + streamResponse: false, + streamResponseConfig: { enabled: true }, + } + replaceSlackStreamAuthoringConfig( + providerConfig, + normalizeSlackStreamResponseConfig(providerConfig, {}) + ) + expect(providerConfig.streamResponseConfig).toBeUndefined() + }) +}) diff --git a/apps/sim/lib/webhooks/slack-stream-config.ts b/apps/sim/lib/webhooks/slack-stream-config.ts new file mode 100644 index 00000000000..dbeb71bf70b --- /dev/null +++ b/apps/sim/lib/webhooks/slack-stream-config.ts @@ -0,0 +1,183 @@ +import { isRecordLike } from '@sim/utils/object' +import { + formatInternalOutputSelector, + parsePublicOutputSelector, + resolveOutputBlockRef, +} from '@/lib/workflows/streaming/output-selector' +import { normalizeName } from '@/executor/constants' + +export const SLACK_STREAM_RESPONSE_EVENTS = [ + 'message', + 'app_mention', + 'assistant_thread_started', +] as const + +export interface SlackStreamOutputConfig { + workflowId?: string + blockId: string + path: string +} + +export interface SlackStreamResponseConfig { + enabled: true + outputConfigs: SlackStreamOutputConfig[] + includeThinking: boolean + includeToolCalls: boolean + taskTitle: string + taskDisplayMode: 'timeline' | 'plan' +} + +const SLACK_TASK_TITLE_LIMIT = 256 + +function parseSlackOutputSelector( + selector: string, + currentBlocks: Record, + currentBlockRefs: ReadonlySet +): SlackStreamOutputConfig { + const parsed = parsePublicOutputSelector(selector, { currentBlockRefs }) + if (!parsed.path) { + throw new Error(`Invalid Slack stream output selector: ${selector}`) + } + const blockId = parsed.workflowId + ? parsed.blockId + : resolveOutputBlockRef(parsed.blockId, currentBlocks) + return { ...parsed, blockId } +} + +/** Converts trigger authoring fields into the durable Slack streaming contract. */ +export function normalizeSlackStreamResponseConfig( + providerConfig: Record, + blocks: Record +): SlackStreamResponseConfig | null { + if (providerConfig.streamResponse !== true) return null + + if ( + typeof providerConfig.eventType !== 'string' || + !SLACK_STREAM_RESPONSE_EVENTS.includes( + providerConfig.eventType as (typeof SLACK_STREAM_RESPONSE_EVENTS)[number] + ) + ) { + throw new Error('Slack streaming is only supported for reply-capable trigger events') + } + if (!Array.isArray(providerConfig.streamOutputs) || providerConfig.streamOutputs.length === 0) { + throw new Error('Select at least one workflow output to stream to Slack') + } + const selectors = providerConfig.streamOutputs.map((value) => { + if (typeof value !== 'string' || !value) { + throw new Error('Slack stream output selectors must be non-empty strings') + } + return value + }) + const taskDisplayMode = providerConfig.streamTaskDisplayMode ?? 'timeline' + if (taskDisplayMode !== 'timeline' && taskDisplayMode !== 'plan') { + throw new Error('Slack stream task display mode must be timeline or plan') + } + const rawTaskTitle = providerConfig.streamTaskTitle ?? '' + if (typeof rawTaskTitle !== 'string') { + throw new Error('Slack stream response status label must be a string') + } + const taskTitle = rawTaskTitle.trim() || 'Running' + if (taskTitle.length > SLACK_TASK_TITLE_LIMIT) { + throw new Error( + `Slack stream response status label must be ${SLACK_TASK_TITLE_LIMIT} characters or fewer` + ) + } + const currentBlockRefs = new Set() + for (const block of Object.values(blocks)) { + currentBlockRefs.add(block.id) + if (block.name) currentBlockRefs.add(normalizeName(block.name)) + } + + return { + enabled: true, + outputConfigs: selectors.map((selector) => + parseSlackOutputSelector(selector, blocks, currentBlockRefs) + ), + includeThinking: providerConfig.streamIncludeThinking === true, + includeToolCalls: providerConfig.streamIncludeToolCalls !== false, + taskTitle, + taskDisplayMode, + } +} + +/** Reads and validates the normalized config stored on a deployed webhook. */ +export function readSlackStreamResponseConfig( + providerConfig: Record +): SlackStreamResponseConfig | null { + const value = providerConfig.streamResponseConfig + if (value === undefined) return null + if (!isRecordLike(value) || value.enabled !== true) { + throw new Error('Invalid persisted Slack stream response configuration') + } + if (!Array.isArray(value.outputConfigs) || value.outputConfigs.length === 0) { + throw new Error('Persisted Slack stream configuration has no outputs') + } + const outputConfigs = value.outputConfigs.map((output) => { + if (!isRecordLike(output) || typeof output.blockId !== 'string' || !output.blockId) { + throw new Error('Persisted Slack stream output is missing a block ID') + } + if (typeof output.path !== 'string' || !output.path) { + throw new Error('Persisted Slack stream output is missing an output path') + } + if ( + output.workflowId !== undefined && + (typeof output.workflowId !== 'string' || !output.workflowId) + ) { + throw new Error('Persisted Slack stream output has an invalid workflow ID') + } + formatInternalOutputSelector( + output.blockId, + output.path, + typeof output.workflowId === 'string' ? output.workflowId : undefined + ) + return { + ...(typeof output.workflowId === 'string' ? { workflowId: output.workflowId } : {}), + blockId: output.blockId, + path: output.path, + } + }) + if (typeof value.includeThinking !== 'boolean' || typeof value.includeToolCalls !== 'boolean') { + throw new Error('Persisted Slack stream visibility settings are invalid') + } + if (value.taskTitle !== undefined && typeof value.taskTitle !== 'string') { + throw new Error('Persisted Slack stream response status label is invalid') + } + const taskTitle = value.taskTitle?.trim() || 'Running' + if (taskTitle.length > SLACK_TASK_TITLE_LIMIT) { + throw new Error('Persisted Slack stream response status label is too long') + } + if (value.taskDisplayMode !== 'timeline' && value.taskDisplayMode !== 'plan') { + throw new Error('Persisted Slack stream task display mode is invalid') + } + return { + enabled: true, + outputConfigs, + includeThinking: value.includeThinking, + includeToolCalls: value.includeToolCalls, + taskTitle, + taskDisplayMode: value.taskDisplayMode, + } +} + +export function isSlackStreamResponseRequested(providerConfig: Record): boolean { + return ( + providerConfig.streamResponse === true || + (isRecordLike(providerConfig.streamResponseConfig) && + providerConfig.streamResponseConfig.enabled === true) + ) +} + +/** Replaces editor-only fields with the normalized durable contract. */ +export function replaceSlackStreamAuthoringConfig( + providerConfig: Record, + normalized: SlackStreamResponseConfig | null +): void { + if (normalized) providerConfig.streamResponseConfig = normalized + else providerConfig.streamResponseConfig = undefined + providerConfig.streamResponse = undefined + providerConfig.streamOutputs = undefined + providerConfig.streamIncludeThinking = undefined + providerConfig.streamIncludeToolCalls = undefined + providerConfig.streamTaskTitle = undefined + providerConfig.streamTaskDisplayMode = undefined +} diff --git a/apps/sim/lib/webhooks/slack-stream-sessions.ts b/apps/sim/lib/webhooks/slack-stream-sessions.ts new file mode 100644 index 00000000000..1bf09e6d41e --- /dev/null +++ b/apps/sim/lib/webhooks/slack-stream-sessions.ts @@ -0,0 +1,111 @@ +import { isRecordLike } from '@sim/utils/object' +import { getRedisClient } from '@/lib/core/config/redis' + +const SESSION_TTL_SECONDS = 24 * 60 * 60 + +export interface SlackStreamSessionTarget { + channel: string + threadTs: string +} + +export interface SlackStreamSessionExecution { + executionId: string + workflowId: string + userId: string + workspaceId: string +} + +function getSessionKey(credentialId: string, target: SlackStreamSessionTarget): string { + return `slack:agent-session:${credentialId}:${target.channel}:${target.threadTs}` +} + +function requireRedis() { + const redis = getRedisClient() + if (!redis) { + throw new Error('Redis is required for Slack agent session streaming') + } + return redis +} + +function parseExecution(value: string): SlackStreamSessionExecution { + const parsed = JSON.parse(value) as unknown + if ( + !isRecordLike(parsed) || + typeof parsed.executionId !== 'string' || + typeof parsed.workflowId !== 'string' || + typeof parsed.userId !== 'string' || + typeof parsed.workspaceId !== 'string' + ) { + throw new Error('Invalid Slack agent session execution record') + } + return { + executionId: parsed.executionId, + workflowId: parsed.workflowId, + userId: parsed.userId, + workspaceId: parsed.workspaceId, + } +} + +export async function registerSlackStreamSession( + credentialId: string, + target: SlackStreamSessionTarget, + execution: SlackStreamSessionExecution +): Promise { + const redis = requireRedis() + const key = getSessionKey(credentialId, target) + await redis + .multi() + .hset(key, execution.executionId, JSON.stringify(execution)) + .expire(key, SESSION_TTL_SECONDS) + .exec() +} + +export async function unregisterSlackStreamSession( + credentialId: string, + target: SlackStreamSessionTarget, + executionId: string +): Promise { + const redis = requireRedis() + const key = getSessionKey(credentialId, target) + await redis.eval( + ` +redis.call('HDEL', KEYS[1], ARGV[1]) +if redis.call('HLEN', KEYS[1]) == 0 then + redis.call('DEL', KEYS[1]) +end +return 1 +`, + 1, + key, + executionId + ) +} + +export async function listSlackStreamSessions( + credentialId: string, + target: SlackStreamSessionTarget +): Promise { + const redis = requireRedis() + const values = await redis.hvals(getSessionKey(credentialId, target)) + if (!Array.isArray(values)) { + throw new Error('Invalid Redis response for Slack agent session lookup') + } + return values.map((value) => { + if (typeof value !== 'string') { + throw new Error('Invalid Slack agent session value in Redis') + } + return parseExecution(value) + }) +} + +export function resolveStoppedSlackSession(body: unknown): SlackStreamSessionTarget | null { + if (!isRecordLike(body) || !isRecordLike(body.event)) return null + if (body.event.type !== 'agent_session_stopped') return null + if (typeof body.event.channel !== 'string' || !body.event.channel) { + throw new Error('Slack agent_session_stopped event is missing channel') + } + if (typeof body.event.thread_ts !== 'string' || !body.event.thread_ts) { + throw new Error('Slack agent_session_stopped event is missing thread_ts') + } + return { channel: body.event.channel, threadTs: body.event.thread_ts } +} diff --git a/apps/sim/lib/workflows/api/index.ts b/apps/sim/lib/workflows/api/index.ts index 26e4ce576dd..44fde23abf1 100644 --- a/apps/sim/lib/workflows/api/index.ts +++ b/apps/sim/lib/workflows/api/index.ts @@ -2,6 +2,7 @@ export { createInternalWorkflowErrorPolicy, internalWorkflowErrorPolicies, internalWorkflowReadAuth, + internalWorkflowSessionOrApiKeyAuth, internalWorkflowSessionOrExecutorAuth, v2WorkflowErrorPolicies, WORKFLOW_NOT_FOUND_MESSAGE, diff --git a/apps/sim/lib/workflows/api/route-policies.ts b/apps/sim/lib/workflows/api/route-policies.ts index 488d747b343..e445a2766ef 100644 --- a/apps/sim/lib/workflows/api/route-policies.ts +++ b/apps/sim/lib/workflows/api/route-policies.ts @@ -1,4 +1,10 @@ -import type { Principal } from '@sim/auth/principal' +import type { + PersonalApiKeyPrincipal, + Principal, + SessionPrincipal, + WorkspaceApiKeyPrincipal, +} from '@sim/auth/principal' +import { v2CancelWorkflowRunDataSchema } from '@/lib/api/contracts/v2/workflows' import { createInternalResourceConcealmentPolicy, createInternalSessionOrExecutorAuth, @@ -8,15 +14,38 @@ import { InternalUnauthenticatedError, internalErrorResponse, internalOrchestrationErrorPolicy, + internalSessionAuth, type V2ErrorPolicy, v2OrchestrationErrorPolicy, } from '@/lib/api/server/routes' import { authenticateApiKeyFromHeader, updateApiKeyLastUsed } from '@/lib/api-key/service' import { asOrchestrationError, statusForOrchestrationError } from '@/lib/core/orchestration/types' +import { WorkflowRunAlreadyTerminalError } from '@/lib/execution/workflow-run-already-terminal-error' import { WORKFLOW_DELEGATION_AUDIENCE } from '@/lib/workflows/application/authorization' import { WorkflowImportError } from '@/lib/workflows/application/workflow-import-error' import { WorkflowOperationsNotAppliedError } from '@/lib/workflows/application/workflow-operations-error' -import { v2CaughtOrchestrationError, v2ErrorForOrchestration } from '@/app/api/v2/lib/response' +import { + v2CaughtOrchestrationError, + v2Data, + v2ErrorForOrchestration, +} from '@/app/api/v2/lib/response' + +function v2CancelRunErrorResponse(error: unknown) { + if (error instanceof WorkflowRunAlreadyTerminalError) { + return v2Data( + v2CancelWorkflowRunDataSchema.parse({ + success: true, + runId: error.executionId, + redisAvailable: error.redisAvailable, + durablyRecorded: false, + locallyAborted: error.locallyAborted, + pausedCancelled: false, + reason: error.executionStatus === 'completed' ? 'already_completed' : 'already_failed', + }) + ) + } + return v2CaughtOrchestrationError(error) +} export const v2WorkflowErrorPolicies = { default: v2OrchestrationErrorPolicy, @@ -55,31 +84,50 @@ export const v2WorkflowErrorPolicies = { concealRunAuthorization: createV2ResourceConcealmentPolicy({ notFoundMessage: 'Run not found', }), + cancelRun: createV2ResourceConcealmentPolicy({ + notFoundMessage: 'Run not found', + render: v2CancelRunErrorResponse, + }), } as const export const internalWorkflowSessionOrExecutorAuth = createInternalSessionOrExecutorAuth({ audience: WORKFLOW_DELEGATION_AUDIENCE, }) +type WorkflowApiKeyPrincipal = PersonalApiKeyPrincipal | WorkspaceApiKeyPrincipal + +async function authenticateWorkflowApiKey(rawApiKey: string): Promise { + const result = await authenticateApiKeyFromHeader(rawApiKey) + if (!result.success || !result.keyId || !result.keyType) { + throw new InternalUnauthenticatedError('Unauthorized') + } + await updateApiKeyLastUsed(result.keyId) + + if (result.keyType === 'workspace') { + if (!result.workspaceId) throw new Error('Workspace API key is missing its workspace scope') + return { kind: 'workspace_api_key', workspaceId: result.workspaceId, keyId: result.keyId } + } + if (!result.userId) throw new Error('Personal API key is missing its credential owner') + return { kind: 'personal_api_key', userId: result.userId, keyId: result.keyId } +} + +export const internalWorkflowSessionOrApiKeyAuth: InternalAuthPolicy< + SessionPrincipal | WorkflowApiKeyPrincipal +> = { + async authenticate(request) { + const rawApiKey = request.headers.get('x-api-key') + if (!rawApiKey) return internalSessionAuth.authenticate() + return authenticateWorkflowApiKey(rawApiKey) + }, +} + export const internalWorkflowReadAuth: InternalAuthPolicy = { async authenticate(request, params) { const rawApiKey = request.headers.get('x-api-key') if (!rawApiKey) { return internalWorkflowSessionOrExecutorAuth.authenticate(request, params) } - - const result = await authenticateApiKeyFromHeader(rawApiKey) - if (!result.success || !result.keyId || !result.keyType) { - throw new InternalUnauthenticatedError('Unauthorized') - } - await updateApiKeyLastUsed(result.keyId) - - if (result.keyType === 'workspace') { - if (!result.workspaceId) throw new Error('Workspace API key is missing its workspace scope') - return { kind: 'workspace_api_key', workspaceId: result.workspaceId, keyId: result.keyId } - } - if (!result.userId) throw new Error('Personal API key is missing its credential owner') - return { kind: 'personal_api_key', userId: result.userId, keyId: result.keyId } + return authenticateWorkflowApiKey(rawApiKey) }, } @@ -125,4 +173,8 @@ export const internalWorkflowErrorPolicies = { base: internalOrchestrationErrorPolicy, notFoundMessage: WORKFLOW_NOT_FOUND_MESSAGE, }), + concealRunAuthorization: createInternalResourceConcealmentPolicy({ + base: internalOrchestrationErrorPolicy, + notFoundMessage: 'Execution not found', + }), } as const diff --git a/apps/sim/lib/workflows/application/apply-workflow-operations.test.ts b/apps/sim/lib/workflows/application/apply-workflow-operations.test.ts index a9acf0dc61a..a88a6d652b4 100644 --- a/apps/sim/lib/workflows/application/apply-workflow-operations.test.ts +++ b/apps/sim/lib/workflows/application/apply-workflow-operations.test.ts @@ -85,8 +85,18 @@ vi.mock('@/lib/billing/core/subscription', () => ({ hasWorkspaceSandboxAccess: mocks.sandboxAccess, })) vi.mock('@/lib/core/config/block-visibility', () => ({ getBlockVisibility: mocks.blockVisibility })) -vi.mock('@/ee/access-control/utils/permission-check', () => ({ +vi.mock('@/lib/permission-groups/resolve.server', () => ({ getUserPermissionConfig: mocks.permissionConfig, + /** + * The use case passes the organization it already loaded, so the resolver + * takes its verified-context branch rather than looking the workspace up + * again. + */ + resolveVerifiedUserAccessControlContext: async ( + userId: string, + workspaceId: string, + _organizationId: string | null + ) => ({ config: await mocks.permissionConfig(userId, workspaceId) }), })) vi.mock('@/blocks/visibility/server-context', () => ({ withBlockVisibility: (_state: unknown, run: () => unknown) => run(), diff --git a/apps/sim/lib/workflows/application/apply-workflow-operations.ts b/apps/sim/lib/workflows/application/apply-workflow-operations.ts index ff5a34aebb9..0aff805b423 100644 --- a/apps/sim/lib/workflows/application/apply-workflow-operations.ts +++ b/apps/sim/lib/workflows/application/apply-workflow-operations.ts @@ -9,6 +9,7 @@ import { ForbiddenOperationError, principalAuditSource } from '@/lib/core/applic import { getBlockVisibility } from '@/lib/core/config/block-visibility' import { OrchestrationError } from '@/lib/core/orchestration/types' import { MAX_PLAN_REQUIRED } from '@/lib/execution/remote-sandbox/workspace-sandboxes' +import { resolvePermissionGroupConfig } from '@/lib/permission-groups/config-scope.server' import { notifyWorkflowUpdated } from '@/lib/realtime/notify' import { defineAuthorizedWorkflowUseCase } from '@/lib/workflows/application/authorized-workflow-use-case' import { @@ -54,7 +55,6 @@ import { import { loadWorkflowFromNormalizedTables } from '@/lib/workflows/persistence/utils' import { validateWorkflowState } from '@/lib/workflows/sanitization/validation' import { withBlockVisibility } from '@/blocks/visibility/server-context' -import { getUserPermissionConfig } from '@/ee/access-control/utils/permission-check' import { generateLoopBlocks, generateParallelBlocks } from '@/stores/workflows/workflow/utils' import { normalizeWorkflowState } from '@/stores/workflows/workflow/validation' @@ -274,7 +274,11 @@ export const applyWorkflowOperations = defineAuthorizedWorkflowUseCase({ const baseGraph = await resolveBaseGraph(principal, input, context) const [permissionConfig, blockVisibility] = await Promise.all([ - getUserPermissionConfig(subjectUserId, context.workspaceId), + resolvePermissionGroupConfig( + subjectUserId, + context.workspaceId, + context.workspaceOrganizationId + ), getBlockVisibility({ userId: subjectUserId, orgId: context.workspaceOrganizationId }), ]) @@ -417,6 +421,12 @@ export const applyWorkflowOperations = defineAuthorizedWorkflowUseCase({ workflowId: context.workflowId, workspaceId: context.workspaceId, attributedUserId: subjectUserId, + /** + * The same id line 287 already resolves the permission config against. + * This operation denies workspace API keys, so the attribution and the + * governed subject are the same human and cannot diverge here. + */ + subjectUserId, state: { blocks: graph.blocks, edges: graph.edges }, }) diff --git a/apps/sim/lib/workflows/application/cancel-run.ts b/apps/sim/lib/workflows/application/cancel-run.ts index 848fb3c64cc..f946231c6c9 100644 --- a/apps/sim/lib/workflows/application/cancel-run.ts +++ b/apps/sim/lib/workflows/application/cancel-run.ts @@ -4,13 +4,14 @@ import { cancelWorkflowExecution, WorkflowExecutionNotFoundError, } from '@/lib/execution/cancel-workflow-execution' +import { captureServerEvent } from '@/lib/posthog/server' import { defineAuthorizedWorkflowUseCase } from '@/lib/workflows/application/authorized-workflow-use-case' import { resolveActiveWorkflowRunApplicationContext } from '@/lib/workflows/application/context' import { workflowOperations } from '@/lib/workflows/application/operations' export interface CancelWorkflowRunInput { - workflowId: string runId: string + abortSignal?: AbortSignal } export const cancelWorkflowRun = defineAuthorizedWorkflowUseCase({ @@ -18,9 +19,8 @@ export const cancelWorkflowRun = defineAuthorizedWorkflowUseCase({ resolveContext: ({ input }: { input: CancelWorkflowRunInput }) => resolveActiveWorkflowRunApplicationContext({ runId: input.runId, - assertedWorkflowId: input.workflowId, }), - async execute({ principal, context }) { + async execute({ principal, context, input }) { const attribution = resolvePrincipalAttribution(principal, { workspaceBillingOwnerUserId: context.billedAccountUserId, }) @@ -28,9 +28,9 @@ export const cancelWorkflowRun = defineAuthorizedWorkflowUseCase({ const result = await cancelWorkflowExecution({ executionId: context.runId, workflowId: context.workflowId, - userId: attribution.attributedUserId, + attributedUserId: attribution.attributedUserId, workspaceId: context.workspaceId, - captureAnalytics: false, + abortSignal: input.abortSignal, }) return { ...result, workflowId: context.workflowId, workspaceId: context.workspaceId } } catch (error) { @@ -40,4 +40,16 @@ export const cancelWorkflowRun = defineAuthorizedWorkflowUseCase({ throw error } }, + afterSuccess({ principal, context, result }) { + if (!result.success || result.reason === 'already_cancelled') return + const attribution = resolvePrincipalAttribution(principal, { + workspaceBillingOwnerUserId: context.billedAccountUserId, + }) + captureServerEvent( + attribution.attributedUserId, + 'workflow_execution_cancelled', + { workflow_id: context.workflowId, workspace_id: context.workspaceId }, + { groups: { workspace: context.workspaceId } } + ) + }, }) diff --git a/apps/sim/lib/workflows/application/chat-deployments.ts b/apps/sim/lib/workflows/application/chat-deployments.ts index fa125332cfe..44b27bede10 100644 --- a/apps/sim/lib/workflows/application/chat-deployments.ts +++ b/apps/sim/lib/workflows/application/chat-deployments.ts @@ -14,20 +14,17 @@ import { getChatDeploymentIdOwningIdentifier, getLiveChatDeploymentForWorkflow, } from '@/lib/chat-deployments/queries' -import { ForbiddenOperationError } from '@/lib/core/application' import { OrchestrationError } from '@/lib/core/orchestration/types' import { defineAuthorizedWorkflowUseCase } from '@/lib/workflows/application/authorized-workflow-use-case' import { resolveActiveWorkflowApplicationContext } from '@/lib/workflows/application/context' import { workflowOperations } from '@/lib/workflows/application/operations' import { assertedWorkflowWorkspaceId } from '@/lib/workflows/application/principal-scope' import { performChatDeploy, performChatUndeploy } from '@/lib/workflows/orchestration' -import { - ChatDeployAuthNotAllowedError, - validateChatDeployAuth, -} from '@/ee/access-control/utils/permission-check' +import { formatInternalOutputSelector } from '@/lib/workflows/streaming/output-selector' +import { validateChatDeployAuth } from '@/ee/access-control/utils/permission-check' type ChatAuthType = 'public' | 'password' | 'email' | 'sso' -type ChatOutputConfig = { blockId: string; path: string } +type ChatOutputConfig = { workflowId?: string; blockId: string; path: string } type ChatCustomizations = { primaryColor?: string welcomeMessage?: string @@ -73,12 +70,22 @@ function parseChatOutputConfigs(value: unknown[] | undefined): ChatOutputConfig[ 'blockId' in entry && typeof entry.blockId === 'string' && entry.blockId.length > 0 && + (!('workflowId' in entry) || + entry.workflowId === undefined || + (typeof entry.workflowId === 'string' && entry.workflowId.length > 0)) && 'path' in entry && typeof entry.path === 'string' ) ) { throw new OrchestrationError('validation', 'Invalid chat output configuration') } + try { + for (const config of value) { + formatInternalOutputSelector(config.blockId, config.path, config.workflowId) + } + } catch { + throw new OrchestrationError('validation', 'Invalid chat output configuration') + } return value } @@ -161,14 +168,7 @@ export const deployWorkflowChat = defineAuthorizedWorkflowUseCase({ const subjectUserId = requirePrincipalSubjectUserId(principal) if (authType !== existingDeployment?.authType) { - try { - await validateChatDeployAuth(subjectUserId, context.workspaceId, authType) - } catch (error) { - if (error instanceof ChatDeployAuthNotAllowedError) { - throw new ForbiddenOperationError('CHAT_AUTH_MODE_NOT_PERMITTED', error.message) - } - throw error - } + await validateChatDeployAuth(subjectUserId, context.workspaceId, authType) } const attribution = resolvePrincipalAttribution(principal, { diff --git a/apps/sim/lib/workflows/application/create-workflow.ts b/apps/sim/lib/workflows/application/create-workflow.ts index 53580cd0e21..295edac51c3 100644 --- a/apps/sim/lib/workflows/application/create-workflow.ts +++ b/apps/sim/lib/workflows/application/create-workflow.ts @@ -6,7 +6,7 @@ import { OrchestrationError } from '@/lib/core/orchestration/types' import { PlatformEvents } from '@/lib/core/telemetry' import { MAX_FOLDERS_PER_WORKSPACE } from '@/lib/folders/constants' import { loadActiveFolderPathIndex } from '@/lib/folders/queries' -import { notifyWorkflowUpdated } from '@/lib/realtime/notify' +import { notifyWorkflowUpdated, notifyWorkspaceWorkflowsChanged } from '@/lib/realtime/notify' import { defineAuthorizedWorkflowUseCase } from '@/lib/workflows/application/authorized-workflow-use-case' import { workflowOperations } from '@/lib/workflows/application/operations' import { requireWorkflowTransition } from '@/lib/workflows/application/transition-result' @@ -98,7 +98,10 @@ export const createWorkflow = defineAuthorizedWorkflowUseCase({ }, }), async afterSuccess({ result }) { - await notifyWorkflowUpdated(result.workflow.id) + await Promise.all([ + notifyWorkflowUpdated(result.workflow.id), + notifyWorkspaceWorkflowsChanged(result.workflow.workspaceId), + ]) try { PlatformEvents.workflowCreated({ workflowId: result.workflow.id, diff --git a/apps/sim/lib/workflows/application/delete-workflow.ts b/apps/sim/lib/workflows/application/delete-workflow.ts index cfa01b44119..4a845fc9fcd 100644 --- a/apps/sim/lib/workflows/application/delete-workflow.ts +++ b/apps/sim/lib/workflows/application/delete-workflow.ts @@ -3,7 +3,7 @@ import { type Principal, resolvePrincipalAttribution } from '@sim/auth/principal import { createLogger } from '@sim/logger' import { assertWorkflowMutable, WorkflowLockedError } from '@sim/platform-authz/workflow' import { OrchestrationError } from '@/lib/core/orchestration/types' -import { notifyWorkflowDeleted } from '@/lib/realtime/notify' +import { notifyWorkflowDeleted, notifyWorkspaceWorkflowsChanged } from '@/lib/realtime/notify' import { defineAuthorizedWorkflowUseCase } from '@/lib/workflows/application/authorized-workflow-use-case' import { resolveActiveWorkflowApplicationContext } from '@/lib/workflows/application/context' import { workflowOperations } from '@/lib/workflows/application/operations' @@ -69,6 +69,11 @@ export const deleteWorkflow = defineAuthorizedWorkflowUseCase({ metadata: { archived: true }, } : [], - afterSuccess: ({ context, result }) => - result.archived ? notifyWorkflowDeleted(context.workflowId) : undefined, + async afterSuccess({ context, result }) { + if (!result.archived) return + await Promise.all([ + notifyWorkflowDeleted(context.workflowId), + notifyWorkspaceWorkflowsChanged(context.workspaceId), + ]) + }, }) diff --git a/apps/sim/lib/workflows/application/duplicate-workflow.ts b/apps/sim/lib/workflows/application/duplicate-workflow.ts index a1a3abe0902..024b5fb6066 100644 --- a/apps/sim/lib/workflows/application/duplicate-workflow.ts +++ b/apps/sim/lib/workflows/application/duplicate-workflow.ts @@ -7,7 +7,7 @@ import { OrchestrationError } from '@/lib/core/orchestration/types' import { generateRequestId } from '@/lib/core/utils/request' import { MAX_FOLDERS_PER_WORKSPACE } from '@/lib/folders/constants' import { loadActiveFolderPathIndex } from '@/lib/folders/queries' -import { notifyWorkflowUpdated } from '@/lib/realtime/notify' +import { notifyWorkflowUpdated, notifyWorkspaceWorkflowsChanged } from '@/lib/realtime/notify' import { defineAuthorizedWorkflowUseCase } from '@/lib/workflows/application/authorized-workflow-use-case' import { resolveActiveWorkflowApplicationContext } from '@/lib/workflows/application/context' import { workflowOperations } from '@/lib/workflows/application/operations' @@ -100,5 +100,10 @@ export const duplicateWorkflow = defineAuthorizedWorkflowUseCase({ source: principalAuditSource(principal), }, }), - afterSuccess: ({ result }) => notifyWorkflowUpdated(result.id), + async afterSuccess({ context, result }) { + await Promise.all([ + notifyWorkflowUpdated(result.id), + notifyWorkspaceWorkflowsChanged(context.workspaceId), + ]) + }, }) diff --git a/apps/sim/lib/workflows/application/import-export.test.ts b/apps/sim/lib/workflows/application/import-export.test.ts index c205d225361..3b36db1a7c5 100644 --- a/apps/sim/lib/workflows/application/import-export.test.ts +++ b/apps/sim/lib/workflows/application/import-export.test.ts @@ -12,6 +12,7 @@ const mocks = vi.hoisted(() => ({ folderLock: vi.fn(), loadIndex: vi.fn(), recordAudit: vi.fn(), + notifyWorkspace: vi.fn(), })) vi.mock('@/lib/workspaces/application/workspace-context', () => ({ @@ -41,6 +42,10 @@ vi.mock('@/lib/folders/queries', () => ({ resolveFolderPathFromIndex: (index: { idByPath: Map }, path: string) => path === '/' ? null : index.idByPath.get(path), })) +vi.mock('@/lib/realtime/notify', () => ({ + notifyWorkspaceWorkflowsChanged: mocks.notifyWorkspace, +})) + vi.mock('@/lib/workflows/operations/import-workflow', () => ({ importWorkflowIntoWorkspaceTransition: mocks.importTransition, })) @@ -153,6 +158,7 @@ describe('workflow import and export application operations', () => { }), }) ) + expect(mocks.notifyWorkspace).toHaveBeenCalledWith('ws-1') }) it('preserves classified import details and does not audit a failure', async () => { diff --git a/apps/sim/lib/workflows/application/import-export.ts b/apps/sim/lib/workflows/application/import-export.ts index ecad2d84428..414557433f5 100644 --- a/apps/sim/lib/workflows/application/import-export.ts +++ b/apps/sim/lib/workflows/application/import-export.ts @@ -1,10 +1,12 @@ import { AuditAction, AuditResourceType } from '@sim/audit' import { resolvePrincipalAttribution } from '@sim/auth/principal' +import { capabilityGovernedPrincipalUserId } from '@/lib/core/application' import type { OrchestrationErrorCode } from '@/lib/core/orchestration/types' import { OrchestrationError } from '@/lib/core/orchestration/types' import { generateRequestId } from '@/lib/core/utils/request' import { MAX_FOLDERS_PER_WORKSPACE } from '@/lib/folders/constants' import { loadActiveFolderPathIndex } from '@/lib/folders/queries' +import { notifyWorkspaceWorkflowsChanged } from '@/lib/realtime/notify' import { defineAuthorizedWorkflowUseCase } from '@/lib/workflows/application/authorized-workflow-use-case' import { resolveActiveWorkflowApplicationContext } from '@/lib/workflows/application/context' import { workflowOperations } from '@/lib/workflows/application/operations' @@ -48,6 +50,7 @@ export interface ExportWorkflowResult { function importErrorCode(status: number): OrchestrationErrorCode { if (status === 400) return 'validation' if (status === 404) return 'not_found' + if (status === 403) return 'forbidden' if (status === 409) return 'conflict' if (status === 423) return 'locked' return 'internal' @@ -70,6 +73,7 @@ export const importWorkflow = defineAuthorizedWorkflowUseCase({ description: input.description, workflow: input.workflow, userId: attribution.attributedUserId, + capabilityUserId: capabilityGovernedPrincipalUserId(principal), requestId: generateRequestId(), }) if (!result.success) { @@ -96,6 +100,7 @@ export const importWorkflow = defineAuthorizedWorkflowUseCase({ }, } }, + afterSuccess: ({ result }) => notifyWorkspaceWorkflowsChanged(result.workflow.workspaceId), }) export const exportWorkflow = defineAuthorizedWorkflowUseCase({ diff --git a/apps/sim/lib/workflows/application/list-workflow-runs.test.ts b/apps/sim/lib/workflows/application/list-workflow-runs.test.ts new file mode 100644 index 00000000000..17a21a13841 --- /dev/null +++ b/apps/sim/lib/workflows/application/list-workflow-runs.test.ts @@ -0,0 +1,125 @@ +/** + * @vitest-environment node + * + * `logs.cost` is a PROJECTION, not a gate — a group withholds the figure from + * the response rather than refusing the read, which is why `workflows.listRuns` + * correctly declares `capability: 'none'`. + * + * This listing carries the same per-run total every other log surface withholds, + * and applied none of it: an enterprise member whose group hides spend read it + * in full here through a personal API key. These run the real use case against + * the real `resolveLogFieldProjection`, so they fail if this surface stops + * projecting. + */ +import { + permissionGroupScopeMock, + permissionGroupScopeMockFns, + resetPermissionGroupScopeMock, +} from '@sim/testing' +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const mocks = vi.hoisted(() => ({ + loadWorkspace: vi.fn(), + resolvePermission: vi.fn(), + resolveWorkflowContext: vi.fn(), + listExecutions: vi.fn(), + recordAudit: vi.fn(), +})) + +vi.mock('@/lib/permission-groups/config-scope.server', () => permissionGroupScopeMock) + +vi.mock('@/lib/workspaces/application/workspace-context', () => ({ + loadActiveWorkspaceApplicationContext: mocks.loadWorkspace, +})) + +vi.mock('@sim/platform-authz/workspace', () => ({ + permissionSatisfies: (permission: string | null, required: string) => + permission === 'admin' || permission === 'write' || permission === required, + resolveEffectiveWorkspacePermission: mocks.resolvePermission, +})) + +vi.mock('@/lib/workflows/application/context', () => ({ + resolveActiveWorkflowApplicationContext: mocks.resolveWorkflowContext, +})) + +vi.mock('@/lib/workflows/executor/execution-queries', () => ({ + listWorkflowExecutions: mocks.listExecutions, +})) + +vi.mock('@sim/audit', () => ({ recordAudit: mocks.recordAudit })) + +import { DEFAULT_PERMISSION_GROUP_CONFIG } from '@/lib/permission-groups/fields' +import { listWorkflowRuns } from '@/lib/workflows/application/list-workflow-runs' + +const WORKSPACE_ID = 'workspace-1' +const WORKFLOW_ID = 'workflow-1' + +const sessionPrincipal = { kind: 'session' as const, userId: 'user-1' } +const workspaceKeyPrincipal = { + kind: 'workspace_api_key' as const, + workspaceId: WORKSPACE_ID, + keyId: 'key-1', +} + +const input = { workflowId: WORKFLOW_ID, limit: 10, order: 'desc' as const } + +function runRow(costTotal: string | null) { + return { rowId: 1, executionId: 'run-1', startedAt: new Date(), status: 'success', costTotal } +} + +beforeEach(() => { + vi.clearAllMocks() + resetPermissionGroupScopeMock() + mocks.loadWorkspace.mockResolvedValue({ + workspaceId: WORKSPACE_ID, + workspaceOrganizationId: 'organization-1', + allowPersonalApiKeys: true, + billedAccountUserId: 'billing-owner-1', + }) + mocks.resolvePermission.mockResolvedValue('admin') + mocks.resolveWorkflowContext.mockResolvedValue({ + workspaceId: WORKSPACE_ID, + workspaceOrganizationId: 'organization-1', + allowPersonalApiKeys: true, + billedAccountUserId: 'billing-owner-1', + workflowId: WORKFLOW_ID, + }) + mocks.listExecutions.mockResolvedValue({ data: [runRow('0.75')], nextCursor: null }) + permissionGroupScopeMockFns.mockResolvePermissionGroupConfig.mockResolvedValue(null) +}) + +describe('listWorkflowRuns cost projection', () => { + it('blanks the per-run total when the group hides cost', async () => { + permissionGroupScopeMockFns.mockResolvePermissionGroupConfig.mockResolvedValue({ + ...DEFAULT_PERMISSION_GROUP_CONFIG, + hideCostInfo: true, + }) + + const result = await listWorkflowRuns.execute({ principal: sessionPrincipal, input }) + + expect(result.data[0].costTotal).toBeNull() + }) + + it('returns the total when the group withholds nothing', async () => { + permissionGroupScopeMockFns.mockResolvePermissionGroupConfig.mockResolvedValue({ + ...DEFAULT_PERMISSION_GROUP_CONFIG, + }) + + const result = await listWorkflowRuns.execute({ principal: sessionPrincipal, input }) + + expect(result.data[0].costTotal).toBe('0.75') + }) + + it('returns the total when no group governs the caller', async () => { + const result = await listWorkflowRuns.execute({ principal: sessionPrincipal, input }) + + expect(result.data[0].costTotal).toBe('0.75') + }) + + it('withholds nothing from a workspace API key, and never resolves a group', async () => { + const result = await listWorkflowRuns.execute({ principal: workspaceKeyPrincipal, input }) + + expect(result.data[0].costTotal).toBe('0.75') + expect(permissionGroupScopeMockFns.mockResolvePermissionGroupConfig).not.toHaveBeenCalled() + }) +}) diff --git a/apps/sim/lib/workflows/application/list-workflow-runs.ts b/apps/sim/lib/workflows/application/list-workflow-runs.ts index ab2b88b6a1c..5e4c078cfbc 100644 --- a/apps/sim/lib/workflows/application/list-workflow-runs.ts +++ b/apps/sim/lib/workflows/application/list-workflow-runs.ts @@ -1,3 +1,4 @@ +import { logProjectionSubjectUserId, resolveLogFieldProjection } from '@/lib/logs/log-projection' import { defineAuthorizedWorkflowUseCase } from '@/lib/workflows/application/authorized-workflow-use-case' import { resolveActiveWorkflowApplicationContext } from '@/lib/workflows/application/context' import { workflowOperations } from '@/lib/workflows/application/operations' @@ -14,7 +15,23 @@ export const listWorkflowRuns = defineAuthorizedWorkflowUseCase({ operation: workflowOperations.listRuns, resolveContext: ({ input }: { input: ListWorkflowRunsInput }) => resolveActiveWorkflowApplicationContext({ workflowId: input.workflowId }), - async execute({ context, input }) { + async execute({ principal, context, input }) { + /** + * The per-run total this listing carries is the same figure `hideCostInfo` + * withholds on every other log surface, so it is projected here rather than + * in the presenter — the withholding travels with the read. + * + * {@link logProjectionSubjectUserId} names nobody for a workspace API key, + * which represents no user and therefore no group — the key's creator is + * never substituted — nor for an executor delegation, which carries a role + * and no capabilities. This listing publishes no cost sort or filter, so + * there is no query surface to refuse alongside the value. + */ + const projection = await resolveLogFieldProjection( + logProjectionSubjectUserId(principal), + context.workspaceId, + context.workspaceOrganizationId + ) const result = await listWorkflowExecutions({ workflowId: context.workflowId, status: input.status, @@ -25,6 +42,13 @@ export const listWorkflowRuns = defineAuthorizedWorkflowUseCase({ cursor: input.cursor, order: input.order, }) - return { ...result, workflowId: context.workflowId, order: input.order } + return { + ...result, + data: projection.hideCostInfo + ? result.data.map((row) => ({ ...row, costTotal: null })) + : result.data, + workflowId: context.workflowId, + order: input.order, + } }, }) diff --git a/apps/sim/lib/workflows/application/move-workflows-bulk.test.ts b/apps/sim/lib/workflows/application/move-workflows-bulk.test.ts index 20260e59dfd..b50532261be 100644 --- a/apps/sim/lib/workflows/application/move-workflows-bulk.test.ts +++ b/apps/sim/lib/workflows/application/move-workflows-bulk.test.ts @@ -15,6 +15,7 @@ const { FolderLockedError, WorkflowLockedError, mocks } = vi.hoisted(() => { assertWorkflowMutable: vi.fn(), audit: vi.fn(), notify: vi.fn(), + notifyWorkspace: vi.fn(), permission: vi.fn(), resolveContext: vi.fn(), updateWorkflow: vi.fn(), @@ -48,7 +49,10 @@ vi.mock('@/lib/workflows/orchestration', () => ({ updateWorkflowRecord: mocks.updateWorkflow, })) -vi.mock('@/lib/realtime/notify', () => ({ notifyWorkflowUpdated: mocks.notify })) +vi.mock('@/lib/realtime/notify', () => ({ + notifyWorkflowUpdated: mocks.notify, + notifyWorkspaceWorkflowsChanged: mocks.notifyWorkspace, +})) import { moveWorkflowsBulk } from '@/lib/workflows/application/move-workflows-bulk' @@ -118,6 +122,7 @@ describe('moveWorkflowsBulk', () => { ) expect(mocks.notify).toHaveBeenCalledWith('workflow-1') expect(mocks.notify).not.toHaveBeenCalledWith('workflow-2') + expect(mocks.notifyWorkspace).toHaveBeenCalledWith('workspace-1') }) it('conceals cross-workspace workflow IDs as failed items', async () => { diff --git a/apps/sim/lib/workflows/application/move-workflows-bulk.ts b/apps/sim/lib/workflows/application/move-workflows-bulk.ts index 2232a78150a..ec102f57692 100644 --- a/apps/sim/lib/workflows/application/move-workflows-bulk.ts +++ b/apps/sim/lib/workflows/application/move-workflows-bulk.ts @@ -11,7 +11,7 @@ import { import { and, eq, inArray, isNull } from 'drizzle-orm' import { principalAuditSource } from '@/lib/core/application' import { asOrchestrationError, OrchestrationError } from '@/lib/core/orchestration/types' -import { notifyWorkflowUpdated } from '@/lib/realtime/notify' +import { notifyWorkflowUpdated, notifyWorkspaceWorkflowsChanged } from '@/lib/realtime/notify' import { defineAuthorizedWorkflowUseCase } from '@/lib/workflows/application/authorized-workflow-use-case' import { workflowOperations } from '@/lib/workflows/application/operations' import { requireWorkflowTransition } from '@/lib/workflows/application/transition-result' @@ -171,9 +171,11 @@ export const moveWorkflowsBulk = defineAuthorizedWorkflowUseCase({ source: principalAuditSource(principal), }, })), - afterSuccess: async ({ result }) => { - for (const workflowId of result.moved) { - await notifyWorkflowUpdated(workflowId) - } + afterSuccess: async ({ context, result }) => { + if (result.moved.length === 0) return + await Promise.all([ + ...result.moved.map((workflowId) => notifyWorkflowUpdated(workflowId)), + notifyWorkspaceWorkflowsChanged(context.workspaceId), + ]) }, }) diff --git a/apps/sim/lib/workflows/application/operations.test.ts b/apps/sim/lib/workflows/application/operations.test.ts index c79db22ca80..b978a3e3bcd 100644 --- a/apps/sim/lib/workflows/application/operations.test.ts +++ b/apps/sim/lib/workflows/application/operations.test.ts @@ -134,4 +134,14 @@ describe('workflow operation registry', () => { expect(operation.id).toMatch(/^workflows\.manual\.execute/) } }) + + it('protects paused execution detail as a workflow read', () => { + expect(workflowOperations.readPausedExecution).toMatchObject({ + id: 'workflows.paused_executions.read', + minimumRole: 'read', + workspaceApiKey: 'allow', + principalKinds: ['session', 'personal_api_key', 'workspace_api_key', 'delegated'], + delegatedServices: ['copilot'], + }) + }) }) diff --git a/apps/sim/lib/workflows/application/operations.ts b/apps/sim/lib/workflows/application/operations.ts index 87591f517bc..3d83968be1b 100644 --- a/apps/sim/lib/workflows/application/operations.ts +++ b/apps/sim/lib/workflows/application/operations.ts @@ -26,52 +26,68 @@ const COPILOT_WORKFLOW_PRINCIPAL_POLICY = { } as const export const workflowOperations = { + // permission-group-exempt: listing the workflows in a workspace is governed by workspace role; no group hides the workflow module list: defineWorkspaceOperation({ id: 'workflows.list', minimumRole: 'read', workspaceApiKey: 'allow', + capability: 'none', ...ALL_WORKFLOW_PRINCIPAL_POLICY, }), + // permission-group-exempt: reading a workflow is governed by workspace role, not by a group capability read: defineWorkspaceOperation({ id: 'workflows.read', minimumRole: 'read', workspaceApiKey: 'allow', + capability: 'none', ...WORKFLOW_READ_PRINCIPAL_POLICY, }), + // permission-group-exempt: reporting where a workflow is already deployed is a read of existing state; a group withholds the act of deploying, not the record of it readDeploymentOverview: defineWorkspaceOperation({ id: 'workflows.deployment_overview.read', minimumRole: 'read', workspaceApiKey: 'deny', + capability: 'none', ...COPILOT_WORKFLOW_PRINCIPAL_POLICY, }), + // permission-group-exempt: reading a workflow's run inputs is workflow content; Chat itself is withheld by copilot.use at the chat surface readCopilotRunOptions: defineWorkspaceOperation({ id: 'workflows.copilot.run_options.read', minimumRole: 'read', workspaceApiKey: 'deny', + capability: 'none', ...COPILOT_WORKFLOW_PRINCIPAL_POLICY, }), + // permission-group-exempt: reading a block's declared outputs is workflow content; Chat itself is withheld by copilot.use at the chat surface readCopilotBlockOutputs: defineWorkspaceOperation({ id: 'workflows.copilot.block_outputs.read', minimumRole: 'read', workspaceApiKey: 'deny', + capability: 'none', ...COPILOT_WORKFLOW_PRINCIPAL_POLICY, }), + // permission-group-exempt: resolving which upstream blocks a block may reference is workflow content; Chat itself is withheld by copilot.use at the chat surface readCopilotUpstreamReferences: defineWorkspaceOperation({ id: 'workflows.copilot.upstream_references.read', minimumRole: 'read', workspaceApiKey: 'deny', + capability: 'none', ...COPILOT_WORKFLOW_PRINCIPAL_POLICY, }), + // permission-group-exempt: the workflow module has no hide key, so creating a workflow is governed by workspace role alone create: defineWorkspaceOperation({ id: 'workflows.create', minimumRole: 'write', workspaceApiKey: 'allow', + capability: 'none', ...ALL_WORKFLOW_PRINCIPAL_POLICY, }), + // permission-group-exempt: renaming or re-describing a workflow is governed by workspace role update: defineWorkspaceOperation({ id: 'workflows.update', minimumRole: 'write', workspaceApiKey: 'allow', + capability: 'none', ...ALL_WORKFLOW_PRINCIPAL_POLICY, }), /** @@ -87,11 +103,14 @@ export const workflowOperations = { * * Personal keys keep the capability, so headless authoring is unaffected for a * credential that names a human. + * + * permission-group-exempt: which blocks a member may store is judged against allowedIntegrations inside replaceWorkflowNormalizedState, which this use case passes the principal's human subject to, not by a capability the authorization funnel can apply */ replaceState: defineWorkspaceOperation({ id: 'workflows.state.replace', minimumRole: 'write', workspaceApiKey: 'deny', + capability: 'none', ...HUMAN_WORKFLOW_PRINCIPAL_POLICY, }), /** @@ -110,150 +129,195 @@ export const workflowOperations = { * Personal keys keep the capability, so headless editing is unaffected for a * credential that names a human. Re-open this to workspace keys only once the * three lookups can express a workspace-scoped policy that fails closed. + * + * permission-group-exempt: which blocks a member may store is judged against allowedIntegrations inside replaceWorkflowNormalizedState, which this use case passes the principal's human subject to, not by a capability the authorization funnel can apply */ applyOperations: defineWorkspaceOperation({ id: 'workflows.operations.apply', minimumRole: 'write', workspaceApiKey: 'deny', + capability: 'none', ...HUMAN_WORKFLOW_PRINCIPAL_POLICY, }), + // permission-group-exempt: restoring a soft-deleted workflow is governed by workspace role restore: defineWorkspaceOperation({ id: 'workflows.restore', minimumRole: 'write', workspaceApiKey: 'allow', + capability: 'none', ...ALL_WORKFLOW_PRINCIPAL_POLICY, }), + // permission-group-exempt: a workflow's run policy is workspace-admin configuration; no group capability withholds it updatePolicy: defineWorkspaceOperation({ id: 'workflows.policy.update', minimumRole: 'admin', workspaceApiKey: 'deny', + capability: 'none', principalKinds: ['session'], }), + // permission-group-exempt: workflow variables are workflow content, governed by workspace role applyVariableOperations: defineWorkspaceOperation({ id: 'workflows.variables.apply_operations', minimumRole: 'write', workspaceApiKey: 'allow', + capability: 'none', ...ALL_WORKFLOW_PRINCIPAL_POLICY, }), + // permission-group-exempt: toggling a block edits workflow content; which integrations a member may use is allowedIntegrations, enforced against the block type rather than the operation setBlockEnabled: defineWorkspaceOperation({ id: 'workflows.blocks.set_enabled', minimumRole: 'write', workspaceApiKey: 'deny', + capability: 'none', ...COPILOT_WORKFLOW_PRINCIPAL_POLICY, }), + // permission-group-exempt: moving workflows between folders is placement, governed by workspace role moveBulk: defineWorkspaceOperation({ id: 'workflows.bulk.move', minimumRole: 'write', workspaceApiKey: 'allow', + capability: 'none', ...ALL_WORKFLOW_PRINCIPAL_POLICY, }), + // permission-group-exempt: the workflow file tree has no hide key; arranging it is governed by workspace role createVfsFolders: defineWorkspaceOperation({ id: 'workflows.vfs.folders.create', minimumRole: 'write', workspaceApiKey: 'deny', + capability: 'none', ...COPILOT_WORKFLOW_PRINCIPAL_POLICY, }), + // permission-group-exempt: the workflow file tree has no hide key; arranging it is governed by workspace role moveVfsItems: defineWorkspaceOperation({ id: 'workflows.vfs.move', minimumRole: 'write', workspaceApiKey: 'deny', + capability: 'none', ...COPILOT_WORKFLOW_PRINCIPAL_POLICY, }), + // permission-group-exempt: the workflow file tree has no hide key; arranging it is governed by workspace role copyVfsItems: defineWorkspaceOperation({ id: 'workflows.vfs.copy', minimumRole: 'write', workspaceApiKey: 'deny', + capability: 'none', ...COPILOT_WORKFLOW_PRINCIPAL_POLICY, }), + // permission-group-exempt: the workflow file tree has no hide key; arranging it is governed by workspace role deleteVfsItems: defineWorkspaceOperation({ id: 'workflows.vfs.delete', minimumRole: 'write', workspaceApiKey: 'deny', + capability: 'none', ...COPILOT_WORKFLOW_PRINCIPAL_POLICY, }), + // permission-group-exempt: duplicating copies a graph the caller may already read into the same workspace, so it crosses no capability boundary duplicate: defineWorkspaceOperation({ id: 'workflows.duplicate', minimumRole: 'write', workspaceApiKey: 'allow', + capability: 'none', ...ALL_WORKFLOW_PRINCIPAL_POLICY, }), + // permission-group-exempt: running a workflow is governed by workspace role; Chat itself is withheld by copilot.use at the chat surface runFromCopilot: defineWorkspaceOperation({ id: 'workflows.copilot.run', minimumRole: 'write', workspaceApiKey: 'deny', + capability: 'none', principalKinds: ['delegated'], delegatedServices: ['copilot'], }), + // permission-group-exempt: running a workflow is governed by workspace role; Chat itself is withheld by copilot.use at the chat surface runUntilFromCopilot: defineWorkspaceOperation({ id: 'workflows.copilot.run_until', minimumRole: 'write', workspaceApiKey: 'deny', + capability: 'none', ...COPILOT_WORKFLOW_PRINCIPAL_POLICY, }), + // permission-group-exempt: running a workflow is governed by workspace role; Chat itself is withheld by copilot.use at the chat surface runFromBlockFromCopilot: defineWorkspaceOperation({ id: 'workflows.copilot.run_from_block', minimumRole: 'write', workspaceApiKey: 'deny', + capability: 'none', ...COPILOT_WORKFLOW_PRINCIPAL_POLICY, }), + // permission-group-exempt: running a single block is governed by workspace role; Chat itself is withheld by copilot.use at the chat surface runBlockFromCopilot: defineWorkspaceOperation({ id: 'workflows.copilot.run_block', minimumRole: 'write', workspaceApiKey: 'deny', + capability: 'none', ...COPILOT_WORKFLOW_PRINCIPAL_POLICY, }), + // permission-group-exempt: deleting a workflow is governed by workspace role delete: defineWorkspaceOperation({ id: 'workflows.delete', minimumRole: 'write', workspaceApiKey: 'allow', + capability: 'none', ...ALL_WORKFLOW_PRINCIPAL_POLICY, }), + // permission-group-exempt: the workflow folder tree has no hide key; reading it is governed by workspace role listFolders: defineWorkspaceOperation({ id: 'workflows.folders.list', minimumRole: 'read', workspaceApiKey: 'allow', + capability: 'none', ...ALL_WORKFLOW_PRINCIPAL_POLICY, }), + // permission-group-exempt: the workflow folder tree has no hide key; arranging it is governed by workspace role createFolder: defineWorkspaceOperation({ id: 'workflows.folders.create', minimumRole: 'write', workspaceApiKey: 'allow', + capability: 'none', ...ALL_WORKFLOW_PRINCIPAL_POLICY, }), + // permission-group-exempt: the workflow folder tree has no hide key; arranging it is governed by workspace role relocateFolder: defineWorkspaceOperation({ id: 'workflows.folders.relocate', minimumRole: 'write', workspaceApiKey: 'allow', + capability: 'none', ...ALL_WORKFLOW_PRINCIPAL_POLICY, }), + // permission-group-exempt: the workflow folder tree has no hide key; arranging it is governed by workspace role deleteFolder: defineWorkspaceOperation({ id: 'workflows.folders.delete', minimumRole: 'write', workspaceApiKey: 'allow', + capability: 'none', ...ALL_WORKFLOW_PRINCIPAL_POLICY, }), deploy: defineWorkspaceOperation({ id: 'workflows.deploy', minimumRole: 'admin', workspaceApiKey: 'deny', + capability: 'deploy.api', ...WORKFLOW_DEPLOYMENT_PRINCIPAL_POLICY, }), undeploy: defineWorkspaceOperation({ id: 'workflows.undeploy', minimumRole: 'admin', workspaceApiKey: 'deny', + capability: 'deploy.api', ...WORKFLOW_DEPLOYMENT_PRINCIPAL_POLICY, }), deployChat: defineWorkspaceOperation({ id: 'workflows.chat.deploy', minimumRole: 'admin', workspaceApiKey: 'deny', + capability: 'deploy.chat', ...HUMAN_WORKFLOW_PRINCIPAL_POLICY, }), undeployChat: defineWorkspaceOperation({ id: 'workflows.chat.undeploy', minimumRole: 'admin', workspaceApiKey: 'deny', + capability: 'deploy.chat', ...HUMAN_WORKFLOW_PRINCIPAL_POLICY, }), /** @@ -263,89 +327,125 @@ export const workflowOperations = { * a principal here: the operation removes the authentication requirement from * a deployed workflow, which needs an accountable human rather than a machine * credential or an agent acting on a prompt. + * + * permission-group-exempt: `public_api.use` is asserted inside the use case and only for the enabling direction, because a group that withholds public execution must still let an admin withdraw execution a workflow already has */ updatePublicApi: defineWorkspaceOperation({ id: 'workflows.public_api.update', minimumRole: 'admin', workspaceApiKey: 'deny', + capability: 'none', principalKinds: ['session', 'personal_api_key'], }), activateVersion: defineWorkspaceOperation({ id: 'workflows.versions.activate', minimumRole: 'admin', workspaceApiKey: 'deny', + capability: 'deploy.api', ...WORKFLOW_DEPLOYMENT_PRINCIPAL_POLICY, }), + // permission-group-exempt: reverting the draft to an earlier version edits workflow content; deployment capabilities govern what is served, not what is edited revertVersion: defineWorkspaceOperation({ id: 'workflows.versions.revert', minimumRole: 'admin', workspaceApiKey: 'deny', + capability: 'none', ...HUMAN_WORKFLOW_PRINCIPAL_POLICY, }), + // permission-group-exempt: a version's name and description are metadata on workflow content, governed by workspace role updateVersion: defineWorkspaceOperation({ id: 'workflows.versions.update', minimumRole: 'write', workspaceApiKey: 'allow', + capability: 'none', ...ALL_WORKFLOW_PRINCIPAL_POLICY, }), + // permission-group-exempt: version history is workflow content, governed by workspace role listVersions: defineWorkspaceOperation({ id: 'workflows.versions.list', minimumRole: 'read', workspaceApiKey: 'allow', + capability: 'none', ...WORKFLOW_READ_PRINCIPAL_POLICY, }), + // permission-group-exempt: version history is workflow content, governed by workspace role readVersion: defineWorkspaceOperation({ id: 'workflows.versions.read', minimumRole: 'read', workspaceApiKey: 'allow', + capability: 'none', ...WORKFLOW_READ_PRINCIPAL_POLICY, }), + // permission-group-exempt: comparing references across two versions reads workflow content the caller may already open compareReferences: defineWorkspaceOperation({ id: 'workflows.versions.compare_references', minimumRole: 'read', workspaceApiKey: 'deny', + capability: 'none', ...COPILOT_WORKFLOW_PRINCIPAL_POLICY, }), + // permission-group-exempt: an export returns the graph its reader can already open; logs.export withholds execution logs, not definitions export: defineWorkspaceOperation({ id: 'workflows.export', minimumRole: 'read', workspaceApiKey: 'allow', + capability: 'none', ...ALL_WORKFLOW_PRINCIPAL_POLICY, }), + // permission-group-exempt: importing is workflow authoring governed by workspace role; the blocks the payload carries are judged against allowedIntegrations before they are persisted import: defineWorkspaceOperation({ id: 'workflows.import', minimumRole: 'write', workspaceApiKey: 'allow', + capability: 'none', ...ALL_WORKFLOW_PRINCIPAL_POLICY, }), + // permission-group-exempt: running a workflow from an authenticated surface is governed by workspace role; public_api.use withholds the unauthenticated surface, which does not reach this operation execute: defineWorkspaceOperation({ id: 'workflows.execute', minimumRole: 'read', workspaceApiKey: 'allow', + capability: 'none', ...ALL_WORKFLOW_PRINCIPAL_POLICY, }), + // permission-group-exempt: a manual run is governed by workspace role; public_api.use withholds the unauthenticated surface, which does not reach this operation executeManual: defineWorkspaceOperation({ id: 'workflows.manual.execute', minimumRole: 'write', workspaceApiKey: 'deny', + capability: 'none', principalKinds: ['personal_api_key'], }), + // permission-group-exempt: a manual run is governed by workspace role; public_api.use withholds the unauthenticated surface, which does not reach this operation executeManualFromBlock: defineWorkspaceOperation({ id: 'workflows.manual.execute_from_block', minimumRole: 'write', workspaceApiKey: 'deny', + capability: 'none', principalKinds: ['personal_api_key'], }), + // permission-group-exempt: execution history is governed by workspace role; logs.cost and logs.trace_spans withhold fields inside a run, not the right to read one listRuns: defineWorkspaceOperation({ id: 'workflows.runs.list', minimumRole: 'read', workspaceApiKey: 'allow', + capability: 'none', ...ALL_WORKFLOW_PRINCIPAL_POLICY, }), + // permission-group-exempt: execution history is governed by workspace role; logs.cost and logs.trace_spans withhold fields inside a run, not the right to read one readRun: defineWorkspaceOperation({ id: 'workflows.runs.read', minimumRole: 'read', workspaceApiKey: 'allow', + capability: 'none', + ...ALL_WORKFLOW_PRINCIPAL_POLICY, + }), + // permission-group-exempt: a paused execution's detail is pause points and resume state, not the run's execution data — the fields logs.cost and logs.trace_spans withhold never appear here + readPausedExecution: defineWorkspaceOperation({ + id: 'workflows.paused_executions.read', + minimumRole: 'read', + workspaceApiKey: 'allow', + capability: 'none', ...ALL_WORKFLOW_PRINCIPAL_POLICY, }), /** @@ -354,23 +454,43 @@ export const workflowOperations = { * the run resource does not; it keeps `readRun`'s policy because the resource * being authorized is still the run — a run file is reachable only through * the run that recorded it, never as a standalone workspace file. + * + * `logs.trace_spans` is deliberately not a gate here, and the distinction is + * the one that capability draws everywhere else: it withholds *fields* inside + * a run, not the right to read one. `readRun` therefore withholds the file + * *listing* from a viewer whose group hides execution data — the descriptors + * are that data, and `includeFileBase64` is its bytes — while this operation, + * which resolves one already-named file id, stays governed by workspace role. + * A run file id exists nowhere but the execution data the same projection + * withholds, so hiding the listing removes the way to name a file rather than + * the right to fetch a named one. + * + * If that ever needs to become a refusal rather than a projection, it belongs + * in `capability` on this operation, where the funnel applies it — not in a + * check at one of the surfaces that reach it. */ + // permission-group-exempt: a run's own output bytes belong to the run its reader may already open; files.bulk_download withholds the workspace file store, and logs.trace_spans withholds the run's listed fields — including readRun's file list — not the right to fetch one named file downloadRunFile: defineWorkspaceOperation({ id: 'workflows.download_run_file', minimumRole: 'read', workspaceApiKey: 'allow', + capability: 'none', ...ALL_WORKFLOW_PRINCIPAL_POLICY, }), + // permission-group-exempt: stopping a run already in flight is governed by workspace role cancelRun: defineWorkspaceOperation({ id: 'workflows.runs.cancel', minimumRole: 'write', workspaceApiKey: 'allow', + capability: 'none', ...ALL_WORKFLOW_PRINCIPAL_POLICY, }), + // permission-group-exempt: answering a paused run is governed by workspace role resumeRun: defineWorkspaceOperation({ id: 'workflows.runs.resume', minimumRole: 'write', workspaceApiKey: 'allow', + capability: 'none', ...ALL_WORKFLOW_PRINCIPAL_POLICY, }), } as const diff --git a/apps/sim/lib/workflows/application/read-paused-workflow-execution.test.ts b/apps/sim/lib/workflows/application/read-paused-workflow-execution.test.ts new file mode 100644 index 00000000000..1786ba37513 --- /dev/null +++ b/apps/sim/lib/workflows/application/read-paused-workflow-execution.test.ts @@ -0,0 +1,218 @@ +/** + * @vitest-environment node + */ +import type { Principal } from '@sim/auth/principal' +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const mocks = vi.hoisted(() => ({ + getPausedExecutionDetail: vi.fn(), + resolvePermission: vi.fn(), + resolveWorkflowContext: vi.fn(), +})) + +vi.mock('@sim/platform-authz/workspace', () => ({ + permissionSatisfies: (actual: string | null, required: string) => { + const rank = { read: 1, write: 2, admin: 3 } as const + return ( + actual !== null && rank[actual as keyof typeof rank] >= rank[required as keyof typeof rank] + ) + }, + resolveEffectiveWorkspacePermission: mocks.resolvePermission, +})) + +vi.mock('@/lib/workflows/application/context', () => ({ + resolveActiveWorkflowApplicationContext: mocks.resolveWorkflowContext, +})) + +vi.mock('@/lib/workflows/executor/human-in-the-loop-manager', () => ({ + PauseResumeManager: { + getPausedExecutionDetail: mocks.getPausedExecutionDetail, + }, +})) + +import { readPausedWorkflowExecution } from '@/lib/workflows/application/read-paused-workflow-execution' + +const workflowContext = { + workflowId: 'workflow-1', + workflow: { id: 'workflow-1' }, + workspaceId: 'workspace-1', + workspaceOrganizationId: null, + allowPersonalApiKeys: true, + billedAccountUserId: 'billing-owner-1', +} + +const detail = { + id: 'paused-1', + workflowId: 'workflow-1', + executionId: 'execution-1', +} + +const allowedPrincipals: Principal[] = [ + { kind: 'session', userId: 'user-1', sessionId: 'session-1' }, + { kind: 'personal_api_key', userId: 'user-1', keyId: 'personal-key-1' }, + { kind: 'workspace_api_key', workspaceId: 'workspace-1', keyId: 'workspace-key-1' }, + { + kind: 'delegated', + serviceId: 'copilot', + subjectUserId: 'user-1', + workspaceId: 'workspace-1', + delegationId: 'copilot-delegation-1', + audience: 'sim:workflows', + issuedAt: new Date('2026-01-01T00:00:00.000Z'), + expiresAt: new Date('2999-01-01T00:00:00.000Z'), + }, +] + +describe('readPausedWorkflowExecution', () => { + beforeEach(() => { + vi.clearAllMocks() + mocks.resolvePermission.mockResolvedValue('read') + mocks.resolveWorkflowContext.mockResolvedValue(workflowContext) + mocks.getPausedExecutionDetail.mockResolvedValue(detail) + }) + + it.each(allowedPrincipals)( + 'authorizes $kind before loading paused execution detail', + async (principal) => { + const result = await readPausedWorkflowExecution.execute({ + principal, + input: { workflowId: 'workflow-1', executionId: 'execution-1' }, + }) + + expect(result).toBe(detail) + expect(mocks.resolveWorkflowContext).toHaveBeenCalledWith({ workflowId: 'workflow-1' }) + expect(mocks.getPausedExecutionDetail).toHaveBeenCalledWith({ + workflowId: 'workflow-1', + executionId: 'execution-1', + }) + } + ) + + it('finishes session authorization before loading paused execution detail', async () => { + await readPausedWorkflowExecution.execute({ + principal: allowedPrincipals[0], + input: { workflowId: 'workflow-1', executionId: 'execution-1' }, + }) + + expect(mocks.resolvePermission.mock.invocationCallOrder[0]).toBeLessThan( + mocks.getPausedExecutionDetail.mock.invocationCallOrder[0] + ) + }) + + it('supports an authorization-only preflight without loading paused execution detail', async () => { + expect(readPausedWorkflowExecution.authorize).toBeTypeOf('function') + + await readPausedWorkflowExecution.authorize?.({ + principal: allowedPrincipals[0], + input: { workflowId: 'workflow-1', executionId: 'execution-1' }, + }) + + expect(mocks.resolveWorkflowContext).toHaveBeenCalledWith({ workflowId: 'workflow-1' }) + expect(mocks.resolvePermission).toHaveBeenCalled() + expect(mocks.getPausedExecutionDetail).not.toHaveBeenCalled() + }) + + it('rejects executor delegation before canonical lookup', async () => { + const principal: Principal = { + kind: 'delegated', + serviceId: 'executor', + workspaceId: 'workspace-1', + delegationId: 'execution-delegation-1', + audience: 'sim:workflows', + issuedAt: new Date('2026-01-01T00:00:00.000Z'), + expiresAt: new Date('2999-01-01T00:00:00.000Z'), + delegationContext: { kind: 'workflow_execution', workflowId: 'workflow-1' }, + } + + await expect( + readPausedWorkflowExecution.execute({ + principal, + input: { workflowId: 'workflow-1', executionId: 'execution-1' }, + }) + ).rejects.toMatchObject({ name: 'DelegatedServiceAuthorizationError' }) + expect(mocks.resolveWorkflowContext).not.toHaveBeenCalled() + expect(mocks.getPausedExecutionDetail).not.toHaveBeenCalled() + }) + + it('rejects a disallowed principal before canonical lookup', async () => { + const principal: Principal = { + kind: 'system', + serviceId: 'internal', + workspaceId: 'workspace-1', + workflowId: 'workflow-1', + } + + await expect( + readPausedWorkflowExecution.execute({ + principal, + input: { workflowId: 'workflow-1', executionId: 'execution-1' }, + }) + ).rejects.toMatchObject({ name: 'PrincipalKindAuthorizationError' }) + expect(mocks.resolveWorkflowContext).not.toHaveBeenCalled() + expect(mocks.getPausedExecutionDetail).not.toHaveBeenCalled() + }) + + it('rejects a workspace key outside the canonical workspace before loading detail', async () => { + await expect( + readPausedWorkflowExecution.execute({ + principal: { + kind: 'workspace_api_key', + workspaceId: 'workspace-2', + keyId: 'workspace-key-2', + }, + input: { workflowId: 'workflow-1', executionId: 'execution-1' }, + }) + ).rejects.toMatchObject({ code: 'forbidden' }) + expect(mocks.getPausedExecutionDetail).not.toHaveBeenCalled() + }) + + it('rejects a session without current workspace access before loading detail', async () => { + mocks.resolvePermission.mockResolvedValueOnce(null) + + await expect( + readPausedWorkflowExecution.execute({ + principal: allowedPrincipals[0], + input: { workflowId: 'workflow-1', executionId: 'execution-1' }, + }) + ).rejects.toMatchObject({ name: 'NoWorkspaceAccessError' }) + expect(mocks.getPausedExecutionDetail).not.toHaveBeenCalled() + }) + + it('enforces the workspace personal-key policy before loading detail', async () => { + mocks.resolveWorkflowContext.mockResolvedValueOnce({ + ...workflowContext, + allowPersonalApiKeys: false, + }) + + await expect( + readPausedWorkflowExecution.execute({ + principal: allowedPrincipals[1], + input: { workflowId: 'workflow-1', executionId: 'execution-1' }, + }) + ).rejects.toMatchObject({ name: 'PersonalApiKeysDisabledError' }) + expect(mocks.getPausedExecutionDetail).not.toHaveBeenCalled() + }) + + it('returns a semantic not-found error when no paused execution matches', async () => { + mocks.getPausedExecutionDetail.mockResolvedValueOnce(null) + + await expect( + readPausedWorkflowExecution.execute({ + principal: allowedPrincipals[0], + input: { workflowId: 'workflow-1', executionId: 'missing-execution' }, + }) + ).rejects.toMatchObject({ code: 'not_found', message: 'Paused execution not found' }) + }) + + it('propagates manager infrastructure failures', async () => { + const infrastructureError = new Error('database unavailable') + mocks.getPausedExecutionDetail.mockRejectedValueOnce(infrastructureError) + + await expect( + readPausedWorkflowExecution.execute({ + principal: allowedPrincipals[0], + input: { workflowId: 'workflow-1', executionId: 'execution-1' }, + }) + ).rejects.toBe(infrastructureError) + }) +}) diff --git a/apps/sim/lib/workflows/application/read-paused-workflow-execution.ts b/apps/sim/lib/workflows/application/read-paused-workflow-execution.ts new file mode 100644 index 00000000000..958bfdea234 --- /dev/null +++ b/apps/sim/lib/workflows/application/read-paused-workflow-execution.ts @@ -0,0 +1,24 @@ +import { OrchestrationError } from '@/lib/core/orchestration/types' +import { defineAuthorizedWorkflowUseCase } from '@/lib/workflows/application/authorized-workflow-use-case' +import { resolveActiveWorkflowApplicationContext } from '@/lib/workflows/application/context' +import { workflowOperations } from '@/lib/workflows/application/operations' +import { PauseResumeManager } from '@/lib/workflows/executor/human-in-the-loop-manager' + +export interface ReadPausedWorkflowExecutionInput { + workflowId: string + executionId: string +} + +export const readPausedWorkflowExecution = defineAuthorizedWorkflowUseCase({ + operation: workflowOperations.readPausedExecution, + resolveContext: ({ input }: { input: ReadPausedWorkflowExecutionInput }) => + resolveActiveWorkflowApplicationContext({ workflowId: input.workflowId }), + async execute({ context, input }) { + const detail = await PauseResumeManager.getPausedExecutionDetail({ + workflowId: context.workflowId, + executionId: input.executionId, + }) + if (!detail) throw new OrchestrationError('not_found', 'Paused execution not found') + return detail + }, +}) diff --git a/apps/sim/lib/workflows/application/read-workflow-copilot-metadata.ts b/apps/sim/lib/workflows/application/read-workflow-copilot-metadata.ts index ef6c2af4151..62e2a4f9de9 100644 --- a/apps/sim/lib/workflows/application/read-workflow-copilot-metadata.ts +++ b/apps/sim/lib/workflows/application/read-workflow-copilot-metadata.ts @@ -13,7 +13,7 @@ import { loadWorkflowFromNormalizedTables } from '@/lib/workflows/persistence/ut import { resolveTriggerRunOptions, toPublicRunOption } from '@/lib/workflows/triggers/run-options' import { hasTriggerCapability } from '@/lib/workflows/triggers/trigger-utils' import { getBlock } from '@/blocks/registry' -import { normalizeName } from '@/executor/constants' +import { isHumanInTheLoopBlock, normalizeName } from '@/executor/constants' const MAX_COPILOT_BLOCK_IDS = 100 @@ -240,7 +240,7 @@ export const readCopilotWorkflowUpstreamReferences = defineAuthorizedWorkflowUse for (const accessibleBlockId of accessibleIds) { const block = blocks[accessibleBlockId] if (!block?.type) continue - const canSelfReference = block.type === 'approval' || block.type === 'human_in_the_loop' + const canSelfReference = block.type === 'approval' || isHumanInTheLoopBlock(block.type) if (accessibleBlockId === blockId && !canSelfReference) continue const blockName = block.name || block.type diff --git a/apps/sim/lib/workflows/application/read-workflow-run.test.ts b/apps/sim/lib/workflows/application/read-workflow-run.test.ts index a4a1d7cb735..10e8e25c54c 100644 --- a/apps/sim/lib/workflows/application/read-workflow-run.test.ts +++ b/apps/sim/lib/workflows/application/read-workflow-run.test.ts @@ -32,7 +32,7 @@ vi.mock('@/lib/workflows/application/context', () => ({ resolveActiveWorkflowRunApplicationContext: mocks.resolveContext, })) vi.mock('@/lib/workflows/executor/execution-status', () => ({ - getWorkflowExecutionStatus: mocks.getStatus, + getProjectedWorkflowExecutionStatus: mocks.getStatus, })) vi.mock('@/lib/workflows/executor/execution-run-files', () => ({ getWorkflowRunFiles: mocks.getRunFiles, @@ -53,29 +53,81 @@ const context = { const principal = { kind: 'personal_api_key' as const, userId: 'user-1', keyId: 'key-1' } +const NO_PROJECTION = { hideTraceSpans: false, hideCostInfo: false } + const BLOCK_ID = '2f9c2d4e-1a3b-4c5d-8e7f-0a1b2c3d4e5f' function input(selectedOutputs: string[]) { return { workflowId: 'workflow-1', runId: 'run-1', includeOutput: true, selectedOutputs } } +/** + * `logs.cost` and `logs.trace_spans` withhold fields inside a run, and the shared + * read applies them — but only for the subject this use case names. A workspace + * API key authorizes as the workspace and represents no user, so it must resolve + * to none: substituting the key's creator would apply a bystander's group to + * every caller of a shared credential. + */ +describe('readWorkflowRun projection subject', () => { + beforeEach(() => { + vi.clearAllMocks() + mocks.resolveContext.mockResolvedValue(context) + mocks.resolvePermission.mockResolvedValue('read') + mocks.getRunFiles.mockResolvedValue(null) + mocks.getStatus.mockResolvedValue({ + status: { status: 'completed', blockOutputs: {} }, + projection: NO_PROJECTION, + }) + }) + + it('names the acting user as the projection subject', async () => { + await readWorkflowRun.execute({ principal, input: input([]) }) + + expect(mocks.getStatus).toHaveBeenCalledWith( + expect.objectContaining({ + workspaceId: 'workspace-1', + workspaceOrganizationId: null, + viewerUserId: 'user-1', + }) + ) + }) + + it('names no subject for a workspace API key', async () => { + const workspaceKey = { + kind: 'workspace_api_key' as const, + workspaceId: 'workspace-1', + keyId: 'key-1', + } + + await readWorkflowRun.execute({ principal: workspaceKey, input: input([]) }) + + expect(mocks.getStatus).toHaveBeenCalledWith(expect.objectContaining({ viewerUserId: null })) + }) +}) + describe('readWorkflowRun selector resolution', () => { beforeEach(() => { vi.clearAllMocks() mocks.resolveContext.mockResolvedValue(context) mocks.resolvePermission.mockResolvedValue('read') mocks.getRunFiles.mockResolvedValue(null) - mocks.getStatus.mockResolvedValue({ status: 'completed', blockOutputs: {} }) + mocks.getStatus.mockResolvedValue({ + status: { status: 'completed', blockOutputs: {} }, + projection: NO_PROJECTION, + }) }) it('rejects a block-name selector against a recorded output projection', async () => { mocks.getStatus.mockResolvedValue({ - status: 'completed', - blockOutputs: { [BLOCK_ID]: { content: 'hi' } }, + status: { status: 'completed', blockOutputs: { [BLOCK_ID]: { content: 'hi' } } }, + projection: NO_PROJECTION, + }) + await expect( + readWorkflowRun.execute({ principal, input: input(['Agent 1']) }) + ).rejects.toMatchObject({ + code: 'validation', + message: expect.stringContaining('did not resolve to any block on this run: Agent 1'), }) - await expect(readWorkflowRun.execute({ principal, input: input(['Agent 1']) })).rejects.toThrow( - /did not resolve to any block on this run: Agent 1/ - ) }) /** @@ -84,14 +136,20 @@ describe('readWorkflowRun selector resolution', () => { * nothing selected — the silent empty answer the check exists to remove. */ it('rejects a block-name selector on a run with no recorded output projection', async () => { - mocks.getStatus.mockResolvedValue({ status: 'queued', blockOutputs: null }) + mocks.getStatus.mockResolvedValue({ + status: { status: 'queued', blockOutputs: null }, + projection: NO_PROJECTION, + }) await expect(readWorkflowRun.execute({ principal, input: input(['Agent 1']) })).rejects.toThrow( /did not resolve to any block on this run: Agent 1/ ) }) it('accepts a well-formed block id on a run with no recorded output projection', async () => { - mocks.getStatus.mockResolvedValue({ status: 'queued', blockOutputs: null }) + mocks.getStatus.mockResolvedValue({ + status: { status: 'queued', blockOutputs: null }, + projection: NO_PROJECTION, + }) await expect( readWorkflowRun.execute({ principal, input: input([`${BLOCK_ID}.content`]) }) ).resolves.toMatchObject({ status: 'queued', blockOutputs: null }) @@ -103,3 +161,63 @@ describe('readWorkflowRun selector resolution', () => { ).resolves.toMatchObject({ status: 'completed', blockOutputs: {} }) }) }) + +/** + * A run's output files are its execution data: the descriptors name what the + * run produced and `includeFileBase64` returns the bytes. When the viewer's + * group withholds execution data under `logs.trace_spans`, the file list has to + * go with `finalOutput` and `blockOutputs` — otherwise the withheld output + * comes back one field over. + */ +describe('readWorkflowRun file projection', () => { + const WITHHELD = { hideTraceSpans: true, hideCostInfo: false } + + beforeEach(() => { + vi.clearAllMocks() + mocks.resolveContext.mockResolvedValue(context) + mocks.resolvePermission.mockResolvedValue('read') + mocks.getRunFiles.mockResolvedValue({ + terminal: true, + workspaceId: 'workspace-1', + filesById: new Map([['file-1', { key: 'k', name: 'out.csv' }]]), + }) + mocks.describeRunFiles.mockResolvedValue([{ id: 'file-1', name: 'out.csv' }]) + }) + + it('lists the run files when nothing is withheld', async () => { + mocks.getStatus.mockResolvedValue({ + status: { status: 'completed', blockOutputs: {}, finalOutput: { ok: true } }, + projection: NO_PROJECTION, + }) + + await expect(readWorkflowRun.execute({ principal, input: input([]) })).resolves.toMatchObject({ + files: [{ id: 'file-1' }], + }) + }) + + it('withholds the file list when the group withholds execution data', async () => { + mocks.getStatus.mockResolvedValue({ + status: { status: 'completed', blockOutputs: null, finalOutput: null }, + projection: WITHHELD, + }) + + await expect(readWorkflowRun.execute({ principal, input: input([]) })).resolves.toMatchObject({ + files: null, + }) + }) + + it('does not read the run files at all when the group withholds execution data', async () => { + mocks.getStatus.mockResolvedValue({ + status: { status: 'completed', blockOutputs: null, finalOutput: null }, + projection: WITHHELD, + }) + + await readWorkflowRun.execute({ + principal, + input: { ...input([]), includeFileBase64: true }, + }) + + expect(mocks.getRunFiles).not.toHaveBeenCalled() + expect(mocks.describeRunFiles).not.toHaveBeenCalled() + }) +}) diff --git a/apps/sim/lib/workflows/application/read-workflow-run.ts b/apps/sim/lib/workflows/application/read-workflow-run.ts index 8674a45208b..cc235d56c7c 100644 --- a/apps/sim/lib/workflows/application/read-workflow-run.ts +++ b/apps/sim/lib/workflows/application/read-workflow-run.ts @@ -4,6 +4,7 @@ import { FUNCTIONAL_OUTPUTS_UNAVAILABLE_MESSAGE, FunctionalOutputsUnavailableError, } from '@/lib/logs/execution/functional-outputs' +import { logProjectionSubjectUserId } from '@/lib/logs/log-projection' import { defineAuthorizedWorkflowUseCase } from '@/lib/workflows/application/authorized-workflow-use-case' import { resolveActiveWorkflowRunApplicationContext } from '@/lib/workflows/application/context' import { workflowOperations } from '@/lib/workflows/application/operations' @@ -12,7 +13,7 @@ import { getWorkflowRunFiles, type WorkflowRunFileDescriptor, } from '@/lib/workflows/executor/execution-run-files' -import { getWorkflowExecutionStatus } from '@/lib/workflows/executor/execution-status' +import { getProjectedWorkflowExecutionStatus } from '@/lib/workflows/executor/execution-status' /** * Selectors this resource can never answer, so the caller hears about them. @@ -59,16 +60,32 @@ export const readWorkflowRun = defineAuthorizedWorkflowUseCase({ runId: input.runId, assertedWorkflowId: input.workflowId, }), - async execute({ context, input }) { + async execute({ principal, context, input }) { try { - const status = await getWorkflowExecutionStatus({ + /** + * The projection subject, not an attribution: a workspace API key + * authorizes as the workspace and represents no user, so it resolves to + * `undefined` and reads the run whole. Substituting the key's creator would + * apply a bystander's group to every caller of a shared credential. + */ + const projected = await getProjectedWorkflowExecutionStatus({ workflowId: context.workflowId, executionId: context.runId, includeOutput: input.includeOutput, selectedOutputs: input.selectedOutputs, + workspaceId: context.workspaceId, + workspaceOrganizationId: context.workspaceOrganizationId, + viewerUserId: logProjectionSubjectUserId(principal), }) - if (!status) throw new OrchestrationError('not_found', 'Run not found') + if (!projected) throw new OrchestrationError('not_found', 'Run not found') + const { status, projection } = projected + /** + * A run whose `blockOutputs` the viewer's group withholds joins the same + * set as a queued or `includeOutput: false` run: the selector is judged on + * its shape alone. A block *name* still hears that this resource matches + * ids, and a well-formed id still gets the legitimate empty answer. + */ const unresolvable = unresolvableSelectors(input.selectedOutputs, status.blockOutputs) if (unresolvable.length > 0) { throw new OrchestrationError( @@ -83,14 +100,24 @@ export const readWorkflowRun = defineAuthorizedWorkflowUseCase({ * a list it did not request. Derived from the run's own recording, which * is also where the download endpoint re-derives each storage key. * + * They follow the viewer's projection for the same reason. A run's output + * files *are* its execution data — the descriptors name them, and + * `includeFileBase64` hands back their bytes — so a group that withholds + * `finalOutput` and `blockOutputs` under `logs.trace_spans` and then let + * the file list through would return the withheld output one field over. + * The list is `null`, exactly as for a caller that asked for no output, + * and the read is skipped rather than performed and discarded. + * * This re-reads the run rather than reusing what the status read already * loaded, and must: the status read materializes execution data *for * display*, a projection that strips `key` and `context` — exactly the * fields a file descriptor needs — and it also answers from the job queue * for runs that have no log row yet. + * + * permission-group-enforced: logs.trace_spans */ let files: WorkflowRunFileDescriptor[] | null = null - if (input.includeOutput) { + if (input.includeOutput && !projection.hideTraceSpans) { const runFiles = await getWorkflowRunFiles({ workflowId: context.workflowId, runId: context.runId, diff --git a/apps/sim/lib/workflows/application/replace-workflow-state.test.ts b/apps/sim/lib/workflows/application/replace-workflow-state.test.ts index 8e3354813f6..fc8a9171360 100644 --- a/apps/sim/lib/workflows/application/replace-workflow-state.test.ts +++ b/apps/sim/lib/workflows/application/replace-workflow-state.test.ts @@ -137,6 +137,7 @@ describe('replaceWorkflowState', () => { }) expect(mocks.replace).toHaveBeenCalledWith({ + subjectUserId: 'user-1', workflowId: 'workflow-1', workspaceId: 'workspace-1', attributedUserId: 'user-1', diff --git a/apps/sim/lib/workflows/application/replace-workflow-state.ts b/apps/sim/lib/workflows/application/replace-workflow-state.ts index 627fce476a4..5cc97b8d2da 100644 --- a/apps/sim/lib/workflows/application/replace-workflow-state.ts +++ b/apps/sim/lib/workflows/application/replace-workflow-state.ts @@ -135,10 +135,12 @@ export const replaceWorkflowState = defineAuthorizedWorkflowUseCase({ * the reference pass is skipped for them rather than resolved against the * billing owner. See {@link buildWorkflowLintReport}. */ + const subjectUserId = humanSubjectUserId(principal) + const lint = await buildWorkflowLintReport(graph, { workflowId: context.workflowId, workspaceId: context.workspaceId, - subjectUserId: humanSubjectUserId(principal), + subjectUserId, }) if (input.dryRun) { @@ -180,6 +182,15 @@ export const replaceWorkflowState = defineAuthorizedWorkflowUseCase({ workflowId: context.workflowId, workspaceId: context.workspaceId, attributedUserId: attribution.attributedUserId, + /** + * The same human the lint pass resolved above, and never + * `attribution.attributedUserId`: that answers a workspace API key with + * the billing owner, so reusing it would judge a caller-supplied graph + * against a bystander's grants. This operation admits only principals + * that name a human, so the `null` branch is a fail-safe rather than a + * reachable state. + */ + subjectUserId, state: { blocks: graph.blocks, edges: graph.edges, diff --git a/apps/sim/lib/workflows/application/resolve-workflow-outputs.test.ts b/apps/sim/lib/workflows/application/resolve-workflow-outputs.test.ts index 6ccf06f983a..f750feea8b8 100644 --- a/apps/sim/lib/workflows/application/resolve-workflow-outputs.test.ts +++ b/apps/sim/lib/workflows/application/resolve-workflow-outputs.test.ts @@ -7,7 +7,9 @@ import { OrchestrationError } from '@/lib/core/orchestration/types' const mocks = vi.hoisted(() => ({ flatten: vi.fn(), + loadDeployed: vi.fn(), load: vi.fn(), + NoActiveDeploymentError: class NoActiveDeploymentError extends Error {}, order: vi.fn(), resolveContext: vi.fn(), resolvePermission: vi.fn(), @@ -33,10 +35,15 @@ vi.mock('@/lib/workflows/blocks/flatten-outputs', () => ({ })) vi.mock('@/lib/workflows/persistence/utils', () => ({ + NoActiveDeploymentError: mocks.NoActiveDeploymentError, + loadDeployedWorkflowState: mocks.loadDeployed, loadWorkflowFromNormalizedTables: mocks.load, })) -import { resolveWorkflowOutputs } from '@/lib/workflows/application/resolve-workflow-outputs' +import { + loadResolvedDeployedWorkflowOutputs, + resolveWorkflowOutputs, +} from '@/lib/workflows/application/resolve-workflow-outputs' const principal = { kind: 'delegated' as const, @@ -59,12 +66,16 @@ describe('resolveWorkflowOutputs', () => { workspaceOrganizationId: null, allowPersonalApiKeys: true, billedAccountUserId: 'billing-owner-1', - workflow: { id: 'workflow-1' }, + workflow: { id: 'workflow-1', isDeployed: true }, }) mocks.load.mockResolvedValue({ blocks: { block1: { id: 'block-1', type: 'agent', name: 'Agent', subBlocks: {} } }, edges: [], }) + mocks.loadDeployed.mockResolvedValue({ + blocks: { block1: { id: 'block-1', type: 'agent', name: 'Agent', subBlocks: {} } }, + edges: [], + }) mocks.flatten.mockReturnValue([ { blockId: 'block-1', @@ -110,6 +121,42 @@ describe('resolveWorkflowOutputs', () => { expect(mocks.load).not.toHaveBeenCalled() }) + it('resolves table mappings from the active deployment state', async () => { + const context = await mocks.resolveContext() + + await expect(loadResolvedDeployedWorkflowOutputs(context)).resolves.toMatchObject({ + workflowId: 'workflow-1', + outputs: [{ blockId: 'block-1', path: 'content' }], + }) + + expect(mocks.loadDeployed).toHaveBeenCalledWith('workflow-1', 'workspace-1') + expect(mocks.load).not.toHaveBeenCalled() + }) + + it('rejects a workflow without an active deployment before resolving mappings', async () => { + const context = { + ...(await mocks.resolveContext()), + workflow: { id: 'workflow-1', isDeployed: false }, + } + + await expect(loadResolvedDeployedWorkflowOutputs(context)).rejects.toMatchObject({ + code: 'validation', + message: 'Workflow must have an active deployment', + }) + expect(mocks.loadDeployed).not.toHaveBeenCalled() + }) + + it('rejects inconsistent deployment metadata without returning draft mappings', async () => { + const context = await mocks.resolveContext() + mocks.loadDeployed.mockRejectedValueOnce(new mocks.NoActiveDeploymentError()) + + await expect(loadResolvedDeployedWorkflowOutputs(context)).rejects.toMatchObject({ + code: 'validation', + message: 'Workflow must have an active deployment', + }) + expect(mocks.load).not.toHaveBeenCalled() + }) + it('rejects expired delegated scope before loading workflow state', async () => { await expect( resolveWorkflowOutputs.execute({ diff --git a/apps/sim/lib/workflows/application/resolve-workflow-outputs.ts b/apps/sim/lib/workflows/application/resolve-workflow-outputs.ts index 209d26107f6..cb60823c006 100644 --- a/apps/sim/lib/workflows/application/resolve-workflow-outputs.ts +++ b/apps/sim/lib/workflows/application/resolve-workflow-outputs.ts @@ -1,3 +1,4 @@ +import { OrchestrationError } from '@/lib/core/orchestration/types' import { defineAuthorizedWorkflowUseCase } from '@/lib/workflows/application/authorized-workflow-use-case' import { type ActiveWorkflowApplicationContext, @@ -9,7 +10,11 @@ import { flattenWorkflowOutputs, getBlockExecutionOrder, } from '@/lib/workflows/blocks/flatten-outputs' -import { loadWorkflowFromNormalizedTables } from '@/lib/workflows/persistence/utils' +import { + loadDeployedWorkflowState, + loadWorkflowFromNormalizedTables, + NoActiveDeploymentError, +} from '@/lib/workflows/persistence/utils' export interface ResolveWorkflowOutputsInput { workflowId: string @@ -22,14 +27,14 @@ export interface ResolveWorkflowOutputsResult { executionOrderByBlockId: Record } -/** Loads output metadata after a top-level application command has authorized this workflow context. */ -export async function loadResolvedWorkflowOutputs( - context: ActiveWorkflowApplicationContext -): Promise { - const normalized = await loadWorkflowFromNormalizedTables(context.workflowId) - if (!normalized) { - return { workflowId: context.workflowId, outputs: null, executionOrderByBlockId: {} } - } +type ResolvableWorkflowState = + | NonNullable>> + | Awaited> + +function resolveWorkflowOutputsFromState( + workflowId: string, + normalized: ResolvableWorkflowState +): ResolveWorkflowOutputsResult { const blocks = Object.values(normalized.blocks ?? {}).map((block) => ({ id: block.id, type: block.type, @@ -38,12 +43,41 @@ export async function loadResolvedWorkflowOutputs( subBlocks: block.subBlocks as Record | undefined, })) return { - workflowId: context.workflowId, + workflowId, outputs: flattenWorkflowOutputs(blocks, normalized.edges ?? []), executionOrderByBlockId: getBlockExecutionOrder(blocks, normalized.edges ?? []), } } +/** Loads output metadata after a top-level application command has authorized this workflow context. */ +export async function loadResolvedWorkflowOutputs( + context: ActiveWorkflowApplicationContext +): Promise { + const normalized = await loadWorkflowFromNormalizedTables(context.workflowId) + if (!normalized) { + return { workflowId: context.workflowId, outputs: null, executionOrderByBlockId: {} } + } + return resolveWorkflowOutputsFromState(context.workflowId, normalized) +} + +/** Loads output metadata from the active deployment after workflow authorization. */ +export async function loadResolvedDeployedWorkflowOutputs( + context: ActiveWorkflowApplicationContext +): Promise { + if (!context.workflow.isDeployed) { + throw new OrchestrationError('validation', 'Workflow must have an active deployment') + } + try { + const normalized = await loadDeployedWorkflowState(context.workflowId, context.workspaceId) + return resolveWorkflowOutputsFromState(context.workflowId, normalized) + } catch (error) { + if (error instanceof NoActiveDeploymentError) { + throw new OrchestrationError('validation', 'Workflow must have an active deployment') + } + throw error + } +} + export const resolveWorkflowOutputs = defineAuthorizedWorkflowUseCase({ operation: workflowOperations.read, resolveContext: ({ input }: { input: ResolveWorkflowOutputsInput }) => diff --git a/apps/sim/lib/workflows/application/restore-workflow.test.ts b/apps/sim/lib/workflows/application/restore-workflow.test.ts index def7c6931a9..bba5720353d 100644 --- a/apps/sim/lib/workflows/application/restore-workflow.test.ts +++ b/apps/sim/lib/workflows/application/restore-workflow.test.ts @@ -10,6 +10,7 @@ const mocks = vi.hoisted(() => ({ resolveContext: vi.fn(), resolvePermission: vi.fn(), notify: vi.fn(), + notifyWorkspace: vi.fn(), restoreRecord: vi.fn(), folderIndex: vi.fn(), })) @@ -33,7 +34,10 @@ vi.mock('@sim/platform-authz/workspace', () => ({ vi.mock('@/lib/workflows/application/context', () => ({ resolveArchivedWorkflowApplicationContext: mocks.resolveContext, })) -vi.mock('@/lib/realtime/notify', () => ({ notifyWorkflowUpdated: mocks.notify })) +vi.mock('@/lib/realtime/notify', () => ({ + notifyWorkflowUpdated: mocks.notify, + notifyWorkspaceWorkflowsChanged: mocks.notifyWorkspace, +})) vi.mock('@/lib/workflows/lifecycle', () => ({ restoreWorkflow: mocks.restoreRecord })) vi.mock('@/lib/folders/queries', () => ({ loadActiveFolderPathIndex: mocks.folderIndex })) @@ -87,6 +91,8 @@ describe('restoreWorkflow', () => { }) ) expect(mocks.recordAudit).toHaveBeenCalledBefore(mocks.notify) + expect(mocks.notify).toHaveBeenCalledWith('workflow-1') + expect(mocks.notifyWorkspace).toHaveBeenCalledWith('workspace-1') }) it('refuses a workflow that is not archived as a conflict', async () => { diff --git a/apps/sim/lib/workflows/application/restore-workflow.ts b/apps/sim/lib/workflows/application/restore-workflow.ts index 44573566e88..3d8149aaf40 100644 --- a/apps/sim/lib/workflows/application/restore-workflow.ts +++ b/apps/sim/lib/workflows/application/restore-workflow.ts @@ -11,7 +11,7 @@ import { OrchestrationError } from '@/lib/core/orchestration/types' import { generateRequestId } from '@/lib/core/utils/request' import { MAX_FOLDERS_PER_WORKSPACE } from '@/lib/folders/constants' import { loadActiveFolderPathIndex } from '@/lib/folders/queries' -import { notifyWorkflowUpdated } from '@/lib/realtime/notify' +import { notifyWorkflowUpdated, notifyWorkspaceWorkflowsChanged } from '@/lib/realtime/notify' import { defineAuthorizedWorkflowUseCase } from '@/lib/workflows/application/authorized-workflow-use-case' import { resolveArchivedWorkflowApplicationContext } from '@/lib/workflows/application/context' import { workflowOperations } from '@/lib/workflows/application/operations' @@ -94,5 +94,10 @@ export const restoreWorkflow = defineAuthorizedWorkflowUseCase({ source: principalAuditSource(principal), }, }), - afterSuccess: ({ context }) => notifyWorkflowUpdated(context.workflowId), + async afterSuccess({ context }) { + await Promise.all([ + notifyWorkflowUpdated(context.workflowId), + notifyWorkspaceWorkflowsChanged(context.workspaceId), + ]) + }, }) diff --git a/apps/sim/lib/workflows/application/run-workflow-from-copilot.test.ts b/apps/sim/lib/workflows/application/run-workflow-from-copilot.test.ts index 593516912b4..d8976c7264e 100644 --- a/apps/sim/lib/workflows/application/run-workflow-from-copilot.test.ts +++ b/apps/sim/lib/workflows/application/run-workflow-from-copilot.test.ts @@ -391,4 +391,133 @@ describe('Copilot workflow run application commands', () => { expect(readAttemptedExecutionId(error)).toBeUndefined() }) }) + + describe('failed-run provenance crossing', () => { + function trackingLifecycle() { + const importCrossingProvenance = vi.fn().mockResolvedValue(true) + return { + importCrossingProvenance, + lifecycle: { + resolvedSecretTraceRegistry: { + exportProvenanceForValue: vi.fn(() => undefined), + beginPendingActivation: vi.fn(() => vi.fn()), + importCrossingProvenance, + }, + }, + } + } + + async function runExpectingFailure(input: { lifecycle: unknown }) { + await expect( + runWorkflowFromCopilot.execute({ + principal, + input: { + workflowId: 'workflow-1', + useDraftState: true, + lifecycle: input.lifecycle, + hasWorkflowInput: false, + useMockPayload: true, + }, + }) + ).rejects.toThrow() + } + + /** + * The executor attaches its result to every throw, so a failure without one never reached a + * block. Nothing crossed, and saying so keeps the caller's tool result — and the reason its + * run could not start — instead of reducing it to "result unavailable". + */ + it('vouches for a failure that never reached the engine', async () => { + const { importCrossingProvenance, lifecycle: tracked } = trackingLifecycle() + mocks.executeWorkflow.mockRejectedValueOnce(new Error('workflow is not deployed')) + + await runExpectingFailure({ lifecycle: tracked }) + + expect(importCrossingProvenance).toHaveBeenCalledWith( + { version: 1, complete: true, entries: [] }, + expect.objectContaining({ thrownMessage: 'workflow is not deployed' }), + expect.objectContaining({ origin: 'copilotWorkflowMutation.failedRunCrossing' }) + ) + }) + + /** + * The post-run crossing is inside the same try, so its failure reaches the catch with no + * execution result — the same evidence a never-started run leaves. An execution exists and + * its provenance was never imported, so this must not be vouched for. + */ + it('does not vouch when the crossing threw after the run returned', async () => { + const importCrossingProvenance = vi + .fn() + .mockImplementationOnce(() => { + throw new Error('crossing import failed') + }) + .mockResolvedValue(true) + + await runExpectingFailure({ + lifecycle: { + resolvedSecretTraceRegistry: { + exportProvenanceForValue: vi.fn(() => undefined), + beginPendingActivation: vi.fn(() => vi.fn()), + importCrossingProvenance, + }, + }, + }) + + expect(importCrossingProvenance).toHaveBeenNthCalledWith( + 2, + undefined, + expect.objectContaining({ thrownMessage: 'crossing import failed' }), + expect.objectContaining({ origin: 'copilotWorkflowMutation.failedRunCrossing' }) + ) + }) + + /** + * The executor's post-execution work can throw after a run has already produced a result. + * `executeWorkflow` carries it on that throw, so this reaches the catch with a result and + * must not be claimed as never-started. + */ + it('does not vouch when post-execution work threw after the engine ran', async () => { + const { importCrossingProvenance, lifecycle: tracked } = trackingLifecycle() + const incomplete = { version: 1 as const, complete: false, entries: [] } + mocks.executeWorkflow.mockRejectedValueOnce( + Object.assign(new Error('post-execution persistence failed'), { + executionResult: { + success: true, + output: { ran: true }, + executionState: { resolvedSecretTraceProvenance: incomplete }, + }, + }) + ) + + await runExpectingFailure({ lifecycle: tracked }) + + expect(importCrossingProvenance).toHaveBeenCalledWith( + incomplete, + expect.objectContaining({ output: { ran: true } }), + expect.objectContaining({ origin: 'copilotWorkflowMutation.failedRunCrossing' }) + ) + }) + + /** A run that did execute and could not vouch still hands back its incomplete envelope. */ + it('passes through an incomplete envelope from a run that did execute', async () => { + const { importCrossingProvenance, lifecycle: tracked } = trackingLifecycle() + const incomplete = { version: 1 as const, complete: false, entries: [] } + const failure = Object.assign(new Error('block failed'), { + executionResult: { + success: false, + output: { partial: true }, + executionState: { resolvedSecretTraceProvenance: incomplete }, + }, + }) + mocks.executeWorkflow.mockRejectedValueOnce(failure) + + await runExpectingFailure({ lifecycle: tracked }) + + expect(importCrossingProvenance).toHaveBeenCalledWith( + incomplete, + expect.objectContaining({ output: { partial: true } }), + expect.objectContaining({ origin: 'copilotWorkflowMutation.failedRunCrossing' }) + ) + }) + }) }) diff --git a/apps/sim/lib/workflows/application/run-workflow-from-copilot.ts b/apps/sim/lib/workflows/application/run-workflow-from-copilot.ts index 6467156f445..31ed03811fc 100644 --- a/apps/sim/lib/workflows/application/run-workflow-from-copilot.ts +++ b/apps/sim/lib/workflows/application/run-workflow-from-copilot.ts @@ -29,11 +29,14 @@ import { } from '@/lib/workflows/triggers/run-options' import type { SerializableExecutionState } from '@/executor/execution/types' import type { ExecutionResult } from '@/executor/types' -import { attachAttemptedExecutionId } from '@/executor/utils/errors' +import { attachAttemptedExecutionId, hasExecutionResult } from '@/executor/utils/errors' const logger = createLogger('CopilotWorkflowRun') -import type { ResolvedSecretTraceRegistry } from '@/executor/utils/resolved-secret-trace-registry' +import { + emptyResolvedSecretTraceProvenance, + type ResolvedSecretTraceRegistry, +} from '@/executor/utils/resolved-secret-trace-registry' export interface CopilotWorkflowRunLifecycle { billingAttribution?: BillingAttributionSnapshot @@ -250,6 +253,13 @@ async function executeCopilotRun(params: { params.executionInput ) const completePendingActivation = registry?.beginPendingActivation() + /** + * The run's own result, once the executor returns it. The post-run crossing below is inside the + * same `try`, so its failure reaches the catch carrying nothing — and on that evidence alone it + * is indistinguishable from a run that never started. Holding the result here keeps the real + * envelope available to describe content that certainly exists. + */ + let runResult: ExecutionResult | undefined /** * The executor call is the first statement of this `try`, so everything caught below is * post-dispatch by construction, while authorization, admission and provenance export all @@ -302,6 +312,7 @@ async function executeCopilotRun(params: { }, childExecutionId ) + runResult = result if (registry) { await registry.importCrossingProvenance( result.executionState?.resolvedSecretTraceProvenance, @@ -325,16 +336,23 @@ async function executeCopilotRun(params: { * as never started and invite the duplicate this id exists to prevent. */ if (registry) { - const executionResult = - typeof error === 'object' && - error !== null && - 'executionResult' in error && - typeof error.executionResult === 'object' - ? (error.executionResult as ExecutionResult) - : undefined + /** + * Either source counts as proof a run exists: the error carries the result when the run or + * its post-execution work threw, and `runResult` holds it when the failure came later still + * — from the crossing below, after the executor had already returned. + */ + const executionResult = hasExecutionResult(error) ? error.executionResult : runResult try { + /** + * Only a failure with no result from either source can claim nothing ran, and saying so + * keeps the caller's failure reason instead of reducing the tool result to "result + * unavailable" for a message that named no secret because none had been resolved yet. + * Every other failure hands back the envelope it has, and an incomplete one still latches. + */ await registry.importCrossingProvenance( - executionResult?.executionState?.resolvedSecretTraceProvenance, + executionResult + ? executionResult.executionState?.resolvedSecretTraceProvenance + : emptyResolvedSecretTraceProvenance(), { output: executionResult?.output, logs: executionResult?.logs, diff --git a/apps/sim/lib/workflows/application/update-workflow-content.test.ts b/apps/sim/lib/workflows/application/update-workflow-content.test.ts index 86383f35fc8..9b93febdf20 100644 --- a/apps/sim/lib/workflows/application/update-workflow-content.test.ts +++ b/apps/sim/lib/workflows/application/update-workflow-content.test.ts @@ -230,6 +230,7 @@ describe('setWorkflowBlockEnabled', () => { ).resolves.toMatchObject({ changed: true, affectedBlockIds: ['block-1'] }) expect(mocks.replace).toHaveBeenCalledWith({ + subjectUserId: null, workflowId: 'workflow-1', workspaceId: 'workspace-1', attributedUserId: 'user-1', diff --git a/apps/sim/lib/workflows/application/update-workflow-content.ts b/apps/sim/lib/workflows/application/update-workflow-content.ts index 0ae8ef511a1..a50cda3848b 100644 --- a/apps/sim/lib/workflows/application/update-workflow-content.ts +++ b/apps/sim/lib/workflows/application/update-workflow-content.ts @@ -238,6 +238,18 @@ export const setWorkflowBlockEnabled = defineAuthorizedWorkflowUseCase({ workflowId: context.workflowId, workspaceId: context.workspaceId, attributedUserId: attribution.attributedUserId, + /** + * Actorless on purpose. This operation writes back the graph it just read + * under the row lock with one block's `enabled` flipped — the caller + * supplies no blocks, so there is no caller-chosen block type for an + * allowlist to judge. Governing it would only mean refusing a member the + * ability to *disable* a block their group withholds. + * + * `attribution.attributedUserId` is deliberately not reused: it answers a + * workspace API key with the workspace's billing owner, which is right for + * custom-tool ownership and wrong for anything reading a person's grants. + */ + subjectUserId: null, state: async (tx) => { const locked = await loadWorkflowFromNormalizedTables(context.workflowId, tx) if (!locked) { diff --git a/apps/sim/lib/workflows/application/update-workflow.ts b/apps/sim/lib/workflows/application/update-workflow.ts index 9002d9839df..d56cd95445a 100644 --- a/apps/sim/lib/workflows/application/update-workflow.ts +++ b/apps/sim/lib/workflows/application/update-workflow.ts @@ -11,7 +11,7 @@ import type { WorkspaceUseCaseAuditEntry } from '@/lib/core/application' import { OrchestrationError } from '@/lib/core/orchestration/types' import { MAX_FOLDERS_PER_WORKSPACE } from '@/lib/folders/constants' import { loadActiveFolderPathIndex } from '@/lib/folders/queries' -import { notifyWorkflowUpdated } from '@/lib/realtime/notify' +import { notifyWorkflowUpdated, notifyWorkspaceWorkflowsChanged } from '@/lib/realtime/notify' import { defineAuthorizedWorkflowUseCase } from '@/lib/workflows/application/authorized-workflow-use-case' import { type ActiveWorkflowApplicationContext, @@ -249,11 +249,15 @@ function projectWorkflowUpdateAudit(args: { return entries } -function notifyAfterWorkflowUpdate(args: { +async function notifyAfterWorkflowUpdate(args: { context: ActiveWorkflowApplicationContext result: WorkflowUpdateResult }) { - return args.result.changes.length > 0 ? notifyWorkflowUpdated(args.context.workflowId) : undefined + if (args.result.changes.length === 0) return + await Promise.all([ + notifyWorkflowUpdated(args.context.workflowId), + notifyWorkspaceWorkflowsChanged(args.context.workspaceId), + ]) } export const updateWorkflow = defineAuthorizedWorkflowUseCase({ diff --git a/apps/sim/lib/workflows/application/workflow-crud.test.ts b/apps/sim/lib/workflows/application/workflow-crud.test.ts index 8c7168eab0d..b2004215b6e 100644 --- a/apps/sim/lib/workflows/application/workflow-crud.test.ts +++ b/apps/sim/lib/workflows/application/workflow-crud.test.ts @@ -22,6 +22,7 @@ const mocks = vi.hoisted(() => ({ readVersion: vi.fn(), loadNormalized: vi.fn(), notifyWorkflowUpdated: vi.fn(), + notifyWorkspaceWorkflowsChanged: vi.fn(), workflowCreated: vi.fn(), })) @@ -91,6 +92,7 @@ vi.mock('@/lib/workflows/persistence/utils', () => ({ vi.mock('@/lib/realtime/notify', () => ({ notifyWorkflowUpdated: mocks.notifyWorkflowUpdated, + notifyWorkspaceWorkflowsChanged: mocks.notifyWorkspaceWorkflowsChanged, })) vi.mock('@/lib/core/telemetry', () => ({ @@ -237,6 +239,7 @@ describe('authorized workflow CRUD and version reads', () => { }) ) expect(mocks.notifyWorkflowUpdated).toHaveBeenCalledWith(WORKFLOW_ID) + expect(mocks.notifyWorkspaceWorkflowsChanged).toHaveBeenCalledWith(WORKSPACE_ID) expect(mocks.workflowCreated).toHaveBeenCalledWith( expect.objectContaining({ workflowId: WORKFLOW_ID, workspaceId: WORKSPACE_ID }) ) diff --git a/apps/sim/lib/workflows/application/workflow-run-control.test.ts b/apps/sim/lib/workflows/application/workflow-run-control.test.ts index 7478d7d6f91..a7440c3c14a 100644 --- a/apps/sim/lib/workflows/application/workflow-run-control.test.ts +++ b/apps/sim/lib/workflows/application/workflow-run-control.test.ts @@ -11,6 +11,7 @@ const { MockWorkflowExecutionNotFoundError, mocks } = vi.hoisted(() => { mocks: { audit: vi.fn(), cancel: vi.fn(), + capture: vi.fn(), resolvePermission: vi.fn(), resolveRunContext: vi.fn(), resume: vi.fn(), @@ -19,6 +20,7 @@ const { MockWorkflowExecutionNotFoundError, mocks } = vi.hoisted(() => { }) vi.mock('@sim/audit', () => ({ recordAudit: mocks.audit })) +vi.mock('@/lib/posthog/server', () => ({ captureServerEvent: mocks.capture })) vi.mock('@sim/platform-authz/workspace', () => ({ permissionSatisfies: (actual: string | null, required: string) => { @@ -115,20 +117,25 @@ describe('workflow run-control application use cases', () => { async ({ principal, actorUserId }) => { await cancelWorkflowRun.execute({ principal, - input: { workflowId: 'workflow-1', runId: 'parent-run-1' }, + input: { runId: 'parent-run-1' }, }) expect(mocks.resolveRunContext).toHaveBeenCalledWith({ runId: 'parent-run-1', - assertedWorkflowId: 'workflow-1', }) expect(mocks.cancel).toHaveBeenCalledWith({ executionId: 'parent-run-1', workflowId: 'workflow-1', - userId: actorUserId, + attributedUserId: actorUserId, workspaceId: 'workspace-1', - captureAnalytics: false, + abortSignal: undefined, }) + expect(mocks.capture).toHaveBeenCalledWith( + actorUserId, + 'workflow_execution_cancelled', + { workflow_id: 'workflow-1', workspace_id: 'workspace-1' }, + { groups: { workspace: 'workspace-1' } } + ) expect(mocks.audit).not.toHaveBeenCalled() } ) @@ -166,14 +173,14 @@ describe('workflow run-control application use cases', () => { } ) - it('stops cancellation and resume before authorization when workflow/run scope disagrees', async () => { + it('stops cancellation and resume before authorization when canonical run resolution fails', async () => { mocks.resolveRunContext.mockRejectedValue(new OrchestrationError('not_found', 'Run not found')) const principal = principals[0].principal await expect( cancelWorkflowRun.execute({ principal, - input: { workflowId: 'wrong-workflow', runId: 'parent-run-1' }, + input: { runId: 'parent-run-1' }, }) ).rejects.toMatchObject({ code: 'not_found' }) await expect( @@ -200,7 +207,7 @@ describe('workflow run-control application use cases', () => { await expect( cancelWorkflowRun.execute({ principal, - input: { workflowId: 'workflow-1', runId: 'parent-run-1' }, + input: { runId: 'parent-run-1' }, }) ).rejects.toMatchObject({ code: 'forbidden' }) await expect( @@ -225,7 +232,7 @@ describe('workflow run-control application use cases', () => { await expect( cancelWorkflowRun.execute({ principal: principals[0].principal, - input: { workflowId: 'workflow-1', runId: 'parent-run-1' }, + input: { runId: 'parent-run-1' }, }) ).rejects.toMatchObject({ code: 'not_found', message: 'Run not found' }) }) @@ -238,7 +245,7 @@ describe('workflow run-control application use cases', () => { await expect( cancelWorkflowRun.execute({ principal: principals[2].principal, - input: { workflowId: 'workflow-1', runId: 'parent-run-1' }, + input: { runId: 'parent-run-1' }, }) ).rejects.toBe(cancelFailure) diff --git a/apps/sim/lib/workflows/application/workflow-runs.test.ts b/apps/sim/lib/workflows/application/workflow-runs.test.ts index e910167f43d..c299a8fcdd2 100644 --- a/apps/sim/lib/workflows/application/workflow-runs.test.ts +++ b/apps/sim/lib/workflows/application/workflow-runs.test.ts @@ -34,7 +34,7 @@ vi.mock('@/lib/workflows/executor/execution-queries', () => ({ })) vi.mock('@/lib/workflows/executor/execution-status', () => ({ - getWorkflowExecutionStatus: mocks.getStatus, + getProjectedWorkflowExecutionStatus: mocks.getStatus, })) vi.mock('@/lib/workflows/executor/execution-run-files', () => ({ @@ -81,9 +81,8 @@ describe('workflow run application use cases', () => { mocks.resolveRunContext.mockResolvedValue(runContext) mocks.list.mockResolvedValue({ data: [], nextCursor: null }) mocks.getStatus.mockResolvedValue({ - executionId: 'run-1', - workflowId: 'workflow-1', - status: 'completed', + status: { executionId: 'run-1', workflowId: 'workflow-1', status: 'completed' }, + projection: { hideTraceSpans: false, hideCostInfo: false }, }) mocks.getRunFiles.mockResolvedValue({ terminal: true, @@ -130,55 +129,12 @@ describe('workflow run application use cases', () => { executionId: 'run-1', includeOutput: true, selectedOutputs: ['4f1c2b3a-0000-4000-8000-000000000001.value'], + workspaceId: 'workspace-1', + workspaceOrganizationId: null, + viewerUserId: null, }) }) - it('refuses a selector that is not headed by a block id instead of answering an empty selection', async () => { - mocks.getStatus.mockResolvedValueOnce({ - executionId: 'run-1', - workflowId: 'workflow-1', - status: 'completed', - blockOutputs: {}, - }) - - await expect( - readWorkflowRun.execute({ - principal: principals[2], - input: { - workflowId: 'workflow-1', - runId: 'run-1', - includeOutput: true, - selectedOutputs: ['doubler.doubled'], - }, - }) - ).rejects.toMatchObject({ code: 'validation' }) - }) - - /** - * A well-formed id that produced nothing is a legitimate empty answer — the - * block may simply not have run on this path. - */ - it('allows a block id that produced no output on this run', async () => { - mocks.getStatus.mockResolvedValueOnce({ - executionId: 'run-1', - workflowId: 'workflow-1', - status: 'completed', - blockOutputs: {}, - }) - - const result = await readWorkflowRun.execute({ - principal: principals[2], - input: { - workflowId: 'workflow-1', - runId: 'run-1', - includeOutput: true, - selectedOutputs: ['4f1c2b3a-0000-4000-8000-000000000001.value'], - }, - }) - - expect(result.blockOutputs).toEqual({}) - }) - /** * File descriptors follow `output`'s gating: a caller that did not ask for * output must not receive a file list it did not request. diff --git a/apps/sim/lib/workflows/autolayout/change-set.ts b/apps/sim/lib/workflows/autolayout/change-set.ts index ecf12598bc3..e18635672eb 100644 --- a/apps/sim/lib/workflows/autolayout/change-set.ts +++ b/apps/sim/lib/workflows/autolayout/change-set.ts @@ -1,4 +1,4 @@ -import type { Edge } from 'reactflow' +import type { Edge } from '@xyflow/react' import { getBlockMetrics } from '@/lib/workflows/autolayout/utils' import type { WorkflowState } from '@/stores/workflows/workflow/types' diff --git a/apps/sim/lib/workflows/autolayout/containers.ts b/apps/sim/lib/workflows/autolayout/containers.ts index f3b7a244199..f56302aca82 100644 --- a/apps/sim/lib/workflows/autolayout/containers.ts +++ b/apps/sim/lib/workflows/autolayout/containers.ts @@ -1,5 +1,5 @@ import { createLogger } from '@sim/logger' -import { CONTAINER_DIMENSIONS } from '@sim/workflow-renderer' +import { CONTAINER_DIMENSIONS } from '@sim/workflow-renderer/dimensions' import { CONTAINER_PADDING_X, CONTAINER_PADDING_Y, diff --git a/apps/sim/lib/workflows/autolayout/core.ts b/apps/sim/lib/workflows/autolayout/core.ts index 5f827f76a7d..05cfb2a42a3 100644 --- a/apps/sim/lib/workflows/autolayout/core.ts +++ b/apps/sim/lib/workflows/autolayout/core.ts @@ -1,5 +1,5 @@ import { createLogger } from '@sim/logger' -import { HANDLE_POSITIONS } from '@sim/workflow-renderer' +import { HANDLE_POSITIONS } from '@sim/workflow-renderer/dimensions' import { CONTAINER_LAYOUT_OPTIONS, DEFAULT_LAYOUT_OPTIONS, diff --git a/apps/sim/lib/workflows/autolayout/targeted.ts b/apps/sim/lib/workflows/autolayout/targeted.ts index bf09c74cfed..245cf4b54f8 100644 --- a/apps/sim/lib/workflows/autolayout/targeted.ts +++ b/apps/sim/lib/workflows/autolayout/targeted.ts @@ -1,4 +1,4 @@ -import { CONTAINER_DIMENSIONS } from '@sim/workflow-renderer' +import { CONTAINER_DIMENSIONS } from '@sim/workflow-renderer/dimensions' import { CONTAINER_PADDING, DEFAULT_HORIZONTAL_SPACING, diff --git a/apps/sim/lib/workflows/autolayout/types.ts b/apps/sim/lib/workflows/autolayout/types.ts index 88fae3f503f..a2cb5953776 100644 --- a/apps/sim/lib/workflows/autolayout/types.ts +++ b/apps/sim/lib/workflows/autolayout/types.ts @@ -1,6 +1,6 @@ import type { BlockState, Position } from '@/stores/workflows/workflow/types' -export type { Edge } from 'reactflow' +export type { Edge } from '@xyflow/react' export interface LayoutOptions { horizontalSpacing?: number diff --git a/apps/sim/lib/workflows/autolayout/utils.ts b/apps/sim/lib/workflows/autolayout/utils.ts index bccedd3cdd6..91e9eefba2d 100644 --- a/apps/sim/lib/workflows/autolayout/utils.ts +++ b/apps/sim/lib/workflows/autolayout/utils.ts @@ -3,8 +3,8 @@ import { CONTAINER_DIMENSIONS, clampNoteBlockTotalHeight, getNoteBlockHeight, - isNoteContentEmpty, -} from '@sim/workflow-renderer' +} from '@sim/workflow-renderer/dimensions' +import { isNoteContentEmpty } from '@sim/workflow-renderer/note-content' import { AUTO_LAYOUT_EXCLUDED_TYPES, CONTAINER_BLOCK_TYPES, diff --git a/apps/sim/lib/workflows/blocks/block-outputs.ts b/apps/sim/lib/workflows/blocks/block-outputs.ts index b36f9ec3ac3..3556a5150c0 100644 --- a/apps/sim/lib/workflows/blocks/block-outputs.ts +++ b/apps/sim/lib/workflows/blocks/block-outputs.ts @@ -22,6 +22,7 @@ import { type OutputCondition, type OutputFieldDefinition, } from '@/blocks/types' +import { isHumanInTheLoopBlock } from '@/executor/constants' import { getToolOutputsMetadata } from '@/tools/metadata-outputs' import { getTrigger, isTriggerValid } from '@/triggers' @@ -322,7 +323,7 @@ export function getBlockOutputs( return getUnifiedStartOutputs(subBlocks) } - if (blockType === 'human_in_the_loop') { + if (isHumanInTheLoopBlock(blockType)) { // Start with block config outputs (respects hiddenFromDisplay via filterOutputsByCondition) const baseOutputs = filterOutputsByCondition( { ...(blockConfig.outputs || {}) } as OutputDefinition, diff --git a/apps/sim/lib/workflows/blocks/block-reference-tags.ts b/apps/sim/lib/workflows/blocks/block-reference-tags.ts index f48fb20a504..cfeadb3518a 100644 --- a/apps/sim/lib/workflows/blocks/block-reference-tags.ts +++ b/apps/sim/lib/workflows/blocks/block-reference-tags.ts @@ -2,7 +2,7 @@ import { getEffectiveBlockOutputPaths } from '@/lib/workflows/blocks/block-outpu import { hasTriggerCapability } from '@/lib/workflows/triggers/trigger-utils' import { TRIGGER_TYPES } from '@/lib/workflows/triggers/triggers' import { getBlock } from '@/blocks' -import { normalizeName } from '@/executor/constants' +import { isHumanInTheLoopBlock, normalizeName } from '@/executor/constants' interface ReferenceableBlock { id: string @@ -61,7 +61,7 @@ export function getBlockReferenceTags({ const allTags = outputPaths.map((path) => `${normalizedBlockName}.${path}`) let blockTags: string[] - if (block.type === 'human_in_the_loop' && block.id === currentBlockId) { + if (isHumanInTheLoopBlock(block.type) && block.id === currentBlockId) { blockTags = allTags.filter((tag) => tag.endsWith('.url') || tag.endsWith('.resumeEndpoint')) } else if (allTags.length === 0) { blockTags = [normalizedBlockName] diff --git a/apps/sim/lib/workflows/blocks/deterministic-dimensions.ts b/apps/sim/lib/workflows/blocks/deterministic-dimensions.ts index 87ea892767c..9af3b6fc178 100644 --- a/apps/sim/lib/workflows/blocks/deterministic-dimensions.ts +++ b/apps/sim/lib/workflows/blocks/deterministic-dimensions.ts @@ -1,4 +1,4 @@ -import { BLOCK_DIMENSIONS } from '@sim/workflow-renderer' +import { BLOCK_DIMENSIONS } from '@sim/workflow-renderer/dimensions' interface WorkflowBlockDimensionsInput { blockType: string diff --git a/apps/sim/lib/workflows/blocks/flatten-outputs.ts b/apps/sim/lib/workflows/blocks/flatten-outputs.ts index 44d393b6323..7c3feb35739 100644 --- a/apps/sim/lib/workflows/blocks/flatten-outputs.ts +++ b/apps/sim/lib/workflows/blocks/flatten-outputs.ts @@ -10,12 +10,17 @@ import { isRecordLike } from '@sim/utils/object' import { getEffectiveBlockOutputs } from '@/lib/workflows/blocks/block-outputs' +import { HUMAN_IN_THE_LOOP_BLOCK_TYPES } from '@/executor/constants' /** * Block types whose "outputs" are really workflow inputs (Start/starter) or flow * control and should never appear in an output picker. */ -export const EXCLUDED_OUTPUT_TYPES = new Set(['starter', 'start_trigger', 'human_in_the_loop']) +export const EXCLUDED_OUTPUT_TYPES = new Set([ + 'starter', + 'start_trigger', + ...HUMAN_IN_THE_LOOP_BLOCK_TYPES, +]) export interface FlattenedBlockOutput { blockId: string diff --git a/apps/sim/lib/workflows/blocks/retry-eligibility.ts b/apps/sim/lib/workflows/blocks/retry-eligibility.ts index a9c0d16c235..9a176af2d6f 100644 --- a/apps/sim/lib/workflows/blocks/retry-eligibility.ts +++ b/apps/sim/lib/workflows/blocks/retry-eligibility.ts @@ -1,4 +1,8 @@ -import { BlockType, isMetadataOnlyBlockType, isSentinelBlockType } from '@/executor/constants' +import { + isHumanInTheLoopBlock, + isMetadataOnlyBlockType, + isSentinelBlockType, +} from '@/executor/constants' interface RetryEligibilityInput { blockType: string | undefined @@ -26,7 +30,7 @@ export function isRetryEligibleBlock({ triggerMode, }: RetryEligibilityInput): boolean { if (!blockType) return false - if (blockType === BlockType.HUMAN_IN_THE_LOOP) return false + if (isHumanInTheLoopBlock(blockType)) return false if (triggerMode === true || category === 'triggers') return false return !isSentinelBlockType(blockType) && !isMetadataOnlyBlockType(blockType) } diff --git a/apps/sim/lib/workflows/canonical/reported-bug.test.ts b/apps/sim/lib/workflows/canonical/reported-bug.test.ts index 8232f244a56..0cae18ab05d 100644 --- a/apps/sim/lib/workflows/canonical/reported-bug.test.ts +++ b/apps/sim/lib/workflows/canonical/reported-bug.test.ts @@ -16,9 +16,14 @@ import { describe, expect, it, vi } from 'vitest' /** * The canonical form reads declared defaults, so the globally-mocked registry - * (every block reduced to `subBlocks: []`) would make this pass vacuously. + * (every block reduced to `subBlocks: []`) would make this pass vacuously. Only + * the webhook block is read, so only it is registered. */ vi.unmock('@/blocks/registry') +vi.mock('@/blocks/registry-maps', async () => { + const { partialBlockRegistry } = await import('@sim/testing/mocks/block-registry.mock') + return partialBlockRegistry(await import('@/blocks/blocks/generic_webhook')) +}) import { generateWorkflowDiffSummary } from '@/lib/workflows/comparison/compare' import type { WorkflowState } from '@/stores/workflows/workflow/types' diff --git a/apps/sim/lib/workflows/comparison/format-description.test.ts b/apps/sim/lib/workflows/comparison/format-description.test.ts index 037b685406e..b0c8d2f20e9 100644 --- a/apps/sim/lib/workflows/comparison/format-description.test.ts +++ b/apps/sim/lib/workflows/comparison/format-description.test.ts @@ -3,8 +3,9 @@ */ import { beforeEach, describe, expect, it, vi } from 'vitest' -const { mockGetBlock } = vi.hoisted(() => ({ +const { mockGetBlock, mockLoadAllSelectorOptions } = vi.hoisted(() => ({ mockGetBlock: vi.fn(), + mockLoadAllSelectorOptions: vi.fn(), })) vi.mock('@/lib/workflows/subblocks/visibility', () => ({ @@ -17,7 +18,7 @@ vi.mock('@/triggers/constants', () => ({ })) vi.mock('@/blocks/types', () => ({ - SELECTOR_TYPES_HYDRATION_REQUIRED: [], + SELECTOR_TYPES_HYDRATION_REQUIRED: ['channel-selector'], })) vi.mock('@/executor/constants', () => ({ @@ -31,20 +32,13 @@ vi.mock('@/blocks/registry', () => ({ registry: {}, })) -vi.mock('@/lib/workflows/subblocks/context', () => ({ - buildSelectorContextFromBlock: vi.fn(() => ({})), -})) - vi.mock('@/hooks/queries/oauth/oauth-credentials', () => ({ fetchOAuthCredentialDetail: vi.fn(() => []), })) -vi.mock('@/hooks/selectors/registry', () => ({ - getSelectorDefinition: vi.fn(() => ({ fetchList: vi.fn(() => []) })), -})) - -vi.mock('@/hooks/selectors/resolution', () => ({ - resolveSelectorForSubBlock: vi.fn(), +vi.mock('@/lib/selectors/client/execute-selector', () => ({ + executeSelectorRequest: vi.fn(() => ({ kind: 'detail', item: null })), + loadAllSelectorOptions: mockLoadAllSelectorOptions, })) import { WorkflowBuilder } from '@sim/testing' @@ -54,7 +48,11 @@ import { formatDiffSummaryForDescription, formatDiffSummaryForDescriptionAsync, } from '@/lib/workflows/comparison/describe' -import { formatValueForDisplay, resolveFieldLabel } from '@/lib/workflows/comparison/resolve-values' +import { + formatValueForDisplay, + resolveFieldLabel, + resolveValueForDisplay, +} from '@/lib/workflows/comparison/resolve-values' function emptyDiffSummary(overrides: Partial = {}): WorkflowDiffSummary { return { @@ -79,6 +77,7 @@ function emptyDiffSummary(overrides: Partial = {}): Workflo beforeEach(() => { vi.clearAllMocks() + mockLoadAllSelectorOptions.mockResolvedValue({ items: [], truncated: false }) }) describe('resolveFieldLabel', () => { @@ -131,6 +130,36 @@ describe('formatValueForDisplay', () => { }) }) +describe('resolveValueForDisplay', () => { + it('preserves a raw selector ID when the loaded catalog is incomplete', async () => { + mockGetBlock.mockReturnValue({ + subBlocks: [ + { + id: 'channel', + title: 'Channel', + type: 'channel-selector', + selectorKey: 'slack.channels', + }, + ], + }) + mockLoadAllSelectorOptions.mockResolvedValue({ items: [], truncated: true }) + + const channelId = 'C12345678' + const result = await resolveValueForDisplay(channelId, { + blockType: 'slack', + subBlockId: 'channel', + workflowId: 'wf-1', + currentState: new WorkflowBuilder().build(), + }) + + expect(result).toEqual({ + original: channelId, + displayLabel: channelId, + resolved: false, + }) + }) +}) + describe('formatDiffSummaryForDescription', () => { it('returns no-changes message for empty diff', () => { const result = formatDiffSummaryForDescription(emptyDiffSummary()) diff --git a/apps/sim/lib/workflows/comparison/normalize.ts b/apps/sim/lib/workflows/comparison/normalize.ts index ac3d3b74365..2466701617c 100644 --- a/apps/sim/lib/workflows/comparison/normalize.ts +++ b/apps/sim/lib/workflows/comparison/normalize.ts @@ -8,7 +8,7 @@ import { normalizeWorkflowEdgeSourceHandle, normalizeWorkflowEdgeTargetHandle, } from '@sim/workflow-types/workflow' -import type { Edge } from 'reactflow' +import type { Edge } from '@xyflow/react' import { isNonEmptyValue } from '@/lib/workflows/subblocks/visibility' import { isSyntheticToolSubBlockId } from '@/lib/workflows/tool-input/synthetic-subblocks' import type { diff --git a/apps/sim/lib/workflows/comparison/resolve-values.ts b/apps/sim/lib/workflows/comparison/resolve-values.ts index f2b25b2b15b..19913ba21c0 100644 --- a/apps/sim/lib/workflows/comparison/resolve-values.ts +++ b/apps/sim/lib/workflows/comparison/resolve-values.ts @@ -1,13 +1,17 @@ import { createLogger } from '@sim/logger' import { truncate } from '@sim/utils/string' -import { buildSelectorContextFromBlock } from '@/lib/workflows/subblocks/context' +import { + executeSelectorRequest, + loadAllSelectorOptions, +} from '@/lib/selectors/client/execute-selector' +import { buildSelectorRawContext, projectSelectorContext } from '@/lib/selectors/context' +import { getSelectorManifestEntry, type SelectorKey } from '@/lib/selectors/manifest' +import type { SelectorContext, SelectorScope } from '@/lib/selectors/types' +import { getDependsOnFields } from '@/lib/workflows/subblocks/dependencies' import { getBlock } from '@/blocks/registry' import { SELECTOR_TYPES_HYDRATION_REQUIRED, type SubBlockConfig } from '@/blocks/types' import { isUuid } from '@/executor/constants' import { fetchOAuthCredentialDetail } from '@/hooks/queries/oauth/oauth-credentials' -import { getSelectorDefinition, loadAllSelectorOptions } from '@/hooks/selectors/registry' -import { resolveSelectorForSubBlock } from '@/hooks/selectors/resolution' -import type { SelectorContext, SelectorKey } from '@/hooks/selectors/types' import type { WorkflowState } from '@/stores/workflows/workflow/types' import { formatParameterLabel } from '@/tools/params' @@ -25,6 +29,11 @@ interface ResolvedValue { resolved: boolean } +interface ResolvedSelectorValue { + label: string | null + incomplete: boolean +} + /** * Context needed to resolve values for display */ @@ -55,8 +64,8 @@ async function resolveCredential(credentialId: string, workflowId: string): Prom } return null - } catch (error) { - logger.warn('Failed to resolve credential', { credentialId, error }) + } catch { + logger.warn('Failed to resolve credential display label') return null } } @@ -65,18 +74,15 @@ async function resolveWorkflow(workflowId: string, workspaceId?: string): Promis if (!workspaceId) return null try { - const definition = getSelectorDefinition('sim.workflows') - if (definition.fetchById) { - const result = await definition.fetchById({ - key: 'sim.workflows', - context: { workspaceId }, - detailId: workflowId, - }) - return result?.label ?? null - } - return null - } catch (error) { - logger.warn('Failed to resolve workflow', { workflowId, error }) + const result = await executeSelectorRequest({ + selectorKey: 'sim.workflows', + scope: { kind: 'workspace', workspaceId }, + context: {}, + request: { kind: 'detail', id: workflowId }, + }) + return result.kind === 'detail' ? (result.item?.label ?? null) : null + } catch { + logger.warn('Failed to resolve workflow display label') return null } } @@ -84,31 +90,40 @@ async function resolveWorkflow(workflowId: string, workspaceId?: string): Promis async function resolveSelectorValue( value: string, selectorKey: SelectorKey, - selectorContext: SelectorContext -): Promise { + selectorContext: SelectorContext, + scope: SelectorScope +): Promise { try { - const definition = getSelectorDefinition(selectorKey) + const manifest = getSelectorManifestEntry(selectorKey) - if (definition.fetchById) { - const result = await definition.fetchById({ - key: selectorKey, + if (manifest.supportsDetail) { + const result = await executeSelectorRequest({ + selectorKey, + scope, context: selectorContext, - detailId: value, + request: { kind: 'detail', id: value }, }) - if (result?.label) { - return result.label + if (result.kind === 'detail' && result.item?.label) { + return { label: result.item.label, incomplete: false } } } - const options = await loadAllSelectorOptions(definition, { - key: selectorKey, + const catalog = await loadAllSelectorOptions({ + selectorKey, + scope, context: selectorContext, }) - const match = options.find((opt) => opt.id === value) - return match?.label ?? null - } catch (error) { - logger.warn('Failed to resolve selector value', { value, selectorKey, error }) - return null + const match = catalog.items.find((option) => option.id === value) + const incomplete = !match && catalog.truncated + if (incomplete) { + logger.warn('Selector catalog was truncated before display label could be resolved', { + selectorKey, + }) + } + return { label: match?.label ?? null, incomplete } + } catch { + logger.warn('Failed to resolve selector display label', { selectorKey }) + return { label: null, incomplete: false } } } @@ -170,16 +185,19 @@ export function formatValueForDisplay(value: unknown): string { function extractSelectorContext( blockId: string, currentState: WorkflowState, - workflowId: string, - workspaceId?: string + selectorKey: SelectorKey, + subBlockConfig: SubBlockConfig ): SelectorContext { const block = currentState.blocks?.[blockId] - if (!block?.subBlocks) return { workflowId, workspaceId } - return buildSelectorContextFromBlock(block.type, block.subBlocks, { - workflowId, - workspaceId, + if (!block?.subBlocks) return {} + return buildSelectorRawContext({ + selectorKey, + blockType: block.type, + subBlocks: block.subBlocks, + dependsOn: getDependsOnFields(subBlockConfig.dependsOn), canonicalModes: block.data?.canonicalModes, triggerMode: block.triggerMode, + staticContext: { mimeType: subBlockConfig.mimeType }, }) } @@ -210,15 +228,6 @@ export async function resolveValueForDisplay( } const semanticFallback = getSemanticFallback(subBlockConfig) - const selectorCtx = context.blockId - ? extractSelectorContext( - context.blockId, - context.currentState, - context.workflowId, - context.workspaceId - ) - : { workflowId: context.workflowId, workspaceId: context.workspaceId } - const isCredentialField = subBlockConfig.type === 'oauth-input' || context.subBlockId === 'credential' @@ -231,7 +240,7 @@ export async function resolveValueForDisplay( } if (subBlockConfig.type === 'workflow-selector' && isUuid(value)) { - const label = await resolveWorkflow(value, selectorCtx.workspaceId) + const label = await resolveWorkflow(value, context.workspaceId) if (label) { return { original: value, displayLabel: label, resolved: true } } @@ -249,22 +258,32 @@ export async function resolveValueForDisplay( if (label) { return { original: value, displayLabel: label, resolved: true } } - } catch (error) { - logger.warn('Failed to resolve dropdown label', { - value, - subBlockId: context.subBlockId, - error, - }) + } catch { + logger.warn('Failed to resolve dropdown display label') } } if (SELECTOR_TYPES_HYDRATION_REQUIRED.includes(subBlockConfig.type)) { - const resolution = resolveSelectorForSubBlock(subBlockConfig, selectorCtx) - - if (resolution?.key) { - const label = await resolveSelectorValue(value, resolution.key, selectorCtx) - if (label) { - return { original: value, displayLabel: label, resolved: true } + const selectorKey = subBlockConfig.selectorKey + const scope: SelectorScope | undefined = context.workflowId + ? { + kind: 'workflow', + workflowId: context.workflowId, + ...(context.workspaceId ? { workspaceId: context.workspaceId } : {}), + } + : context.workspaceId + ? { kind: 'workspace', workspaceId: context.workspaceId } + : undefined + if (selectorKey && scope) { + const selectorContext = context.blockId + ? extractSelectorContext(context.blockId, context.currentState, selectorKey, subBlockConfig) + : projectSelectorContext(selectorKey, { mimeType: subBlockConfig.mimeType }) + const selectorValue = await resolveSelectorValue(value, selectorKey, selectorContext, scope) + if (selectorValue.label) { + return { original: value, displayLabel: selectorValue.label, resolved: true } + } + if (selectorValue.incomplete) { + return { original: value, displayLabel: formatValueForDisplay(value), resolved: false } } } return { original: value, displayLabel: semanticFallback, resolved: true } diff --git a/apps/sim/lib/workflows/custom-blocks/operations.ts b/apps/sim/lib/workflows/custom-blocks/operations.ts index 4b9018dfa2e..a35f980b7e1 100644 --- a/apps/sim/lib/workflows/custom-blocks/operations.ts +++ b/apps/sim/lib/workflows/custom-blocks/operations.ts @@ -8,10 +8,12 @@ import { } from '@sim/db/schema' import { createLogger } from '@sim/logger' import { generateId, generateShortId } from '@sim/utils/id' -import { and, eq, isNull, sql } from 'drizzle-orm' +import { and, eq, isNull, ne, sql } from 'drizzle-orm' import { isOrganizationFeatureEntitled } from '@/lib/billing/core/subscription' +import { acquireOrganizationMutationLock } from '@/lib/billing/organizations/membership' import { isBillingEnabled, isCustomBlocksEnabled } from '@/lib/core/config/env-flags' import { mapWithConcurrency } from '@/lib/core/utils/concurrency' +import type { DbOrTx } from '@/lib/db/types' import { extractInputFieldsFromBlocks, type WorkflowInputField } from '@/lib/workflows/input-format' import { loadDeployedWorkflowState } from '@/lib/workflows/persistence/utils' import { getWorkspaceWithOwner } from '@/lib/workspaces/permissions/utils' @@ -305,11 +307,23 @@ export async function getCustomBlockManageContext(id: string): Promise<{ * executor to run the bound workflow under the invocation-boundary model: the * consumer needs no permission on the source workflow. Returns the authoritative * `workflowId` from the DB (never trust a serialized value) plus the source - * workflow's **owner** (`workflow.userId`) — the same identity a normal deployed - * API/schedule/webhook run executes as. Using the owner (not the publisher) means - * the owner always has read on their own workflow, and owner deletion cascade- - * deletes the workflow → the custom_block row, so there is never an orphaned block. - * `null` when no enabled block matches the type. + * workflow's **owner** (`workflow.userId`). Using the owner (not the publisher) + * means the owner always has read on their own workflow, and owner deletion + * cascade-deletes the workflow → the custom_block row, so there is never an + * orphaned block. `null` when no enabled block matches the type. + * + * `ownerUserId` carries further than the owner does on any other trigger. It is + * the child run's actor, the personal-variable identity, and the subject of its + * delegated tool calls, because a custom block publishes a fixed behavior to + * consumers who can see none of its internals and the publisher's own + * integrations and personal keys are part of that behavior. + * + * It is NOT the identity for the two things a workspace owns. Workspace + * variables authorize against the source workspace's billing account, and that + * account is the payer, exactly as they would for a schedule on the same + * workflow — see the environment resolution in `workflow-handler`. Reading those + * as the owner too gave a published block a narrower workspace-secret selection + * than the workflow got on every other trigger, which no consumer could see. */ export async function getCustomBlockAuthority( type: string, @@ -495,41 +509,59 @@ export async function publishCustomBlock(params: { throw new CustomBlockValidationError('You can only publish a workflow from its own workspace') } - const ws = wf.workspaceId ? await getWorkspaceWithOwner(wf.workspaceId) : null - if (!ws?.organizationId || ws.organizationId !== organizationId) { - throw new CustomBlockValidationError('Workflow does not belong to this organization') - } - - // One block per workflow: the (org, type) unique index doesn't prevent the same - // workflow being published under a fresh `custom_block_*` type, so guard here. - const [existing] = await db - .select({ id: customBlock.id }) - .from(customBlock) - .where(eq(customBlock.workflowId, workflowId)) - .limit(1) - if (existing) { - throw new CustomBlockValidationError('This workflow is already published as a block') - } - const id = generateId() const type = `${CUSTOM_BLOCK_TYPE_PREFIX}${generateShortId(10).toLowerCase()}` const now = new Date() - await db.insert(customBlock).values({ - id, - organizationId, - workflowId, - type, - name, - description, - iconUrl: iconUrl ?? null, - inputs: inputs ?? [], - outputs: exposedOutputs ?? [], - enabled: true, - traceChildRuns, - createdBy: userId, - createdAt: now, - updatedAt: now, + /** + * The org-belongs check and the insert run under the organization mutation + * lock, together, because an admin workspace move holds that same lock while + * it re-homes a workspace and unpublishes the blocks bound to its workflows. + * Reading the workspace's organization outside the lock lets a publish that + * validated against the OLD organization commit after the move's cleanup + * scan, leaving a source-organization block bound to a workflow that now + * lives in another tenant — which `getCustomBlockAuthority` would resolve and + * execute under the wrong owner's credentials and billing. + */ + const ws = await db.transaction(async (tx) => { + await acquireOrganizationMutationLock(tx, organizationId) + + const workspaceRow = wf.workspaceId + ? await getWorkspaceWithOwner(wf.workspaceId, { executor: tx }) + : null + if (!workspaceRow?.organizationId || workspaceRow.organizationId !== organizationId) { + throw new CustomBlockValidationError('Workflow does not belong to this organization') + } + + // One block per workflow: the (org, type) unique index doesn't prevent the same + // workflow being published under a fresh `custom_block_*` type, so guard here. + const [existing] = await tx + .select({ id: customBlock.id }) + .from(customBlock) + .where(eq(customBlock.workflowId, workflowId)) + .limit(1) + if (existing) { + throw new CustomBlockValidationError('This workflow is already published as a block') + } + + await tx.insert(customBlock).values({ + id, + organizationId, + workflowId, + type, + name, + description, + iconUrl: iconUrl ?? null, + inputs: inputs ?? [], + outputs: exposedOutputs ?? [], + enabled: true, + traceChildRuns, + createdBy: userId, + createdAt: now, + updatedAt: now, + }) + + return workspaceRow }) logger.info('Published custom block', { id, type, organizationId, workflowId }) @@ -588,9 +620,16 @@ export async function updateCustomBlock( await db.update(customBlock).set(patch).where(eq(customBlock.id, id)) } -/** Unpublish (hard-delete) a custom block. */ -export async function deleteCustomBlock(id: string): Promise { - await db.delete(customBlock).where(eq(customBlock.id, id)) +/** + * Unpublish (hard-delete) a custom block. + * + * Accepts an executor so a caller that must unpublish atomically with something + * else can enlist it — the admin workspace move unpublishes blocks in the same + * transaction that re-homes their bound workflow, keeping a block and its + * workflow from ever being visible in two different organizations. + */ +export async function deleteCustomBlock(id: string, executor: DbOrTx = db): Promise { + await executor.delete(customBlock).where(eq(customBlock.id, id)) } /** @@ -603,11 +642,14 @@ export async function deleteCustomBlock(id: string): Promise { */ export async function getCustomBlockUsageCounts( organizationId: string, - blockType: string + blockType: string, + scope?: { onlyWorkspaceId?: string; excludeWorkspaceId?: string } ): Promise<{ usageCount: number; deployedUsageCount: number }> { const orgActiveWorkflow = and( eq(workspace.organizationId, organizationId), - isNull(workflow.archivedAt) + isNull(workflow.archivedAt), + scope?.onlyWorkspaceId ? eq(workflow.workspaceId, scope.onlyWorkspaceId) : undefined, + scope?.excludeWorkspaceId ? ne(workflow.workspaceId, scope.excludeWorkspaceId) : undefined ) // Escape LIKE wildcards — the `_`s in `custom_block_` would otherwise match // any character and let unrelated states through to the jsonb parse. diff --git a/apps/sim/lib/workflows/diff/diff-engine.ts b/apps/sim/lib/workflows/diff/diff-engine.ts index 024bbfbb6f4..6a3b3918b6e 100644 --- a/apps/sim/lib/workflows/diff/diff-engine.ts +++ b/apps/sim/lib/workflows/diff/diff-engine.ts @@ -2,7 +2,7 @@ import { createLogger } from '@sim/logger' import { getErrorMessage, toError } from '@sim/utils/errors' import { generateId } from '@sim/utils/id' import { blockRetryEquals } from '@sim/workflow-types/workflow' -import type { Edge } from 'reactflow' +import type { Edge } from '@xyflow/react' import { getTargetedLayoutImpact } from '@/lib/workflows/autolayout' import type { BlockWithDiff } from '@/lib/workflows/diff/types' import { isValidKey } from '@/lib/workflows/sanitization/key-validation' diff --git a/apps/sim/lib/workflows/editing/builders.ts b/apps/sim/lib/workflows/editing/builders.ts index 534333e938a..2218ea5915d 100644 --- a/apps/sim/lib/workflows/editing/builders.ts +++ b/apps/sim/lib/workflows/editing/builders.ts @@ -7,6 +7,8 @@ import { normalizeBlockRetryWaitMs, } from '@sim/workflow-types/workflow' import { isIntegrationDeploymentAvailableForVisibility } from '@/lib/integrations/availability.server' +import { capabilityDeniedBy } from '@/lib/permission-groups/capability-assertions' +import type { PermissionGroupConfig } from '@/lib/permission-groups/fields' import { createModelAccessGate } from '@/lib/permission-groups/model-access' import { createToolAccessGate, @@ -14,7 +16,6 @@ import { MODEL_SUBBLOCK_ID, OPERATION_SUBBLOCK_ID, } from '@/lib/permission-groups/operation-access' -import type { PermissionGroupConfig } from '@/lib/permission-groups/types' import { getEffectiveBlockOutputs } from '@/lib/workflows/blocks/block-outputs' import { isRetryEligibleBlock } from '@/lib/workflows/blocks/retry-eligibility' import { @@ -790,7 +791,7 @@ export function filterDisallowedTools( const isToolAllowed = createToolAccessGate(permissionConfig.deniedTools) const allowedTools: any[] = [] for (const tool of deploymentAvailableTools) { - if (tool.type === 'custom-tool' && permissionConfig.disableCustomTools) { + if (tool.type === 'custom-tool' && capabilityDeniedBy('custom_tools.use', permissionConfig)) { logSkippedItem(skippedItems, { type: 'tool_not_allowed', operationType: 'add', @@ -800,7 +801,7 @@ export function filterDisallowedTools( }) continue } - if (tool.type === 'mcp' && permissionConfig.disableMcpTools) { + if (tool.type === 'mcp' && capabilityDeniedBy('mcp_tools.use', permissionConfig)) { logSkippedItem(skippedItems, { type: 'tool_not_allowed', operationType: 'add', diff --git a/apps/sim/lib/workflows/editing/engine.ts b/apps/sim/lib/workflows/editing/engine.ts index b58642d30ad..ba98f968ac4 100644 --- a/apps/sim/lib/workflows/editing/engine.ts +++ b/apps/sim/lib/workflows/editing/engine.ts @@ -1,5 +1,5 @@ import { createLogger } from '@sim/logger' -import type { PermissionGroupConfig } from '@/lib/permission-groups/types' +import type { PermissionGroupConfig } from '@/lib/permission-groups/fields' import { isValidKey } from '@/lib/workflows/sanitization/key-validation' import { validateEdges } from '@/stores/workflows/workflow/edge-validation' import { generateLoopBlocks, generateParallelBlocks } from '@/stores/workflows/workflow/utils' @@ -257,7 +257,6 @@ export function applyOperationsToWorkflowState( removeInvalidScopeEdges(modifiedState, skippedItems) // Regenerate loops and parallels after modifications - ;(modifiedState as any).loops = generateLoopBlocks((modifiedState as any).blocks) ;(modifiedState as any).parallels = generateParallelBlocks((modifiedState as any).blocks) diff --git a/apps/sim/lib/workflows/editing/operations.test.ts b/apps/sim/lib/workflows/editing/operations.test.ts index c049bc5e34a..c9f8511bc60 100644 --- a/apps/sim/lib/workflows/editing/operations.test.ts +++ b/apps/sim/lib/workflows/editing/operations.test.ts @@ -2,7 +2,7 @@ * @vitest-environment node */ import { describe, expect, it, vi } from 'vitest' -import { DEFAULT_PERMISSION_GROUP_CONFIG } from '@/lib/permission-groups/types' +import { DEFAULT_PERMISSION_GROUP_CONFIG } from '@/lib/permission-groups/fields' import { sanitizeForCopilot } from '@/lib/workflows/sanitization/json-sanitizer' import { applyOperationsToWorkflowState } from './engine' diff --git a/apps/sim/lib/workflows/editing/types.ts b/apps/sim/lib/workflows/editing/types.ts index c2350f009f2..a775b85e137 100644 --- a/apps/sim/lib/workflows/editing/types.ts +++ b/apps/sim/lib/workflows/editing/types.ts @@ -1,5 +1,5 @@ import { createLogger } from '@sim/logger' -import type { PermissionGroupConfig } from '@/lib/permission-groups/types' +import type { PermissionGroupConfig } from '@/lib/permission-groups/fields' /** Selector subblock types that can be validated */ export const SELECTOR_TYPES = new Set([ diff --git a/apps/sim/lib/workflows/editing/validation.ts b/apps/sim/lib/workflows/editing/validation.ts index 1925cf3174a..5a676c57e0c 100644 --- a/apps/sim/lib/workflows/editing/validation.ts +++ b/apps/sim/lib/workflows/editing/validation.ts @@ -4,7 +4,8 @@ import { omit } from '@sim/utils/object' import { isHosted as isHostedDeployment } from '@/lib/core/config/env-flags' import { isIntegrationDeploymentAvailableForVisibility } from '@/lib/integrations/availability.server' import { isBlockTypeAccessControlExempt } from '@/lib/permission-groups/block-access' -import type { PermissionGroupConfig } from '@/lib/permission-groups/types' +import type { PermissionGroupConfig } from '@/lib/permission-groups/fields' +import { resolveAccessControlBlockType } from '@/lib/permission-groups/integration-allowlist' import { getCustomToolById } from '@/lib/workflows/custom-tools/operations' import { validateSelectorIds } from '@/lib/workflows/editing/selector-validator' import { getSkillById } from '@/lib/workflows/skills/operations' @@ -1000,7 +1001,16 @@ export function validateTargetHandle(targetHandle: string): EdgeHandleValidation } /** - * Checks if a block type is allowed by the permission group config + * Whether a block may be added to a graph by this viewer. + * + * Two questions, not one: whether the viewer can see the block at all + * (deployment visibility — an unrevealed preview block, a kill-switched type) + * and whether their permission group's integration allowlist permits it. + * Refusing to *add* something a viewer cannot see is right. + * + * Refusing to *store* it is not, which is why the persist-time guard in + * `@/lib/workflows/persistence/block-access-guard` checks the allowlist alone: + * a graph exported before a block was gated must still save. */ export function isBlockTypeAllowed( blockType: string, @@ -1013,7 +1023,9 @@ export function isBlockTypeAllowed( if (!permissionConfig || permissionConfig.allowedIntegrations === null) { return true } - return permissionConfig.allowedIntegrations.includes(blockType.toLowerCase()) + return permissionConfig.allowedIntegrations.includes( + resolveAccessControlBlockType(blockType).toLowerCase() + ) } /** diff --git a/apps/sim/lib/workflows/executor/execute-service.ts b/apps/sim/lib/workflows/executor/execute-service.ts index 1bfbf058b89..3ebb700717f 100644 --- a/apps/sim/lib/workflows/executor/execute-service.ts +++ b/apps/sim/lib/workflows/executor/execute-service.ts @@ -2,7 +2,8 @@ import type { WorkflowExecutionPrincipal } from '@sim/auth/principal' import type { workflow as workflowTable } from '@sim/db/schema' import { createLogger } from '@sim/logger' import { getErrorMessage, toError } from '@sim/utils/errors' -import { generateId, isValidUuid } from '@sim/utils/id' +import { generateId } from '@sim/utils/id' +import type { BlockState } from '@sim/workflow-types/workflow' import { releaseExecutionSlot } from '@/lib/billing/calculations/usage-reservation' import type { BillingAttributionSnapshot } from '@/lib/billing/core/billing-attribution' import { createTimeoutAbortController, getTimeoutErrorMessage } from '@/lib/core/execution-limits' @@ -33,13 +34,13 @@ import { loadWorkflowFromNormalizedTables, } from '@/lib/workflows/persistence/utils' import { shouldEmitAgentStreamEvents } from '@/lib/workflows/streaming/agent-stream-protocol' +import { resolveOutputSelectors } from '@/lib/workflows/streaming/resolve-output-selectors' import { agentStreamProtocolResponseHeaders, createStreamingResponse, } from '@/lib/workflows/streaming/streaming' import { workflowHasResponseBlock } from '@/lib/workflows/utils' import { withCustomBlockOverlay } from '@/blocks/custom/server-overlay' -import { normalizeName } from '@/executor/constants' import { ExecutionSnapshot } from '@/executor/execution/snapshot' import type { ExecutionMetadata, SerializableExecutionState } from '@/executor/execution/types' import type { NormalizedBlockOutput } from '@/executor/types' @@ -472,7 +473,17 @@ export async function executeWorkflowService( } if (mode === 'stream') { - const resolvedSelectedOutputs = resolveOutputIds(selectedOutputs, workflowBlocks) + let resolvedSelectedOutputs: string[] | undefined + try { + resolvedSelectedOutputs = await resolveOutputIds(selectedOutputs, workflowBlocks) + } catch (error) { + await releaseExecutionSlot(executionId) + return failure({ + kind: 'input', + message: `Invalid selectedOutputs: ${getErrorMessage(error)}`, + statusCode: 400, + }) + } const streamWorkflow = { id: workflow.id, /** @@ -833,55 +844,12 @@ export async function executeWorkflowService( * `.path`) to internal `_` ids — same normalization the * v1 streaming path applies. */ -export function resolveOutputIds( +export async function resolveOutputIds( selectedOutputs: string[] | undefined, blocks: Record -): string[] | undefined { - if (!selectedOutputs || selectedOutputs.length === 0) { - return selectedOutputs - } - - return selectedOutputs.map((outputId) => { - const underscoreIndex = outputId.indexOf('_') - const dotIndex = outputId.indexOf('.') - if (underscoreIndex > 0) { - const maybeUuid = outputId.substring(0, underscoreIndex) - if (isValidUuid(maybeUuid)) { - return outputId - } - } - - if (dotIndex > 0) { - const maybeUuid = outputId.substring(0, dotIndex) - if (isValidUuid(maybeUuid)) { - return `${outputId.substring(0, dotIndex)}_${outputId.substring(dotIndex + 1)}` - } - } - - if (isValidUuid(outputId)) { - return outputId - } - - if (dotIndex === -1) { - logger.warn(`Invalid output ID format (missing dot): ${outputId}`) - return outputId - } - - const blockName = outputId.substring(0, dotIndex) - const path = outputId.substring(dotIndex + 1) - - const normalizedBlockName = normalizeName(blockName) - const block = Object.values(blocks).find((candidate) => { - const record = candidate as { name?: string } - return normalizeName(record.name || '') === normalizedBlockName - }) - - if (!block) { - logger.warn(`Block not found for name: ${blockName} (from output ID: ${outputId})`) - return outputId - } - - const resolvedId = `${(block as { id: string }).id}_${path}` - return resolvedId +): Promise { + return resolveOutputSelectors({ + selectedOutputs, + currentBlocks: blocks as Record, }) } diff --git a/apps/sim/lib/workflows/executor/execute-workflow.test.ts b/apps/sim/lib/workflows/executor/execute-workflow.test.ts index 4240058f7d3..9f6b48317fe 100644 --- a/apps/sim/lib/workflows/executor/execute-workflow.test.ts +++ b/apps/sim/lib/workflows/executor/execute-workflow.test.ts @@ -56,6 +56,7 @@ vi.mock('@/lib/workflows/executor/pause-persistence', () => ({ })) import { executeWorkflow } from '@/lib/workflows/executor/execute-workflow' +import { hasExecutionResult } from '@/executor/utils/errors' const workflowExecutionLoggerCallIndex = loggerMock.createLogger.mock.calls.findIndex( ([name]) => name === 'WorkflowExecution' @@ -234,6 +235,32 @@ describe('executeWorkflow', () => { ) }) + it('forwards a trusted immutable workflow state to the execution snapshot', async () => { + const workflowStateOverride = { + blocks: { 'block-1': { id: 'block-1', type: 'start_trigger' } }, + edges: [], + loops: {}, + parallels: {}, + variables: { + 'variable-1': { id: 'variable-1', name: 'deployed', value: 'frozen' }, + }, + deploymentVersionId: 'deployment-version-1', + } + + await executeWorkflow(workflow, 'request-1', { prompt: 'hello' }, 'actor-1', { + enabled: true, + principal, + billingAttribution, + workflowStateOverride, + }) + + const coreParams = executeWorkflowCoreMock.mock.calls[0]?.[0] as { + snapshot: ExecutionSnapshot + } + expect(coreParams.snapshot.metadata.workflowStateOverride).toEqual(workflowStateOverride) + expect(coreParams.snapshot.workflowVariables).toEqual(workflowStateOverride.variables) + }) + it('waits for post-execution persistence before resolving', async () => { let resolvePostExecution!: () => void waitForPostExecutionMock.mockReturnValueOnce( @@ -296,6 +323,44 @@ describe('executeWorkflow', () => { expect(executionSettled).toBe(true) }) + /** + * Post-execution work runs after the core has produced a result and the executor never sees + * its failure, so this layer is the only one that can carry the result onto it. Callers read a + * missing result as proof that no block ran — a Copilot run would report an executed workflow + * as never started and vouch for content it cannot describe. + */ + it('carries the execution result onto a post-execution failure', async () => { + const result = { success: true, output: { ran: true }, logs: [] } + executeWorkflowCoreMock.mockResolvedValueOnce(result) + handlePostExecutionPauseStateMock.mockRejectedValueOnce(new Error('pause persistence failed')) + + const thrown = await executeWorkflow(workflow, 'request-1', undefined, 'actor-1', { + enabled: true, + principal, + billingAttribution, + }).catch((error: unknown) => error) + + expect(hasExecutionResult(thrown)).toBe(true) + expect((thrown as { executionResult?: unknown }).executionResult).toBe(result) + }) + + /** A non-Error cannot carry the result, so it is normalized before anything reads it. */ + it('normalizes a non-Error post-execution failure so it can carry the result', async () => { + const result = { success: true, output: { ran: true }, logs: [] } + executeWorkflowCoreMock.mockResolvedValueOnce(result) + handlePostExecutionPauseStateMock.mockRejectedValueOnce('pause persistence exploded') + + const thrown = await executeWorkflow(workflow, 'request-1', undefined, 'actor-1', { + enabled: true, + principal, + billingAttribution, + }).catch((error: unknown) => error) + + expect(thrown).toBeInstanceOf(Error) + expect(hasExecutionResult(thrown)).toBe(true) + expect((thrown as { executionResult?: unknown }).executionResult).toBe(result) + }) + it('transfers post-execution ownership with successful streaming metadata', async () => { const result = await executeWorkflow(workflow, 'request-1', undefined, 'actor-1', { enabled: true, diff --git a/apps/sim/lib/workflows/executor/execute-workflow.ts b/apps/sim/lib/workflows/executor/execute-workflow.ts index 8336ea332d2..241e649be37 100644 --- a/apps/sim/lib/workflows/executor/execute-workflow.ts +++ b/apps/sim/lib/workflows/executor/execute-workflow.ts @@ -1,5 +1,6 @@ import type { WorkflowExecutionPrincipal } from '@sim/auth/principal' import { createLogger } from '@sim/logger' +import { toError } from '@sim/utils/errors' import { generateId } from '@sim/utils/id' import { assertBillingAttributionSnapshot, @@ -11,8 +12,13 @@ import { captureServerEvent } from '@/lib/posthog/server' import { executeWorkflowCore } from '@/lib/workflows/executor/execution-core' import { handlePostExecutionPauseState } from '@/lib/workflows/executor/pause-persistence' import { ExecutionSnapshot } from '@/executor/execution/snapshot' -import type { ExecutionMetadata, SerializableExecutionState } from '@/executor/execution/types' +import type { + BlockCompletionCallbackData, + ExecutionMetadata, + SerializableExecutionState, +} from '@/executor/execution/types' import type { ExecutionResult, StreamingExecution } from '@/executor/types' +import { attachExecutionResult, hasExecutionResult } from '@/executor/utils/errors' import type { ResolvedSecretTraceProvenanceV1 } from '@/executor/utils/resolved-secret-trace-registry' import type { CoreTriggerType } from '@/stores/logs/filters/types' @@ -38,7 +44,7 @@ export interface ExecuteWorkflowOptions { blockType: string, executionOrder: number ) => Promise - onBlockComplete?: (blockId: string, output: unknown) => Promise + onBlockComplete?: (blockId: string, output: unknown, outputBlockId?: string) => Promise /** Transfers post-execution logging ownership to the streaming caller after execution succeeds. */ skipLoggingComplete?: boolean includeFileBase64?: boolean @@ -48,6 +54,8 @@ export interface ExecuteWorkflowOptions { abortSignal?: AbortSignal /** Use the live/draft workflow state instead of the deployed state. Used by copilot. */ useDraftState?: boolean + /** Immutable workflow state selected by a trusted server-side trigger boundary. */ + workflowStateOverride?: NonNullable /** Stop execution after this block completes. Used for "run until block" feature. */ stopAfterBlockId?: string /** Run-from-block configuration using a prior execution snapshot. */ @@ -82,6 +90,12 @@ export interface ExecuteWorkflowOptions { * Callers set this only when the surface consumes thinking/tool events. */ agentEvents?: boolean + /** + * Gate subject for this run, separate from the billing actor + * (see {@link ExecutionMetadata.capabilityGovernedUserId}). Omit unless the + * trigger genuinely has a person distinct from the one it bills. + */ + capabilityGovernedUserId?: string | null } export interface WorkflowInfo { @@ -128,6 +142,12 @@ export async function executeWorkflow( loggingSession.setTrustedExecutionCorrelation(streamConfig.trustedExecutionCorrelation) } let postExecutionOwnershipTransferred = false + /** + * Held outside the `try` so the catch can carry it. The executor attaches its result when the + * run itself throws, but the post-execution work below can throw after a run has already + * produced one — and callers read a missing result as proof that no block ran. + */ + let executionResult: ExecutionResult | undefined try { const metadata: ExecutionMetadata = { @@ -136,12 +156,14 @@ export async function executeWorkflow( workflowId, workspaceId, userId: actorUserId, + capabilityGovernedUserId: streamConfig?.capabilityGovernedUserId, principal, billingAttribution, workflowUserId: workflow.userId, triggerType, triggerBlockId: streamConfig?.triggerBlockId, useDraftState: streamConfig?.useDraftState ?? false, + workflowStateOverride: streamConfig?.workflowStateOverride, startTime: new Date().toISOString(), isClientSession: false, enforceCredentialAccess: streamConfig?.enforceCredentialAccess ?? false, @@ -163,13 +185,13 @@ export async function executeWorkflow( metadata, workflow, input, - workflow.variables || {}, + streamConfig?.workflowStateOverride?.variables ?? workflow.variables ?? {}, streamConfig?.selectedOutputs || [] ) const executionStartMs = Date.now() - const result = await executeWorkflowCore({ + const result = (executionResult = await executeWorkflowCore({ snapshot, callbacks: { onStream: streamConfig?.onStream, @@ -184,8 +206,13 @@ export async function executeWorkflow( } : undefined, onBlockComplete: streamConfig?.onBlockComplete - ? async (blockId: string, _blockName: string, _blockType: string, output: unknown) => { - await streamConfig.onBlockComplete!(blockId, output) + ? async ( + blockId: string, + _blockName: string, + _blockType: string, + data: BlockCompletionCallbackData + ) => { + await streamConfig.onBlockComplete!(blockId, data.output, data.outputBlockId) } : undefined, }, @@ -197,7 +224,7 @@ export async function executeWorkflow( trustedInitialResolvedSecretTraceProvenance: streamConfig?.trustedInitialResolvedSecretTraceProvenance, runFromBlock: streamConfig?.runFromBlock, - }) + })) const blockTypes = [ ...new Set( @@ -240,7 +267,22 @@ export async function executeWorkflow( } return result - } catch (error: unknown) { + } catch (caught: unknown) { + /** + * Normalized before anything reads it, for the reason the executor normalizes its own throw: + * a value that cannot carry the result would otherwise reach callers bare, and they read a + * missing result as proof that no block ran. `toError` returns an `Error` unchanged, so a + * custom error class keeps its identity and every ordinary failure is untouched. + */ + const error = toError(caught) + /** + * Carries the run's result on a failure raised after it produced one — the post-execution + * work below the executor call can throw, and the executor never saw it. Skipped when the + * executor already attached its own, which is the more specific record. + */ + if (executionResult && !hasExecutionResult(error)) { + attachExecutionResult(error, executionResult) + } const errorDiagnostic = loggingSession.projectDiagnosticError(error) logger.error(`[${requestId}] Workflow execution failed`, errorDiagnostic) diff --git a/apps/sim/lib/workflows/executor/execution-core.ts b/apps/sim/lib/workflows/executor/execution-core.ts index f5ec36774a7..595fa7c35a0 100644 --- a/apps/sim/lib/workflows/executor/execution-core.ts +++ b/apps/sim/lib/workflows/executor/execution-core.ts @@ -10,8 +10,8 @@ import { createLogger } from '@sim/logger' import { getErrorMessage } from '@sim/utils/errors' import { filterUndefined, isPlainRecord, isRecordLike } from '@sim/utils/object' import { mergeSubblockStateWithValues } from '@sim/workflow-persistence/subblocks' +import type { Edge } from '@xyflow/react' import { eq } from 'drizzle-orm' -import type { Edge } from 'reactflow' import { z } from 'zod' import { type EffectivePiiRedaction, resolveEffectivePiiRedaction } from '@/lib/billing/retention' import { diff --git a/apps/sim/lib/workflows/executor/execution-status-projection.test.ts b/apps/sim/lib/workflows/executor/execution-status-projection.test.ts new file mode 100644 index 00000000000..823629e610c --- /dev/null +++ b/apps/sim/lib/workflows/executor/execution-status-projection.test.ts @@ -0,0 +1,180 @@ +/** + * @vitest-environment node + * + * `logs.trace_spans` and `logs.cost` are PROJECTIONS, not gates — a group + * withholds those fields from the response rather than refusing the read. + * + * The run-detail family applied none of it: a member whose group hides spend saw + * `cost` blanked on the log list and then read `cost.total` on the run one click + * deeper, and a member whose group hides execution detail got `finalOutput` and + * `blockOutputs` back whole from both the internal executions route and + * `/api/v2/workflows/{id}/runs/{runId}`. These run the real + * `getWorkflowExecutionStatus` against the real `resolveLogFieldProjection` — the + * same helper `readLogDetail` and the v1 routes resolve their flags through — so + * they fail if this read stops projecting. + */ +import { + dbChainMockFns, + permissionGroupScopeMock, + permissionGroupScopeMockFns, + queueTableRows, + resetDbChainMock, + resetPermissionGroupScopeMock, + schemaMock, +} from '@sim/testing' +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const { mockGetJob, mockMaterializeForDisplayWithBlockOutputs } = vi.hoisted(() => ({ + mockGetJob: vi.fn(), + mockMaterializeForDisplayWithBlockOutputs: vi.fn(), +})) + +vi.mock('@/lib/permission-groups/config-scope.server', () => permissionGroupScopeMock) + +vi.mock('@/lib/core/async-jobs', () => ({ + getJobQueue: vi.fn().mockResolvedValue({ getJob: mockGetJob }), +})) + +vi.mock('@/lib/logs/execution/trace-store', () => ({ + materializeExecutionDataForDisplayWithBlockOutputs: mockMaterializeForDisplayWithBlockOutputs, +})) + +vi.mock('@/lib/workflows/executor/paused-execution-metadata', () => ({ + getAutomaticResumeWaitingMetadata: vi.fn().mockReturnValue(null), +})) + +import { DEFAULT_PERMISSION_GROUP_CONFIG } from '@/lib/permission-groups/fields' +import { getWorkflowExecutionStatus } from '@/lib/workflows/executor/execution-status' + +const BLOCK_ID = 'block-1' + +function queueCompletedRun(): void { + queueTableRows(schemaMock.workflowExecutionLogs, [ + { + executionId: 'execution-1', + workflowId: 'workflow-1', + workspaceId: 'workspace-1', + status: 'completed', + level: 'info', + trigger: 'api', + startedAt: new Date('2026-08-05T12:00:00.000Z'), + endedAt: new Date('2026-08-05T12:00:01.000Z'), + totalDurationMs: 1000, + executionData: { executionState: {} }, + costTotal: '0.75', + }, + ]) + queueTableRows(schemaMock.resumeQueue, []) + queueTableRows(schemaMock.pausedExecutions, []) + mockMaterializeForDisplayWithBlockOutputs.mockResolvedValueOnce({ + executionData: { finalOutput: { answer: 'a customer address' } }, + blockOutputs: new Map([[BLOCK_ID, { answer: 'a customer address' }]]), + }) +} + +function readRun(viewerUserId: string | null | undefined) { + return getWorkflowExecutionStatus({ + workflowId: 'workflow-1', + executionId: 'execution-1', + includeOutput: true, + selectedOutputs: [BLOCK_ID], + workspaceId: 'workspace-1', + workspaceOrganizationId: 'organization-1', + viewerUserId, + }) +} + +describe('run-detail field projection', () => { + beforeEach(() => { + vi.clearAllMocks() + resetDbChainMock() + resetPermissionGroupScopeMock() + mockMaterializeForDisplayWithBlockOutputs.mockResolvedValue({ + executionData: {}, + blockOutputs: new Map(), + }) + }) + + it('reads the run whole for a member no group governs', async () => { + queueCompletedRun() + + const status = await readRun('user-1') + + expect(status?.cost).toEqual({ total: 0.75 }) + expect(status?.finalOutput).toEqual({ answer: 'a customer address' }) + expect(status?.blockOutputs).toEqual({ [BLOCK_ID]: { answer: 'a customer address' } }) + }) + + it('withholds the run total from a member whose group hides spend', async () => { + permissionGroupScopeMockFns.mockResolvePermissionGroupConfig.mockResolvedValue({ + ...DEFAULT_PERMISSION_GROUP_CONFIG, + hideCostInfo: true, + }) + queueCompletedRun() + + const status = await readRun('user-1') + + expect(status?.cost).toBeNull() + expect(status?.status).toBe('completed') + expect(status?.finalOutput).toEqual({ answer: 'a customer address' }) + }) + + it('withholds the execution payloads from a member whose group hides trace spans', async () => { + permissionGroupScopeMockFns.mockResolvePermissionGroupConfig.mockResolvedValue({ + ...DEFAULT_PERMISSION_GROUP_CONFIG, + hideTraceSpans: true, + }) + queueCompletedRun() + + const status = await readRun('user-1') + + expect(status?.finalOutput).toBeNull() + expect(status?.blockOutputs).toBeNull() + expect(status?.cost).toEqual({ total: 0.75 }) + expect(JSON.stringify(status)).not.toContain('a customer address') + }) + + /** + * A workspace API key authorizes as the workspace and represents no user, so + * its caller resolves to no subject. Substituting the key's creator would apply + * a bystander's group to every caller of a shared credential — which is why + * this asserts the resolver is never reached, not merely that the run came back + * whole. + */ + it('reads whole and resolves no group for a subjectless caller', async () => { + permissionGroupScopeMockFns.mockResolvePermissionGroupConfig.mockResolvedValue({ + ...DEFAULT_PERMISSION_GROUP_CONFIG, + hideCostInfo: true, + hideTraceSpans: true, + }) + queueCompletedRun() + + const status = await readRun(undefined) + + expect(status?.cost).toEqual({ total: 0.75 }) + expect(status?.finalOutput).toEqual({ answer: 'a customer address' }) + expect(status?.blockOutputs).toEqual({ [BLOCK_ID]: { answer: 'a customer address' } }) + expect(permissionGroupScopeMockFns.mockResolvePermissionGroupConfig).not.toHaveBeenCalled() + }) + + /** The queue branch answers before any log row exists, and is projected too. */ + it('withholds a queued run output from a member whose group hides trace spans', async () => { + permissionGroupScopeMockFns.mockResolvePermissionGroupConfig.mockResolvedValue({ + ...DEFAULT_PERMISSION_GROUP_CONFIG, + hideTraceSpans: true, + }) + dbChainMockFns.limit.mockResolvedValueOnce([]).mockResolvedValueOnce([]) + mockGetJob.mockResolvedValue({ + status: 'completed', + createdAt: new Date('2026-08-05T12:00:00.000Z'), + completedAt: new Date('2026-08-05T12:00:01.000Z'), + output: { output: { answer: 'a customer address' } }, + metadata: { workflowId: 'workflow-1', correlation: { triggerType: 'api' } }, + }) + + const status = await readRun('user-1') + + expect(status?.status).toBe('completed') + expect(status?.finalOutput).toBeNull() + }) +}) diff --git a/apps/sim/lib/workflows/executor/execution-status.test.ts b/apps/sim/lib/workflows/executor/execution-status.test.ts index 94f83e9453b..e593b9614fc 100644 --- a/apps/sim/lib/workflows/executor/execution-status.test.ts +++ b/apps/sim/lib/workflows/executor/execution-status.test.ts @@ -29,6 +29,9 @@ const input = { executionId: 'execution-1', includeOutput: false, selectedOutputs: [], + workspaceId: 'workspace-1', + /** No governing subject: field projection has its own suite next door. */ + viewerUserId: undefined, } describe('getWorkflowExecutionStatus queue projection', () => { diff --git a/apps/sim/lib/workflows/executor/execution-status.ts b/apps/sim/lib/workflows/executor/execution-status.ts index bc6cf1370df..873dc5182ed 100644 --- a/apps/sim/lib/workflows/executor/execution-status.ts +++ b/apps/sim/lib/workflows/executor/execution-status.ts @@ -5,6 +5,11 @@ import type { WorkflowExecutionStatusResponse } from '@/lib/api/contracts/workfl import { getJobQueue } from '@/lib/core/async-jobs' import type { Job } from '@/lib/core/async-jobs/types' import { materializeExecutionDataForDisplayWithBlockOutputs } from '@/lib/logs/execution/trace-store' +import { + type LogFieldProjection, + projectCostTotal, + resolveLogFieldProjection, +} from '@/lib/logs/log-projection' import { RESUME_EXECUTION_JOB_ID_PREFIX, WORKFLOW_EXECUTION_JOB_ID_PREFIX, @@ -120,10 +125,101 @@ export interface GetWorkflowExecutionStatusInput { executionId: string includeOutput: boolean selectedOutputs: string[] + /** + * The workspace the caller already authorized against. Passed in rather than + * read off the log row because this resource also answers from the job queue, + * for a run that has no log row yet — and that branch must be projected too. + */ + workspaceId: string + /** + * The user whose permission group governs the projection, or `null`/`undefined` + * when none does — a workspace API key (which authorizes as the workspace and + * whose reported user is only the key's creator) and an executor delegation + * (which carries a role but no capabilities) both read whole. + * + * Required rather than optional on purpose: a new consumer of this read has to + * name its subject to compile, instead of silently inheriting an unprojected + * response. + */ + viewerUserId: string | null | undefined + /** The workspace's organization, when the caller already loaded it. */ + workspaceOrganizationId?: string | null +} + +/** + * Applies a viewer's log projection to a run-detail resource. + * + * `finalOutput` and `blockOutputs` are the run-shaped spellings of `finalOutput` + * and `blockExecutions` on the withheld list in `withheldExecutionData` — the + * same per-block execution detail the log-detail path strips — so + * `logs.trace_spans` withholds them here too. A caller that could still name a + * block in `selectedOutputs` and get its output back would read exactly what the + * detail surface refuses, one query parameter later. + * + * `status`, `error` and the timings stay: these are projections, not gates, and + * withholding whether a run failed is not what the two capabilities restrict. + * + * permission-group-enforced: logs.trace_spans + * permission-group-enforced: logs.cost + */ +function projectExecutionStatus( + status: WorkflowExecutionStatusResponse, + projection: LogFieldProjection +): WorkflowExecutionStatusResponse { + if (!projection.hideCostInfo && !projection.hideTraceSpans) return status + return { + ...status, + cost: projectCostTotal(status.cost?.total ?? null, projection), + finalOutput: projection.hideTraceSpans ? null : status.finalOutput, + blockOutputs: projection.hideTraceSpans ? null : status.blockOutputs, + } +} + +/** A projected status resource together with the projection that produced it. */ +export interface ProjectedWorkflowExecutionStatus { + status: WorkflowExecutionStatusResponse + projection: LogFieldProjection +} + +/** + * Reads the execution status resource, projected for the viewer, and reports + * the projection alongside it. + * + * Projection lives in this shared read rather than in each of its route + * adapters so the rule has one copy and the next consumer inherits it, and it + * runs after the caller's authorization, on whichever branch answered. + * + * The projection is returned rather than kept private because a caller that + * appends more of the run's execution data to this resource has to withhold it + * on the same terms — a run's output *files* are the clearest case. Deriving + * that answer from the applied projection is what keeps the two from drifting; + * resolving the viewer's group a second time would be a second copy of the rule. + */ +export async function getProjectedWorkflowExecutionStatus( + input: GetWorkflowExecutionStatusInput +): Promise { + const status = await readWorkflowExecutionStatus(input) + if (!status) return null + const projection = await resolveLogFieldProjection( + input.viewerUserId, + input.workspaceId, + input.workspaceOrganizationId + ) + return { status: projectExecutionStatus(status, projection), projection } } +/** + * The projected status resource alone, for a caller that renders nothing beyond + * it. + */ export async function getWorkflowExecutionStatus( input: GetWorkflowExecutionStatusInput +): Promise { + return (await getProjectedWorkflowExecutionStatus(input))?.status ?? null +} + +async function readWorkflowExecutionStatus( + input: GetWorkflowExecutionStatusInput ): Promise { const { workflowId, executionId, includeOutput, selectedOutputs } = input diff --git a/apps/sim/lib/workflows/executor/human-in-the-loop-manager.test.ts b/apps/sim/lib/workflows/executor/human-in-the-loop-manager.test.ts index 664249646a8..33dd706d8a5 100644 --- a/apps/sim/lib/workflows/executor/human-in-the-loop-manager.test.ts +++ b/apps/sim/lib/workflows/executor/human-in-the-loop-manager.test.ts @@ -1282,6 +1282,44 @@ describe('PauseResumeManager paused cancellation after pause release', () => { expect(dbChainMockFns.set).not.toHaveBeenCalled() }) + it('finalizes staged pause state without mutating a terminal parent log', async () => { + queueTableRows(workflowExecutionLogs, [{ status: 'completed' }]) + queueTableRows(pausedExecutions, [{ id: 'paused-exec-1', status: 'cancelling' }]) + queueTableRows(resumeQueue, [{ id: 'resume-entry-1' }]) + + await expect( + PauseResumeManager.finalizePausedCancellationForTerminalRun('execution-1', 'workflow-1', [ + 'resume-entry-1', + ]) + ).resolves.toBe(true) + + expect(dbChainMockFns.set).toHaveBeenNthCalledWith(1, { + status: 'cancelled', + updatedAt: expect.any(Date), + nextResumeAt: null, + }) + expect(dbChainMockFns.set).toHaveBeenNthCalledWith(2, { + status: 'failed', + completedAt: expect.any(Date), + failureReason: 'Paused execution cancelled', + }) + expect(dbChainMockFns.update).not.toHaveBeenCalledWith(workflowExecutionLogs) + expect(mockReleaseExecutionSlot).toHaveBeenCalledWith('resume-entry-1') + }) + + it('does not finalize or release an unconfirmed claimed resume', async () => { + queueTableRows(workflowExecutionLogs, [{ status: 'completed' }]) + queueTableRows(pausedExecutions, [{ id: 'paused-exec-1', status: 'cancelling' }]) + queueTableRows(resumeQueue, [{ id: 'resume-entry-1' }]) + + await expect( + PauseResumeManager.finalizePausedCancellationForTerminalRun('execution-1', 'workflow-1', []) + ).resolves.toBe(false) + + expect(dbChainMockFns.set).not.toHaveBeenCalled() + expect(mockReleaseExecutionSlot).not.toHaveBeenCalled() + }) + it('restores only cancellation-staged queue entries while the workflow remains active', async () => { queueTableRows(workflowExecutionLogs, [{ status: 'running' }]) queueTableRows(pausedExecutions, [{ id: 'paused-exec-1' }]) diff --git a/apps/sim/lib/workflows/executor/human-in-the-loop-manager.ts b/apps/sim/lib/workflows/executor/human-in-the-loop-manager.ts index 367026277d7..4cf96093dc2 100644 --- a/apps/sim/lib/workflows/executor/human-in-the-loop-manager.ts +++ b/apps/sim/lib/workflows/executor/human-in-the-loop-manager.ts @@ -4,8 +4,8 @@ import { createLogger } from '@sim/logger' import { toError } from '@sim/utils/errors' import { generateId } from '@sim/utils/id' import { isRecordLike, omit } from '@sim/utils/object' +import type { Edge } from '@xyflow/react' import { and, asc, desc, eq, inArray, lt, type SQL, sql } from 'drizzle-orm' -import type { Edge } from 'reactflow' import { releaseExecutionSlot } from '@/lib/billing/calculations/usage-reservation' import { assertBillingAttributionSnapshot } from '@/lib/billing/core/billing-attribution' import { @@ -1317,7 +1317,9 @@ export class PauseResumeManager { ) if (blockLogIndex !== -1) { // Filter output for logging using shared utility - // 'resume' is redundant with url/resumeEndpoint so we filter it out + // 'resume' is redundant with url/resumeEndpoint so we filter it out. + // The type is only used to read the block's `outputs` for `hiddenFromDisplay`, + // and v2 inherits that map from v1 verbatim — so both versions filter alike. const filteredOutput = filterOutputForLog('human_in_the_loop', mergedOutput, { additionalHiddenKeys: ['resume'], }) @@ -2732,6 +2734,109 @@ export class PauseResumeManager { return transition.cancelled } + /** + * Finalizes only pause and resume state when a non-cancellation terminal + * transition wins the parent execution race. The parent log is locked and + * inspected but never mutated, so a late claimed resume cannot revive it. + * Every claimed resume must be stopped before its queue row is finalized. + */ + static async finalizePausedCancellationForTerminalRun( + executionId: string, + workflowId: string, + stoppedResumeEntryIds: string[] + ): Promise { + const now = new Date() + + const transition = await execDb.transaction(async (tx) => { + const executionLog = await tx + .select({ status: workflowExecutionLogs.status }) + .from(workflowExecutionLogs) + .where( + and( + eq(workflowExecutionLogs.executionId, executionId), + eq(workflowExecutionLogs.workflowId, workflowId) + ) + ) + .for('update') + .limit(1) + .then((rows) => rows[0]) + + if (executionLog?.status === 'running' || executionLog?.status === 'pending') { + return { finalized: false, claimedResumeEntryIds: [] as string[] } + } + + const pausedExecution = await tx + .select({ id: pausedExecutions.id, status: pausedExecutions.status }) + .from(pausedExecutions) + .where( + and( + eq(pausedExecutions.executionId, executionId), + eq(pausedExecutions.workflowId, workflowId), + inArray(pausedExecutions.status, ['cancelling', 'cancelled']) + ) + ) + .for('update') + .limit(1) + .then((rows) => rows[0]) + + if (!pausedExecution) { + return { finalized: true, claimedResumeEntryIds: [] as string[] } + } + + const claimedResumeEntries = await tx + .select({ id: resumeQueue.id }) + .from(resumeQueue) + .where( + and( + eq(resumeQueue.parentExecutionId, executionId), + eq(resumeQueue.pausedExecutionId, pausedExecution.id), + eq(resumeQueue.status, 'claimed') + ) + ) + .for('update') + + const stoppedResumeEntryIdSet = new Set(stoppedResumeEntryIds) + if (claimedResumeEntries.some((entry) => !stoppedResumeEntryIdSet.has(entry.id))) { + return { finalized: false, claimedResumeEntryIds: [] as string[] } + } + + if (pausedExecution.status !== 'cancelled') { + await tx + .update(pausedExecutions) + .set({ status: 'cancelled', updatedAt: now, nextResumeAt: null }) + .where( + and( + eq(pausedExecutions.id, pausedExecution.id), + eq(pausedExecutions.status, 'cancelling') + ) + ) + } + + await tx + .update(resumeQueue) + .set({ + status: 'failed', + completedAt: now, + failureReason: 'Paused execution cancelled', + }) + .where( + and( + eq(resumeQueue.parentExecutionId, executionId), + eq(resumeQueue.pausedExecutionId, pausedExecution.id), + inArray(resumeQueue.status, ['pending', 'claimed']) + ) + ) + + return { + finalized: true, + claimedResumeEntryIds: claimedResumeEntries.map((entry) => entry.id), + } + }) + + await releaseCancelledResumeReservations(transition.claimedResumeEntryIds) + return transition.finalized + } + static async blockQueuedResumesForCancellation( executionId: string, workflowId: string @@ -2790,7 +2895,18 @@ export class PauseResumeManager { executionId: string, workflowId: string ): Promise { - const activeResume = await execDb + const activeResumes = await PauseResumeManager.getActiveResumeCancellationTargets( + executionId, + workflowId + ) + return activeResumes[0] ?? null + } + + static async getActiveResumeCancellationTargets( + executionId: string, + workflowId: string + ): Promise { + return await execDb .select({ resumeEntryId: resumeQueue.id, pausedExecutionId: resumeQueue.pausedExecutionId, @@ -2808,10 +2924,6 @@ export class PauseResumeManager { ) ) .orderBy(desc(resumeQueue.claimedAt)) - .limit(1) - .then((rows) => rows[0]) - - return activeResume ?? null } static async rollbackActiveResumeCancellation( diff --git a/apps/sim/lib/workflows/migrations/subblock-migrations.test.ts b/apps/sim/lib/workflows/migrations/subblock-migrations.test.ts index 6b2f7c5b2f9..84483755628 100644 --- a/apps/sim/lib/workflows/migrations/subblock-migrations.test.ts +++ b/apps/sim/lib/workflows/migrations/subblock-migrations.test.ts @@ -11,6 +11,7 @@ import * as blocksBarrel from '@/blocks' import { getBlock as getRealBlock } from '@/blocks/registry' import { backfillCanonicalModes, + migrateCanonicalModeIds, migrateSubblockIds, SUBBLOCK_ID_MIGRATIONS, } from './subblock-migrations' @@ -745,6 +746,73 @@ describe('migrateSubblockIds', () => { }) }) +describe('migrateCanonicalModeIds', () => { + function mistralBlock(data: Record, subBlocks: Record) { + return makeBlock({ type: 'mistral_parse_v3', data, subBlocks } as never) + } + + it('carries the selection across the document -> file rename', () => { + const { blocks, migrated } = migrateCanonicalModeIds({ + b1: mistralBlock({ canonicalModes: { document: 'advanced' } }, {}), + }) + + expect(migrated).toBe(true) + const modes = blocks.b1.data?.canonicalModes as Record + expect(modes).toEqual({ file: 'advanced' }) + }) + + /** + * The case the backfill alone cannot recover. `setBlockCanonicalMode` writes + * the mode without clearing the sibling, so a workflow that uploaded a file, + * switched to advanced, then typed a reference holds both values — and + * `resolveCanonicalMode` prefers basic whenever the basic side is populated. + * Without the rename the run would silently switch to the uploaded file. + */ + it('preserves advanced when both sides hold a value, which the backfill would not', () => { + const both = { + fileUpload: { id: 'fileUpload', type: 'file-upload', value: { name: 'a.pdf' } }, + fileReference: { id: 'fileReference', type: 'short-input', value: '' }, + } + + const { blocks } = migrateCanonicalModeIds({ + b1: mistralBlock({ canonicalModes: { document: 'advanced' } }, both), + }) + expect((blocks.b1.data?.canonicalModes as Record).file).toBe('advanced') + + // Same input through the backfill alone resolves to basic — the regression + // this migration exists to prevent. + const { blocks: backfilled } = backfillCanonicalModes({ + b1: mistralBlock({ canonicalModes: {} }, both), + }) + expect((backfilled.b1.data?.canonicalModes as Record).file).toBe('basic') + }) + + it('leaves a block that already stores the current id alone', () => { + const { blocks, migrated } = migrateCanonicalModeIds({ + b1: mistralBlock({ canonicalModes: { file: 'basic' } }, {}), + }) + + expect(migrated).toBe(false) + expect(blocks.b1.data?.canonicalModes).toEqual({ file: 'basic' }) + }) + + it('prefers a value already written under the current id over the legacy one', () => { + const { blocks } = migrateCanonicalModeIds({ + b1: mistralBlock({ canonicalModes: { document: 'advanced', file: 'basic' } }, {}), + }) + + expect(blocks.b1.data?.canonicalModes).toEqual({ file: 'basic' }) + }) + + it('does not touch a block type with no canonical rename', () => { + const { migrated } = migrateCanonicalModeIds({ + b1: makeBlock({ type: 'knowledge', data: { canonicalModes: { document: 'advanced' } } }), + }) + + expect(migrated).toBe(false) + }) +}) + describe('backfillCanonicalModes', () => { it('should add missing canonicalModes entry for knowledge block with basic value', () => { const input: Record = { diff --git a/apps/sim/lib/workflows/migrations/subblock-migrations.ts b/apps/sim/lib/workflows/migrations/subblock-migrations.ts index 52f91909997..c12c94fcab8 100644 --- a/apps/sim/lib/workflows/migrations/subblock-migrations.ts +++ b/apps/sim/lib/workflows/migrations/subblock-migrations.ts @@ -140,6 +140,14 @@ export const SUBBLOCK_ID_MIGRATIONS: Record): { return { blocks: result, migrated: anyMigrated } } +/** + * One legacy-to-current `canonicalParamId` rename for a block type. + * + * Renaming a canonical id is otherwise invisible to persistence — values are + * stored under subblock `id`, which does not move — with one exception: + * `data.canonicalModes` is keyed by canonical id, so the old entry is orphaned + * and the pair reverts to whatever {@link backfillCanonicalModes} infers. + * + * That inference is right whenever exactly one side holds a value, which is why + * this is a narrow migration rather than a general one. It is wrong when BOTH + * sides hold values: `setBlockCanonicalMode` writes the mode without clearing + * the sibling, so a workflow that uploaded a file, switched to advanced, and + * typed a reference has both — and `resolveCanonicalMode` prefers basic, which + * would silently swap which value the run uses. + */ +export interface CanonicalIdMigration { + /** The canonical id a legacy saved state stores the mode under. */ + from: string + /** The canonical id the block definition declares now. */ + to: string +} + +/** Canonical-id renames per block type. */ +export const CANONICAL_ID_MIGRATIONS: Record = { + /** + * The tool parameter is `file`, and `check-block-registry.ts` requires the + * canonical id to match it once the parameter is `user-only` — which it + * became so a direct `POST /api/v2/tools/{toolId}/execute` caller could + * supply the document at all. + */ + mistral_parse_v3: [{ from: 'document', to: 'file' }], +} + +/** + * Renames persisted canonical-mode keys whose block definition moved them. + * + * Runs before {@link backfillCanonicalModes} so a carried-over selection is + * already present and the backfill leaves it alone; anything genuinely missing + * still gets inferred there. + */ +export function migrateCanonicalModeIds(blocks: Record): { + blocks: Record + migrated: boolean +} { + let anyMigrated = false + const result: Record = {} + + for (const [blockId, block] of Object.entries(blocks)) { + const migrations = CANONICAL_ID_MIGRATIONS[block.type] + const modes = block.data?.canonicalModes + if (!migrations || !isPlainRecord(modes)) { + result[blockId] = block + continue + } + + type CanonicalModes = Record + let patched: CanonicalModes | null = null + for (const { from, to } of migrations) { + if (!(from in modes)) continue + const next: CanonicalModes = patched ?? { ...(modes as CanonicalModes) } + // A value already stored under the current id wins: it was written by the + // current definition, so it is newer than the legacy one. + if (!(to in next)) next[to] = next[from] + delete next[from] + patched = next + } + + if (!patched) { + result[blockId] = block + continue + } + + logger.info('Migrated legacy canonical-mode ids', { blockId: block.id, blockType: block.type }) + anyMigrated = true + result[blockId] = { ...block, data: { ...block.data, canonicalModes: patched } } + } + + return { blocks: result, migrated: anyMigrated } +} + /** * Backfills missing `canonicalModes` entries in block data. * diff --git a/apps/sim/lib/workflows/migrations/whatsapp-interactive-type.test.ts b/apps/sim/lib/workflows/migrations/whatsapp-interactive-type.test.ts index 87056804a6b..0b24fe2e408 100644 --- a/apps/sim/lib/workflows/migrations/whatsapp-interactive-type.test.ts +++ b/apps/sim/lib/workflows/migrations/whatsapp-interactive-type.test.ts @@ -4,7 +4,15 @@ import { afterAll, describe, expect, it, vi } from 'vitest' import type { BlockState } from '@/stores/workflows/workflow/types' +/** + * The backfill reads the WhatsApp block's declared sub-blocks, which the global + * registry stub empties. Only that block is registered. + */ vi.unmock('@/blocks/registry') +vi.mock('@/blocks/registry-maps', async () => { + const { partialBlockRegistry } = await import('@sim/testing/mocks/block-registry.mock') + return partialBlockRegistry(await import('@/blocks/blocks/whatsapp')) +}) import * as blocksBarrel from '@/blocks' import { getBlock as getRealBlock } from '@/blocks/registry' diff --git a/apps/sim/lib/workflows/operations/export-workflow.test.ts b/apps/sim/lib/workflows/operations/export-workflow.test.ts index 18ca2c7abff..07a32708768 100644 --- a/apps/sim/lib/workflows/operations/export-workflow.test.ts +++ b/apps/sim/lib/workflows/operations/export-workflow.test.ts @@ -32,6 +32,13 @@ vi.mock('@/blocks/registry', () => ({ import { buildWorkflowExportPayload } from '@/lib/workflows/operations/export-workflow' +/** + * Asserts real tool params and outputs, which the global `@/tools/metadata` + * and `@/tools/metadata-outputs` mocks in vitest.setup.ts empty. + */ +vi.unmock('@/tools/metadata') +vi.unmock('@/tools/metadata-outputs') + describe('buildWorkflowExportPayload', () => { beforeEach(() => { vi.clearAllMocks() diff --git a/apps/sim/lib/workflows/operations/export-workflow.ts b/apps/sim/lib/workflows/operations/export-workflow.ts index 74bfec7cc06..81ee238a3a6 100644 --- a/apps/sim/lib/workflows/operations/export-workflow.ts +++ b/apps/sim/lib/workflows/operations/export-workflow.ts @@ -1,4 +1,4 @@ -import type { Edge } from 'reactflow' +import type { Edge } from '@xyflow/react' import { loadWorkflowFromNormalizedTables } from '@/lib/workflows/persistence/utils' import { type ExportWorkflowState, diff --git a/apps/sim/lib/workflows/operations/import-export.test.ts b/apps/sim/lib/workflows/operations/import-export.test.ts index d6c0ed0e2f9..26fa1968ad8 100644 --- a/apps/sim/lib/workflows/operations/import-export.test.ts +++ b/apps/sim/lib/workflows/operations/import-export.test.ts @@ -1,6 +1,18 @@ import { describe, expect, it, vi } from 'vitest' +/** + * Import parsing migrates sub-block ids against each block's declared config, + * which the global registry stub empties. Only the blocks the fixtures name are + * registered. + */ vi.unmock('@/blocks/registry') +vi.mock('@/blocks/registry-maps', async () => { + const { partialBlockRegistry } = await import('@sim/testing/mocks/block-registry.mock') + return partialBlockRegistry( + await import('@/blocks/blocks/knowledge'), + await import('@/blocks/blocks/start_trigger') + ) +}) vi.mock('@/lib/api/client/request', () => ({ requestJson: vi.fn().mockResolvedValue({}), diff --git a/apps/sim/lib/workflows/operations/import-workflow.test.ts b/apps/sim/lib/workflows/operations/import-workflow.test.ts new file mode 100644 index 00000000000..f4d3046d256 --- /dev/null +++ b/apps/sim/lib/workflows/operations/import-workflow.test.ts @@ -0,0 +1,136 @@ +/** + * @vitest-environment node + */ +import { queueTableRows, resetDbChainMock, schemaMock } from '@sim/testing' +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const mocks = vi.hoisted(() => ({ + getUserPermissionConfig: vi.fn(), + performCreateWorkflow: vi.fn(), + performCreateWorkflowTransition: vi.fn(), + saveWorkflowToNormalizedTables: vi.fn(), + extractAndPersistCustomTools: vi.fn(), +})) + +vi.mock('@/lib/permission-groups/resolve.server', () => ({ + getUserPermissionConfig: mocks.getUserPermissionConfig, +})) +vi.mock('@/lib/workflows/orchestration', () => ({ + performCreateWorkflow: mocks.performCreateWorkflow, + performCreateWorkflowTransition: mocks.performCreateWorkflowTransition, +})) +vi.mock('@/lib/workflows/persistence/utils', () => ({ + saveWorkflowToNormalizedTables: mocks.saveWorkflowToNormalizedTables, +})) +vi.mock('@/lib/workflows/persistence/custom-tools-persistence', () => ({ + extractAndPersistCustomTools: mocks.extractAndPersistCustomTools, +})) + +import { importWorkflowIntoWorkspace } from '@/lib/workflows/operations/import-workflow' + +function block(id: string, type: string) { + return { + id, + type, + name: id, + position: { x: 0, y: 0 }, + subBlocks: {}, + outputs: {}, + enabled: true, + } +} + +function payload(...blocks: ReturnType[]) { + return { + blocks: Object.fromEntries(blocks.map((entry) => [entry.id, entry])), + edges: [], + loops: {}, + parallels: {}, + } +} + +function params(workflowPayload: Record) { + return { + workspaceId: 'workspace-1', + userId: 'user-1', + capabilityUserId: 'user-1', + requestId: 'request-1', + workflow: workflowPayload, + } +} + +describe('importWorkflowIntoWorkspace block access', () => { + beforeEach(() => { + vi.clearAllMocks() + resetDbChainMock() + queueTableRows(schemaMock.workspace, [{ id: 'workspace-1' }]) + mocks.getUserPermissionConfig.mockResolvedValue(null) + mocks.performCreateWorkflow.mockResolvedValue({ + success: true, + workflow: { + id: 'workflow-1', + name: 'Imported Workflow', + description: null, + folderId: null, + sortOrder: 0, + createdAt: new Date(), + updatedAt: new Date(), + }, + }) + mocks.saveWorkflowToNormalizedTables.mockResolvedValue({ success: true }) + mocks.extractAndPersistCustomTools.mockResolvedValue({ saved: 0, errors: [] }) + }) + + /** + * The bypass this closes: import never went through the editing operations, + * so a denied integration reached the normalized tables and was refused only + * at run time, if ever. + */ + it('refuses a payload carrying a block type the permission group withholds', async () => { + mocks.getUserPermissionConfig.mockResolvedValue({ allowedIntegrations: ['slack'] }) + + const result = await importWorkflowIntoWorkspace( + params(payload(block('b1', 'slack'), block('b2', 'gmail'))) + ) + + expect(result).toMatchObject({ success: false, status: 403 }) + expect(result.success === false && result.error).toContain('gmail') + }) + + /** Nothing may be written before the refusal, or the caller is left an orphan. */ + it('refuses before any workflow row is created', async () => { + mocks.getUserPermissionConfig.mockResolvedValue({ allowedIntegrations: ['slack'] }) + + await importWorkflowIntoWorkspace(params(payload(block('b1', 'gmail')))) + + expect(mocks.performCreateWorkflow).not.toHaveBeenCalled() + expect(mocks.saveWorkflowToNormalizedTables).not.toHaveBeenCalled() + }) + + it('imports a payload whose block types the allowlist names', async () => { + mocks.getUserPermissionConfig.mockResolvedValue({ allowedIntegrations: ['slack'] }) + + const result = await importWorkflowIntoWorkspace(params(payload(block('b1', 'slack')))) + + expect(result.success).toBe(true) + expect(mocks.performCreateWorkflow).toHaveBeenCalledOnce() + }) + + /** + * `workflows.import` allows a workspace API key, which has no user and so no + * permission group. The attribution field still names someone — the billing + * owner, or the key's creator — and judging the payload against that + * bystander's allowlist is what this separates. + */ + it('judges no allowlist for a caller no permission group governs', async () => { + mocks.getUserPermissionConfig.mockResolvedValue({ allowedIntegrations: ['slack'] }) + + const result = await importWorkflowIntoWorkspace({ + ...params(payload(block('b1', 'gmail'))), + capabilityUserId: null, + }) + + expect(result.success).toBe(true) + expect(mocks.getUserPermissionConfig).not.toHaveBeenCalled() + }) +}) diff --git a/apps/sim/lib/workflows/operations/import-workflow.ts b/apps/sim/lib/workflows/operations/import-workflow.ts index 5103b939c9a..a6c42d03dbf 100644 --- a/apps/sim/lib/workflows/operations/import-workflow.ts +++ b/apps/sim/lib/workflows/operations/import-workflow.ts @@ -23,6 +23,10 @@ import { performCreateWorkflow, performCreateWorkflowTransition, } from '@/lib/workflows/orchestration' +import { + findWithheldBlockType, + withheldBlockTypeMessage, +} from '@/lib/workflows/persistence/block-access-guard' import { extractAndPersistCustomTools } from '@/lib/workflows/persistence/custom-tools-persistence' import { prepareWorkflowStateForPersistence } from '@/lib/workflows/persistence/prepare-state' import { saveWorkflowToNormalizedTables } from '@/lib/workflows/persistence/utils' @@ -59,7 +63,21 @@ export interface ImportWorkflowParams { description?: string /** Export envelope, bare state, or a JSON string of either. */ workflow: string | Record + /** Legacy attribution field: who the created workflow is recorded against. */ userId: string + /** + * The person whose permission group judges the payload's block types, or + * `null` when no group governs the caller — a workspace API key, which + * `workflows.import` allows and which has no user at all. + * + * Deliberately not {@link ImportWorkflowParams.userId}. That one is an + * attribution field: for a workspace key it holds the billing owner (the + * application path) or the key's creator (v1), and running either one's + * integration allowlist against a shared key's import would refuse it on a + * bystander's policy — and break the key outright once that person's group + * changed. + */ + capabilityUserId: string | null requestId: string } @@ -176,7 +194,7 @@ async function executeImportWorkflowIntoWorkspace( params: ImportWorkflowParams, createWorkflow: (params: PerformCreateWorkflowParams) => Promise ): Promise { - const { workspaceId, folderId, userId, requestId } = params + const { workspaceId, folderId, userId, capabilityUserId, requestId } = params const [workspaceData] = await db .select({ id: workspace.id }) @@ -256,6 +274,27 @@ async function executeImportWorkflowIntoWorkspace( const workflowState: WorkflowState = { ...parsedState, ...preparedState } + /** + * Nothing has been written yet, which is why the check sits here: an import + * carries blocks the caller never added through the editing operations, so + * this is the only place the workspace's integration allowlist is consulted + * before the graph becomes a stored workflow. + */ + const withheldBlockType = capabilityUserId + ? await findWithheldBlockType({ + userId: capabilityUserId, + workspaceId, + blocks: Object.values(workflowState.blocks), + }) + : null + if (withheldBlockType) { + return { + success: false, + status: 403, + error: withheldBlockTypeMessage(withheldBlockType), + } + } + let parsedPayload: unknown = rawWorkflow if (typeof rawWorkflow === 'string') { try { @@ -297,7 +336,19 @@ async function executeImportWorkflowIntoWorkspace( */ try { await db.transaction(async (tx) => { - const saveResult = await saveWorkflowToNormalizedTables(workflowId, workflowState, tx) + const saveResult = await saveWorkflowToNormalizedTables( + workflowId, + workflowState, + /** + * The same subject the pre-check above used. The pre-check stays because + * it renders this door's own 403 before the shell workflow row is + * created — a refusal after that point would have to roll the row back — + * and the two agree by construction: both read `capabilityUserId`, and + * an import with no governed user passes `null` to both. + */ + { workspaceId, subjectUserId: capabilityUserId ?? null }, + tx + ) if (!saveResult.success) { throw new Error(saveResult.error || 'Failed to save workflow state') } diff --git a/apps/sim/lib/workflows/orchestration/deploy.test.ts b/apps/sim/lib/workflows/orchestration/deploy.test.ts index 262a45c30d7..7a43664cdfa 100644 --- a/apps/sim/lib/workflows/orchestration/deploy.test.ts +++ b/apps/sim/lib/workflows/orchestration/deploy.test.ts @@ -163,6 +163,8 @@ describe('performRevertToVersion', () => { }, }, }), + /** A revert restores a graph the workspace already deployed, so it writes as nobody. */ + { workspaceId: null, subjectUserId: null }, dbChainMock.db ) expect(dbChainMockFns.set).toHaveBeenCalledWith( @@ -896,6 +898,7 @@ describe('mutation lock on the orchestration entry points', () => { expect(result.success).toBe(false) expect(result.error).toContain('locked') + expect(result.errorCode).toBe('locked') expect(mockRecordAudit).not.toHaveBeenCalled() }) diff --git a/apps/sim/lib/workflows/orchestration/deploy.ts b/apps/sim/lib/workflows/orchestration/deploy.ts index 5584a52412f..097a296a83e 100644 --- a/apps/sim/lib/workflows/orchestration/deploy.ts +++ b/apps/sim/lib/workflows/orchestration/deploy.ts @@ -151,7 +151,7 @@ export async function performFullDeploy( // Backstop for every caller — routes may assert first to render their own 423, // but the copilot deploy tools call this directly. const lockDenial = await workflowLockDenial(workflowId) - if (lockDenial) return { success: false, error: lockDenial, errorCode: 'validation' } + if (lockDenial) return { success: false, error: lockDenial, errorCode: 'locked' } const [workflowRecord] = await db .select() @@ -509,6 +509,7 @@ export interface PerformFullUndeployParams { export interface PerformFullUndeployResult { success: boolean error?: string + errorCode?: OrchestrationErrorCode warnings?: string[] } @@ -526,7 +527,7 @@ export async function performFullUndeploy( const requestId = params.requestId ?? generateRequestId() const lockDenial = await workflowLockDenial(workflowId) - if (lockDenial) return { success: false, error: lockDenial } + if (lockDenial) return { success: false, error: lockDenial, errorCode: 'locked' } const [workflowRecord] = await db .select() @@ -661,7 +662,7 @@ export async function performActivateVersion( const idempotencyKey = params.idempotencyKey ?? generateId() const lockDenial = await workflowLockDenial(workflowId) - if (lockDenial) return { success: false, error: lockDenial, errorCode: 'validation' } + if (lockDenial) return { success: false, error: lockDenial, errorCode: 'locked' } const [versionRow] = await db .select({ @@ -965,7 +966,22 @@ export async function performRevertToVersion( restoredState.variables = deployedState.variables || {} } - const result = await saveWorkflowToNormalizedTables(workflowId, restoredState, tx) + const result = await saveWorkflowToNormalizedTables( + workflowId, + restoredState, + { + /** + * Actorless, and deliberately so. This is the executor-adjacent path: + * it writes back a graph the workspace already deployed. A run + * persisting its own state must not be refused because the member who + * triggered it is in a group that withholds a block the deployment + * uses — the deployment was authorized when it was created. + */ + workspaceId: null, + subjectUserId: null, + }, + tx + ) if (!result.success) return result await tx diff --git a/apps/sim/lib/workflows/orchestration/workflow-lifecycle.ts b/apps/sim/lib/workflows/orchestration/workflow-lifecycle.ts index 7c5414f63ae..d2098cbd418 100644 --- a/apps/sim/lib/workflows/orchestration/workflow-lifecycle.ts +++ b/apps/sim/lib/workflows/orchestration/workflow-lifecycle.ts @@ -239,7 +239,19 @@ export async function performCreateWorkflowTransition( variables: {}, }) - await saveWorkflowToNormalizedTables(workflowId, workflowState, tx) + await saveWorkflowToNormalizedTables( + workflowId, + workflowState, + { + /** + * Actorless: the starter graph a new workflow is seeded with is the + * platform's, not a member's choice of blocks. + */ + workspaceId: null, + subjectUserId: null, + }, + tx + ) }) break } catch (error) { diff --git a/apps/sim/lib/workflows/persistence/block-access-guard.test.ts b/apps/sim/lib/workflows/persistence/block-access-guard.test.ts new file mode 100644 index 00000000000..4e61d7bd556 --- /dev/null +++ b/apps/sim/lib/workflows/persistence/block-access-guard.test.ts @@ -0,0 +1,64 @@ +/** + * @vitest-environment node + */ +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const mocks = vi.hoisted(() => ({ + getUserPermissionConfig: vi.fn(), +})) + +vi.mock('@/lib/permission-groups/resolve.server', () => ({ + getUserPermissionConfig: mocks.getUserPermissionConfig, +})) + +import { findWithheldBlockType } from '@/lib/workflows/persistence/block-access-guard' + +const PARAMS = { userId: 'user-1', workspaceId: 'workspace-1' } + +describe('findWithheldBlockType', () => { + beforeEach(() => { + vi.clearAllMocks() + mocks.getUserPermissionConfig.mockResolvedValue(null) + }) + + it('permits every block type when no permission group governs the workspace', async () => { + await expect( + findWithheldBlockType({ ...PARAMS, blocks: [{ type: 'gmail' }, { type: 'slack' }] }) + ).resolves.toBeNull() + }) + + it('permits every block type when the allowlist names every integration', async () => { + mocks.getUserPermissionConfig.mockResolvedValue({ allowedIntegrations: null }) + + await expect( + findWithheldBlockType({ ...PARAMS, blocks: [{ type: 'gmail' }] }) + ).resolves.toBeNull() + }) + + it('names the first block type the allowlist withholds', async () => { + mocks.getUserPermissionConfig.mockResolvedValue({ allowedIntegrations: ['slack'] }) + + await expect( + findWithheldBlockType({ + ...PARAMS, + blocks: [{ type: 'slack' }, { type: 'gmail' }, { type: 'notion' }], + }) + ).resolves.toBe('gmail') + }) + + /** + * Containers resolve to no integration, so an allowlist naming every + * permitted one would still withhold them — and a graph the editor happily + * builds could never be written back. + */ + it('does not withhold loop and parallel containers', async () => { + mocks.getUserPermissionConfig.mockResolvedValue({ allowedIntegrations: ['slack'] }) + + await expect( + findWithheldBlockType({ + ...PARAMS, + blocks: [{ type: 'loop' }, { type: 'parallel' }, { type: 'slack' }], + }) + ).resolves.toBeNull() + }) +}) diff --git a/apps/sim/lib/workflows/persistence/block-access-guard.ts b/apps/sim/lib/workflows/persistence/block-access-guard.ts new file mode 100644 index 00000000000..e7d624f012d --- /dev/null +++ b/apps/sim/lib/workflows/persistence/block-access-guard.ts @@ -0,0 +1,124 @@ +import { OrchestrationError } from '@/lib/core/orchestration/types' +import { isBlockTypeAccessControlExempt } from '@/lib/permission-groups/block-access' +import { resolvePermissionGroupConfig } from '@/lib/permission-groups/config-scope.server' +import { + resolveAccessControlBlockType, + toAccessControlAllowlist, +} from '@/lib/permission-groups/integration-allowlist' +import { BlockType } from '@/executor/constants' + +/** + * Loop and parallel are canvas containers rather than registry blocks, so they + * resolve to no integration and an allowlist naming every permitted integration + * would still withhold them. The editing operations skip them for the same + * reason, and the two paths must agree or a graph the editor accepts would be + * refused when it is written back. + */ +const CONTAINER_BLOCK_TYPES: ReadonlySet = new Set([BlockType.LOOP, BlockType.PARALLEL]) + +/** + * The first block type in `blocks` that the user's permission group withholds, + * or `null` when every one of them is permitted. + * + * `allowedIntegrations` is checked when a block is added through the editing + * operations, but a whole-graph write does not go through that path: the caller + * hands over the finished blocks, naming whatever types it likes. Validating at + * persist time is what makes the allowlist a property of what is *stored* + * rather than of one authoring route — otherwise a withheld integration lands + * in the workspace and is caught only by the executor refusing it mid-run, + * after the workflow has been saved, shared, and possibly deployed. + * + * Only the allowlist, deliberately — not the editor's `isBlockTypeAllowed`, + * which also refuses blocks hidden from the current viewer. Those two questions + * differ on a whole-graph write: refusing to *add* a preview block a viewer + * cannot see is right, while refusing to *store* a graph that already contains + * one would reject an export taken before the block was gated, and would make a + * save fail for a reason no permission group set. + */ +export async function findWithheldBlockType(params: { + userId: string + workspaceId: string + blocks: Iterable<{ type?: string }> +}): Promise { + const permissionConfig = await resolvePermissionGroupConfig( + params.userId, + params.workspaceId, + undefined + ) + const allowed = toAccessControlAllowlist(permissionConfig?.allowedIntegrations ?? null) + + /** + * Hoisted out of the loop: an unrestricted group is the common case, and every + * workflow save in every ungoverned workspace would otherwise pay two registry + * lookups per block to reach the same answer. + */ + if (allowed === null) return null + + for (const block of params.blocks) { + const blockType = block.type + if (!blockType || CONTAINER_BLOCK_TYPES.has(blockType)) continue + if (isBlockTypeAccessControlExempt(blockType)) continue + if (!allowed.has(resolveAccessControlBlockType(blockType).toLowerCase())) return blockType + } + + return null +} + +/** The refusal text every persist path renders for a withheld block type. */ +export function withheldBlockTypeMessage(blockType: string): string { + return `Block type "${blockType}" is not allowed by your organization's permission group` +} + +/** + * Who a normalized-state write is performed *as*, for permission-group purposes. + * + * Both fields are required at every call site, and `null` is spelled out rather + * than omitted, for the reason `capability` is required on + * `defineWorkspaceOperation`: an absent declaration cannot be told apart from an + * unreviewed one. The guard used to sit at individual doors, and the two that + * never grew one — `PUT /api/v2/workflows/{id}/state` and the Copilot + * materialize-import — were exactly the doors nobody remembered to add it to. + * + * `subjectUserId` is `null` only when the write is not a member's authoring + * action: an executor run persisting its own graph, a workspace fork copying + * rows, or workspace creation seeding a starter workflow. A member's group must + * not govern those, because the writer is the platform rather than the member. + */ +export interface WorkflowPersistGovernance { + /** Canonical workspace whose permission groups govern the write, or `null` when it has none. */ + workspaceId: string | null + /** The human this write is performed as, or `null` when it is performed as no human. */ + subjectUserId: string | null +} + +/** + * Refuses a normalized-state write carrying a block type the writer's permission + * group withholds. + * + * Lives on the shared persistence primitive rather than at each door so a new + * caller inherits the check instead of having to remember it. A `null` subject + * or workspace no-ops: there is no group to resolve, and inventing one would + * either fail open against a bystander's grants or block the executor. + * + * Throws {@link OrchestrationError} rather than returning a union, matching the + * rest of the persistence layer: `statusForOrchestrationError` renders + * `forbidden` as the 403 the pre-consolidation doors returned, and + * `messageForOrchestrationError` passes this message through unchanged, so the + * refusal a caller sees is byte-identical to the one it rendered itself. + */ +export async function assertNoWithheldBlockType( + governance: WorkflowPersistGovernance, + blocks: Iterable<{ type?: string }> +): Promise { + const { workspaceId, subjectUserId } = governance + if (!workspaceId || !subjectUserId) return + + const withheldBlockType = await findWithheldBlockType({ + userId: subjectUserId, + workspaceId, + blocks, + }) + if (withheldBlockType) { + throw new OrchestrationError('forbidden', withheldBlockTypeMessage(withheldBlockType)) + } +} diff --git a/apps/sim/lib/workflows/persistence/duplicate.ts b/apps/sim/lib/workflows/persistence/duplicate.ts index 5d6ad4329c0..30b394c3f42 100644 --- a/apps/sim/lib/workflows/persistence/duplicate.ts +++ b/apps/sim/lib/workflows/persistence/duplicate.ts @@ -410,7 +410,6 @@ export async function duplicateWorkflow( updatedConfig = structuredClone(subflow.config) as LoopConfig | ParallelConfig // Update the config ID to match the new subflow ID - ;(updatedConfig as any).id = newSubflowId /** diff --git a/apps/sim/lib/workflows/persistence/persist-block-access-gate.test.ts b/apps/sim/lib/workflows/persistence/persist-block-access-gate.test.ts new file mode 100644 index 00000000000..2645bb64bdd --- /dev/null +++ b/apps/sim/lib/workflows/persistence/persist-block-access-gate.test.ts @@ -0,0 +1,145 @@ +/** + * @vitest-environment node + * + * The gate lives on the shared write rather than at each door, so this is where + * it is proved: `saveWorkflowToNormalizedTables` is the one primitive every + * normalized-table write funnels through, and the assertions below are about + * the primitive, not about any caller that happens to reach it. + */ +import { + dbChainMock, + permissionGroupScopeMock, + permissionGroupScopeMockFns, + resetDbChainMock, + resetPermissionGroupScopeMock, +} from '@sim/testing' +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const mocks = vi.hoisted(() => ({ + saveRaw: vi.fn(), + lock: vi.fn(), +})) + +vi.mock('@/lib/permission-groups/config-scope.server', () => permissionGroupScopeMock) +vi.mock('@sim/workflow-persistence/save', () => ({ + saveWorkflowToNormalizedTables: mocks.saveRaw, +})) + +import { OrchestrationError } from '@/lib/core/orchestration/types' +import { saveWorkflowToNormalizedTables } from '@/lib/workflows/persistence/utils' +import type { WorkflowState } from '@/stores/workflows/workflow/types' + +function stateWith(type: string): WorkflowState { + return { + blocks: { + 'block-1': { + id: 'block-1', + type, + name: 'Block', + position: { x: 0, y: 0 }, + subBlocks: {}, + outputs: {}, + enabled: true, + }, + }, + edges: [], + loops: {}, + parallels: {}, + } as unknown as WorkflowState +} + +const GOVERNED = { workspaceId: 'workspace-1', subjectUserId: 'user-1' } + +describe('saveWorkflowToNormalizedTables permission-group gate', () => { + beforeEach(() => { + vi.clearAllMocks() + resetDbChainMock() + resetPermissionGroupScopeMock() + mocks.saveRaw.mockResolvedValue({ success: true }) + }) + + it('refuses a block type the governed subject’s allowlist withholds, before any write', async () => { + permissionGroupScopeMockFns.mockResolvePermissionGroupConfig.mockResolvedValue({ + allowedIntegrations: ['slack'], + }) + + await expect( + saveWorkflowToNormalizedTables('workflow-1', stateWith('gmail'), GOVERNED) + ).rejects.toMatchObject({ + name: 'OrchestrationError', + code: 'forbidden', + message: expect.stringContaining('gmail'), + }) + expect(mocks.saveRaw).not.toHaveBeenCalled() + }) + + it('writes a block type the allowlist names', async () => { + permissionGroupScopeMockFns.mockResolvePermissionGroupConfig.mockResolvedValue({ + allowedIntegrations: ['slack'], + }) + + await expect( + saveWorkflowToNormalizedTables('workflow-1', stateWith('slack'), GOVERNED, dbChainMock.db) + ).resolves.toMatchObject({ success: true }) + expect(mocks.saveRaw).toHaveBeenCalled() + }) + + /** + * The executor exemption. A run — or a revert, or a fork copy — persists a + * graph the workspace already holds, and blocking it on the triggering + * member's group would fail a run for a block the deployment was authorized + * with. Every such caller states that by passing a `null` subject. + */ + it('writes for an actorless caller even when the workspace withholds the block type', async () => { + permissionGroupScopeMockFns.mockResolvePermissionGroupConfig.mockResolvedValue({ + allowedIntegrations: ['slack'], + }) + + await expect( + saveWorkflowToNormalizedTables( + 'workflow-1', + stateWith('gmail'), + { workspaceId: 'workspace-1', subjectUserId: null }, + dbChainMock.db + ) + ).resolves.toMatchObject({ success: true }) + expect(mocks.saveRaw).toHaveBeenCalled() + expect(permissionGroupScopeMockFns.mockResolvePermissionGroupConfig).not.toHaveBeenCalled() + }) + + it('writes when no workspace, and therefore no permission group, scopes the workflow', async () => { + permissionGroupScopeMockFns.mockResolvePermissionGroupConfig.mockResolvedValue({ + allowedIntegrations: ['slack'], + }) + + await expect( + saveWorkflowToNormalizedTables( + 'workflow-1', + stateWith('gmail'), + { workspaceId: null, subjectUserId: 'user-1' }, + dbChainMock.db + ) + ).resolves.toMatchObject({ success: true }) + expect(mocks.saveRaw).toHaveBeenCalled() + }) + + /** + * The refusal must not be folded into the `{ success: false }` union: every + * caller renders that as a 500, and this one is a 403. + */ + it('throws the refusal rather than returning it, on the external-transaction path too', async () => { + permissionGroupScopeMockFns.mockResolvePermissionGroupConfig.mockResolvedValue({ + allowedIntegrations: ['slack'], + }) + + const thrown = await saveWorkflowToNormalizedTables( + 'workflow-1', + stateWith('gmail'), + GOVERNED, + dbChainMock.db + ).catch((error: unknown) => error) + + expect(thrown).toBeInstanceOf(OrchestrationError) + expect(mocks.saveRaw).not.toHaveBeenCalled() + }) +}) diff --git a/apps/sim/lib/workflows/persistence/replace-normalized-state.test.ts b/apps/sim/lib/workflows/persistence/replace-normalized-state.test.ts index 516e73fb092..17c4fb5acc3 100644 --- a/apps/sim/lib/workflows/persistence/replace-normalized-state.test.ts +++ b/apps/sim/lib/workflows/persistence/replace-normalized-state.test.ts @@ -49,6 +49,7 @@ function input(overrides: Record = {}) { workflowId: 'workflow-1', workspaceId: 'workspace-1', attributedUserId: 'user-1', + subjectUserId: 'user-1', state: { blocks: { 'block-1': BLOCK }, edges: [] }, ...overrides, } as Parameters[0] @@ -107,6 +108,7 @@ describe('replaceWorkflowNormalizedState', () => { expect(mocks.save).toHaveBeenCalledWith( 'workflow-1', expect.objectContaining({ blocks: PREPARED.blocks, edges: PREPARED.edges }), + { workspaceId: 'workspace-1', subjectUserId: 'user-1' }, expect.anything() ) expect(mocks.prepare).toHaveBeenCalledBefore(mocks.save) diff --git a/apps/sim/lib/workflows/persistence/replace-normalized-state.ts b/apps/sim/lib/workflows/persistence/replace-normalized-state.ts index 9355a0bc45e..131645d8499 100644 --- a/apps/sim/lib/workflows/persistence/replace-normalized-state.ts +++ b/apps/sim/lib/workflows/persistence/replace-normalized-state.ts @@ -5,6 +5,7 @@ import { getPostgresConstraintName, getPostgresErrorCode } from '@sim/utils/erro import { and, eq, inArray, isNull, ne } from 'drizzle-orm' import { OrchestrationError } from '@/lib/core/orchestration/types' import type { DbOrTx } from '@/lib/db/types' +import { assertNoWithheldBlockType } from '@/lib/workflows/persistence/block-access-guard' import { extractAndPersistCustomTools } from '@/lib/workflows/persistence/custom-tools-persistence' import { type PreparedWorkflowState, @@ -189,6 +190,18 @@ export interface ReplaceWorkflowNormalizedStateInput { workspaceId: string | null /** Owner recorded on any custom tool this graph defines. */ attributedUserId: string + /** + * The human this replace is performed as, or `null` when it is performed as no + * human. + * + * Deliberately separate from `attributedUserId`, which answers a workspace API + * key with the workspace's billing owner: attribution is a billing question + * and fails open here, where this one decides whether a member's own + * permission group may store a block type. Required, and `null` spelled out, + * so an actorless write is a claim the caller made rather than an argument it + * forgot. + */ + subjectUserId: string | null /** * The graph to write, or a reader that produces it. * @@ -230,9 +243,24 @@ export interface ReplaceWorkflowNormalizedStateResult { export async function replaceWorkflowNormalizedState( input: ReplaceWorkflowNormalizedStateInput ): Promise { - const { workflowId, workspaceId, attributedUserId, state, requestId } = input + const { workflowId, workspaceId, attributedUserId, subjectUserId, state, requestId } = input const logPrefix = requestId ? `[${requestId}] ` : '' + /** + * Hoisted ahead of the transaction even though the shared write checks it + * again: the second call is answered from the request-scoped memo, and + * refusing here means a withheld block type never takes the workflow's row + * lock or reaches drizzle's transaction wrapper — so the thrown + * `OrchestrationError` arrives at callers unwrapped. + * + * A caller that produces its graph from a reader is unaffected: the reader + * composes a graph from what is already stored, and this pass covers the + * blocks it hands back through the inner check. + */ + if (typeof state !== 'function') { + await assertNoWithheldBlockType({ workspaceId, subjectUserId }, Object.values(state.blocks)) + } + let preparedState!: PreparedWorkflowState let warnings: string[] = [] let workflowState!: WorkflowState @@ -277,7 +305,12 @@ export async function replaceWorkflowNormalizedState( let result: Awaited> try { - result = await saveWorkflowToNormalizedTables(workflowId, workflowState, tx) + result = await saveWorkflowToNormalizedTables( + workflowId, + workflowState, + { workspaceId, subjectUserId }, + tx + ) } catch (error) { if (isGraphIdUniqueViolation(error)) { throw new OrchestrationError( diff --git a/apps/sim/lib/workflows/persistence/save-normalized-state.ts b/apps/sim/lib/workflows/persistence/save-normalized-state.ts index a1c2a036513..c5e4239f84b 100644 --- a/apps/sim/lib/workflows/persistence/save-normalized-state.ts +++ b/apps/sim/lib/workflows/persistence/save-normalized-state.ts @@ -101,6 +101,15 @@ export async function saveWorkflowNormalizedState(params: { workflowId, workspaceId: workflowData.workspaceId ?? null, attributedUserId: userId, + /** + * This door authorizes by bare `userId`, so the writer and the governed + * subject are the same person. The integration allowlist that used to be + * checked inline here now lives on the shared write, which refuses a + * withheld block type as a `forbidden` `OrchestrationError` — read below + * by the same `asOrchestrationError` branch that classifies the rest, and + * rendered as the identical 403 and message. + */ + subjectUserId: userId, state: { blocks: state.blocks as Record, edges: state.edges as WorkflowState['edges'], diff --git a/apps/sim/lib/workflows/persistence/save-workflow-normalized-state.test.ts b/apps/sim/lib/workflows/persistence/save-workflow-normalized-state.test.ts index 99c1dde6531..b18ca24a384 100644 --- a/apps/sim/lib/workflows/persistence/save-workflow-normalized-state.test.ts +++ b/apps/sim/lib/workflows/persistence/save-workflow-normalized-state.test.ts @@ -12,6 +12,11 @@ import { beforeEach, describe, expect, it, vi } from 'vitest' const mocks = vi.hoisted(() => ({ replace: vi.fn(), notify: vi.fn(), + getUserPermissionConfig: vi.fn(), +})) + +vi.mock('@/lib/permission-groups/resolve.server', () => ({ + getUserPermissionConfig: mocks.getUserPermissionConfig, })) vi.mock('@/lib/workflows/persistence/replace-normalized-state', async () => { @@ -68,6 +73,58 @@ describe('saveWorkflowNormalizedState', () => { }) workflowAuthzMockFns.mockAssertWorkflowMutable.mockResolvedValue(undefined) mocks.replace.mockResolvedValue({ warnings: ['dropped an edge'], state: STATE }) + mocks.getUserPermissionConfig.mockResolvedValue(null) + }) + + /** + * The bypass this closes: a graph replace never went through the editing + * operations, so a member whose group withholds an integration could still + * store a block using it and have it refused only at run time, if ever. + * + * The check itself now lives on the shared write, so what this door owes is + * naming the right subject and rendering the primitive's `forbidden` refusal + * as the 403 it used to build inline. + */ + it('names the authorizing user as the subject the permission group governs', async () => { + await saveWorkflowNormalizedState(params()) + + expect(mocks.replace).toHaveBeenCalledWith( + expect.objectContaining({ subjectUserId: 'user-1', workspaceId: 'workspace-1' }) + ) + }) + + it('refuses a state carrying a block type the permission group withholds', async () => { + mocks.replace.mockRejectedValue( + new OrchestrationError( + 'forbidden', + 'Block type "gmail" is not allowed by your organization\'s permission group' + ) + ) + + const result = await saveWorkflowNormalizedState(params()) + + expect(result).toMatchObject({ success: false, status: 403 }) + expect(result.success === false && result.error).toContain('gmail') + expect(mocks.notify).not.toHaveBeenCalled() + }) + + it('stores a state whose block types the allowlist names', async () => { + mocks.getUserPermissionConfig.mockResolvedValue({ allowedIntegrations: ['starter'] }) + + await expect(saveWorkflowNormalizedState(params())).resolves.toMatchObject({ success: true }) + }) + + /** A workflow with no workspace has no permission group to resolve. */ + it('skips the block-type check for a workflow outside any workspace', async () => { + workflowAuthzMockFns.mockAuthorizeWorkflowByWorkspacePermission.mockResolvedValue({ + allowed: true, + status: 200, + workflow: { id: 'workflow-1', workspaceId: null }, + workspacePermission: 'write', + }) + + await expect(saveWorkflowNormalizedState(params())).resolves.toMatchObject({ success: true }) + expect(mocks.getUserPermissionConfig).not.toHaveBeenCalled() }) it('returns success with the preparation warnings and notifies once', async () => { diff --git a/apps/sim/lib/workflows/persistence/utils.test.ts b/apps/sim/lib/workflows/persistence/utils.test.ts index 0059d781832..b4b50ab88a0 100644 --- a/apps/sim/lib/workflows/persistence/utils.test.ts +++ b/apps/sim/lib/workflows/persistence/utils.test.ts @@ -293,6 +293,13 @@ const mockWorkflowState = createWorkflowState({ }, }) +/** + * The ungoverned write every characterization here performs: these exercise the + * table mechanics, not the permission-group gate, and a `null` subject is how a + * caller declares the write is not a member's authoring action. + */ +const UNGOVERNED = { workspaceId: null, subjectUserId: null } + describe('Database Helpers', () => { beforeEach(() => { vi.clearAllMocks() @@ -510,7 +517,8 @@ describe('Database Helpers', () => { it('should successfully save workflow data to normalized tables', async () => { const result = await dbHelpers.saveWorkflowToNormalizedTables( mockWorkflowId, - asAppState(mockWorkflowState) + asAppState(mockWorkflowState), + UNGOVERNED ) expect(result.success).toBe(true) @@ -523,7 +531,8 @@ describe('Database Helpers', () => { const result = await dbHelpers.saveWorkflowToNormalizedTables( mockWorkflowId, - asAppState(emptyWorkflowState) + asAppState(emptyWorkflowState), + UNGOVERNED ) expect(result.success).toBe(true) @@ -534,7 +543,8 @@ describe('Database Helpers', () => { const result = await dbHelpers.saveWorkflowToNormalizedTables( mockWorkflowId, - asAppState(mockWorkflowState) + asAppState(mockWorkflowState), + UNGOVERNED ) expect(result.success).toBe(false) @@ -549,7 +559,8 @@ describe('Database Helpers', () => { const result = await dbHelpers.saveWorkflowToNormalizedTables( mockWorkflowId, - asAppState(mockWorkflowState) + asAppState(mockWorkflowState), + UNGOVERNED ) expect(result.success).toBe(false) @@ -557,7 +568,11 @@ describe('Database Helpers', () => { }) it('should properly format block data for database insertion', async () => { - await dbHelpers.saveWorkflowToNormalizedTables(mockWorkflowId, asAppState(mockWorkflowState)) + await dbHelpers.saveWorkflowToNormalizedTables( + mockWorkflowId, + asAppState(mockWorkflowState), + UNGOVERNED + ) const [capturedBlockInserts = []] = insertedRowsFor(schemaMock.workflowBlocks) const [capturedEdgeInserts = []] = insertedRowsFor(schemaMock.workflowEdges) @@ -631,7 +646,11 @@ describe('Database Helpers', () => { staleWorkflowState.loops = {} staleWorkflowState.parallels = {} - await dbHelpers.saveWorkflowToNormalizedTables(mockWorkflowId, asAppState(staleWorkflowState)) + await dbHelpers.saveWorkflowToNormalizedTables( + mockWorkflowId, + asAppState(staleWorkflowState), + UNGOVERNED + ) const [capturedSubflowInserts = []] = insertedRowsFor(schemaMock.workflowSubflows) @@ -737,7 +756,8 @@ describe('Database Helpers', () => { const result = await dbHelpers.saveWorkflowToNormalizedTables( mockWorkflowId, - asAppState(largeWorkflowState) + asAppState(largeWorkflowState), + UNGOVERNED ) expect(result.success).toBe(true) @@ -869,7 +889,8 @@ describe('Database Helpers', () => { const saveResult = await dbHelpers.saveWorkflowToNormalizedTables( mockWorkflowId, - workflowState + workflowState, + UNGOVERNED ) expect(saveResult.success).toBe(true) @@ -940,7 +961,8 @@ describe('Database Helpers', () => { const saveResult = await dbHelpers.saveWorkflowToNormalizedTables( mockWorkflowId, - asAppState(testWorkflowState) + asAppState(testWorkflowState), + UNGOVERNED ) expect(saveResult.success).toBe(true) diff --git a/apps/sim/lib/workflows/persistence/utils.ts b/apps/sim/lib/workflows/persistence/utils.ts index 80da0d470d6..fc1f6984680 100644 --- a/apps/sim/lib/workflows/persistence/utils.ts +++ b/apps/sim/lib/workflows/persistence/utils.ts @@ -21,18 +21,23 @@ import { collectErrorSourceBlockIds, normalizeWorkflowEdgeHandles, } from '@sim/workflow-types/workflow' +import type { Edge } from '@xyflow/react' import type { InferSelectModel } from 'drizzle-orm' import { and, desc, eq, inArray, lt, sql } from 'drizzle-orm' import { LRUCache } from 'lru-cache' -import type { Edge } from 'reactflow' import { releaseWebhookPathClaims } from '@/lib/webhooks/path-claims' import { remapConditionBlockIds, remapConditionEdgeHandle } from '@/lib/workflows/condition-ids' import { isDynamicHandleSubblock } from '@/lib/workflows/dynamic-handle-topology' import { backfillCanonicalModes, + migrateCanonicalModeIds, migrateSubblockIds, } from '@/lib/workflows/migrations/subblock-migrations' import { backfillWhatsAppInteractiveType } from '@/lib/workflows/migrations/whatsapp-interactive-type' +import { + assertNoWithheldBlockType, + type WorkflowPersistGovernance, +} from '@/lib/workflows/persistence/block-access-guard' import { supersedeInFlightDeploymentOperations } from '@/lib/workflows/persistence/deployment-operations' import { sanitizeAgentToolsInBlocks } from '@/lib/workflows/sanitization/validation' @@ -367,6 +372,11 @@ const applyBlockMigrations = createMigrationPipeline([ return { ...ctx, blocks, migrated: ctx.migrated || migrated } }, + (ctx) => { + const { blocks, migrated } = migrateCanonicalModeIds(ctx.blocks) + return { ...ctx, blocks, migrated: ctx.migrated || migrated } + }, + (ctx) => { const { blocks, migrated } = backfillCanonicalModes(ctx.blocks) return { ...ctx, blocks, migrated: ctx.migrated || migrated } @@ -638,11 +648,30 @@ export function buildWorkflowDeploymentSnapshot( } } +/** + * The one door every normalized-table write goes through, and therefore the one + * place the workspace's integration allowlist can be enforced for all of them. + * + * `governance` is required rather than optional: a whole-graph write hands over + * finished blocks naming whatever types it likes, so every caller has to state + * whose grants judge them. Passing `{ subjectUserId: null }` is how a caller + * declares itself actorless — the executor persisting a run's own graph, a fork + * copying rows, workspace creation seeding a starter workflow — and that is a + * claim a reader can check, where an omitted argument was not. + * + * The check runs before any transaction is opened so a refusal never holds the + * workflow's row lock, and it throws rather than folding into the `{ success }` + * union: the union collapses to a 500 at every caller, and this refusal is a + * 403. + */ export async function saveWorkflowToNormalizedTables( workflowId: string, state: WorkflowState, + governance: WorkflowPersistGovernance, externalTx?: DbOrTx ): Promise<{ success: boolean; error?: string }> { + await assertNoWithheldBlockType(governance, Object.values(state.blocks)) + if (externalTx) { return saveWorkflowToNormalizedTablesRaw(workflowId, state, externalTx) } diff --git a/apps/sim/lib/workflows/sanitization/json-sanitizer.ts b/apps/sim/lib/workflows/sanitization/json-sanitizer.ts index a25be379e7d..01a7c0b4a8b 100644 --- a/apps/sim/lib/workflows/sanitization/json-sanitizer.ts +++ b/apps/sim/lib/workflows/sanitization/json-sanitizer.ts @@ -1,6 +1,6 @@ import { isRecordLike, sortObjectKeysDeep } from '@sim/utils/object' import { normalizeWorkflowEdgeSourceHandle } from '@sim/workflow-types/workflow' -import type { Edge } from 'reactflow' +import type { Edge } from '@xyflow/react' import { getBaseUrl } from '@/lib/core/utils/urls' import { sanitizeWorkflowForSharing } from '@/lib/workflows/credentials/credential-extractor' import { getBlock } from '@/blocks/registry' diff --git a/apps/sim/lib/workflows/sanitization/subblocks.test.ts b/apps/sim/lib/workflows/sanitization/subblocks.test.ts index 5dd15da3801..61f0e043bb0 100644 --- a/apps/sim/lib/workflows/sanitization/subblocks.test.ts +++ b/apps/sim/lib/workflows/sanitization/subblocks.test.ts @@ -3,7 +3,18 @@ */ import { describe, expect, it, vi } from 'vitest' +/** + * Sanitization reads each block's declared sub-block types, which the global + * registry stub empties. Only the blocks the cases below name are registered. + */ vi.unmock('@/blocks/registry') +vi.mock('@/blocks/registry-maps', async () => { + const { partialBlockRegistry } = await import('@sim/testing/mocks/block-registry.mock') + return partialBlockRegistry( + await import('@/blocks/blocks/condition'), + await import('@/blocks/blocks/function') + ) +}) import { migrateSubblockIds } from '@/lib/workflows/migrations/subblock-migrations' import { sanitizeMalformedSubBlocks } from '@/lib/workflows/sanitization/subblocks' diff --git a/apps/sim/lib/workflows/search-replace/indexer-selector-context.test.ts b/apps/sim/lib/workflows/search-replace/indexer-selector-context.test.ts new file mode 100644 index 00000000000..9b0c4bebcc0 --- /dev/null +++ b/apps/sim/lib/workflows/search-replace/indexer-selector-context.test.ts @@ -0,0 +1,78 @@ +/** + * @vitest-environment node + */ +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const { getSubBlocksForToolInput } = vi.hoisted(() => ({ + getSubBlocksForToolInput: vi.fn(), +})) + +vi.mock('@/tools/params', () => ({ + formatParameterLabel: (id: string) => id, + getToolIdForOperation: () => 'test_list', + getSubBlocksForToolInput, +})) + +import { getToolInputParamConfigs } from '@/lib/workflows/search-replace/indexer' + +/** + * The credential and the selector that depends on it, in the shape + * `getSubBlocksForToolInput` now returns for every user-facing param — whether the + * block declares the sub-block or it was synthesized from the param's declared type. + */ +const SELECTOR_SUB_BLOCKS = [ + { + id: 'credential', + title: 'Credential', + type: 'short-input', + canonicalParamId: 'oauthCredential', + }, + { + id: 'resourceId', + title: 'Resource', + type: 'dropdown', + selectorKey: 'gmail.labels', + dependsOn: ['credential'], + }, +] + +describe('tool-input selector context', () => { + beforeEach(() => getSubBlocksForToolInput.mockReset()) + + it.each([ + ['from the tool params alone', SELECTOR_SUB_BLOCKS], + [ + 'alongside an unrelated block sub-block', + [...SELECTOR_SUB_BLOCKS, { id: 'message', title: 'Message', type: 'short-input' }], + ], + ])('resolves a selector dependency %s', (_state, subBlocks) => { + getSubBlocksForToolInput.mockReturnValue({ subBlocks }) + + const configs = getToolInputParamConfigs({ + tool: { + type: 'test', + operation: 'list', + params: { + credential: 'credential-1', + resourceId: 'resource-1', + message: 'hello', + }, + }, + }) + + expect(configs.find((config) => config.paramId === 'resourceId')?.selectorContext).toEqual({ + oauthCredential: 'credential-1', + }) + }) + + it('returns the generic fallback when the tool has no registry definition', () => { + getSubBlocksForToolInput.mockReturnValue(null) + + const configs = getToolInputParamConfigs({ + tool: { type: 'test', operation: 'list', params: { message: 'hello' } }, + }) + + expect(configs.map((config) => config.paramId)).toEqual(['message']) + expect(configs[0].authoritative).toBe(false) + }) +}) diff --git a/apps/sim/lib/workflows/search-replace/indexer.test.ts b/apps/sim/lib/workflows/search-replace/indexer.test.ts index 099e99f3036..43e247c32f9 100644 --- a/apps/sim/lib/workflows/search-replace/indexer.test.ts +++ b/apps/sim/lib/workflows/search-replace/indexer.test.ts @@ -1,7 +1,7 @@ /** * @vitest-environment node */ -import { describe, expect, it } from 'vitest' +import { describe, expect, it, vi } from 'vitest' import { getToolInputParamConfigs, indexWorkflowSearchMatches, @@ -15,15 +15,11 @@ import { WORKFLOW_SEARCH_SUBFLOW_FIELD_IDS } from '@/lib/workflows/search-replac import { NoteBlock } from '@/blocks/blocks/note' /** - * Uses the real tool registry. Nothing here imports it directly — the dependency - * is transitive: the search-replace planner resolves tool input params through - * real subblock configs, so the global `@/tools/registry` mock in - * vitest.setup.ts empties the data these assertions read. - * - * Not a no-op, despite the lack of a direct import. Dropping this opt-out fails - * 8 tests across this file and its sibling suite. + * Asserts real tool params and outputs, which the global `@/tools/metadata` + * and `@/tools/metadata-outputs` mocks in vitest.setup.ts empty. */ -vi.unmock('@/tools/registry') +vi.unmock('@/tools/metadata') +vi.unmock('@/tools/metadata-outputs') describe('indexWorkflowSearchMatches', () => { it('marks generic tool-param fallbacks as non-authoritative', () => { diff --git a/apps/sim/lib/workflows/search-replace/indexer.ts b/apps/sim/lib/workflows/search-replace/indexer.ts index 8eb9e60b37e..575d2e96db6 100644 --- a/apps/sim/lib/workflows/search-replace/indexer.ts +++ b/apps/sim/lib/workflows/search-replace/indexer.ts @@ -3,6 +3,8 @@ import { forEachSearchOccurrence, projectEscapedMarkdownForSearch } from '@sim/u import { DEFAULT_SUBBLOCK_TYPE } from '@sim/workflow-persistence/subblocks' import type { SubBlockType } from '@sim/workflow-types/blocks' import { isWorkflowBlockProtected } from '@sim/workflow-types/workflow' +import { buildSelectorContextFromValues } from '@/lib/selectors/context' +import type { SelectorKey } from '@/lib/selectors/manifest' import { COMPARISON_OPERATORS, LOGICAL_OPERATORS } from '@/lib/table/query-builder/constants' import { getSearchableJsonStringLeaves, @@ -22,10 +24,10 @@ import type { WorkflowSearchBlockState, WorkflowSearchIndexerOptions, WorkflowSearchMatch, + WorkflowSearchSelectorContext, WorkflowSearchValuePath, } from '@/lib/workflows/search-replace/types' import { pathToKey, walkStringValues } from '@/lib/workflows/search-replace/value-walker' -import { SELECTOR_CONTEXT_FIELDS } from '@/lib/workflows/subblocks/context' import { getTransitiveSubBlockDependents } from '@/lib/workflows/subblocks/dependencies' import { resolveStoredToolName } from '@/lib/workflows/subblocks/display' import { @@ -48,13 +50,10 @@ import { type ParsedStoredTool, parseStoredToolInputValue } from '@/lib/workflow import { getBlock } from '@/blocks/registry' import type { SubBlockConfig } from '@/blocks/types' import { isReference } from '@/executor/constants' -import type { SelectorContext } from '@/hooks/selectors/types' import { formatParameterLabel, getSubBlocksForToolInput, getToolIdForOperation, - getToolParametersConfig, - type ToolParameterConfig, } from '@/tools/params' /** @@ -176,8 +175,11 @@ function looksLikeStructuredString(value: string): boolean { ) } -function getFallbackToolParamType(value: unknown, paramType?: string): SubBlockType { - if (paramType === 'object') return 'workflow-input-mapper' +/** + * The searchable shape of a value belonging to a tool with no registry definition — a + * custom or MCP tool, where nothing declares a type. Inferred from the value itself. + */ +function getFallbackToolParamType(value: unknown): SubBlockType { if (isRecordLike(value)) return 'workflow-input-mapper' if (typeof value !== 'string') return DEFAULT_SUBBLOCK_TYPE as SubBlockType @@ -656,41 +658,13 @@ function addTextMatches({ }) } -function buildToolInputSearchConfig(param: ToolParameterConfig): WorkflowSearchSubBlockConfig { - const uiComponent = param.uiComponent - return { - id: param.id, - title: uiComponent?.title ?? param.id, - type: (uiComponent?.type ?? getFallbackToolParamType(undefined, param.type)) as SubBlockType, - placeholder: uiComponent?.placeholder, - condition: uiComponent?.condition as SubBlockConfig['condition'], - serviceId: uiComponent?.serviceId, - selectorKey: uiComponent?.selectorKey, - requiredScopes: uiComponent?.requiredScopes, - mimeType: uiComponent?.mimeType, - canonicalParamId: uiComponent?.canonicalParamId, - mode: uiComponent?.mode, - password: uiComponent?.password, - dependsOn: uiComponent?.dependsOn, - } -} - -function isVisibleToolParameter(param: ToolParameterConfig, values: Record) { - if (param.visibility === 'hidden' || param.visibility === 'llm-only') return false - const condition = param.uiComponent?.condition - return ( - !condition || - evaluateSubBlockCondition(condition as Parameters[0], values) - ) -} - export interface ResolvedToolInputParamConfig { paramId: string config: WorkflowSearchSubBlockConfig value: unknown /** False when the codec had no registered tool definition and inferred only a generic shape. */ authoritative: boolean - selectorContext?: SelectorContext + selectorContext?: WorkflowSearchSelectorContext dependentValuePaths?: WorkflowSearchValuePath[] } @@ -761,39 +735,22 @@ export function getToolInputParamConfigs({ scopedCanonicalModes, blockConfig?.subBlocks ? { subBlocks: blockConfig.subBlocks } : undefined ) - const toolParams = getToolParametersConfig(toolId, tool.type, values) - const displayParams = toolParams?.userInputParameters ?? [] - - if (!toolParams && !subBlocksResult) return genericFallback() - - if (!subBlocksResult?.subBlocks.length) { - const fallbackCanonicalIndex = buildCanonicalIndex([]) - return displayParams - .filter((param) => isVisibleToolParameter(param, values)) - .map((param) => { - const config = buildToolInputSearchConfig(param) - return { - paramId: param.id, - authoritative: true, - config, - value: parseToolParamValue(toolParamValues[param.id], config.type), - selectorContext: - config.selectorKey || config.dependsOn - ? buildSelectorContext({ - subBlockConfig: config, - subBlockValues: values, - canonicalIndex: fallbackCanonicalIndex, - canonicalModes: scopedCanonicalModes, - }) - : undefined, - } - }) - } + if (!subBlocksResult) return genericFallback() + + /** + * The block's own sub-blocks plus the ones synthesized for params it does not declare. + * Selector and `dependsOn` resolution needs every sibling a value could be keyed by, + * including the ones filtered out of the rendered list by a failing condition. + */ + const blockSubBlocks = blockConfig?.subBlocks ?? [] + const blockSubBlockIds = new Set(blockSubBlocks.map((subBlock) => subBlock.id)) + const allToolSubBlocks = [ + ...blockSubBlocks, + ...subBlocksResult.subBlocks.filter((subBlock) => !blockSubBlockIds.has(subBlock.id)), + ] // canonical-index-unscoped: a nested tool's params are always the action surface - const toolCanonicalIndex = buildCanonicalIndex( - blockConfig?.subBlocks ?? subBlocksResult.subBlocks - ) + const toolCanonicalIndex = buildCanonicalIndex(allToolSubBlocks) const visibleSubBlocks = subBlocksResult.subBlocks.filter((subBlock) => isToolParamVisibleForReactiveCondition({ subBlockConfig: subBlock, @@ -803,30 +760,13 @@ export function getToolInputParamConfigs({ credentialTypeById, }) ) - const allToolSubBlocks = blockConfig?.subBlocks ?? subBlocksResult.subBlocks const getDependentValuePaths = (changedSubBlockId: string): WorkflowSearchValuePath[] => getTransitiveSubBlockDependents(allToolSubBlocks, [changedSubBlockId]).map((clear) => [ 'params', clear.subBlockId, ]) - const coveredParamIds = new Set( - visibleSubBlocks.flatMap((subBlock) => { - const ids = [subBlock.id] - if (subBlock.canonicalParamId) ids.push(subBlock.canonicalParamId) - const canonicalId = toolCanonicalIndex.canonicalIdBySubBlockId[subBlock.id] - if (canonicalId) { - const group = toolCanonicalIndex.groupsById[canonicalId] - if (group) { - if (group.basicId) ids.push(group.basicId) - ids.push(...group.advancedIds) - } - } - return ids - }) - ) - - const subBlockParams = visibleSubBlocks.map((config) => ({ + return visibleSubBlocks.map((config) => ({ paramId: config.id, authoritative: true, config, @@ -837,38 +777,18 @@ export function getToolInputParamConfigs({ ? buildSelectorContext({ subBlockConfig: config, subBlockValues: values, + contextConfigs: allToolSubBlocks, canonicalIndex: toolCanonicalIndex, canonicalModes: scopedCanonicalModes, }) : undefined, })) - const uncoveredParams = displayParams - .filter((param) => !coveredParamIds.has(param.id) && isVisibleToolParameter(param, values)) - .map((param) => { - const config = buildToolInputSearchConfig(param) - return { - paramId: param.id, - authoritative: true, - config, - value: parseToolParamValue(toolParamValues[param.id], config.type), - selectorContext: - config.selectorKey || config.dependsOn - ? buildSelectorContext({ - subBlockConfig: config, - subBlockValues: values, - canonicalIndex: toolCanonicalIndex, - canonicalModes: scopedCanonicalModes, - }) - : undefined, - } - }) - - return [...subBlockParams, ...uncoveredParams] } function buildSelectorContext({ subBlockConfig, subBlockValues, + contextConfigs, canonicalIndex, canonicalModes, workspaceId, @@ -876,22 +796,41 @@ function buildSelectorContext({ }: { subBlockConfig?: WorkflowSearchSubBlockConfig subBlockValues: Record + contextConfigs: SubBlockConfig[] canonicalIndex: ReturnType canonicalModes?: CanonicalModeOverrides workspaceId?: string workflowId?: string -}): SelectorContext { - const context: SelectorContext = {} +}): WorkflowSearchSelectorContext { + const context: WorkflowSearchSelectorContext = {} if (workspaceId) context.workspaceId = workspaceId if (workflowId) { context.workflowId = workflowId context.excludeWorkflowId = workflowId } - if (subBlockConfig?.mimeType) context.mimeType = subBlockConfig.mimeType - const { allDependsOnFields } = parseDependsOn(subBlockConfig?.dependsOn) + if (subBlockConfig?.selectorKey) { + const projected = buildSelectorContextFromValues({ + selectorKey: subBlockConfig.selectorKey as SelectorKey, + contextConfigs, + values: subBlockValues, + dependsOn: allDependsOnFields, + canonicalIndex, + canonicalModes, + staticContext: { mimeType: subBlockConfig.mimeType }, + }) + return { + ...projected, + ...(context.excludeWorkflowId ? { excludeWorkflowId: context.excludeWorkflowId } : {}), + ...(context.workflowId ? { workflowId: context.workflowId } : {}), + ...(context.workspaceId ? { workspaceId: context.workspaceId } : {}), + } + } + + if (subBlockConfig?.mimeType) context.mimeType = subBlockConfig.mimeType + for (const subBlockId of allDependsOnFields) { const value = normalizeDependencyValue( resolveDependencyValue(subBlockId, subBlockValues, canonicalIndex, canonicalModes) @@ -906,11 +845,8 @@ function buildSelectorContext({ context.mcpServerId = stringValue continue } - if (SELECTOR_CONTEXT_FIELDS.has(canonicalKey as keyof SelectorContext)) { - context[canonicalKey as keyof SelectorContext] = stringValue - } + context[canonicalKey as keyof WorkflowSearchSelectorContext] = stringValue } - return context } @@ -918,6 +854,7 @@ function buildSearchSelectorContext({ block, subBlockConfig, subBlockValues, + contextConfigs, canonicalIndex, workspaceId, workflowId, @@ -925,13 +862,15 @@ function buildSearchSelectorContext({ block: WorkflowSearchBlockState subBlockConfig?: WorkflowSearchSubBlockConfig subBlockValues: Record + contextConfigs: SubBlockConfig[] canonicalIndex: ReturnType workspaceId?: string workflowId?: string -}): SelectorContext { +}): WorkflowSearchSelectorContext { return buildSelectorContext({ subBlockConfig, subBlockValues, + contextConfigs, canonicalIndex, canonicalModes: getSearchCanonicalModes(block), workspaceId, @@ -1556,6 +1495,7 @@ export function indexWorkflowSearchMatches( block, subBlockConfig, subBlockValues, + contextConfigs: subBlockConfigs, canonicalIndex, workspaceId, workflowId, diff --git a/apps/sim/lib/workflows/search-replace/json-value-fields.ts b/apps/sim/lib/workflows/search-replace/json-value-fields.ts index 2b4c8a0dbbf..37b78cc08f7 100644 --- a/apps/sim/lib/workflows/search-replace/json-value-fields.ts +++ b/apps/sim/lib/workflows/search-replace/json-value-fields.ts @@ -4,6 +4,7 @@ import type { WorkflowSearchValuePath, } from '@/lib/workflows/search-replace/types' import { getValueAtPath, setValueAtPath } from '@/lib/workflows/search-replace/value-walker' +import { holdsObjectValue } from '@/tools/param-shape' const SEARCHABLE_JSON_ARRAY_VALUE_FIELDS: Partial>> = { 'condition-input': { @@ -29,12 +30,6 @@ const SEARCHABLE_JSON_OBJECT_VALUE_FIELDS: Partial> 'workflow-input-mapper': 'Value', } -const SERIALIZED_SUBBLOCK_VALUE_TYPES = new Set([ - 'file-upload', - 'grouped-checkbox-list', - 'table', -]) - export interface SearchableJsonStringLeaf { path: WorkflowSearchValuePath value: string @@ -120,8 +115,7 @@ export function shouldParseSerializedSubBlockValue( ): subBlockType is SubBlockType { return Boolean( subBlockType && - (isSearchableJsonValueSubBlock(subBlockType) || - SERIALIZED_SUBBLOCK_VALUE_TYPES.has(subBlockType)) + (isSearchableJsonValueSubBlock(subBlockType) || holdsObjectValue({ type: subBlockType })) ) } diff --git a/apps/sim/lib/workflows/search-replace/replacements.test.ts b/apps/sim/lib/workflows/search-replace/replacements.test.ts index 340fa434350..752a0e86d44 100644 --- a/apps/sim/lib/workflows/search-replace/replacements.test.ts +++ b/apps/sim/lib/workflows/search-replace/replacements.test.ts @@ -1,7 +1,7 @@ /** * @vitest-environment node */ -import { describe, expect, it } from 'vitest' +import { describe, expect, it, vi } from 'vitest' import { indexWorkflowSearchMatches } from '@/lib/workflows/search-replace/indexer' import { buildWorkflowSearchReplacePlan } from '@/lib/workflows/search-replace/replacements' import { @@ -11,15 +11,11 @@ import { import { WORKFLOW_SEARCH_SUBFLOW_FIELD_IDS } from '@/lib/workflows/search-replace/subflow-fields' /** - * Uses the real tool registry. Nothing here imports it directly — the dependency - * is transitive: the search-replace planner resolves tool input params through - * real subblock configs, so the global `@/tools/registry` mock in - * vitest.setup.ts empties the data these assertions read. - * - * Not a no-op, despite the lack of a direct import. Dropping this opt-out fails - * 8 tests across this file and its sibling suite. + * Asserts real tool params and outputs, which the global `@/tools/metadata` + * and `@/tools/metadata-outputs` mocks in vitest.setup.ts empty. */ -vi.unmock('@/tools/registry') +vi.unmock('@/tools/metadata') +vi.unmock('@/tools/metadata-outputs') describe('buildWorkflowSearchReplacePlan', () => { it('replaces selected text ranges across blocks without touching unselected matches', () => { diff --git a/apps/sim/lib/workflows/search-replace/resources/references.ts b/apps/sim/lib/workflows/search-replace/resources/references.ts index cfd718c833d..e96bc6169ff 100644 --- a/apps/sim/lib/workflows/search-replace/resources/references.ts +++ b/apps/sim/lib/workflows/search-replace/resources/references.ts @@ -7,11 +7,11 @@ import { import type { WorkflowSearchRange, WorkflowSearchResourceMeta, + WorkflowSearchSelectorContext, } from '@/lib/workflows/search-replace/types' import type { SubBlockConfig } from '@/blocks/types' import { normalizeName, REFERENCE } from '@/executor/constants' import { createEnvVarPattern, createReferencePattern } from '@/executor/utils/reference-validation' -import type { SelectorContext } from '@/hooks/selectors/types' export interface ParsedInlineReference { kind: 'environment' | 'workflow-reference' @@ -136,7 +136,7 @@ export function resolveInlineReferenceSearchText( export function parseStructuredResourceReferences( value: unknown, subBlockConfig?: Pick, - selectorContext?: SelectorContext + selectorContext?: WorkflowSearchSelectorContext ): StructuredResourceReference[] { return parseWorkflowSearchSubBlockResources(value, subBlockConfig, selectorContext) } diff --git a/apps/sim/lib/workflows/search-replace/resources/registry.ts b/apps/sim/lib/workflows/search-replace/resources/registry.ts index 37f19619ea4..bf61b54838f 100644 --- a/apps/sim/lib/workflows/search-replace/resources/registry.ts +++ b/apps/sim/lib/workflows/search-replace/resources/registry.ts @@ -4,10 +4,10 @@ import type { WorkflowSearchMatch, WorkflowSearchMatchKind, WorkflowSearchResourceMeta, + WorkflowSearchSelectorContext, WorkflowSearchValuePath, } from '@/lib/workflows/search-replace/types' import type { SubBlockConfig } from '@/blocks/types' -import type { SelectorContext } from '@/hooks/selectors/types' export type StructuredWorkflowSearchResourceKind = Exclude< WorkflowSearchMatchKind, @@ -21,7 +21,7 @@ interface ResourceCodecParseParams { SubBlockConfig, 'type' | 'serviceId' | 'selectorKey' | 'requiredScopes' | 'multiSelect' | 'multiple' > - selectorContext?: SelectorContext + selectorContext?: WorkflowSearchSelectorContext } export interface StructuredResourceReference { @@ -69,7 +69,7 @@ function createResourceMeta({ kind: StructuredWorkflowSearchResourceKind rawValue: string subBlockConfig: Pick - selectorContext?: SelectorContext + selectorContext?: WorkflowSearchSelectorContext }): WorkflowSearchResourceMeta { const resource: WorkflowSearchResourceMeta = { kind, @@ -409,7 +409,7 @@ export function parseWorkflowSearchSubBlockResources( SubBlockConfig, 'type' | 'serviceId' | 'selectorKey' | 'requiredScopes' | 'multiSelect' | 'multiple' >, - selectorContext?: SelectorContext + selectorContext?: WorkflowSearchSelectorContext ): StructuredResourceReference[] { const definition = getWorkflowSearchSubBlockResourceDefinition(subBlockConfig) if (!definition || !subBlockConfig) return [] diff --git a/apps/sim/lib/workflows/search-replace/types.ts b/apps/sim/lib/workflows/search-replace/types.ts index 4eefe464a80..4c3ff0ca36d 100644 --- a/apps/sim/lib/workflows/search-replace/types.ts +++ b/apps/sim/lib/workflows/search-replace/types.ts @@ -1,11 +1,11 @@ import type { SubBlockType } from '@sim/workflow-types/blocks' +import type { SelectorContext } from '@/lib/selectors/types' import type { WorkflowSearchSubflowEditableValue, WorkflowSearchSubflowFieldId, } from '@/lib/workflows/search-replace/subflow-fields' import type { StoredCustomToolRecord } from '@/lib/workflows/subblocks/display' import type { SubBlockConfig } from '@/blocks/types' -import type { SelectorContext } from '@/hooks/selectors/types' import type { BlockState, SubBlockState } from '@/stores/workflows/workflow/types' export type WorkflowSearchMode = 'text' | 'resource' | 'all' @@ -26,6 +26,14 @@ export type WorkflowSearchMatchKind = export type WorkflowSearchValuePath = Array +/** Raw selector inputs plus the resource scope needed by the client transport. */ +export interface WorkflowSearchSelectorContext extends SelectorContext { + workflowId?: string + workspaceId?: string + /** Search metadata for MCP tools; never forwarded to selectors.execute. */ + mcpServerId?: string +} + export interface WorkflowSearchRange { start: number end: number @@ -36,7 +44,7 @@ export interface WorkflowSearchResourceMeta { providerId?: string serviceId?: string selectorKey?: string - selectorContext?: SelectorContext + selectorContext?: WorkflowSearchSelectorContext resourceGroupKey?: string requiredScopes?: string[] token?: string @@ -119,7 +127,7 @@ export interface WorkflowSearchReplacementOption { providerId?: string serviceId?: string selectorKey?: string - selectorContext?: SelectorContext + selectorContext?: WorkflowSearchSelectorContext resourceGroupKey?: string } diff --git a/apps/sim/lib/workflows/streaming/nested-output-options.test.ts b/apps/sim/lib/workflows/streaming/nested-output-options.test.ts new file mode 100644 index 00000000000..8010fc1547f --- /dev/null +++ b/apps/sim/lib/workflows/streaming/nested-output-options.test.ts @@ -0,0 +1,108 @@ +import { describe, expect, it, vi } from 'vitest' + +vi.mock('@/lib/workflows/blocks/flatten-outputs', () => ({ + flattenWorkflowOutputs: (blocks: Iterable<{ id: string; name: string; type: string }>) => + [...blocks] + .filter((candidate) => candidate.type === 'agent') + .map((candidate) => ({ + blockId: candidate.id, + blockName: candidate.name, + blockType: candidate.type, + path: 'content', + })), +})) + +import { + buildWorkflowOutputMenu, + buildWorkflowOutputOptions, + collectReferencedWorkflowIds, + getWorkflowInvocationTarget, +} from '@/lib/workflows/streaming/nested-output-options' + +function block(id: string, type: string, name: string, subBlocks = {}, data = {}) { + return { + id, + type, + name, + subBlocks, + data, + position: { x: 0, y: 0 }, + outputs: {}, + enabled: true, + } +} + +describe('nested workflow output options', () => { + it('uses the active canonical workflow ID', () => { + const workflowBlock = block( + 'invoke', + 'workflow_input', + 'Research', + { + workflowId: { value: 'basic-workflow' }, + manualWorkflowId: { value: 'advanced-workflow' }, + }, + { canonicalModes: { workflowId: 'advanced' } } + ) + + expect(getWorkflowInvocationTarget(workflowBlock)).toBe('advanced-workflow') + }) + + it('builds workflow-scoped selectors and stops cycles', () => { + const root = { + blocks: { + invoke: block('invoke', 'workflow_input', 'Research', { + workflowId: { value: 'child-workflow' }, + }), + }, + edges: [], + } + const child = { + blocks: { + agent: block('agent', 'agent', 'Writer'), + cycle: block('cycle', 'workflow_input', 'Back to root', { + workflowId: { value: 'root-workflow' }, + }), + }, + edges: [], + } + + expect(collectReferencedWorkflowIds([root])).toEqual(['child-workflow']) + const options = buildWorkflowOutputOptions({ + rootWorkflowId: 'root-workflow', + rootState: root, + workflowStates: new Map([ + ['child-workflow', child], + ['root-workflow', root], + ]), + maxChildDepth: 3, + }) + + expect( + options.some( + (option) => + option.id === 'child-workflow.agent_content' && + option.label === 'child-workflow.writer.content' + ) + ).toBe(true) + expect(options.some((option) => option.menuPath.length > 2)).toBe(false) + + expect(buildWorkflowOutputMenu(options)).toMatchObject([ + { + blockId: 'invoke', + blockName: 'Research', + blockType: 'workflow_input', + outputs: [], + children: [ + { + blockId: 'invoke/agent', + blockName: 'Writer', + blockType: 'agent', + outputs: [{ id: 'child-workflow.agent_content', path: 'content' }], + children: [], + }, + ], + }, + ]) + }) +}) diff --git a/apps/sim/lib/workflows/streaming/nested-output-options.ts b/apps/sim/lib/workflows/streaming/nested-output-options.ts new file mode 100644 index 00000000000..1ec3ef71908 --- /dev/null +++ b/apps/sim/lib/workflows/streaming/nested-output-options.ts @@ -0,0 +1,186 @@ +import type { BlockState, WorkflowState } from '@sim/workflow-types/workflow' +import { flattenWorkflowOutputs } from '@/lib/workflows/blocks/flatten-outputs' +import { + formatInternalOutputSelector, + formatPublicOutputSelector, +} from '@/lib/workflows/streaming/output-selector' +import { normalizeName } from '@/executor/constants' + +const WORKFLOW_BLOCK_TYPES = new Set(['workflow', 'workflow_input']) + +export interface WorkflowOutputOption { + id: string + label: string + workflowId?: string + blockId: string + blockName: string + blockType: string + groupKey: string + groupLabel: string + path: string + menuPath: WorkflowOutputMenuSegment[] +} + +export interface WorkflowOutputMenuSegment { + blockId: string + blockName: string + blockType: string +} + +export interface WorkflowOutputMenuNode extends WorkflowOutputMenuSegment { + outputs: WorkflowOutputOption[] + children: WorkflowOutputMenuNode[] +} + +type OutputWorkflowState = Pick + +function unwrapSubBlockValue(value: unknown): unknown { + return value && typeof value === 'object' && 'value' in value + ? (value as { value: unknown }).value + : value +} + +/** Resolves the active literal child workflow selected by a regular Workflow block. */ +export function getWorkflowInvocationTarget(block: BlockState): string | undefined { + if (!WORKFLOW_BLOCK_TYPES.has(block.type)) return undefined + + const basicValue = unwrapSubBlockValue(block.subBlocks.workflowId) + const advancedValue = unwrapSubBlockValue(block.subBlocks.manualWorkflowId) + const mode = block.data?.canonicalModes?.workflowId + const selected = + mode === 'advanced' + ? advancedValue + : mode === 'basic' + ? basicValue + : typeof basicValue === 'string' && basicValue + ? basicValue + : advancedValue + + return typeof selected === 'string' && selected.trim() ? selected.trim() : undefined +} + +export function collectReferencedWorkflowIds( + states: Iterable +): string[] { + const workflowIds = new Set() + for (const state of states) { + if (!state) continue + for (const block of Object.values(state.blocks)) { + const workflowId = getWorkflowInvocationTarget(block) + if (workflowId) workflowIds.add(workflowId) + } + } + return [...workflowIds] +} + +interface BuildWorkflowOutputOptionsInput { + rootWorkflowId: string + rootState: OutputWorkflowState + workflowStates: ReadonlyMap + maxChildDepth: number +} + +/** Builds selectable outputs across regular child-workflow invocation paths. */ +export function buildWorkflowOutputOptions({ + rootWorkflowId, + rootState, + workflowStates, + maxChildDepth, +}: BuildWorkflowOutputOptionsInput): WorkflowOutputOption[] { + const options: WorkflowOutputOption[] = [] + + const visit = ( + workflowId: string, + state: OutputWorkflowState, + invocationPath: WorkflowOutputMenuSegment[], + childDepth: number, + callChain: ReadonlySet + ): void => { + const flattened = flattenWorkflowOutputs(Object.values(state.blocks), state.edges) + for (const output of flattened) { + const selectedWorkflowId = workflowId === rootWorkflowId ? undefined : workflowId + const displayBlockName = normalizeName(output.blockName || `block-${output.blockId}`) + const invocationNames = invocationPath.map((segment) => segment.blockName) + const menuParentId = invocationPath.at(-1)?.blockId + const menuBlockId = menuParentId ? `${menuParentId}/${output.blockId}` : output.blockId + const groupLabel = + invocationNames.length > 0 + ? `${invocationNames.join(' / ')} / ${output.blockName}` + : output.blockName + options.push({ + id: formatInternalOutputSelector(output.blockId, output.path, selectedWorkflowId), + label: formatPublicOutputSelector(displayBlockName, output.path, selectedWorkflowId), + workflowId: selectedWorkflowId, + blockId: output.blockId, + blockName: output.blockName, + blockType: output.blockType, + groupKey: menuBlockId, + groupLabel, + path: output.path, + menuPath: [ + ...invocationPath, + { + blockId: menuBlockId, + blockName: output.blockName, + blockType: output.blockType, + }, + ], + }) + } + + if (childDepth >= maxChildDepth) return + + for (const block of Object.values(state.blocks)) { + const childWorkflowId = getWorkflowInvocationTarget(block) + if (!childWorkflowId || callChain.has(childWorkflowId)) continue + const childState = workflowStates.get(childWorkflowId) + if (!childState) continue + const parentBlockId = invocationPath.at(-1)?.blockId + const blockId = parentBlockId ? `${parentBlockId}/${block.id}` : block.id + visit( + childWorkflowId, + childState, + [ + ...invocationPath, + { + blockId, + blockName: block.name, + blockType: block.type, + }, + ], + childDepth + 1, + new Set([...callChain, childWorkflowId]) + ) + } + } + + visit(rootWorkflowId, rootState, [], 0, new Set([rootWorkflowId])) + return options +} + +export function buildWorkflowOutputMenu( + options: readonly WorkflowOutputOption[] +): WorkflowOutputMenuNode[] { + const roots: WorkflowOutputMenuNode[] = [] + + for (const option of options) { + let siblings = roots + let node: WorkflowOutputMenuNode | undefined + + for (const segment of option.menuPath) { + node = siblings.find((candidate) => candidate.blockId === segment.blockId) + if (!node) { + node = { ...segment, outputs: [], children: [] } + siblings.push(node) + } + siblings = node.children + } + + if (!node) { + throw new Error(`Workflow output is missing its menu path: ${option.id}`) + } + node.outputs.push(option) + } + + return roots +} diff --git a/apps/sim/lib/workflows/streaming/output-selector.test.ts b/apps/sim/lib/workflows/streaming/output-selector.test.ts new file mode 100644 index 00000000000..b4e07d4916f --- /dev/null +++ b/apps/sim/lib/workflows/streaming/output-selector.test.ts @@ -0,0 +1,119 @@ +import { describe, expect, it } from 'vitest' +import { + formatInternalOutputSelector, + formatPublicOutputSelector, + parseInternalOutputSelector, + parsePublicOutputSelector, + parseStoredOutputSelector, + scopeOutputBlockId, + selectChildOutputSelectors, +} from '@/lib/workflows/streaming/output-selector' + +const CHILD_WORKFLOW_ID = '11111111-1111-4111-8111-111111111111' +const CHILD_BLOCK_ID = '22222222-2222-4222-8222-222222222222' + +function block(id: string, name: string) { + return { + id, + type: 'agent', + name, + subBlocks: {}, + position: { x: 0, y: 0 }, + outputs: {}, + enabled: true, + } +} + +describe('output selector scoping', () => { + it('parses root and workflow-scoped internal selectors', () => { + expect(parseInternalOutputSelector('agent_content')).toEqual({ + blockId: 'agent', + path: 'content', + }) + expect( + parseInternalOutputSelector(`${CHILD_WORKFLOW_ID}.${CHILD_BLOCK_ID}_content.text`) + ).toEqual({ + workflowId: CHILD_WORKFLOW_ID, + blockId: CHILD_BLOCK_ID, + path: 'content.text', + }) + }) + + it('uses current workflow block refs to distinguish nested selectors from dotted paths', () => { + const currentBlockRefs = new Set(['rootagent']) + + expect(parsePublicOutputSelector('rootagent.result.text', { currentBlockRefs })).toEqual({ + blockId: 'rootagent', + path: 'result.text', + }) + expect( + parsePublicOutputSelector(`${CHILD_WORKFLOW_ID}.writer.result.text`, { + currentBlockRefs, + }) + ).toEqual({ + workflowId: CHILD_WORKFLOW_ID, + blockId: 'writer', + path: 'result.text', + }) + }) + + it('recognizes stable stored internal selectors without confusing public underscores', () => { + const currentBlockRefs = new Set([CHILD_BLOCK_ID]) + + expect( + parseStoredOutputSelector(`${CHILD_BLOCK_ID}_content.text`, { currentBlockRefs }) + ).toEqual({ + blockId: CHILD_BLOCK_ID, + path: 'content.text', + }) + expect(parseStoredOutputSelector('my_agent.content', { currentBlockRefs })).toEqual({ + blockId: 'my_agent', + path: 'content', + }) + }) + + it('formats workflow-scoped selectors without invocation paths', () => { + expect(formatPublicOutputSelector('writer', 'content', CHILD_WORKFLOW_ID)).toBe( + `${CHILD_WORKFLOW_ID}.writer.content` + ) + expect(formatInternalOutputSelector(CHILD_BLOCK_ID, 'content', CHILD_WORKFLOW_ID)).toBe( + `${CHILD_WORKFLOW_ID}.${CHILD_BLOCK_ID}_content` + ) + expect(scopeOutputBlockId(CHILD_WORKFLOW_ID, CHILD_BLOCK_ID)).toBe( + `${CHILD_WORKFLOW_ID}.${CHILD_BLOCK_ID}` + ) + }) + + it('routes direct selections locally and forwards descendant workflow selections', () => { + const descendantWorkflowId = '33333333-3333-4333-8333-333333333333' + const directSelector = formatInternalOutputSelector('writer', 'content', CHILD_WORKFLOW_ID) + const descendantSelector = formatInternalOutputSelector( + 'reviewer', + 'result.text', + descendantWorkflowId + ) + + const selection = selectChildOutputSelectors( + CHILD_WORKFLOW_ID, + { [CHILD_BLOCK_ID]: block(CHILD_BLOCK_ID, 'Writer') }, + ['root_content', directSelector, descendantSelector] + ) + + expect(selection.selectedOutputs).toEqual([`${CHILD_BLOCK_ID}_content`, descendantSelector]) + expect(selection.selectedBlockRefs.get(CHILD_BLOCK_ID)).toBe('writer') + expect(selection.targetsChildWorkflow).toBe(true) + }) + + it.each([ + '', + ' workflow.agent_content', + 'workflow/agent_content', + '.agent_content', + 'workflow..agent_content', + 'agent_', + 'agent_content.', + 'agent_content..text', + ])('fails fast for malformed selector %j', (selector) => { + expect(() => parseInternalOutputSelector(selector)).toThrow('Invalid') + }) +}) diff --git a/apps/sim/lib/workflows/streaming/output-selector.ts b/apps/sim/lib/workflows/streaming/output-selector.ts new file mode 100644 index 00000000000..ac74dd9bf9b --- /dev/null +++ b/apps/sim/lib/workflows/streaming/output-selector.ts @@ -0,0 +1,264 @@ +import { isValidUuid } from '@sim/utils/id' +import type { BlockState } from '@sim/workflow-types/workflow' +import { normalizeName } from '@/executor/constants' + +const INTERNAL_OUTPUT_PATH_SEPARATOR = '_' +const PUBLIC_OUTPUT_PATH_SEPARATOR = '.' + +export interface ParsedOutputSelector { + /** Child workflow containing the selected block. Omitted for the current workflow. */ + workflowId?: string + /** Stable block ID internally, or normalized block name at a public boundary. */ + blockId: string + /** Dot path within the selected block output. Empty selects the whole block. */ + path: string +} + +export interface PublicOutputSelectorContext { + /** IDs and normalized names belonging to the workflow being executed. */ + currentBlockRefs: ReadonlySet + /** Known reachable child workflows. UUID workflow IDs are also recognized without preloading. */ + childWorkflowIds?: ReadonlySet +} + +export interface ChildOutputSelection { + selectedOutputs: string[] + /** Actual child block ID to the caller-supplied ref used in the scoped selector. */ + selectedBlockRefs: ReadonlyMap + targetsChildWorkflow: boolean +} + +function assertValidSelectorPart(value: string, label: string): void { + if (!value || value.trim() !== value || value.includes('/') || value.includes('.')) { + throw new Error(`Invalid output selector ${label}: ${value}`) + } +} + +function assertValidOutputPath(path: string): void { + if ( + path.trim() !== path || + path + .split(PUBLIC_OUTPUT_PATH_SEPARATOR) + .some((segment) => !segment || segment.trim() !== segment) + ) { + throw new Error(`Invalid output selector path: ${path}`) + } +} + +function assertSelector(selector: string): void { + if (!selector || selector.trim() !== selector || selector.includes('/')) { + throw new Error(`Invalid output selector: ${selector}`) + } +} + +function decodeInternalSelectorPart(value: string): string { + try { + return decodeURIComponent(value) + } catch { + throw new Error(`Invalid encoded output selector part: ${value}`) + } +} + +function encodeInternalSelectorPart(value: string): string { + assertValidSelectorPart(value, 'part') + return encodeURIComponent(value).replaceAll(INTERNAL_OUTPUT_PATH_SEPARATOR, '%5F') +} + +function parseScopedBlockRef( + value: string, + decodeInternal = false +): Pick { + const segments = value.split(PUBLIC_OUTPUT_PATH_SEPARATOR) + if (segments.length > 2) { + throw new Error(`Invalid output selector block reference: ${value}`) + } + const [rawFirst, rawSecond] = segments + const first = decodeInternal ? decodeInternalSelectorPart(rawFirst) : rawFirst + const second = rawSecond + ? decodeInternal + ? decodeInternalSelectorPart(rawSecond) + : rawSecond + : undefined + if (!first || (segments.length === 2 && !second)) { + throw new Error(`Invalid output selector block reference: ${value}`) + } + if (second) { + assertValidSelectorPart(first, 'workflow ID') + assertValidSelectorPart(second, 'block reference') + return { workflowId: first, blockId: second } + } + assertValidSelectorPart(first, 'block reference') + return { blockId: first } +} + +/** Parses caller-facing selectors using the current workflow to disambiguate dot paths. */ +export function parsePublicOutputSelector( + selector: string, + context?: PublicOutputSelectorContext +): ParsedOutputSelector { + assertSelector(selector) + const segments = selector.split(PUBLIC_OUTPUT_PATH_SEPARATOR) + if (segments.some((segment) => !segment || segment.trim() !== segment)) { + throw new Error(`Invalid output selector: ${selector}`) + } + + const [first, second, ...pathSegments] = segments + assertValidSelectorPart(first, 'block reference') + if (!second) return { blockId: first, path: '' } + + if (context?.currentBlockRefs.has(first)) { + const path = [second, ...pathSegments].join(PUBLIC_OUTPUT_PATH_SEPARATOR) + assertValidOutputPath(path) + return { blockId: first, path } + } + + const selectsChildWorkflow = context?.childWorkflowIds?.has(first) === true || isValidUuid(first) + if (context && selectsChildWorkflow && pathSegments.length > 0) { + assertValidSelectorPart(second, 'block reference') + const path = pathSegments.join(PUBLIC_OUTPUT_PATH_SEPARATOR) + assertValidOutputPath(path) + return { workflowId: first, blockId: second, path } + } + + const path = [second, ...pathSegments].join(PUBLIC_OUTPUT_PATH_SEPARATOR) + assertValidOutputPath(path) + return { blockId: first, path } +} + +/** Parses the executor-internal `blockId_path` or `workflowId.blockId_path` form. */ +export function parseInternalOutputSelector(selector: string): ParsedOutputSelector { + assertSelector(selector) + const separatorIndex = selector.indexOf(INTERNAL_OUTPUT_PATH_SEPARATOR) + const scopedBlockRef = separatorIndex > 0 ? selector.slice(0, separatorIndex) : selector + const path = separatorIndex > 0 ? selector.slice(separatorIndex + 1) : '' + if (separatorIndex === 0 || (separatorIndex > 0 && !path)) { + throw new Error(`Invalid output selector: ${selector}`) + } + const parsed = parseScopedBlockRef(scopedBlockRef, true) + if (parsed.workflowId && !path) { + throw new Error(`Nested output selector is missing its output path: ${selector}`) + } + if (path) assertValidOutputPath(path) + return { ...parsed, path } +} + +/** Parses output-picker state, whose canonical form is the internal selector form. */ +export function parseStoredOutputSelector( + selector: string, + context?: PublicOutputSelectorContext +): ParsedOutputSelector { + const separatorIndex = selector.indexOf(INTERNAL_OUTPUT_PATH_SEPARATOR) + if (separatorIndex > 0) { + const scopedBlockRef = selector.slice(0, separatorIndex) + const scopedSegments = scopedBlockRef.split(PUBLIC_OUTPUT_PATH_SEPARATOR) + const isCurrentStableBlock = context?.currentBlockRefs.has(scopedBlockRef) === true + const isChildStableBlock = + scopedSegments.length === 2 && + isValidUuid(scopedSegments[0]) && + isValidUuid(scopedSegments[1]) + const isEncodedInternalBlock = scopedBlockRef.includes('%') + if (isCurrentStableBlock || isChildStableBlock || isEncodedInternalBlock || !context) { + return parseInternalOutputSelector(selector) + } + } + return parsePublicOutputSelector(selector, context) +} + +function formatPublicScopedBlockRef(blockId: string, workflowId?: string): string { + assertValidSelectorPart(blockId, 'block reference') + if (!workflowId) return blockId + assertValidSelectorPart(workflowId, 'workflow ID') + return `${workflowId}${PUBLIC_OUTPUT_PATH_SEPARATOR}${blockId}` +} + +function formatInternalScopedBlockRef(blockId: string, workflowId?: string): string { + const encodedBlockId = encodeInternalSelectorPart(blockId) + if (!workflowId) return encodedBlockId + return `${encodeInternalSelectorPart(workflowId)}${PUBLIC_OUTPUT_PATH_SEPARATOR}${encodedBlockId}` +} + +/** Formats the caller-facing `block.path` or `workflow.block.path` selector. */ +export function formatPublicOutputSelector( + blockId: string, + path = '', + workflowId?: string +): string { + if (workflowId && !path) { + throw new Error('Nested output selectors require an output path') + } + const scopedBlockRef = formatPublicScopedBlockRef(blockId, workflowId) + if (path) assertValidOutputPath(path) + return path ? `${scopedBlockRef}${PUBLIC_OUTPUT_PATH_SEPARATOR}${path}` : scopedBlockRef +} + +/** Formats the canonical `block_path` or `workflow.block_path` executor selector. */ +export function formatInternalOutputSelector( + blockId: string, + path = '', + workflowId?: string +): string { + if (workflowId && !path) { + throw new Error('Nested output selectors require an output path') + } + const scopedBlockRef = formatInternalScopedBlockRef(blockId, workflowId) + if (path) assertValidOutputPath(path) + return path ? `${scopedBlockRef}${INTERNAL_OUTPUT_PATH_SEPARATOR}${path}` : scopedBlockRef +} + +export const formatOutputSelector = formatInternalOutputSelector + +/** Creates the external block identity emitted by a selected child workflow. */ +export function scopeOutputBlockId(workflowId: string, childBlockId: string): string { + if (childBlockId.includes(PUBLIC_OUTPUT_PATH_SEPARATOR)) { + parseScopedBlockRef(childBlockId, true) + return childBlockId + } + return formatInternalScopedBlockRef(childBlockId, workflowId) +} + +export function resolveOutputBlockRef( + blockRef: string, + blocks: Record +): string { + const exact = blocks[blockRef] + if (exact) return exact.id + + const blockValues = Object.values(blocks) + const idMatches = blockValues.filter((block) => block.id === blockRef) + if (idMatches.length === 1) return idMatches[0].id + + const normalizedRef = normalizeName(blockRef) + const matches = blockValues.filter((block) => normalizeName(block.name || '') === normalizedRef) + if (matches.length !== 1) { + throw new Error(`Selected output block does not resolve: ${blockRef}`) + } + return matches[0].id +} + +/** Routes workflow-scoped selections into a child executor. */ +export function selectChildOutputSelectors( + childWorkflowId: string, + childBlocks: Record, + selectedOutputs: readonly string[] | undefined +): ChildOutputSelection { + assertValidSelectorPart(childWorkflowId, 'workflow ID') + const childSelectors: string[] = [] + const selectedBlockRefs = new Map() + let targetsChildWorkflow = false + + for (const selector of selectedOutputs ?? []) { + const parsed = parseInternalOutputSelector(selector) + if (!parsed.workflowId) continue + if (parsed.workflowId !== childWorkflowId) { + childSelectors.push(selector) + continue + } + + targetsChildWorkflow = true + const childBlockId = resolveOutputBlockRef(parsed.blockId, childBlocks) + selectedBlockRefs.set(childBlockId, parsed.blockId) + childSelectors.push(formatInternalOutputSelector(childBlockId, parsed.path)) + } + + return { selectedOutputs: childSelectors, selectedBlockRefs, targetsChildWorkflow } +} diff --git a/apps/sim/lib/workflows/streaming/resolve-output-selectors.test.ts b/apps/sim/lib/workflows/streaming/resolve-output-selectors.test.ts new file mode 100644 index 00000000000..5e52d36c095 --- /dev/null +++ b/apps/sim/lib/workflows/streaming/resolve-output-selectors.test.ts @@ -0,0 +1,64 @@ +import { describe, expect, it } from 'vitest' +import { resolveOutputSelectors } from '@/lib/workflows/streaming/resolve-output-selectors' + +const ROOT_BLOCK_ID = '11111111-1111-4111-8111-111111111111' +const CHILD_WORKFLOW_ID = '22222222-2222-4222-8222-222222222222' + +function block(id: string, name: string) { + return { + id, + type: 'agent', + name, + subBlocks: {}, + position: { x: 0, y: 0 }, + outputs: {}, + enabled: true, + } +} + +describe('resolveOutputSelectors', () => { + it('resolves current names and defers child names to the authorized child loader', () => { + expect( + resolveOutputSelectors({ + selectedOutputs: [ + 'rootagent.result.text', + `${CHILD_WORKFLOW_ID}.answer_writer.result.text`, + ], + currentBlocks: { [ROOT_BLOCK_ID]: block(ROOT_BLOCK_ID, 'Root Agent') }, + }) + ).toEqual([`${ROOT_BLOCK_ID}_result.text`, `${CHILD_WORKFLOW_ID}.answer%5Fwriter_result.text`]) + }) + + it('rejects invocation-scoped slash selectors', () => { + expect(() => + resolveOutputSelectors({ + selectedOutputs: ['workflow-block/agent.content'], + currentBlocks: { [ROOT_BLOCK_ID]: block(ROOT_BLOCK_ID, 'Root Agent') }, + }) + ).toThrow('Invalid output selector') + }) + + it('uses referenced workflow IDs even when the ID is not UUID-shaped', () => { + const invocation = { + ...block('invoke', 'Research'), + type: 'workflow_input', + subBlocks: { workflowId: { value: 'child-workflow' } }, + } + + expect( + resolveOutputSelectors({ + selectedOutputs: ['child-workflow.writer.content'], + currentBlocks: { invoke: invocation }, + }) + ).toEqual(['child-workflow.writer_content']) + }) + + it('does not reinterpret an unknown root block name as a child workflow', () => { + expect(() => + resolveOutputSelectors({ + selectedOutputs: ['missing.result.text'], + currentBlocks: { [ROOT_BLOCK_ID]: block(ROOT_BLOCK_ID, 'Root Agent') }, + }) + ).toThrow('Selected output block does not resolve: missing') + }) +}) diff --git a/apps/sim/lib/workflows/streaming/resolve-output-selectors.ts b/apps/sim/lib/workflows/streaming/resolve-output-selectors.ts new file mode 100644 index 00000000000..bb83bd40600 --- /dev/null +++ b/apps/sim/lib/workflows/streaming/resolve-output-selectors.ts @@ -0,0 +1,38 @@ +import type { BlockState } from '@sim/workflow-types/workflow' +import { getWorkflowInvocationTarget } from '@/lib/workflows/streaming/nested-output-options' +import { + formatInternalOutputSelector, + parseStoredOutputSelector, + resolveOutputBlockRef, +} from '@/lib/workflows/streaming/output-selector' +import { normalizeName } from '@/executor/constants' + +interface ResolveOutputSelectorsOptions { + selectedOutputs: readonly string[] | undefined + currentBlocks: Record +} + +/** Resolves current-workflow names and leaves child names for its authorized loader. */ +export function resolveOutputSelectors({ + selectedOutputs, + currentBlocks, +}: ResolveOutputSelectorsOptions): string[] | undefined { + if (!selectedOutputs || selectedOutputs.length === 0) return selectedOutputs?.slice() + + const currentBlockRefs = new Set() + const childWorkflowIds = new Set() + for (const block of Object.values(currentBlocks)) { + currentBlockRefs.add(block.id) + currentBlockRefs.add(normalizeName(block.name || '')) + const childWorkflowId = getWorkflowInvocationTarget(block) + if (childWorkflowId) childWorkflowIds.add(childWorkflowId) + } + + return selectedOutputs.map((selector) => { + const parsed = parseStoredOutputSelector(selector, { currentBlockRefs, childWorkflowIds }) + const blockId = parsed.workflowId + ? parsed.blockId + : resolveOutputBlockRef(parsed.blockId, currentBlocks) + return formatInternalOutputSelector(blockId, parsed.path, parsed.workflowId) + }) +} diff --git a/apps/sim/lib/workflows/streaming/streaming.test.ts b/apps/sim/lib/workflows/streaming/streaming.test.ts index 5e5391358cc..1400be73a1a 100644 --- a/apps/sim/lib/workflows/streaming/streaming.test.ts +++ b/apps/sim/lib/workflows/streaming/streaming.test.ts @@ -9,6 +9,7 @@ import { agentStreamProtocolResponseHeaders, createStreamingResponse, } from '@/lib/workflows/streaming/streaming' +import type { AgentStreamSink } from '@/providers/stream-events' const workflowStreamingLoggerCallIndex = loggerMock.createLogger.mock.calls.findIndex( ([name]) => name === 'WorkflowStreaming' @@ -180,6 +181,7 @@ describe('createStreamingResponse', () => { streamConfig: {}, executeFn: async ({ onStream }) => { await onStream({ + blockId: 'agent-1', stream: new ReadableStream({ start(controller) { controller.error(rawError) @@ -217,6 +219,46 @@ describe('createStreamingResponse', () => { expect(rawError.message).toBe(message) }) + it('emits workflow-scoped block IDs for a nested agent stream', async () => { + const stream = await createStreamingResponse({ + requestId: 'request-nested-agent', + executionId: 'execution-1', + streamConfig: { + selectedOutputs: ['child-workflow.agent-1_content'], + includeFileBase64: false, + }, + executeFn: async ({ onStream }) => { + await onStream({ + blockId: 'child-workflow.agent-1', + stream: new ReadableStream({ + start(controller) { + controller.enqueue(new TextEncoder().encode('Nested answer')) + controller.close() + }, + }), + execution: { + success: true, + output: { content: 'Nested answer' }, + logs: [], + metadata: {}, + }, + }) + return { + success: true, + output: {}, + logs: [], + metadata: { duration: 1 }, + } + }, + }) + + const events = await collectSSEEvents(stream) + expect(events).toContainEqual({ + blockId: 'child-workflow.agent-1', + chunk: 'Nested answer', + }) + }) + it('extracts block-level selected outputs from JSON content payloads', async () => { const output = { content: JSON.stringify({ answer: 'ok' }) } const stream = await createStreamingResponse({ @@ -944,6 +986,7 @@ describe('createStreamingResponse agent-events-v1', () => { }) const onStreamPromise = onStream({ + blockId: 'agent-1', stream: textStream, streamFormat: 'text', subscribe: (nextSink: { onEvent: (event: unknown) => void | Promise }) => { @@ -1165,7 +1208,7 @@ describe('createStreamingResponse agent-events-v1', () => { }, executeFn: async ({ onStream }) => { let textController!: ReadableStreamDefaultController - let sink: { onEvent: (event: unknown) => void | Promise } | undefined + let sink: AgentStreamSink | undefined const textStream = new ReadableStream({ start(controller) { textController = controller @@ -1173,9 +1216,10 @@ describe('createStreamingResponse agent-events-v1', () => { }) const onStreamPromise = onStream({ + blockId: 'agent-1', stream: textStream, streamFormat: 'text', - subscribe: (nextSink: { onEvent: (event: unknown) => void | Promise }) => { + subscribe: (nextSink: AgentStreamSink) => { sink = nextSink return () => {} }, @@ -1254,7 +1298,7 @@ describe('createStreamingResponse agent-events-v1', () => { }, executeFn: async ({ onStream }) => { let textController!: ReadableStreamDefaultController - let sink: { onEvent: (event: unknown) => void | Promise } | undefined + let sink: AgentStreamSink | undefined const textStream = new ReadableStream({ start(controller) { textController = controller @@ -1262,9 +1306,10 @@ describe('createStreamingResponse agent-events-v1', () => { }) const onStreamPromise = onStream({ + blockId: 'agent-1', stream: textStream, streamFormat: 'text', - subscribe: (nextSink: { onEvent: (event: unknown) => void | Promise }) => { + subscribe: (nextSink: AgentStreamSink) => { sink = nextSink return () => {} }, @@ -1342,6 +1387,7 @@ describe('createStreamingResponse agent-events-v1', () => { }) const onStreamPromise = onStream({ + blockId: 'agent-1', stream: textStream, streamFormat: 'text', subscribe: (nextSink: { onEvent: (event: unknown) => void | Promise }) => { @@ -1498,7 +1544,7 @@ describe('createStreamingResponse agent-events-v1', () => { }, executeFn: async ({ onStream }) => { let textController!: ReadableStreamDefaultController - let sink: { onEvent: (event: unknown) => void | Promise } | undefined + let sink: AgentStreamSink | undefined const textStream = new ReadableStream({ start(controller) { textController = controller @@ -1506,9 +1552,10 @@ describe('createStreamingResponse agent-events-v1', () => { }) const onStreamPromise = onStream({ + blockId: 'agent-1', stream: textStream, streamFormat: 'text', - subscribe: (nextSink: any) => { + subscribe: (nextSink: AgentStreamSink) => { sink = nextSink return () => { sink = undefined diff --git a/apps/sim/lib/workflows/streaming/streaming.ts b/apps/sim/lib/workflows/streaming/streaming.ts index c330388dd0b..dda38e49dab 100644 --- a/apps/sim/lib/workflows/streaming/streaming.ts +++ b/apps/sim/lib/workflows/streaming/streaming.ts @@ -43,14 +43,6 @@ import { navigatePathAsync } from '@/executor/variables/resolvers/reference-asyn import type { ToolCallEndStatus } from '@/providers/stream-events' import { DEFAULT_MAX_THINKING_CHARS } from '@/providers/stream-pump' -/** - * Extended streaming execution type that includes blockId on the execution. - * The runtime passes blockId but the base StreamingExecution type doesn't declare it. - */ -interface StreamingExecutionWithBlockId extends Omit { - execution?: StreamingExecution['execution'] & { blockId?: string } -} - const logger = createLogger('WorkflowStreaming') const DANGEROUS_KEYS = ['__proto__', 'constructor', 'prototype'] @@ -88,7 +80,7 @@ interface StreamingConfig { export type StreamingExecutorFn = (callbacks: { onStream: (streamingExec: StreamingExecution) => Promise - onBlockComplete: (blockId: string, output: unknown) => Promise + onBlockComplete: (blockId: string, output: unknown, outputBlockId?: string) => Promise abortSignal: AbortSignal }) => Promise @@ -602,8 +594,8 @@ export async function createStreamingResponse( * Subscribe synchronously before the first await so the executor pump * can attach sinks before pulling provider chunks. */ - const onStreamCallback = async (streamingExec: StreamingExecutionWithBlockId) => { - const blockId = streamingExec.execution?.blockId + const onStreamCallback = async (streamingExec: StreamingExecution) => { + const blockId = streamingExec.blockId if (!blockId) { logger.warn(`[${requestId}] Streaming execution missing blockId`) return @@ -708,19 +700,24 @@ export async function createStreamingResponse( const includeFileBase64 = streamConfig.includeFileBase64 ?? true const base64MaxBytes = streamConfig.base64MaxBytes - const onBlockCompleteCallback = async (blockId: string, output: unknown) => { - state.completedBlockIds.add(blockId) + const onBlockCompleteCallback = async ( + blockId: string, + output: unknown, + outputBlockId?: string + ) => { + const selectedOutputBlockId = outputBlockId ?? blockId + state.completedBlockIds.add(selectedOutputBlockId) if (!streamConfig.selectedOutputs?.length) { return } - if (state.streamedChunks.has(blockId)) { + if (state.streamedChunks.has(selectedOutputBlockId)) { return } const matchingOutputs = getSelectedOutputDescriptors(streamConfig.selectedOutputs).filter( - (descriptor) => descriptor.blockId === blockId + (descriptor) => descriptor.blockId === selectedOutputBlockId ) /** @@ -792,14 +789,14 @@ export async function createStreamingResponse( getInlineJsonByteLength(hydratedOutput) ?? 0, Buffer.byteLength(formattedOutput, 'utf8') ) - sendChunk(blockId, formattedOutput, { + sendChunk(selectedOutputBlockId, formattedOutput, { selectedOutputKey: descriptor.key, selectedOutputBytes, }) } } catch (error) { logger.warn(`[${requestId}] Failed to materialize selected output`, { - blockId, + blockId: selectedOutputBlockId, outputId: descriptor.outputId, ...projectResolvedSecretDiagnosticError(error, undefined), }) @@ -807,7 +804,7 @@ export async function createStreamingResponse( state.selectedOutputError ??= errorMessage const frame: ChatStreamErrorFrame = { event: 'error', - blockId, + blockId: selectedOutputBlockId, error: errorMessage, } controller.enqueue(encodeSSE(frame)) diff --git a/apps/sim/lib/workflows/subblocks/context.test.ts b/apps/sim/lib/workflows/subblocks/context.test.ts index 554605d084d..ccf5762bb7c 100644 --- a/apps/sim/lib/workflows/subblocks/context.test.ts +++ b/apps/sim/lib/workflows/subblocks/context.test.ts @@ -5,9 +5,9 @@ import { afterAll, describe, expect, it, vi } from 'vitest' vi.unmock('@/blocks/registry') +import { isSelectorReady } from '@/lib/selectors/manifest' import * as blocksBarrel from '@/blocks' import { getAllBlocks, getBlock as getRealBlock } from '@/blocks/registry' -import { bitbucketSelectors } from '@/hooks/selectors/providers/bitbucket/selectors' import { buildSelectorContextFromBlock, getSelectorContextSubBlocks, @@ -190,6 +190,105 @@ describe('buildSelectorContextFromBlock', () => { ).toBeUndefined() }) + it('preserves exact environment references through the strict selector context path', () => { + const subBlocks = subBlocksFromValues({ + credential: '{{GMAIL_BASIC_CREDENTIAL}}', + manualCredential: '{{GMAIL_SHARED_CREDENTIAL_ID}}', + }) + + expect( + buildSelectorContextFromBlock('gmail', subBlocks, { + selectorKey: 'gmail.labels', + dependsOn: ['credential', 'manualCredential'], + }).oauthCredential + ).toBe('{{GMAIL_BASIC_CREDENTIAL}}') + expect( + buildSelectorContextFromBlock('gmail', subBlocks, { + selectorKey: 'gmail.labels', + dependsOn: ['credential', 'manualCredential'], + canonicalModes: { oauthCredential: 'advanced' }, + }).oauthCredential + ).toBe('{{GMAIL_SHARED_CREDENTIAL_ID}}') + }) + + it('includes Google impersonation as an explicit active selector hint', () => { + const context = buildSelectorContextFromBlock( + 'gmail', + subBlocksFromValues({ + credential: '{{GMAIL_CREDENTIAL_ID}}', + impersonateUserEmail: '{{GMAIL_IMPERSONATE_EMAIL}}', + }), + { + selectorKey: 'gmail.labels', + dependsOn: ['credential'], + } + ) + + expect(context).toEqual({ + oauthCredential: '{{GMAIL_CREDENTIAL_ID}}', + impersonateUserEmail: '{{GMAIL_IMPERSONATE_EMAIL}}', + }) + }) + + it('projects only the active Slack auth source plus trigger credentials', () => { + const oauthAction = buildSelectorContextFromBlock( + 'slack', + subBlocksFromValues({ + authMethod: 'oauth', + credential: 'active-oauth', + botToken: 'xoxb-dormant', + }), + { + selectorKey: 'slack.channels', + dependsOn: ['authMethod', 'credential', 'botToken'], + } + ) + expect(oauthAction.oauthCredential).toBe('active-oauth') + + const botAction = buildSelectorContextFromBlock( + 'slack', + subBlocksFromValues({ + authMethod: 'bot_token', + credential: 'dormant-oauth', + botToken: '{{SLACK_BOT_TOKEN}}', + }), + { + selectorKey: 'slack.channels', + dependsOn: ['authMethod', 'credential', 'botToken'], + } + ) + expect(botAction.oauthCredential).toBe('{{SLACK_BOT_TOKEN}}') + + const trigger = buildSelectorContextFromBlock( + 'slack_v2', + subBlocksFromValues({ + eventType: 'message', + customBotCredential: '{{SLACK_TRIGGER_CREDENTIAL}}', + }), + { + selectorKey: 'slack.channels', + dependsOn: ['customBotCredential'], + triggerMode: true, + } + ) + expect(trigger.oauthCredential).toBe('{{SLACK_TRIGGER_CREDENTIAL}}') + }) + + it('projects the optional Microsoft Excel drive cascade input', () => { + const excel = buildSelectorContextFromBlock( + 'microsoft_excel', + subBlocksFromValues({ + credential: 'excel-credential', + driveId: '{{SHAREPOINT_DRIVE_ID}}', + }), + { + selectorKey: 'microsoft.excel', + dependsOn: ['credential', 'driveId'], + } + ) + expect(excel.driveId).toBe('{{SHAREPOINT_DRIVE_ID}}') + }) + it('uses trigger credentials with and without canonical metadata after action conversion', () => { const clickupValues = { selectedTriggerId: 'clickup_task_created', @@ -221,6 +320,25 @@ describe('buildSelectorContextFromBlock', () => { ) }) + it('uses only active trigger dependencies in the strict selector context path', () => { + const context = buildSelectorContextFromBlock( + 'clickup', + subBlocksFromValues({ + selectedTriggerId: 'clickup_task_created', + credential: 'dormant-action', + triggerCredentials: '{{CLICKUP_SHARED_CREDENTIAL}}', + teamId: '', + }), + { + selectorKey: 'clickup.spaces', + dependsOn: ['triggerCredentials', 'teamId'], + triggerMode: true, + } + ) + + expect(context).toEqual({ oauthCredential: '{{CLICKUP_SHARED_CREDENTIAL}}' }) + }) + it('does not leak a dormant action credential when an unmapped trigger credential is blank', () => { const ctx = buildSelectorContextFromBlock( 'airtable', @@ -258,7 +376,7 @@ describe('buildSelectorContextFromBlock', () => { oauthCredential: 'credential-1', workspaceSlug: 'acme-platform', }) - expect(bitbucketSelectors['bitbucket.repositories'].enabled({ context } as never)).toBe(true) + expect(isSelectorReady('bitbucket.repositories', context)).toBe(true) }) it('should ignore subblock keys not in SELECTOR_CONTEXT_FIELDS', () => { @@ -274,6 +392,7 @@ describe('buildSelectorContextFromBlock', () => { describe('SELECTOR_CONTEXT_FIELDS validation', () => { it('every entry must be a canonicalParamId (if a canonical pair exists) or a direct subblock ID', () => { + const explicitSurfaceFields = new Set(['excludeWorkflowId']) const allCanonicalParamIds = new Set() const allSubBlockIds = new Set() const idsInCanonicalPairs = new Set() @@ -299,6 +418,7 @@ describe('SELECTOR_CONTEXT_FIELDS validation', () => { for (const field of SELECTOR_CONTEXT_FIELDS) { const f = field as string + if (explicitSurfaceFields.has(f)) continue if (allCanonicalParamIds.has(f)) continue if (idsInCanonicalPairs.has(f)) { diff --git a/apps/sim/lib/workflows/subblocks/context.ts b/apps/sim/lib/workflows/subblocks/context.ts index fe7ff8eeec4..ca14a0fb749 100644 --- a/apps/sim/lib/workflows/subblocks/context.ts +++ b/apps/sim/lib/workflows/subblocks/context.ts @@ -1,62 +1,22 @@ -import { getBlock } from '@/blocks' -import type { SubBlockConfig } from '@/blocks/types' -import { isReference } from '@/executor/constants' -import type { SelectorContext } from '@/hooks/selectors/types' -import type { SubBlockState } from '@/stores/workflows/workflow/types' +import { + buildSelectorRawContext, + getSelectorContextSubBlocks as getSharedSelectorContextSubBlocks, + SELECTOR_CONTEXT_FIELDS, +} from '@/lib/selectors/context' +import type { SelectorKey } from '@/lib/selectors/manifest' +import type { SelectorContext } from '@/lib/selectors/types' import { buildCanonicalIndex, buildSubBlockValues, type CanonicalModeOverrides, - evaluateSubBlockCondition, resolveActiveCanonicalValue, -} from './visibility' +} from '@/lib/workflows/subblocks/visibility' +import { getBlock } from '@/blocks' +import type { SubBlockConfig } from '@/blocks/types' +import { isReference } from '@/executor/constants' +import type { SubBlockState } from '@/stores/workflows/workflow/types' -/** - * Canonical param IDs (or raw subblock IDs) that correspond to SelectorContext fields. - * A subblock's resolved canonical key is set on the context only if it appears here. - */ -export const SELECTOR_CONTEXT_FIELDS = new Set([ - 'oauthCredential', - 'domain', - 'teamId', - 'projectId', - 'knowledgeBaseId', - 'planId', - 'siteId', - 'collectionId', - 'spreadsheetId', - 'driveId', - 'fileId', - 'baseId', - 'datasetId', - 'serviceDeskId', - 'impersonateUserEmail', - 'boardId', - 'spaceId', - 'listSpaceId', - 'folderId', - 'awsAccessKeyId', - 'awsSecretAccessKey', - 'awsRegion', - 'logGroupName', - 'tableId', - 'jobId', - 'orgId', - 'database', - 'schema', - 'workspaceSlug', - 'objectType', - 'customObjectTypeId', - 'pipelineId', - 'environmentType', - 'credentialGroupId', - 'language', - 'host', - 'port', - 'secure', - 'username', - 'password', -]) +export { SELECTOR_CONTEXT_FIELDS } /** * Selects the block fields allowed to contribute to selector context for the active mode. @@ -66,12 +26,7 @@ export function getSelectorContextSubBlocks( values: Record, triggerMode?: boolean ): SubBlockConfig[] { - if (!triggerMode) return subBlocks - return subBlocks.filter( - (subBlock) => - (subBlock.mode === 'trigger' || subBlock.mode === 'trigger-advanced') && - evaluateSubBlockCondition(subBlock.condition, values) - ) + return getSharedSelectorContextSubBlocks(subBlocks, values, triggerMode) } /** @@ -90,74 +45,65 @@ export function buildSelectorContextFromBlock( workspaceId?: string canonicalModes?: CanonicalModeOverrides triggerMode?: boolean + selectorKey?: SelectorKey + dependsOn?: readonly string[] + staticContext?: Readonly> } ): SelectorContext { - const context: SelectorContext = {} - if (opts?.workflowId) context.workflowId = opts.workflowId - if (opts?.workspaceId) context.workspaceId = opts.workspaceId + if (!opts?.selectorKey) { + const context: SelectorContext & { workflowId?: string; workspaceId?: string } = {} + if (opts?.workflowId) context.workflowId = opts.workflowId + if (opts?.workspaceId) context.workspaceId = opts.workspaceId - const blockConfig = getBlock(blockType) - if (!blockConfig) return context + const blockConfig = getBlock(blockType) + if (!blockConfig) return context + const values = buildSubBlockValues(subBlocks) + const configs = getSelectorContextSubBlocks(blockConfig.subBlocks, values, opts?.triggerMode) + const configById = new Map(configs.map((config) => [config.id, config])) + const canonicalIndex = buildCanonicalIndex(configs) + const resolvedGroups = new Set() - const values = buildSubBlockValues(subBlocks) - const contextConfigs = getSelectorContextSubBlocks( - blockConfig.subBlocks, - values, - opts?.triggerMode - ) - const canonicalIndex = buildCanonicalIndex(contextConfigs) - const contextSubBlockIds = opts?.triggerMode - ? new Set(contextConfigs.map((subBlock) => subBlock.id)) - : undefined - const resolvedGroups = new Set() - - const setField = (key: string, value: unknown) => { - if (value === null || value === undefined) return - const strValue = typeof value === 'string' ? value : String(value) - if (!strValue) return - // A `` reference only resolves at run time; handing the literal text to a - // selector would issue a request for a resource that cannot exist (mirrors useSelectorSetup). - if (isReference(strValue)) return - if (SELECTOR_CONTEXT_FIELDS.has(key as keyof SelectorContext)) { - context[key as keyof SelectorContext] = strValue + const setField = (field: string, value: unknown) => { + if (!SELECTOR_CONTEXT_FIELDS.has(field as keyof SelectorContext)) return + if (value === null || value === undefined) return + const normalized = typeof value === 'string' ? value : String(value) + if (!normalized || isReference(normalized)) return + context[field as keyof SelectorContext] = normalized } - } - for (const [subBlockId, subBlock] of Object.entries(subBlocks)) { - if (contextSubBlockIds && !contextSubBlockIds.has(subBlockId)) continue - const canonicalId = canonicalIndex.canonicalIdBySubBlockId[subBlockId] - if (canonicalId) { - // A canonical group resolves to its ACTIVE member only (no last-write-wins between a - // basic/advanced pair when both hold values), honoring an explicit mode override. + for (const [subBlockId, subBlock] of Object.entries(subBlocks)) { + if (!configById.has(subBlockId)) continue + const canonicalId = canonicalIndex.canonicalIdBySubBlockId[subBlockId] + if (!canonicalId) { + setField(subBlockId, subBlock.value) + continue + } if (resolvedGroups.has(canonicalId)) continue resolvedGroups.add(canonicalId) - const group = canonicalIndex.groupsById[canonicalId] - setField(canonicalId, resolveActiveCanonicalValue(group, values, opts?.canonicalModes)) - continue + setField( + canonicalId, + resolveActiveCanonicalValue( + canonicalIndex.groupsById[canonicalId], + values, + opts?.canonicalModes + ) + ) } - setField(subBlockId, subBlock?.value) - } - // A credential field IS the oauth credential, whatever the block calls its subblock. Most - // blocks say so with `canonicalParamId: 'oauthCredential'`, but that id is also the block's - // serialized param name — so requiring it would mean renaming a shipped block's param just - // to make its pickers resolvable, which is not a rename any picker should be able to force. - // Reading it off the subblock TYPE keeps the two decisions independent. - // - // Only fills a gap: a block that does declare the canonical id has already set it above, - // including the basic/advanced active-member resolution this loop cannot express. - if (!context.oauthCredential && !resolvedGroups.has('oauthCredential')) { - for (const [subBlockId, subBlock] of Object.entries(subBlocks)) { - if (contextConfigs.find((cfg) => cfg.id === subBlockId)?.type !== 'oauth-input') { - continue - } - const value = subBlock?.value - if (typeof value === 'string' && value) { - context.oauthCredential = value - break - } + if (!context.oauthCredential && !resolvedGroups.has('oauthCredential')) { + const credential = configs.find((config) => config.type === 'oauth-input') + if (credential) setField('oauthCredential', subBlocks[credential.id]?.value) } + return context } - return context + return buildSelectorRawContext({ + selectorKey: opts.selectorKey, + blockType, + subBlocks, + dependsOn: opts.dependsOn, + canonicalModes: opts.canonicalModes, + triggerMode: opts.triggerMode, + staticContext: opts.staticContext, + }) } diff --git a/apps/sim/lib/workflows/tool-input/synthetic-subblocks.ts b/apps/sim/lib/workflows/tool-input/synthetic-subblocks.ts index 59d948a36c6..194bbf93be0 100644 --- a/apps/sim/lib/workflows/tool-input/synthetic-subblocks.ts +++ b/apps/sim/lib/workflows/tool-input/synthetic-subblocks.ts @@ -1,3 +1,5 @@ +import { encodeToolParamValue } from '@/tools/param-shape' + const TOOL_SUBBLOCK_INFIX = '-tool-' const SYNTHETIC_TOOL_SUBBLOCK_RE = new RegExp(`${TOOL_SUBBLOCK_INFIX}\\d+-`) @@ -48,12 +50,7 @@ export function resolveToolParamSync( ): ToolParamSyncAction { if (storeValue === undefined) return { action: 'reproject' } - const stringified = - storeValue === null - ? '' - : typeof storeValue === 'string' - ? storeValue - : JSON.stringify(storeValue) + const stringified = encodeToolParamValue(storeValue) if (stringified === syncedValue) return { action: 'noop' } return { action: 'mirror', value: stringified } diff --git a/apps/sim/lib/workflows/types.ts b/apps/sim/lib/workflows/types.ts index 9e51d7ff1a7..28a68489132 100644 --- a/apps/sim/lib/workflows/types.ts +++ b/apps/sim/lib/workflows/types.ts @@ -12,6 +12,15 @@ export const USER_FILE_ACCESSIBLE_PROPERTIES = [ 'size', 'type', 'base64', + /** + * Path to the file on the sandbox filesystem, mounted on demand. + * + * The counterpart to `base64`: that one inlines the bytes and is JavaScript- + * only, while this one hands any language a real path to open — which is what + * a CLI or a library like pandas or ffmpeg actually needs. Referencing it runs + * the block in the remote sandbox, since the isolated VM has no filesystem. + */ + 'path', ] as const export type UserFileAccessibleProperty = (typeof USER_FILE_ACCESSIBLE_PROPERTIES)[number] @@ -23,6 +32,7 @@ export const USER_FILE_PROPERTY_TYPES: Record { - it('projects style and compiled-check failures without constructing responses', () => { + it('projects style failures without constructing responses', () => { expect( internalFileErrorPolicies.style.project(new StyleExtractionUnsupportedError('Unsupported')) ).toEqual({ status: 422, body: { error: 'Unsupported' }, headers: undefined }) - expect( - internalFileErrorPolicies.compiledCheck.project(new CompiledCheckUnsupportedError()) - ).toMatchObject({ status: 422 }) - expect( - internalFileErrorPolicies.compiledCheck.project(new CompiledCheckTooLargeError()) - ).toMatchObject({ status: 413 }) }) it('conceals forbidden inline resources with the legacy not-found envelope', () => { diff --git a/apps/sim/lib/workspace-files/api/internal-error-policies.ts b/apps/sim/lib/workspace-files/api/internal-error-policies.ts index 2e899c8e6b2..49a65b0b9b9 100644 --- a/apps/sim/lib/workspace-files/api/internal-error-policies.ts +++ b/apps/sim/lib/workspace-files/api/internal-error-policies.ts @@ -9,10 +9,6 @@ import { import { StorageLimitExceededError } from '@/lib/billing/storage' import { asOrchestrationError, statusForOrchestrationError } from '@/lib/core/orchestration/types' import { ArchiveError, statusForArchiveError } from '@/lib/uploads/archive' -import { - CompiledCheckTooLargeError, - CompiledCheckUnsupportedError, -} from '@/lib/workspace-files/application/compiled-check-workspace-file' import { StyleExtractionUnsupportedError } from '@/lib/workspace-files/application/style-workspace-file' const logger = createLogger('InternalWorkspaceFileErrors') @@ -22,16 +18,6 @@ const style = extendInternalErrorPolicy(internalOrchestrationErrorPolicy, (error return internalErrorResponse(422, { error: error.message }) }) -const compiledCheck = extendInternalErrorPolicy(internalOrchestrationErrorPolicy, (error) => { - if (error instanceof CompiledCheckUnsupportedError) { - return internalErrorResponse(422, { error: error.message }) - } - if (error instanceof CompiledCheckTooLargeError) { - return internalErrorResponse(413, { error: error.message }) - } - return null -}) - const content = extendInternalErrorPolicy(internalOrchestrationErrorPolicy, (error) => { if (!(error instanceof StorageLimitExceededError)) return null return internalErrorResponse(402, { error: error.message }) @@ -105,7 +91,6 @@ export const internalFileErrorPolicies = { notFoundMessage: FILE_NOT_FOUND_MESSAGE, }), style, - compiledCheck, downloadUrl, downloadArchive, extractArchive, diff --git a/apps/sim/lib/workspace-files/application/compiled-check-workspace-file.test.ts b/apps/sim/lib/workspace-files/application/compiled-check-workspace-file.test.ts deleted file mode 100644 index 27967b13cee..00000000000 --- a/apps/sim/lib/workspace-files/application/compiled-check-workspace-file.test.ts +++ /dev/null @@ -1,245 +0,0 @@ -/** - * @vitest-environment node - */ -import { beforeEach, describe, expect, it, vi } from 'vitest' - -const mocks = vi.hoisted(() => ({ - getE2BDocFormat: vi.fn(), - getFile: vi.fn(), - loadContext: vi.fn(), - resolvePermission: vi.fn(), - fetchBuffer: vi.fn(), - runE2BCompiledCheck: vi.fn(), - runSandboxTask: vi.fn(), - validateMermaidSource: vi.fn(), -})) - -vi.mock('@sim/platform-authz/workspace', () => ({ - permissionSatisfies: () => true, - resolveEffectiveWorkspacePermission: mocks.resolvePermission, -})) - -vi.mock('@/lib/copilot/tools/server/files/doc-compile', () => ({ - getE2BDocFormat: mocks.getE2BDocFormat, -})) - -vi.mock('@/lib/copilot/tools/server/files/doc-recalc', () => ({ - runE2BCompiledCheck: mocks.runE2BCompiledCheck, -})) - -vi.mock('@/lib/core/config/env-flags', () => ({ isDocSandboxEnabled: true })) - -vi.mock('@/lib/execution/constants', () => ({ - BINARY_DOC_TASKS: { pptx: 'document-pptx' }, - MAX_DOCUMENT_PREVIEW_CODE_BYTES: 1_000, -})) - -vi.mock('@/lib/execution/sandbox/run-task', () => ({ - runSandboxTask: mocks.runSandboxTask, - SandboxUserCodeError: class SandboxUserCodeError extends Error { - constructor(message: string, name: string) { - super(message) - this.name = name - } - }, -})) - -vi.mock('@/lib/mermaid/validate', () => ({ - validateMermaidSource: mocks.validateMermaidSource, -})) - -vi.mock('@/lib/uploads/contexts/workspace/workspace-file-manager', () => ({ - loadActiveWorkspaceFileContext: mocks.loadContext, -})) - -vi.mock('@/lib/uploads/contexts/workspace', () => ({ - fetchWorkspaceFileBuffer: mocks.fetchBuffer, - getWorkspaceFile: mocks.getFile, -})) - -import { SandboxUserCodeError } from '@/lib/execution/sandbox/run-task' -import { compiledCheckWorkspaceFile } from '@/lib/workspace-files/application/compiled-check-workspace-file' - -const sessionPrincipal = { - kind: 'session' as const, - userId: 'current-user', - sessionId: 'session-1', -} - -function mockFile(name: string) { - mocks.getFile.mockResolvedValue({ - id: 'file-1', - workspaceId: 'workspace-1', - name, - size: 20, - uploadedBy: 'original-uploader', - }) - mocks.fetchBuffer.mockResolvedValue(Buffer.from('source code')) -} - -describe('compiledCheckWorkspaceFile', () => { - beforeEach(() => { - vi.clearAllMocks() - mocks.getE2BDocFormat.mockResolvedValue(null) - mocks.resolvePermission.mockResolvedValue('admin') - mocks.loadContext.mockResolvedValue({ - fileId: 'file-1', - workspaceId: 'workspace-1', - workspaceOrganizationId: null, - allowPersonalApiKeys: false, - billedAccountUserId: 'billing-owner', - }) - mocks.runSandboxTask.mockResolvedValue(Buffer.from('compiled')) - }) - - it('rejects API-key principals before canonical loading or business execution', async () => { - const unsupportedPrincipals = [ - { kind: 'personal_api_key' as const, userId: 'user-1', keyId: 'personal-key' }, - { - kind: 'workspace_api_key' as const, - workspaceId: 'workspace-1', - keyId: 'workspace-key', - }, - ] - - /** - * A workspace key is refused with the code naming *why* — the operation - * denies workspace keys, so the remedy is a personal key — while any other - * disallowed kind gets the generic kind refusal. - */ - const expectedDetailCode = { - personal_api_key: 'PRINCIPAL_KIND_NOT_PERMITTED', - workspace_api_key: 'WORKSPACE_KEY_OPERATION_NOT_PERMITTED', - } as const - - for (const principal of unsupportedPrincipals) { - await expect( - compiledCheckWorkspaceFile.execute({ - principal, - input: { fileId: 'file-1', assertedWorkspaceId: 'workspace-1' }, - }) - ).rejects.toMatchObject({ - code: 'forbidden', - detailCode: expectedDetailCode[principal.kind], - }) - } - - expect(compiledCheckWorkspaceFile.operation).toMatchObject({ - id: 'files.compiled_check', - minimumRole: 'read', - workspaceApiKey: 'deny', - principalKinds: ['session'], - }) - expect(mocks.loadContext).not.toHaveBeenCalled() - expect(mocks.getFile).not.toHaveBeenCalled() - expect(mocks.fetchBuffer).not.toHaveBeenCalled() - }) - - it('uses the current session user as the legacy sandbox owner, never the uploader', async () => { - mockFile('report.pptx') - - await expect( - compiledCheckWorkspaceFile.execute({ - principal: sessionPrincipal, - input: { fileId: 'file-1', assertedWorkspaceId: 'workspace-1' }, - }) - ).resolves.toEqual({ ok: true }) - - expect(mocks.runSandboxTask).toHaveBeenCalledWith( - 'document-pptx', - { code: 'source code', workspaceId: 'workspace-1' }, - { ownerKey: 'user:current-user' } - ) - expect(mocks.runSandboxTask).not.toHaveBeenCalledWith(expect.anything(), expect.anything(), { - ownerKey: 'user:original-uploader', - }) - expect(mocks.loadContext).toHaveBeenCalledTimes(1) - expect(mocks.resolvePermission).toHaveBeenCalledTimes(1) - expect(mocks.getFile).toHaveBeenCalledTimes(1) - expect(mocks.fetchBuffer).toHaveBeenCalledTimes(1) - }) - - it('preserves legacy sandbox user-code failures in the successful response envelope', async () => { - mockFile('report.pptx') - mocks.runSandboxTask.mockRejectedValue( - new SandboxUserCodeError('Presentation source is invalid', 'SyntaxError') - ) - - await expect( - compiledCheckWorkspaceFile.execute({ - principal: sessionPrincipal, - input: { fileId: 'file-1', assertedWorkspaceId: 'workspace-1' }, - }) - ).resolves.toEqual({ - ok: false, - error: 'Presentation source is invalid', - errorName: 'SyntaxError', - }) - }) - - it('keeps Mermaid validation on the in-process path', async () => { - mockFile('diagram.mmd') - mocks.validateMermaidSource.mockResolvedValue({ - ok: false, - error: 'Unexpected token', - errorName: 'MermaidError', - }) - - await expect( - compiledCheckWorkspaceFile.execute({ - principal: sessionPrincipal, - input: { fileId: 'file-1', assertedWorkspaceId: 'workspace-1' }, - }) - ).resolves.toEqual({ - ok: false, - error: 'Unexpected token', - errorName: 'MermaidError', - }) - - expect(mocks.validateMermaidSource).toHaveBeenCalledWith('source code') - expect(mocks.runE2BCompiledCheck).not.toHaveBeenCalled() - expect(mocks.runSandboxTask).not.toHaveBeenCalled() - }) - - it('keeps E2B user-code failures in the successful response envelope', async () => { - mockFile('report.pptx') - mocks.getE2BDocFormat.mockResolvedValue({ ext: 'pptx' }) - mocks.runE2BCompiledCheck.mockResolvedValue({ ok: false, error: 'Script failed' }) - - await expect( - compiledCheckWorkspaceFile.execute({ - principal: sessionPrincipal, - input: { fileId: 'file-1', assertedWorkspaceId: 'workspace-1' }, - }) - ).resolves.toEqual({ - ok: false, - error: 'Script failed', - errorName: 'CompiledCheckError', - }) - - expect(mocks.runE2BCompiledCheck).toHaveBeenCalledWith({ - source: 'source code', - fileName: 'report.pptx', - workspaceId: 'workspace-1', - ext: 'pptx', - principal: sessionPrincipal, - }) - expect(mocks.runSandboxTask).not.toHaveBeenCalled() - }) - - it('propagates E2B infrastructure failures', async () => { - mockFile('report.pptx') - mocks.getE2BDocFormat.mockResolvedValue({ ext: 'pptx' }) - const failure = new Error('E2B unavailable') - mocks.runE2BCompiledCheck.mockRejectedValue(failure) - - await expect( - compiledCheckWorkspaceFile.execute({ - principal: sessionPrincipal, - input: { fileId: 'file-1', assertedWorkspaceId: 'workspace-1' }, - }) - ).rejects.toBe(failure) - - expect(mocks.runSandboxTask).not.toHaveBeenCalled() - }) -}) diff --git a/apps/sim/lib/workspace-files/application/compiled-check-workspace-file.ts b/apps/sim/lib/workspace-files/application/compiled-check-workspace-file.ts deleted file mode 100644 index f10d806d9a4..00000000000 --- a/apps/sim/lib/workspace-files/application/compiled-check-workspace-file.ts +++ /dev/null @@ -1,114 +0,0 @@ -import { getE2BDocFormat } from '@/lib/copilot/tools/server/files/doc-compile' -import { runE2BCompiledCheck } from '@/lib/copilot/tools/server/files/doc-recalc' -import type { AuthorizedWorkspaceUseCaseContext } from '@/lib/core/application' -import { isDocSandboxEnabled } from '@/lib/core/config/env-flags' -import { OrchestrationError } from '@/lib/core/orchestration/types' -import { BINARY_DOC_TASKS, MAX_DOCUMENT_PREVIEW_CODE_BYTES } from '@/lib/execution/constants' -import { runSandboxTask, SandboxUserCodeError } from '@/lib/execution/sandbox/run-task' -import { validateMermaidSource } from '@/lib/mermaid/validate' -import { fetchWorkspaceFileBuffer, getWorkspaceFile } from '@/lib/uploads/contexts/workspace' -import type { ActiveWorkspaceFileContext } from '@/lib/uploads/contexts/workspace/workspace-file-manager' -import { defineAuthorizedWorkspaceFileUseCase } from '@/lib/workspace-files/application/authorized-workspace-file-use-case' -import { fileOperations } from '@/lib/workspace-files/application/operations' -import { resolveActiveWorkspaceFileContext } from '@/lib/workspace-files/application/workspace-file-context' - -export class CompiledCheckUnsupportedError extends Error { - constructor() { - super('Compiled check only supports .docx, .pptx, .pdf, .xlsx, and .mmd files') - this.name = 'CompiledCheckUnsupportedError' - } -} - -export class CompiledCheckTooLargeError extends Error { - constructor() { - super('File source exceeds maximum size') - this.name = 'CompiledCheckTooLargeError' - } -} - -export interface CompiledCheckWorkspaceFileInput { - fileId: string - assertedWorkspaceId?: string -} - -export type CompiledCheckWorkspaceFileResult = - | { ok: true } - | { ok: false; error: string; errorName: string } - -function normalizeCompiledCheckResult(result: { - ok: boolean - error?: string - errorName?: string -}): CompiledCheckWorkspaceFileResult { - if (result.ok) return { ok: true } - return { - ok: false, - error: result.error ?? 'Compiled check failed', - errorName: result.errorName ?? 'CompiledCheckError', - } -} - -async function executeCompiledCheckWorkspaceFile({ - principal, - context, -}: AuthorizedWorkspaceUseCaseContext< - typeof fileOperations.compiledCheck, - CompiledCheckWorkspaceFileInput, - ActiveWorkspaceFileContext ->): Promise { - const file = await getWorkspaceFile(context.workspaceId, context.fileId, { - throwOnError: true, - }) - if (!file) throw new OrchestrationError('not_found', 'File not found') - const ext = file.name.split('.').pop()?.toLowerCase() ?? '' - const e2bFmt = isDocSandboxEnabled ? await getE2BDocFormat(file.name) : null - const taskId = BINARY_DOC_TASKS[ext] - const isMermaidFile = ext === 'mmd' || ext === 'mermaid' - if (!e2bFmt && !taskId && !isMermaidFile) throw new CompiledCheckUnsupportedError() - - if (file.size > MAX_DOCUMENT_PREVIEW_CODE_BYTES) { - throw new CompiledCheckTooLargeError() - } - - const content = await fetchWorkspaceFileBuffer(file, { - maxBytes: MAX_DOCUMENT_PREVIEW_CODE_BYTES, - }) - - const code = content.toString('utf-8') - if (Buffer.byteLength(code, 'utf-8') > MAX_DOCUMENT_PREVIEW_CODE_BYTES) { - throw new CompiledCheckTooLargeError() - } - if (isMermaidFile) return normalizeCompiledCheckResult(await validateMermaidSource(code)) - if (e2bFmt) { - return normalizeCompiledCheckResult( - await runE2BCompiledCheck({ - source: code, - fileName: file.name, - workspaceId: file.workspaceId, - ext, - principal, - }) - ) - } - - try { - if (!taskId) throw new CompiledCheckUnsupportedError() - await runSandboxTask( - taskId, - { code, workspaceId: file.workspaceId }, - { ownerKey: `user:${principal.userId}` } - ) - return { ok: true } - } catch (error) { - if (error instanceof SandboxUserCodeError) { - return { ok: false, error: error.message, errorName: error.name } - } - throw error - } -} - -export const compiledCheckWorkspaceFile = defineAuthorizedWorkspaceFileUseCase({ - operation: fileOperations.compiledCheck, - resolveContext: ({ input }) => resolveActiveWorkspaceFileContext(input), - execute: executeCompiledCheckWorkspaceFile, -}) diff --git a/apps/sim/lib/workspace-files/application/download-workspace-file-items.test.ts b/apps/sim/lib/workspace-files/application/download-workspace-file-items.test.ts index d858615398a..e8374108526 100644 --- a/apps/sim/lib/workspace-files/application/download-workspace-file-items.test.ts +++ b/apps/sim/lib/workspace-files/application/download-workspace-file-items.test.ts @@ -14,6 +14,7 @@ const { mockIsGenerated, mockIsRenderable, mockIsDocNotReady, + mockGetUserPermissionConfig, } = vi.hoisted(() => ({ events: [] as string[], mockLoadContext: vi.fn(), @@ -25,6 +26,15 @@ const { mockIsGenerated: vi.fn(), mockIsRenderable: vi.fn(), mockIsDocNotReady: vi.fn(), + mockGetUserPermissionConfig: vi.fn(), +})) + +vi.mock('@/lib/permission-groups/resolve.server', () => ({ + getUserPermissionConfig: mockGetUserPermissionConfig, + /** The use case passes the organization the authorized context already loaded. */ + resolveVerifiedUserAccessControlContext: async (userId: string, workspaceId: string) => ({ + config: await mockGetUserPermissionConfig(userId, workspaceId), + }), })) vi.mock('@/lib/uploads/contexts/workspace', () => ({ @@ -60,6 +70,7 @@ vi.mock('@sim/audit', () => ({ recordAudit: mockRecordAudit, })) +import { DEFAULT_PERMISSION_GROUP_CONFIG } from '@/lib/permission-groups/fields' import { downloadWorkspaceFileItems } from '@/lib/workspace-files/application/download-workspace-file-items' const principal = { kind: 'session' as const, userId: 'u1', sessionId: 's1' } @@ -105,6 +116,7 @@ describe('downloadWorkspaceFileItems', () => { mockIsGenerated.mockReturnValue(false) mockIsRenderable.mockReturnValue(false) mockIsDocNotReady.mockReturnValue(false) + mockGetUserPermissionConfig.mockResolvedValue(null) }) it('authorizes the workspace once and returns the bounded selection', async () => { @@ -264,4 +276,66 @@ describe('downloadWorkspaceFileItems', () => { expect(result.filesToZip.map((item) => item.id)).toEqual(['f1']) }) + + describe('permission-group capability', () => { + beforeEach(() => { + mockGetUserPermissionConfig.mockResolvedValue({ + ...DEFAULT_PERMISSION_GROUP_CONFIG, + disableBulkFileDownload: true, + }) + mockListFolders.mockResolvedValue([{ id: 'folder-1', parentId: null, name: 'Reports' }]) + mockListFiles.mockImplementation(async () => { + events.push('execute') + return [file('f1', 'clip.mp4'), file('f2', 'notes.txt', 'folder-1')] + }) + }) + + it('refuses a folder archive when the group withholds files.bulk_download', async () => { + await expect( + downloadWorkspaceFileItems.execute({ + principal, + input: { workspaceId: 'ws-1', fileIds: [], folderIds: ['folder-1'] }, + }) + ).rejects.toMatchObject({ capability: 'files.bulk_download' }) + + expect(events).not.toContain('execute') + }) + + it('refuses a multi-file archive, which is the same bulk extraction', async () => { + await expect( + downloadWorkspaceFileItems.execute({ + principal, + input: { workspaceId: 'ws-1', fileIds: ['f1', 'f2'], folderIds: [] }, + }) + ).rejects.toMatchObject({ capability: 'files.bulk_download' }) + }) + + /** + * A run carries the role of whoever triggered it but not their capabilities + * — `authorizeWorkspaceOperation` exempts a subject-bearing executor — and + * an assertion that read the subject straight off the principal re-applied + * here exactly what the funnel exempts. + */ + it('does not apply the capability to a delegated executor carrying a subject', async () => { + await expect( + downloadWorkspaceFileItems.execute({ + principal: { + ...delegatedPrincipal, + serviceId: 'executor' as const, + resourceScope: {}, + }, + input: { workspaceId: 'ws-1', fileIds: [], folderIds: ['folder-1'] }, + }) + ).resolves.toMatchObject({ filesToZip: [expect.objectContaining({ id: 'f2' })] }) + }) + + it('still allows downloading a single named file, which the key does not withhold', async () => { + await expect( + downloadWorkspaceFileItems.execute({ + principal, + input: { workspaceId: 'ws-1', fileIds: ['f1'], folderIds: [] }, + }) + ).resolves.toMatchObject({ filesToZip: [expect.objectContaining({ id: 'f1' })] }) + }) + }) }) diff --git a/apps/sim/lib/workspace-files/application/download-workspace-file-items.ts b/apps/sim/lib/workspace-files/application/download-workspace-file-items.ts index 69bedcc7bc7..011dabaf0c7 100644 --- a/apps/sim/lib/workspace-files/application/download-workspace-file-items.ts +++ b/apps/sim/lib/workspace-files/application/download-workspace-file-items.ts @@ -1,8 +1,12 @@ import { AuditAction, AuditResourceType } from '@sim/audit' -import type { AuthorizedWorkspaceUseCaseContext } from '@/lib/core/application' +import { + type AuthorizedWorkspaceUseCaseContext, + capabilityGovernedPrincipalUserId, +} from '@/lib/core/application' import { OrchestrationError } from '@/lib/core/orchestration/types' import { PayloadSizeLimitError } from '@/lib/core/utils/stream-limits' import { parseFolderPath } from '@/lib/folders/paths' +import { assertWorkspaceCapability } from '@/lib/permission-groups/capability-assertions' import { buildWorkspaceFileFolderPathMap, listWorkspaceFileFolders, @@ -118,6 +122,31 @@ async function executeDownloadWorkspaceFileItems({ validationError('No files selected for download') } + /** + * permission-group-enforced: files.bulk_download — one operation serves both + * a single file and a whole folder tree, and only the archive is what the key + * withholds; declaring the capability on `files.download` would take away + * saving one file too. `context.fileId` is the same single-file predicate the + * resource authorization already resolved, reused so the two cannot drift. + * Asserted against whoever the funnel would have judged, from its own rule — + * nobody, for a workspace key or an executor run. A run carries the role of + * whoever triggered it but not their capabilities, and reading the subject + * straight off the principal would have re-applied here exactly the + * capability `authorizeWorkspaceOperation` exempts a subject-bearing executor + * from. + */ + if (context.fileId === undefined) { + const actingUserId = capabilityGovernedPrincipalUserId(principal) + if (actingUserId) { + await assertWorkspaceCapability( + actingUserId, + context.workspaceId, + 'files.bulk_download', + context.workspaceOrganizationId + ) + } + } + const [files, folders] = await Promise.all([ listWorkspaceFiles(context.workspaceId, { hydrateFolderPaths: false, throwOnError: true }), listWorkspaceFileFolders(context.workspaceId), diff --git a/apps/sim/lib/workspace-files/application/operations.test.ts b/apps/sim/lib/workspace-files/application/operations.test.ts index d07f3edebfe..dd9135a101f 100644 --- a/apps/sim/lib/workspace-files/application/operations.test.ts +++ b/apps/sim/lib/workspace-files/application/operations.test.ts @@ -44,6 +44,7 @@ describe('file operation registry', () => { expect(executorOperationIds).toEqual([ 'files.read_metadata', 'files.read_content', + 'files.search_content', 'files.download', 'files.create', 'files.update_content', diff --git a/apps/sim/lib/workspace-files/application/operations.ts b/apps/sim/lib/workspace-files/application/operations.ts index 2752a576521..09b480c84fb 100644 --- a/apps/sim/lib/workspace-files/application/operations.ts +++ b/apps/sim/lib/workspace-files/application/operations.ts @@ -21,42 +21,56 @@ export const fileOperations = { id: 'files.list', minimumRole: 'read', workspaceApiKey: 'allow', + capability: 'files.use', ...ALL_COPILOT_PRINCIPAL_POLICY, }), readMetadata: defineWorkspaceOperation({ id: 'files.read_metadata', minimumRole: 'read', workspaceApiKey: 'allow', + capability: 'files.use', ...ALL_FILE_TOOL_PRINCIPAL_POLICY, }), readContent: defineWorkspaceOperation({ id: 'files.read_content', minimumRole: 'read', workspaceApiKey: 'allow', + capability: 'files.use', + ...ALL_FILE_TOOL_PRINCIPAL_POLICY, + }), + searchContent: defineWorkspaceOperation({ + id: 'files.search_content', + minimumRole: 'read', + workspaceApiKey: 'allow', + capability: 'files.use', ...ALL_FILE_TOOL_PRINCIPAL_POLICY, }), download: defineWorkspaceOperation({ id: 'files.download', minimumRole: 'read', workspaceApiKey: 'allow', + capability: 'files.use', ...ALL_FILE_TOOL_PRINCIPAL_POLICY, }), compiledCheck: defineWorkspaceOperation({ id: 'files.compiled_check', minimumRole: 'read', workspaceApiKey: 'deny', + capability: 'files.use', principalKinds: ['session'], }), create: defineWorkspaceOperation({ id: 'files.create', minimumRole: 'write', workspaceApiKey: 'allow', + capability: 'files.use', ...ALL_FILE_TOOL_PRINCIPAL_POLICY, }), rename: defineWorkspaceOperation({ id: 'files.rename', minimumRole: 'write', workspaceApiKey: 'allow', + capability: 'files.use', ...ALL_COPILOT_PRINCIPAL_POLICY, }), /** @@ -75,30 +89,35 @@ export const fileOperations = { id: 'files.extract_archive', minimumRole: 'write', workspaceApiKey: 'allow', + capability: 'files.use', principalKinds: ['session', 'personal_api_key', 'workspace_api_key'], }), updateContent: defineWorkspaceOperation({ id: 'files.update_content', minimumRole: 'write', workspaceApiKey: 'allow', + capability: 'files.use', ...ALL_FILE_TOOL_PRINCIPAL_POLICY, }), updateMetadata: defineWorkspaceOperation({ id: 'files.update_metadata', minimumRole: 'write', workspaceApiKey: 'allow', + capability: 'files.use', ...ALL_COPILOT_PRINCIPAL_POLICY, }), move: defineWorkspaceOperation({ id: 'files.move', minimumRole: 'write', workspaceApiKey: 'allow', + capability: 'files.use', ...ALL_FILE_TOOL_PRINCIPAL_POLICY, }), createVfsFolders: defineWorkspaceOperation({ id: 'files.vfs.folders.create', minimumRole: 'write', workspaceApiKey: 'deny', + capability: 'files.use', principalKinds: ['delegated'], delegatedServices: ['copilot'], }), @@ -106,6 +125,7 @@ export const fileOperations = { id: 'files.vfs.relocate', minimumRole: 'write', workspaceApiKey: 'deny', + capability: 'files.use', principalKinds: ['delegated'], delegatedServices: ['copilot'], }), @@ -113,6 +133,7 @@ export const fileOperations = { id: 'files.vfs.delete', minimumRole: 'write', workspaceApiKey: 'deny', + capability: 'files.use', principalKinds: ['delegated'], delegatedServices: ['copilot'], }), @@ -120,60 +141,70 @@ export const fileOperations = { id: 'files.delete', minimumRole: 'write', workspaceApiKey: 'allow', + capability: 'files.use', ...ALL_COPILOT_PRINCIPAL_POLICY, }), restore: defineWorkspaceOperation({ id: 'files.restore', minimumRole: 'write', workspaceApiKey: 'allow', + capability: 'files.use', ...ALL_COPILOT_PRINCIPAL_POLICY, }), readShare: defineWorkspaceOperation({ id: 'files.share.read', minimumRole: 'read', workspaceApiKey: 'allow', + capability: 'files.use', ...ALL_FILE_TOOL_PRINCIPAL_POLICY, }), updateShare: defineWorkspaceOperation({ id: 'files.share.update', minimumRole: 'write', workspaceApiKey: 'deny', + capability: 'files.use', ...HUMAN_FILE_TOOL_PRINCIPAL_POLICY, }), listFolders: defineWorkspaceOperation({ id: 'files.folders.list', minimumRole: 'read', workspaceApiKey: 'allow', + capability: 'files.use', ...ALL_COPILOT_PRINCIPAL_POLICY, }), createFolder: defineWorkspaceOperation({ id: 'files.folders.create', minimumRole: 'write', workspaceApiKey: 'allow', + capability: 'files.use', ...ALL_FILE_TOOL_PRINCIPAL_POLICY, }), updateFolder: defineWorkspaceOperation({ id: 'files.folders.update', minimumRole: 'write', workspaceApiKey: 'allow', + capability: 'files.use', ...ALL_COPILOT_PRINCIPAL_POLICY, }), deleteFolder: defineWorkspaceOperation({ id: 'files.folders.delete', minimumRole: 'write', workspaceApiKey: 'allow', + capability: 'files.use', ...ALL_COPILOT_PRINCIPAL_POLICY, }), restoreFolder: defineWorkspaceOperation({ id: 'files.folders.restore', minimumRole: 'write', workspaceApiKey: 'allow', + capability: 'files.use', ...ALL_COPILOT_PRINCIPAL_POLICY, }), uploadCreate: defineWorkspaceOperation({ id: 'files.upload.create', minimumRole: 'write', workspaceApiKey: 'allow', + capability: 'files.use', ...UPLOAD_PRINCIPAL_POLICY, }), /** @@ -186,24 +217,28 @@ export const fileOperations = { id: 'files.upload.read', minimumRole: 'read', workspaceApiKey: 'allow', + capability: 'files.use', ...UPLOAD_PRINCIPAL_POLICY, }), uploadParts: defineWorkspaceOperation({ id: 'files.upload.parts', minimumRole: 'write', workspaceApiKey: 'allow', + capability: 'files.use', ...UPLOAD_PRINCIPAL_POLICY, }), uploadComplete: defineWorkspaceOperation({ id: 'files.upload.complete', minimumRole: 'write', workspaceApiKey: 'allow', + capability: 'files.use', ...UPLOAD_PRINCIPAL_POLICY, }), uploadCancel: defineWorkspaceOperation({ id: 'files.upload.cancel', minimumRole: 'write', workspaceApiKey: 'allow', + capability: 'files.use', ...UPLOAD_PRINCIPAL_POLICY, }), } as const diff --git a/apps/sim/lib/workspace-files/application/read-workspace-file-secret-provenance.ts b/apps/sim/lib/workspace-files/application/read-workspace-file-secret-provenance.ts index 45075640fa7..27c8a695108 100644 --- a/apps/sim/lib/workspace-files/application/read-workspace-file-secret-provenance.ts +++ b/apps/sim/lib/workspace-files/application/read-workspace-file-secret-provenance.ts @@ -11,6 +11,8 @@ import { resolveActiveWorkspaceFileContext } from '@/lib/workspace-files/applica export interface ReadWorkspaceFileSecretProvenanceInput { fileId: string assertedWorkspaceId?: string + /** Fails closed when the caller's derived content no longer matches the canonical file revision. */ + expectedContentUpdatedAt?: Date } export const readWorkspaceFileSecretProvenance = defineAuthorizedWorkspaceFileUseCase({ @@ -28,6 +30,7 @@ export const readWorkspaceFileSecretProvenance = defineAuthorizedWorkspaceFileUs fileId: file.id, key: file.key, context: 'workspace', + contentUpdatedAt: input.expectedContentUpdatedAt, }), ownerUserId: file.uploadedBy, } diff --git a/apps/sim/lib/workspace-files/application/resolve-workspace-file-reference.test.ts b/apps/sim/lib/workspace-files/application/resolve-workspace-file-reference.test.ts index cd69a123118..2601f6f55d3 100644 --- a/apps/sim/lib/workspace-files/application/resolve-workspace-file-reference.test.ts +++ b/apps/sim/lib/workspace-files/application/resolve-workspace-file-reference.test.ts @@ -90,6 +90,7 @@ describe('workspace file reference application service', () => { minimumRole: 'write', workspaceApiKey: 'deny', principalKinds: ['session'], + capability: 'files.use', }) await expect( diff --git a/apps/sim/lib/workspace-files/application/search-workspace-file-content.ts b/apps/sim/lib/workspace-files/application/search-workspace-file-content.ts new file mode 100644 index 00000000000..bc80e2db3be --- /dev/null +++ b/apps/sim/lib/workspace-files/application/search-workspace-file-content.ts @@ -0,0 +1,59 @@ +import { OrchestrationError } from '@/lib/core/orchestration/types' +import { loadActiveWorkspaceContext } from '@/lib/uploads/contexts/workspace' +import { defineAuthorizedWorkspaceFileUseCase } from '@/lib/workspace-files/application/authorized-workspace-file-use-case' +import { fileOperations } from '@/lib/workspace-files/application/operations' +import { + compileFileSearchPattern, + type FileSearchMode, + FileSearchPatternError, +} from '@/lib/workspace-files/search/pattern' +import { + searchWorkspaceFileIndex, + WorkspaceFileSearchUnavailableError, +} from '@/lib/workspace-files/search/repository' + +export interface SearchWorkspaceFileContentInput { + workspaceId: string + query: string + mode: FileSearchMode + maxResults: number + signal?: AbortSignal +} + +async function resolveSearchWorkspaceFileContext(input: SearchWorkspaceFileContentInput) { + input.signal?.throwIfAborted() + const workspace = await loadActiveWorkspaceContext(input.workspaceId) + input.signal?.throwIfAborted() + if (!workspace) throw new OrchestrationError('not_found', 'Workspace not found') + return workspace +} + +export const searchWorkspaceFileContent = defineAuthorizedWorkspaceFileUseCase({ + operation: fileOperations.searchContent, + resolveContext: ({ input }: { input: SearchWorkspaceFileContentInput }) => + resolveSearchWorkspaceFileContext(input), + execute: async ({ input, context }) => { + try { + return await searchWorkspaceFileIndex({ + workspaceId: context.workspaceId, + pattern: compileFileSearchPattern(input.query, input.mode), + maxResults: input.maxResults, + signal: input.signal, + }) + } catch (error) { + /** + * A rejected or too-expensive pattern is the caller's to fix, and the + * message names the construct and the supported alternative — so it is + * classified rather than left to become the surface's generic failure text. + */ + if (error instanceof FileSearchPatternError) { + throw new OrchestrationError('validation', error.message) + } + /** Nothing is wrong with the query, so the caller is told to retry, not to rewrite it. */ + if (error instanceof WorkspaceFileSearchUnavailableError) { + throw new OrchestrationError('locked', error.message) + } + throw error + } + }, +}) diff --git a/apps/sim/lib/workspace-files/application/share-workspace-file.ts b/apps/sim/lib/workspace-files/application/share-workspace-file.ts index b1584f0f977..b854f42b43a 100644 --- a/apps/sim/lib/workspace-files/application/share-workspace-file.ts +++ b/apps/sim/lib/workspace-files/application/share-workspace-file.ts @@ -2,7 +2,6 @@ import { AuditAction, AuditResourceType } from '@sim/audit' import { resolvePrincipalExecutionActorUserId } from '@sim/auth/principal' import { createLogger } from '@sim/logger' import type { ShareAuthType, ShareRecord } from '@/lib/api/contracts/public-shares' -import { ForbiddenOperationError } from '@/lib/core/application/forbidden' import { OrchestrationError } from '@/lib/core/orchestration/types' import { getShareForResource, @@ -13,10 +12,7 @@ import { getWorkspaceFile } from '@/lib/uploads/contexts/workspace/workspace-fil import { defineAuthorizedWorkspaceFileUseCase } from '@/lib/workspace-files/application/authorized-workspace-file-use-case' import { fileOperations } from '@/lib/workspace-files/application/operations' import { resolveActiveWorkspaceFileContext } from '@/lib/workspace-files/application/workspace-file-context' -import { - PublicFileSharingNotAllowedError, - validatePublicFileSharing, -} from '@/ee/access-control/utils/permission-check' +import { validatePublicFileSharing } from '@/ee/access-control/utils/permission-check' const logger = createLogger('WorkspaceFileShare') @@ -87,13 +83,7 @@ export const updateWorkspaceFileShare = defineAuthorizedWorkspaceFileUseCase({ if (input.isActive) { const effectiveAuthType = input.authType ?? existingShare?.authType ?? 'public' - try { - await validatePublicFileSharing(userId, context.workspaceId, effectiveAuthType) - } catch (error) { - if (error instanceof PublicFileSharingNotAllowedError) - throw new ForbiddenOperationError('PUBLIC_SHARING_NOT_ALLOWED', error.message) - throw error - } + await validatePublicFileSharing(userId, context.workspaceId, effectiveAuthType) } let share: ShareRecord diff --git a/apps/sim/lib/workspace-files/page-compile-limits.test.ts b/apps/sim/lib/workspace-files/page-compile-limits.test.ts new file mode 100644 index 00000000000..7afcf241adc --- /dev/null +++ b/apps/sim/lib/workspace-files/page-compile-limits.test.ts @@ -0,0 +1,135 @@ +/** + * @vitest-environment node + */ +import { describe, expect, it } from 'vitest' +import { + collectSimPageDiagnostics, + compileSimPage, + isSimPageSource, +} from '@/lib/workspace-files/page-compile' + +/** + * A `sim:table` payload whose cell count is the PRODUCT of two alias lists while + * its source length is their SUM: `columns` is anchored once and every row is an + * alias to it, so `n` rows over `n` columns cost ~13n source bytes and render + * n² cells. This is the shape that makes the fence renderers amplify — a deeply + * nested alias chain does not, because every payload schema is at most + * `array of array of scalar` and rejects depth 3 without descending. + */ +function aliasedTable(n: number): string { + const columns = Array.from({ length: n }, () => ' - x').join('\n') + const rows = Array.from({ length: n }, () => ' - *c').join('\n') + return `columns: &c\n${columns}\nrows:\n${rows}\n` +} + +/** A `sim:steps` payload that aliases one large markdown body `n` times. */ +function aliasedSteps(n: number, markdownBytes: number): string { + const markdown = 'lorem ipsum dolor sit amet '.repeat(Math.ceil(markdownBytes / 27)) + const repeats = Array.from({ length: n - 1 }, () => '- *s').join('\n') + return `- &s\n title: T\n markdown: "${markdown.slice(0, markdownBytes)}"\n${repeats}\n` +} + +function page(kind: string, payload: string): string { + return `---\ntitle: T\n---\n\`\`\`sim:${kind}\n${payload}\`\`\`\n` +} + +describe('page compile YAML expansion limits', () => { + it('renders a table whose expanded size is within the budget', () => { + const html = compileSimPage(page('table', aliasedTable(40))) + expect(html).toContain('') + expect(collectSimPageDiagnostics(page('table', aliasedTable(40)))).toEqual([]) + }) + + it('skips a table whose aliases expand past the budget', () => { + const source = page('table', aliasedTable(400)) + const html = compileSimPage(source) + + expect(html).not.toContain('
') + expect(collectSimPageDiagnostics(source)).toEqual([ + expect.stringContaining('sim:table block starting "columns: &c" skipped:'), + ]) + expect(collectSimPageDiagnostics(source)[0]).toContain('too large to render') + }) + + it('bounds the compile cost of an alias bomb that would otherwise be quadratic', () => { + // 2000 x 2000 renders 4M cells through marked.parseInline — seconds of CPU + // and tens of MB of HTML per request, from 25 KB of source. + const source = page('table', aliasedTable(2000)) + + const started = performance.now() + const html = compileSimPage(source) + const elapsed = performance.now() - started + + expect(html).not.toContain('') + expect(html).toContain('
') + expect(html.length).toBeLessThan(64 * 1024) + expect(elapsed).toBeLessThan(1000) + }) + + it('charges aliased strings by their expanded bytes, not their node count', () => { + // Only ~2000 nodes, but 2000 x 4 KB of markdown reaches the renderer. + const source = page('steps', aliasedSteps(2000, 4096)) + const html = compileSimPage(source) + + expect(html).not.toContain('
    ') + expect(collectSimPageDiagnostics(source)[0]).toContain('maximum serialized size') + }) + + it('shares one budget across every block, so splitting buys no extra rendering', () => { + const oneBlock = page('table', aliasedTable(150)) + expect(collectSimPageDiagnostics(oneBlock)).toEqual([]) + + const manyBlocks = `---\ntitle: T\n---\n${Array.from( + { length: 6 }, + () => `\`\`\`sim:table\n${aliasedTable(150)}\`\`\`\n` + ).join('\n')}` + const diagnostics = collectSimPageDiagnostics(manyBlocks) + + expect(diagnostics.length).toBeGreaterThan(0) + expect(diagnostics.length).toBeLessThan(6) + expect(diagnostics.at(-1)).toContain('spent its whole structured-block budget') + }) + + it('refuses to recognize page source whose frontmatter expands past the budget', () => { + const nav = Array.from({ length: 400 }, () => ' - *g').join('\n') + const pages = Array.from({ length: 400 }, () => ' - "[A](sim:file/a)"').join('\n') + const source = `---\ntitle: T\nnav:\n - &g\n pages:\n${pages}\n${nav}\n---\nBody.\n` + + expect(isSimPageSource(source)).toBe(false) + }) + + it('leaves an ordinary page and its diagnostics untouched', () => { + const source = [ + '---', + 'title: Report', + '---', + 'Intro prose.', + '```sim:table', + 'columns: [Name, Count:num]', + 'rows:', + ' - [alpha, 1]', + ' - [beta, 2]', + '```', + '```sim:kv', + '- key: Owner', + ' value: Ops', + '```', + '```sim:table', + 'columns: nope', + '```', + ].join('\n') + + const html = compileSimPage(source) + expect(html).toContain('
alpha
Columns` helpers // must keep naming every doomed column away, including ones deprecated later. const skipFiles = new Set([fileURLToPath(import.meta.url)]) - const namePattern = new RegExp(`\\b(${[...pendingTables.keys()].join('|')}|alias)\\b`) + const pendingTableNames = new Set(pendingTables.keys()) const violations: Violation[] = [] for (const file of SCAN_DIRS.flatMap((dir) => collectSources(dir))) { if (skipFiles.has(file) || /\.test\.(ts|tsx|mts|cts)$/.test(file)) continue const source = readFileSync(file, 'utf8') - if (!namePattern.test(source)) continue + if (file !== SCHEMA_PATH && !mayReferencePendingTable(source, pendingTableNames)) continue violations.push(...auditFile(file, source, pendingTables)) } @@ -568,4 +591,4 @@ function main(): void { process.exit(1) } -main() +if (import.meta.main) main() diff --git a/scripts/check-permission-group-enforcement.test.ts b/scripts/check-permission-group-enforcement.test.ts new file mode 100644 index 00000000000..f1da09f02fa --- /dev/null +++ b/scripts/check-permission-group-enforcement.test.ts @@ -0,0 +1,329 @@ +import { describe, expect, it } from 'vitest' +import { + parseCapabilityIds, + parseFieldEnforcement, + parseOperationCapabilities, + parseOperationRegistryMembers, +} from './check-permission-group-enforcement' + +describe('operation capability parsing', () => { + it('reads a direct declaration', () => { + const { declarations, unreadable } = parseOperationCapabilities(` + export const tableOperations = { + create: defineWorkspaceOperation({ + id: 'tables.create', + minimumRole: 'write', + capability: 'tables.create', + }), + } as const + `) + + expect(declarations).toEqual([ + expect.objectContaining({ id: 'tables.create', capability: 'tables.create' }), + ]) + expect(unreadable).toEqual([]) + }) + + it('resolves call sites of a function factory without reporting the factory itself', () => { + const { declarations, unreadable } = parseOperationCapabilities(` + function tableOperation(id: string, capability: string) { + return defineWorkspaceOperation({ id, minimumRole: 'write', capability }) + } + + export const listRows = tableOperation('tables.rows.list', 'tables.use') + export const readRow = tableOperation('tables.rows.read', 'tables.use') + `) + + expect(declarations.map((declaration) => declaration.id)).toEqual([ + 'tables.rows.list', + 'tables.rows.read', + ]) + expect(declarations.every((declaration) => declaration.capability === 'tables.use')).toBe(true) + expect(unreadable).toEqual([]) + }) + + /** + * The two silent-drop forms. Each used to vanish from the count with the audit + * still printing a tick; both are now findings. + */ + it('reports a declaration whose id is a const reference', () => { + const { declarations, unreadable } = parseOperationCapabilities(` + const TABLE_CREATE_ID = 'tables.create' + + export const create = defineWorkspaceOperation({ + id: TABLE_CREATE_ID, + minimumRole: 'write', + capability: 'tables.create', + }) + `) + + expect(declarations).toEqual([]) + expect(unreadable).toHaveLength(1) + }) + + it('reports a wrapper written as an arrow const rather than a function', () => { + const { declarations, unreadable } = parseOperationCapabilities(` + const tableOperation = (id: string, capability: string) => + defineWorkspaceOperation({ id, minimumRole: 'write', capability }) + + export const listRows = tableOperation('tables.rows.list', 'tables.use') + `) + + expect(declarations).toEqual([]) + expect(unreadable).toHaveLength(1) + }) +}) + +describe('registry parsing', () => { + it('reads capability ids in declaration order', () => { + expect( + parseCapabilityIds(` + export const CAPABILITY_IDS = ['tables.use', 'files.use'] as const + `) + ).toEqual(['tables.use', 'files.use']) + }) + + /** Keys are matched at the registry's own two-space indentation. */ + it('reads each config key declared enforcement', () => { + const enforcement = parseFieldEnforcement( + [ + 'export const PERMISSION_GROUP_FIELDS = {', + " allowedIntegrations: allowlist(z.string(), 'executor', {", + " limited: 'x',", + " empty: 'y',", + ' }),', + " hideTablesTab: booleanRestriction('capability', {", + " id: 'hide-tables',", + " hint: 'Hide the Tables module from the sidebar.',", + ' }),', + '} satisfies Record', + ].join('\n') + ) + + expect(enforcement.get('allowedIntegrations')).toBe('executor') + expect(enforcement.get('hideTablesTab')).toBe('capability') + }) +}) + +/** + * The blind spot that shipped five ungated OAuth-connection operations: a domain + * that mints operations through a builder of its own, never calling + * `defineWorkspaceOperation`, so nothing read what it declared and the audit + * still printed a tick. Both halves of the fix are pinned here — the parsers now + * follow the `define*Operation` family, and the registry check names any member + * they still could not read. + */ +describe('operation builders other than defineWorkspaceOperation', () => { + it('reads a domain builder that takes an id and a capability positionally', () => { + const { declarations, unreadable } = parseOperationCapabilities(` + function defineCredentialUserOperation(id: string, capability: string) { + return Object.freeze({ id, capability, principalKinds: ['session'] }) + } + + export const credentialUserOperations = { + listOAuthConnections: defineCredentialUserOperation( + 'credentials.oauth_connections.list', + 'integrations.manage' + ), + disconnectOAuth: defineCredentialUserOperation( + 'credentials.oauth_connections.disconnect', + 'integrations.manage' + ), + } as const + `) + + expect(declarations).toEqual([ + expect.objectContaining({ + id: 'credentials.oauth_connections.list', + capability: 'integrations.manage', + }), + expect.objectContaining({ + id: 'credentials.oauth_connections.disconnect', + capability: 'integrations.manage', + }), + ]) + expect(unreadable).toEqual([]) + }) + + it('reads a domain builder that passes an object literal straight through', () => { + const { declarations, unreadable } = parseOperationCapabilities(` + export const auditLogOperations = { + list: defineAuditLogOperation({ + id: 'audit_logs.list', + capability: 'none', + }), + } as const + `) + + expect(declarations).toEqual([ + expect.objectContaining({ id: 'audit_logs.list', capability: 'none' }), + ]) + expect(unreadable).toEqual([]) + }) + + /** + * The wrapper form. Counting the outer and the inner call separately would + * double every credential operation, so the nested match is skipped — its id + * and capability are already carried by the outer call's text. + */ + it('counts a wrapped operation once', () => { + const { declarations } = parseOperationCapabilities(` + export const credentialOperations = { + read: defineCredentialOperation( + defineWorkspaceOperation({ + id: 'credentials.read', + minimumRole: 'read', + capability: 'integrations.manage', + }), + 'member' + ), + } as const + `) + + expect(declarations).toEqual([ + expect.objectContaining({ id: 'credentials.read', capability: 'integrations.manage' }), + ]) + }) + + it('reports a builder that mints the operation itself from a bare id argument', () => { + const { declarations, unreadable } = parseOperationCapabilities(` + function defineCredentialUserOperation(id: string) { + return Object.freeze({ id, principalKinds: ['session'] }) + } + + export const credentialUserOperations = { + listOAuthConnections: defineCredentialUserOperation('credentials.oauth_connections.list'), + } as const + `) + + expect(declarations).toEqual([]) + expect(unreadable).toHaveLength(1) + }) +}) + +describe('registry completeness', () => { + const registrySource = ` + export const probeOperations = { + list: Object.freeze({ id: 'probe.list' as const }), + read: defineWorkspaceOperation({ + id: 'probe.read', + minimumRole: 'read', + capability: 'tables.use', + }), + } as const + ` + + it('enumerates each member with the line span it occupies', () => { + const members = parseOperationRegistryMembers(registrySource) + + expect(members.map((member) => `${member.registry}.${member.member}`)).toEqual([ + 'probeOperations.list', + 'probeOperations.read', + ]) + const [list, read] = members + expect(list.startLine).toBe(3) + expect(read.startLine).toBe(4) + expect(read.endLine).toBe(8) + }) + + /** + * The check the audit runs: a member no parsed declaration falls inside is a + * member nothing read. `list` is minted by no builder at all, so it yields + * nothing — and yielding nothing is what used to read as success. + */ + it('leaves a member no parser read outside every parsed line', () => { + const { declarations, unreadable } = parseOperationCapabilities(registrySource) + const readLines = [...declarations.map((declaration) => declaration.line), ...unreadable] + + const unread = parseOperationRegistryMembers(registrySource).filter( + (member) => !readLines.some((line) => line >= member.startLine && line <= member.endLine) + ) + + expect(unread.map((member) => member.member)).toEqual(['list']) + }) + + it('ignores a comment or a string that looks like a member key', () => { + const members = parseOperationRegistryMembers(` + export const probeOperations = { + // permission-group-exempt: nothing: here is a member + read: defineWorkspaceOperation({ + id: 'probe.read', + minimumRole: 'read', + capability: 'tables.use', + }), + } as const + `) + + expect(members.map((member) => member.member)).toEqual(['read']) + }) + + it('reads no registry from a module that exports none', () => { + expect( + parseOperationRegistryMembers(` + export const applyWorkflowOperations = defineAuthorizedWorkflowUseCase({ + operation: workflowOperations.applyOperations, + }) + `) + ).toEqual([]) + }) +}) + +describe('a factory that admits a Partial override of the operation', () => { + /** + * Nothing in the tree does this today, which is why it is probed here rather + * than caught in the wild: the capability the parsers read is the literal in + * the factory body, and a `Partial` spread over the result + * can replace it with `'none'` after the audit has already approved it. The + * factory is reported as unparseable rather than resolved — the override's + * value lives at the call site, and following it is the call graph this audit + * does not have. + */ + it('reports an `overrides?: Partial` parameter', () => { + const { overridable } = parseOperationCapabilities( + 'function defineTableOperation(id: string, overrides?: Partial) {\n' + + " return defineWorkspaceOperation({ id, capability: 'tables.use', ...overrides })\n" + + '}\n' + ) + + expect(overridable).toEqual([1]) + }) + + it('reports the override however the parameter is spelled', () => { + const { overridable } = parseOperationCapabilities( + 'function defineKnowledgeOperation(\n' + + ' id: string,\n' + + ' patch: Partial = {}\n' + + ') {\n' + + " return defineWorkspaceOperation({ id, capability: 'knowledge.use', ...patch })\n" + + '}\n' + ) + + expect(overridable).toEqual([1]) + }) + + /** + * `Partial` over something that is not an operation is ordinary code — a + * factory taking a partial audit payload has nothing to say about capability. + */ + it('leaves a Partial of an unrelated type alone', () => { + const { overridable } = parseOperationCapabilities( + 'function defineTableOperation(id: string, audit?: Partial) {\n' + + " return defineWorkspaceOperation({ id, capability: 'tables.use', audit })\n" + + '}\n' + ) + + expect(overridable).toEqual([]) + }) + + it('leaves a factory with named parameters alone', () => { + const { declarations, overridable } = parseOperationCapabilities( + 'function defineTableOperation(id: string, capability: string) {\n' + + ' return defineWorkspaceOperation({ id, capability })\n' + + '}\n' + + "defineTableOperation('table.read', 'tables.use')\n" + ) + + expect(overridable).toEqual([]) + expect(declarations).toEqual([{ id: 'table.read', line: 4, capability: 'tables.use' }]) + }) +}) diff --git a/scripts/check-permission-group-enforcement.ts b/scripts/check-permission-group-enforcement.ts new file mode 100644 index 00000000000..6f16b5c825f --- /dev/null +++ b/scripts/check-permission-group-enforcement.ts @@ -0,0 +1,680 @@ +#!/usr/bin/env bun +/** + * Connects a permission-group config key to the server gate that enforces it. + * + * Twelve keys shipped with an admin checkbox, a hint describing what they + * restrict, and no server check at all — an organization that set + * `hideCopilot` or `hideDeployChatbot` believed it had withheld a capability + * while every API route still answered. Nothing connected "this key is offered + * to admins" to "something refuses when it is set", because the two live in + * different files and neither knows about the other. This audit connects them. + * + * It asserts, in order of what actually goes wrong: + * + * A every workspace operation declares a capability, or `'none'` with a + * reason — an omission cannot be told apart from an unreviewed operation + * B every declared capability exists + * C every capability in the registry is reachable: named by an operation, or + * by an annotated call site for the ones the funnel cannot apply + * D every key claiming `enforcement: 'capability'` is read by some rule + * E no key claiming a weaker mechanism is read by one, so a key cannot gain + * enforcement while still documented as cosmetic or execution-scoped + * F every member of an exported `*Operations` registry was read by A's + * parsers. Everything above can only speak about what they found, and an + * operation minted in a form they do not follow yields nothing — which + * reads exactly like a clean file + * + * A capability the funnel cannot apply — one needing a request value, like an + * auth mode — is declared at its call site instead: + * + * // permission-group-enforced: deploy.chat.auth_mode — asserted from the use case + * + * An operation no group governs says so explicitly: + * + * // permission-group-exempt: + * + * `capability` is a required field on `defineWorkspaceOperation`, so the half of + * assertion A that asks whether an operation declared one cannot fail through + * the type system. It survives because this audit reads source text rather than + * the type: an operation written in a form the parsers cannot follow yields no + * capability, and without the check it would be skipped in silence — counted as + * reviewed while nothing had actually read what it declares. + * + * ## What "enforced" means here, and where it stops + * + * Two different strengths of evidence sit behind assertion C, and the printed + * total does not distinguish them: + * + * - A capability NAMED BY AN OPERATION is enforced by construction. The funnel + * applies it; the declaration and the gate are the same fact. + * - A capability reachable only through a `permission-group-enforced:` comment + * is enforced by ASSERTION OF THE AUTHOR. This audit matches the comment + * text; it does not verify that anything below it gates. Today that is 18 of + * 35 capabilities — among them `logs.cost`, `inbox.use`, `personal_api_key.use` + * and `copilot.tool_auto_approval` — so it is the majority of the registry, + * not a rounding error. {@link parseEnforcedAnnotations} records which cheap + * lookahead shapes were measured against the tree and why each is wrong more + * often than right; closing this properly wants a call graph. What still + * holds is narrow but real: deleting a gate AND its comment is reported by C + * immediately, and assertion E stops an annotation from inventing enforcement + * for a key whose field says `ui-only` or `executor`. The uncovered case is + * exactly one — a gate deleted while its comment is left behind. + * + * {@link SCAN_ROOTS} is the other boundary. `background/`, `connectors/`, + * `tools/`, `enrichments/`, `triggers/` and every `.tsx` are unscanned, and as + * of this writing each contains ZERO operation declarations and ZERO capability + * gate calls — checked, not assumed. The boundary is documented rather than + * widened because a scan root that finds nothing costs walk time and teaches a + * reader that operations might live there. If one ever does, two things change + * together: `SCAN_ROOTS`, and the `.ts`-only filter in `walk`. + */ +import { readdirSync, readFileSync, statSync } from 'node:fs' +import { dirname, join, relative, resolve } from 'node:path' +import { fileURLToPath } from 'node:url' + +const SCRIPT_DIR = dirname(fileURLToPath(import.meta.url)) +const ROOT = resolve(SCRIPT_DIR, '..') +/** + * Where a `defineWorkspaceOperation` can live. Every operation declared today is + * under one of these, and `.tsx` is excluded because an operation is policy data + * that no component declares. Both are deliberate rather than incidental: an + * operation landing in `background`, `tools`, `triggers`, `connectors` or + * `enrichments`, or in a `.tsx`, would be invisible here — so widen this list + * (and `walk`) at the same time as moving one, not afterwards. + */ +const SCAN_ROOTS = ['apps/sim/lib', 'apps/sim/app', 'apps/sim/ee', 'apps/sim/executor'] +const CAPABILITIES_FILE = 'apps/sim/lib/permission-groups/capabilities.ts' +const FIELDS_FILE = 'apps/sim/lib/permission-groups/fields.ts' +const ENFORCED_ANNOTATION = 'permission-group-enforced:' +const EXEMPT_ANNOTATION = 'permission-group-exempt:' +const MAX_ANNOTATION_LOOKBACK = 3 + +/** + * The naming convention every operation-minting builder follows: + * `defineWorkspaceOperation`, `defineOperation`, and each domain's own + * `defineOperation`. + * + * The audit used to look for `defineWorkspaceOperation` and nothing else, so a + * domain that minted operations through a builder of its own — an object frozen + * by hand rather than passed to the shared one — was read by neither the + * builder's definition-time guard nor this script. Twenty-one operations across + * six domains were invisible that way, and the failure was silent: the files + * were scanned, some other operation in them was counted, and the audit printed + * a tick. Matching the family rather than the one name is what makes a new + * builder visible by default instead of on purpose. + */ +const MINTING_NAME = /^define[A-Za-z0-9_$]*Operation$/ +/** + * A factory parameter that admits a `Partial<…>` override of the operation it + * mints — `overrides?: Partial` and its relatives. + * + * The capability this audit reads is the one written in the factory body or at + * the call site. A parameter spread over the result afterwards can replace it, + * including with `'none'`, and the audit would keep reporting whatever the + * literal said — a green tick over an operation whose capability is decided by + * whoever calls it. No factory in the tree does this today; the point of + * refusing it here is that the first one to try is reported rather than + * discovered later. + * + * `Partial` and not `Omit`/`Pick`: those narrow a type, they do not make a + * declared field optional to overwrite. + */ +const OVERRIDE_PARAMETER = /\bPartial\s*<[^>]*Operation\b/ +const MINTING_CALL_SOURCE = String.raw`\b(define[A-Za-z0-9_$]*Operation)\s*\(` +const mintingCallPattern = () => new RegExp(MINTING_CALL_SOURCE, 'g') +/** Whether a module mints an operation at all, so files that do not are skipped cheaply. */ +const MINTS_AN_OPERATION = new RegExp(MINTING_CALL_SOURCE) +/** An exported `*Operations` registry object, the second route by which operations reach a surface. */ +const OPERATION_REGISTRY = + /(?:^|\n)\s*export const ([A-Za-z0-9_$]+Operations)\s*(?::[^=\n]*)?=\s*\{/g +const DECLARES_A_REGISTRY = /(?:^|\n)\s*export const [A-Za-z0-9_$]+Operations\s*(?::[^=\n]*)?=\s*\{/ + +interface Finding { + file: string + line?: number + message: string +} + +/** The 1-based line `index` falls on. */ +function lineAt(source: string, index: number): number { + return source.slice(0, index).split('\n').length +} + +const CLOSING: Record = { '(': ')', '{': '}', '[': ']' } + +/** Text of the balanced `(...)`, `{...}` or `[...]` group that starts at `openIndex`. */ +function balancedGroup(source: string, openIndex: number): string { + const open = source[openIndex] + const close = CLOSING[open] + let depth = 0 + for (let index = openIndex; index < source.length; index++) { + const char = source[index] + if (char === open) depth++ + else if (char === close) { + depth-- + if (depth === 0) return source.slice(openIndex, index + 1) + } + } + return source.slice(openIndex) +} + +function walk(directory: string, into: string[]): string[] { + for (const entry of readdirSync(directory)) { + if (entry === 'node_modules' || entry === '.next') continue + const full = join(directory, entry) + if (statSync(full).isDirectory()) walk(full, into) + else if (full.endsWith('.ts') && !full.endsWith('.test.ts')) into.push(full) + } + return into +} + +/** The capability ids the registry declares, in declaration order. */ +export function parseCapabilityIds(source: string): string[] { + const start = source.indexOf('CAPABILITY_IDS = [') + if (start === -1) return [] + const group = balancedGroup(source, source.indexOf('[', start)) + return [...group.matchAll(/'([a-z0-9_.]+)'/g)].map((match) => match[1]) +} + +/** Each capability's rule kind and the config keys it reads. */ +export function parseCapabilityRules( + source: string +): Map { + const rules = new Map() + const start = source.indexOf('CAPABILITY_RULES = {') + if (start === -1) return rules + + const body = balancedGroup(source, source.indexOf('{', start)) + const entryPattern = /'([a-z0-9_.]+)'\s*:\s*\{/g + for (let match = entryPattern.exec(body); match; match = entryPattern.exec(body)) { + const entry = balancedGroup(body, body.indexOf('{', match.index + match[0].length - 1)) + const kind = /kind\s*:\s*'([a-z]+)'/.exec(entry)?.[1] ?? '' + const keysGroup = /configKeys\s*:\s*\[([^\]]*)\]/.exec(entry)?.[1] ?? '' + const configKeys = [...keysGroup.matchAll(/'([A-Za-z0-9_]+)'/g)].map((key) => key[1]) + rules.set(match[1], { kind, configKeys }) + } + return rules +} + +/** Each config key's declared enforcement, from the field registry. */ +export function parseFieldEnforcement(source: string): Map { + const enforcement = new Map() + const start = source.indexOf('PERMISSION_GROUP_FIELDS = {') + if (start === -1) return enforcement + + const body = balancedGroup(source, source.indexOf('{', start)) + const entryPattern = + /(?:^|\n)\s{2}([A-Za-z0-9_]+)\s*:\s*(allowlist|denylist|booleanRestriction)\(/g + for (let match = entryPattern.exec(body); match; match = entryPattern.exec(body)) { + const call = balancedGroup(body, body.indexOf('(', match.index + match[0].length - 1)) + const declared = /'(capability|executor|ui-only)'/.exec(call)?.[1] + if (declared) enforcement.set(match[1], declared) + } + return enforcement +} + +interface OperationDeclaration { + id: string + line: number + capability: string | undefined +} + +interface ParsedOperations { + declarations: OperationDeclaration[] + /** + * Lines of same-file factories whose parameters admit a `Partial<…>` override + * of the operation itself. See {@link OVERRIDE_PARAMETER}. + */ + overridable: number[] + /** + * Lines of operation-minting calls this parser could not read an id from, + * and which no recognized factory accounts for. + * + * A call whose `id` is a const reference, or one minted by a wrapper written + * as an arrow const rather than a `function`, used to be dropped in silence — + * the operation simply stopped being counted, and the audit still printed a + * tick. Reported instead, because an audit that quietly stops watching a + * domain is the failure it exists to prevent. + */ + unreadable: number[] +} + +/** + * Every operation minted in a module and the capability it declares, resolved + * through a same-file factory when a domain wraps a builder (the table + * operations take only an id and a capability). + */ +export function parseOperationCapabilities(source: string): ParsedOperations { + const declarations: OperationDeclaration[] = [] + const unreadable: number[] = [] + const overridable: number[] = [] + + /** + * Domains that wrap a builder in a same-file factory declare the capability + * one of three ways: fixed in the factory body, when every operation it makes + * belongs to one capability; taken as a second argument when they differ; or + * passed straight through from an object literal at the call site. The first + * two are read from their call sites below, the third by the direct scan, + * which reads the literal exactly as it reads a builder's own. + */ + const factoryCapabilities = new Map() + const factoryRanges: Array<[number, number]> = [] + const factoryPattern = /(?:^|\n)\s*(?:export\s+)?function\s+([A-Za-z0-9_$]+)\s*[<(]/g + for (let match = factoryPattern.exec(source); match; match = factoryPattern.exec(source)) { + const bodyIndex = source.indexOf('{', match.index + match[0].length - 1) + if (bodyIndex === -1) continue + const body = balancedGroup(source, bodyIndex) + if (!MINTS_AN_OPERATION.test(body) && !MINTING_NAME.test(match[1])) continue + factoryRanges.push([bodyIndex, bodyIndex + body.length]) + const parameterIndex = source.indexOf('(', match.index + match[0].length - 1) + if (parameterIndex !== -1 && parameterIndex < bodyIndex) { + const parameters = balancedGroup(source, parameterIndex) + if (OVERRIDE_PARAMETER.test(parameters)) overridable.push(lineAt(source, match.index)) + } + const fixed = /capability\s*:\s*'([a-z0-9_.]+)'/.exec(body)?.[1] + if (fixed) factoryCapabilities.set(match[1], fixed) + else if (/capability\s*[,:}]/.test(body)) factoryCapabilities.set(match[1], 'positional') + } + + /** A call inside a recognized factory takes its id from a parameter; its call sites are read below. */ + const insideFactory = (index: number) => + factoryRanges.some(([start, end]) => index >= start && index < end) + + /** + * Ranges of calls already read. A domain wrapper such as + * `defineCredentialOperation(defineWorkspaceOperation({...}), 'admin')` matches + * twice over one operation; the outer match already carries the inner's `id` + * and `capability`, so the nested one is skipped rather than counted again. + */ + const accepted: Array<[number, number]> = [] + + const directPattern = mintingCallPattern() + for (let match = directPattern.exec(source); match; match = directPattern.exec(source)) { + const name = match[1] + if (/\bfunction\s+$/.test(source.slice(Math.max(0, match.index - 16), match.index))) continue + if (factoryCapabilities.has(name)) continue + if (insideFactory(match.index)) continue + const openIndex = source.indexOf('(', match.index) + if (accepted.some(([start, end]) => openIndex > start && openIndex < end)) continue + const call = balancedGroup(source, openIndex) + accepted.push([openIndex, openIndex + call.length]) + const id = /id\s*:\s*'([^']+)'/.exec(call)?.[1] + if (!id) { + unreadable.push(lineAt(source, match.index)) + continue + } + declarations.push({ + id, + line: lineAt(source, match.index), + capability: /capability\s*:\s*'([a-z0-9_.]+)'/.exec(call)?.[1], + }) + } + + for (const [factory, capability] of factoryCapabilities) { + const callPattern = + capability === 'positional' + ? new RegExp(`\\b${factory}\\s*\\(\\s*'([^']+)'\\s*,\\s*'([a-z0-9_.]+)'`, 'g') + : new RegExp(`\\b${factory}\\s*\\(\\s*'([^']+)'`, 'g') + for (let match = callPattern.exec(source); match; match = callPattern.exec(source)) { + if (insideFactory(match.index)) continue + declarations.push({ + id: match[1], + line: lineAt(source, match.index), + capability: capability === 'positional' ? match[2] : capability, + }) + } + } + + return { declarations, unreadable, overridable } +} + +export interface OperationRegistryMember { + registry: string + member: string + startLine: number + endLine: number +} + +/** Index just past the string literal that starts at `openIndex`. */ +function skipStringLiteral(body: string, openIndex: number): number { + const quote = body[openIndex] + for (let index = openIndex + 1; index < body.length; index++) { + if (body[index] === '\\') { + index++ + continue + } + if (body[index] === quote) return index + 1 + } + return body.length +} + +/** The keyed members of an object literal, at its own top level only. */ +function topLevelMembers(body: string): Array<{ key: string; start: number; end: number }> { + const members: Array<{ key: string; start: number; end: number }> = [] + let depth = 0 + let index = 0 + let pending: { key: string; start: number } | null = null + const flush = (end: number) => { + if (pending) members.push({ ...pending, end }) + pending = null + } + + while (index < body.length) { + const char = body[index] + if (char === '/' && body[index + 1] === '/') { + const newline = body.indexOf('\n', index) + index = newline === -1 ? body.length : newline + 1 + continue + } + if (char === '/' && body[index + 1] === '*') { + const close = body.indexOf('*/', index) + index = close === -1 ? body.length : close + 2 + continue + } + if (char === "'" || char === '"' || char === '`') { + index = skipStringLiteral(body, index) + continue + } + if (char === '{' || char === '(' || char === '[') { + depth++ + index++ + continue + } + if (char === '}' || char === ')' || char === ']') { + depth-- + if (depth === 0) flush(index) + index++ + continue + } + if (depth === 1) { + if (char === ',') { + flush(index) + index++ + continue + } + if (!pending && /[\s{,]/.test(body[index - 1] ?? '{')) { + const key = /^([A-Za-z0-9_$]+)\s*:/.exec(body.slice(index)) + if (key) { + pending = { key: key[1], start: index } + index += key[0].length + continue + } + } + } + index++ + } + flush(body.length) + return members +} + +/** + * The members of every exported `*Operations` registry, with the line span each + * one occupies. + * + * This is the completeness half of the audit, and it asks a different question + * from everything above: not *does this operation declare a capability*, but + * *did this audit read this operation at all*. Assertion A can only speak about + * operations the parsers found; a member minted by a form they do not follow + * yields nothing, and nothing is indistinguishable from a clean file. Comparing + * the registry a surface actually imports against what was parsed is what turns + * that silence into a failure — an undercount reads exactly like success, which + * is how five OAuth-connection operations shipped ungated. + */ +export function parseOperationRegistryMembers(source: string): OperationRegistryMember[] { + const members: OperationRegistryMember[] = [] + OPERATION_REGISTRY.lastIndex = 0 + for ( + let match = OPERATION_REGISTRY.exec(source); + match; + match = OPERATION_REGISTRY.exec(source) + ) { + const openIndex = source.indexOf('{', match.index + match[0].length - 1) + const body = balancedGroup(source, openIndex) + for (const member of topLevelMembers(body)) { + members.push({ + registry: match[1], + member: member.key, + startLine: lineAt(source, openIndex + member.start), + endLine: lineAt(source, openIndex + member.end), + }) + } + } + return members +} + +/** + * Capabilities declared enforced at a call site the funnel cannot reach. + * + * Deliberately a bare scan of the whole file, with no check that anything below + * the annotation actually gates: an annotation left behind after its gate was + * deleted still counts the capability as reachable, and assertion C stays + * green. That is a real gap, and it is left open because every cheap shape that + * would close it is wrong more often than it is right. + * + * The obvious shapes were measured against the 70-odd annotations in the tree: + * + * - "the capability id appears in code in the same file" misses 7, among them + * `logs/application/list-public-logs.ts` and `get-public-log.ts`, which + * annotate `logs.cost` and `logs.trace_spans` over a call to + * `resolveLogFieldProjection` — the one place those two are read, named + * nowhere else because naming them twice is how one copy stops redacting. + * - "a capability sink is called within N lines below" misses 7 at any N, + * including `core/application/workspace-authorization.ts` (delegates to + * `requirePersonalApiKeysAllowed`), `invitations/workspace-invitations.ts` + * (`validateInvitationsAllowed`) and the three `integrations.manage` sites in + * `auth/oauth/credentials/route.ts` (`checkOAuthCredentialAccess`). + * + * Both fail on the same case, and it is the common one: the annotation sits + * above a call to a DOMAIN helper that enforces the group somewhere else. A + * lookahead would have to enumerate every such helper — which is the open-ended + * set the annotation exists to describe in the first place, so the list would + * go stale in exactly the direction that makes the audit lie. + * + * What does hold the line is assertion E's other half: an annotation cannot + * invent enforcement for a key whose field says `ui-only` or `executor`, and a + * capability whose gate is deleted along with its annotation is reported by + * assertion C immediately. The gap is narrow — a gate deleted while its comment + * is kept — and closing it wants a call-graph, not a regex. + */ +export function parseEnforcedAnnotations(source: string): string[] { + return [...source.matchAll(new RegExp(`${ENFORCED_ANNOTATION}\\s*([a-z0-9_.]+)`, 'g'))].map( + (match) => match[1] + ) +} + +/** Whether an operation's `capability: 'none'` carries a reason. */ +export function hasExemptAnnotation(source: string, line: number): boolean { + const lines = source.split('\n') + for (let back = line - 2; back >= 0 && back >= line - 2 - MAX_ANNOTATION_LOOKBACK; back--) { + const candidate = lines[back]?.trim() ?? '' + if (candidate === '') continue + if (!candidate.startsWith('//') && !candidate.startsWith('*')) break + if (candidate.includes(EXEMPT_ANNOTATION)) { + return ( + candidate.slice(candidate.indexOf(EXEMPT_ANNOTATION) + EXEMPT_ANNOTATION.length).trim() !== + '' + ) + } + } + return false +} + +function main(): void { + const capabilitiesSource = readFileSync(join(ROOT, CAPABILITIES_FILE), 'utf8') + const fieldsSource = readFileSync(join(ROOT, FIELDS_FILE), 'utf8') + + const capabilityIds = new Set(parseCapabilityIds(capabilitiesSource)) + const rules = parseCapabilityRules(capabilitiesSource) + const enforcement = parseFieldEnforcement(fieldsSource) + + /** + * This audit reads source text, so a rename it does not know about makes its + * parsers return nothing — and every assertion below would then pass over an + * empty set. An audit that goes quiet when it breaks is worse than no audit, + * so refuse to report success on an obviously empty parse. + */ + if (capabilityIds.size === 0 || rules.size === 0 || enforcement.size === 0) { + console.error( + 'Permission-group enforcement audit could not read its own inputs:\n' + + ` capabilities parsed: ${capabilityIds.size}, rules: ${rules.size}, config keys: ${enforcement.size}\n\n` + + 'One of CAPABILITY_IDS, CAPABILITY_RULES or PERMISSION_GROUP_FIELDS was renamed or\n' + + 'reshaped. Update the parsers in this script rather than leaving it passing vacuously.\n' + ) + process.exit(1) + } + if (rules.size !== capabilityIds.size) { + console.error( + `Permission-group enforcement audit parsed ${capabilityIds.size} capabilities but ${rules.size} rules; the registry and its rules disagree.\n` + ) + process.exit(1) + } + + const sourceFiles = SCAN_ROOTS.flatMap((root) => walk(join(ROOT, root), [])) + + const findings: Finding[] = [] + const usedCapabilities = new Set() + let declaredOperations = 0 + + for (const file of sourceFiles) { + const relativePath = relative(ROOT, file) + const source = readFileSync(file, 'utf8') + + for (const capability of parseEnforcedAnnotations(source)) { + usedCapabilities.add(capability) + if (!capabilityIds.has(capability)) { + findings.push({ + file: relativePath, + message: `declares enforcement for unknown capability '${capability}'`, + }) + } + } + + if (!MINTS_AN_OPERATION.test(source) && !DECLARES_A_REGISTRY.test(source)) continue + + const { declarations, unreadable, overridable } = parseOperationCapabilities(source) + + for (const line of overridable) { + findings.push({ + file: relativePath, + line, + message: + "mints operations through a factory that takes a `Partial<…Operation>` override. The capability this audit reads is the one in the factory body or at the call site, and a partial spread over the result can replace it — including with 'none' — so what is declared here stops being what ships. Take the fields the factory varies as named parameters rather than an open override", + }) + } + + for (const line of unreadable) { + findings.push({ + file: relativePath, + line, + message: + 'operation-minting call this audit cannot read an id from — a const-reference id, an id taken as a bare argument by a builder that mints the operation itself, or a wrapper written as an arrow const rather than a `function`. Teach parseOperationCapabilities the form rather than letting the operation drop out of the count in silence', + }) + } + + /** + * A file that calls the builder and yields nothing means the parsers no + * longer understand it. Per-file rather than a count floor: a floor rots on + * every legitimate addition and invites bumping the number. + */ + if (MINTS_AN_OPERATION.test(source) && declarations.length === 0 && unreadable.length === 0) { + findings.push({ + file: relativePath, + message: + 'mints an operation but this audit parsed none from it — the declaration form changed and every operation in this file is now unchecked', + }) + } + + /** + * Assertion F: every member of an exported registry was read. + * + * Everything above can only speak about what the parsers found. This asks + * whether they found each thing a surface can actually import, which is the + * only question whose answer distinguishes a clean file from one the + * parsers walked straight past. + */ + const readLines = [...declarations.map((declaration) => declaration.line), ...unreadable] + for (const member of parseOperationRegistryMembers(source)) { + if (readLines.some((line) => line >= member.startLine && line <= member.endLine)) continue + findings.push({ + file: relativePath, + line: member.startLine, + message: `'${member.registry}.${member.member}' is exported as an operation but this audit read no operation from it — it is minted by a form the parsers do not follow, so nothing here checks what capability it declares. Name the builder \`defineOperation\` and pass it an object literal with a string \`id\` and \`capability\`, or teach parseOperationCapabilities the form; do not leave the member counted as reviewed while unread`, + }) + } + + for (const declaration of declarations) { + declaredOperations++ + if (declaration.capability === undefined) { + findings.push({ + file: relativePath, + line: declaration.line, + message: `operation '${declaration.id}' declares a capability this audit cannot read — the field is required, so this is a declaration form the parsers do not follow; teach parseOperationCapabilities about it rather than leaving the operation unchecked`, + }) + continue + } + if (declaration.capability === 'none') { + if (!hasExemptAnnotation(source, declaration.line)) { + findings.push({ + file: relativePath, + line: declaration.line, + message: `operation '${declaration.id}' declares capability 'none' without a reason — put '${EXEMPT_ANNOTATION} ' in a comment directly above it`, + }) + } + continue + } + usedCapabilities.add(declaration.capability) + if (!capabilityIds.has(declaration.capability)) { + findings.push({ + file: relativePath, + line: declaration.line, + message: `operation '${declaration.id}' names unknown capability '${declaration.capability}'`, + }) + } + } + } + + /** A capability nothing names is a key an admin can set to no effect. */ + for (const capability of capabilityIds) { + if (usedCapabilities.has(capability)) continue + findings.push({ + file: CAPABILITIES_FILE, + message: `capability '${capability}' is declared but nothing enforces it — name it on an operation, or annotate its call site with '${ENFORCED_ANNOTATION} ${capability} — '`, + }) + } + + const enforcedByRule = new Set([...rules.values()].flatMap((rule) => rule.configKeys)) + for (const [key, declared] of enforcement) { + if (declared === 'capability' && !enforcedByRule.has(key)) { + findings.push({ + file: FIELDS_FILE, + message: `config key '${key}' claims capability enforcement but no rule reads it — give it a rule, or declare it 'executor' or 'ui-only'`, + }) + } + if (declared !== 'capability' && enforcedByRule.has(key)) { + findings.push({ + file: FIELDS_FILE, + message: `config key '${key}' is declared '${declared}' but a capability rule reads it — set enforcement to 'capability' so the key stops being documented as something weaker`, + }) + } + } + + if (findings.length > 0) { + console.error('Permission-group enforcement audit failed:\n') + for (const finding of findings) { + const where = finding.line ? `${finding.file}:${finding.line}` : finding.file + console.error(` ${where}\n ${finding.message}\n`) + } + console.error( + 'A permission-group key that reaches the admin editor without a server gate is a\n' + + 'restriction an organization believes it applied. Wire the gate, or declare the\n' + + "key 'ui-only' so it is documented as a rendering hint rather than a control.\n" + ) + process.exit(1) + } + + console.log( + `✓ permission-group enforcement: ${declaredOperations} operations declare a capability, ${capabilityIds.size} capabilities all enforced` + ) +} + +if (import.meta.main) main() diff --git a/scripts/check-script-test-coverage.ts b/scripts/check-script-test-coverage.ts index 3e126d5b164..87dc148c458 100644 --- a/scripts/check-script-test-coverage.ts +++ b/scripts/check-script-test-coverage.ts @@ -1,11 +1,14 @@ #!/usr/bin/env bun /** - * Asserts every `scripts/*.test.ts` file is reachable from the root `test` script. + * Asserts every `scripts/*.test.ts` file is collected by the root Vitest config. * - * The root `test` script chains a hand-maintained list of `test:*` entries, and a hand-maintained - * list silently drifts from the files on disk: a test added without a matching entry never runs, - * in CI or locally, and nothing reports it. `scripts/check-migrations-safety.test.ts` sat - * unreferenced and green for exactly that reason. + * The root `test` script once chained a hand-maintained list of `test:*` entries, and a + * hand-maintained list silently drifts from the files on disk: a test added without a matching + * entry never runs, in CI or locally, and nothing reports it. `scripts/check-migrations-safety.test.ts` + * sat unreferenced and green for exactly that reason. The root `vitest.scripts.config.ts` now collects + * the directory by glob, so drift can only come from a file the glob does not match (a test in a + * subdirectory, a different suffix) or from the `test` script no longer chaining `test:scripts`. + * This guard checks both by asking Vitest which files it would run. * * `run-audits.ts` derives its own list from the `check:*` namespace precisely so a new audit is * picked up by default, so this guard registers itself simply by being named `check:*` — it cannot @@ -15,57 +18,59 @@ import { readdirSync } from 'node:fs' import path from 'node:path' const ROOT = path.resolve(import.meta.dir, '..') -const TEST_FILE_PATTERN = /scripts\/[\w.-]+\.test\.ts/g const SUB_SCRIPT_PATTERN = /bun run ([\w:-]+)/g const manifest = await Bun.file(path.join(ROOT, 'package.json')).json() const commands = manifest.scripts as Record -/** Walks the `test` script and every `test:*` entry it chains, collecting referenced test files. */ -function reachableTestFiles(entry: string): Set { - const referenced = new Set() +/** Walks the `test` script and every entry it chains. */ +function reachableScripts(entry: string): Set { const seen = new Set() const queue = [entry] - while (queue.length > 0) { const name = queue.pop() as string if (seen.has(name)) continue seen.add(name) + for (const match of (commands[name] ?? '').matchAll(SUB_SCRIPT_PATTERN)) queue.push(match[1]) + } + return seen +} - const command = commands[name] - if (command === undefined) continue +if (!reachableScripts('test').has('test:scripts')) { + console.error('The root `test` script no longer chains `test:scripts`, so no script test runs.') + process.exit(1) +} - for (const match of command.matchAll(TEST_FILE_PATTERN)) referenced.add(match[0]) - for (const match of command.matchAll(SUB_SCRIPT_PATTERN)) queue.push(match[1]) +const listed = Bun.spawnSync( + ['bunx', 'vitest', 'list', '--json', '--filesOnly', '--config', 'vitest.scripts.config.ts'], + { + cwd: ROOT, } - - return referenced +) +if (listed.exitCode !== 0) { + console.error(`\`vitest list\` failed:\n${listed.stderr.toString()}`) + process.exit(1) } +const collected = new Set( + (JSON.parse(listed.stdout.toString()) as Array<{ file: string }>).map((entry) => + path.relative(ROOT, entry.file).split(path.sep).join('/') + ) +) const onDisk = readdirSync(path.join(ROOT, 'scripts')) .filter((file) => file.endsWith('.test.ts')) .map((file) => `scripts/${file}`) .sort() -const reachable = reachableTestFiles('test') -const orphaned = onDisk.filter((file) => !reachable.has(file)) -const missing = [...reachable].filter((file) => !onDisk.includes(file)).sort() - -if (orphaned.length > 0 || missing.length > 0) { - if (orphaned.length > 0) { - console.error( - `Script tests never run by \`bun run test\`:\n${orphaned.map((file) => ` - ${file}`).join('\n')}\n` + - 'Add a `test:*` entry for each and chain it into the root `test` script.' - ) - } - if (missing.length > 0) { - console.error( - `Root \`test\` script references script tests that do not exist:\n${missing.map((file) => ` - ${file}`).join('\n')}` - ) - } +const orphaned = onDisk.filter((file) => !collected.has(file)) +if (orphaned.length > 0) { + console.error( + `Script tests never run by \`bun run test\`:\n${orphaned.map((file) => ` - ${file}`).join('\n')}\n` + + 'Make sure the root `vitest.scripts.config.ts` include glob matches them.' + ) process.exit(1) } console.log( - `Script test coverage passed: ${onDisk.length} script tests reachable from \`bun run test\`.` + `Script test coverage passed: ${onDisk.length} script tests collected by the root Vitest config.` ) diff --git a/scripts/check-sql-date-binding.ts b/scripts/check-sql-date-binding.ts index 514e8884fd4..3078a7b89db 100644 --- a/scripts/check-sql-date-binding.ts +++ b/scripts/check-sql-date-binding.ts @@ -17,6 +17,7 @@ import { readdirSync, readFileSync } from 'node:fs' import { dirname, extname, join, relative, resolve } from 'node:path' import { fileURLToPath } from 'node:url' import { parse } from '@babel/parser' +import ts from '@typescript/typescript6' const SCRIPT_DIR = dirname(fileURLToPath(import.meta.url)) const ROOT = resolve(SCRIPT_DIR, '..') @@ -27,6 +28,13 @@ const ALLOW_ANNOTATION = '// sql-date-bound:' const DRIZZLE_MODULE = 'drizzle-orm' const STATEMENT_TYPE = /(Statement|Declaration)$/ +function isDrizzleModule(value: unknown): value is string { + return ( + typeof value === 'string' && + (value === DRIZZLE_MODULE || value.startsWith(`${DRIZZLE_MODULE}/`)) + ) +} + interface Violation { file: string line: number @@ -222,10 +230,7 @@ function collectSqlBindings(program: SyntaxNode): SqlBindings { const visit = (node: SyntaxNode) => { if (node.type === 'ImportDeclaration' && isSyntaxNode(node.source)) { const source = node.source.value - const isDrizzle = - typeof source === 'string' && - (source === DRIZZLE_MODULE || source.startsWith(`${DRIZZLE_MODULE}/`)) - if (isDrizzle && Array.isArray(node.specifiers)) { + if (isDrizzleModule(source) && Array.isArray(node.specifiers)) { for (const specifier of node.specifiers) { if (!isSyntaxNode(specifier) || !isSyntaxNode(specifier.local)) continue const local = specifier.local.name @@ -269,10 +274,7 @@ function isDrizzleImportCall(node: unknown): boolean { const args = Array.isArray(current.arguments) ? current.arguments : [] const source = isSyntaxNode(current.source) ? current.source : args.find(isSyntaxNode) const value = source?.value - return ( - typeof value === 'string' && - (value === DRIZZLE_MODULE || value.startsWith(`${DRIZZLE_MODULE}/`)) - ) + return isDrizzleModule(value) } const unwrapAwait = (node: SyntaxNode): unknown => @@ -548,12 +550,32 @@ function analyzeSource(source: string, file = 'source.ts'): FileAnalysis { * Skips the parse for files that cannot bind the tag. * * `collectSqlBindings` and `isDrizzleImportCall` both match the specifier as a string literal, - * so a source that never names the module yields no bindings and no violations. That is all but - * ~590 of the ~13,900 scanned files, and not parsing them takes the audit from ~4.5s to ~0.8s. - * An escaped specifier (`'drizzle\x2dorm'`) would evade the substring; the repo contains none. + * so a source that never names the module yields no bindings and no violations. The scanner + * decodes escaped string tokens before comparing them and also requires a possible `sql` binding. */ -function mayBindDrizzleSql(source: string): boolean { - return source.includes(DRIZZLE_MODULE) +export function mayBindDrizzleSql(source: string): boolean { + const scanner = ts.createScanner(ts.ScriptTarget.Latest, true, ts.LanguageVariant.JSX, source) + let hasDrizzleModule = false + let hasSqlToken = false + + for (let token = scanner.scan(); token !== ts.SyntaxKind.EndOfFileToken; token = scanner.scan()) { + const value = scanner.getTokenValue() + if ( + (token === ts.SyntaxKind.StringLiteral || + token === ts.SyntaxKind.NoSubstitutionTemplateLiteral) && + isDrizzleModule(value) + ) { + hasDrizzleModule = true + } + if ( + (token === ts.SyntaxKind.Identifier || token === ts.SyntaxKind.StringLiteral) && + value === 'sql' + ) { + hasSqlToken = true + } + if (hasDrizzleModule && hasSqlToken) return true + } + return false } function collectSources(dir: string, found: string[] = []): string[] { diff --git a/scripts/check-tool-param-reachability.ts b/scripts/check-tool-param-reachability.ts new file mode 100644 index 00000000000..e4d4a693eb9 --- /dev/null +++ b/scripts/check-tool-param-reachability.ts @@ -0,0 +1,206 @@ +#!/usr/bin/env bun +/** + * Fails when a tool declares a required parameter nothing can fill. + * + * `visibility: 'hidden'` means "not shown to user or LLM" (`tools/types.ts`), so + * a hidden parameter has no caller. Something else has to supply it, and only + * two mechanisms do: OAuth credential resolution, which assigns the fields in + * {@link RESOLVER_GUARANTEED} once a credential is bound, or the fields a tool declares in `authoritativeParams`, and hosted-key + * injection, which assigns `hosting.apiKeyParam`. A required hidden parameter + * outside both is unreachable by every caller except the block that happens to + * construct it during serialization. + * + * The failure this exists to prevent is silent. `createUserToolSchema` omits + * hidden parameters, so an agent is never told to send one; a tool that also + * omits its `oauth` declaration is never asked for a credential either; and + * `validateRequiredParametersAfterMerge` only validates `user-or-llm`, so + * nothing rejects the call. The request is built with `undefined` in place of + * the value and the provider answers a 401 that names nothing — which is how + * 117 parameters across four integrations reached production broken for every + * direct caller (Copilot's `call_integration_tool` and `POST + * /api/v2/tools/{toolId}/execute`) while working inside a workflow. + * + * The fix is one of three, decided by what actually supplies the value: + * + * - the user types it into a block field -> `visibility: 'user-only'` + * (`mailchimp.apiKey`, `zendesk.apiToken`). This does not widen what the + * model sees: `createLLMToolSchema` skips `user-only` and `hidden` alike. + * It only lets a caller send it, and lets `{{VAR}}` references resolve. + * - a bound OAuth credential supplies it -> declare `oauth` on the tool + * (`pipedrive`, `wealthbox`, whose `accessToken: 'hidden'` was already + * right; the missing declaration was the bug). + * - a block composes it from sibling fields -> publish the composed shape as + * `visibility: 'user-or-llm'` (`calcom_create_booking.attendee`). The block + * keeps composing it — `tools.config.params` runs before execution, so the + * merge validation still sees a value — and a direct caller sends the object + * itself. + * + * Choosing `user-only` carries an obligation: `check-block-registry.ts` requires + * every required `user-only` parameter to have a subBlock whose `id` or + * `canonicalParamId` equals the parameter id, because the serializer resolves it + * by direct lookup. A block whose canonical key differs has to be aligned on the + * parameter id — safe to do, because canonical ids are config-derived rather + * than stored, and `backfillCanonicalModes` re-derives a renamed pair's mode + * from whichever value is populated. + * + * There is deliberately no allowlist. All three answers leave the parameter + * reachable, so a parameter needing an exemption is one no caller can supply — + * exactly what this audit exists to reject. + * + * Usage: + * bun run scripts/check-tool-param-reachability.ts + */ +import { tools } from '../apps/sim/tools/registry' +import type { ToolConfig } from '../apps/sim/tools/types' + +/** + * The one parameter credential resolution assigns unconditionally. + * + * `executeToolImplementation` writes `contextParams.accessToken = data.accessToken` + * with no guard, so an OAuth tool's hidden `accessToken` is filled whenever a + * credential resolves at all. Nothing else is: every other token-response field + * is assigned under `if (data.X)`, present on some providers' credentials and + * absent on others. + */ +const RESOLVER_GUARANTEED = 'accessToken' + +/** + * Token-response fields a tool may declare its credential supplies. + * + * These are assigned conditionally — `idToken`, `instanceUrl`, `apiDomain`, + * `cloudId`, `domain`, `authStyle` under `if (data.X)`, and `credentialType` + * additionally only when listed here. Whether a given credential carries one + * is a fact about the provider, not the resolver, and the resolver cannot + * vouch for it. The tool can: `oauth.authoritativeParams` is the declaration + * that the token response supplies the named field, so a required hidden + * parameter in this set is exempt only when its tool lists it there. A tool + * that hides one without declaring it is asserting a filler the resolver may + * never run — the exact shape this audit exists to reject. + * + * Kept in step with the assignments in `apps/sim/tools/index.ts` and the + * `authoritativeParams` union in `tools/types.ts`. + */ +const TOKEN_RESPONSE_FIELDS = new Set([ + 'credentialType', + 'idToken', + 'instanceUrl', + 'apiDomain', + 'cloudId', + 'domain', + 'authStyle', +]) + +interface Finding { + toolId: string + param: string + reason: string +} + +/** + * A hosted tool must not declare its own `cost` output. + * + * Direct execution (`POST /api/v2/tools/{toolId}/execute`) bills hosted-key + * spend by reading `output.cost` on a tool with `hosting` — because on such a + * tool that field has exactly one writer, `applyHostedKeyCostToResult`, which + * runs only when the registry actually used Sim's key on a successful call. A + * BYOK call leaves it absent and so is not billed. A hosted tool that also + * reported its own cost there would break that reading: its self-reported + * number would bill as Sim's spend on a BYOK call or a caller-keyed call. Tools + * without `hosting` may report cost freely; the meter never looks at them. + */ +function findHostedToolsReportingCost(): string[] { + return Object.entries(tools as Record) + .filter( + ([, config]) => config.hosting && config.outputs && Object.hasOwn(config.outputs, 'cost') + ) + .map(([toolId]) => toolId) + .sort() +} + +function findUnreachableParams(): Finding[] { + const findings: Finding[] = [] + + for (const [toolId, config] of Object.entries(tools as Record)) { + /** + * Only unconditional hosting is a guarantee. A `hosting.enabled` predicate + * can decline for a given parameter combination, and this audit has no + * params to evaluate it against — so a conditionally-hosted key is treated + * as unfilled, which is the answer that fails closed. + */ + const hostedKeyParam = config.hosting?.enabled ? undefined : config.hosting?.apiKeyParam + + for (const [param, declaration] of Object.entries(config.params ?? {})) { + if (!declaration || declaration.visibility !== 'hidden' || !declaration.required) continue + if (config.oauth && param === RESOLVER_GUARANTEED) continue + if ( + config.oauth && + TOKEN_RESPONSE_FIELDS.has(param) && + (config.oauth.authoritativeParams as readonly string[] | undefined)?.includes(param) + ) { + continue + } + if (hostedKeyParam && param === hostedKeyParam) continue + + findings.push({ + toolId, + param, + reason: config.oauth + ? TOKEN_RESPONSE_FIELDS.has(param) + ? `declares oauth (${config.oauth.provider}) but not \`authoritativeParams: ['${param}']\`, and the resolver assigns '${param}' only when the credential carries it` + : `declares oauth (${config.oauth.provider}), which does not supply '${param}'` + : config.hosting?.enabled + ? `hosting is conditional, so it is not a guarantee for '${param}'` + : config.hosting + ? `hosting supplies '${config.hosting.apiKeyParam}', not '${param}'` + : 'declares neither oauth nor hosting', + }) + } + } + + return findings.sort((a, b) => a.toolId.localeCompare(b.toolId) || a.param.localeCompare(b.param)) +} + +function main(): void { + const findings = findUnreachableParams() + const toolCount = Object.keys(tools).length + const costReporters = findHostedToolsReportingCost() + + if (costReporters.length > 0) { + console.error('Tool parameter reachability audit failed:\n') + for (const toolId of costReporters) { + console.error( + ` ${toolId} — declares hosting AND a 'cost' output; direct execution reads output.cost on a hosted tool as "Sim's key paid", so a self-reported cost would bill BYOK and caller-keyed calls` + ) + } + process.exit(1) + } + + if (findings.length === 0) { + console.log( + `✓ tool parameter reachability: all ${toolCount} tools supply every required hidden parameter through oauth, hosting, or a published shape` + ) + return + } + + console.error('Tool parameter reachability audit failed:\n') + for (const { toolId, param, reason } of findings) { + console.error(` ${toolId} — required hidden parameter '${param}': ${reason}`) + } + console.error( + [ + '', + 'A required hidden parameter has no caller. Supply it by declaration, not by hoping:', + " - the user types it into a block field -> visibility: 'user-only'", + ' - a bound OAuth credential supplies it -> declare oauth on the tool', + " - Sim's hosted key supplies it -> declare hosting with this apiKeyParam", + " - a block composes it from siblings -> publish the shape as 'user-or-llm'", + '', + 'Leaving it hidden means every direct caller sends undefined and reads an', + 'upstream 401 that names nothing, while the block path keeps working — so the', + 'break is invisible until someone calls the tool outside a workflow.', + ].join('\n') + ) + process.exit(1) +} + +main() diff --git a/scripts/check-tool-registry-boundary.baseline.json b/scripts/check-tool-registry-boundary.baseline.json index 34c38e86d06..295695c36c5 100644 --- a/scripts/check-tool-registry-boundary.baseline.json +++ b/scripts/check-tool-registry-boundary.baseline.json @@ -6,76 +6,76 @@ }, "entries": { "app/api/v2/blocks/[blockId]/route.ts": { - "modules": 1652, + "modules": 1609, "gateways": { - "apps/sim/blocks/registry.ts": 532, - "apps/sim/triggers/index.ts": 474, - "apps/sim/triggers/registry.ts": 472, - "apps/sim/lib/api/server/routes/index.ts": 364, - "apps/sim/lib/api/server/routes/internal-json-route.ts": 322, - "apps/sim/lib/auth/index.ts": 311, - "apps/sim/blocks/blocks/credential-group.ts": 189, - "apps/sim/stores/workflows/registry/store.ts": 169 + "apps/sim/triggers/index.ts": 485, + "apps/sim/triggers/registry.ts": 483, + "apps/sim/blocks/registry.ts": 448, + "apps/sim/lib/api/server/routes/index.ts": 393, + "apps/sim/lib/api/server/routes/internal-json-route.ts": 349, + "apps/sim/lib/auth/index.ts": 336, + "apps/sim/lib/webhooks/providers/index.ts": 110, + "apps/sim/lib/webhooks/providers/registry.ts": 108 } }, "app/api/v2/blocks/route.ts": { - "modules": 1651, + "modules": 1608, "gateways": { - "apps/sim/blocks/registry.ts": 532, - "apps/sim/triggers/index.ts": 474, - "apps/sim/triggers/registry.ts": 472, - "apps/sim/lib/api/server/routes/index.ts": 357, - "apps/sim/lib/api/server/routes/internal-json-route.ts": 324, - "apps/sim/lib/auth/index.ts": 313, - "apps/sim/blocks/blocks/credential-group.ts": 189, - "apps/sim/stores/workflows/registry/store.ts": 169 + "apps/sim/triggers/index.ts": 485, + "apps/sim/triggers/registry.ts": 483, + "apps/sim/blocks/registry.ts": 448, + "apps/sim/lib/api/server/routes/index.ts": 386, + "apps/sim/lib/api/server/routes/internal-json-route.ts": 351, + "apps/sim/lib/auth/index.ts": 338, + "apps/sim/lib/webhooks/providers/index.ts": 110, + "apps/sim/lib/webhooks/providers/registry.ts": 108 } }, "app/api/v2/connector-types/route.ts": { - "modules": 1714, + "modules": 1672, "gateways": { - "apps/sim/blocks/registry.ts": 532, - "apps/sim/triggers/index.ts": 474, - "apps/sim/triggers/registry.ts": 472, - "apps/sim/lib/api/server/routes/index.ts": 366, - "apps/sim/lib/api/server/routes/internal-json-route.ts": 324, - "apps/sim/lib/auth/index.ts": 313, - "apps/sim/blocks/blocks/credential-group.ts": 189, - "apps/sim/stores/workflows/registry/store.ts": 169 + "apps/sim/triggers/index.ts": 485, + "apps/sim/triggers/registry.ts": 483, + "apps/sim/blocks/registry.ts": 448, + "apps/sim/lib/api/server/routes/index.ts": 395, + "apps/sim/lib/api/server/routes/internal-json-route.ts": 351, + "apps/sim/lib/auth/index.ts": 338, + "apps/sim/lib/webhooks/providers/index.ts": 110, + "apps/sim/lib/webhooks/providers/registry.ts": 108 } }, "app/api/v2/tools/[toolId]/route.ts": { - "modules": 1649, + "modules": 1606, "gateways": { - "apps/sim/blocks/registry.ts": 532, - "apps/sim/triggers/index.ts": 474, - "apps/sim/triggers/registry.ts": 472, - "apps/sim/lib/api/server/routes/index.ts": 364, - "apps/sim/lib/api/server/routes/internal-json-route.ts": 322, - "apps/sim/lib/auth/index.ts": 311, - "apps/sim/blocks/blocks/credential-group.ts": 189, - "apps/sim/stores/workflows/registry/store.ts": 169 + "apps/sim/triggers/index.ts": 485, + "apps/sim/triggers/registry.ts": 483, + "apps/sim/blocks/registry.ts": 448, + "apps/sim/lib/api/server/routes/index.ts": 393, + "apps/sim/lib/api/server/routes/internal-json-route.ts": 349, + "apps/sim/lib/auth/index.ts": 336, + "apps/sim/lib/webhooks/providers/index.ts": 110, + "apps/sim/lib/webhooks/providers/registry.ts": 108 } }, "app/api/v2/tools/route.ts": { - "modules": 1650, + "modules": 1607, "gateways": { - "apps/sim/blocks/registry.ts": 532, - "apps/sim/triggers/index.ts": 474, - "apps/sim/triggers/registry.ts": 472, - "apps/sim/lib/api/server/routes/index.ts": 355, - "apps/sim/lib/api/server/routes/internal-json-route.ts": 322, - "apps/sim/lib/auth/index.ts": 311, - "apps/sim/blocks/blocks/credential-group.ts": 189, - "apps/sim/stores/workflows/registry/store.ts": 169 + "apps/sim/triggers/index.ts": 485, + "apps/sim/triggers/registry.ts": 483, + "apps/sim/blocks/registry.ts": 448, + "apps/sim/lib/api/server/routes/index.ts": 384, + "apps/sim/lib/api/server/routes/internal-json-route.ts": 349, + "apps/sim/lib/auth/index.ts": 336, + "apps/sim/lib/webhooks/providers/index.ts": 110, + "apps/sim/lib/webhooks/providers/registry.ts": 108 } }, "app/workspace/[workspaceId]/chat/[chatId]/error.tsx": { - "modules": 142, + "modules": 146, "gateways": { - "apps/sim/app/workspace/[workspaceId]/components/index.ts": 141, - "apps/sim/app/workspace/[workspaceId]/components/message-actions/index.ts": 107, - "apps/sim/hooks/queries/copilot-feedback.ts": 70 + "apps/sim/app/workspace/[workspaceId]/components/index.ts": 145, + "apps/sim/app/workspace/[workspaceId]/components/message-actions/index.ts": 110, + "apps/sim/hooks/queries/copilot-feedback.ts": 73 } }, "app/workspace/[workspaceId]/chat/[chatId]/layout.tsx": { @@ -83,89 +83,89 @@ "gateways": {} }, "app/workspace/[workspaceId]/chat/[chatId]/page.tsx": { - "modules": 2882, + "modules": 2872, "gateways": { - "apps/sim/app/workspace/[workspaceId]/home/home.tsx": 1261, - "apps/sim/app/workspace/[workspaceId]/home/components/mothership-view/mothership-view.tsx": 906, - "apps/sim/app/workspace/[workspaceId]/home/components/mothership-view/components/index.ts": 758, - "apps/sim/app/workspace/[workspaceId]/home/components/mothership-view/components/resource-content/index.ts": 755, - "apps/sim/triggers/registry.ts": 472, - "apps/sim/blocks/registry.ts": 318, - "apps/sim/lib/auth/index.ts": 238, - "apps/sim/app/workspace/[workspaceId]/w/[workflowId]/workflow.tsx": 198 + "apps/sim/app/workspace/[workspaceId]/home/home.tsx": 1301, + "apps/sim/app/workspace/[workspaceId]/home/components/mothership-view/mothership-view.tsx": 923, + "apps/sim/app/workspace/[workspaceId]/home/components/mothership-view/components/index.ts": 774, + "apps/sim/app/workspace/[workspaceId]/home/components/mothership-view/components/resource-content/index.ts": 771, + "apps/sim/triggers/registry.ts": 483, + "apps/sim/blocks/registry.ts": 325, + "apps/sim/lib/auth/index.ts": 248, + "apps/sim/app/workspace/[workspaceId]/w/[workflowId]/workflow.tsx": 199 } }, "app/workspace/[workspaceId]/error.tsx": { - "modules": 142, + "modules": 146, "gateways": { - "apps/sim/app/workspace/[workspaceId]/components/index.ts": 141, - "apps/sim/app/workspace/[workspaceId]/components/message-actions/index.ts": 107, - "apps/sim/hooks/queries/copilot-feedback.ts": 70 + "apps/sim/app/workspace/[workspaceId]/components/index.ts": 145, + "apps/sim/app/workspace/[workspaceId]/components/message-actions/index.ts": 110, + "apps/sim/hooks/queries/copilot-feedback.ts": 73 } }, "app/workspace/[workspaceId]/files/[fileId]/loading.tsx": { - "modules": 144, + "modules": 148, "gateways": { - "apps/sim/app/workspace/[workspaceId]/components/index.ts": 141, - "apps/sim/app/workspace/[workspaceId]/components/message-actions/index.ts": 107, - "apps/sim/hooks/queries/copilot-feedback.ts": 70 + "apps/sim/app/workspace/[workspaceId]/components/index.ts": 145, + "apps/sim/app/workspace/[workspaceId]/components/message-actions/index.ts": 110, + "apps/sim/hooks/queries/copilot-feedback.ts": 73 } }, "app/workspace/[workspaceId]/files/[fileId]/page.tsx": { - "modules": 1952, + "modules": 1945, "gateways": { - "apps/sim/triggers/registry.ts": 472, - "apps/sim/blocks/registry.ts": 341, - "apps/sim/app/workspace/[workspaceId]/files/files.tsx": 276, - "apps/sim/lib/auth/index.ts": 245, - "apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/index.ts": 154, - "apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/file-viewer.tsx": 137, - "apps/sim/lib/webhooks/providers/index.ts": 109, - "apps/sim/lib/webhooks/providers/registry.ts": 107 + "apps/sim/triggers/registry.ts": 483, + "apps/sim/blocks/registry.ts": 350, + "apps/sim/app/workspace/[workspaceId]/files/files.tsx": 305, + "apps/sim/lib/auth/index.ts": 262, + "apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/index.ts": 179, + "apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/file-viewer.tsx": 146, + "apps/sim/lib/webhooks/providers/index.ts": 110, + "apps/sim/lib/webhooks/providers/registry.ts": 108 } }, "app/workspace/[workspaceId]/files/[fileId]/view/page.tsx": { - "modules": 61, + "modules": 62, "gateways": { - "apps/sim/app/workspace/[workspaceId]/files/[fileId]/view/file-viewer.tsx": 60, - "apps/sim/hooks/queries/workspace-files.ts": 56 + "apps/sim/app/workspace/[workspaceId]/files/[fileId]/view/file-viewer.tsx": 61, + "apps/sim/hooks/queries/workspace-files.ts": 57 } }, "app/workspace/[workspaceId]/files/error.tsx": { - "modules": 142, + "modules": 146, "gateways": { - "apps/sim/app/workspace/[workspaceId]/components/index.ts": 141, - "apps/sim/app/workspace/[workspaceId]/components/message-actions/index.ts": 107, - "apps/sim/hooks/queries/copilot-feedback.ts": 70 + "apps/sim/app/workspace/[workspaceId]/components/index.ts": 145, + "apps/sim/app/workspace/[workspaceId]/components/message-actions/index.ts": 110, + "apps/sim/hooks/queries/copilot-feedback.ts": 73 } }, "app/workspace/[workspaceId]/files/loading.tsx": { - "modules": 142, + "modules": 146, "gateways": { - "apps/sim/app/workspace/[workspaceId]/components/index.ts": 141, - "apps/sim/app/workspace/[workspaceId]/components/message-actions/index.ts": 107, - "apps/sim/hooks/queries/copilot-feedback.ts": 70 + "apps/sim/app/workspace/[workspaceId]/components/index.ts": 145, + "apps/sim/app/workspace/[workspaceId]/components/message-actions/index.ts": 110, + "apps/sim/hooks/queries/copilot-feedback.ts": 73 } }, "app/workspace/[workspaceId]/files/page.tsx": { - "modules": 1952, + "modules": 1945, "gateways": { - "apps/sim/triggers/registry.ts": 472, - "apps/sim/blocks/registry.ts": 341, - "apps/sim/app/workspace/[workspaceId]/files/files.tsx": 278, - "apps/sim/lib/auth/index.ts": 245, - "apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/index.ts": 154, - "apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/file-viewer.tsx": 137, - "apps/sim/lib/webhooks/providers/index.ts": 109, - "apps/sim/lib/webhooks/providers/registry.ts": 107 + "apps/sim/triggers/registry.ts": 483, + "apps/sim/blocks/registry.ts": 350, + "apps/sim/app/workspace/[workspaceId]/files/files.tsx": 307, + "apps/sim/lib/auth/index.ts": 262, + "apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/index.ts": 179, + "apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/file-viewer.tsx": 146, + "apps/sim/lib/webhooks/providers/index.ts": 110, + "apps/sim/lib/webhooks/providers/registry.ts": 108 } }, "app/workspace/[workspaceId]/home/error.tsx": { - "modules": 142, + "modules": 146, "gateways": { - "apps/sim/app/workspace/[workspaceId]/components/index.ts": 141, - "apps/sim/app/workspace/[workspaceId]/components/message-actions/index.ts": 107, - "apps/sim/hooks/queries/copilot-feedback.ts": 70 + "apps/sim/app/workspace/[workspaceId]/components/index.ts": 145, + "apps/sim/app/workspace/[workspaceId]/components/message-actions/index.ts": 110, + "apps/sim/hooks/queries/copilot-feedback.ts": 73 } }, "app/workspace/[workspaceId]/home/layout.tsx": { @@ -173,192 +173,192 @@ "gateways": {} }, "app/workspace/[workspaceId]/home/page.tsx": { - "modules": 2882, + "modules": 2872, "gateways": { - "apps/sim/app/workspace/[workspaceId]/home/home.tsx": 1261, - "apps/sim/app/workspace/[workspaceId]/home/components/mothership-view/mothership-view.tsx": 906, - "apps/sim/app/workspace/[workspaceId]/home/components/mothership-view/components/index.ts": 758, - "apps/sim/app/workspace/[workspaceId]/home/components/mothership-view/components/resource-content/index.ts": 755, - "apps/sim/triggers/registry.ts": 472, - "apps/sim/blocks/registry.ts": 318, - "apps/sim/lib/auth/index.ts": 238, - "apps/sim/app/workspace/[workspaceId]/w/[workflowId]/workflow.tsx": 198 + "apps/sim/app/workspace/[workspaceId]/home/home.tsx": 1301, + "apps/sim/app/workspace/[workspaceId]/home/components/mothership-view/mothership-view.tsx": 923, + "apps/sim/app/workspace/[workspaceId]/home/components/mothership-view/components/index.ts": 774, + "apps/sim/app/workspace/[workspaceId]/home/components/mothership-view/components/resource-content/index.ts": 771, + "apps/sim/triggers/registry.ts": 483, + "apps/sim/blocks/registry.ts": 325, + "apps/sim/lib/auth/index.ts": 248, + "apps/sim/app/workspace/[workspaceId]/w/[workflowId]/workflow.tsx": 199 } }, "app/workspace/[workspaceId]/integrations/[block]/page.tsx": { - "modules": 1247, + "modules": 1170, "gateways": { - "apps/sim/app/workspace/[workspaceId]/integrations/[block]/integration-block-detail.tsx": 1221, - "apps/sim/triggers/index.ts": 510, - "apps/sim/triggers/registry.ts": 508, - "apps/sim/blocks/registry.ts": 491, - "apps/sim/blocks/blocks/credential-group.ts": 145, - "apps/sim/stores/workflows/registry/store.ts": 128, - "apps/sim/hooks/queries/deployments.ts": 121, - "apps/sim/lib/workflows/comparison/describe.ts": 111 + "apps/sim/app/workspace/[workspaceId]/integrations/[block]/integration-block-detail.tsx": 1141, + "apps/sim/triggers/index.ts": 521, + "apps/sim/triggers/registry.ts": 519, + "apps/sim/blocks/registry.ts": 398, + "apps/sim/app/workspace/[workspaceId]/integrations/[block]/integration-skills-section.tsx": 60, + "apps/sim/app/workspace/[workspaceId]/components/index.ts": 57, + "apps/sim/blocks/blocks/credential-group.ts": 46, + "apps/sim/lib/api/contracts/index.ts": 36 } }, "app/workspace/[workspaceId]/integrations/connected/[credentialId]/page.tsx": { - "modules": 1225, + "modules": 1148, "gateways": { - "apps/sim/app/workspace/[workspaceId]/integrations/connected/[credentialId]/connected-credential-detail.tsx": 1224, - "apps/sim/triggers/registry.ts": 508, - "apps/sim/blocks/registry.ts": 349, - "apps/sim/stores/workflows/registry/store.ts": 124, - "apps/sim/hooks/queries/deployments.ts": 121, - "apps/sim/lib/workflows/comparison/describe.ts": 111, - "apps/sim/hooks/selectors/registry.ts": 106, - "apps/sim/app/workspace/[workspaceId]/components/credential-detail/index.ts": 54 + "apps/sim/app/workspace/[workspaceId]/integrations/connected/[credentialId]/connected-credential-detail.tsx": 1147, + "apps/sim/triggers/registry.ts": 519, + "apps/sim/blocks/registry.ts": 366, + "apps/sim/app/workspace/[workspaceId]/components/credential-detail/index.ts": 57, + "apps/sim/components/permissions/index.ts": 44, + "apps/sim/lib/api/contracts/index.ts": 42, + "apps/sim/components/permissions/add-people-modal.tsx": 35, + "apps/sim/app/workspace/[workspaceId]/providers/workspace-permissions-provider.tsx": 33 } }, "app/workspace/[workspaceId]/integrations/error.tsx": { - "modules": 142, + "modules": 146, "gateways": { - "apps/sim/app/workspace/[workspaceId]/components/index.ts": 141, - "apps/sim/app/workspace/[workspaceId]/components/message-actions/index.ts": 107, - "apps/sim/hooks/queries/copilot-feedback.ts": 70 + "apps/sim/app/workspace/[workspaceId]/components/index.ts": 145, + "apps/sim/app/workspace/[workspaceId]/components/message-actions/index.ts": 110, + "apps/sim/hooks/queries/copilot-feedback.ts": 73 } }, "app/workspace/[workspaceId]/integrations/page.tsx": { - "modules": 1232, + "modules": 1153, "gateways": { - "apps/sim/app/workspace/[workspaceId]/integrations/integrations.tsx": 1090, - "apps/sim/blocks/registry.ts": 1005, - "apps/sim/triggers/index.ts": 510, - "apps/sim/triggers/registry.ts": 508, - "apps/sim/blocks/blocks/credential-group.ts": 146, - "apps/sim/stores/workflows/registry/store.ts": 128, - "apps/sim/hooks/queries/deployments.ts": 121, - "apps/sim/lib/workflows/comparison/describe.ts": 111 + "apps/sim/app/workspace/[workspaceId]/integrations/integrations.tsx": 1007, + "apps/sim/blocks/registry.ts": 923, + "apps/sim/triggers/index.ts": 521, + "apps/sim/triggers/registry.ts": 519, + "apps/sim/app/workspace/[workspaceId]/components/index.ts": 58, + "apps/sim/blocks/blocks/credential-group.ts": 47, + "apps/sim/lib/api/contracts/index.ts": 38, + "apps/sim/triggers/clickup/index.ts": 32 } }, "app/workspace/[workspaceId]/knowledge/[id]/[documentId]/loading.tsx": { - "modules": 144, + "modules": 148, "gateways": { - "apps/sim/app/workspace/[workspaceId]/components/index.ts": 141, - "apps/sim/app/workspace/[workspaceId]/components/message-actions/index.ts": 107, - "apps/sim/hooks/queries/copilot-feedback.ts": 70 + "apps/sim/app/workspace/[workspaceId]/components/index.ts": 145, + "apps/sim/app/workspace/[workspaceId]/components/message-actions/index.ts": 110, + "apps/sim/hooks/queries/copilot-feedback.ts": 73 } }, "app/workspace/[workspaceId]/knowledge/[id]/[documentId]/page.tsx": { - "modules": 1432, + "modules": 1367, "gateways": { - "apps/sim/app/workspace/[workspaceId]/knowledge/[id]/[documentId]/document.tsx": 1287, - "apps/sim/triggers/registry.ts": 508, - "apps/sim/blocks/registry.ts": 344, - "apps/sim/blocks/registry-maps.ts": 341, - "apps/sim/hooks/selectors/registry.ts": 91, - "apps/sim/connectors/registry.ts": 65, - "apps/sim/app/workspace/[workspaceId]/components/index.ts": 60, - "apps/sim/app/workspace/[workspaceId]/knowledge/[id]/components/index.ts": 51 + "apps/sim/app/workspace/[workspaceId]/knowledge/[id]/[documentId]/document.tsx": 1218, + "apps/sim/triggers/registry.ts": 519, + "apps/sim/blocks/registry.ts": 358, + "apps/sim/blocks/registry-maps.ts": 355, + "apps/sim/connectors/registry.ts": 66, + "apps/sim/app/workspace/[workspaceId]/components/index.ts": 59, + "apps/sim/app/workspace/[workspaceId]/knowledge/[id]/components/index.ts": 50, + "apps/sim/lib/api/contracts/index.ts": 40 } }, "app/workspace/[workspaceId]/knowledge/[id]/error.tsx": { - "modules": 142, + "modules": 146, "gateways": { - "apps/sim/app/workspace/[workspaceId]/components/index.ts": 141, - "apps/sim/app/workspace/[workspaceId]/components/message-actions/index.ts": 107, - "apps/sim/hooks/queries/copilot-feedback.ts": 70 + "apps/sim/app/workspace/[workspaceId]/components/index.ts": 145, + "apps/sim/app/workspace/[workspaceId]/components/message-actions/index.ts": 110, + "apps/sim/hooks/queries/copilot-feedback.ts": 73 } }, "app/workspace/[workspaceId]/knowledge/[id]/loading.tsx": { - "modules": 145, + "modules": 149, "gateways": { - "apps/sim/app/workspace/[workspaceId]/components/index.ts": 141, - "apps/sim/app/workspace/[workspaceId]/components/message-actions/index.ts": 107, - "apps/sim/hooks/queries/copilot-feedback.ts": 70 + "apps/sim/app/workspace/[workspaceId]/components/index.ts": 145, + "apps/sim/app/workspace/[workspaceId]/components/message-actions/index.ts": 110, + "apps/sim/hooks/queries/copilot-feedback.ts": 73 } }, "app/workspace/[workspaceId]/knowledge/[id]/page.tsx": { - "modules": 1435, + "modules": 1370, "gateways": { - "apps/sim/app/workspace/[workspaceId]/knowledge/[id]/base.tsx": 1289, - "apps/sim/triggers/registry.ts": 508, - "apps/sim/blocks/registry.ts": 349, - "apps/sim/blocks/registry-maps.ts": 346, - "apps/sim/hooks/selectors/registry.ts": 91, - "apps/sim/connectors/registry.ts": 65, - "apps/sim/app/workspace/[workspaceId]/components/index.ts": 60, - "apps/sim/app/workspace/[workspaceId]/knowledge/[id]/components/index.ts": 44 + "apps/sim/app/workspace/[workspaceId]/knowledge/[id]/base.tsx": 1220, + "apps/sim/triggers/registry.ts": 519, + "apps/sim/blocks/registry.ts": 366, + "apps/sim/blocks/registry-maps.ts": 363, + "apps/sim/connectors/registry.ts": 66, + "apps/sim/app/workspace/[workspaceId]/components/index.ts": 59, + "apps/sim/app/workspace/[workspaceId]/knowledge/[id]/components/index.ts": 43, + "apps/sim/lib/api/contracts/index.ts": 40 } }, "app/workspace/[workspaceId]/knowledge/error.tsx": { - "modules": 142, + "modules": 146, "gateways": { - "apps/sim/app/workspace/[workspaceId]/components/index.ts": 141, - "apps/sim/app/workspace/[workspaceId]/components/message-actions/index.ts": 107, - "apps/sim/hooks/queries/copilot-feedback.ts": 70 + "apps/sim/app/workspace/[workspaceId]/components/index.ts": 145, + "apps/sim/app/workspace/[workspaceId]/components/message-actions/index.ts": 110, + "apps/sim/hooks/queries/copilot-feedback.ts": 73 } }, "app/workspace/[workspaceId]/knowledge/loading.tsx": { - "modules": 144, + "modules": 148, "gateways": { - "apps/sim/app/workspace/[workspaceId]/components/index.ts": 141, - "apps/sim/app/workspace/[workspaceId]/components/message-actions/index.ts": 107, - "apps/sim/hooks/queries/copilot-feedback.ts": 70 + "apps/sim/app/workspace/[workspaceId]/components/index.ts": 145, + "apps/sim/app/workspace/[workspaceId]/components/message-actions/index.ts": 110, + "apps/sim/hooks/queries/copilot-feedback.ts": 73 } }, "app/workspace/[workspaceId]/knowledge/page.tsx": { - "modules": 2173, + "modules": 2158, "gateways": { - "apps/sim/triggers/registry.ts": 472, - "apps/sim/blocks/registry.ts": 339, - "apps/sim/app/workspace/[workspaceId]/knowledge/prefetch.ts": 290, - "apps/sim/lib/knowledge/application/knowledge-bases.ts": 234, - "apps/sim/lib/auth/index.ts": 193, - "apps/sim/app/workspace/[workspaceId]/knowledge/knowledge.tsx": 154, - "apps/sim/lib/knowledge/orchestration/index.ts": 145, - "apps/sim/lib/knowledge/orchestration/connectors.ts": 141 + "apps/sim/triggers/registry.ts": 483, + "apps/sim/blocks/registry.ts": 350, + "apps/sim/app/workspace/[workspaceId]/knowledge/prefetch.ts": 305, + "apps/sim/lib/knowledge/application/knowledge-bases.ts": 249, + "apps/sim/lib/auth/index.ts": 204, + "apps/sim/app/workspace/[workspaceId]/knowledge/knowledge.tsx": 161, + "apps/sim/lib/knowledge/orchestration/index.ts": 148, + "apps/sim/lib/knowledge/orchestration/connectors.ts": 144 } }, "app/workspace/[workspaceId]/layout.tsx": { - "modules": 2033, + "modules": 2034, "gateways": { - "apps/sim/triggers/registry.ts": 472, - "apps/sim/blocks/registry.ts": 340, - "apps/sim/lib/auth/index.ts": 317, - "apps/sim/app/workspace/[workspaceId]/components/workspace-chrome/index.ts": 309, - "apps/sim/app/workspace/[workspaceId]/w/components/sidebar/sidebar.tsx": 304, - "apps/sim/app/workspace/[workspaceId]/w/components/sidebar/components/index.ts": 209, - "apps/sim/lib/webhooks/providers/index.ts": 109, - "apps/sim/lib/webhooks/providers/registry.ts": 107 + "apps/sim/triggers/registry.ts": 483, + "apps/sim/blocks/registry.ts": 349, + "apps/sim/app/workspace/[workspaceId]/components/workspace-chrome/index.ts": 340, + "apps/sim/lib/auth/index.ts": 340, + "apps/sim/app/workspace/[workspaceId]/w/components/sidebar/sidebar.tsx": 335, + "apps/sim/app/workspace/[workspaceId]/w/components/sidebar/components/index.ts": 229, + "apps/sim/lib/webhooks/providers/index.ts": 110, + "apps/sim/lib/webhooks/providers/registry.ts": 108 } }, "app/workspace/[workspaceId]/logs/error.tsx": { - "modules": 142, + "modules": 146, "gateways": { - "apps/sim/app/workspace/[workspaceId]/components/index.ts": 141, - "apps/sim/app/workspace/[workspaceId]/components/message-actions/index.ts": 107, - "apps/sim/hooks/queries/copilot-feedback.ts": 70 + "apps/sim/app/workspace/[workspaceId]/components/index.ts": 145, + "apps/sim/app/workspace/[workspaceId]/components/message-actions/index.ts": 110, + "apps/sim/hooks/queries/copilot-feedback.ts": 73 } }, "app/workspace/[workspaceId]/logs/loading.tsx": { - "modules": 142, + "modules": 146, "gateways": { - "apps/sim/app/workspace/[workspaceId]/components/index.ts": 141, - "apps/sim/app/workspace/[workspaceId]/components/message-actions/index.ts": 107, - "apps/sim/hooks/queries/copilot-feedback.ts": 70 + "apps/sim/app/workspace/[workspaceId]/components/index.ts": 145, + "apps/sim/app/workspace/[workspaceId]/components/message-actions/index.ts": 110, + "apps/sim/hooks/queries/copilot-feedback.ts": 73 } }, "app/workspace/[workspaceId]/logs/page.tsx": { - "modules": 1684, + "modules": 1663, "gateways": { - "apps/sim/app/workspace/[workspaceId]/logs/logs.tsx": 1541, - "apps/sim/triggers/registry.ts": 508, - "apps/sim/app/workspace/[workspaceId]/logs/components/log-details/components/execution-snapshot/execution-snapshot.tsx": 362, - "apps/sim/blocks/registry.ts": 335, - "apps/sim/app/workspace/[workspaceId]/w/components/preview/components/preview-editor/index.ts": 316, - "apps/sim/app/workspace/[workspaceId]/w/[workflowId]/components/panel/components/editor/components/index.ts": 310, - "apps/sim/app/workspace/[workspaceId]/w/[workflowId]/components/panel/components/editor/components/sub-block/sub-block.tsx": 272, - "apps/sim/app/workspace/[workspaceId]/w/[workflowId]/components/panel/components/editor/components/sub-block/components/index.ts": 261 + "apps/sim/app/workspace/[workspaceId]/logs/logs.tsx": 1516, + "apps/sim/triggers/registry.ts": 519, + "apps/sim/app/workspace/[workspaceId]/logs/components/log-details/components/execution-snapshot/execution-snapshot.tsx": 410, + "apps/sim/app/workspace/[workspaceId]/w/components/preview/components/preview-editor/index.ts": 361, + "apps/sim/app/workspace/[workspaceId]/w/[workflowId]/components/panel/components/editor/components/index.ts": 355, + "apps/sim/blocks/registry.ts": 345, + "apps/sim/app/workspace/[workspaceId]/w/[workflowId]/components/panel/components/editor/components/sub-block/sub-block.tsx": 318, + "apps/sim/app/workspace/[workspaceId]/w/[workflowId]/components/panel/components/editor/components/sub-block/components/index.ts": 307 } }, "app/workspace/[workspaceId]/not-found.tsx": { - "modules": 142, + "modules": 146, "gateways": { - "apps/sim/app/workspace/[workspaceId]/components/index.ts": 141, - "apps/sim/app/workspace/[workspaceId]/components/message-actions/index.ts": 107, - "apps/sim/hooks/queries/copilot-feedback.ts": 70 + "apps/sim/app/workspace/[workspaceId]/components/index.ts": 145, + "apps/sim/app/workspace/[workspaceId]/components/message-actions/index.ts": 110, + "apps/sim/hooks/queries/copilot-feedback.ts": 73 } }, "app/workspace/[workspaceId]/page.tsx": { @@ -366,11 +366,11 @@ "gateways": {} }, "app/workspace/[workspaceId]/settings/[section]/error.tsx": { - "modules": 142, + "modules": 146, "gateways": { - "apps/sim/app/workspace/[workspaceId]/components/index.ts": 141, - "apps/sim/app/workspace/[workspaceId]/components/message-actions/index.ts": 107, - "apps/sim/hooks/queries/copilot-feedback.ts": 70 + "apps/sim/app/workspace/[workspaceId]/components/index.ts": 145, + "apps/sim/app/workspace/[workspaceId]/components/message-actions/index.ts": 110, + "apps/sim/hooks/queries/copilot-feedback.ts": 73 } }, "app/workspace/[workspaceId]/settings/[section]/layout.tsx": { @@ -382,16 +382,16 @@ "gateways": {} }, "app/workspace/[workspaceId]/settings/[section]/page.tsx": { - "modules": 2130, + "modules": 2199, "gateways": { - "apps/sim/triggers/registry.ts": 472, - "apps/sim/app/workspace/[workspaceId]/settings/[section]/settings.tsx": 470, - "apps/sim/blocks/registry.ts": 342, - "apps/sim/lib/auth/index.ts": 299, - "apps/sim/lib/webhooks/providers/index.ts": 109, - "apps/sim/lib/webhooks/providers/registry.ts": 107, - "apps/sim/hooks/selectors/registry.ts": 71, - "apps/sim/app/workspace/[workspaceId]/settings/[section]/prefetch.ts": 51 + "apps/sim/app/workspace/[workspaceId]/settings/[section]/settings.tsx": 576, + "apps/sim/triggers/registry.ts": 483, + "apps/sim/blocks/registry.ts": 345, + "apps/sim/lib/auth/index.ts": 309, + "apps/sim/lib/webhooks/providers/index.ts": 110, + "apps/sim/lib/webhooks/providers/registry.ts": 108, + "apps/sim/ee/access-control/components/access-control.tsx": 74, + "apps/sim/ee/access-control/components/group-detail.tsx": 72 } }, "app/workspace/[workspaceId]/settings/billing/credit-usage/layout.tsx": { @@ -403,24 +403,24 @@ "gateways": {} }, "app/workspace/[workspaceId]/settings/billing/credit-usage/page.tsx": { - "modules": 1604, + "modules": 1563, "gateways": { - "apps/sim/lib/auth/index.ts": 1467, - "apps/sim/blocks/registry.ts": 530, - "apps/sim/blocks/registry-maps.ts": 527, - "apps/sim/triggers/index.ts": 474, - "apps/sim/triggers/registry.ts": 472, - "apps/sim/blocks/blocks/credential-group.ts": 185, - "apps/sim/stores/workflows/registry/store.ts": 168, - "apps/sim/hooks/queries/deployments.ts": 160 + "apps/sim/lib/auth/index.ts": 1426, + "apps/sim/triggers/index.ts": 485, + "apps/sim/triggers/registry.ts": 483, + "apps/sim/blocks/registry.ts": 445, + "apps/sim/blocks/registry-maps.ts": 442, + "apps/sim/lib/webhooks/providers/index.ts": 110, + "apps/sim/lib/webhooks/providers/registry.ts": 108, + "apps/sim/blocks/blocks/credential-group.ts": 95 } }, "app/workspace/[workspaceId]/settings/error.tsx": { - "modules": 142, + "modules": 146, "gateways": { - "apps/sim/app/workspace/[workspaceId]/components/index.ts": 141, - "apps/sim/app/workspace/[workspaceId]/components/message-actions/index.ts": 107, - "apps/sim/hooks/queries/copilot-feedback.ts": 70 + "apps/sim/app/workspace/[workspaceId]/components/index.ts": 145, + "apps/sim/app/workspace/[workspaceId]/components/message-actions/index.ts": 110, + "apps/sim/hooks/queries/copilot-feedback.ts": 73 } }, "app/workspace/[workspaceId]/settings/layout.tsx": { @@ -432,29 +432,29 @@ "gateways": {} }, "app/workspace/[workspaceId]/settings/secrets/[credentialId]/loading.tsx": { - "modules": 1194, + "modules": 1116, "gateways": { - "apps/sim/app/workspace/[workspaceId]/components/credential-detail/index.ts": 1191, - "apps/sim/components/permissions/index.ts": 1074, - "apps/sim/components/permissions/add-people-modal.tsx": 1065, - "apps/sim/app/workspace/[workspaceId]/providers/workspace-permissions-provider.tsx": 1063, - "apps/sim/triggers/registry.ts": 508, - "apps/sim/blocks/registry.ts": 351, - "apps/sim/blocks/registry-maps.ts": 348, - "apps/sim/stores/workflows/registry/store.ts": 124 + "apps/sim/app/workspace/[workspaceId]/components/credential-detail/index.ts": 1113, + "apps/sim/components/permissions/index.ts": 989, + "apps/sim/components/permissions/add-people-modal.tsx": 980, + "apps/sim/app/workspace/[workspaceId]/providers/workspace-permissions-provider.tsx": 978, + "apps/sim/triggers/registry.ts": 519, + "apps/sim/blocks/registry.ts": 368, + "apps/sim/blocks/registry-maps.ts": 365, + "apps/sim/lib/api/contracts/index.ts": 47 } }, "app/workspace/[workspaceId]/settings/secrets/[credentialId]/page.tsx": { - "modules": 1272, + "modules": 1194, "gateways": { - "apps/sim/triggers/registry.ts": 508, - "apps/sim/blocks/registry.ts": 351, - "apps/sim/blocks/registry-maps.ts": 348, - "apps/sim/stores/workflows/registry/store.ts": 123, - "apps/sim/hooks/queries/deployments.ts": 120, - "apps/sim/lib/workflows/comparison/describe.ts": 110, - "apps/sim/hooks/selectors/registry.ts": 105, - "apps/sim/app/workspace/[workspaceId]/settings/secrets/[credentialId]/secret-detail.tsx": 77 + "apps/sim/triggers/registry.ts": 519, + "apps/sim/blocks/registry.ts": 368, + "apps/sim/blocks/registry-maps.ts": 365, + "apps/sim/app/workspace/[workspaceId]/settings/secrets/[credentialId]/secret-detail.tsx": 77, + "apps/sim/app/workspace/[workspaceId]/components/index.ts": 56, + "apps/sim/app/workspace/[workspaceId]/components/credential-detail/index.ts": 54, + "apps/sim/components/permissions/index.ts": 43, + "apps/sim/lib/api/contracts/index.ts": 43 } }, "app/workspace/[workspaceId]/settings/usage/events/layout.tsx": { @@ -466,219 +466,219 @@ "gateways": {} }, "app/workspace/[workspaceId]/settings/usage/events/page.tsx": { - "modules": 1626, + "modules": 1572, "gateways": { - "apps/sim/lib/auth/index.ts": 1482, - "apps/sim/blocks/registry.ts": 535, - "apps/sim/blocks/registry-maps.ts": 532, - "apps/sim/triggers/index.ts": 474, - "apps/sim/triggers/registry.ts": 472, - "apps/sim/blocks/blocks/credential-group.ts": 187, - "apps/sim/stores/workflows/registry/store.ts": 169, - "apps/sim/hooks/queries/deployments.ts": 161 + "apps/sim/lib/auth/index.ts": 1428, + "apps/sim/triggers/index.ts": 485, + "apps/sim/triggers/registry.ts": 483, + "apps/sim/blocks/registry.ts": 447, + "apps/sim/blocks/registry-maps.ts": 444, + "apps/sim/lib/webhooks/providers/index.ts": 110, + "apps/sim/lib/webhooks/providers/registry.ts": 108, + "apps/sim/blocks/blocks/credential-group.ts": 97 } }, "app/workspace/[workspaceId]/skills/[skillId]/page.tsx": { - "modules": 1354, + "modules": 1311, "gateways": { - "apps/sim/app/workspace/[workspaceId]/skills/[skillId]/skill-detail.tsx": 1353, - "apps/sim/triggers/registry.ts": 508, - "apps/sim/blocks/registry.ts": 349, - "apps/sim/blocks/registry-maps.ts": 347, - "apps/sim/app/workspace/[workspaceId]/skills/components/skill-fields/index.ts": 95, - "apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/rich-markdown-field.tsx": 92, - "apps/sim/hooks/queries/deployments.ts": 90, - "apps/sim/lib/workflows/comparison/describe.ts": 83 + "apps/sim/app/workspace/[workspaceId]/skills/[skillId]/skill-detail.tsx": 1310, + "apps/sim/triggers/registry.ts": 519, + "apps/sim/blocks/registry.ts": 362, + "apps/sim/blocks/registry-maps.ts": 360, + "apps/sim/app/workspace/[workspaceId]/skills/components/skill-fields/index.ts": 131, + "apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/rich-markdown-field.tsx": 128, + "apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/mention/index.ts": 64, + "apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/mention/use-markdown-mentions.ts": 62 } }, "app/workspace/[workspaceId]/skills/error.tsx": { - "modules": 142, + "modules": 146, "gateways": { - "apps/sim/app/workspace/[workspaceId]/components/index.ts": 141, - "apps/sim/app/workspace/[workspaceId]/components/message-actions/index.ts": 107, - "apps/sim/hooks/queries/copilot-feedback.ts": 70 + "apps/sim/app/workspace/[workspaceId]/components/index.ts": 145, + "apps/sim/app/workspace/[workspaceId]/components/message-actions/index.ts": 110, + "apps/sim/hooks/queries/copilot-feedback.ts": 73 } }, "app/workspace/[workspaceId]/skills/new/page.tsx": { - "modules": 1352, + "modules": 1309, "gateways": { - "apps/sim/app/workspace/[workspaceId]/skills/new/skill-create.tsx": 1351, - "apps/sim/triggers/registry.ts": 508, - "apps/sim/blocks/registry.ts": 349, - "apps/sim/blocks/registry-maps.ts": 347, - "apps/sim/app/workspace/[workspaceId]/skills/components/skill-fields/index.ts": 95, - "apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/rich-markdown-field.tsx": 92, - "apps/sim/hooks/queries/deployments.ts": 90, - "apps/sim/lib/workflows/comparison/describe.ts": 83 + "apps/sim/app/workspace/[workspaceId]/skills/new/skill-create.tsx": 1308, + "apps/sim/triggers/registry.ts": 519, + "apps/sim/blocks/registry.ts": 362, + "apps/sim/blocks/registry-maps.ts": 360, + "apps/sim/app/workspace/[workspaceId]/skills/components/skill-fields/index.ts": 131, + "apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/rich-markdown-field.tsx": 128, + "apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/mention/index.ts": 64, + "apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/mention/use-markdown-mentions.ts": 62 } }, "app/workspace/[workspaceId]/skills/page.tsx": { - "modules": 1215, + "modules": 1133, "gateways": { - "apps/sim/app/workspace/[workspaceId]/skills/skills.tsx": 1073, - "apps/sim/app/workspace/[workspaceId]/integrations/components/showcase-with-explore/index.ts": 1061, - "apps/sim/blocks/registry.ts": 1049, - "apps/sim/blocks/registry-maps.ts": 1047, - "apps/sim/triggers/index.ts": 510, - "apps/sim/triggers/registry.ts": 508, - "apps/sim/blocks/blocks/credential-group.ts": 160, - "apps/sim/stores/workflows/registry/store.ts": 140 + "apps/sim/app/workspace/[workspaceId]/skills/skills.tsx": 987, + "apps/sim/app/workspace/[workspaceId]/integrations/components/showcase-with-explore/index.ts": 975, + "apps/sim/blocks/registry.ts": 966, + "apps/sim/blocks/registry-maps.ts": 964, + "apps/sim/triggers/index.ts": 521, + "apps/sim/triggers/registry.ts": 519, + "apps/sim/blocks/blocks/credential-group.ts": 62, + "apps/sim/app/workspace/[workspaceId]/components/index.ts": 58 } }, "app/workspace/[workspaceId]/tables/[tableId]/error.tsx": { - "modules": 142, + "modules": 146, "gateways": { - "apps/sim/app/workspace/[workspaceId]/components/index.ts": 141, - "apps/sim/app/workspace/[workspaceId]/components/message-actions/index.ts": 107, - "apps/sim/hooks/queries/copilot-feedback.ts": 70 + "apps/sim/app/workspace/[workspaceId]/components/index.ts": 145, + "apps/sim/app/workspace/[workspaceId]/components/message-actions/index.ts": 110, + "apps/sim/hooks/queries/copilot-feedback.ts": 73 } }, "app/workspace/[workspaceId]/tables/[tableId]/loading.tsx": { - "modules": 142, + "modules": 146, "gateways": { - "apps/sim/app/workspace/[workspaceId]/components/index.ts": 141, - "apps/sim/app/workspace/[workspaceId]/components/message-actions/index.ts": 107, - "apps/sim/hooks/queries/copilot-feedback.ts": 70 + "apps/sim/app/workspace/[workspaceId]/components/index.ts": 145, + "apps/sim/app/workspace/[workspaceId]/components/message-actions/index.ts": 110, + "apps/sim/hooks/queries/copilot-feedback.ts": 73 } }, "app/workspace/[workspaceId]/tables/[tableId]/page.tsx": { - "modules": 1803, + "modules": 1768, "gateways": { - "apps/sim/app/workspace/[workspaceId]/tables/[tableId]/table.tsx": 1659, - "apps/sim/triggers/registry.ts": 508, - "apps/sim/app/workspace/[workspaceId]/w/components/preview/index.ts": 332, - "apps/sim/blocks/registry.ts": 319, - "apps/sim/app/workspace/[workspaceId]/w/components/preview/components/preview-editor/index.ts": 286, - "apps/sim/app/workspace/[workspaceId]/w/[workflowId]/components/panel/components/editor/components/index.ts": 282, - "apps/sim/app/workspace/[workspaceId]/w/[workflowId]/components/panel/components/editor/components/sub-block/sub-block.tsx": 249, - "apps/sim/app/workspace/[workspaceId]/w/[workflowId]/components/panel/components/editor/components/sub-block/components/index.ts": 238 + "apps/sim/app/workspace/[workspaceId]/tables/[tableId]/table.tsx": 1621, + "apps/sim/triggers/registry.ts": 519, + "apps/sim/app/workspace/[workspaceId]/w/components/preview/index.ts": 342, + "apps/sim/blocks/registry.ts": 326, + "apps/sim/app/workspace/[workspaceId]/w/components/preview/components/preview-editor/index.ts": 295, + "apps/sim/app/workspace/[workspaceId]/w/[workflowId]/components/panel/components/editor/components/index.ts": 291, + "apps/sim/app/workspace/[workspaceId]/w/[workflowId]/components/panel/components/editor/components/sub-block/sub-block.tsx": 259, + "apps/sim/app/workspace/[workspaceId]/w/[workflowId]/components/panel/components/editor/components/sub-block/components/index.ts": 248 } }, "app/workspace/[workspaceId]/tables/error.tsx": { - "modules": 142, + "modules": 146, "gateways": { - "apps/sim/app/workspace/[workspaceId]/components/index.ts": 141, - "apps/sim/app/workspace/[workspaceId]/components/message-actions/index.ts": 107, - "apps/sim/hooks/queries/copilot-feedback.ts": 70 + "apps/sim/app/workspace/[workspaceId]/components/index.ts": 145, + "apps/sim/app/workspace/[workspaceId]/components/message-actions/index.ts": 110, + "apps/sim/hooks/queries/copilot-feedback.ts": 73 } }, "app/workspace/[workspaceId]/tables/loading.tsx": { - "modules": 142, + "modules": 146, "gateways": { - "apps/sim/app/workspace/[workspaceId]/components/index.ts": 141, - "apps/sim/app/workspace/[workspaceId]/components/message-actions/index.ts": 107, - "apps/sim/hooks/queries/copilot-feedback.ts": 70 + "apps/sim/app/workspace/[workspaceId]/components/index.ts": 145, + "apps/sim/app/workspace/[workspaceId]/components/message-actions/index.ts": 110, + "apps/sim/hooks/queries/copilot-feedback.ts": 73 } }, "app/workspace/[workspaceId]/tables/page.tsx": { - "modules": 1792, + "modules": 1778, "gateways": { - "apps/sim/triggers/registry.ts": 472, - "apps/sim/blocks/registry.ts": 341, - "apps/sim/lib/auth/index.ts": 339, - "apps/sim/lib/webhooks/providers/index.ts": 109, - "apps/sim/lib/webhooks/providers/registry.ts": 107, - "apps/sim/app/workspace/[workspaceId]/tables/tables.tsx": 106, - "apps/sim/stores/workflows/registry/store.ts": 98, - "apps/sim/lib/workflows/comparison/describe.ts": 88 + "apps/sim/triggers/registry.ts": 483, + "apps/sim/lib/auth/index.ts": 369, + "apps/sim/blocks/registry.ts": 350, + "apps/sim/app/workspace/[workspaceId]/tables/tables.tsx": 125, + "apps/sim/lib/webhooks/providers/index.ts": 110, + "apps/sim/lib/webhooks/providers/registry.ts": 108, + "apps/sim/app/workspace/[workspaceId]/components/index.ts": 59, + "apps/sim/lib/workflows/lifecycle.ts": 49 } }, "app/workspace/[workspaceId]/upgrade/page.tsx": { - "modules": 132, + "modules": 136, "gateways": { - "apps/sim/app/workspace/[workspaceId]/upgrade/upgrade.tsx": 125, - "apps/sim/app/workspace/[workspaceId]/upgrade/hooks/index.ts": 77, - "apps/sim/lib/billing/client/upgrade.ts": 69, - "apps/sim/hooks/queries/organization.ts": 65, - "apps/sim/hooks/queries/workspace.ts": 56, - "apps/sim/lib/api/contracts/index.ts": 54 + "apps/sim/app/workspace/[workspaceId]/upgrade/upgrade.tsx": 129, + "apps/sim/app/workspace/[workspaceId]/upgrade/hooks/index.ts": 81, + "apps/sim/lib/billing/client/upgrade.ts": 73, + "apps/sim/hooks/queries/organization.ts": 69, + "apps/sim/hooks/queries/workspace.ts": 59, + "apps/sim/lib/api/contracts/index.ts": 57 } }, "app/workspace/[workspaceId]/w/[workflowId]/layout.tsx": { - "modules": 145, + "modules": 148, "gateways": { - "apps/sim/app/workspace/[workspaceId]/w/[workflowId]/components/error/index.tsx": 144, - "apps/sim/app/workspace/[workspaceId]/components/index.ts": 142, - "apps/sim/app/workspace/[workspaceId]/components/message-actions/index.ts": 107, - "apps/sim/hooks/queries/copilot-feedback.ts": 70 + "apps/sim/app/workspace/[workspaceId]/w/[workflowId]/components/error/index.tsx": 147, + "apps/sim/app/workspace/[workspaceId]/components/index.ts": 145, + "apps/sim/app/workspace/[workspaceId]/components/message-actions/index.ts": 110, + "apps/sim/hooks/queries/copilot-feedback.ts": 73 } }, "app/workspace/[workspaceId]/w/[workflowId]/page.tsx": { - "modules": 2053, + "modules": 2018, "gateways": { - "apps/sim/app/workspace/[workspaceId]/w/[workflowId]/workflow.tsx": 2052, - "apps/sim/triggers/registry.ts": 508, - "apps/sim/app/workspace/[workspaceId]/w/[workflowId]/components/index.ts": 344, - "apps/sim/blocks/registry.ts": 338, - "apps/sim/app/workspace/[workspaceId]/w/[workflowId]/components/panel/index.ts": 307, - "apps/sim/app/workspace/[workspaceId]/w/[workflowId]/components/panel/components/index.ts": 245, - "apps/sim/app/workspace/[workspaceId]/w/[workflowId]/components/panel/components/editor/components/index.ts": 151, - "apps/sim/app/workspace/[workspaceId]/w/[workflowId]/components/panel/components/editor/components/sub-block/sub-block.tsx": 144 + "apps/sim/app/workspace/[workspaceId]/w/[workflowId]/workflow.tsx": 2017, + "apps/sim/triggers/registry.ts": 519, + "apps/sim/app/workspace/[workspaceId]/w/[workflowId]/components/index.ts": 351, + "apps/sim/blocks/registry.ts": 345, + "apps/sim/app/workspace/[workspaceId]/w/[workflowId]/components/panel/index.ts": 314, + "apps/sim/app/workspace/[workspaceId]/w/[workflowId]/components/panel/components/index.ts": 251, + "apps/sim/app/workspace/[workspaceId]/w/[workflowId]/components/panel/components/editor/components/index.ts": 155, + "apps/sim/app/workspace/[workspaceId]/w/[workflowId]/components/panel/components/editor/components/sub-block/sub-block.tsx": 148 } }, "app/workspace/[workspaceId]/w/page.tsx": { - "modules": 2035, + "modules": 2003, "gateways": { - "apps/sim/app/workspace/[workspaceId]/w/[workflowId]/components/index.ts": 818, - "apps/sim/app/workspace/[workspaceId]/w/[workflowId]/components/panel/index.ts": 542, - "apps/sim/triggers/registry.ts": 508, - "apps/sim/blocks/registry.ts": 338, - "apps/sim/app/workspace/[workspaceId]/w/[workflowId]/components/panel/components/index.ts": 310, - "apps/sim/app/workspace/[workspaceId]/w/[workflowId]/components/panel/components/editor/components/index.ts": 153, - "apps/sim/app/workspace/[workspaceId]/w/[workflowId]/components/panel/components/editor/components/sub-block/sub-block.tsx": 146, - "apps/sim/app/workspace/[workspaceId]/w/[workflowId]/components/panel/components/editor/components/sub-block/components/index.ts": 140 + "apps/sim/app/workspace/[workspaceId]/w/[workflowId]/components/index.ts": 848, + "apps/sim/app/workspace/[workspaceId]/w/[workflowId]/components/panel/index.ts": 554, + "apps/sim/triggers/registry.ts": 519, + "apps/sim/blocks/registry.ts": 345, + "apps/sim/app/workspace/[workspaceId]/w/[workflowId]/components/panel/components/index.ts": 317, + "apps/sim/app/workspace/[workspaceId]/w/[workflowId]/components/panel/components/editor/components/index.ts": 157, + "apps/sim/app/workspace/[workspaceId]/w/[workflowId]/components/panel/components/editor/components/sub-block/sub-block.tsx": 150, + "apps/sim/app/workspace/[workspaceId]/w/[workflowId]/components/panel/components/editor/components/sub-block/components/index.ts": 144 } }, "app/workspace/layout.tsx": { - "modules": 1160, + "modules": 1082, "gateways": { - "apps/sim/app/workspace/providers/socket-provider.tsx": 1150, - "apps/sim/triggers/registry.ts": 508, - "apps/sim/blocks/registry.ts": 351, - "apps/sim/blocks/registry-maps.ts": 348, - "apps/sim/stores/workflows/registry/store.ts": 180, - "apps/sim/hooks/queries/deployments.ts": 177, - "apps/sim/lib/workflows/comparison/describe.ts": 166, - "apps/sim/hooks/selectors/registry.ts": 106 + "apps/sim/app/workspace/providers/socket-provider.tsx": 1072, + "apps/sim/triggers/registry.ts": 519, + "apps/sim/blocks/registry.ts": 368, + "apps/sim/blocks/registry-maps.ts": 365, + "apps/sim/stores/workflows/registry/store.ts": 82, + "apps/sim/hooks/queries/deployments.ts": 79, + "apps/sim/lib/workflows/comparison/describe.ts": 67, + "apps/sim/lib/api/contracts/index.ts": 47 } }, "app/workspace/page.tsx": { - "modules": 1157, + "modules": 1082, "gateways": { - "apps/sim/lib/auth/stale-session-recovery.ts": 1068, - "apps/sim/triggers/index.ts": 510, - "apps/sim/triggers/registry.ts": 508, - "apps/sim/blocks/registry.ts": 351, - "apps/sim/blocks/registry-maps.ts": 348, - "apps/sim/stores/workflows/registry/store.ts": 130, - "apps/sim/hooks/queries/deployments.ts": 123, - "apps/sim/lib/workflows/comparison/describe.ts": 113 + "apps/sim/lib/auth/stale-session-recovery.ts": 990, + "apps/sim/triggers/index.ts": 521, + "apps/sim/triggers/registry.ts": 519, + "apps/sim/blocks/registry.ts": 368, + "apps/sim/blocks/registry-maps.ts": 365, + "apps/sim/lib/api/contracts/index.ts": 44, + "apps/sim/stores/workflows/registry/store.ts": 35, + "apps/sim/triggers/clickup/index.ts": 32 } }, "lib/catalog/projection/block-detail.ts": { - "modules": 1138, + "modules": 1060, "gateways": { - "apps/sim/lib/catalog/projection/block-summary.ts": 597, - "apps/sim/blocks/registry-maps.ts": 593, - "apps/sim/triggers/index.ts": 510, - "apps/sim/triggers/registry.ts": 508, - "apps/sim/blocks/blocks/credential-group.ts": 219, - "apps/sim/stores/workflows/registry/store.ts": 190, - "apps/sim/hooks/queries/deployments.ts": 181, - "apps/sim/lib/workflows/comparison/describe.ts": 170 + "apps/sim/triggers/index.ts": 521, + "apps/sim/triggers/registry.ts": 519, + "apps/sim/lib/catalog/projection/block-summary.ts": 504, + "apps/sim/blocks/registry-maps.ts": 500, + "apps/sim/blocks/blocks/credential-group.ts": 123, + "apps/sim/stores/workflows/registry/store.ts": 93, + "apps/sim/hooks/queries/deployments.ts": 84, + "apps/sim/lib/workflows/comparison/describe.ts": 72 } }, "lib/catalog/projection/block-summary.ts": { - "modules": 1134, + "modules": 1056, "gateways": { - "apps/sim/blocks/registry.ts": 1126, - "apps/sim/blocks/registry-maps.ts": 1123, - "apps/sim/triggers/index.ts": 510, - "apps/sim/triggers/registry.ts": 508, - "apps/sim/blocks/blocks/credential-group.ts": 220, - "apps/sim/stores/workflows/registry/store.ts": 190, - "apps/sim/hooks/queries/deployments.ts": 181, - "apps/sim/lib/workflows/comparison/describe.ts": 170 + "apps/sim/blocks/registry.ts": 1047, + "apps/sim/blocks/registry-maps.ts": 1044, + "apps/sim/triggers/index.ts": 521, + "apps/sim/triggers/registry.ts": 519, + "apps/sim/blocks/blocks/credential-group.ts": 124, + "apps/sim/stores/workflows/registry/store.ts": 93, + "apps/sim/hooks/queries/deployments.ts": 84, + "apps/sim/lib/workflows/comparison/describe.ts": 72 } }, "lib/catalog/projection/connector-type.ts": { @@ -694,16 +694,16 @@ "gateways": {} }, "lib/copilot/tools/server/blocks/get-blocks-metadata-tool.ts": { - "modules": 1257, - "gateways": { - "apps/sim/blocks/registry.ts": 553, - "apps/sim/blocks/registry-maps.ts": 551, - "apps/sim/triggers/index.ts": 510, - "apps/sim/triggers/registry.ts": 508, - "apps/sim/blocks/blocks/credential-group.ts": 205, - "apps/sim/stores/workflows/registry/store.ts": 181, - "apps/sim/hooks/queries/deployments.ts": 172, - "apps/sim/lib/workflows/comparison/describe.ts": 161 + "modules": 1161, + "gateways": { + "apps/sim/triggers/index.ts": 521, + "apps/sim/triggers/registry.ts": 519, + "apps/sim/blocks/registry.ts": 493, + "apps/sim/blocks/registry-maps.ts": 491, + "apps/sim/blocks/blocks/credential-group.ts": 118, + "apps/sim/lib/permission-groups/config-scope.server.ts": 92, + "apps/sim/lib/permission-groups/resolve.server.ts": 90, + "apps/sim/stores/workflows/registry/store.ts": 89 } } } diff --git a/scripts/check-tool-registry-boundary.ts b/scripts/check-tool-registry-boundary.ts index 544dca12c82..dfe5fefbae5 100644 --- a/scripts/check-tool-registry-boundary.ts +++ b/scripts/check-tool-registry-boundary.ts @@ -84,6 +84,18 @@ function hasDefaultExport(file: string): boolean { const isWorkspaceEntry = (filename: string, fullPath: string) => WORKSPACE_ENTRY_FILENAMES.has(filename) && hasDefaultExport(fullPath) const isRouteEntry = (filename: string) => filename === 'route.ts' +/** + * Whether a route file sits under an `execute` segment *within the app*. + * + * Two ways to get this wrong, both silent. Testing the absolute path matches a + * checkout that merely happens to live under a directory named `execute`, which + * would exclude every catalog route and quietly retire the guard. Testing for + * the substring `'/execute/'` stops matching on Windows, where `join` emits + * backslashes, and re-includes the route so the audit fails on every run. So: + * relative to the app first, then split on either separator. + */ +const isUnderExecute = (fullPath: string) => + relative(APP, fullPath).split(/[/\\]/).includes('execute') const isSourceModule = (filename: string) => filename.endsWith('.ts') && !filename.endsWith('.test.ts') @@ -118,8 +130,17 @@ const ENTRY_SOURCES: readonly EntrySource[] = [ reason: 'the public block catalog, which reads block metadata only', }, { + /** + * The catalog routes only — `POST /tools/{toolId}/execute` is deliberately + * outside. Reading a tool and running one are different jobs: the reads + * project `params`/`outputs`, which `@/tools/metadata` covers, while + * execution has to reach the executable registry by definition. That is the + * same reason the ~122 execute/deploy/import/webhook routes are not covered + * wholesale, and it keeps the rule meaningful for its four siblings: a + * `getTool` import in the list or detail route is still always a mistake. + */ root: 'app/api/v2/tools', - matches: isRouteEntry, + matches: (filename, fullPath) => isRouteEntry(filename) && !isUnderExecute(fullPath), reason: 'the public tool catalog, which reads tool metadata only', }, { diff --git a/scripts/check-tool-request-boundary.test.ts b/scripts/check-tool-request-boundary.test.ts index 5291d3154f7..39657d39d33 100644 --- a/scripts/check-tool-request-boundary.test.ts +++ b/scripts/check-tool-request-boundary.test.ts @@ -1,5 +1,5 @@ import { describe, expect, it } from 'vitest' -import { auditToolSelfHops } from './check-tool-request-boundary' +import { auditToolSelfHops, mayAccessToolRequest } from './check-tool-request-boundary' const ENCODED_ID_TEMPLATE = '$' + '{encodeURIComponent(params.id)}' const GET_BASE_URL_TEMPLATE = '$' + '{getBaseUrl()}' @@ -1085,3 +1085,17 @@ describe('tool self-hop audit', () => { ]) }) }) + +describe('tool request access candidate scan', () => { + it('finds direct request member access', () => { + expect(mayAccessToolRequest('const url = tool.request.url')).toBe(true) + }) + + it('decodes escaped identifiers and property strings', () => { + expect(mayAccessToolRequest(String.raw`const url = tool.req\u0075est['\u0075rl']`)).toBe(true) + }) + + it('ignores request objects that are never executed directly', () => { + expect(mayAccessToolRequest("const request = { endpoint: '/v1/items' }")).toBe(false) + }) +}) diff --git a/scripts/check-tool-request-boundary.ts b/scripts/check-tool-request-boundary.ts index dc2f7d2ce1f..12da20a5eb9 100644 --- a/scripts/check-tool-request-boundary.ts +++ b/scripts/check-tool-request-boundary.ts @@ -10,12 +10,14 @@ import { existsSync, readdirSync, readFileSync, statSync } from 'node:fs' import { dirname, extname, join, relative, resolve } from 'node:path' import { fileURLToPath } from 'node:url' import { parse } from '@babel/parser' +import ts from '@typescript/typescript6' const SCRIPT_DIR = dirname(fileURLToPath(import.meta.url)) const ROOT = resolve(SCRIPT_DIR, '..') const APP = join(ROOT, 'apps/sim') const CANONICAL_TRANSPORT = join(APP, 'tools/request-transport.ts') const REQUEST_MEMBERS = new Set(['url', 'method', 'headers', 'body']) +const REQUEST_CANDIDATE_TOKENS = new Set(['request', ...REQUEST_MEMBERS]) const SIM_URLS_MODULE = '@/lib/core/utils/urls' const SIM_ORIGIN_EXPORTS = new Set(['getBaseUrl', 'getInternalApiBaseUrl']) const SIM_URL_BUILDER_EXPORTS = new Set(['ensureAbsoluteUrl']) @@ -1596,8 +1598,7 @@ function getResolvedObjectProperties( } /** Rejects tool definitions that route execution back through this Sim app. */ -export function auditToolSelfHops(source: string, file = 'source.ts'): ToolSelfHopAudit { - const program = parseProgram(source, file) +function auditToolSelfHopProgram(program: SyntaxNode, file: string): ToolSelfHopAudit { const violations: ToolSelfHopViolation[] = [] let detectedSelfHops = 0 let legacyInternalPolicies = 0 @@ -1753,6 +1754,10 @@ export function auditToolSelfHops(source: string, file = 'source.ts'): ToolSelfH return { violations, detectedSelfHops, legacyInternalPolicies } } +export function auditToolSelfHops(source: string, file = 'source.ts'): ToolSelfHopAudit { + return auditToolSelfHopProgram(parseProgram(source, file), file) +} + function getStaticMemberAccess( expression: SyntaxNode ): { target: SyntaxNode; member: string } | undefined { @@ -1809,17 +1814,11 @@ function isLikelyToolIdentifier(expression: SyntaxNode): boolean { ) } -function findToolRequestBoundaryViolations(source: string, file = 'source.ts'): Violation[] { - const extension = extname(file) - const syntaxTree = parse(source, { - sourceFilename: file, - sourceType: 'unambiguous', - errorRecovery: true, - plugins: [ - ...(extension === '.jsx' || extension === '.tsx' ? (['jsx'] as const) : []), - ...(!['.js', '.jsx', '.mjs', '.cjs'].includes(extension) ? (['typescript'] as const) : []), - ], - }) +function findToolRequestBoundaryViolations( + program: SyntaxNode, + source: string, + file: string +): Violation[] { const requestAliases = new Set() const violations: Violation[] = [] const seen = new Set() @@ -1850,7 +1849,7 @@ function findToolRequestBoundaryViolations(source: string, file = 'source.ts'): } for (const child of getChildNodes(node)) collectAliases(child) } - collectAliases(syntaxTree.program) + collectAliases(program) const visit = (node: SyntaxNode) => { if ( @@ -1905,19 +1904,52 @@ function findToolRequestBoundaryViolations(source: string, file = 'source.ts'): } for (const child of getChildNodes(node)) visit(child) } - visit(syntaxTree.program) + visit(program) return violations } +export function mayAccessToolRequest(source: string): boolean { + const scanner = ts.createScanner(ts.ScriptTarget.Latest, true, ts.LanguageVariant.JSX, source) + let hasRequest = false + let hasRequestMember = false + + for (let token = scanner.scan(); token !== ts.SyntaxKind.EndOfFileToken; token = scanner.scan()) { + if ( + token !== ts.SyntaxKind.Identifier && + token !== ts.SyntaxKind.StringLiteral && + token !== ts.SyntaxKind.NoSubstitutionTemplateLiteral + ) { + continue + } + const value = scanner.getTokenValue() + if (!REQUEST_CANDIDATE_TOKENS.has(value)) continue + if (value === 'request') hasRequest = true + else hasRequestMember = true + if (hasRequest && hasRequestMember) return true + } + return false +} + function main(): void { const productionSources = collectProductionSources(APP) - const violations = productionSources - .filter((file) => file !== CANONICAL_TRANSPORT) - .flatMap((file) => findToolRequestBoundaryViolations(readFileSync(file, 'utf8'), file)) - const selfHopAudits = productionSources - .filter((file) => file.startsWith(join(APP, 'tools'))) - .map((file) => auditToolSelfHops(readFileSync(file, 'utf8'), file)) + const violations: Violation[] = [] + const selfHopAudits: ToolSelfHopAudit[] = [] + + for (const file of productionSources) { + const isToolSource = file.startsWith(join(APP, 'tools')) + const source = readFileSync(file, 'utf8') + const auditsDirectAccess = file !== CANONICAL_TRANSPORT && mayAccessToolRequest(source) + if (!isToolSource && !auditsDirectAccess) continue + + const program = parseProgram(source, file) + if (auditsDirectAccess) { + violations.push(...findToolRequestBoundaryViolations(program, source, file)) + } + if (isToolSource) { + selfHopAudits.push(auditToolSelfHopProgram(program, file)) + } + } const selfHopViolations = selfHopAudits.flatMap((audit) => audit.violations) if (violations.length > 0) { diff --git a/scripts/check-utils-enforcement.ts b/scripts/check-utils-enforcement.ts index 3239ef8aa9c..a44162f89e1 100644 --- a/scripts/check-utils-enforcement.ts +++ b/scripts/check-utils-enforcement.ts @@ -171,23 +171,33 @@ async function main() { if (ALLOWLISTED_FILES.has(rel)) continue const content = await readFile(file, 'utf8') - const lines = content.split('\n') - const lineStarts = buildLineStarts(content) + const matches: Array<{ + index: number + description: string + suggestion: string + }> = [] for (const { pattern, description, suggestion } of BANNED_PATTERNS) { pattern.lastIndex = 0 for (let match = pattern.exec(content); match !== null; match = pattern.exec(content)) { - const line = lineAt(lineStarts, match.index) - if (hasAllow(lines, line)) continue - violations.push({ - file: rel, - line, - description, - suggestion, - snippet: (lines[line - 1] ?? '').trim(), - }) + matches.push({ index: match.index, description, suggestion }) } } + if (matches.length === 0) continue + + const lines = content.split('\n') + const lineStarts = buildLineStarts(content) + for (const match of matches) { + const line = lineAt(lineStarts, match.index) + if (hasAllow(lines, line)) continue + violations.push({ + file: rel, + line, + description: match.description, + suggestion: match.suggestion, + snippet: (lines[line - 1] ?? '').trim(), + }) + } } if (violations.length === 0) { diff --git a/scripts/format-generated-source.ts b/scripts/format-generated-source.ts index 538d3643489..63de2c80dde 100644 --- a/scripts/format-generated-source.ts +++ b/scripts/format-generated-source.ts @@ -1,7 +1,8 @@ import { spawnSync } from 'node:child_process' +import { localBin } from './local-bin' export function formatGeneratedSource(source: string, stdinFilePath: string, cwd: string): string { - const result = spawnSync('bunx', ['biome', 'format', '--stdin-file-path', stdinFilePath], { + const result = spawnSync(localBin('biome'), ['format', '--stdin-file-path', stdinFilePath], { cwd, encoding: 'utf8', input: source, diff --git a/scripts/generate-block-successors.test.ts b/scripts/generate-block-successors.test.ts new file mode 100644 index 00000000000..3a6d40e581a --- /dev/null +++ b/scripts/generate-block-successors.test.ts @@ -0,0 +1,43 @@ +import { describe, expect, it } from 'vitest' +import { flattenSuccessors } from './generate-block-successors' + +const registered = (blocks: readonly string[]) => (blockType: string) => blocks.includes(blockType) + +describe('flattenSuccessors', () => { + it('follows a chain to the current version so one lookup answers it', () => { + const map = flattenSuccessors({ a: 'b', b: 'c' }, registered(['a', 'b', 'c'])) + + expect(map.get('a')).toBe('c') + expect(map.get('b')).toBe('c') + }) + + it('stops rather than looping when successors point at each other', () => { + const map = flattenSuccessors({ a: 'b', b: 'a' }, registered(['a', 'b'])) + + expect(map.get('a')).toBe('b') + expect(map.get('b')).toBe('a') + }) + + /** + * A `replacedBy` naming a block that was never registered — a typo, or a + * successor removed later — must leave the retired id as its own answer. The + * editor still offers it as an allowlist row under that id, so resolving it to + * a type nothing can be permitted as would deny it with no row to fix it. + */ + it('keeps its own identity when the named successor is not registered', () => { + const map = flattenSuccessors({ a: 'gone' }, registered(['a'])) + + expect(map.has('a')).toBe(false) + }) + + it('emits nothing for a block that is already current', () => { + expect(flattenSuccessors({}, registered(['slack_v2'])).size).toBe(0) + }) + + /** No key may also be a value, or the runtime's single lookup would be short. */ + it('produces a closed map', () => { + const map = flattenSuccessors({ a: 'b', b: 'c' }, registered(['a', 'b', 'c'])) + + for (const successor of map.values()) expect(map.has(successor)).toBe(false) + }) +}) diff --git a/scripts/generate-block-successors.ts b/scripts/generate-block-successors.ts new file mode 100644 index 00000000000..5a60ddaa047 --- /dev/null +++ b/scripts/generate-block-successors.ts @@ -0,0 +1,158 @@ +#!/usr/bin/env bun +/** + * Generates the access-control successor map from the block registry. + * + * The map answers one question — "which block type is an allowlist decision + * about this id really made against?" — and it has to be answerable from + * `lib/permission-groups/`, which `scripts/check-application-graph.ts` forbids + * from importing `blocks/`: the authorization funnel would pull every block + * definition into every surface that authorizes anything. Before this file the + * answer was reachable only through `getBlock`, so the env allowlist was + * intersected with the group allowlist *textually*, and a deployment naming + * `slack` against a group naming `slack_v2` intersected to nothing — refusing an + * integration both policies allow. + * + * Entries are flattened to the terminal successor; see {@link flattenSuccessors} + * for the walk and its stopping rules. + * + * Usage: + * bun run scripts/generate-block-successors.ts + * bun run scripts/generate-block-successors.ts --check + */ +import { readFile, writeFile } from 'node:fs/promises' +import { dirname, resolve } from 'node:path' +import { fileURLToPath } from 'node:url' +import { formatGeneratedSource } from './format-generated-source' + +const SCRIPT_DIR = dirname(fileURLToPath(import.meta.url)) +const ROOT = resolve(SCRIPT_DIR, '..') +const OUTPUT_PATH = resolve(ROOT, 'apps/sim/lib/permission-groups/block-successors.generated.ts') +const CHECK_MODE = process.argv.includes('--check') + +interface SunsetBlock { + sunset?: { status: string; replacedBy?: string } +} + +/** + * The block registry, loaded lazily. + * + * A static import would run on every import of this module, including the unit + * test for {@link flattenSuccessors}, which needs no registry and cannot + * resolve the `@/` specifiers every block file uses from the repo-root vitest + * project. + */ +async function loadRegistry(): Promise> { + const { BLOCK_REGISTRY } = await import('../apps/sim/blocks/registry-maps') + return BLOCK_REGISTRY as unknown as Record +} + +/** + * Flattens one-hop `replacedBy` edges to terminal successors. + * + * Reproduces the walk the runtime used to perform against the registry, with + * both of its stopping rules: a cycle stops at the last id visited rather than + * spinning, and an edge naming an unregistered block is not followed, leaving + * the id as its own answer. Only ids whose answer differs from themselves are + * returned, so a lookup that misses is a block with no successor. + */ +export function flattenSuccessors( + directSuccessors: Readonly>, + isRegistered: (blockType: string) => boolean +): ReadonlyMap { + const terminal = (blockType: string): string => { + const seen = new Set([blockType]) + let current = blockType + + while (true) { + const successor = directSuccessors[current] + if (!successor || seen.has(successor) || !isRegistered(successor)) return current + seen.add(successor) + current = successor + } + } + + const successors = new Map() + for (const blockType of Object.keys(directSuccessors).sort()) { + const resolved = terminal(blockType) + if (resolved !== blockType) successors.set(blockType, resolved) + } + return successors +} + +export async function buildBlockSuccessors(): Promise> { + const registry = await loadRegistry() + + /** `getBlock`'s own-key lookup, with its dash-to-underscore normalization. */ + const lookup = (type: string): SunsetBlock | undefined => { + if (Object.hasOwn(registry, type)) return registry[type] + const normalized = type.replace(/-/g, '_') + return Object.hasOwn(registry, normalized) ? registry[normalized] : undefined + } + + const directSuccessors: Record = {} + for (const blockType of Object.keys(registry)) { + const successor = registry[blockType]?.sunset?.replacedBy + if (successor) directSuccessors[blockType] = successor + } + return flattenSuccessors(directSuccessors, (blockType) => lookup(blockType) !== undefined) +} + +function render(successors: ReadonlyMap): string { + const quote = (value: string) => `'${value.replace(/\\/g, '\\\\').replace(/'/g, "\\'")}'` + const entries = [...successors] + .map( + ([blockType, successor]) => + ` ${/^[A-Za-z_$][\w$]*$/.test(blockType) ? blockType : quote(blockType)}: ${quote(successor)},` + ) + .join('\n') + + return `/** + * Generated by \`bun run generate:block-successors\` from the block registry. + * Do not edit this file directly. + * + * Maps a retired block type to the *terminal* type an access-control decision + * about it is made against — \`sunset.replacedBy\`, followed transitively. It + * exists as a generated projection because \`lib/permission-groups/\` may not + * import \`blocks/\`; see \`scripts/generate-block-successors.ts\`. + */ +export const BLOCK_ACCESS_SUCCESSORS: Record = { +${entries} +} +` +} + +async function main(): Promise { + const successors = await buildBlockSuccessors() + + /** + * The map must be closed: no key may also be a value, or a lookup would need + * a second hop and the runtime does exactly one. Flattening guarantees it, so + * a violation means the flattening itself regressed. + */ + for (const successor of successors.values()) { + if (successors.has(successor)) { + throw new Error( + `Block successor map is not flattened: '${successor}' is both a successor and a retired id.` + ) + } + } + + const generated = formatGeneratedSource(render(successors), OUTPUT_PATH, ROOT) + + if (CHECK_MODE) { + const current = await readFile(OUTPUT_PATH, 'utf8').catch(() => '') + if (current !== generated) { + console.error( + 'Block successor map is stale. Run `bun run generate:block-successors` and commit the result.' + ) + process.exit(1) + } + process.stdout.write('Block successor map is current.\n') + return + } + + await writeFile(OUTPUT_PATH, generated) + process.stdout.write(`Generated ${OUTPUT_PATH}\n`) +} + +if (import.meta.main) await main() diff --git a/scripts/generate-docs.test.ts b/scripts/generate-docs.test.ts index af9c8e15c0e..2f6448145d6 100644 --- a/scripts/generate-docs.test.ts +++ b/scripts/generate-docs.test.ts @@ -12,6 +12,15 @@ import { } from './generate-docs' describe('documentation tool metadata', () => { + it('uses evaluated outputs for factory-defined tools', async () => { + const approve = await getToolInfo('sailpoint_approve_access_request') + const identity = await getToolInfo('sailpoint_get_identity') + + expect(Object.keys(approve?.outputs ?? {})).toEqual(['accepted', 'status']) + expect(Object.keys(identity?.outputs ?? {})).toEqual(['identity']) + expect(identity?.outputs.identity.properties).toHaveProperty('name') + }, 15_000) + it('keeps legitimate parameters named params', async () => { const tool = await getToolInfo('supabase_rpc') @@ -336,13 +345,7 @@ describe('subBlock param extraction', () => { * harmless today because no `notion_*` tool carries a hidden param besides `accessToken`. */ it('reports a subBlocks array of only unfollowable spreads as UNKNOWN, not empty', () => { - for (const blockFile of [ - 'imap.ts', - 'generic_webhook.ts', - 'circleback.ts', - 'rss.ts', - 'sim_workspace_event.ts', - ]) { + for (const blockFile of ['imap.ts', 'generic_webhook.ts', 'rss.ts', 'sim_workspace_event.ts']) { expect(extractUserSettableParamIds(blockSource(blockFile))).toBeNull() } @@ -621,6 +624,25 @@ describe('an unreadable subBlocks array', () => { expect(readableConfig.userSettableParamIds).toEqual(['query']) }) + it('reads action ids from tools.access when another access array appears earlier', () => { + const [config] = extractAllBlockConfigs(` + import type { BlockConfig } from '@/blocks/types' + + export const GovernBlock: BlockConfig = { + type: 'govern', + name: 'Govern', + description: 'A synthetic block', + canvasPresentation: { + sentences: { byOperation: { govern_request: ['Request access'] } }, + }, + subBlocks: [{ id: 'operation' }], + tools: { access: ['govern_request', 'govern_review'] }, + } + `) + + expect(config.tools?.access).toEqual(['govern_request', 'govern_review']) + }) + /** * The whole point of the UNKNOWN state: `[]` asserts the block supplies nothing and strips * every hidden param, so the two must not be spelled the same way. diff --git a/scripts/generate-docs.ts b/scripts/generate-docs.ts index 9c131d84571..44df9dd0131 100755 --- a/scripts/generate-docs.ts +++ b/scripts/generate-docs.ts @@ -6,6 +6,7 @@ import { isVersionedType, stripVersionSuffix } from '@sim/utils/string' import { glob } from 'glob' import type { BlockCategory } from '../apps/sim/blocks/types' import { IntegrationType } from '../apps/sim/blocks/types' +import type { ToolOutputProperty } from '../apps/sim/tools/types' /** * Cache for resolved const definitions from types files. @@ -41,6 +42,34 @@ const LANDING_INTEGRATIONS_DATA_PATH = path.join( 'apps/sim/app/(landing)/integrations/data' ) const TRIGGERS_PATH = path.join(rootDir, 'apps/sim/triggers') +const sourceFileCache = new Map() +const sourceGlobCache = new Map>() +const blockConfigCache = new Map>() + +function readSourceFile(filePath: string): string { + const cached = sourceFileCache.get(filePath) + if (cached !== undefined) return cached + const source = fs.readFileSync(filePath, 'utf-8') + sourceFileCache.set(filePath, source) + return source +} + +async function sourceGlob(pattern: string): Promise { + let pending = sourceGlobCache.get(pattern) + if (!pending) { + pending = glob(pattern) + sourceGlobCache.set(pattern, pending) + } + return [...(await pending)] +} + +function blockConfigsForFile(filePath: string): ReturnType { + const cached = blockConfigCache.get(filePath) + if (cached) return cached + const configs = extractAllBlockConfigs(readSourceFile(filePath)) + blockConfigCache.set(filePath, configs) + return configs +} // Integration triggers are merged into the same per-service page as the service's // actions (one block per integration: actions + an optional Trigger). const TRIGGER_DOCS_OUTPUT_PATH = DOCS_OUTPUT_PATH @@ -335,6 +364,16 @@ async function loadToolMetadata(): Promise> { return toolMetadata } +/** Evaluated tool output schemas, keyed by tool id and kept in sync with the registry by CI. */ +let toolOutputs: Record> | null = null + +async function loadToolOutputs(): Promise>> { + if (toolOutputs) return toolOutputs + const module = await import(path.join(rootDir, 'apps/sim/tools/generated/tool-outputs.ts')) + toolOutputs = module.default as Record> + return toolOutputs +} + /** Human-facing tool names, keyed by tool id. Kept in sync with the registry by CI. */ let toolDisplayNames: Map | null = null @@ -462,7 +501,7 @@ function copyIconsFile(): void { return } - const iconsContent = fs.readFileSync(ICONS_PATH, 'utf-8') + const iconsContent = readSourceFile(ICONS_PATH) emitGeneratedFile(DOCS_ICONS_PATH, iconsContent) if (!CHECK_ONLY) console.log('✓ Icons successfully copied to docs app') @@ -478,12 +517,16 @@ function copyIconsFile(): void { * instead of the two-letter fallback. Never overwrites a block-derived entry — * the block is the canonical icon source when one exists. */ -async function addTriggerProviderIcons(iconMapping: Record): Promise { - const triggerFiles = (await glob(`${TRIGGERS_PATH}/**/*.ts`)).filter((f) => !f.includes('.test.')) +async function addTriggerProviderIcons( + iconMappings: readonly Record[] +): Promise { + const triggerFiles = (await sourceGlob(`${TRIGGERS_PATH}/**/*.ts`)).filter( + (f) => !f.includes('.test.') + ) const previewOnly = await collectPreviewOnlyTriggerIds() for (const file of triggerFiles) { - const fileContent = fs.readFileSync(file, 'utf-8') + const fileContent = readSourceFile(file) const source = stripSourceComments(fileContent) // Pair each trigger's `id` with the `provider` that follows it in the same @@ -494,7 +537,7 @@ async function addTriggerProviderIcons(iconMapping: Record): Pr for (const match of source.matchAll(configRegex)) { const [, triggerId, provider] = match - if (iconMapping[provider]) continue + if (iconMappings.every((iconMapping) => iconMapping[provider])) continue // Preview-only triggers get no page, so they need no provider icon. if (previewOnly.has(triggerId)) continue @@ -502,7 +545,10 @@ async function addTriggerProviderIcons(iconMapping: Record): Pr const iconName = extractIconNameFromContent(source.slice(match.index)) if (!iconName) continue - iconMapping[provider] = { name: iconName, source: resolveIconSource(fileContent, iconName) } + const iconRef = { name: iconName, source: resolveIconSource(fileContent, iconName) } + for (const iconMapping of iconMappings) { + if (!iconMapping[provider]) iconMapping[provider] = iconRef + } } } } @@ -512,17 +558,19 @@ async function addTriggerProviderIcons(iconMapping: Record): Pr * Docs need hidden historical version keys so old BlockInfoCard references and * versioned docs links still render icons, while landing only needs visible blocks. */ -async function generateIconMapping(options: { - includeHidden: boolean -}): Promise> { +async function generateIconMappings(): Promise<{ + docs: Record + visible: Record +}> { try { console.log('Generating icon mapping from block definitions...') - const iconMapping: Record = {} - const blockFiles = (await glob(`${BLOCKS_PATH}/*.ts`)).sort() + const docs: Record = {} + const visible: Record = {} + const blockFiles = (await sourceGlob(`${BLOCKS_PATH}/*.ts`)).sort() for (const blockFile of blockFiles) { - const fileContent = fs.readFileSync(blockFile, 'utf-8') + const fileContent = readSourceFile(blockFile) // For icon mapping, we need ALL blocks including hidden ones // because V2 blocks inherit icons from legacy blocks via spread @@ -592,26 +640,30 @@ async function generateIconMapping(options: { * hidden versioned block. Without this it renders as a text tile. */ const isSunsetBlockType = /sunset\s*:\s*\{/.test(stripSourceComments(blockContent)) - if ( - !hideFromToolbar || - (options.includeHidden && (isVersionedBlockType || isSunsetBlockType)) - ) { - iconMapping[blockType] = { - name: iconName, - source: resolveIconSource(fileContent, iconName), - } + const iconRef = { + name: iconName, + source: resolveIconSource(fileContent, iconName), + } + if (!hideFromToolbar) { + docs[blockType] = iconRef + visible[blockType] = iconRef + } else if (isVersionedBlockType || isSunsetBlockType) { + docs[blockType] = iconRef } } } } - await addTriggerProviderIcons(iconMapping) + await addTriggerProviderIcons([docs, visible]) - console.log(`✓ Generated icon mapping for ${Object.keys(iconMapping).length} blocks`) - return iconMapping + console.log( + `✓ Generated icon mappings for ${Object.keys(docs).length} docs blocks and ` + + `${Object.keys(visible).length} visible blocks` + ) + return { docs, visible } } catch (error) { console.error('Error generating icon mapping:', error) - return {} + return { docs: {}, visible: {} } } } @@ -1216,11 +1268,11 @@ async function buildToolDescriptionMap(): Promise { const desc = new Map() const name = new Map() try { - const toolFiles = await glob(`${toolsDir}/**/*.ts`) + const toolFiles = await sourceGlob(`${toolsDir}/**/*.ts`) for (const file of toolFiles) { const basename = path.basename(file) if (basename === 'index.ts' || basename === 'types.ts') continue - const content = fs.readFileSync(file, 'utf-8') + const content = readSourceFile(file) // Find every `id: 'tool_id'` occurrence in the file. For each, search // the next ~600 characters for `name:` and `description:` fields, cutting @@ -1679,13 +1731,13 @@ async function buildTriggerRegistry(): Promise> { const registry = new Map() const SKIP = new Set(['index.ts', 'registry.ts', 'types.ts', 'constants.ts', 'utils.ts']) - const triggerFiles = (await glob(`${TRIGGERS_PATH}/**/*.ts`)).filter( + const triggerFiles = (await sourceGlob(`${TRIGGERS_PATH}/**/*.ts`)).filter( (f) => !SKIP.has(path.basename(f)) && !f.includes('.test.') ) for (const file of triggerFiles) { try { - const content = fs.readFileSync(file, 'utf-8') + const content = readSourceFile(file) // A file may export multiple TriggerConfig objects (e.g. v1 + v2 in // the same file). Extract all exported configs by splitting on the @@ -1799,12 +1851,12 @@ async function writeIntegrationsJson(iconMapping: Record): Prom const integrations: IntegrationEntry[] = [] const seenBaseTypes = new Set() - const blockFiles = (await glob(`${BLOCKS_PATH}/*.ts`)).sort() + const blockFiles = (await sourceGlob(`${BLOCKS_PATH}/*.ts`)).sort() for (const blockFile of blockFiles) { - const fileContent = fs.readFileSync(blockFile, 'utf-8') + const fileContent = readSourceFile(blockFile) const switchCaseMap = extractSwitchCaseToolMapping(fileContent) - const configs = extractAllBlockConfigs(fileContent) + const configs = blockConfigsForFile(blockFile) for (const config of configs) { const blockType = config.type @@ -2473,7 +2525,15 @@ function extractOutputsFromContent(content: string): Record { } function extractToolsAccessFromContent(content: string): string[] { - const accessMatch = content.match(/access\s*:\s*\[\s*([^\]]+)\s*\]/) + const toolsMatch = /\btools\s*:\s*\{/.exec(content) + if (!toolsMatch) return [] + + const toolsStart = toolsMatch.index + toolsMatch[0].lastIndexOf('{') + const toolsEnd = findMatchingClose(content, toolsStart) + if (toolsEnd === -1) return [] + + const toolsContent = content.substring(toolsStart, toolsEnd) + const accessMatch = toolsContent.match(/access\s*:\s*\[\s*([^\]]+)\s*\]/) if (!accessMatch) return [] return [...accessMatch[1].matchAll(/['"]([^'"]+)['"]/g)].map((m) => m[1]) } @@ -2526,7 +2586,7 @@ function resolveConstReference( return null } - const typesContent = fs.readFileSync(typesFilePath, 'utf-8') + const typesContent = readSourceFile(typesFilePath) // Find the const definition // Pattern: export const CONST_NAME = { ... } as const @@ -2917,7 +2977,7 @@ function resolveFactorySource(fileContent: string, toolFilePath: string, rootDir : path.resolve(path.dirname(toolFilePath), specifier) for (const candidate of [`${resolved}.ts`, path.join(resolved, 'index.ts')]) { - if (fs.existsSync(candidate)) return fs.readFileSync(candidate, 'utf-8') + if (fs.existsSync(candidate)) return readSourceFile(candidate) } return '' } @@ -2947,7 +3007,7 @@ function readImportedModuleSource( if (!resolved) return '' for (const candidate of [`${resolved}.ts`, path.join(resolved, 'index.ts')]) { - if (fs.existsSync(candidate)) return fs.readFileSync(candidate, 'utf-8') + if (fs.existsSync(candidate)) return readSourceFile(candidate) } return '' } @@ -3008,7 +3068,7 @@ export function extractToolInfo( ): { description: string params: Array<{ name: string; type: string; required: boolean; description: string }> - outputs: Record + outputs: Record } | null { try { // First, try to find the specific tool definition by its ID @@ -3788,6 +3848,7 @@ export async function getToolInfo( try { const metadata = (await loadToolMetadata())[toolName] + const generatedOutputs = (await loadToolOutputs())[toolName] const parts = toolName.split('_') let toolPrefix = '' @@ -3855,7 +3916,7 @@ export async function getToolInfo( for (const location of possibleLocations) { if (fs.existsSync(location.path)) { - const content = fs.readFileSync(location.path, 'utf-8') + const content = readSourceFile(location.path) const toolIdRegex = new RegExp(`id:\\s*['"]${toolName}['"]`) if (toolIdRegex.test(content)) { @@ -3880,11 +3941,11 @@ export async function getToolInfo( if (!foundExactId) { const prefixDir = path.join(rootDir, `apps/sim/tools/${toolPrefix}`) if (fs.existsSync(prefixDir)) { - const dirFiles = await glob(`${prefixDir}/**/*.ts`) + const dirFiles = await sourceGlob(`${prefixDir}/**/*.ts`) const toolIdRegex = new RegExp(`id:\\s*['"]${toolName}['"]`) for (const dirFile of dirFiles) { if (dirFile.endsWith('.test.ts')) continue - const content = fs.readFileSync(dirFile, 'utf-8') + const content = readSourceFile(dirFile) if (toolIdRegex.test(content)) { toolFileContent = content foundFile = dirFile @@ -3899,7 +3960,7 @@ export async function getToolInfo( if (!toolFileContent) { for (const location of possibleLocations) { if (fs.existsSync(location.path)) { - toolFileContent = fs.readFileSync(location.path, 'utf-8') + toolFileContent = readSourceFile(location.path) foundFile = location.path break } @@ -3948,7 +4009,10 @@ export async function getToolInfo( return { description: metadata.description ?? sourceInfo?.description ?? 'No description available', params, - outputs: sourceInfo?.outputs ?? {}, + outputs: + toolPrefix === 'sailpoint' + ? (generatedOutputs ?? sourceInfo?.outputs ?? {}) + : (sourceInfo?.outputs ?? generatedOutputs ?? {}), } } catch (error) { console.error(`Error getting info for tool ${toolName}:`, error) @@ -4023,10 +4087,10 @@ async function generateBlockDoc(blockPath: string) { return } - const fileContent = fs.readFileSync(blockPath, 'utf-8') + const fileContent = readSourceFile(blockPath) // Extract ALL block configs from the file (already filters out hideFromToolbar: true) - const blockConfigs = extractAllBlockConfigs(fileContent) + const blockConfigs = blockConfigsForFile(blockPath) if (blockConfigs.length === 0) { console.warn(`Skipping ${blockFileName} - no valid block configs found`) @@ -4264,11 +4328,10 @@ ${toolsSection} */ async function getCanonicalToolDocNames(): Promise> { const validToolDocs = new Set() - const blockFiles = (await glob(`${BLOCKS_PATH}/*.ts`)).sort() + const blockFiles = (await sourceGlob(`${BLOCKS_PATH}/*.ts`)).sort() for (const blockFile of blockFiles) { - const fileContent = fs.readFileSync(blockFile, 'utf-8') - const configs = extractAllBlockConfigs(fileContent) + const configs = blockConfigsForFile(blockFile) for (const config of configs) { // Match the writer filter: integration blocks, the documented @@ -4548,14 +4611,14 @@ async function buildFullTriggerRegistry(): Promise> const registry = new Map() const SKIP = new Set(['index.ts', 'registry.ts', 'types.ts', 'constants.ts', 'utils.ts']) - const triggerFiles = (await glob(`${TRIGGERS_PATH}/**/*.ts`)).filter( + const triggerFiles = (await sourceGlob(`${TRIGGERS_PATH}/**/*.ts`)).filter( (f) => !SKIP.has(path.basename(f)) && !f.includes('.test.') ) const registryTriggers = await loadTriggerRegistry() for (const file of triggerFiles) { try { - const content = fs.readFileSync(file, 'utf-8') + const content = readSourceFile(file) const exportRegex = /export\s+const\s+\w+\s*:\s*TriggerConfig\s*=\s*\{/g let exportMatch: RegExpExecArray | null @@ -4757,11 +4820,10 @@ ${buildTriggersSection(triggers)}` */ async function buildProviderColorMap(): Promise> { const colorMap = new Map() - const blockFiles = (await glob(`${BLOCKS_PATH}/*.ts`)).sort() + const blockFiles = (await sourceGlob(`${BLOCKS_PATH}/*.ts`)).sort() for (const blockFile of blockFiles) { - const fileContent = fs.readFileSync(blockFile, 'utf-8') - const configs = extractAllBlockConfigs(fileContent) + const configs = blockConfigsForFile(blockFile) for (const config of configs) { if (config.bgColor && config.type) { const baseType = stripVersionSuffix(config.type) @@ -4789,9 +4851,9 @@ async function collectPreviewOnlyTriggerIds(): Promise> { const listedByReleased = new Set() const listedByPreview = new Set() - const blockFiles = (await glob(`${BLOCKS_PATH}/*.ts`)).sort() + const blockFiles = (await sourceGlob(`${BLOCKS_PATH}/*.ts`)).sort() for (const blockFile of blockFiles) { - const fileContent = fs.readFileSync(blockFile, 'utf-8') + const fileContent = readSourceFile(blockFile) const exportRegex = /export\s+const\s+(\w+)Block\s*:\s*BlockConfig[^=]*=\s*\{/g let match: RegExpExecArray | null @@ -4898,12 +4960,11 @@ async function generateAllTriggerDocs(): Promise { async function generateAllBlockDocs() { try { - const blockFiles = (await glob(`${BLOCKS_PATH}/*.ts`)).sort() + const blockFiles = (await sourceGlob(`${BLOCKS_PATH}/*.ts`)).sort() copyIconsFile() - const docsIconMapping = await generateIconMapping({ includeHidden: true }) - const visibleIconMapping = await generateIconMapping({ includeHidden: false }) + const { docs: docsIconMapping, visible: visibleIconMapping } = await generateIconMappings() writeIconMapping(docsIconMapping) await writeIntegrationsJson(visibleIconMapping) diff --git a/scripts/local-bin.ts b/scripts/local-bin.ts index de00cc5f2c5..8ef07336b67 100644 --- a/scripts/local-bin.ts +++ b/scripts/local-bin.ts @@ -1,6 +1,7 @@ import path from 'node:path' +import { fileURLToPath } from 'node:url' -const ROOT = path.resolve(import.meta.dir, '..') +const ROOT = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..') /** * Absolute path to a locally-installed executable. diff --git a/scripts/openapi/documents.test.ts b/scripts/openapi/documents.test.ts index 69930110e71..a3e14d01498 100644 --- a/scripts/openapi/documents.test.ts +++ b/scripts/openapi/documents.test.ts @@ -40,9 +40,20 @@ const EXPECTED_OPERATION_COUNTS = new Map([ ['apps/docs/openapi-v2-tables.json', 53], ['apps/docs/openapi-v2-knowledge.json', 44], ['apps/docs/openapi-v2-billing.json', 2], - ['apps/docs/openapi-v2-resources.json', 45], + ['apps/docs/openapi-v2-resources.json', 46], ]) +const generatedDocuments = new Map<(typeof DOCUMENTS)[number], JsonObject>() + +function generatedDocument(document: (typeof DOCUMENTS)[number]): JsonObject { + const cached = generatedDocuments.get(document) + if (cached) return cached + + const generated = generateOpenApiDocument(document) + generatedDocuments.set(document, generated) + return generated +} + function getOperation(spec: JsonObject, path: string, method: string): JsonObject { const paths = spec.paths as JsonObject return (paths[path] as JsonObject)[method] as JsonObject @@ -147,7 +158,7 @@ describe('generated OpenAPI documents', () => { let totalOperations = 0 for (const document of DOCUMENTS) { - const spec = generateOpenApiDocument(document) + const spec = generatedDocument(document) const documentOperations = operations(spec) const expectedCount = EXPECTED_OPERATION_COUNTS.get(document.output) @@ -172,11 +183,11 @@ describe('generated OpenAPI documents', () => { }) } } - expect(totalOperations).toBe(212) + expect(totalOperations).toBe(213) }) it('documents mixed workflow execution and resume responses', () => { - const spec = generateOpenApiDocument(workflowsOpenApiDocument) + const spec = generatedDocument(workflowsOpenApiDocument) const execute = getOperation(spec, '/api/v2/workflows/{workflowId}/execute', 'post') const executeResponses = execute.responses as JsonObject const executeOk = executeResponses['200'] as JsonObject @@ -220,7 +231,7 @@ describe('generated OpenAPI documents', () => { }) it('documents multipart uploads, dual-status secret sets, and nullable file shares', () => { - const knowledgeSpec = generateOpenApiDocument(knowledgeOpenApiDocument) + const knowledgeSpec = generatedDocument(knowledgeOpenApiDocument) const upload = getOperation( knowledgeSpec, '/api/v2/knowledge/{knowledgeBaseId}/documents', @@ -238,13 +249,13 @@ describe('generated OpenAPI documents', () => { expect(Object.keys(uploadContent)).toEqual(['multipart/form-data']) expect(uploadProperties.file).toMatchObject({ type: 'string', format: 'binary' }) - const resourcesSpec = generateOpenApiDocument(resourcesOpenApiDocument) + const resourcesSpec = generatedDocument(resourcesOpenApiDocument) const setSecret = getOperation(resourcesSpec, '/api/v2/secrets/{name}', 'put') expect( Object.keys(setSecret.responses as JsonObject).filter((status) => status.startsWith('2')) ).toEqual(['200', '201']) - const filesSpec = generateOpenApiDocument(filesAuditOpenApiDocument) + const filesSpec = generatedDocument(filesAuditOpenApiDocument) const fileSchemas = (filesSpec.components as JsonObject).schemas as JsonObject const fileMetadata = fileSchemas.V2FileMetadata as JsonObject const fileMetadataProperties = fileMetadata.properties as JsonObject @@ -254,12 +265,12 @@ describe('generated OpenAPI documents', () => { }) it('documents public resource owner email addresses', () => { - const knowledgeSpec = generateOpenApiDocument(knowledgeOpenApiDocument) + const knowledgeSpec = generatedDocument(knowledgeOpenApiDocument) const knowledgeSchemas = (knowledgeSpec.components as JsonObject).schemas as JsonObject const knowledgeBase = knowledgeSchemas.V2KnowledgeBase as JsonObject const knowledgeBaseProperties = knowledgeBase.properties as JsonObject - const tablesSpec = generateOpenApiDocument(tablesOpenApiDocument) + const tablesSpec = generatedDocument(tablesOpenApiDocument) const tableSchemas = (tablesSpec.components as JsonObject).schemas as JsonObject const table = tableSchemas.V2ApiTable as JsonObject const tableProperties = table.properties as JsonObject @@ -269,14 +280,14 @@ describe('generated OpenAPI documents', () => { }) it('keeps billing as its own API reference group', () => { - const spec = generateOpenApiDocument(billingOpenApiDocument) + const spec = generatedDocument(billingOpenApiDocument) expect((spec.tags as JsonObject[]).map((tag) => tag.name)).toEqual(['Billing']) expect(getOperation(spec, '/api/v2/billing/status', 'get').tags).toEqual(['Billing']) expect(getOperation(spec, '/api/v2/billing/logs', 'get').tags).toEqual(['Billing']) }) it('documents workspace details as a named schema without internal mode', () => { - const resourcesSpec = generateOpenApiDocument(resourcesOpenApiDocument) + const resourcesSpec = generatedDocument(resourcesOpenApiDocument) const schemas = (resourcesSpec.components as JsonObject).schemas as JsonObject const response = schemas.GetWorkspaceResponse as JsonObject const responseProperties = response.properties as JsonObject @@ -297,7 +308,7 @@ describe('generated OpenAPI documents', () => { }) it('publishes Agent tools as named integration, custom, and MCP schemas', () => { - const workflowsSpec = generateOpenApiDocument(workflowsOpenApiDocument) + const workflowsSpec = generatedDocument(workflowsOpenApiDocument) const schemas = (workflowsSpec.components as JsonObject).schemas as JsonObject const agentToolInput = schemas.AgentToolInput as JsonObject const agentTool = schemas.AgentTool as JsonObject @@ -354,7 +365,7 @@ describe('generated OpenAPI documents', () => { it('uses named schemas for top-level response objects and list items', () => { for (const document of DOCUMENTS) { - expect(anonymousTopLevelResponseObjects(generateOpenApiDocument(document))).toEqual([]) + expect(anonymousTopLevelResponseObjects(generatedDocument(document))).toEqual([]) } }) @@ -438,7 +449,7 @@ describe('documented error sets', () => { * selection is an empty page. `getAuditLog` does 404 and keeps it. */ it('does not publish a 404 the audit-log list cannot emit', () => { - const spec = generateOpenApiDocument(filesAuditOpenApiDocument) + const spec = generatedDocument(filesAuditOpenApiDocument) expect( Object.keys(getOperation(spec, '/api/v2/audit-logs', 'get').responses as JsonObject) ).not.toContain('404') @@ -483,7 +494,7 @@ describe('shared parameter descriptions do not fork', () => { const descriptionsByParameter = new Map>() for (const document of DOCUMENTS) { - const spec = generateOpenApiDocument(document) + const spec = generatedDocument(document) for (const operation of operations(spec)) { for (const parameter of (operation.parameters ?? []) as JsonObject[]) { const name = parameter.name as string diff --git a/scripts/openapi/generator.test.ts b/scripts/openapi/generator.test.ts index c44ee3d0451..b6b9b453fb9 100644 --- a/scripts/openapi/generator.test.ts +++ b/scripts/openapi/generator.test.ts @@ -21,6 +21,18 @@ import { } from './generator' type JsonObject = Record +type OpenApiDocument = Parameters[0] + +const generatedDocuments = new Map() + +function generatedDocument(document: OpenApiDocument): JsonObject { + const cached = generatedDocuments.get(document) + if (cached) return cached + + const generated = generateOpenApiDocument(document) + generatedDocuments.set(document, generated) + return generated +} const ERROR_SCHEMA = z .object({ @@ -578,7 +590,7 @@ describe('OpenAPI generator', () => { }) it('documents nullable file share metadata from the response schema', () => { - const spec = generateOpenApiDocument(filesAuditOpenApiDocument) + const spec = generatedDocument(filesAuditOpenApiDocument) const schemas = (spec.components as JsonObject).schemas as JsonObject const metadata = schemas.V2FileMetadata as JsonObject const properties = metadata.properties as JsonObject @@ -588,7 +600,7 @@ describe('OpenAPI generator', () => { }) it('documents v2 billing storage coverage from the response schema', () => { - const spec = generateOpenApiDocument(billingOpenApiDocument) + const spec = generatedDocument(billingOpenApiDocument) const paths = spec.paths as JsonObject const schemas = (spec.components as JsonObject).schemas as JsonObject const response = schemas.V2BillingStatusResponse as JsonObject @@ -626,7 +638,7 @@ describe('OpenAPI generator', () => { * is why it stays and is pinned here instead. */ it('uses string wire values for a stringbool query param', () => { - const spec = generateOpenApiDocument(filesAuditOpenApiDocument) + const spec = generatedDocument(filesAuditOpenApiDocument) const deleteFolder = getOperation(spec, '/api/v2/files/folders', 'delete') const deleteFolderParameters = deleteFolder.parameters as JsonObject[] const recursive = deleteFolderParameters.find((parameter) => parameter.name === 'recursive') @@ -641,14 +653,14 @@ describe('OpenAPI generator', () => { * callers to send a string for what four sibling params took as a boolean. */ it('documents boolean query flags as booleans', () => { - const auditSpec = generateOpenApiDocument(filesAuditOpenApiDocument) + const auditSpec = generatedDocument(filesAuditOpenApiDocument) const listAuditLogParameters = getOperation(auditSpec, '/api/v2/audit-logs', 'get') .parameters as JsonObject[] const includeDeparted = listAuditLogParameters.find( (parameter) => parameter.name === 'includeDeparted' ) - const workflowSpec = generateOpenApiDocument(workflowsOpenApiDocument) + const workflowSpec = generatedDocument(workflowsOpenApiDocument) const getRunParameters = getOperation( workflowSpec, '/api/v2/workflows/{workflowId}/runs/{runId}', @@ -661,7 +673,7 @@ describe('OpenAPI generator', () => { }) it('documents binary download response headers', () => { - const spec = generateOpenApiDocument(filesAuditOpenApiDocument) + const spec = generatedDocument(filesAuditOpenApiDocument) const operation = getOperation(spec, '/api/v2/files/{fileId}', 'get') const response = (operation.responses as JsonObject)['200'] as JsonObject @@ -717,7 +729,7 @@ describe('OpenAPI generator', () => { }) it('publishes a distinct example under every documented error status', () => { - const spec = generateOpenApiDocument(workflowsOpenApiDocument) + const spec = generatedDocument(workflowsOpenApiDocument) const responses = (spec.components as JsonObject).responses as JsonObject const byStatus = new Map>() diff --git a/scripts/openapi/generator.ts b/scripts/openapi/generator.ts index cc8cd84c0f2..550113a1ce8 100644 --- a/scripts/openapi/generator.ts +++ b/scripts/openapi/generator.ts @@ -10,6 +10,7 @@ import type { } from '@/lib/api/openapi/types' type JsonObject = Record +type SchemaIo = 'input' | 'output' const HTTP_SUCCESS_MIN = 200 const HTTP_SUCCESS_MAX = 299 @@ -41,6 +42,12 @@ const outputExampleValidator = new Ajv2020({ allErrors: true, validateFormats: false, }) +const comparableSchemaCache = new WeakMap>() +const generatedSchemaCache = new WeakMap>() +const outputValidatorCache = new WeakMap< + ApiSchema, + ReturnType +>() function invariant(condition: unknown, message: string): asserts condition { if (!condition) throw new Error(message) @@ -98,8 +105,11 @@ function stripLegacySchemaIds(value: unknown): unknown { ) } -function comparableSchema(schema: ApiSchema, io: 'input' | 'output'): unknown { - return stripSchemaDocumentation( +function comparableSchema(schema: ApiSchema, io: SchemaIo): unknown { + const cached = comparableSchemaCache.get(schema)?.get(io) + if (cached) return cached + + const comparable = stripSchemaDocumentation( sanitizeSchema( z.toJSONSchema(schema, { io, @@ -110,6 +120,10 @@ function comparableSchema(schema: ApiSchema, io: 'input' | 'output'): unknown { }) ) ) + const byIo = comparableSchemaCache.get(schema) ?? new Map() + byIo.set(io, comparable) + comparableSchemaCache.set(schema, byIo) + return comparable } function addComponent( @@ -178,28 +192,35 @@ function validateNoSilentOpaqueSchemas(schema: JsonObject, label: string, path = function generateSchema( schema: ApiSchema, - io: 'input' | 'output', + io: SchemaIo, components: JsonObject, label: string, includeRootComponent = true ): { name: string; schema: JsonObject; metadata: z.core.GlobalMeta } { const metadata = schemaMetadata(schema, label) - const { $defs: definitions, ...generated } = z.toJSONSchema(schema, { - io, - target: 'draft-2020-12', - unrepresentable: 'any', - cycles: 'ref', - reused: 'inline', - override: ({ zodSchema, path }) => { - const current = zodSchema as ApiSchema - validateExamples( - current, - z.globalRegistry.get(current)?.examples, - io, - `${label} at ${path.join('.') || ''}` - ) - }, - }) as JsonObject + let generatedWithDefinitions = generatedSchemaCache.get(schema)?.get(io) + if (!generatedWithDefinitions) { + generatedWithDefinitions = z.toJSONSchema(schema, { + io, + target: 'draft-2020-12', + unrepresentable: 'any', + cycles: 'ref', + reused: 'inline', + override: ({ zodSchema, path }) => { + const current = zodSchema as ApiSchema + validateExamples( + current, + z.globalRegistry.get(current)?.examples, + io, + `${label} at ${path.join('.') || ''}` + ) + }, + }) as JsonObject + const byIo = generatedSchemaCache.get(schema) ?? new Map() + byIo.set(io, generatedWithDefinitions) + generatedSchemaCache.set(schema, byIo) + } + const { $defs: definitions, ...generated } = generatedWithDefinitions if (definitions !== undefined) { invariant( @@ -377,12 +398,7 @@ function statusSuccessContent( return content } -function validateExamples( - schema: ApiSchema, - examples: unknown, - io: 'input' | 'output', - label: string -): void { +function validateExamples(schema: ApiSchema, examples: unknown, io: SchemaIo, label: string): void { if (examples === undefined) return invariant( Array.isArray(examples) && examples.length > 0, @@ -397,14 +413,18 @@ function validateExamples( ) continue } - const outputSchema = z.toJSONSchema(schema, { - io: 'output', - target: 'draft-2020-12', - unrepresentable: 'any', - cycles: 'ref', - reused: 'inline', - }) - const validate = outputExampleValidator.compile(stripLegacySchemaIds(outputSchema)) + let validate = outputValidatorCache.get(schema) + if (!validate) { + const outputSchema = z.toJSONSchema(schema, { + io: 'output', + target: 'draft-2020-12', + unrepresentable: 'any', + cycles: 'ref', + reused: 'inline', + }) + validate = outputExampleValidator.compile(stripLegacySchemaIds(outputSchema)) + outputValidatorCache.set(schema, validate) + } invariant( validate(example), `${label} example ${index + 1} is invalid for the output schema: ${outputExampleValidator.errorsText(validate.errors)}` @@ -494,8 +514,9 @@ function operationFor( if (!contractSchema) continue invariant(documentedSchema, `${label} is missing its documented ${name} schema`) invariant( - JSON.stringify(comparableSchema(contractSchema, io)) === - JSON.stringify(comparableSchema(documentedSchema, io)), + contractSchema === documentedSchema || + JSON.stringify(comparableSchema(contractSchema, io)) === + JSON.stringify(comparableSchema(documentedSchema, io)), `${label} documented ${name} schema does not match the contract schema` ) } @@ -529,8 +550,9 @@ function operationFor( } for (const status of expectedStatuses) { invariant( - JSON.stringify(comparableSchema(contractStatusSchemas[status], 'output')) === - JSON.stringify(comparableSchema(schemas.responses[status], 'output')), + contractStatusSchemas[status] === schemas.responses[status] || + JSON.stringify(comparableSchema(contractStatusSchemas[status], 'output')) === + JSON.stringify(comparableSchema(schemas.responses[status], 'output')), `${label} documented schema for status ${status} does not match the contract schema` ) } diff --git a/turbo.json b/turbo.json index 619997da6ee..587768aa8e9 100644 --- a/turbo.json +++ b/turbo.json @@ -37,6 +37,7 @@ }, "test": { "dependsOn": ["^build"], + "env": ["SIM_TEST_SHARD"], "outputs": [] }, "lint": { diff --git a/vitest.scripts.config.ts b/vitest.scripts.config.ts new file mode 100644 index 00000000000..80d14604d7b --- /dev/null +++ b/vitest.scripts.config.ts @@ -0,0 +1,18 @@ +import { defineConfig } from 'vitest/config' + +/** + * Repo-level scripts have their own suites. One invocation for all of them + * replaces eleven sequential `vitest run ` processes, each of which paid + * its own startup. `scripts/openapi` keeps its own config and runs under + * `check:openapi`. + * + * Deliberately not named `vitest.config.ts`: Vitest walks up from a package's + * directory looking for that name, so a root config would silently replace + * the defaults of every workspace package that has none of its own. + */ +export default defineConfig({ + test: { + environment: 'node', + include: ['scripts/*.test.ts'], + }, +})