Skip to content
Closed
1 change: 1 addition & 0 deletions apps/server/src/environment/ServerEnvironment.ts
Original file line number Diff line number Diff line change
Expand Up @@ -229,6 +229,7 @@ export const make = Effect.gen(function* () {
threadAutoSettlement: true,
storageCleanup: true,
projectWorktreeCleanup: true,
worktreeDisposablePaths: true,
worktreesDirectory: true,
threadRestartContinuation: true,
projectSettingsOverrides: true,
Expand Down
97 changes: 96 additions & 1 deletion apps/server/src/storageCleanup.test.ts
Original file line number Diff line number Diff line change
@@ -1,4 +1,8 @@
import { describe, expect, it } from "vite-plus/test";
// @effect-diagnostics nodeBuiltinImport:off - Effect's FileSystem cannot create junctions.
import * as NodeFS from "node:fs";

import * as NodeServices from "@effect/platform-node/NodeServices";
import { it } from "@effect/vitest";
import {
ProjectId,
ProviderInstanceId,
Expand All @@ -8,11 +12,20 @@ import {
type OrchestrationV2ThreadShell,
} from "@t3tools/contracts";
import * as DateTime from "effect/DateTime";
import * as Effect from "effect/Effect";
import * as FileSystem from "effect/FileSystem";
import * as Layer from "effect/Layer";
import * as Path from "effect/Path";
import { describe, expect } from "vite-plus/test";

import * as ServerConfig from "./config.ts";
import {
storageCleanupActivityAt,
storageCleanupKeepsIgnoredFiles,
storageCleanupPullRequestMerged,
storageCleanupThreadIdle,
} from "./storageCleanup.ts";
import * as GitVcsDriver from "./vcs/GitVcsDriver.ts";

const NOW_MS = Date.parse("2026-06-10T12:00:00.000Z");
const DAY_MS = 24 * 60 * 60 * 1_000;
Expand Down Expand Up @@ -151,6 +164,88 @@ describe("V2 storage cleanup eligibility", () => {
}
});

const GitLayer = GitVcsDriver.layer.pipe(
Layer.provide(ServerConfig.layerTest(process.cwd(), { prefix: "t3-storage-cleanup-test-" })),
Layer.provideMerge(NodeServices.layer),
);

const makeRepo = (files: Record<string, string>) =>
Effect.gen(function* () {
const fs = yield* FileSystem.FileSystem;
const path = yield* Path.Path;
const git = yield* GitVcsDriver.GitVcsDriver;
const cwd = yield* fs.makeTempDirectoryScoped({ prefix: "storage-cleanup-repo-" });
const write = (relativePath: string, contents: string) =>
fs
.makeDirectory(path.dirname(path.join(cwd, relativePath)), { recursive: true })
.pipe(Effect.andThen(fs.writeFileString(path.join(cwd, relativePath), contents)));
yield* git.initRepo({ cwd });
yield* write(".gitignore", "node_modules/\nbuild/\n.dart_tool/\n*.env\nlinked\n");
yield* git.execute({ operation: "test.add", cwd, args: ["add", ".gitignore"] });
yield* git.execute({
operation: "test.commit",
cwd,
args: ["-c", "user.name=Test", "-c", "user.email=test@test.com", "commit", "-m", "init"],
});
for (const [relativePath, contents] of Object.entries(files))
yield* write(relativePath, contents);
return cwd;
});

it.layer(GitLayer)("storage cleanup ignored files", (it) => {
const flutterBuild = {
"build/app/output.apk": "apk",
".dart_tool/package_config.json": "{}",
"node_modules/pkg/index.js": "",
// An untracked package left with only its dependency install.
"packages/removed/node_modules/pkg/index.js": "",
};

it.effect("keeps a worktree whose ignored files are not disposable", () =>
Effect.gen(function* () {
const cwd = yield* makeRepo(flutterBuild);
expect(yield* storageCleanupKeepsIgnoredFiles(cwd, ["node_modules/"])).toBe(true);
expect(yield* storageCleanupKeepsIgnoredFiles(cwd, [])).toBe(true);
}),
);

it.effect("allows removing a worktree whose ignored files all match disposable patterns", () =>
Effect.gen(function* () {
const cwd = yield* makeRepo(flutterBuild);
expect(
yield* storageCleanupKeepsIgnoredFiles(cwd, ["node_modules/", "build/", ".dart_tool/"]),
).toBe(false);
}),
);

it.effect("keeps unknown ignored files next to disposable ones", () =>
Effect.gen(function* () {
const cwd = yield* makeRepo({ ...flutterBuild, "packages/removed/local.env": "TOKEN=1" });
expect(
yield* storageCleanupKeepsIgnoredFiles(cwd, ["node_modules/", "build/", ".dart_tool/"]),
).toBe(true);
}),
);

it.effect("never lets an ignored link keep a worktree", () =>
Effect.gen(function* () {
const fs = yield* FileSystem.FileSystem;
const path = yield* Path.Path;
const cwd = yield* makeRepo({ "node_modules/pkg/index.js": "" });
const target = yield* fs.makeTempDirectoryScoped({ prefix: "storage-cleanup-target-" });
yield* fs.writeFileString(path.join(target, "secret.env"), "TOKEN=1");
NodeFS.symlinkSync(target, path.join(cwd, "linked"), "junction");
NodeFS.mkdirSync(path.join(cwd, "infra"));
NodeFS.symlinkSync(target, path.join(cwd, "infra", "linked"), "junction");
expect(yield* storageCleanupKeepsIgnoredFiles(cwd, ["node_modules/"])).toBe(false);

yield* fs.makeDirectory(path.join(cwd, "build"));
yield* fs.writeFileString(path.join(cwd, "build", "output.bin"), "");
expect(yield* storageCleanupKeepsIgnoredFiles(cwd, ["node_modules/"])).toBe(true);
}),
);
});

describe("merged pull request cleanup", () => {
const HEAD_SHA = "a".repeat(40);
const integrated = {
Expand Down
101 changes: 68 additions & 33 deletions apps/server/src/storageCleanup.ts
Original file line number Diff line number Diff line change
Expand Up @@ -111,6 +111,59 @@ export function storageCleanupActivityAt(thread: OrchestrationV2ThreadShell): nu
);
}

// Ignored files can contain secrets or local datasets, so each one keeps the
// worktree unless it matches a disposable gitignore pattern or is a symlink or
// junction. Removal unlinks a link and never touches its target.
export const storageCleanupKeepsIgnoredFiles = Effect.fn("StorageCleanup.keepsIgnoredFiles")(
function* (worktreePath: string, disposablePaths: ReadonlyArray<string>) {
const git = yield* GitVcsDriver.GitVcsDriver;
const fs = yield* FileSystem.FileSystem;
const path = yield* Path.Path;
const listIgnored = (excludes: ReadonlyArray<string>) =>
git
.execute({
operation: "StorageCleanup.ignoredFiles",
cwd: worktreePath,
args: ["ls-files", "--others", "--ignored", "--directory", "-z", ...excludes],
maxOutputBytes: 64 * 1024,
})
.pipe(
Effect.map((result) =>
result.stdoutTruncated ? null : result.stdout.split("\0").filter(Boolean),
),
);
const ignored = yield* listIgnored(["--exclude-standard"]);
if (ignored === null) return true;
if (ignored.length === 0) return false;
// Git matches the patterns, so they mean exactly what they mean in .gitignore.
const disposable =
disposablePaths.length === 0
? []
: yield* listIgnored(disposablePaths.map((pattern) => `--exclude=${pattern}`));
if (disposable === null) return true;
for (const entry of ignored) {
// Git also lists an untracked directory that holds only ignored files.
// Its contents are listed separately and decide on their own.
if (
entry.endsWith("/") &&
ignored.some((other) => other !== entry && other.startsWith(entry))
)
continue;
if (
disposable.some(
(match) => entry === match || (match.endsWith("/") && entry.startsWith(match)),
)
)
continue;
const link = yield* fs
.readLink(path.join(worktreePath, entry.replace(/\/$/, "")))
.pipe(Effect.option);
if (Option.isNone(link)) return true;
}
return false;
},
);

/**
* Whether the host's pull request proves this worktree's head was merged. A
* squash or rebase merge leaves the head outside the default branch, so the
Expand Down Expand Up @@ -203,6 +256,14 @@ export const make = Effect.gen(function* () {
return false;
});

const ignoredFilesContext = yield* Effect.context<
GitVcsDriver.GitVcsDriver | FileSystem.FileSystem | Path.Path
>();
const keepsIgnoredFiles = (worktreePath: string, disposablePaths: ReadonlyArray<string>) =>
storageCleanupKeepsIgnoredFiles(worktreePath, disposablePaths).pipe(
Effect.provideContext(ignoredFilesContext),
);

const cleanWorktrees = Effect.fn("StorageCleanup.cleanWorktrees")(function* (
serverSettings: ServerSettings,
now: number,
Expand All @@ -222,6 +283,7 @@ export const make = Effect.gen(function* () {
if (root !== null && !isFilesystemRoot(root, path)) roots.push(root);
}
if (roots.length === 0) return;
const disposablePaths = serverSettings.storageCleanup.worktreeDisposablePaths;
const hasDeleteRule = anyWorktreePolicy(serverSettings, (rules) => rules.worktreeOnDelete);
const deletedRows = hasDeleteRule
? yield* sql<{ payload_json: string; workspaceRoot: string }>`
Expand Down Expand Up @@ -280,21 +342,7 @@ export const make = Effect.gen(function* () {
if (!status.isRepo || status.branch !== thread.branch || status.hasWorkingTreeChanges)
return;
const head = yield* git.resolveCommit({ cwd: worktreePath, revision: "HEAD" });
const ignored = yield* git.execute({
operation: "StorageCleanup.ignoredFiles",
cwd: worktreePath,
args: ["ls-files", "--others", "--ignored", "--exclude-standard", "--directory", "-z"],
maxOutputBytes: 64 * 1024,
});
// Ignored files can contain secrets or local datasets. Dependency installs
// are reproducible; every other ignored path prevents automatic removal.
if (
ignored.stdoutTruncated ||
ignored.stdout
.split("\0")
.some((entry) => entry !== "" && !/(^|\/)node_modules\/$/.test(entry))
)
return;
if (yield* keepsIgnoredFiles(worktreePath, disposablePaths)) return;
const old =
!deleted &&
settings.worktreeAfterDays !== null &&
Expand Down Expand Up @@ -399,28 +447,15 @@ export const make = Effect.gen(function* () {
head.commitSha
)
return;
const finalIgnored = yield* git.execute({
operation: "StorageCleanup.ignoredFiles",
cwd: worktreePath,
args: ["ls-files", "--others", "--ignored", "--exclude-standard", "--directory", "-z"],
maxOutputBytes: 64 * 1024,
});
if (
finalIgnored.stdoutTruncated ||
finalIgnored.stdout
.split("\0")
.some((entry) => entry !== "" && !/(^|\/)node_modules\/$/.test(entry))
)
return;
const current = resolveWorktreeCleanup(
yield* settingsService.getSettings,
thread.projectId,
);
if (yield* keepsIgnoredFiles(worktreePath, disposablePaths)) return;
const latestSettings = yield* settingsService.getSettings;
const current = resolveWorktreeCleanup(latestSettings, thread.projectId);
if (
Object.keys(settings).some(
(key) =>
current[key as keyof typeof settings] !== settings[key as keyof typeof settings],
)
) ||
!Equal.equals(latestSettings.storageCleanup.worktreeDisposablePaths, disposablePaths)
)
return;
yield* git.removeWorktree({ cwd: project.workspaceRoot, path: worktreePath, force: false });
Expand Down
62 changes: 59 additions & 3 deletions apps/web/src/components/settings/StorageSettings.tsx
Original file line number Diff line number Diff line change
@@ -1,10 +1,12 @@
import type { StorageCleanupSettings, WorktreeCleanupRules } from "@t3tools/contracts";
import { resolveWorktreeCleanup } from "@t3tools/shared/projectSettings";
import * as Equal from "effect/Equal";
import { useRef, useState } from "react";

import { Input } from "../ui/input";
import { Select, SelectItem, SelectPopup, SelectTrigger, SelectValue } from "../ui/select";
import { Switch } from "../ui/switch";
import { Textarea } from "../ui/textarea";
import {
NumberField,
NumberFieldDecrement,
Expand Down Expand Up @@ -163,9 +165,12 @@ export function StorageSettingsPanel() {
const ruleStatus = (key: keyof StorageCleanupSettings) =>
targets.some(
(target) =>
({ ...target.settings.storageCleanup, ...resolveWorktreeCleanup(target.settings, null) })[
key
] !== settings[key],
!Equal.equals(
{ ...target.settings.storageCleanup, ...resolveWorktreeCleanup(target.settings, null) }[
key
],
settings[key],
),
)
? "Mixed across selected machines"
: undefined;
Expand All @@ -175,6 +180,17 @@ export function StorageSettingsPanel() {
isProjectScope
? updateSettings({ worktreeCleanup: { mode: "custom", rules: patch } })
: update(patch);
// Machine-wide only: a project's custom rules can't carry the list.
const showDisposablePaths =
!isProjectScope &&
connectedEnvironments.every(
Comment thread
coderabbitai[bot] marked this conversation as resolved.
(environment) =>
environment.serverConfig?.environment.capabilities.worktreeDisposablePaths === true,
);
const disposablePathsStatus = ruleStatus("worktreeDisposablePaths");
const disposablePathsEdited = useRef(false);
const disposablePathsText = settings.worktreeDisposablePaths.join("\n");
const scopeKey = targets.map((entry) => `${entry.environmentId}:${entry.projectId}`).join(",");

if (
isProjectScope &&
Expand Down Expand Up @@ -312,6 +328,46 @@ export function StorageSettingsPanel() {
/>
}
/>
{showDisposablePaths && (
<SettingsRow
title="Disposable ignored paths"
status={disposablePathsStatus}
description="Ignored files keep a worktree because they can hold secrets or local data. List .gitignore patterns that are safe to delete, one per line. Ignored symlinks and junctions never keep a worktree."
serverScoped
>
<div className="mt-3 max-w-2xl pb-3.5">
<Textarea
key={`${scopeKey}:${disposablePathsStatus !== undefined}:${disposablePathsText}`}
aria-label="Disposable ignored paths"
autoCapitalize="none"
spellCheck={false}
rows={3}
defaultValue={disposablePathsStatus ? "" : disposablePathsText}
placeholder={
disposablePathsStatus
? "Mixed. Enter patterns to apply to all selected targets."
: "No disposable paths"
}
onChange={() => {
disposablePathsEdited.current = true;
}}
onBlur={(event) => {
const worktreeDisposablePaths = event.target.value
.split("\n")
.map((line) => line.trim())
.filter((line) => line !== "");
if (
disposablePathsEdited.current &&
(disposablePathsStatus ||
!Equal.equals(worktreeDisposablePaths, settings.worktreeDisposablePaths))
)
update({ worktreeDisposablePaths });
disposablePathsEdited.current = false;
}}
/>
</div>
</SettingsRow>
)}
</>
)}
</SettingsSection>
Expand Down
2 changes: 1 addition & 1 deletion apps/web/src/components/settings/settingsSearch.ts
Original file line number Diff line number Diff line change
Expand Up @@ -140,7 +140,7 @@ export const SETTINGS_SEARCH_ITEMS = [
to: "/settings/storage",
scope: "project-defaults",
searchTerms: [
"disk storage delete deleted archived threads old inactive merged unchanged worktrees retention days project inherit off custom",
"disk storage delete deleted archived threads old inactive merged unchanged worktrees retention days project inherit off custom ignored gitignore disposable paths node_modules build",
],
},
{
Expand Down
10 changes: 7 additions & 3 deletions docs/user/project-settings.md
Original file line number Diff line number Diff line change
Expand Up @@ -139,12 +139,16 @@ hourly. Offline machines keep their existing policies.
Select a project to set **Automatic worktree cleanup** to **Inherit**, **Off**, or **Custom**.
Inherit follows each machine's rules; Off keeps that project's worktrees until you remove them
manually. Custom applies separate worktree rules to the selected project or checkout. Browser
captures and log retention remain machine-wide.
captures, log retention and disposable ignored paths remain machine-wide.

Worktrees can be removed after a chosen number of inactive days, after merging, or when they
have no commits beyond the default branch. Only T3-managed worktrees are eligible. Active
sessions, shared worktrees, uncommitted changes, and ignored files other than `node_modules`
prevent removal. Branches and thread history stay; starting another turn recreates the checkout.
sessions, shared worktrees, uncommitted changes, and ignored files prevent removal, because
ignored files can hold secrets or local data. Ignored symlinks and junctions don't count, and
neither do paths listed under **Disposable ignored paths**, which uses `.gitignore` patterns and
starts with `node_modules/`. Add build output such as `build/` and `.dart_tool/` there. Patterns
only match ignored paths, so a project without that folder is unaffected.
Branches and thread history stay; starting another turn recreates the checkout.
Merge cleanup requires a merged pull request whose commits are included in the remote default
branch. A squash or rebase merge on GitHub also counts when the pull request targeted the default
branch and the worktree is still at the pull request's last commit.
Expand Down
2 changes: 2 additions & 0 deletions packages/contracts/src/environment.ts
Original file line number Diff line number Diff line change
Expand Up @@ -126,6 +126,8 @@ export const ExecutionEnvironmentCapabilities = Schema.Struct({
threadAutoSettlement: Schema.optionalKey(Schema.Boolean),
storageCleanup: Schema.optionalKey(Schema.Boolean),
projectWorktreeCleanup: Schema.optionalKey(Schema.Boolean),
/** Server honors `worktreeDisposablePaths`; older servers drop the key from patches. */
worktreeDisposablePaths: Schema.optionalKey(Schema.Boolean),
/** Server honors the `worktreesDirectory` setting. */
worktreesDirectory: Schema.optionalKey(Schema.Boolean),
/** Server persists the opt-in for continuing interrupted threads after restarts. */
Expand Down
Loading
Loading