test(snapshots): reduce repeated setup and worker idle time

.github/workflows/ci.yml

@@ -970,6 +970,9 @@ jobs:
       - name: Build snapshot test binaries
         run: cargo test -p vp_cli_snapshots --no-run --message-format=json > "$RUNNER_TEMP/snapshot-test-build.json"
 
+      - name: Test snapshot runner utilities
+        run: cargo test -p vp_cli_snapshots --test redact_unit --test shard_unit --test registry_pack_unit --test report_unit --test schedule_unit
+
       - name: Package CLI test binaries
         run: |
           artifact_dir="$(mktemp -d)"
@@ -1102,14 +1105,24 @@ jobs:
           VP_SNAP_JS_RUNTIME_DIR="$HOME/.vite-plus/js_runtime" \
           VP_SNAP_FISH_BIN="$(command -v fish)" \
           VP_SNAP_NU_BIN="$(command -v nu)" \
-          ./target/snapshot-tests/cli_snapshots
+          ./target/snapshot-tests/cli_snapshots --test-threads 8
         env:
           RUST_BACKTRACE: '1'
           # Relocate fixture and helper paths while keeping the same test
           # process and VP_SNAP_SHARD partitioning as cargo test.
           CARGO_MANIFEST_DIR: ${{ github.workspace }}/crates/vp_cli_snapshots
           CARGO_BIN_EXE_vpt: ${{ github.workspace }}/target/snapshot-tests/vpt
           VP_SNAP_SHARD: ${{ matrix.shard }}/3
+          VP_SNAP_ARTIFACTS_DIR: ${{ runner.temp }}/snapshot-artifacts
+
+      - name: Upload snapshot diagnostics and timings
+        if: ${{ !cancelled() }}
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+        with:
+          name: snapshot-diagnostics-${{ matrix.target }}-${{ matrix.shard }}
+          path: ${{ runner.temp }}/snapshot-artifacts
+          if-no-files-found: ignore
+          retention-days: 7
 
   # Runs the PTY snapshot suite (crates/vp_cli_snapshots) on Windows with
   # BOTH vp flavors, without a Rust toolchain on the runner: the test binary
@@ -1233,32 +1246,54 @@ jobs:
           # Keep pwsh at its canonical installation path; copying pwsh.exe
           # alone can break its adjacent runtime dependencies.
           export VP_SNAP_PWSH_BIN="$(cygpath -w "$(command -v pwsh.exe)")"
+          # Reserve all nextest workers for cases that need the execution gate.
+          # Generate the overrides from the fixtures, including automatic Ctrl-C
+          # isolation, so this configuration cannot drift from the runner.
+          VP_SNAP_NEXTEST_CONFIG="$RUNNER_TEMP/snapshot-nextest.toml" \
+            cargo-nextest nextest list --archive-file windows-snapshot-tests.tar.zst --workspace-remap . \
+            > "$RUNNER_TEMP/snapshot-test-list.txt"
+          nextest_run_args=(
+            --test-threads 8 --config-file "$RUNNER_TEMP/snapshot-nextest.toml"
+            --archive-file windows-snapshot-tests.tar.zst --workspace-remap . --no-fail-fast
+          )
           # --no-fail-fast: on a snapshot suite every diff is diagnostic
           # signal; cancelling on the first failure hides the rest.
           test_exit=0
           # This fixture requires case-sensitive directory support, which the
           # default temp directory on these runners does not provide.
-          cargo-nextest nextest run --archive-file windows-snapshot-tests.tar.zst --workspace-remap . --no-fail-fast --partition hash:${{ matrix.shard }}/3 \
+          cargo-nextest nextest run "${nextest_run_args[@]}" --partition hash:${{ matrix.shard }}/3 \
             -E 'not test(windows_case_sensitive_shims)' || test_exit=$?
 
           if [[ '${{ matrix.shard }}' == '1' ]]; then
             # Run both flavors on NTFS once. Rust's GetTempPath2 ignores
             # TEMP/TMP under the SYSTEM account, so override SystemTemp.
             SystemTemp='${{ steps.snapshot-temp.outputs.directory }}' \
-              cargo-nextest nextest run --archive-file windows-snapshot-tests.tar.zst --workspace-remap . --no-fail-fast \
+              cargo-nextest nextest run "${nextest_run_args[@]}" \
                 -E 'test(windows_case_sensitive_shims)' || test_exit=$?
 
             # Exercise the same environment wrapper under Windows PowerShell 5.1.
             VP_SNAP_PWSH_BIN="$(cygpath -w "$(command -v powershell.exe)")" \
-              cargo-nextest nextest run --archive-file windows-snapshot-tests.tar.zst --workspace-remap . --no-fail-fast \
+              cargo-nextest nextest run "${nextest_run_args[@]}" \
                 -E 'test(command_env_powershell)' || test_exit=$?
           fi
           exit "$test_exit"
         env:
           RUST_BACKTRACE: '1'
+          # Share immutable tarballs across nextest processes in this job.
+          VP_SNAP_PACKAGES_DIR: ${{ runner.temp }}/snapshot-packages
+          VP_SNAP_ARTIFACTS_DIR: ${{ runner.temp }}/snapshot-artifacts
           # Keep Windows env parity with the `test` recipe in justfile.
           __COMPAT_LAYER: RunAsInvoker
 
+      - name: Upload snapshot diagnostics and timings
+        if: ${{ !cancelled() }}
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+        with:
+          name: snapshot-diagnostics-windows-${{ matrix.shard }}
+          path: ${{ runner.temp }}/snapshot-artifacts
+          if-no-files-found: ignore
+          retention-days: 7
+
   cli-e2e-test-musl:
     name: CLI E2E test (Linux x64 musl)
     needs:

crates/vp_cli_snapshots/tests/cli_snapshots/README.md

@@ -61,22 +61,82 @@ CI runs three shards per platform. Linux and macOS use `VP_SNAP_SHARD=1/3`
 happens before name filtering, so filtered runs keep the same assignment.
 Leave the variable unset to run the whole suite. Windows uses the existing
 nextest runner with `--partition hash:1/3` (then `2/3` and `3/3`).
+Each CI shard uses eight workers so process and network waits can overlap.
+Local runs retain libtest's default worker count; override it with
+`--test-threads <count>`.
+
+Within each shard, the in-process runner executes parallel cases before isolated
+cases. This keeps workers available for ready work instead of blocking them on
+the execution gate. Scheduling happens after partitioning and does not change
+shard membership or test listing order.
+The existing gate still protects `serial = true` and Ctrl-C cases, including
+against other runner processes.
 
 Environment overrides, mainly for CI:
 
-| Variable                    | Effect                                                              |
-| --------------------------- | ------------------------------------------------------------------- |
-| `VP_SNAP_GLOBAL_VP`         | Path to a prebuilt global `vp` binary (skips the target-dir lookup) |
-| `VP_SNAP_LOCAL_CLI_BIN_DIR` | Local CLI bin dir (default `<repo>/packages/cli/bin`)               |
-| `VP_SNAP_JS_RUNTIME_DIR`    | Provisioned managed runtime to seed case homes with                 |
-| `VP_SNAP_SH_BIN`            | POSIX `sh` binary for cases that execute the generated `env` file   |
-| `VP_SNAP_BASH_BIN`          | Bash binary for cases that execute the generated `env` file         |
-| `VP_SNAP_ZSH_BIN`           | Zsh binary for cases that execute the generated `env` file          |
-| `VP_SNAP_CMD_BIN`           | System cmd.exe for cases that execute generated batch files         |
-| `VP_SNAP_FISH_BIN`          | Fish binary for cases that execute generated `env.fish` files       |
-| `VP_SNAP_NU_BIN`            | Nushell binary for cases that execute generated `env.nu` files      |
-| `VP_SNAP_PWSH_BIN`          | PowerShell binary for cases that execute generated `env.ps1` files  |
-| `VP_SNAP_SKIP_FLAVORS`      | Comma-separated flavors to skip registering (e.g. `local`)          |
+| Variable                    | Effect                                                                        |
+| --------------------------- | ----------------------------------------------------------------------------- |
+| `VP_SNAP_GLOBAL_VP`         | Path to a prebuilt global `vp` binary (skips the target-dir lookup)           |
+| `VP_SNAP_LOCAL_CLI_BIN_DIR` | Local CLI bin dir (default `<repo>/packages/cli/bin`)                         |
+| `VP_SNAP_JS_RUNTIME_DIR`    | Provisioned managed runtime to seed case homes with                           |
+| `VP_SNAP_SH_BIN`            | POSIX `sh` binary for cases that execute the generated `env` file             |
+| `VP_SNAP_BASH_BIN`          | Bash binary for cases that execute the generated `env` file                   |
+| `VP_SNAP_ZSH_BIN`           | Zsh binary for cases that execute the generated `env` file                    |
+| `VP_SNAP_CMD_BIN`           | System cmd.exe for cases that execute generated batch files                   |
+| `VP_SNAP_FISH_BIN`          | Fish binary for cases that execute generated `env.fish` files                 |
+| `VP_SNAP_NU_BIN`            | Nushell binary for cases that execute generated `env.nu` files                |
+| `VP_SNAP_PWSH_BIN`          | PowerShell binary for cases that execute generated `env.ps1` files            |
+| `VP_SNAP_SKIP_FLAVORS`      | Comma-separated flavors to skip registering (e.g. `local`)                    |
+| `VP_SNAP_PACKAGES_DIR`      | Run-scoped directory for sharing packed packages across test processes        |
+| `VP_SNAP_ARTIFACTS_DIR`     | Directory for phase timings and failure diagnostics; unset disables artifacts |
+| `VP_SNAP_NEXTEST_CONFIG`    | With `--list`, write nextest overrides for the discovered isolated cases      |
+
+Windows CI generates its nextest configuration from the same case definitions.
+Exact nextest runs read only the selected fixture; listing and native shard
+assignment still discover all cases.
+
+The overrides reserve all test workers for an isolated case and schedule these
+cases last. The file lock remains a fallback when running without the generated
+configuration. To use these overrides locally:
+
+```bash
+VP_SNAP_NEXTEST_CONFIG="$PWD/target/snapshot-nextest.toml" cargo nextest list -p vp_cli_snapshots
+cargo nextest run -p vp_cli_snapshots --config-file target/snapshot-nextest.toml
+```
+
+`VP_SNAP_PACKAGES_DIR` packs the checkout on the first registry case, under a
+cross-process file lock. Later cases reuse the completed tarballs. Use a fresh
+directory for each test run and keep the build unchanged while tests run. The
+runner checks the checkout path and input/tarball file sizes and modification
+times; a changed build fails instead of silently testing stale packages. This
+is a per-run preparation directory, not a persistent content-addressed cache.
+Each case still has its own registry server, overlays, and mutable caches.
+Windows CI uses this to avoid repacking in every nextest process.
+
+With `VP_SNAP_ARTIFACTS_DIR` set, each runner process creates a unique `run-*`
+directory. `runner/timing.json` records discovery, test execution, and final
+cleanup. `cases/<fixture>/<snapshot>/timing.json` records gate waiting,
+provisioning, package preparation/reuse, registry startup, individual steps,
+rendering, comparison, and cleanup. `registry-pack` appears only in the process
+that actually packs; it is nested inside `registry-pack-or-reuse`. Phase
+durations are milliseconds; nested phases must not be added together. The
+existing console timings still exclude gate waiting.
+
+The `case-setup/*` phases divide `case-setup` into home creation, binary and
+package installation, preference seeding, and first-start setup. These phases are
+nested inside `case-setup`; use them to identify preparation costs without
+counting the parent duration twice.
+
+`workspace-cleanup` removes the case workspace after comparison, while other
+workers can still run tests. The final `run-cleanup` removes shared run files and
+any case files left by a panic or an unsuccessful earlier cleanup attempt.
+
+Failed cases also write `error.txt`, `expected.md` (when present), `actual.md`
+(when a complete or partial snapshot was rendered), and `output.txt`. The output
+contains the last 1 MiB of unredacted rendered step output, including successful
+hidden steps; it is not the raw PTY byte stream. Artifacts survive temporary
+workspace cleanup. CI uploads these files and timings on successful and failed
+runs. Listing tests with `--list` neither packs packages nor creates artifacts.
 
 ## Case reference
 
@@ -102,14 +162,24 @@ after = [ ... ]               # cleanup steps, never snapshotted
 
 `vp` picks which CLI runs the case. Both flavors install the built Rust binary
 into the case's `VP_HOME/current/bin`, install the checkout package under that
-case home, and run `vp env setup` before steps. `"global"` exposes only
+case home, and run first-start setup before steps. `"global"` exposes only
 `VP_HOME/bin`; `"local"` also exposes the case-local
 `VP_HOME/current/node_modules/vite-plus/bin` package bin. On Windows, local
 flavor exposes sibling `.cmd` shims under
 `VP_HOME/current/node_modules/.vite-plus-bin` instead. The list form registers
 one trial and one snapshot per flavor; use it for parity cases (help output,
 routing, error messages) where both surfaces must agree.
 
+Each case starts with explicit managed-mode preferences for npm, pnpm, Yarn,
+and Bun. The runner writes these preferences before starting the unmarked
+`vp` binary with no arguments. First-start setup creates the environment files
+and shims, then exits. This avoids a second `vp env setup --refresh`. Tests of
+preference inference or an empty installation must create a separate home, as the
+`shim_package_manager_setup` fixture does.
+The preparation command sets `VP_SELF_SETUP_NO_MODIFY_PATH=1` so Windows
+self-setup does not add temporary case homes to the user's persistent `PATH`.
+This override applies only to runner preparation, not to fixture commands.
+
 A step is a bare argv array or a table:
 
 ```toml

crates/vp_cli_snapshots/tests/cli_snapshots/fixtures/migration_pnpm_vite_identity/packages/utils/vite.config.ts

@@ -1,6 +1,10 @@
 import { defineConfig } from 'vite-plus';
 
 export default defineConfig({
+  test: {
+    // Keep passing test details independent of machine load in snapshots.
+    slowTestThreshold: 60_000,
+  },
   run: {
     tasks: {
       test: { command: 'vp test run' },

crates/vp_cli_snapshots/tests/cli_snapshots/fixtures/snapshot_setup/package.json

@@ -0,0 +1,4 @@
+{
+  "name": "snapshot-setup",
+  "private": true
+}

crates/vp_cli_snapshots/tests/cli_snapshots/fixtures/snapshot_setup/snapshots.toml

@@ -0,0 +1,6 @@
+[[case]]
+name = "prepared_home_matches_cli_setup"
+vp = ["global", "local"]
+steps = [
+  { argv = ["node", "verify.mjs"], comment = "The prepared home has the same preferences, environment files, and shims as setup followed by enabling package-manager management." },
+]

crates/vp_cli_snapshots/tests/cli_snapshots/fixtures/snapshot_setup/snapshots/prepared_home_matches_cli_setup.global.md

@@ -0,0 +1,10 @@
+# prepared_home_matches_cli_setup
+
+## `node verify.mjs`
+
+The prepared home has the same preferences, environment files, and shims as setup followed by enabling package-manager management.
+
+```
+Prepared preferences, environment files, and shims match the CLI setup sequence.
+Configuration changes remain isolated to their own home.
+```

crates/vp_cli_snapshots/tests/cli_snapshots/fixtures/snapshot_setup/snapshots/prepared_home_matches_cli_setup.local.md

@@ -0,0 +1,10 @@
+# prepared_home_matches_cli_setup
+
+## `node verify.mjs`
+
+The prepared home has the same preferences, environment files, and shims as setup followed by enabling package-manager management.
+
+```
+Prepared preferences, environment files, and shims match the CLI setup sequence.
+Configuration changes remain isolated to their own home.
+```

crates/vp_cli_snapshots/tests/cli_snapshots/fixtures/snapshot_setup/verify.mjs

@@ -0,0 +1,111 @@
+import assert from 'node:assert/strict';
+import { spawnSync } from 'node:child_process';
+import { createHash } from 'node:crypto';
+import fs from 'node:fs';
+import path from 'node:path';
+
+const prepared = fs.realpathSync(process.env.VP_HOME);
+const referenceHome = path.resolve('reference-home');
+const reference = path.join(referenceHome, '.vite-plus');
+const windows = process.platform === 'win32';
+assert.equal(process.env.VP_SELF_SETUP_NO_MODIFY_PATH, undefined);
+if (windows) {
+  const result = spawnSync(
+    'powershell.exe',
+    [
+      '-NoProfile',
+      '-NonInteractive',
+      '-Command',
+      '[Environment]::GetEnvironmentVariable("Path", "User")',
+    ],
+    { encoding: 'utf8', timeout: 30000 },
+  );
+  assert.equal(result.status, 0, result.error?.message ?? result.stdout + result.stderr);
+  assert.ok(!result.stdout.toLowerCase().includes(prepared.toLowerCase()));
+}
+const binaryName = windows ? 'vp.exe' : 'vp';
+const bin = path.join(reference, 'current', 'bin');
+fs.mkdirSync(bin, { recursive: true });
+for (const name of windows ? [binaryName, 'vp-shim.exe'] : [binaryName]) {
+  fs.copyFileSync(path.join(prepared, 'current', 'bin', name), path.join(bin, name));
+}
+
+const env = { ...process.env };
+for (const name of Object.keys(env)) {
+  if (name.startsWith('VP_') || name.startsWith('XDG_')) {
+    delete env[name];
+  }
+}
+Object.assign(env, {
+  VP_HOME: reference,
+  VP_CLI_TEST: '1',
+  VP_SELF_SETUP_NO_MODIFY_PATH: '1',
+  HOME: referenceHome,
+  USERPROFILE: referenceHome,
+  PATH: windows
+    ? (process.env.PATH ?? '')
+        .split(path.delimiter)
+        .filter((entry) => !entry.startsWith(path.dirname(prepared)))
+        .join(path.delimiter)
+    : '/usr/bin:/bin:/usr/sbin:/sbin',
+});
+for (const args of [
+  ['env', 'setup', '--refresh'],
+  ['env', 'on', 'pm'],
+]) {
+  const result = spawnSync(path.join(bin, binaryName), args, {
+    env,
+    encoding: 'utf8',
+    timeout: 30000,
+  });
+  assert.equal(result.status, 0, result.error?.message ?? result.stdout + result.stderr);
+}
+
+function installation(home) {
+  const files = {};
+  const normalize = (value) => value.replaceAll(home, '<home>');
+  function visit(relative) {
+    const file = path.join(home, relative);
+    const stat = fs.lstatSync(file);
+    if (stat.isSymbolicLink()) {
+      files[relative] = { target: normalize(fs.readlinkSync(file)) };
+    } else if (stat.isDirectory()) {
+      files[relative] = 'directory';
+      for (const name of fs.readdirSync(file).sort()) {
+        visit(path.join(relative, name));
+      }
+    } else if (file.endsWith('.exe')) {
+      files[relative] = {
+        sha256: createHash('sha256').update(fs.readFileSync(file)).digest('hex'),
+      };
+    } else {
+      files[relative] = { contents: normalize(fs.readFileSync(file, 'utf8')) };
+    }
+  }
+  for (const name of fs.readdirSync(home).sort()) {
+    if (
+      name === 'config.json' ||
+      name === 'bin' ||
+      name === 'fallback-bin' ||
+      /^env(?:\.|$)/.test(name)
+    ) {
+      visit(name);
+    }
+  }
+  return files;
+}
+
+const actual = installation(prepared);
+const expected = installation(reference);
+assert.deepEqual(Object.keys(actual), Object.keys(expected));
+for (const name of Object.keys(expected)) {
+  assert.deepEqual(actual[name], expected[name], name);
+}
+console.log('Prepared preferences, environment files, and shims match the CLI setup sequence.');
+
+// Mutating the reference must not change the case's own configuration.
+const config = path.join(prepared, 'config.json');
+const original = fs.readFileSync(config, 'utf8');
+fs.writeFileSync(path.join(reference, 'config.json'), '{}');
+assert.equal(fs.readFileSync(config, 'utf8'), original);
+console.log('Configuration changes remain isolated to their own home.');