Add new ESRP + changesets publish system

.changeset/esrp-publish.md

@@ -0,0 +1,5 @@
+---
+"@typescript/vfs": patch
+---
+
+Switch to new release pipeline

.github/workflows/CI.yml

@@ -102,12 +102,23 @@ jobs:
         # git checkout origin/v2 -- <ignored files>
         pnpm changeset status --since=origin/v2
 
+  release-pipeline:
+    name: release pipeline
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+    - uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0
+      with:
+        node-version: 'lts/*'
+    - run: node build/azure-pipelines/pack-changed-packages.mjs --check-stage-capacity
+
   required:
     runs-on: ubuntu-latest
     if: ${{ always() }}
     needs:
       - tests
       - changesets
+      - release-pipeline
 
     steps:
       - name: Check required jobs

.github/workflows/publish-packages.yml

@@ -28,14 +28,9 @@ jobs:
       - uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0
         with:
           node-version: "22.x"
-          registry-url: "https://registry.npmjs.org/"
       - uses: pnpm/action-setup@ea17c68df8912ef543352723c149a84f56e3d413 # v6.1.0
 
-      - name: Prepare website v2
-        run: |
-          pnpm install
-          pnpm bootstrap
-          pnpm build
+      - run: pnpm install
 
       - uses: azure/login@a641126d1b8aa4d1fa005f4f92df94a3a4c4c906 # v3.1.0
         with:
@@ -65,6 +60,5 @@ jobs:
       - uses: changesets/action@ae32849d5ba541f9ae29e40e22a623bc13562f51 # v2.1.2
         with:
           github-token: ${{ steps.app-token.outputs.token }}
-          publish-script: pnpm ci:publish
-        env:
-          NODE_AUTH_TOKEN: ${{ secrets.NPM_PUBLISH_TOKEN }}
+          # Packages are published to npm by build/azure-pipelines/release.yml.
+          publish-script: pnpm changeset git-tag

build/azure-pipelines/pack-changed-packages.mjs

@@ -0,0 +1,203 @@
+import { execFileSync } from "node:child_process";
+import fs from "node:fs";
+import path from "node:path";
+
+const root = process.cwd();
+const releasePipelinePath = path.join(root, "build/azure-pipelines/release.yml");
+const checkStageCapacity = process.argv.includes("--check-stage-capacity");
+const outDir = process.argv.find(arg => !arg.startsWith("--") && arg !== process.argv[0] && arg !== process.argv[1]) || process.env.BUILD_ARTIFACTSTAGINGDIRECTORY;
+
+function readJson(file) {
+  return JSON.parse(fs.readFileSync(file, "utf8"));
+}
+
+function collectWorkspacePackages() {
+  const packages = new Map();
+  for (const dir of fs.readdirSync(path.join(root, "packages"))) {
+    const packageJsonPath = path.join(root, "packages", dir, "package.json");
+    if (!fs.existsSync(packageJsonPath)) continue;
+
+    const packageJson = readJson(packageJsonPath);
+    if (!packageJson.name || packageJson.private) continue;
+
+    packages.set(packageJson.name, {
+      dir: path.relative(root, path.dirname(packageJsonPath)),
+      packageJson,
+    });
+  }
+  return packages;
+}
+
+function dependenciesOnPackedPackages(pkg, packedNames) {
+  const dependencies = new Set();
+  for (const section of ["dependencies", "peerDependencies", "optionalDependencies"]) {
+    for (const dependency of Object.keys(pkg.packageJson[section] || {})) {
+      if (packedNames.has(dependency)) dependencies.add(dependency);
+    }
+  }
+  return dependencies;
+}
+
+function computeDepths(packages, packageNames) {
+  const packageNameSet = new Set(packageNames);
+  const memo = new Map();
+  const visiting = new Set();
+
+  function depth(name) {
+    if (memo.has(name)) return memo.get(name);
+    if (visiting.has(name)) throw new Error(`Workspace dependency cycle involving ${name}`);
+
+    visiting.add(name);
+    const deps = dependenciesOnPackedPackages(packages.get(name), packageNameSet);
+    const value = deps.size === 0 ? 0 : 1 + Math.max(...[...deps].map(depth));
+    visiting.delete(name);
+    memo.set(name, value);
+    return value;
+  }
+
+  for (const name of packageNames) depth(name);
+  return memo;
+}
+
+function validateReleasePipelineStageCapacity(packages) {
+  const allPackageNames = [...packages.keys()];
+  const maxRequiredDepth = Math.max(0, ...computeDepths(packages, allPackageNames).values());
+  const releaseYaml = fs.readFileSync(releasePipelinePath, "utf8");
+  const stageNumbers = [...releaseYaml.matchAll(/^\s+-\s+index:\s+(\d+)\s*$/gm)]
+    .map(match => Number(match[1]));
+  const maxAvailableStage = Math.max(-1, ...stageNumbers);
+  const contiguous = stageNumbers.length === maxAvailableStage + 1 && stageNumbers.every((n, i) => n === i);
+
+  if (!contiguous || maxRequiredDepth !== maxAvailableStage) {
+    throw new Error(`Workspace publish graph requires stages 0-${maxRequiredDepth}, but ${releasePipelinePath} defines stages ${stageNumbers.join(", ") || "(none)"}`);
+  }
+
+  console.log(`Workspace publish graph requires stages 0-${maxRequiredDepth}; release pipeline defines stages 0-${maxAvailableStage}.`);
+  return maxAvailableStage + 1;
+}
+
+const packages = collectWorkspacePackages();
+const maxStages = validateReleasePipelineStageCapacity(packages);
+
+if (checkStageCapacity) {
+  process.exit(0);
+}
+
+if (!outDir) {
+  throw new Error("Usage: node build/azure-pipelines/pack-changed-packages.mjs <artifact-staging-directory>");
+}
+
+// PRs are always squash merged, so a version bump from the "Version Packages"
+// PR is always in the tip commit of its own push.
+function readPreviousPackageJson(pkg) {
+  try {
+    return JSON.parse(execFileSync("git", ["show", `HEAD~1:${pkg.dir}/package.json`], { encoding: "utf8" }));
+  }
+  catch {
+    return undefined;
+  }
+}
+
+function versionChanged(pkg) {
+  const previous = readPreviousPackageJson(pkg);
+  return previous?.version !== pkg.packageJson.version
+    || previous?.name !== pkg.packageJson.name
+    || !!previous?.private;
+}
+
+// changesets/action only publishes once every changeset has been consumed by
+// a "Version Packages" PR, so do the same. Otherwise merging a PR that adds a
+// new package would publish it at its placeholder version.
+function findPendingChangesets() {
+  const changesetDir = path.join(root, ".changeset");
+  const preStatePath = path.join(changesetDir, "pre.json");
+  const consumed = new Set(fs.existsSync(preStatePath) ? readJson(preStatePath).changesets : []);
+  return fs.readdirSync(changesetDir)
+    .filter(file => file.endsWith(".md") && file !== "README.md")
+    .filter(file => !consumed.has(file.slice(0, -".md".length)));
+}
+
+function findChangedPackageNames() {
+  const releaseBranch = `refs/heads/${readJson(path.join(root, ".changeset/config.json")).baseBranch}`;
+  const sourceBranch = process.env.BUILD_SOURCEBRANCH;
+  if (sourceBranch !== undefined && sourceBranch !== releaseBranch) {
+    console.log(`Building ${sourceBranch}, not ${releaseBranch}; not staging any packages.`);
+    return [];
+  }
+
+  const pendingChangesets = findPendingChangesets();
+  if (pendingChangesets.length) {
+    console.log(`Changesets are still pending (${pendingChangesets.join(", ")}); not staging any packages.`);
+    return [];
+  }
+
+  const changed = [...packages.values()].filter(versionChanged);
+  for (const pkg of changed) {
+    // ESRP publishes to the publishConfig.tag dist-tag, or latest without one.
+    if (pkg.packageJson.version.includes("-") && !pkg.packageJson.publishConfig?.tag) {
+      throw new Error(`${pkg.packageJson.name}@${pkg.packageJson.version} is a prerelease but has no publishConfig.tag, so it would be published as latest.`);
+    }
+  }
+  return changed.map(pkg => pkg.packageJson.name);
+}
+
+const changedPackageNames = findChangedPackageNames();
+
+let packed = [];
+if (changedPackageNames.length) {
+  const packedDir = path.join(outDir, "packed");
+  fs.mkdirSync(packedDir, { recursive: true });
+
+  const filters = changedPackageNames.flatMap(name => ["--filter", name]);
+  const output = execFileSync("pnpm", ["-r", ...filters, "pack", "--pack-destination", packedDir, "--json"], {
+    encoding: "utf8",
+    stdio: ["ignore", "pipe", "inherit"],
+  });
+  const parsed = output.trim() ? JSON.parse(output) : [];
+  packed = Array.isArray(parsed) ? parsed : [parsed];
+
+  for (const packedPackage of packed) {
+    packedPackage.filename = path.join(packedDir, path.basename(packedPackage.filename));
+  }
+}
+
+packed = packed.filter(pkg => packages.has(pkg.name));
+const packedNames = packed.map(pkg => pkg.name);
+const packedByName = new Map(packed.map(pkg => [pkg.name, pkg]));
+const depths = computeDepths(packages, packedNames);
+const manifest = { stages: Array.from({ length: maxStages }, () => []) };
+for (let i = 0; i < maxStages; i++) {
+  fs.mkdirSync(path.join(outDir, `stage-${i}`), { recursive: true });
+}
+
+for (const name of packedNames) {
+  const depth = depths.get(name);
+  if (depth >= maxStages) {
+    throw new Error(`${name} requires publish stage ${depth}, but max stage is ${maxStages - 1}`);
+  }
+
+  const packedPackage = packedByName.get(name);
+  const filename = path.basename(packedPackage.filename);
+  const stageDir = path.join(outDir, `stage-${depth}`);
+  fs.mkdirSync(stageDir, { recursive: true });
+  fs.copyFileSync(packedPackage.filename, path.join(stageDir, filename));
+  manifest.stages[depth].push({
+    name,
+    version: packedPackage.version,
+    filename,
+  });
+}
+
+fs.rmSync(path.join(outDir, "packed"), { recursive: true, force: true });
+
+for (let i = 0; i < maxStages; i++) {
+  console.log(`Stage ${i}: ${manifest.stages[i].map(pkg => `${pkg.name}@${pkg.version}`).join(", ") || "(empty)"}`);
+  console.log(`##vso[task.setvariable variable=PublishStage${i};isOutput=true]${manifest.stages[i].length > 0}`);
+}
+
+const shouldPublish = packedNames.length > 0;
+console.log(`##vso[task.setvariable variable=ShouldPublish]${shouldPublish}`);
+console.log(`##vso[task.setvariable variable=ShouldPublish;isOutput=true]${shouldPublish}`);
+if (shouldPublish && process.env.BUILD_BUILDNUMBER) {
+  console.log(`##vso[build.updatebuildnumber]${process.env.BUILD_BUILDNUMBER}-release`);
+}