From 8ea4c3b722708a4f149d74e342237bcb9fb79026 Mon Sep 17 00:00:00 2001 From: Theo Browne Date: Sun, 8 Nov 2026 17:06:25 -0800 Subject: [PATCH] fix(release): Windows signing retries transient Azure Trusted Signing errors Azure Trusted Signing sometimes answers with InternalError, which failed both Windows release legs three times in one hour. jsign now retries only recognized transient errors (InternalError, HTTP 429/5xx, dropped connections) up to three times with backoff. Each attempt signs a fresh copy that replaces the file only after jsign succeeds, so a failed attempt never changes the input. Credential and input errors still fail at once. Co-Authored-By: Claude Opus 5.5 --- scripts/lib/windows-jsign.test.ts | 152 ++++++++++++++++++++++++++++++ scripts/lib/windows-jsign.ts | 115 ++++++++++++++++------ 2 files changed, 239 insertions(+), 28 deletions(-) create mode 100644 scripts/lib/windows-jsign.test.ts diff --git a/scripts/lib/windows-jsign.test.ts b/scripts/lib/windows-jsign.test.ts new file mode 100644 index 0000000000000..6174db045c108 --- /dev/null +++ b/scripts/lib/windows-jsign.test.ts @@ -0,0 +1,152 @@ +import * as NodeServices from "@effect/platform-node/NodeServices"; +import { assert, it } from "@effect/vitest"; +import * as Cause from "effect/Cause"; +import * as Clock from "effect/Clock"; +import * as ConfigProvider from "effect/ConfigProvider"; +import * as Duration from "effect/Duration"; +import * as Effect from "effect/Effect"; +import * as Exit from "effect/Exit"; +import * as FileSystem from "effect/FileSystem"; +import * as Option from "effect/Option"; +import * as Path from "effect/Path"; +import * as Sink from "effect/Sink"; +import * as Stream from "effect/Stream"; +import { ChildProcess, ChildProcessSpawner } from "effect/process"; + +import { signWithJsign } from "./windows-jsign.ts"; + +const UNSIGNED = "MZ unsigned payload"; +const TRANSIENT_CHAIN = + "jsign: Unable to retrieve the certificate chain 'acct/profile'\nCaused by: java.io.IOException: InternalError - An internal error has occurred.\n"; +const TRANSIENT_SIGN = + "jsign: Couldn't sign t3.exe\nCaused by: java.security.GeneralSecurityException: java.io.IOException: InternalError - An internal error has occurred.\n"; +const BAD_SECRET = + "jsign: Unable to retrieve the certificate chain 'acct/profile'\nCaused by: java.io.IOException: AADSTS7000215: Invalid client secret provided.\n"; + +/** What the stub does with the file jsign was given: sign it, or write junk and fail. */ +type Reply = { readonly exitCode: number; readonly stderr?: string; readonly write?: string }; + +/** Stands in for jsign. Records the bytes it received and applies each reply in turn. */ +function stubJsign(replies: ReadonlyArray) { + const inputs: Array<{ readonly path: string; readonly content: string }> = []; + const spawner = ChildProcessSpawner.make((command) => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const args = ChildProcess.isStandardCommand(command) ? command.args.map(String) : []; + const target = args.at(-1)!; + inputs.push({ path: target, content: yield* fs.readFileString(target) }); + const reply = replies[inputs.length - 1] ?? replies.at(-1)!; + if (reply.write !== undefined) yield* fs.writeFileString(target, reply.write); + return ChildProcessSpawner.makeHandle({ + pid: ChildProcessSpawner.ProcessId(1), + exitCode: Effect.succeed(ChildProcessSpawner.ExitCode(reply.exitCode)), + isRunning: Effect.succeed(false), + kill: () => Effect.void, + unref: Effect.succeed(Effect.void), + stdin: Sink.drain, + stdout: Stream.empty, + stderr: reply.stderr ? Stream.make(new TextEncoder().encode(reply.stderr)) : Stream.empty, + all: Stream.empty, + getInputFd: () => Sink.drain, + getOutputFd: () => Stream.empty, + }); + }).pipe(Effect.provide(NodeServices.layer), Effect.orDie), + ); + return { inputs, spawner }; +} + +const signed = (content: string): Reply => ({ exitCode: 0, write: `${content}+signature` }); + +const runSign = Effect.fn("test.runSign")(function* (replies: ReadonlyArray) { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const dir = yield* fs.makeTempDirectoryScoped({ prefix: "t3-jsign-" }); + const file = path.join(dir, "t3.exe"); + yield* fs.writeFileString(file, UNSIGNED); + const stub = stubJsign(replies); + const sleeps: Array = []; + const clock = yield* Clock.Clock; + const exit = yield* signWithJsign(file).pipe( + Effect.provideService(ChildProcessSpawner.ChildProcessSpawner, stub.spawner), + // Records the backoff instead of waiting it out. + Effect.provideService(Clock.Clock, { + ...clock, + sleep: (duration) => Effect.sync(() => sleeps.push(Duration.toSeconds(duration))), + }), + Effect.provide( + ConfigProvider.layer( + ConfigProvider.fromEnv({ + env: { + AZURE_TRUSTED_SIGNING_ENDPOINT: "https://eus.codesigning.azure.net/", + AZURE_TRUSTED_SIGNING_ACCOUNT_NAME: "acct", + AZURE_TRUSTED_SIGNING_CERTIFICATE_PROFILE_NAME: "profile", + }, + }), + ), + ), + Effect.exit, + ); + return { + ok: Exit.isSuccess(exit), + error: Exit.isFailure(exit) + ? Option.getOrUndefined(Cause.findErrorOption(exit.cause)) + : undefined, + sleeps, + inputs: stub.inputs, + // jsign only ever sees a disposable copy, never the file itself. + signedInPlace: stub.inputs.some((input) => input.path === file), + content: yield* fs.readFileString(file), + // Only the original file is left: every attempt's copy is cleaned up. + entries: yield* fs.readDirectory(dir), + }; +}); + +it.layer(NodeServices.layer)("signWithJsign", (it) => { + it.effect("retries transient Azure errors on a fresh copy and keeps only the signed file", () => + Effect.gen(function* () { + const result = yield* runSign([ + { exitCode: 1, stderr: TRANSIENT_CHAIN }, + // A partial write must not reach the next attempt or the original. + { exitCode: 1, stderr: TRANSIENT_SIGN, write: "MZ half-written" }, + signed(UNSIGNED), + ]); + assert.isTrue(result.ok); + assert.strictEqual(result.content, `${UNSIGNED}+signature`); + assert.deepStrictEqual( + result.inputs.map((input) => input.content), + [UNSIGNED, UNSIGNED, UNSIGNED], + ); + assert.isFalse(result.signedInPlace); + assert.deepStrictEqual(result.sleeps, [5, 10]); + assert.deepStrictEqual(result.entries, ["t3.exe"]); + }), + ); + + it.effect("fails once without retrying on errors that are not transient", () => + Effect.gen(function* () { + for (const reply of [ + { exitCode: 1, stderr: BAD_SECRET }, + { exitCode: 1, stderr: "jsign: Unsupported file: t3.exe\n", write: "MZ half-written" }, + ]) { + const result = yield* runSign([reply, signed(UNSIGNED)]); + assert.strictEqual(result.error?._tag, "JsignFailedError"); + assert.isFalse(result.error?._tag === "JsignFailedError" && result.error.transient); + assert.strictEqual(result.inputs.length, 1); + assert.deepStrictEqual(result.sleeps, []); + assert.strictEqual(result.content, UNSIGNED); + assert.deepStrictEqual(result.entries, ["t3.exe"]); + } + }), + ); + + it.effect("gives up after three retries and leaves the input unsigned", () => + Effect.gen(function* () { + const result = yield* runSign([{ exitCode: 1, stderr: TRANSIENT_SIGN, write: "MZ junk" }]); + assert.isTrue(result.error?._tag === "JsignFailedError" && result.error.transient); + assert.strictEqual(result.inputs.length, 4); + assert.deepStrictEqual(result.sleeps, [5, 10, 20]); + assert.strictEqual(result.content, UNSIGNED); + assert.deepStrictEqual(result.entries, ["t3.exe"]); + }), + ); +}); diff --git a/scripts/lib/windows-jsign.ts b/scripts/lib/windows-jsign.ts index ad95aebadb863..b9115640711d1 100644 --- a/scripts/lib/windows-jsign.ts +++ b/scripts/lib/windows-jsign.ts @@ -5,18 +5,54 @@ import * as NodeServices from "@effect/platform-node/NodeServices"; import * as Config from "effect/Config"; import * as Effect from "effect/Effect"; +import * as FileSystem from "effect/FileSystem"; +import * as Path from "effect/Path"; +import * as Schedule from "effect/Schedule"; import * as Schema from "effect/Schema"; +import * as Stream from "effect/Stream"; import { ChildProcess, ChildProcessSpawner } from "effect/process"; +// Upstream failures worth another attempt: Azure's `InternalError` (seen in +// release runs on both the certificate chain and the sign call), HTTP 429/5xx, +// and dropped connections. Credential, permission and input errors never match. +const TRANSIENT_JSIGN_ERROR = + /\bInternalError - |HTTP Error (?:429|5\d\d)\b|java\.net\.(?:SocketTimeoutException|ConnectException)\b|Connection reset\b/; +const JSIGN_RETRIES = 3; +const STDERR_TAIL_CHARS = 64 * 1024; + export class JsignFailedError extends Schema.TaggedError()("JsignFailedError", { file: Schema.String, exitCode: Schema.Number, + transient: Schema.Boolean, }) { override get message() { - return `jsign exited with code ${String(this.exitCode)} signing ${this.file}.`; + return `jsign exited with code ${String(this.exitCode)} signing ${this.file}${this.transient ? " (transient Azure Trusted Signing error)" : ""}.`; } } +/** Runs jsign, forwarding its stderr as it arrives and returning a tail of it for classification. */ +const runJsign = Effect.fn("runJsign")(function* (jsign: string, args: ReadonlyArray) { + const spawner = yield* ChildProcessSpawner.ChildProcessSpawner; + const child = yield* spawner.spawn( + ChildProcess.make(jsign, args, { stdout: "inherit", stderr: "pipe" }), + ); + const decoder = new TextDecoder(); + let stderr = ""; + const [exitCode] = yield* Effect.all( + [ + child.exitCode.pipe(Effect.map(Number)), + Stream.runForEach(child.stderr, (chunk) => + Effect.sync(() => { + process.stderr.write(chunk); + stderr = (stderr + decoder.decode(chunk, { stream: true })).slice(-STDERR_TAIL_CHARS); + }), + ), + ], + { concurrency: "unbounded" }, + ); + return { exitCode, stderr }; +}); + export const signWithJsign = Effect.fn("signWithJsign")(function* (file: string) { const { jsign, endpoint, account, profile } = yield* Config.all({ jsign: Config.String("T3CODE_JSIGN").pipe(Config.withDefault("jsign")), @@ -24,34 +60,57 @@ export const signWithJsign = Effect.fn("signWithJsign")(function* (file: string) account: Config.String("AZURE_TRUSTED_SIGNING_ACCOUNT_NAME"), profile: Config.String("AZURE_TRUSTED_SIGNING_CERTIFICATE_PROFILE_NAME"), }); - const spawner = yield* ChildProcessSpawner.ChildProcessSpawner; - const child = yield* spawner.spawn( - ChildProcess.make( - jsign, - [ - "--storetype", - "TRUSTEDSIGNING", - "--keystore", - new URL(endpoint).host, - "--alias", - `${account}/${profile}`, - "--alg", - "SHA-256", - "--tsaurl", - "http://timestamp.acs.microsoft.com", - "--tsmode", - "RFC3161", - // jsign appends a nested signature by default; signtool replaces. - "--replace", - file, - ], - { stdout: "inherit", stderr: "inherit" }, - ), + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const args = [ + "--storetype", + "TRUSTEDSIGNING", + "--keystore", + new URL(endpoint).host, + "--alias", + `${account}/${profile}`, + "--alg", + "SHA-256", + "--tsaurl", + "http://timestamp.acs.microsoft.com", + "--tsmode", + "RFC3161", + // jsign appends a nested signature by default; signtool replaces. + "--replace", + ]; + + let attempt = 0; + // jsign rewrites its input in place, so each attempt signs a fresh copy and + // only a copy jsign finished with replaces the file (same directory, so the + // rename is atomic). A failed attempt never touches the original. + const signCopy = Effect.gen(function* () { + attempt++; + const dir = yield* fs.makeTempDirectoryScoped({ + directory: path.dirname(file), + prefix: ".jsign-", + }); + const copy = path.join(dir, path.basename(file)); + yield* fs.copyFile(file, copy); + const { exitCode, stderr } = yield* runJsign(jsign, [...args, copy]); + if (exitCode !== 0) { + const transient = TRANSIENT_JSIGN_ERROR.test(stderr); + if (transient && attempt <= JSIGN_RETRIES) { + yield* Effect.logWarning( + `[jsign] Transient Azure Trusted Signing error signing ${file} (attempt ${String(attempt)} of ${String(JSIGN_RETRIES + 1)}); retrying.`, + ); + } + return yield* new JsignFailedError({ file, exitCode, transient }); + } + yield* fs.rename(copy, file); + }).pipe(Effect.scoped); + + yield* signCopy.pipe( + Effect.retry({ + schedule: Schedule.exponential("5 seconds"), + times: JSIGN_RETRIES, + while: (error) => error._tag === "JsignFailedError" && error.transient, + }), ); - const exitCode = Number(yield* child.exitCode); - if (exitCode !== 0) { - return yield* new JsignFailedError({ file, exitCode }); - } }); export default function sign(configuration: { readonly path: string }): Promise {