diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index f073e4849..ffc448036 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -651,9 +651,9 @@ jobs: # bsdtar (libarchive-tools) is fpm's mtree generator for the pacman target. # rpmbuild (rpm) is what fpm shells out to for the rpm target; without it the - # build fails at packaging, not at config parse. + # build fails at packaging, not at config parse. zsync generates AppImageUpdate sidecars. - name: Install Linux packaging dependencies - run: sudo apt-get update && sudo apt-get install -y libarchive-tools rpm + run: sudo apt-get update && sudo apt-get install -y libarchive-tools rpm zsync # patchelf is what build-linux-compositor-addon.mjs renames the ffmpeg symbols # with, so the addon cannot bind to Chromium's bundled ffmpeg — an unconditional @@ -708,6 +708,21 @@ jobs: exit 1 fi + - name: Verify AppImage update information and sidecars + run: | + VERSION="$(node -p 'require("./package.json").version')" + EXPECTED='gh-releases-zsync|getopenscreen|openscreen|latest|Openscreen-Linux-*.AppImage.zsync' + # Helper-only prereleases can be selected by AppImageUpdate's latest-all channel. + # RCs keep using electron-updater; their .zsync is available by its explicit URL. + if [[ "$VERSION" == *-* ]]; then EXPECTED=''; fi + while IFS= read -r -d '' IMAGE; do + test -s "${IMAGE}.zsync" \ + || { echo "::error::${IMAGE}.zsync missing — external AppImage updates have no sidecar"; exit 1; } + INFO="$("$IMAGE" --appimage-updateinformation)" + [[ "$INFO" == "$EXPECTED" ]] \ + || { echo "::error::Unexpected AppImage update information in ${IMAGE}: ${INFO}"; exit 1; } + done < <(find release -type f -name '*.AppImage' -print0) + # The Linux counterpart of the Windows job's "Verify native binaries load under # package identity", added for the same reason and after the same kind of miss: # 1.9.1 shipped three sonames that nothing declared and nothing bundled — libgbm.so.1 @@ -747,16 +762,12 @@ jobs: name: openscreen-linux path: | release/**/*.AppImage + release/**/*.AppImage.zsync release/**/*.deb release/**/*.pacman release/**/*.rpm release/**/latest-linux.yml - # Still no *.zsync, and there never will be: zsync was electron-updater's old delta - # format and app-builder-lib 26.x dropped it in favour of the block map it embeds - # directly in the AppImage. That glob had matched nothing since the dependency bump, - # silently — `if-no-files-found: error` evaluates the union of these patterns, so one - # dead glob among live ones never fails. Hence the explicit assertion below. - # + # .zsync serves external AppImage tools; electron-updater uses the embedded blockmap. # latest-linux.yml serves all four formats from one file: each updater picks its own # extension out of the `files:` list. app-update.yml and the `package-type` marker are # already inside the deb/rpm/pacman payloads — only this feed was missing. diff --git a/electron-builder.json5 b/electron-builder.json5 index 8da5be310..8b3d17f8f 100644 --- a/electron-builder.json5 +++ b/electron-builder.json5 @@ -47,6 +47,9 @@ // (unknown scene fields are #[serde(default)]) rather than erroring. See the script header // and technical-documentation/engineering/build-and-packaging.md. "beforePack": "scripts/before-pack.cjs", + // Standard AppImage updates need .upd_info and a .zsync alongside electron-updater's blockmap. + "artifactBuildCompleted": "scripts/appimage-updates.cjs", + "afterAllArtifactBuild": "scripts/appimage-updates.cjs", "npmRebuild": true, // sharp ships ABI-stable (napi) prebuilt binaries with bundled libvips. Building it from source // needs a system libvips we don't provide and breaks on CI/local ("vips-cpp.42 not found"), so we diff --git a/scripts/appimage-updates.cjs b/scripts/appimage-updates.cjs new file mode 100644 index 000000000..f4a1a6e8d --- /dev/null +++ b/scripts/appimage-updates.cjs @@ -0,0 +1,131 @@ +// Standard AppImage update information complements electron-updater's embedded blockmap. +// artifactBuildCompleted runs before artifactCreated, where electron-builder schedules the feed +// and upload. Mutating the file after that point would leave latest-linux.yml's hashes stale. +const childProcess = require("node:child_process"); +const fs = require("node:fs"); +const path = require("node:path"); + +const REPOSITORY = "getopenscreen/openscreen"; +const ASSET_PATTERN = "Openscreen-Linux-*.AppImage.zsync"; + +function readBuffer(fd, offset, length, limit) { + if ( + !Number.isSafeInteger(offset) || + !Number.isSafeInteger(length) || + offset < 0 || + length < 0 || + offset + length > limit + ) { + throw new Error("AppImage ELF section extends beyond the artifact"); + } + const buffer = Buffer.alloc(length); + let read = 0; + while (read < length) { + const count = fs.readSync(fd, buffer, read, length - read, offset + read); + if (count === 0) throw new Error("AppImage ELF section is truncated"); + read += count; + } + return buffer; +} + +function findUpdateSection(fd, payloadSize) { + const header = readBuffer(fd, 0, 64, payloadSize); + if ( + header.subarray(0, 4).toString("hex") !== "7f454c46" || + header.subarray(8, 11).toString("hex") !== "414902" || + ![1, 2].includes(header[4]) || + header[5] !== 1 + ) { + throw new Error("Expected a little-endian ELF32 or ELF64 type-2 AppImage"); + } + const is64 = header[4] === 2; + const tableOffset = is64 ? Number(header.readBigUInt64LE(40)) : header.readUInt32LE(32); + const entrySize = header.readUInt16LE(is64 ? 58 : 46); + const entryCount = header.readUInt16LE(is64 ? 60 : 48); + const namesIndex = header.readUInt16LE(is64 ? 62 : 50); + if (entrySize !== (is64 ? 64 : 40) || namesIndex >= entryCount) { + throw new Error("AppImage ELF section table is invalid"); + } + const table = readBuffer(fd, tableOffset, entrySize * entryCount, payloadSize); + const section = (index) => { + const offset = index * entrySize; + return { + name: table.readUInt32LE(offset), + offset: is64 ? Number(table.readBigUInt64LE(offset + 24)) : table.readUInt32LE(offset + 16), + size: is64 ? Number(table.readBigUInt64LE(offset + 32)) : table.readUInt32LE(offset + 20), + }; + }; + const namesSection = section(namesIndex); + const names = readBuffer(fd, namesSection.offset, namesSection.size, payloadSize); + for (let index = 0; index < entryCount; index++) { + const entry = section(index); + const end = names.indexOf(0, entry.name); + if (end < 0) throw new Error("AppImage ELF section name is invalid"); + if (names.subarray(entry.name, end).toString("utf8") === ".upd_info") { + // Validate the complete reserved range before changing any bytes. + readBuffer(fd, entry.offset, entry.size, payloadSize); + return entry; + } + } + throw new Error("AppImage runtime has no reserved .upd_info section"); +} + +exports.artifactBuildCompleted = async function artifactBuildCompleted(event) { + if (event.target?.name !== "appImage" || !event.file?.endsWith(".AppImage")) return; + const version = event.packager.appInfo.version; + // Stable installs discover stable releases. AppImageUpdate's latest-all / latest-pre + // choose a release before matching assets, so helper-only prereleases such as + // v0.0.0-stt-models would break RC discovery. Leave RCs without automatic discovery; + // their tagged .zsync still supports an explicitly selected update. + const information = version.includes("-") + ? Buffer.alloc(0) + : Buffer.from(`gh-releases-zsync|${REPOSITORY.replace("/", "|")}|latest|${ASSET_PATTERN}`); + const blockMapSize = event.updateInfo?.blockMapSize; + const fd = fs.openSync(event.file, "r+"); + try { + const size = fs.fstatSync(fd).size; + if (!Number.isSafeInteger(blockMapSize) || blockMapSize <= 0 || blockMapSize + 4 >= size) { + throw new Error("AppImage embedded blockmap size is invalid"); + } + const trailer = readBuffer(fd, size - 4, 4, size); + if (trailer.readUInt32BE(0) !== blockMapSize) { + throw new Error("AppImage embedded blockmap trailer disagrees with update metadata"); + } + const payloadSize = size - blockMapSize - 4; + const section = findUpdateSection(fd, payloadSize); + if (information.length >= section.size) { + throw new Error("AppImage update information does not fit in .upd_info"); + } + const padded = Buffer.alloc(section.size); + information.copy(padded); + fs.ftruncateSync(fd, payloadSize); + fs.writeSync(fd, padded, 0, padded.length, section.offset); + } finally { + fs.closeSync(fd); + } + // This internal helper is verified against app-builder-lib 26.15.3. Recheck its return + // shape and artifactBuildCompleted ordering when upgrading electron-builder. + const { appendBlockmap } = require("app-builder-lib/out/targets/differentialUpdateInfoBuilder"); + event.updateInfo = await appendBlockmap(event.file); + const filename = path.basename(event.file); + childProcess.execFileSync( + "zsyncmake", + [ + "-e", + "-f", + filename, + "-u", + `https://github.com/${REPOSITORY}/releases/download/v${version}/${encodeURIComponent(filename)}`, + "-o", + `${event.file}.zsync`, + event.file, + ], + { stdio: "inherit" }, + ); +}; + +exports.afterAllArtifactBuild = function afterAllArtifactBuild(result) { + return result.artifactPaths + .filter((file) => file.endsWith(".AppImage")) + .map((file) => `${file}.zsync`); +}; diff --git a/scripts/appimage-updates.test.mjs b/scripts/appimage-updates.test.mjs new file mode 100644 index 000000000..dc47adf58 --- /dev/null +++ b/scripts/appimage-updates.test.mjs @@ -0,0 +1,199 @@ +import childProcess from "node:child_process"; +import { createHash } from "node:crypto"; +import fs from "node:fs"; +import { createRequire } from "node:module"; +import os from "node:os"; +import path from "node:path"; +import { inflateRawSync } from "node:zlib"; +import { afterEach, describe, expect, it, vi } from "vitest"; + +const require = createRequire(import.meta.url); +const { blake2b } = require("@noble/hashes/blake2.js"); +const { appendBlockmap } = require("app-builder-lib/out/targets/differentialUpdateInfoBuilder"); +const { artifactBuildCompleted, afterAllArtifactBuild } = require("./appimage-updates.cjs"); +const temporaryDirectories = []; +const UPDATE_OFFSET = 1536; +const UPDATE_SIZE = 1024; + +afterEach(() => { + vi.restoreAllMocks(); + for (const directory of temporaryDirectories.splice(0)) { + fs.rmSync(directory, { recursive: true, force: true }); + } +}); + +// A small ELF header, section table and reserved runtime area, followed by payload bytes. +// Real legacy/static runtimes use these same ELF32/64 layouts; no Linux executable is needed. +function runtimeFixture({ + is64 = true, + sectionName = ".upd_info", + sectionSize = UPDATE_SIZE, + updateInformation = "", +} = {}) { + const buffer = Buffer.alloc(4096, 0x5a); + buffer.fill(0, 0, 64); + Buffer.from("7f454c46", "hex").copy(buffer); + buffer[4] = is64 ? 2 : 1; + buffer[5] = 1; + Buffer.from("414902", "hex").copy(buffer, 8); + const tableOffset = 512; + const entrySize = is64 ? 64 : 40; + if (is64) buffer.writeBigUInt64LE(BigInt(tableOffset), 40); + else buffer.writeUInt32LE(tableOffset, 32); + buffer.writeUInt16LE(entrySize, is64 ? 58 : 46); + buffer.writeUInt16LE(3, is64 ? 60 : 48); + buffer.writeUInt16LE(1, is64 ? 62 : 50); + buffer.fill(0, tableOffset, tableOffset + entrySize * 3); + const names = Buffer.from(`\0.shstrtab\0${sectionName}\0`); + names.copy(buffer, 1024); + const setSection = (index, name, offset, size) => { + const at = tableOffset + index * entrySize; + buffer.writeUInt32LE(name, at); + if (is64) { + buffer.writeBigUInt64LE(BigInt(offset), at + 24); + buffer.writeBigUInt64LE(BigInt(size), at + 32); + } else { + buffer.writeUInt32LE(offset, at + 16); + buffer.writeUInt32LE(size, at + 20); + } + }; + setSection(1, 1, 1024, names.length); + setSection(2, 11, UPDATE_OFFSET, sectionSize); + buffer.fill(0, UPDATE_OFFSET, UPDATE_OFFSET + UPDATE_SIZE); + Buffer.from(updateInformation).copy(buffer, UPDATE_OFFSET); + return buffer; +} + +async function artifact({ version = "2.0.1", filename, ...fixtureOptions } = {}) { + const directory = fs.mkdtempSync(path.join(os.tmpdir(), "openscreen-appimage-update-")); + temporaryDirectories.push(directory); + const file = path.join(directory, filename ?? `Openscreen-Linux-${version}.AppImage`); + fs.writeFileSync(file, runtimeFixture(fixtureOptions)); + const updateInfo = await appendBlockmap(file); + return { + file, + target: { name: "appImage" }, + packager: { appInfo: { version } }, + updateInfo, + }; +} + +describe("AppImage standard update information", () => { + it.each([ + true, + false, + ])("preserves payload and refreshes all update hashes (ELF64=%s)", async (is64) => { + const event = await artifact({ is64 }); + const before = fs.readFileSync(event.file); + const beforeInfo = event.updateInfo; + const payloadSize = before.length - beforeInfo.blockMapSize - 4; + const zsync = vi.spyOn(childProcess, "execFileSync").mockImplementation(() => { + const finalBytes = fs.readFileSync(event.file); + expect(event.updateInfo.sha512).toBe( + createHash("sha512").update(finalBytes).digest("base64"), + ); + fs.writeFileSync(`${event.file}.zsync`, "generated from final bytes"); + }); + + await artifactBuildCompleted(event); + + const after = fs.readFileSync(event.file); + expect(after.subarray(0, UPDATE_OFFSET)).toEqual(before.subarray(0, UPDATE_OFFSET)); + expect(after.subarray(UPDATE_OFFSET + UPDATE_SIZE, payloadSize)).toEqual( + before.subarray(UPDATE_OFFSET + UPDATE_SIZE, payloadSize), + ); + const information = after.subarray(UPDATE_OFFSET, UPDATE_OFFSET + UPDATE_SIZE); + const end = information.indexOf(0); + expect(information.subarray(0, end).toString()).toBe( + "gh-releases-zsync|getopenscreen|openscreen|latest|Openscreen-Linux-*.AppImage.zsync", + ); + expect(information.subarray(end).every((byte) => byte === 0)).toBe(true); + expect(event.updateInfo).not.toBe(beforeInfo); + expect(event.updateInfo.sha512).not.toBe(beforeInfo.sha512); + expect(event.updateInfo.size).toBe(after.length); + expect(after.readUInt32BE(after.length - 4)).toBe(event.updateInfo.blockMapSize); + const map = JSON.parse( + inflateRawSync(after.subarray(payloadSize, after.length - 4)).toString(), + ); + expect(map.files[0].sizes).toEqual([payloadSize]); + expect(map.files[0].checksums).toEqual([ + Buffer.from(blake2b(after.subarray(0, payloadSize), { dkLen: 18 })).toString("base64"), + ]); + expect(zsync).toHaveBeenCalledWith( + "zsyncmake", + [ + "-e", + "-f", + "Openscreen-Linux-2.0.1.AppImage", + "-u", + "https://github.com/getopenscreen/openscreen/releases/download/v2.0.1/Openscreen-Linux-2.0.1.AppImage", + "-o", + `${event.file}.zsync`, + event.file, + ], + { stdio: "inherit" }, + ); + expect( + afterAllArtifactBuild({ + artifactPaths: [event.file, path.join(path.dirname(event.file), "other.deb")], + }), + ).toEqual([`${event.file}.zsync`]); + }); + + it("leaves RC discovery empty while generating a tagged zsync URL", async () => { + const event = await artifact({ + version: "2.1.0-rc.2", + filename: "Openscreen-Linux-2.1.0-rc.2 beta.AppImage", + updateInformation: "gh-releases-zsync|getopenscreen|openscreen|latest-all|*.AppImage.zsync", + }); + const zsync = vi.spyOn(childProcess, "execFileSync").mockImplementation(() => undefined); + await artifactBuildCompleted(event); + const after = fs.readFileSync(event.file); + // latest-all / latest-pre may select a helper-only release without an AppImage asset. + expect( + after.subarray(UPDATE_OFFSET, UPDATE_OFFSET + UPDATE_SIZE).every((byte) => byte === 0), + ).toBe(true); + expect(event.updateInfo.size).toBe(after.length); + expect(event.updateInfo.sha512).toBe(createHash("sha512").update(after).digest("base64")); + expect(zsync.mock.calls[0][1]).toContain( + "https://github.com/getopenscreen/openscreen/releases/download/v2.1.0-rc.2/Openscreen-Linux-2.1.0-rc.2%20beta.AppImage", + ); + }); + + it.each([ + ["a missing reserved section", { sectionName: ".other" }, /no reserved .upd_info/], + ["an undersized reserved section", { sectionSize: 8 }, /does not fit/], + ["an out-of-range reserved section", { sectionSize: 4096 }, /beyond the artifact/], + ])("rejects %s before changing any bytes", async (_label, fixtureOptions, message) => { + const event = await artifact(fixtureOptions); + const before = fs.readFileSync(event.file); + const zsync = vi.spyOn(childProcess, "execFileSync"); + await expect(artifactBuildCompleted(event)).rejects.toThrow(message); + expect(fs.readFileSync(event.file)).toEqual(before); + expect(zsync).not.toHaveBeenCalled(); + }); + + it.each([ + undefined, + -1, + 0, + 1, + Number.MAX_SAFE_INTEGER, + ])("rejects invalid or stale blockmap size %s before changing any bytes", async (blockMapSize) => { + const event = await artifact(); + event.updateInfo.blockMapSize = blockMapSize; + const before = fs.readFileSync(event.file); + await expect(artifactBuildCompleted(event)).rejects.toThrow(/blockmap/); + expect(fs.readFileSync(event.file)).toEqual(before); + }); + + it.each([ + { target: null }, + { target: { name: "deb" }, file: "missing.deb" }, + { target: { name: "appImage" }, file: "missing.blockmap" }, + ])("ignores an unrelated artifact %j", async (event) => { + const zsync = vi.spyOn(childProcess, "execFileSync"); + await artifactBuildCompleted(event); + expect(zsync).not.toHaveBeenCalled(); + }); +}); diff --git a/technical-documentation/engineering/build-and-packaging.md b/technical-documentation/engineering/build-and-packaging.md index d7608ff0e..11aea19e0 100644 --- a/technical-documentation/engineering/build-and-packaging.md +++ b/technical-documentation/engineering/build-and-packaging.md @@ -23,6 +23,14 @@ OpenScreen builds its renderer, Electron main process, preload bridge, native he `vite.config.ts` uses `vite-plugin-electron` to compile `electron/main.ts` and `electron/preload.ts` into `dist-electron/` while Vite emits the renderer to `dist/`. The main `tsconfig.json` is strict, covers `src` and `electron`, and has `noEmit`; TypeScript is therefore a check while Vite performs emission. `build-vite` is the renderer/Electron-bundle build used when an installer is not needed, whereas `build` continues through electron-builder. +## AppImage updates + +Linux packaging also requires `zsync` (`sudo apt-get install zsync`), which supplies `zsyncmake`. Each AppImage ships a `.zsync` sidecar for external tools such as AppImageUpdate, alongside the embedded blockmap and `latest-linux.yml` used by electron-updater. + +`scripts/appimage-updates.cjs` fills the runtime's reserved `.upd_info` section without moving its appended filesystem. The `artifactBuildCompleted` hook removes the old blockmap, writes the update information, rebuilds the blockmap and replaces the artifact's update metadata **before** electron-builder schedules publication. It then generates `.zsync` from the final bytes; `afterAllArtifactBuild` includes that sidecar when publishing directly. Recheck the internal `appendBlockmap` helper and hook ordering when upgrading electron-builder from 26.15.3. + +Stable builds use `gh-releases-zsync` with the upstream repository's `latest` channel. RCs still produce sidecars with explicit tagged download URLs, but leave external discovery empty: AppImageUpdate's `latest-all` can select a helper-only prerelease with no AppImage asset. RC updates continue through electron-updater. CI checks both policies and uploads the sidecars as release assets. + ## Native artifacts A usable full package depends on generated artifacts that are not committed: