diff --git a/AGENTS.md b/AGENTS.md index 5a803cb..1b772aa 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -18,10 +18,11 @@ ## Important Files - [index.d.ts](index.d.ts): The spec for this package. When the user updates this spec, the agent should update the native implementation to match +- [README.md.template](README.md.template): editable source for the README wrapper content - [src/binding.c](src/binding.c): native implementation - [index.mjs](index.mjs): generated ESM wrapper - [index.js](index.js): CommonJS loader via `node-gyp-build` -- [scripts/codegen-index-mjs.mts](scripts/codegen-index-mjs.mts): generates `index.mjs` from `index.d.ts` +- [scripts/codegen-index-mjs.mts](scripts/codegen-index-mjs.mts): generates `index.mjs` and the README API section from `index.d.ts` - [scripts/zig-build-prebuilds.mts](scripts/zig-build-prebuilds.mts): builds and publishes prebuilds - [test/register-unsafe-pointer-tests.js](test/register-unsafe-pointer-tests.js): shared runtime tests - [dist-test/index.test.js](dist-test/index.test.js): packed-tarball verification test @@ -42,15 +43,17 @@ - `index.d.ts` is the source of truth for the JS-facing API. - `index.mjs` is generated from `index.d.ts`. Do not hand-edit it. +- `README.md` is generated from `README.md.template` and `index.d.ts`. Do not hand-edit it. - `bun run build:js` must be stable. Running it should not introduce formatter drift. ## Native/API Sync Rules - If the public API changes, update all of: + - [index.d.ts](index.d.ts) - [src/binding.c](src/binding.c) - [test/register-unsafe-pointer-tests.js](test/register-unsafe-pointer-tests.js) - - [README.md](README.md) - - generated [index.mjs](index.mjs), via `bun run build:js` + - [README.md.template](README.md.template) if the surrounding docs need changes + - generated [index.mjs](index.mjs) and [README.md](README.md), via `bun run build:js` - Keep the native implementation simple and direct. Avoid duplicating logic, share with helper functions. - The addon is plain C with Node-API. Keep it small and dependency-free. - `binding.gyp` exists for source builds. Do not remove the `node-gyp` fallback. @@ -101,6 +104,6 @@ - Keep docs dry and concise. - Use sentence case. Sentences start with a capital letter and end with punctuation. - Prefer TypeScript in README examples. -- Author the README API section from [index.d.ts](index.d.ts). +- Author the README API section from [index.d.ts](index.d.ts). Edit [README.md.template](README.md.template) for the rest. - Do not soften the safety language. This package can crash the process, corrupt memory, or enable attacker-controlled execution. - Preserve existing naming and packaging conventions unless there is a concrete reason to change them. diff --git a/README.md b/README.md index c0ea20f..92aef69 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,4 @@ + # unsafe-pointer Unsafely turn `ArrayBuffer` values into raw pointers and raw pointers into @@ -11,25 +12,78 @@ especially via FFI like [koffi](https://koffi.dev). ## API +#### unsafePointerOf(buf) + +```ts +export function unsafePointerOf(buf: ArrayBufferLike | ArrayBufferView): T +``` + +Returns a pointer to the first byte of the given `ArrayBuffer` or `ArrayBufferView` as a number. + +[See Bun's explanation on `number` vs `bigint` pointers](https://bun.com/docs/runtime/ffi#pointers) + +```ts +type MyPointerType = number & { __ptr: true } +const ptr: MyPointerType = unsafePointerOf(buffer) +``` + +- **unsafe**: The JavaScript runtime may move or deallocate objects at will, leading to invalid pointer access. Invalid pointer access can lead to attackers controlling your users' computers. +- **see**: `unsafeBigIntPointerOf` for bigint pointers + +#### unsafeBigIntPointerOf(buf) + +```ts +export function unsafeBigIntPointerOf(buf: ArrayBufferLike | ArrayBufferView): T +``` + +Returns a pointer to the first byte of the given `ArrayBuffer` or `ArrayBufferView` as a bigint. +Bigints are slower than numbers but theoretically a safer way to represent pointers. + +[See Bun's explanation on `number` vs `bigint` pointers](https://bun.com/docs/runtime/ffi#pointers) + +```ts +type MyPointerType = bigint & { __ptr: true } +const ptr: MyPointerType = unsafeBigIntPointerOf(buffer) +``` + +- **unsafe**: The JavaScript runtime may move or deallocate objects at will, leading to invalid pointer access. Invalid pointer access can lead to attackers controlling your users' computers. +- **see**: `unsafePointerOf` for number pointers + +#### unsafeArrayBufferAt(ptr, offset, byteLength) + +```ts +export function unsafeArrayBufferAt( + ptr: T, + offset: number | undefined, + byteLength: number, +): ArrayBuffer +``` + +Unsafely create an `ArrayBuffer` aliasing the memory at `pointer + offset` with the given length. + ```ts -import { - unsafePointerOf, - unsafeBigIntPointerOf, - unsafeArrayBufferAt, - unsafeCountNonNullBytes, -} from "unsafe-pointer" +type Point3DPointer = number & { __ptr: true, __type: "Point3D" } +type Point3D = Float32Array & { length: 3, __type: "Point3D" } + +function UnsafePoint32(ptr: Point3DPointer): Point3D { + return new Float32Array(unsafeArrayBufferAt(ptr, 0, 3 * Float32Array.BYTES_PER_ELEMENT)) +} +``` + +- **unsafe**: Accessing arbitrary memory can lead to attackers controlling your users' computers. -const view = new Uint8Array([1, 2, 3, 4]) +#### unsafeCountNonNullBytes(ptr, maxBytes) -const ptr = unsafePointerOf(view) -const bigPtr = unsafeBigIntPointerOf(view) +```ts +export function unsafeCountNonNullBytes(ptr: T, maxBytes: number): number +``` -const alias = new Uint8Array(unsafeArrayBufferAt(ptr, 1, 2)) +Iterates from `ptr` until the first null byte is found, or `maxBytes` bytes are reached. +Returns the number of bytes iterated, or `-1` if no null byte found before `maxBytes`. -console.log([...alias]) // [ 2, 3 ] -alias[0] = 99 -console.log([...view]) // [ 1, 99, 3, 4 ] +Pass `-1` for `maxBytes` to count all bytes (which like `strlen` is unsafe). +```ts function unsafeStringAt(ptr: number) { const length = unsafeCountNonNullBytes(ptr, -1) return new TextDecoder().decode(new Uint8Array(unsafeArrayBufferAt(ptr, 0, length))) @@ -40,20 +94,43 @@ const pointer = unsafePointerOf(cstring) console.log(unsafeStringAt(pointer)) // "Hello, world!" ``` -- `unsafePointerOf(buf: ArrayBuffer | ArrayBufferView): T` - Returns a pointer to the first byte of `buf` as a number. -- `unsafeBigIntPointerOf(buf: ArrayBuffer | ArrayBufferView): T` - Returns a pointer to the first byte of `buf` as a bigint. -- `unsafeArrayBufferAt(ptr: T, offset: number | undefined, byteLength: number): ArrayBuffer` - Creates an `ArrayBuffer` alias for memory at `ptr + offset` with the given `byteLength`. -- `unsafeCountNonNullBytes(ptr: T, maxBytes: number): number` - Iterates from `ptr` until the first null byte is found, or until `maxBytes` - bytes are reached. Returns the number of bytes iterated, or `-1` if no null - byte is found before `maxBytes`. Pass `-1` for `maxBytes` to count without a - bound. - -All APIs are unsafe. The runtime may move or reclaim memory. Invalid pointer -access can lead to crashes, memory corruption, or attacker-controlled execution. +- **unsafe**: Accessing arbitrary memory can lead to attackers controlling your users' computers. + +#### type CString + +```ts +export type CString = { + value: string + ptr: T + offset: number + byteLength: number +} +``` + +A copy of a C UTF-8 string who's properties indicate the original pointer, +offset, and byte length. + +As it is a copy, it's safe to use after the pointer is freed. + +- **warning**: It is unsafe to assume the c string at `ptr + offset` is still valid +- **warning**: It is unsafe to assume there is a null terminator at `ptr + offset + byteLength`. + +#### unsafeCStringAt(ptr, offset, byteLength, result) + +```ts +export function unsafeCStringAt>>( + ptr: T, + offset?: number, + byteLength?: number, + result?: R, +): R & CString +``` + +Copy the C UTF-8 string at `ptr` into a JavaScript string. +If `byteLength` not provided, scans for the closing `\0` character. +Pass a negative `byteLength` to scan at most ABS(byteLength) bytes. + +- **unsafe**: Accessing arbitrary memory can lead to attackers controlling your users' computers. ### TypeScript @@ -65,7 +142,7 @@ Examples: ```ts /** * Emulate bun:ffi interface. - * + * * https://bun.com/docs/runtime/ffi */ import { unsafePointerOf, unsafeArrayBufferAt } from "unsafe-pointer" @@ -108,7 +185,7 @@ mise install bun install bun run typecheck bun run rebuild -bun test +bun run test ``` `bun run rebuild` rebuilds the local addon with `node-gyp`. diff --git a/README.md.template b/README.md.template new file mode 100644 index 0000000..ff51dde --- /dev/null +++ b/README.md.template @@ -0,0 +1,103 @@ + +# unsafe-pointer + +Unsafely turn `ArrayBuffer` values into raw pointers and raw pointers into +`ArrayBuffer` aliases. + +Warning: invalid pointer use can crash the process, corrupt memory, or let +attackers control the machine. Using this package voids your warranty. + +May be useful for interacting with native libraries or doing light witchcraft, +especially via FFI like [koffi](https://koffi.dev). + +## API + +__APIDOCS__ + +### TypeScript + +You may use `number` or `bigint` branded sub-types of your choosing to represent +pointers. This can be useful for type safety (not! runtime! safety!). + +Examples: + +```ts +/** + * Emulate bun:ffi interface. + * + * https://bun.com/docs/runtime/ffi + */ +import { unsafePointerOf, unsafeArrayBufferAt } from "unsafe-pointer" +export type Pointer = number & { __pointer__: null } +export const ptr = unsafePointerOf +export const toArrayBuffer = unsafeArrayBufferAt +``` + +```ts +/** + * Do fancy generics. + */ +import * as unsafe from "unsafe-pointer" +const PointerBrand = Symbol("Pointer") +type Pointer = number & { [PointerBrand]: true, __type__: T } + +export function unsafePointerOf(view: T): Pointer { + return unsafe.unsafePointerOf(view) +} + +type TypedArrayConstructor = { + new (buffer: ArrayBuffer, byteOffset: number, length: number): T + BYTES_PER_ELEMENT: number +} + +export function unsafeTypedArrayAt(TypedArray: TypedArrayConstructor, ptr: Pointer, length: number): T { + const arrayBuffer = unsafe.unsafeArrayBufferAt(ptr, 0, length * TypedArray.BYTES_PER_ELEMENT) + return new TypedArray(arrayBuffer, 0, length) +} +``` + +## Development + +```sh +# Install tools. +curl https://mise.run | sh +mise install + +# Develop. +bun install +bun run typecheck +bun run rebuild +bun run test +``` + +`bun run rebuild` rebuilds the local addon with `node-gyp`. + +## Prebuilds + +```sh +bun run build +``` + +`bun run build` builds the full prebuild matrix with `zig-build` and stages +artifacts under `build/zig-build/` before copying them into `prebuilds/`. + +To build a subset of targets: + +```sh +./scripts/zig-build-prebuilds.mts darwin-arm64 linux-x64-glibc win32-x64 +``` + +Supported targets: + +- `darwin-x64` +- `darwin-arm64` +- `linux-x64-glibc` +- `linux-arm64-glibc` +- `linux-x64-musl` +- `linux-arm64-musl` +- `win32-x64` +- `win32-arm64` + +At runtime, `node-gyp-build` loads the matching prebuild from `prebuilds/` when +available. If no matching prebuild is present, source builds via `node-gyp` +should work. diff --git a/package.json b/package.json index b38431f..0d83b9a 100644 --- a/package.json +++ b/package.json @@ -58,6 +58,6 @@ "zig-build": "github:solarwinds/zig-build#fa7428c0a607e4075172346e4d22f7a19ba68fe0" }, "engines": { - "node": ">=18" + "node": ">=22" } } diff --git a/scripts/codegen-index-mjs.mts b/scripts/codegen-index-mjs.mts index b5ecddf..310efb1 100755 --- a/scripts/codegen-index-mjs.mts +++ b/scripts/codegen-index-mjs.mts @@ -5,16 +5,34 @@ import * as path from "node:path" import * as ts from "typescript" import { fileURLToPath } from "node:url" +type DeclarationSource = { + sourceFile: ts.SourceFile + sourceText: string +} + +type DocumentedExport = { + declaration: string + docComment: string | null + heading: string +} + +type MarkdownCodeBlock = { + code: string + lang: string + startLine: number +} + const __filename = fileURLToPath(import.meta.url) const __dirname = path.dirname(__filename) const repoRoot = path.resolve(__dirname, "..") const declarationsPath = path.join(repoRoot, "index.d.ts") -const outputPath = path.join(repoRoot, "index.mjs") +const readmeOutputPath = path.join(repoRoot, "README.md") +const readmeTemplatePath = path.join(repoRoot, "README.md.template") +const esmOutputPath = path.join(repoRoot, "index.mjs") +const readmeApiDocsPlaceholder = "__APIDOCS__" -export function generateIndexMjs(): void { - const sourceText = readFileSync(declarationsPath, "utf8") - const sourceFile = ts.createSourceFile(declarationsPath, sourceText, ts.ScriptTarget.Latest, true, ts.ScriptKind.TS) - const exportNames = getRuntimeExportNames(sourceFile) +export function generateIndexMjs(declarations: DeclarationSource = loadDeclarations()): void { + const exportNames = getRuntimeExportNames(declarations.sourceFile) if (exportNames.length === 0) { throw new Error(`expected ${path.relative(repoRoot, declarationsPath)} to declare at least one runtime export`) @@ -31,7 +49,29 @@ export function generateIndexMjs(): void { "", ].join("\n") - writeFileSync(outputPath, output) + writeFileSync(esmOutputPath, output) +} + +export function generateReadme(declarations: DeclarationSource = loadDeclarations()): void { + const template = readFileSync(readmeTemplatePath, "utf8") + + if (!template.includes(readmeApiDocsPlaceholder)) { + throw new Error(`expected ${path.relative(repoRoot, readmeTemplatePath)} to contain ${readmeApiDocsPlaceholder}`) + } + + const apiDocs = getDocumentedExports(declarations).map(formatDocumentedExport).join("\n\n") + const output = template.replace(readmeApiDocsPlaceholder, apiDocs) + + validateReadmeTypeScriptBlocks(output) + + writeFileSync(readmeOutputPath, output) +} + +function loadDeclarations(): DeclarationSource { + const sourceText = readFileSync(declarationsPath, "utf8") + const sourceFile = ts.createSourceFile(declarationsPath, sourceText, ts.ScriptTarget.Latest, true, ts.ScriptKind.TS) + + return { sourceFile, sourceText } } function getRuntimeExportNames(sourceFile: ts.SourceFile): string[] { @@ -60,6 +100,270 @@ function getRuntimeExportNames(sourceFile: ts.SourceFile): string[] { return names } +function getDocumentedExports({ sourceFile, sourceText }: DeclarationSource): DocumentedExport[] { + const exports: DocumentedExport[] = [] + + for (const statement of sourceFile.statements) { + if (!hasExportModifier(statement) || !isDocumentedExportStatement(statement)) { + continue + } + + exports.push({ + declaration: statement.getText(sourceFile), + docComment: getDocCommentMarkdown(sourceText, statement), + heading: getHeading(statement, sourceFile), + }) + } + + return exports +} + +function isDocumentedExportStatement( + statement: ts.Statement, +): statement is + | ts.ClassDeclaration + | ts.EnumDeclaration + | ts.FunctionDeclaration + | ts.InterfaceDeclaration + | ts.TypeAliasDeclaration + | ts.VariableStatement { + return ( + ts.isClassDeclaration(statement) || + ts.isEnumDeclaration(statement) || + ts.isFunctionDeclaration(statement) || + ts.isInterfaceDeclaration(statement) || + ts.isTypeAliasDeclaration(statement) || + ts.isVariableStatement(statement) + ) +} + +function formatDocumentedExport(entry: DocumentedExport): string { + const lines = [`#### ${entry.heading}`, "", "```ts", entry.declaration, "```"] + + if (entry.docComment) { + lines.push("", entry.docComment) + } + + return lines.join("\n") +} + +function getHeading(statement: DocumentedExportStatement, sourceFile: ts.SourceFile): string { + if (ts.isFunctionDeclaration(statement)) { + const name = requireStatementName(statement) + const parameters = statement.parameters.map((parameter) => parameter.name.getText(sourceFile)).join(", ") + return `${name}(${parameters})` + } + + if (ts.isTypeAliasDeclaration(statement)) { + return `type ${statement.name.text}` + } + + if (ts.isInterfaceDeclaration(statement)) { + return `interface ${statement.name.text}` + } + + if (ts.isClassDeclaration(statement)) { + return `class ${requireStatementName(statement)}` + } + + if (ts.isEnumDeclaration(statement)) { + return `enum ${statement.name.text}` + } + + return getVariableHeading(statement) +} + +type DocumentedExportStatement = + | ts.ClassDeclaration + | ts.EnumDeclaration + | ts.FunctionDeclaration + | ts.InterfaceDeclaration + | ts.TypeAliasDeclaration + | ts.VariableStatement + +function requireStatementName(statement: ts.ClassDeclaration | ts.FunctionDeclaration): string { + if (!statement.name) { + throw new Error("expected exported declaration to have a name") + } + + return statement.name.text +} + +function getVariableHeading(statement: ts.VariableStatement): string { + const names: string[] = [] + + for (const declaration of statement.declarationList.declarations) { + collectBindingNames(declaration.name, names) + } + + if (names.length === 0) { + throw new Error("expected exported variable declaration to contain at least one binding name") + } + + return names.join(", ") +} + +function getDocCommentMarkdown(sourceText: string, statement: ts.Statement): string | null { + const ranges = ts.getLeadingCommentRanges(sourceText, statement.getFullStart()) ?? [] + + for (let index = ranges.length - 1; index >= 0; index--) { + const range = ranges[index] + if (!range) { + continue + } + + const raw = sourceText.slice(range.pos, range.end) + + if (!raw.startsWith("/**")) { + continue + } + + const lines = raw + .replace(/^\/\*\*\s?/, "") + .replace(/\*\/$/, "") + .split(/\r?\n/u) + .map((line) => line.replace(/^\s*\* ?/u, "").trimEnd()) + + while (lines.length > 0 && lines[0] === "") { + lines.shift() + } + + while (lines.length > 0 && lines.at(-1) === "") { + lines.pop() + } + + return normalizeDocCommentLines(lines).join("\n") + } + + return null +} + +function normalizeDocCommentLines(lines: string[]): string[] { + const normalized: string[] = [] + let inCodeBlock = false + + for (const line of lines) { + if (line.trim() === "") { + normalized.push("") + continue + } + + if (line === "```") { + normalized.push(inCodeBlock ? "```" : "```ts") + inCodeBlock = !inCodeBlock + continue + } + + if (inCodeBlock) { + normalized.push(line) + continue + } + + normalized.push(formatDocCommentText(line)) + } + + return normalized +} + +function formatDocCommentText(line: string): string { + const normalized = line.replace(/\{@link\s+([^}]+)\}/gu, "`$1`") + const tagMatch = normalized.match(/^@([A-Za-z][\w-]*)(?:\s+(.*))?$/u) + if (tagMatch) { + const [, tag, body = ""] = tagMatch + return body.length > 0 ? `- **${tag}**: ${body}` : `- **${tag}**` + } + + return normalized +} + +function validateReadmeTypeScriptBlocks(markdown: string): void { + const diagnostics: string[] = [] + + for (const block of getMarkdownCodeBlocks(markdown)) { + if (!isTypeScriptFence(block.lang)) { + continue + } + + const fileName = `${path.relative(repoRoot, readmeOutputPath)}:${block.startLine}` + const transpileResult = ts.transpileModule(block.code, { + compilerOptions: { + target: ts.ScriptTarget.Latest, + }, + fileName, + reportDiagnostics: true, + }) + + for (const diagnostic of transpileResult.diagnostics ?? []) { + if (diagnostic.file === undefined) { + const message = ts.flattenDiagnosticMessageText(diagnostic.messageText, "\n") + diagnostics.push(`${fileName}: ${message}`) + continue + } + + const position = diagnostic.start ?? 0 + const { character, line } = diagnostic.file.getLineAndCharacterOfPosition(position) + const message = ts.flattenDiagnosticMessageText(diagnostic.messageText, "\n") + diagnostics.push( + `${path.relative(repoRoot, readmeOutputPath)}:${block.startLine + line}:${character + 1}: ${message}`, + ) + } + } + + if (diagnostics.length > 0) { + throw new Error(`TypeScript code blocks in README failed to parse:\n${diagnostics.join("\n")}`) + } +} + +function getMarkdownCodeBlocks(markdown: string): MarkdownCodeBlock[] { + const blocks: MarkdownCodeBlock[] = [] + const lines = markdown.split(/\r?\n/u) + let currentLang: string | null = null + let currentLines: string[] = [] + let currentStartLine = 0 + + for (let index = 0; index < lines.length; index++) { + const line = lines[index] + if (line === undefined) { + continue + } + + const fenceMatch = line.match(/^```([^\s`]*)\s*$/u) + if (!fenceMatch) { + if (currentLang !== null) { + currentLines.push(line) + } + + continue + } + + if (currentLang === null) { + currentLang = fenceMatch[1] ?? "" + currentLines = [] + currentStartLine = index + 2 + continue + } + + blocks.push({ + code: currentLines.join("\n"), + lang: currentLang, + startLine: currentStartLine, + }) + currentLang = null + currentLines = [] + currentStartLine = 0 + } + + if (currentLang !== null) { + throw new Error(`${path.relative(repoRoot, readmeOutputPath)} contains an unterminated code fence`) + } + + return blocks +} + +function isTypeScriptFence(lang: string): boolean { + return lang === "ts" || lang === "tsx" || lang === "typescript" +} + function hasExportModifier(node: ts.Node): boolean { if (!ts.canHaveModifiers(node)) { return false @@ -82,5 +386,7 @@ function collectBindingNames(name: ts.BindingName, names: string[]): void { } if (path.resolve(process.argv[1] ?? "") === __filename) { - generateIndexMjs() + const declarations = loadDeclarations() + generateIndexMjs(declarations) + generateReadme(declarations) } diff --git a/scripts/zig-build-prebuilds.mts b/scripts/zig-build-prebuilds.mts index e7be2b5..b03e94e 100755 --- a/scripts/zig-build-prebuilds.mts +++ b/scripts/zig-build-prebuilds.mts @@ -101,6 +101,8 @@ const targetSpecs: Record = { }, } +const validPrebuildFilenamesByDir = createValidPrebuildFilenamesByDir() + main().catch((error: unknown) => { if (error instanceof Error && error.stack) { console.error(error.stack) @@ -126,6 +128,7 @@ async function main(): Promise { mkdirSync(zigLocalCacheDir, { recursive: true }) mkdirSync(zigGlobalCacheDir, { recursive: true }) + ensureStagedBuildDirs(selectedTargets.map(([, spec]) => spec)) for (const batch of createBuildBatches(selectedTargets)) { await build(createBuildTargets(batch), repoRoot) @@ -186,6 +189,12 @@ function createBuildBatches( return sharedBatch.length > 0 ? [sharedBatch, ...windowsBatches] : windowsBatches } +function ensureStagedBuildDirs(specs: readonly TargetSpec[]): void { + for (const spec of specs) { + mkdirSync(path.dirname(stagedBuildPath(spec)), { recursive: true }) + } +} + function createBuildTargets( selectedTargets: readonly (readonly [name: string, spec: TargetSpec])[], ): Record { @@ -232,34 +241,33 @@ function encodePrebuildName(name: string): string { } function publishPrebuildArtifacts(specs: TargetSpec[]): void { - const expectedFilesByDir = new Map>() + const specsByDir = new Map() for (const spec of specs) { - const published = prebuildPath(spec) - const publishedDir = path.dirname(published) - let expectedFiles = expectedFilesByDir.get(publishedDir) - - if (!expectedFiles) { - expectedFiles = new Set() - expectedFilesByDir.set(publishedDir, expectedFiles) + const publishedDir = path.dirname(prebuildPath(spec)) + const dirSpecs = specsByDir.get(publishedDir) + if (dirSpecs) { + dirSpecs.push(spec) + } else { + specsByDir.set(publishedDir, [spec]) } - - expectedFiles.add(path.basename(published)) } - for (const spec of specs) { - const staged = stagedBuildPath(spec) - const published = prebuildPath(spec) - const publishedDir = path.dirname(published) + for (const [publishedDir, dirSpecs] of specsByDir) { + mkdirSync(publishedDir, { recursive: true }) + removeStalePrebuilds(publishedDir, validPrebuildFilenamesByDir.get(publishedDir) || new Set()) - if (!existsSync(staged)) { - throw new Error(`expected zig-build output at ${staged}`) - } + for (const spec of dirSpecs) { + const staged = stagedBuildPath(spec) + const published = prebuildPath(spec) - mkdirSync(publishedDir, { recursive: true }) - removeStalePrebuilds(publishedDir, expectedFilesByDir.get(publishedDir) || new Set()) - copyFileSync(staged, published) - chmodSync(published, statSync(staged).mode) + if (!existsSync(staged)) { + throw new Error(`expected zig-build output at ${staged}`) + } + + copyFileSync(staged, published) + chmodSync(published, statSync(staged).mode) + } } } @@ -280,3 +288,21 @@ function removeStalePrebuilds(dir: string, expectedFiles: ReadonlySet): function prebuildFilename(spec: TargetSpec): string { return spec.platform === "linux" && spec.libc ? `${packageName}.${spec.libc}.node` : `${packageName}.node` } + +function createValidPrebuildFilenamesByDir(): Map> { + const filesByDir = new Map>() + + for (const spec of Object.values(targetSpecs)) { + const dir = path.dirname(prebuildPath(spec)) + let names = filesByDir.get(dir) + + if (!names) { + names = new Set() + filesByDir.set(dir, names) + } + + names.add(prebuildFilename(spec)) + } + + return filesByDir +} diff --git a/src/binding.c b/src/binding.c index 8835278..4a7eeea 100644 --- a/src/binding.c +++ b/src/binding.c @@ -3,11 +3,23 @@ #include #include #include +#include -#define MAX_SAFE_INTEGER_AS_DOUBLE 9007199254740991.0 -#define MAX_SAFE_INTEGER_AS_UINT64 9007199254740991ULL -#define MAX_SAFE_BYTE_LENGTH \ - (SIZE_MAX > MAX_SAFE_INTEGER_AS_UINT64 ? MAX_SAFE_INTEGER_AS_UINT64 : SIZE_MAX) +#define MAX_SAFE_INTEGER 9007199254740991 + +#if SIZE_MAX > MAX_SAFE_INTEGER +#define MAX_SAFE_BYTE_LENGTH MAX_SAFE_INTEGER +#define MAX_SAFE_SCAN_LENGTH ((size_t)MAX_SAFE_INTEGER + 1) +#else +#define MAX_SAFE_BYTE_LENGTH SIZE_MAX +#define MAX_SAFE_SCAN_LENGTH SIZE_MAX +#endif + +#if UINTPTR_MAX > MAX_SAFE_INTEGER +#define MAX_SAFE_POINTER_NUMBER MAX_SAFE_INTEGER +#else +#define MAX_SAFE_POINTER_NUMBER ((int64_t)UINTPTR_MAX) +#endif static void nop_finalize(napi_env env, void* finalize_data, void* finalize_hint) { (void)env; @@ -25,159 +37,57 @@ static napi_value throw_range_error(napi_env env, const char* code, const char* return NULL; } -static bool get_size_t_arg(napi_env env, napi_value value, const char* name, size_t* result) { - double number_value = 0; - char message[128]; - - if (napi_get_value_double(env, value, &number_value) != napi_ok) { - snprintf(message, sizeof(message), "%s must be a number", name); - throw_type_error(env, "ERR_INVALID_ARG_TYPE", message); +static bool get_safe_int64_arg(napi_env env, napi_value value, const char* name, + int64_t min, int64_t max, const char* range_desc, + int64_t* result) { + double d; + char msg[160]; + if (napi_get_value_double(env, value, &d) != napi_ok) { + snprintf(msg, sizeof(msg), "%s must be a number", name); + napi_throw_type_error(env, "ERR_INVALID_ARG_TYPE", msg); return false; } - - if (!isfinite(number_value) || number_value < 0 || number_value > (double)SIZE_MAX) { - snprintf(message, sizeof(message), "%s is out of range", name); - throw_range_error(env, "ERR_OUT_OF_RANGE", message); + if (!isfinite(d) || d < (double)min || d > (double)max || floor(d) != d) { + snprintf(msg, sizeof(msg), "%s must be %s", name, range_desc); + napi_throw_range_error(env, "ERR_OUT_OF_RANGE", msg); return false; } - - if (floor(number_value) != number_value) { - snprintf(message, sizeof(message), "%s must be an integer", name); - throw_range_error(env, "ERR_OUT_OF_RANGE", message); - return false; - } - - *result = (size_t)number_value; + *result = (int64_t)d; return true; } -static bool get_optional_size_t_arg(napi_env env, napi_value value, const char* name, size_t* result) { - napi_valuetype value_type; - if (napi_typeof(env, value, &value_type) != napi_ok) { - napi_throw_error(env, NULL, "Failed to inspect argument type"); - return false; - } - - if (value_type == napi_undefined) { - *result = 0; - return true; - } - - return get_size_t_arg(env, value, name, result); -} - -static bool get_max_byte_count_arg(napi_env env, napi_value value, const char* name, int64_t* result) { - double number_value = 0; - char message[160]; - - if (napi_get_value_double(env, value, &number_value) != napi_ok) { - snprintf(message, sizeof(message), "%s must be a number", name); - throw_type_error(env, "ERR_INVALID_ARG_TYPE", message); - return false; - } - - if (!isfinite(number_value) || number_value < -1 || number_value > MAX_SAFE_INTEGER_AS_DOUBLE) { - snprintf(message, sizeof(message), "%s must be -1 or a non-negative safe integer", name); - throw_range_error(env, "ERR_OUT_OF_RANGE", message); - return false; - } - - if (floor(number_value) != number_value) { - snprintf(message, sizeof(message), "%s must be an integer", name); - throw_range_error(env, "ERR_OUT_OF_RANGE", message); +static bool get_size_t_arg(napi_env env, napi_value value, const char* name, size_t* result) { + int64_t v; + if (!get_safe_int64_arg(env, value, name, 0, MAX_SAFE_BYTE_LENGTH, "a non-negative safe integer", &v)) return false; - } - - *result = (int64_t)number_value; + *result = (size_t)v; return true; } -static bool get_optional_safe_int64_arg(napi_env env, napi_value value, const char* name, bool* provided, int64_t* result) { - napi_valuetype value_type; - double number_value = 0; - char message[160]; - - if (napi_typeof(env, value, &value_type) != napi_ok) { - napi_throw_error(env, NULL, "Failed to inspect argument type"); - return false; - } - - if (value_type == napi_undefined) { - *provided = false; - *result = 0; - return true; - } - - if (napi_get_value_double(env, value, &number_value) != napi_ok) { - snprintf(message, sizeof(message), "%s must be a number", name); - throw_type_error(env, "ERR_INVALID_ARG_TYPE", message); - return false; - } - - if (!isfinite(number_value) || number_value < -MAX_SAFE_INTEGER_AS_DOUBLE || number_value > MAX_SAFE_INTEGER_AS_DOUBLE) { - snprintf(message, sizeof(message), "%s must be a safe integer", name); - throw_range_error(env, "ERR_OUT_OF_RANGE", message); - return false; - } - - if (floor(number_value) != number_value) { - snprintf(message, sizeof(message), "%s must be an integer", name); - throw_range_error(env, "ERR_OUT_OF_RANGE", message); - return false; - } - - *provided = true; - *result = (int64_t)number_value; - return true; +static bool is_nullish(napi_env env, napi_value value) { + napi_valuetype t; + return napi_typeof(env, value, &t) == napi_ok && (t == napi_undefined || t == napi_null); } -static bool get_pointer_number_arg(napi_env env, napi_value value, const char* name, uint64_t* result) { - double number_value = 0; - char message[160]; - - if (napi_get_value_double(env, value, &number_value) != napi_ok) { - snprintf(message, sizeof(message), "%s must be a number", name); - throw_type_error(env, "ERR_INVALID_ARG_TYPE", message); - return false; - } - - if (!isfinite(number_value) || number_value < 0 || number_value > MAX_SAFE_INTEGER_AS_DOUBLE) { - snprintf(message, sizeof(message), "%s must be a non-negative safe integer", name); - throw_range_error(env, "ERR_OUT_OF_RANGE", message); - return false; - } - - if (floor(number_value) != number_value) { - snprintf(message, sizeof(message), "%s must be an integer", name); - throw_range_error(env, "ERR_OUT_OF_RANGE", message); - return false; - } - - *result = (uint64_t)number_value; - return true; +static bool get_optional_size_t_arg(napi_env env, napi_value value, const char* name, size_t* result) { + if (is_nullish(env, value)) { *result = 0; return true; } + return get_size_t_arg(env, value, name, result); } -static bool get_pointer_bigint_arg(napi_env env, napi_value value, const char* name, uint64_t* result) { - bool lossless = false; - char message[128]; - - if (napi_get_value_bigint_uint64(env, value, result, &lossless) != napi_ok) { - snprintf(message, sizeof(message), "%s must be a bigint", name); - throw_type_error(env, "ERR_INVALID_ARG_TYPE", message); - return false; - } - - if (!lossless) { - snprintf(message, sizeof(message), "%s bigint is out of range", name); - throw_range_error(env, "ERR_OUT_OF_RANGE", message); - return false; - } - - return true; +static bool get_optional_safe_int64_arg(napi_env env, napi_value value, const char* name, + bool* provided, int64_t* result) { + if (is_nullish(env, value)) { *provided = false; *result = 0; return true; } + *provided = true; + return get_safe_int64_arg(env, value, name, -MAX_SAFE_BYTE_LENGTH, + MAX_SAFE_BYTE_LENGTH, + "a safe integer within the supported byte range", result); } -static bool get_pointer_arg(napi_env env, napi_value value, const char* name, uint64_t* result) { +static bool get_pointer_arg(napi_env env, napi_value value, const char* name, uintptr_t* result) { napi_valuetype value_type; + int64_t int_value; + uint64_t bigint_value; + bool lossless; char message[128]; if (napi_typeof(env, value, &value_type) != napi_ok) { @@ -186,11 +96,29 @@ static bool get_pointer_arg(napi_env env, napi_value value, const char* name, ui } if (value_type == napi_number) { - return get_pointer_number_arg(env, value, name, result); + if (!get_safe_int64_arg( + env, + value, + name, + 0, + MAX_SAFE_POINTER_NUMBER, + "a non-negative safe integer within the supported pointer range", + &int_value + )) + return false; + *result = (uintptr_t)int_value; + return true; } if (value_type == napi_bigint) { - return get_pointer_bigint_arg(env, value, name, result); + if (napi_get_value_bigint_uint64(env, value, &bigint_value, &lossless) != napi_ok + || !lossless || bigint_value > UINTPTR_MAX) { + snprintf(message, sizeof(message), "%s bigint is out of range", name); + throw_range_error(env, "ERR_OUT_OF_RANGE", message); + return false; + } + *result = (uintptr_t)bigint_value; + return true; } snprintf(message, sizeof(message), "%s must be a number or bigint", name); @@ -208,6 +136,11 @@ static bool create_int64_result(napi_env env, int64_t value, napi_value* result) } static bool create_size_t_number(napi_env env, size_t value, napi_value* result) { + if (value > MAX_SAFE_BYTE_LENGTH) { + throw_range_error(env, "ERR_OUT_OF_RANGE", "number result exceeds safe integer range"); + return false; + } + if (napi_create_double(env, (double)value, result) != napi_ok) { napi_throw_error(env, NULL, "Failed to create number result"); return false; @@ -220,183 +153,59 @@ static bool count_non_null_bytes( napi_env env, const uint8_t* data, bool has_limit, - uint64_t limit, + size_t limit, bool* found_null, - uint64_t* result -) { - uint64_t count = 0; - - while (true) { - if (has_limit && count == limit) { - *found_null = false; - *result = count; - return true; - } - - if (data[count] == 0) { - *found_null = true; - *result = count; - return true; - } - - if (count == MAX_SAFE_BYTE_LENGTH) { - throw_range_error(env, "ERR_OUT_OF_RANGE", "byte count exceeds safe integer range"); - return false; - } - - count++; - } -} - -static bool get_c_string_byte_length( - napi_env env, - const uint8_t* data, - bool has_byte_length, - int64_t requested_byte_length, size_t* result ) { - bool has_limit = false; - uint64_t limit = 0; - bool found_null = false; - uint64_t count = 0; + const size_t scan_length = has_limit ? limit : MAX_SAFE_SCAN_LENGTH; + const uint8_t* null_byte = memchr(data, 0, scan_length); - if (has_byte_length && requested_byte_length >= 0) { - *result = (size_t)requested_byte_length; + if (null_byte != NULL) { + *found_null = true; + *result = (size_t)(null_byte - data); return true; } - if (has_byte_length) { - has_limit = true; - limit = (uint64_t)(-requested_byte_length); + if (has_limit) { + *found_null = false; + *result = limit; + return true; } - if (!count_non_null_bytes( - env, - data, - has_limit, - limit, - &found_null, - &count - )) { - return false; - } + throw_range_error(env, "ERR_OUT_OF_RANGE", "byte count exceeds safe integer range"); + return false; +} - (void)found_null; +static bool get_c_string_byte_length(napi_env env, const uint8_t* data, + bool has_len, int64_t req_len, size_t* result) { + bool found; + size_t count; + if (has_len && req_len >= 0) { *result = (size_t)req_len; return true; } + if (!count_non_null_bytes(env, data, has_len, has_len ? (size_t)(-req_len) : 0, &found, &count)) + return false; *result = (size_t)count; return true; } static bool get_optional_result_object(napi_env env, napi_value value, napi_value* result) { - napi_valuetype value_type; - - if (napi_typeof(env, value, &value_type) != napi_ok) { - napi_throw_error(env, NULL, "Failed to inspect result argument type"); - return false; - } - - if (value_type == napi_undefined) { - if (napi_create_object(env, result) != napi_ok) { - napi_throw_error(env, NULL, "Failed to create result object"); - return false; - } - - return true; - } - - if (value_type != napi_object && value_type != napi_function) { - throw_type_error(env, "ERR_INVALID_ARG_TYPE", "result must be an object"); + napi_valuetype t; + if (napi_typeof(env, value, &t) != napi_ok) { + napi_throw_error(env, NULL, "Failed to inspect argument type"); return false; } - - *result = value; - return true; + if (t == napi_undefined || t == napi_null) + return napi_create_object(env, result) == napi_ok; + if (t == napi_object || t == napi_function) { *result = value; return true; } + throw_type_error(env, "ERR_INVALID_ARG_TYPE", "result must be an object"); + return false; } -static bool get_buffer_data(napi_env env, napi_value value, uint8_t** data) { - bool is_buffer = false; - if (napi_is_buffer(env, value, &is_buffer) != napi_ok) { - napi_throw_error(env, NULL, "Failed to inspect Buffer argument"); - return false; - } - - if (is_buffer) { - size_t byte_length = 0; - if (napi_get_buffer_info(env, value, (void**)data, &byte_length) != napi_ok) { - napi_throw_error(env, NULL, "Failed to read Buffer argument"); - return false; - } - return true; - } - - bool is_typed_array = false; - if (napi_is_typedarray(env, value, &is_typed_array) != napi_ok) { - napi_throw_error(env, NULL, "Failed to inspect TypedArray argument"); - return false; - } - - if (is_typed_array) { - napi_typedarray_type array_type; - size_t element_length = 0; - napi_value array_buffer; - size_t byte_offset = 0; - - if (napi_get_typedarray_info( - env, - value, - &array_type, - &element_length, - (void**)data, - &array_buffer, - &byte_offset - ) != napi_ok) { - napi_throw_error(env, NULL, "Failed to read TypedArray argument"); - return false; - } - - (void)array_type; - (void)element_length; - (void)array_buffer; - (void)byte_offset; - return true; - } - - bool is_dataview = false; - if (napi_is_dataview(env, value, &is_dataview) != napi_ok) { - napi_throw_error(env, NULL, "Failed to inspect DataView argument"); - return false; - } - - if (is_dataview) { - size_t byte_length = 0; - napi_value array_buffer; - size_t byte_offset = 0; - - if (napi_get_dataview_info(env, value, &byte_length, (void**)data, &array_buffer, &byte_offset) != napi_ok) { - napi_throw_error(env, NULL, "Failed to read DataView argument"); - return false; - } - - (void)byte_length; - (void)array_buffer; - (void)byte_offset; - return true; - } - - bool is_array_buffer = false; - if (napi_is_arraybuffer(env, value, &is_array_buffer) != napi_ok) { - napi_throw_error(env, NULL, "Failed to inspect ArrayBuffer argument"); - return false; - } - - if (is_array_buffer) { - size_t byte_length = 0; - if (napi_get_arraybuffer_info(env, value, (void**)data, &byte_length) != napi_ok) { - napi_throw_error(env, NULL, "Failed to read ArrayBuffer argument"); - return false; - } - return true; - } - +static bool get_buffer_data_ptr(napi_env env, napi_value value, void** data) { + size_t n; napi_typedarray_type t; napi_value ab; size_t off; + if (napi_get_buffer_info(env, value, data, &n) == napi_ok) return true; + if (napi_get_typedarray_info(env, value, &t, &n, data, &ab, &off) == napi_ok) return true; + if (napi_get_dataview_info(env, value, &n, data, &ab, &off) == napi_ok) return true; + if (napi_get_arraybuffer_info(env, value, data, &n) == napi_ok) return true; throw_type_error(env, "ERR_INVALID_ARG_TYPE", "buf must be an ArrayBuffer or ArrayBufferView"); return false; } @@ -405,8 +214,8 @@ static napi_value unsafe_pointer_common(napi_env env, napi_callback_info info, b size_t argc = 1; napi_value argv[1]; napi_value result; - uint8_t* data = NULL; - uint64_t pointer = 0; + void* data = NULL; + uintptr_t pointer = 0; if (napi_get_cb_info(env, info, &argc, argv, NULL, NULL) != napi_ok) { napi_throw_error(env, NULL, "Failed to read arguments"); @@ -417,14 +226,14 @@ static napi_value unsafe_pointer_common(napi_env env, napi_callback_info info, b return throw_type_error(env, "ERR_MISSING_ARGS", "buf is required"); } - if (!get_buffer_data(env, argv[0], &data)) { + if (!get_buffer_data_ptr(env, argv[0], &data)) { return NULL; } - pointer = (uint64_t)(uintptr_t)data; + pointer = (uintptr_t)data; if (as_bigint) { - if (napi_create_bigint_uint64(env, pointer, &result) != napi_ok) { + if (napi_create_bigint_uint64(env, (uint64_t)pointer, &result) != napi_ok) { napi_throw_error(env, NULL, "Failed to create pointer result"); return NULL; } @@ -447,7 +256,7 @@ static napi_value unsafeBigIntPointerOf(napi_env env, napi_callback_info info) { static napi_value unsafeArrayBufferAt(napi_env env, napi_callback_info info) { size_t argc = 3; napi_value argv[3]; - uint64_t pointer = 0; + uintptr_t pointer = 0; size_t offset = 0; size_t byte_length = 0; napi_value result; @@ -473,13 +282,13 @@ static napi_value unsafeArrayBufferAt(napi_env env, napi_callback_info info) { return NULL; } - if ((uint64_t)offset > UINT64_MAX - pointer) { + if ((uintptr_t)offset > UINTPTR_MAX - pointer) { return throw_range_error(env, "ERR_OUT_OF_RANGE", "ptr + offset is out of range"); } if (napi_create_external_arraybuffer( env, - (uint8_t*)(uintptr_t)(pointer + (uint64_t)offset), + (void*)(pointer + offset), byte_length, nop_finalize, NULL, @@ -495,12 +304,12 @@ static napi_value unsafeArrayBufferAt(napi_env env, napi_callback_info info) { static napi_value unsafeCountNonNullBytes(napi_env env, napi_callback_info info) { size_t argc = 2; napi_value argv[2]; - uint64_t pointer = 0; + uintptr_t pointer = 0; int64_t max_bytes = 0; napi_value result; const uint8_t* data = NULL; bool found_null = false; - uint64_t count = 0; + size_t count = 0; if (napi_get_cb_info(env, info, &argc, argv, NULL, NULL) != napi_ok) { napi_throw_error(env, NULL, "Failed to read arguments"); @@ -515,7 +324,8 @@ static napi_value unsafeCountNonNullBytes(napi_env env, napi_callback_info info) return NULL; } - if (!get_max_byte_count_arg(env, argv[1], "maxBytes", &max_bytes)) { + if (!get_safe_int64_arg(env, argv[1], "maxBytes", -1, MAX_SAFE_BYTE_LENGTH, + "-1 or a non-negative safe integer", &max_bytes)) { return NULL; } @@ -533,7 +343,7 @@ static napi_value unsafeCountNonNullBytes(napi_env env, napi_callback_info info) env, data, max_bytes >= 0, - max_bytes >= 0 ? (uint64_t)max_bytes : 0, + max_bytes >= 0 ? (size_t)max_bytes : 0, &found_null, &count )) { @@ -550,7 +360,7 @@ static napi_value unsafeCountNonNullBytes(napi_env env, napi_callback_info info) static napi_value unsafeCStringAt(napi_env env, napi_callback_info info) { size_t argc = 4; napi_value argv[4]; - uint64_t pointer = 0; + uintptr_t pointer = 0; size_t offset = 0; bool has_byte_length = false; int64_t requested_byte_length = 0; @@ -582,7 +392,7 @@ static napi_value unsafeCStringAt(napi_env env, napi_callback_info info) { return NULL; } - if ((uint64_t)offset > UINT64_MAX - pointer) { + if ((uintptr_t)offset > UINTPTR_MAX - pointer) { return throw_range_error(env, "ERR_OUT_OF_RANGE", "ptr + offset is out of range"); } @@ -595,7 +405,7 @@ static napi_value unsafeCStringAt(napi_env env, napi_callback_info info) { return NULL; } - data = (const uint8_t*)(uintptr_t)(pointer + (uint64_t)offset); + data = (const uint8_t*)(pointer + (uintptr_t)offset); if (!get_c_string_byte_length(env, data, has_byte_length, requested_byte_length, &byte_length)) { return NULL; } diff --git a/test/register-unsafe-pointer-tests.js b/test/register-unsafe-pointer-tests.js index f536d09..c7b6cb6 100644 --- a/test/register-unsafe-pointer-tests.js +++ b/test/register-unsafe-pointer-tests.js @@ -60,6 +60,20 @@ module.exports = function registerUnsafePointerTests({ assert.equal(view[3], 1) }) + test("unsafeArrayBufferAt rejects offset and byteLength above the safe integer range", () => { + const view = new Uint8Array([1, 2, 3, 4]) + const ptr = unsafePointerOf(view) + + assert.throws( + () => unsafeArrayBufferAt(ptr, Number.MAX_SAFE_INTEGER + 1, 1), + /offset must be a non-negative safe integer/u, + ) + assert.throws( + () => unsafeArrayBufferAt(ptr, 0, Number.MAX_SAFE_INTEGER + 1), + /byteLength must be a non-negative safe integer/u, + ) + }) + test("ArrayBuffer inputs work too", () => { const arrayBuffer = new Uint32Array([11, 12, 13]).buffer const ptr = unsafePointerOf(arrayBuffer) @@ -117,6 +131,17 @@ module.exports = function registerUnsafePointerTests({ }) }) + test("unsafeCStringAt rejects offset and byteLength above the safe integer range", () => { + const cstring = new TextEncoder().encode("abc\0") + const ptr = unsafePointerOf(cstring) + + assert.throws( + () => unsafeCStringAt(ptr, Number.MAX_SAFE_INTEGER + 1), + /offset must be a non-negative safe integer/u, + ) + assert.throws(() => unsafeCStringAt(ptr, 0, Number.MAX_SAFE_INTEGER + 1), /byteLength must be a safe integer/u) + }) + test("unsafeCStringAt decodes UTF-8 and reports byteLength in bytes", () => { const value = "hé🌍" const cstring = new TextEncoder().encode(`${value}\0`)