diff --git a/packages/gittensory-miner/lib/orb-export.d.ts b/packages/gittensory-miner/lib/orb-export.d.ts new file mode 100644 index 0000000000..5595bc83af --- /dev/null +++ b/packages/gittensory-miner/lib/orb-export.d.ts @@ -0,0 +1,42 @@ +import type { NormalizedPrOutcomePayload, PrOutcomeLedgerReader } from "./pr-outcome.js"; + +/** OPT-IN default: a laptop miner exports nothing unless a contributor turns it on. */ +export const ORB_EXPORT_ENABLED_BY_DEFAULT: false; + +/** One anonymized outcome in an export batch — no raw repo name, PR number, or free-text reason. */ +export interface OrbExportRow { + repoHash: string; + prHash: string; + decision: string; + reasonBucket: string; + closedAt: string | null; +} + +/** The local orb-export store: the per-instance anonymization secret + export cursor, in local SQLite. */ +export interface OrbExportStore { + dbPath: string; + getOrCreateAnonSecret(): string; + getCursor(): string | null; + setCursor(cursor: string): void; + close(): void; +} + +/** A pr_outcome record as produced by `readPrOutcomes` (the local ledger's latest-per-PR reduction). */ +export type OrbExportOutcome = NormalizedPrOutcomePayload & { repoFullName: string }; + +export function resolveOrbExportDbPath(env?: Record): string; + +export function hmacAnonymize(value: string | number, secret: string): string; + +export function buildAnonymizedOrbBatch( + outcomes: Iterable | Map, + secret: string, +): OrbExportRow[]; + +export function openOrbExportStore(dbPath?: string): OrbExportStore; + +export function collectOrbExportBatch(options?: { + store: OrbExportStore; + eventLedger: PrOutcomeLedgerReader; + enabled?: boolean; +}): OrbExportRow[] | null; diff --git a/packages/gittensory-miner/lib/orb-export.js b/packages/gittensory-miner/lib/orb-export.js new file mode 100644 index 0000000000..7d56b79f22 --- /dev/null +++ b/packages/gittensory-miner/lib/orb-export.js @@ -0,0 +1,133 @@ +import { chmodSync, mkdirSync } from "node:fs"; +import { homedir } from "node:os"; +import { dirname, join } from "node:path"; +import { DatabaseSync } from "node:sqlite"; +import { createHmac, randomBytes } from "node:crypto"; +import { readPrOutcomes } from "./pr-outcome.js"; + +// Optional anonymized Orb telemetry export (#4277). The self-host Orb collector (src/selfhost/orb-collector.ts, +// #1255) is ALWAYS-ON for a maintainer's own instance; a miner runs on a third-party contributor's laptop with a +// much lower consent bar, so this export is OPT-IN (default OFF) — hence "optional". It mirrors the collector's +// privacy posture: repo/PR identifiers are HMAC-anonymized with a per-instance DEDICATED secret (generated once, +// persisted locally, single-purpose), and only a fixed low-cardinality reason bucket + the decision leave — never +// raw repo names or free text. The data source is the local pr_outcome ledger (pr-outcome.js), not a hosted D1. +// This module builds the anonymized batch and manages the local secret + cursor; performing the network POST is the +// caller's job, so this stays pure over its inputs + local store and needs no network to test. + +/** OPT-IN: a laptop miner exports nothing unless a contributor explicitly turns it on. */ +export const ORB_EXPORT_ENABLED_BY_DEFAULT = false; + +const ANON_SECRET_KEY = "anon_secret"; +const CURSOR_KEY = "export_cursor"; +const defaultDbFileName = "orb-export.sqlite3"; + +export function resolveOrbExportDbPath(env = process.env) { + const explicitPath = + typeof env.GITTENSORY_MINER_ORB_EXPORT_DB === "string" ? env.GITTENSORY_MINER_ORB_EXPORT_DB.trim() : ""; + if (explicitPath) return explicitPath; + + const explicitConfigDir = + typeof env.GITTENSORY_MINER_CONFIG_DIR === "string" ? env.GITTENSORY_MINER_CONFIG_DIR.trim() : ""; + if (explicitConfigDir) return join(explicitConfigDir, defaultDbFileName); + + const configHome = + typeof env.XDG_CONFIG_HOME === "string" && env.XDG_CONFIG_HOME.trim() + ? env.XDG_CONFIG_HOME.trim() + : join(homedir(), ".config"); + return join(configHome, "gittensory-miner", defaultDbFileName); +} + +function normalizeDbPath(dbPath) { + const path = (dbPath ?? resolveOrbExportDbPath()).trim(); + if (!path) throw new Error("invalid_orb_export_db_path"); + return path; +} + +/** HMAC a value with the per-instance secret — mirrors orb-collector.ts's hmacField (sha256, first 24 hex). */ +export function hmacAnonymize(value, secret) { + if (typeof secret !== "string" || !secret) throw new Error("invalid_anon_secret"); + return createHmac("sha256", secret).update(String(value)).digest("hex").slice(0, 24); +} + +/** + * Turn the local pr_outcome map (pr-outcome.js `readPrOutcomes`) into an anonymized export batch: repo and PR + * identifiers are HMAC-hashed, and only the `decision` + a low-cardinality `reasonBucket` (already one of the + * miner's `REJECTION_REASONS`, else `"none"`) + `closedAt` leave. Pure and deterministic (rows sorted by prHash). + * Accepts either the Map `readPrOutcomes` returns or any iterable of outcome records. + */ +export function buildAnonymizedOrbBatch(outcomes, secret) { + const iterable = outcomes && typeof outcomes.values === "function" ? outcomes.values() : outcomes; + const rows = []; + for (const outcome of iterable ?? []) { + if (!outcome || typeof outcome.repoFullName !== "string" || !outcome.repoFullName.trim()) continue; + if (!Number.isInteger(outcome.prNumber) || outcome.prNumber <= 0) continue; + rows.push({ + repoHash: hmacAnonymize(outcome.repoFullName, secret), + prHash: hmacAnonymize(`${outcome.repoFullName}:${outcome.prNumber}`, secret), + decision: outcome.decision, + reasonBucket: typeof outcome.reason === "string" && outcome.reason ? outcome.reason : "none", + closedAt: typeof outcome.closedAt === "string" && outcome.closedAt ? outcome.closedAt : null, + }); + } + rows.sort((a, b) => a.prHash.localeCompare(b.prHash)); + return rows; +} + +/** + * Open/create the local orb-export store: a small key/value SQLite table holding the per-instance anonymization + * secret and the export cursor. Mirrors the other miner ledgers' node:sqlite pattern — a `0o700` config dir and a + * `0o600` file, since the secret must never leave this machine. + */ +export function openOrbExportStore(dbPath = resolveOrbExportDbPath()) { + const resolvedPath = normalizeDbPath(dbPath); + mkdirSync(dirname(resolvedPath), { recursive: true, mode: 0o700 }); + const db = new DatabaseSync(resolvedPath); + chmodSync(resolvedPath, 0o600); + db.exec("PRAGMA busy_timeout = 5000"); + db.exec(`CREATE TABLE IF NOT EXISTS orb_export_meta (key TEXT PRIMARY KEY, value TEXT NOT NULL)`); + + const getStatement = db.prepare("SELECT value FROM orb_export_meta WHERE key = ?"); + const setStatement = db.prepare( + "INSERT INTO orb_export_meta (key, value) VALUES (?, ?) ON CONFLICT(key) DO UPDATE SET value = excluded.value", + ); + const readValue = (key) => { + const row = getStatement.get(key); + return row && typeof row.value === "string" ? row.value : null; + }; + + return { + dbPath: resolvedPath, + /** The per-instance DEDICATED anonymization secret — generated once (256-bit) and persisted, then reused + * forever so a repo/PR always hashes the same way. Single-purpose: only this export uses it. */ + getOrCreateAnonSecret() { + const existing = readValue(ANON_SECRET_KEY); + if (existing) return existing; + const generated = randomBytes(32).toString("hex"); + setStatement.run(ANON_SECRET_KEY, generated); + return generated; + }, + /** The export watermark (opaque string), or null before the first export. */ + getCursor() { + return readValue(CURSOR_KEY); + }, + setCursor(cursor) { + setStatement.run(CURSOR_KEY, String(cursor)); + }, + close() { + db.close(); + }, + }; +} + +/** + * Collect the anonymized Orb export batch from the local pr_outcome ledger. OPT-IN: returns null (exports nothing) + * unless `enabled` is true — a third-party contributor's laptop must explicitly turn this on. Never performs the + * network POST itself; the caller sends the returned batch to the Orb ingest endpoint and then advances the store + * cursor, so this function stays pure over its inputs and the local store. + */ +export function collectOrbExportBatch({ store, eventLedger, enabled = ORB_EXPORT_ENABLED_BY_DEFAULT } = {}) { + if (!enabled) return null; + if (!store || typeof store.getOrCreateAnonSecret !== "function") throw new Error("invalid_orb_export_store"); + const outcomes = readPrOutcomes(eventLedger); + return buildAnonymizedOrbBatch(outcomes, store.getOrCreateAnonSecret()); +} diff --git a/test/unit/miner-orb-export.test.ts b/test/unit/miner-orb-export.test.ts new file mode 100644 index 0000000000..6e6f5ab76c --- /dev/null +++ b/test/unit/miner-orb-export.test.ts @@ -0,0 +1,142 @@ +import { mkdtempSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { afterEach, beforeEach, describe, expect, it } from "vitest"; + +import { + ORB_EXPORT_ENABLED_BY_DEFAULT, + buildAnonymizedOrbBatch, + collectOrbExportBatch, + hmacAnonymize, + openOrbExportStore, +} from "../../packages/gittensory-miner/lib/orb-export.js"; +import type { OrbExportOutcome } from "../../packages/gittensory-miner/lib/orb-export.js"; + +let dir: string; +function storePath() { + return join(dir, "orb-export.sqlite3"); +} + +/** A minimal in-memory event ledger of pr_outcome events, matching pr-outcome.js's readEvents contract. */ +function fakeLedger(events: Array<{ type: string; repoFullName: string; payload: unknown }>) { + return { readEvents: () => events }; +} +function outcomeEvent(repoFullName: string, prNumber: number, decision: "merged" | "closed", reason: string | null) { + return { type: "pr_outcome", repoFullName, payload: { prNumber, decision, closedAt: "2026-01-01T00:00:00Z", reason } }; +} + +beforeEach(() => { + dir = mkdtempSync(join(tmpdir(), "orb-export-")); +}); +afterEach(() => { + rmSync(dir, { recursive: true, force: true }); +}); + +describe("orb-export store (#4277)", () => { + it("defaults to opt-OUT (export disabled unless explicitly enabled)", () => { + expect(ORB_EXPORT_ENABLED_BY_DEFAULT).toBe(false); + }); + + it("generates a stable 256-bit per-instance anon key and persists it across reopens", () => { + const store = openOrbExportStore(storePath()); + const anonKey = store.getOrCreateAnonSecret(); + expect(anonKey).toMatch(/^[0-9a-f]{64}$/); + expect(store.getOrCreateAnonSecret()).toBe(anonKey); // same within a session + store.close(); + + const reopened = openOrbExportStore(storePath()); + expect(reopened.getOrCreateAnonSecret()).toBe(anonKey); // same across reopens + reopened.close(); + }); + + it("tracks an export cursor (null until set)", () => { + const store = openOrbExportStore(storePath()); + expect(store.getCursor()).toBeNull(); + store.setCursor("2026-01-02T00:00:00Z"); + expect(store.getCursor()).toBe("2026-01-02T00:00:00Z"); + store.close(); + }); +}); + +describe("hmacAnonymize", () => { + const anonKey = "a".repeat(64); + + it("is deterministic per (value, key), hides the raw value, and separates distinct values", () => { + const hashed = hmacAnonymize("owner/repo", anonKey); + expect(hashed).toBe(hmacAnonymize("owner/repo", anonKey)); + expect(hashed).toMatch(/^[0-9a-f]{24}$/); + expect(hashed).not.toContain("owner"); + expect(hmacAnonymize("owner/other", anonKey)).not.toBe(hashed); + expect(hmacAnonymize("owner/repo", "b".repeat(64))).not.toBe(hashed); // different key → different hash + }); + + it("throws on a missing key", () => { + expect(() => hmacAnonymize("owner/repo", "")).toThrow(/invalid_anon_secret/); + }); +}); + +describe("buildAnonymizedOrbBatch", () => { + const anonKey = "c".repeat(64); + + it("anonymizes a readPrOutcomes-shaped map, buckets a null reason to 'none', and sorts deterministically", () => { + const outcomes = new Map([ + ["owner/repo:2", { repoFullName: "owner/repo", prNumber: 2, decision: "closed", closedAt: "2026-01-02T00:00:00Z", reason: "gate_close" }], + ["owner/repo:1", { repoFullName: "owner/repo", prNumber: 1, decision: "merged", closedAt: null, reason: null }], + ]); + const batch = buildAnonymizedOrbBatch(outcomes, anonKey); + expect(batch).toHaveLength(2); + // no raw identifiers leak + const json = JSON.stringify(batch); + expect(json).not.toContain("owner/repo"); + expect(json).not.toContain('"prNumber"'); + const merged = batch.find((r) => r.decision === "merged"); + expect(merged?.reasonBucket).toBe("none"); + expect(merged?.closedAt).toBeNull(); + expect(merged?.repoHash).toBe(hmacAnonymize("owner/repo", anonKey)); + const closed = batch.find((r) => r.decision === "closed"); + expect(closed?.reasonBucket).toBe("gate_close"); + // deterministic prHash ordering + expect([...batch].sort((a, b) => a.prHash.localeCompare(b.prHash))).toEqual(batch); + }); + + it("skips malformed outcome records", () => { + const batch = buildAnonymizedOrbBatch( + [ + null, + { repoFullName: "owner/repo", prNumber: 1.5, decision: "merged", reason: null, closedAt: null }, + { repoFullName: "", prNumber: 1, decision: "merged", reason: null, closedAt: null }, + ] as never, + anonKey, + ); + expect(batch).toEqual([]); + }); +}); + +describe("collectOrbExportBatch", () => { + it("returns null when export is not enabled (opt-in gate)", () => { + const store = openOrbExportStore(storePath()); + expect(collectOrbExportBatch({ store, eventLedger: fakeLedger([]), enabled: false })).toBeNull(); + // default (no `enabled`) is also opt-out + expect(collectOrbExportBatch({ store, eventLedger: fakeLedger([]) })).toBeNull(); + store.close(); + }); + + it("builds an anonymized batch from the local pr_outcome ledger when enabled", () => { + const store = openOrbExportStore(storePath()); + const ledger = fakeLedger([ + outcomeEvent("owner/a", 1, "merged", null), + outcomeEvent("owner/b", 2, "closed", "superseded_by_duplicate"), + ]); + const batch = collectOrbExportBatch({ store, eventLedger: ledger, enabled: true }); + expect(batch).not.toBeNull(); + expect(batch).toHaveLength(2); + expect(JSON.stringify(batch)).not.toContain("owner/"); + store.close(); + }); + + it("throws on an invalid store", () => { + expect(() => + collectOrbExportBatch({ store: {} as never, eventLedger: fakeLedger([]), enabled: true }), + ).toThrow(/invalid_orb_export_store/); + }); +});