Skip to content

feat(selfhost): opt-in @sentry/node error tracking with boot/queue/review capture seams #1547

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
JSONbored merged 1 commit into from
Jun 26, 2026
Merged

feat(selfhost): opt-in @sentry/node error tracking with boot/queue/review capture seams #1547

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
320 changes: 315 additions & 5 deletions package-lock.json
View file
Open in desktop

Large diffs are not rendered by default.

1 change: 1 addition & 0 deletions package.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -65,6 +65,7 @@
"@hono/node-server": "^2.0.6",
"@modelcontextprotocol/sdk": "1.29.0",
"@octokit/core": "^7.0.6",
"@sentry/node": "^10.62.0",
"agents": "^0.16.2",
"drizzle-orm": "^0.45.2",
"hono": "^4.12.26",
Expand Down
12 changes: 11 additions & 1 deletion src/queue/processors.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -321,6 +321,7 @@ import {
buildReviewEnrichment,
isEnrichmentEnabled,
} from "../review/enrichment-wire";
import { captureReviewFailure } from "../selfhost/sentry";
import { evaluateWithSurfaceLane } from "../review/content-lane-wire";
import { indexRepo, reindexChangedPaths } from "../review/rag-index";
import {
Expand Down Expand Up @@ -1283,7 +1284,10 @@ async function maybeRunAgentMaintenance(
// Contributor blacklist (#1425): resolve whether the PR author is on the repo's blacklist (the shared/global
// list unions in once its table lands). A match short-circuits the planner to a deterministic label + close
// ahead of merit/CI/AI; only the configured label (default "slop") reaches public actions.
const blacklistEntry = findBlacklistEntry(pr.authorLogin, settings.contributorBlacklist);
const blacklistEntry = findBlacklistEntry(
pr.authorLogin,
settings.contributorBlacklist,
);

const planned = planAgentMaintenanceActions({
conclusion: gate.conclusion,
Expand Down Expand Up @@ -3608,6 +3612,12 @@ export async function runAiReviewForAdvisory(
error: errorMessage(error),
}),
);
captureReviewFailure(error, {
kind: "review",
repo: args.repoFullName,
pr: args.pr.number,
head_sha: args.advisory.headSha,
});
return undefined;
}
}
Expand Down
137 changes: 118 additions & 19 deletions src/selfhost/pg-queue.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@
import type { Pool } from "pg";
import { logAudit, extractPayloadType } from "./audit";
import { incr } from "./metrics";
import { captureError } from "./sentry";
import type { JobMessage } from "../types";

const TABLE = "_selfhost_jobs";
Expand Down Expand Up @@ -46,25 +47,47 @@ export interface PgQueueOptions {
concurrency?: number;
}

export function createPgQueue(pool: Pool, consume: (message: JobMessage) => Promise<void>, opts: PgQueueOptions = {}): PgDurableQueue {
export function createPgQueue(
pool: Pool,
consume: (message: JobMessage) => Promise<void>,
opts: PgQueueOptions = {},
): PgDurableQueue {
const maxRetries = opts.maxRetries ?? 5;
const pollIntervalMs = opts.pollIntervalMs ?? 1000;
const backoff = opts.backoffMs ?? ((attempt: number) => Math.min(60_000, 1000 * 2 ** attempt));
const concurrency = opts.concurrency ?? Math.max(1, Number(process.env.QUEUE_CONCURRENCY ?? "4"));
const backoff =
opts.backoffMs ??
((attempt: number) => Math.min(60_000, 1000 * 2 ** attempt));
const concurrency =
opts.concurrency ??
Math.max(1, Number(process.env.QUEUE_CONCURRENCY ?? "4"));

let running = false;
let active = 0;
let timer: ReturnType<typeof setTimeout> | null = null;

async function init(): Promise<void> {
await pool.query(DDL);
const recovered = (await pool.query(`UPDATE ${TABLE} SET status='pending' WHERE status='processing'`)).rowCount ?? 0;
if (recovered) console.log(JSON.stringify({ event: "selfhost_queue_recovered", count: recovered }));
const recovered =
(
await pool.query(
`UPDATE ${TABLE} SET status='pending' WHERE status='processing'`,
)
).rowCount ?? 0;
if (recovered)
console.log(
JSON.stringify({ event: "selfhost_queue_recovered", count: recovered }),
);
}

async function enqueue(message: JobMessage, delaySeconds: number): Promise<void> {
async function enqueue(
message: JobMessage,
delaySeconds: number,
): Promise<void> {
const now = Date.now();
await pool.query(`INSERT INTO ${TABLE} (payload, status, attempts, run_after, created_at) VALUES ($1,'pending',0,$2,$3)`, [JSON.stringify(message), now + delaySeconds * 1000, now]);
await pool.query(
`INSERT INTO ${TABLE} (payload, status, attempts, run_after, created_at) VALUES ($1,'pending',0,$2,$3)`,
[JSON.stringify(message), now + delaySeconds * 1000, now],
);
incr("gittensory_jobs_enqueued_total");
void pump();
}
Expand All @@ -88,28 +111,87 @@ export function createPgQueue(pool: Pool, consume: (message: JobMessage) => Prom
try {
message = JSON.parse(job.payload) as JobMessage;
} catch {
await pool.query(`UPDATE ${TABLE} SET status='dead', last_error='unparseable payload' WHERE id=$1`, [job.id]);
await pool.query(
`UPDATE ${TABLE} SET status='dead', last_error='unparseable payload' WHERE id=$1`,
[job.id],
);
incr("gittensory_jobs_dead_total");
logAudit({ event: "job_dead", ts: Date.now(), job_id: job.id, latency_ms: Date.now() - claimedAt, attempts: Number(job.attempts) + 1, error: "unparseable payload" });
logAudit({
event: "job_dead",
ts: Date.now(),
job_id: job.id,
latency_ms: Date.now() - claimedAt,
attempts: Number(job.attempts) + 1,
error: "unparseable payload",
});
captureError(new Error("unparseable queue payload"), {
kind: "job_dead",
reason: "unparseable_payload",
jobId: job.id,
});
return true;
}
try {
await consume(message);
await pool.query(`DELETE FROM ${TABLE} WHERE id=$1`, [job.id]);
incr("gittensory_jobs_processed_total");
logAudit({ event: "job_complete", ts: Date.now(), job_id: job.id, payload_type: extractPayloadType(job.payload), latency_ms: Date.now() - claimedAt, attempts: Number(job.attempts) + 1 });
logAudit({
event: "job_complete",
ts: Date.now(),
job_id: job.id,
payload_type: extractPayloadType(job.payload),
latency_ms: Date.now() - claimedAt,
attempts: Number(job.attempts) + 1,
});
} catch (error) {
const attempts = Number(job.attempts) + 1;
const errMsg = error instanceof Error ? error.message : "unknown error";
incr("gittensory_jobs_failed_total");
if (attempts >= maxRetries) {
await pool.query(`UPDATE ${TABLE} SET status='dead', attempts=$1, last_error=$2 WHERE id=$3`, [attempts, errMsg, job.id]);
await pool.query(
`UPDATE ${TABLE} SET status='dead', attempts=$1, last_error=$2 WHERE id=$3`,
[attempts, errMsg, job.id],
);
incr("gittensory_jobs_dead_total");
console.error(JSON.stringify({ level: "error", event: "selfhost_job_dead", id: job.id, attempts, error: errMsg }));
logAudit({ event: "job_dead", ts: Date.now(), job_id: job.id, payload_type: extractPayloadType(job.payload), latency_ms: Date.now() - claimedAt, attempts, error: errMsg });
console.error(
JSON.stringify({
level: "error",
event: "selfhost_job_dead",
id: job.id,
attempts,
error: errMsg,
}),
);
logAudit({
event: "job_dead",
ts: Date.now(),
job_id: job.id,
payload_type: extractPayloadType(job.payload),
latency_ms: Date.now() - claimedAt,
attempts,
error: errMsg,
});
captureError(error, {
kind: "job_dead",
reason: "max_retries_exhausted",
jobType: extractPayloadType(job.payload),
jobId: job.id,
attempts,
});
} else {
await pool.query(`UPDATE ${TABLE} SET status='pending', attempts=$1, run_after=$2, last_error=$3 WHERE id=$4`, [attempts, Date.now() + backoff(attempts), errMsg, job.id]);
logAudit({ event: "job_error", ts: Date.now(), job_id: job.id, payload_type: extractPayloadType(job.payload), latency_ms: Date.now() - claimedAt, attempts, error: errMsg });
await pool.query(
`UPDATE ${TABLE} SET status='pending', attempts=$1, run_after=$2, last_error=$3 WHERE id=$4`,
[attempts, Date.now() + backoff(attempts), errMsg, job.id],
);
logAudit({
event: "job_error",
ts: Date.now(),
job_id: job.id,
payload_type: extractPayloadType(job.payload),
latency_ms: Date.now() - claimedAt,
attempts,
error: errMsg,
});
}
}
return true;
Expand All @@ -128,10 +210,15 @@ export function createPgQueue(pool: Pool, consume: (message: JobMessage) => Prom
}

const binding = {
async send(message: JobMessage, options?: { delaySeconds?: number }): Promise<void> {
async send(
message: JobMessage,
options?: { delaySeconds?: number },
): Promise<void> {
await enqueue(message, options?.delaySeconds ?? 0);
},
async sendBatch(messages: Iterable<{ body: JobMessage; delaySeconds?: number }>): Promise<void> {
async sendBatch(
messages: Iterable<{ body: JobMessage; delaySeconds?: number }>,
): Promise<void> {
for (const m of messages) await enqueue(m.body, m.delaySeconds ?? 0);
},
} as unknown as Queue;
Expand Down Expand Up @@ -161,10 +248,22 @@ export function createPgQueue(pool: Pool, consume: (message: JobMessage) => Prom
await pump();
},
async size() {
return Number((await pool.query(`SELECT COUNT(*) AS c FROM ${TABLE} WHERE status IN ('pending','processing')`)).rows[0].c);
return Number(
(
await pool.query(
`SELECT COUNT(*) AS c FROM ${TABLE} WHERE status IN ('pending','processing')`,
)
).rows[0].c,
);
},
async deadCount() {
return Number((await pool.query(`SELECT COUNT(*) AS c FROM ${TABLE} WHERE status='dead'`)).rows[0].c);
return Number(
(
await pool.query(
`SELECT COUNT(*) AS c FROM ${TABLE} WHERE status='dead'`,
)
).rows[0].c,
);
},
};
}
99 changes: 99 additions & 0 deletions src/selfhost/sentry.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,99 @@
// Self-host-only error tracking (#1468). Opt-in: a complete NO-OP when SENTRY_DSN is unset, mirroring the
// env-gated, dynamically-imported selfhost-integration pattern (Redis/Qdrant/embed-provider in server.ts).
// @sentry/node is NEVER imported at module top level — it loads lazily inside initSentry(), so it never enters
// the Worker bundle (src/index.ts) and cloudflare:* stubbing stays clean. All helpers are safe to call when off.
type SentryNs = typeof import("@sentry/node");
let Sentry: SentryNs | undefined;
let active = false;

const SECRET_KEY =
/(token|secret|key|password|passwd|authorization|auth|dsn|cookie|bearer|credential|private)/i;

/** beforeSend scrubber — redact anything token/secret-like before an event leaves the box (privacy boundary). */
export function scrubEvent<T>(event: T): T {
const redact = (obj: unknown, depth: number): void => {
if (!obj || typeof obj !== "object" || depth > 6) return;
for (const key of Object.keys(obj as Record<string, unknown>)) {
const rec = obj as Record<string, unknown>;
if (SECRET_KEY.test(key)) rec[key] = "[redacted]";
else if (typeof rec[key] === "object") redact(rec[key], depth + 1);
}
};
try {
const e = event as {
request?: { headers?: unknown };
contexts?: unknown;
extra?: unknown;
};
redact(e.request?.headers, 0);
redact(e.contexts, 0);
redact(e.extra, 0);
} catch {
/* scrubbing must never break the send */
}
return event;
}

/** Initialize Sentry from the environment. Returns false (and stays a no-op) when SENTRY_DSN is unset. */
export async function initSentry(env: NodeJS.ProcessEnv): Promise<boolean> {
if (!env.SENTRY_DSN) return false;
Sentry = await import("@sentry/node");
Sentry.init({
dsn: env.SENTRY_DSN,
environment: env.SENTRY_ENVIRONMENT ?? "production",
release: env.SENTRY_RELEASE ?? env.GITTENSORY_VERSION,
tracesSampleRate: Number(env.SENTRY_TRACES_SAMPLE_RATE ?? "0"),
serverName: env.PUBLIC_API_ORIGIN,
beforeSend: (e) => scrubEvent(e),

@superagent-security superagent-security Bot Jun 26, 2026

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

P2: Sentry beforeSend scrubber misses secret-bearing event fields

beforeSend only scrubs headers, contexts, and extra, missing URL query params, breadcrumbs, and error messages where secrets often appear.

Expand scrubEvent to recursively redact secrets across the entire Sentry event, including request.url, breadcrumbs, and exception values.

AI prompt 
Check if this security scanner issue is valid. If so, understand the root cause and fix it. If appropriate, update or add tests. Keep the change focused and preserve intended behavior.

<file name="src/selfhost/sentry.ts">
<violation number="1" location="src/selfhost/sentry.ts:47">
<priority>P2</priority>
<title>Sentry beforeSend scrubber misses secret-bearing event fields</title>
<evidence>The scrubEvent function only redacts request.headers, contexts, and extra, leaving many Sentry event fields unscrubbed. Secrets in error messages (exception.values[].value), request URLs with query parameters (request.url, request.query_string), breadcrumbs, user context, and stack-trace local variables can leak to the configured Sentry endpoint.</evidence>
<recommendation>Expand scrubEvent to recursively redact secret-patterned keys across the entire Sentry event object, not just a whitelist of three fields. Alternatively, specifically target additional secret-bearing fields such as request.url, request.query_string, breadcrumbs, exception.values[].value, and user context. Consider using Sentry&apos;s built-in sendDefaultPii: false and server-side data scrubbing as defense-in-depth.</recommendation>
</violation>
</file>

});
active = true;
return true;
}

/** Capture an error with optional structured context. No-op when Sentry is off. */
export function captureError(
error: unknown,
context?: Record<string, unknown>,
): void {
if (!active || !Sentry) return;
Sentry.withScope((scope) => {
if (context) scope.setContext("gittensory", context);
Sentry!.captureException(
error instanceof Error ? error : new Error(String(error)),
);
});
}

/** Capture a degraded/failed review at WARNING level, tagged by repo/PR/SHA for triage. No-op when off. */
export function captureReviewFailure(
error: unknown,
context?: Record<string, unknown>,
): void {
if (!active || !Sentry) return;
Sentry.withScope((scope) => {
scope.setLevel("warning");
if (context) {
scope.setContext("review", context);
for (const tag of ["owner", "repo", "pr", "head_sha"]) {
const value = context[tag];
if (value !== undefined && value !== null)
scope.setTag(tag, String(value));
}
}
Sentry!.captureException(
error instanceof Error ? error : new Error(String(error)),
);
});
}

/** Flush buffered events before exit. No-op when off. */
export async function flushSentry(timeoutMs = 2000): Promise<void> {
if (!active || !Sentry) return;
await Sentry.flush(timeoutMs).catch(() => undefined);
}

/** Test-only: reset module state between cases. */
export function resetSentryForTest(): void {
Sentry = undefined;
active = false;
}
Loading
Loading