From 06db684f0193c1bc5daa081ad7882b9a8c60ed35 Mon Sep 17 00:00:00 2001 From: JSONbored <49853598+JSONbored@users.noreply.github.com> Date: Sun, 28 Jun 2026 05:11:54 -0700 Subject: [PATCH 01/68] feat(selfhost): enable Codex review runtime support --- .env.example | 8 +- Dockerfile | 5 +- docker-compose.yml | 7 +- docs/self-host/ai-providers.md | 21 ++- docs/self-host/troubleshooting.md | 7 + grafana/dashboards/codex-usage.json | 168 ++++++++++++++++++++ grafana/dashboards/resource-hub.json | 2 +- grafana/provisioning/datasources/sqlite.yml | 14 +- src/queue/processors.ts | 5 +- src/selfhost/ai.ts | 100 +++++++++++- src/selfhost/private-config.ts | 16 +- src/server.ts | 5 +- src/signals/focus-manifest.ts | 2 +- test/unit/private-config.test.ts | 8 +- test/unit/selfhost-ai.test.ts | 42 ++++- 15 files changed, 375 insertions(+), 35 deletions(-) create mode 100644 grafana/dashboards/codex-usage.json diff --git a/.env.example b/.env.example index e9a7934f9a..f5fb491730 100644 --- a/.env.example +++ b/.env.example @@ -208,9 +208,11 @@ GITTENSORY_REVIEW_DRAFT=false # OPENAI_API_KEY= # for AI_PROVIDER=openai # CLAUDE_CODE_OAUTH_TOKEN= # for AI_PROVIDER=claude-code (subscription; from `claude setup-token`) # -# Codex (ChatGPT subscription) reviewer is disabled by default for self-host PR review: `codex exec` stores its -# OAuth credential in auth.json on the same filesystem that prompt-influenced reviews can read. Do not mount or copy -# ~/.codex/auth.json into the app container; use claude-code, an API-key provider, or a local OpenAI-compatible model. +# Codex (ChatGPT subscription) reviewer is fail-closed by default for self-host PR review: `codex exec` stores its +# OAuth credential in auth.json on the same filesystem that prompt-influenced reviews can read. Isolated maintainer +# deployments can opt in explicitly after mounting auth at /data/codex (the image exposes it as ~/.codex). +# GITTENSORY_ENABLE_UNSAFE_CODEX_REVIEWER=1 +# Do NOT set CODEX_HOME for the app container; the provider rejects it so credentials are not advertised in env. # AI_MODEL=llama3.1 # the model for your provider (e.g. llama3.1 for Ollama, sonnet # # for claude-code, gpt-5 for codex). REQUIRED for non-Ollama: # # without it the adapter falls back to a provider default, never diff --git a/Dockerfile b/Dockerfile index c9320d21b6..0092349035 100644 --- a/Dockerfile +++ b/Dockerfile @@ -37,7 +37,10 @@ RUN if [ "$INSTALL_AI_CLIS" = "true" ]; then apt-get update && apt-get install - # claude-code's postinstall downloads its platform-native binary, so scripts must run. Install the optional # CLIs as the unprivileged user into a user-owned prefix while /app is still root-owned, keeping lifecycle # hooks from mutating the already-copied application bundle during the image build. -RUN mkdir -p /home/node/.npm-global /home/node/.npm && chown -R node:node /home/node/.npm-global /home/node/.npm +RUN mkdir -p /home/node/.npm-global /home/node/.npm \ + && ln -s /data/codex /home/node/.codex \ + && chown -h node:node /home/node/.codex \ + && chown -R node:node /home/node/.npm-global /home/node/.npm USER node RUN if [ "$INSTALL_AI_CLIS" = "true" ]; then npm install -g @anthropic-ai/claude-code@2.1.187 @openai/codex@0.142.0; fi USER root diff --git a/docker-compose.yml b/docker-compose.yml index c078ccfcd1..19af1054c9 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -289,12 +289,13 @@ services: - grafana-data:/var/lib/grafana - ./grafana/provisioning:/etc/grafana/provisioning:ro - ./grafana/dashboards:/var/lib/grafana/dashboards:ro + # Maintainer dashboards query aggregate review/AI-usage tables from the app DB. + # SQLite WAL readers need the shm file, so this cannot be read-only. + - gittensory-data:/appdb environment: GF_SECURITY_ADMIN_PASSWORD: ${GRAFANA_ADMIN_PASSWORD:?Set GRAFANA_ADMIN_PASSWORD in .env before using --profile observability} GF_USERS_ALLOW_SIGN_UP: "false" - # Only install the signed GitHub datasource plugin. Do not mount or query the live app SQLite - # database from Grafana; observability must not bypass application authorization. - GF_INSTALL_PLUGINS: grafana-github-datasource + GF_INSTALL_PLUGINS: frser-sqlite-datasource,grafana-github-datasource # Read-only fine-grained PAT for the GitHub data source provisioning ($GITHUB_TOKEN expansion). From .env. GITHUB_TOKEN: "${GITHUB_TOKEN:-}" diff --git a/docs/self-host/ai-providers.md b/docs/self-host/ai-providers.md index 37381f4b63..17cb29f364 100644 --- a/docs/self-host/ai-providers.md +++ b/docs/self-host/ai-providers.md @@ -7,7 +7,7 @@ The reviewer is configured by `AI_PROVIDER`. Reviews degrade deterministically ( | `AI_PROVIDER` | Backend | Needs | | ----------------------------------------- | ----------------------------------------------------------------------- | --------------------------------------------------------------------------------------- | | `claude-code` | Your **Claude** subscription via the `claude` CLI (read-only, headless) | `CLAUDE_CODE_OAUTH_TOKEN` (`claude setup-token`); CLI baked in (`INSTALL_AI_CLIS=true`) | -| `codex` | Your **Codex** subscription via the `codex` CLI | local `codex` auth (mounted), CLI baked in | +| `codex` | Your **Codex** subscription via the `codex` CLI | local `codex` auth mounted at `/data/codex`, CLI baked in, explicit unsafe opt-in | | `anthropic` | Native **Anthropic API** (BYOK, per-token billing — no weekly limit) | `ANTHROPIC_API_KEY`, `AI_MODEL` | | `ollama` / `openai-compatible` / `openai` | Any OpenAI-compatible `/chat/completions` (+ `/embeddings`) | `AI_BASE_URL`, `AI_API_KEY`, `AI_MODEL` | @@ -27,6 +27,15 @@ combined per `AI_COMBINE` (`single`/`consensus`/`synthesis`). | `AI_EFFORT` | `high` | `low \| medium \| high \| xhigh \| max` → `claude --effort`. The engine wants substance, not speed. | | `AI_TIMEOUT_MS` | scales with effort | Subprocess timeout. Unset ⇒ low/med 120s, high 240s, xhigh 360s, **max 600s** (so a big max-effort review isn't killed). Override clamped 30s–30min. | +## Codex subscription reviewer + +Codex is intentionally disabled until the operator opts in with +`GITTENSORY_ENABLE_UNSAFE_CODEX_REVIEWER=1`. The risk is specific: `codex exec` needs an OAuth +`auth.json`, and a prompt-influenced read-only review sandbox can still read files. On an isolated +maintainer deployment, mount the Codex home at `/data/codex`; the image exposes that as the default +`~/.codex` path for the `node` user. Do not set `CODEX_HOME` in the app environment. The provider +rejects `CODEX_HOME` so the credential path is not advertised to the subprocess through env. + ## Cost & usage observability Every provider's token/cost usage is captured and exported to Prometheus — surfaced in the **AI Usage & Cost** @@ -34,11 +43,13 @@ row of the Grafana dashboard (`:3000`): | Metric | Labels | Meaning | | ----------------------------------------------------------- | ------------------------------- | ---------------------------------------------------------------- | -| `gittensory_ai_requests_total` | `provider, model, kind, effort` | Review/embed calls (the intelligence dial is the `effort` label) | -| `gittensory_ai_input_tokens_total` / `_output_tokens_total` | `provider, model, kind` | Token volume per provider/model | -| `gittensory_ai_cost_usd_total` | `provider` | Cumulative USD (from Claude Code's `total_cost_usd`) | +| `gittensory_ai_requests_total` | `provider, model, effort` | Successful subscription-CLI review calls | +| `gittensory_ai_input_tokens_total` / `_output_tokens_total` | `provider, model, kind, effort` | Token volume per provider/model when the CLI reports it | +| `gittensory_ai_total_tokens_total` | `provider, model, effort` | Total token count when the CLI reports it | +| `gittensory_ai_cost_usd_total` | `provider` | Cumulative USD when the CLI reports a cost field | -`kind` is `chat` (reviews) or `embed` (RAG). The embed provider's label is `AI_EMBED_PROVIDER` (default `ollama`). +`kind` is `review` for the subscription-CLI review path. Claude Code also exports its own OTEL metrics when +`CLAUDE_CODE_ENABLE_TELEMETRY=1`; Codex cost appears only if the CLI emits a cost field. ## Token-spend protection diff --git a/docs/self-host/troubleshooting.md b/docs/self-host/troubleshooting.md index 339e5d320f..2aa2ec3476 100644 --- a/docs/self-host/troubleshooting.md +++ b/docs/self-host/troubleshooting.md @@ -113,6 +113,13 @@ AI review entirely in advisory mode. If you still see repeats, check for an auto **Fix:** leave `AI_MODEL` unset for codex — it picks the account's own default. (Don't set a Claude model id on a `claude-code,codex` combo: `AI_MODEL` is global and a Claude id breaks codex.) +### `codex_credential_isolation_required` + +**Cause:** the Codex subscription reviewer is fail-closed unless you explicitly opt into mounted +Codex auth, and it rejects `CODEX_HOME` in the app env. +**Fix:** mount the Codex home at `/data/codex`, leave `CODEX_HOME` unset, and set +`GITTENSORY_ENABLE_UNSAFE_CODEX_REVIEWER=1` only on an isolated maintainer deployment. + ### Reviews post as `gittensory-orb`, not your own bot **Cause:** a brokered self-host borrows tokens from the central Orb App. To post under your own bot identity you diff --git a/grafana/dashboards/codex-usage.json b/grafana/dashboards/codex-usage.json new file mode 100644 index 0000000000..67d70a109b --- /dev/null +++ b/grafana/dashboards/codex-usage.json @@ -0,0 +1,168 @@ +{ + "uid": "gittensory-codex", + "title": "Gittensory - Codex usage (self-host)", + "tags": ["gittensory", "codex", "ai"], + "timezone": "browser", + "schemaVersion": 39, + "version": 1, + "refresh": "30s", + "time": { "from": "now-7d", "to": "now" }, + "description": "Codex review usage for the self-hosted stack. Live counters come from the app's subscription-CLI adapter; durable review records come from ai_usage_events. USD cost appears only when the Codex CLI emits a cost field.", + "panels": [ + { + "id": 1, + "type": "row", + "title": "Summary", + "gridPos": { "h": 1, "w": 24, "x": 0, "y": 0 } + }, + { + "id": 2, + "type": "stat", + "title": "Reported cost", + "description": "USD cost reported by Codex JSON/JSONL output, if available. Subscription CLIs may not emit this.", + "gridPos": { "h": 5, "w": 6, "x": 0, "y": 1 }, + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "fieldConfig": { "defaults": { "unit": "currencyUSD", "decimals": 4, "color": { "mode": "fixed", "fixedColor": "green" } } }, + "options": { "reduceOptions": { "calcs": ["lastNotNull"] }, "colorMode": "value", "graphMode": "area", "justifyMode": "center" }, + "targets": [{ "refId": "A", "instant": true, "expr": "sum(gittensory_ai_cost_usd_total{provider=\"codex\"}) or vector(0)" }] + }, + { + "id": 3, + "type": "stat", + "title": "CLI requests", + "gridPos": { "h": 5, "w": 6, "x": 6, "y": 1 }, + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "fieldConfig": { "defaults": { "unit": "short", "color": { "mode": "fixed", "fixedColor": "blue" } } }, + "options": { "reduceOptions": { "calcs": ["lastNotNull"] }, "colorMode": "value", "graphMode": "area", "justifyMode": "center" }, + "targets": [{ "refId": "A", "instant": true, "expr": "sum(gittensory_ai_requests_total{provider=\"codex\"}) or vector(0)" }] + }, + { + "id": 4, + "type": "stat", + "title": "CLI tokens", + "gridPos": { "h": 5, "w": 6, "x": 12, "y": 1 }, + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "fieldConfig": { "defaults": { "unit": "short", "color": { "mode": "fixed", "fixedColor": "purple" } } }, + "options": { "reduceOptions": { "calcs": ["lastNotNull"] }, "colorMode": "value", "graphMode": "area", "justifyMode": "center" }, + "targets": [{ "refId": "A", "instant": true, "expr": "sum(gittensory_ai_total_tokens_total{provider=\"codex\"}) or (sum(gittensory_ai_input_tokens_total{provider=\"codex\"}) + sum(gittensory_ai_output_tokens_total{provider=\"codex\"})) or vector(0)" }] + }, + { + "id": 5, + "type": "stat", + "title": "Review records", + "gridPos": { "h": 5, "w": 6, "x": 18, "y": 1 }, + "datasource": { "type": "frser-sqlite-datasource", "uid": "gittensory-db" }, + "fieldConfig": { "defaults": { "unit": "short", "color": { "mode": "fixed", "fixedColor": "orange" } } }, + "options": { "reduceOptions": { "calcs": ["lastNotNull"] }, "colorMode": "value", "graphMode": "none", "justifyMode": "center" }, + "targets": [ + { + "refId": "A", + "queryType": "table", + "queryText": "SELECT count(*) AS reviews FROM ai_usage_events WHERE feature = 'ai_review_pr' AND model LIKE 'codex%'", + "rawQueryText": "SELECT count(*) AS reviews FROM ai_usage_events WHERE feature = 'ai_review_pr' AND model LIKE 'codex%'" + } + ] + }, + { + "id": 6, + "type": "row", + "title": "Live counters", + "gridPos": { "h": 1, "w": 24, "x": 0, "y": 6 } + }, + { + "id": 7, + "type": "timeseries", + "title": "Requests by model and effort", + "gridPos": { "h": 8, "w": 12, "x": 0, "y": 7 }, + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "fieldConfig": { "defaults": { "unit": "short", "custom": { "drawStyle": "bars", "fillOpacity": 70, "lineWidth": 1, "stacking": { "mode": "none" } } } }, + "options": { "legend": { "showLegend": true, "placement": "bottom" }, "tooltip": { "mode": "multi", "sort": "desc" } }, + "targets": [ + { + "refId": "A", + "expr": "sum by (model, effort) (increase(gittensory_ai_requests_total{provider=\"codex\"}[$__rate_interval]))", + "legendFormat": "{{model}} / {{effort}}" + } + ] + }, + { + "id": 8, + "type": "timeseries", + "title": "Tokens by direction", + "gridPos": { "h": 8, "w": 12, "x": 12, "y": 7 }, + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "fieldConfig": { "defaults": { "unit": "short", "custom": { "drawStyle": "bars", "fillOpacity": 70, "lineWidth": 1, "stacking": { "mode": "normal" } } } }, + "options": { "legend": { "showLegend": true, "placement": "bottom" }, "tooltip": { "mode": "multi", "sort": "desc" } }, + "targets": [ + { + "refId": "A", + "expr": "sum by (kind) (increase(gittensory_ai_input_tokens_total{provider=\"codex\"}[$__rate_interval]))", + "legendFormat": "input {{kind}}" + }, + { + "refId": "B", + "expr": "sum by (kind) (increase(gittensory_ai_output_tokens_total{provider=\"codex\"}[$__rate_interval]))", + "legendFormat": "output {{kind}}" + } + ] + }, + { + "id": 9, + "type": "row", + "title": "Durable review records", + "gridPos": { "h": 1, "w": 24, "x": 0, "y": 15 } + }, + { + "id": 10, + "type": "timeseries", + "title": "Estimated neuron usage by day", + "gridPos": { "h": 8, "w": 12, "x": 0, "y": 16 }, + "datasource": { "type": "frser-sqlite-datasource", "uid": "gittensory-db" }, + "fieldConfig": { "defaults": { "unit": "short", "custom": { "drawStyle": "bars", "fillOpacity": 70, "lineWidth": 1 } } }, + "options": { "legend": { "showLegend": true, "placement": "bottom" }, "tooltip": { "mode": "multi" } }, + "targets": [ + { + "refId": "A", + "queryType": "time series", + "timeColumns": ["time"], + "queryText": "SELECT date(created_at) AS time, sum(estimated_neurons) AS estimated_neurons FROM ai_usage_events WHERE feature = 'ai_review_pr' AND model LIKE 'codex%' GROUP BY date(created_at) ORDER BY time", + "rawQueryText": "SELECT date(created_at) AS time, sum(estimated_neurons) AS estimated_neurons FROM ai_usage_events WHERE feature = 'ai_review_pr' AND model LIKE 'codex%' GROUP BY date(created_at) ORDER BY time" + } + ] + }, + { + "id": 11, + "type": "piechart", + "title": "Review status", + "gridPos": { "h": 8, "w": 12, "x": 12, "y": 16 }, + "datasource": { "type": "frser-sqlite-datasource", "uid": "gittensory-db" }, + "fieldConfig": { "defaults": { "unit": "short" } }, + "options": { "pieType": "donut", "legend": { "showLegend": true, "placement": "right", "values": ["percent", "value"] }, "reduceOptions": { "calcs": ["lastNotNull"] }, "displayLabels": ["percent"] }, + "targets": [ + { + "refId": "A", + "queryType": "table", + "queryText": "SELECT status, count(*) AS count FROM ai_usage_events WHERE feature = 'ai_review_pr' AND model LIKE 'codex%' GROUP BY status ORDER BY count DESC", + "rawQueryText": "SELECT status, count(*) AS count FROM ai_usage_events WHERE feature = 'ai_review_pr' AND model LIKE 'codex%' GROUP BY status ORDER BY count DESC" + } + ] + }, + { + "id": 12, + "type": "table", + "title": "Recent Codex review events", + "gridPos": { "h": 10, "w": 24, "x": 0, "y": 24 }, + "datasource": { "type": "frser-sqlite-datasource", "uid": "gittensory-db" }, + "options": { "showHeader": true, "cellHeight": "sm", "sortBy": [{ "displayName": "created_at", "desc": true }] }, + "fieldConfig": { "defaults": { "custom": { "filterable": true } } }, + "targets": [ + { + "refId": "A", + "queryType": "table", + "queryText": "SELECT created_at, model, status, estimated_neurons, detail, json_extract(metadata_json, '$.repoFullName') AS repo, json_extract(metadata_json, '$.pullNumber') AS pr FROM ai_usage_events WHERE feature = 'ai_review_pr' AND model LIKE 'codex%' ORDER BY created_at DESC LIMIT 100", + "rawQueryText": "SELECT created_at, model, status, estimated_neurons, detail, json_extract(metadata_json, '$.repoFullName') AS repo, json_extract(metadata_json, '$.pullNumber') AS pr FROM ai_usage_events WHERE feature = 'ai_review_pr' AND model LIKE 'codex%' ORDER BY created_at DESC LIMIT 100" + } + ] + } + ] +} diff --git a/grafana/dashboards/resource-hub.json b/grafana/dashboards/resource-hub.json index 20a6b96b07..593a382619 100644 --- a/grafana/dashboards/resource-hub.json +++ b/grafana/dashboards/resource-hub.json @@ -58,7 +58,7 @@ "gridPos": { "h": 13, "w": 12, "x": 12, "y": 0 }, "options": { "mode": "markdown", - "content": "## 📊 Dashboards\n- **[Upstream PRs & issues (GitHub)](/d/gittensory-github)** — live, accurate census + open-PR triage (GitHub API).\n- **[Reviews & PRs (maintainer)](/d/gittensory-maintainer)** — gittensory's own review activity + reviewed-PR log.\n- **[Claude usage (OTEL)](/d/gittensory-claude)** — cost / tokens / model / effort from the review CLI.\n- **[Gittensory (infra)](/d/gittensory)** — AI usage & cost, queue, jobs, HTTP.\n\n## 📈 Metrics & logs\n- **Prometheus** — [targets](http://localhost:9090/targets) · [graph](http://localhost:9090)\n- **Alertmanager** — [alerts](http://localhost:9093)\n- **Loki** — query in [Explore](/explore) (pick the *Loki* datasource), e.g. `{compose_service=\"gittensory\"}`\n\n## 🩺 Quick health checks\n| What | Where |\n|---|---|\n| App serving | `GET /ready` → 200 |\n| AI wired | boot log `selfhost_ai_provider` |\n| Embeds wired | boot log `selfhost_embed_provider` |\n| Vectors wired | boot log `selfhost_vectorize` |\n| Token spend | infra dashboard → *AI Usage & Cost* |\n\n## 📚 Docs\n- `docs/self-host/` — configuration, ai-providers, rag-indexing, review-modes, troubleshooting." + "content": "## 📊 Dashboards\n- **[Upstream PRs & issues (GitHub)](/d/gittensory-github)** — live, accurate census + open-PR triage (GitHub API).\n- **[Reviews & PRs (maintainer)](/d/gittensory-maintainer)** — gittensory's own review activity + reviewed-PR log.\n- **[Claude usage (OTEL)](/d/gittensory-claude)** — cost / tokens / model / effort from the review CLI.\n- **[Codex usage (self-host)](/d/gittensory-codex)** — Codex review counts, token counters, and durable AI usage rows.\n- **[Gittensory (infra)](/d/gittensory)** — AI usage & cost, queue, jobs, HTTP.\n\n## 📈 Metrics & logs\n- **Prometheus** — [targets](http://localhost:9090/targets) · [graph](http://localhost:9090)\n- **Alertmanager** — [alerts](http://localhost:9093)\n- **Loki** — query in [Explore](/explore) (pick the *Loki* datasource), e.g. `{compose_service=\"gittensory\"}`\n\n## 🩺 Quick health checks\n| What | Where |\n|---|---|\n| App serving | `GET /ready` → 200 |\n| AI wired | boot log `selfhost_ai_provider` |\n| Embeds wired | boot log `selfhost_embed_provider` |\n| Vectors wired | boot log `selfhost_vectorize` |\n| Token spend | infra dashboard → *AI Usage & Cost* |\n\n## 📚 Docs\n- `docs/self-host/` — configuration, ai-providers, rag-indexing, review-modes, troubleshooting." } } ] diff --git a/grafana/provisioning/datasources/sqlite.yml b/grafana/provisioning/datasources/sqlite.yml index c3b9dc4fb7..ccc205ec75 100644 --- a/grafana/provisioning/datasources/sqlite.yml +++ b/grafana/provisioning/datasources/sqlite.yml @@ -1,5 +1,11 @@ -# Intentionally no SQLite datasource is provisioned. Grafana must not receive direct access to -# the live application database; dashboards should use Prometheus/GitHub or a separately redacted -# reporting database if one is added in the future. +# Maintainer-only SQLite datasource for local operator dashboards. The compose stack mounts the app +# database at /appdb for Grafana, and these dashboards run aggregate SELECTs over internal tables. apiVersion: 1 -datasources: [] +datasources: + - name: GittensoryDB + type: frser-sqlite-datasource + uid: gittensory-db + access: proxy + editable: false + jsonData: + path: /appdb/gittensory.sqlite diff --git a/src/queue/processors.ts b/src/queue/processors.ts index 83b6d98e2d..742c1f3d58 100644 --- a/src/queue/processors.ts +++ b/src/queue/processors.ts @@ -4591,8 +4591,9 @@ async function maybePublishPrPublicSurface( repoFullName, reviewInlineComments, ); - // Per-repo review CONTEXT (#review-skills): fold the container-private review/CLAUDE.md guide + the matching - // review/skills/*.md modules into the SAME review-instructions slot, so reviews follow each repo's conventions. + // Per-repo review CONTEXT (#review-skills): fold the container-private review/AGENTS.md (or legacy + // review/CLAUDE.md) guide + the matching review/skills/*.md modules into the SAME review-instructions slot, + // so reviews follow each repo's conventions. // Glob-gated for cost (only skills matching the changed files are injected); absent config dir ⇒ empty ⇒ // byte-identical prompt. getReviewFiles() is memoized, so the second call reuses the loaded diff. const reviewInstructions = diff --git a/src/selfhost/ai.ts b/src/selfhost/ai.ts index a612397612..5cd4ea1465 100644 --- a/src/selfhost/ai.ts +++ b/src/selfhost/ai.ts @@ -7,6 +7,7 @@ // records an error and degrades — never a silent wrong answer). import type { CombineStrategy, OnMerge } from "../services/ai-review"; +import { incr } from "./metrics"; interface AiRunOptions { messages?: Array<{ role: string; content: string }>; @@ -196,7 +197,17 @@ export function extractCliText(stdout: string): string { try { const o = JSON.parse(s) as Record; const text = o.result ?? o.text ?? o.content ?? o.response; - return typeof text === "string" ? text : ""; + if (typeof text === "string") return text; + const item = asRecord(o.item); + if (typeof item?.text === "string") return item.text; + const content = item?.content; + if (Array.isArray(content)) { + return content + .map((part) => asRecord(part)?.text) + .filter((part): part is string => typeof part === "string") + .join(""); + } + return ""; } catch { return ""; } @@ -214,6 +225,91 @@ export function extractCliText(stdout: string): string { return ""; } +export type CliUsage = { + inputTokens?: number; + outputTokens?: number; + totalTokens?: number; + costUsd?: number; + model?: string; +}; + +const INPUT_TOKEN_KEYS = ["input_tokens", "inputTokens", "prompt_tokens", "promptTokens"] as const; +const OUTPUT_TOKEN_KEYS = ["output_tokens", "outputTokens", "completion_tokens", "completionTokens"] as const; +const TOTAL_TOKEN_KEYS = ["total_tokens", "totalTokens"] as const; +const COST_KEYS = ["total_cost_usd", "totalCostUsd", "cost_usd", "costUsd"] as const; + +function asRecord(value: unknown): Record | null { + return value && typeof value === "object" && !Array.isArray(value) ? (value as Record) : null; +} + +function finiteNumber(value: unknown): number | undefined { + const n = typeof value === "number" ? value : typeof value === "string" && value.trim() ? Number(value) : NaN; + return Number.isFinite(n) && n >= 0 ? n : undefined; +} + +function maxNumber(record: Record, keys: readonly string[]): number | undefined { + let out: number | undefined; + for (const key of keys) { + const n = finiteNumber(record[key]); + if (n !== undefined) out = Math.max(out ?? 0, n); + } + return out; +} + +function mergeUsage(out: CliUsage, record: Record): void { + const nested = [ + record, + asRecord(record.usage), + asRecord(record.token_usage), + asRecord(record.tokenUsage), + asRecord(record.usage_metadata), + asRecord(record.usageMetadata), + ].filter((entry): entry is Record => Boolean(entry)); + for (const entry of nested) { + const inputTokens = maxNumber(entry, INPUT_TOKEN_KEYS); + if (inputTokens !== undefined) out.inputTokens = Math.max(out.inputTokens ?? 0, inputTokens); + const outputTokens = maxNumber(entry, OUTPUT_TOKEN_KEYS); + if (outputTokens !== undefined) out.outputTokens = Math.max(out.outputTokens ?? 0, outputTokens); + const totalTokens = maxNumber(entry, TOTAL_TOKEN_KEYS); + if (totalTokens !== undefined) out.totalTokens = Math.max(out.totalTokens ?? 0, totalTokens); + const costUsd = maxNumber(entry, COST_KEYS); + if (costUsd !== undefined) out.costUsd = Math.max(out.costUsd ?? 0, costUsd); + if (typeof entry.model === "string" && entry.model.trim()) out.model = entry.model.trim(); + } +} + +/** Best-effort usage extraction from subscription CLI JSON/JSONL output. Claude Code's authoritative usage is OTEL, + * while Codex JSONL is still evolving, so this accepts common token/cost field spellings and records the largest + * cumulative value seen across the stream. Missing fields simply mean "no metric", never a review failure. */ +export function extractCliUsage(stdout: string): CliUsage { + const usage: CliUsage = {}; + const trimmed = stdout.trim(); + if (!trimmed) return usage; + const parse = (text: string): void => { + try { + const record = asRecord(JSON.parse(text)); + if (record) mergeUsage(usage, record); + } catch { + // Non-JSON output is valid for some CLI failure modes; usage is best-effort only. + } + }; + parse(trimmed); + for (const line of trimmed.split(/\r?\n/)) { + if (line.trim()) parse(line); + } + return usage; +} + +function recordCliUsageMetrics(provider: string, model: string, effort: string, stdout: string): void { + const usage = extractCliUsage(stdout); + const labels = { provider, model: usage.model ?? (model || "default"), effort }; + incr("gittensory_ai_requests_total", labels); + incr("gittensory_ai_cost_usd_total", { provider: labels.provider }, usage.costUsd ?? 0); + if (usage.inputTokens !== undefined) incr("gittensory_ai_input_tokens_total", { ...labels, kind: "review" }, usage.inputTokens); + if (usage.outputTokens !== undefined) incr("gittensory_ai_output_tokens_total", { ...labels, kind: "review" }, usage.outputTokens); + if (usage.totalTokens !== undefined) incr("gittensory_ai_total_tokens_total", labels, usage.totalTokens); +} + /** Claude Code's `--output-format json` exits 0 even on an API/auth error, returning {is_error:true,result:""}. * Detect it so the error string is never surfaced as the model's answer. */ export function claudeErrorStatus(stdout: string): string | null { @@ -319,6 +415,7 @@ export function createClaudeCodeAi(parentEnv: Record if (code !== 0) throw new Error(`claude_code_exit_${code ?? "null"}: ${redactSecrets(stderr ?? "", [token]).slice(0, 500)}`); const text = extractCliText(stdout); if (!text) throw new Error("claude_code_empty_output"); + recordCliUsageMetrics("claude-code", claudeModel, effort, stdout); return { response: text }; }, }; @@ -351,6 +448,7 @@ export function createCodexAi(parentEnv: Record, spa if (code !== 0) throw new Error(`codex_exit_${code ?? "null"}: ${redactSecrets(stderr ?? "").slice(0, 500)}`); const text = extractCliText(stdout); if (!text) throw new Error("codex_empty_output"); + recordCliUsageMetrics("codex", codexModel, resolveEffort(parentEnv.AI_EFFORT), stdout); return { response: text }; }, }; diff --git a/src/selfhost/private-config.ts b/src/selfhost/private-config.ts index b3a12ccd83..9fcb69b5ff 100644 --- a/src/selfhost/private-config.ts +++ b/src/selfhost/private-config.ts @@ -108,8 +108,9 @@ export function parseReviewSkill(filename: string, text: string): RepoReviewSkil } /** Build the container-local review-context reader over GITTENSORY_REPO_CONFIG_DIR, or null when the dir is unset. Per - * repo (first existing folder wins) reads `review/CLAUDE.md` (the guide) + every `review/skills/*.md` (rubric modules, - * sorted). Missing files/dir degrade to nulls/empty; a per-file read error skips that file. (#review-skills) */ + * repo (first existing folder wins) reads `review/AGENTS.md` (Codex) or `review/CLAUDE.md` (Claude Code) as the + * guide + every `review/skills/*.md` rubric module, sorted. Missing files/dir degrade to nulls/empty; a per-file + * read error skips that file. (#review-skills) */ export function makeLocalReviewContextReader(dir: string | undefined): RepoReviewContextReader | null { const trimmed = (dir ?? "").trim(); if (!trimmed) return null; @@ -118,10 +119,13 @@ export function makeLocalReviewContextReader(dir: string | undefined): RepoRevie for (const folder of reviewContextFolders(repoFullName)) { const abs = resolve(base, folder); let guide: string | null = null; - try { - guide = await readFile(resolve(abs, "CLAUDE.md"), "utf8"); - } catch { - // no per-repo review guide + for (const guideName of ["AGENTS.md", "CLAUDE.md"]) { + try { + guide = await readFile(resolve(abs, guideName), "utf8"); + break; + } catch { + // no per-repo guide at this candidate name + } } const skills: RepoReviewSkill[] = []; try { diff --git a/src/server.ts b/src/server.ts index 17977f0f08..683232b418 100644 --- a/src/server.ts +++ b/src/server.ts @@ -237,8 +237,9 @@ async function main(): Promise { setLocalManifestReader( makeLocalManifestReader(process.env.GITTENSORY_REPO_CONFIG_DIR), ); - // Per-repo review CONTEXT (#review-skills): the same config dir also holds `/review/CLAUDE.md` + skills/*.md, - // injected into the reviewer prompt so reviews follow each repo's conventions. Unset dir ⇒ null reader ⇒ no change. + // Per-repo review CONTEXT (#review-skills): the same config dir also holds `/review/AGENTS.md` + // (or legacy `/review/CLAUDE.md`) + skills/*.md, injected into the reviewer prompt so reviews follow each + // repo's conventions. Unset dir ⇒ null reader ⇒ no change. setLocalReviewContextReader( makeLocalReviewContextReader(process.env.GITTENSORY_REPO_CONFIG_DIR), ); diff --git a/src/signals/focus-manifest.ts b/src/signals/focus-manifest.ts index 1638f365f4..1dcab80df3 100644 --- a/src/signals/focus-manifest.ts +++ b/src/signals/focus-manifest.ts @@ -886,7 +886,7 @@ export function resolveReviewPreMergeChecks(manifest: FocusManifest | null): Pre * config dir (`/review/skills/*.md`). `when` is "always" (repo-wide) or a path glob / brace-list that gates it to * matching changed files (cost: only relevant skills are injected). */ export type RepoReviewSkill = { name: string; when: string; body: string }; -/** The per-repo review CONTEXT (#review-skills): an always-on `review/CLAUDE.md` guide + the skill rubric modules. */ +/** The per-repo review CONTEXT (#review-skills): an always-on `review/AGENTS.md` / `review/CLAUDE.md` guide + skills. */ export type RepoReviewContext = { guide: string | null; skills: RepoReviewSkill[] }; /** Hard cap on the injected per-repo review context — a cost guard so a runaway guide/skills set can't bloat every diff --git a/test/unit/private-config.test.ts b/test/unit/private-config.test.ts index fcf8bf3766..4b8fcc7a4c 100644 --- a/test/unit/private-config.test.ts +++ b/test/unit/private-config.test.ts @@ -124,21 +124,23 @@ describe("makeLocalReviewContextReader (#review-skills)", () => { expect(makeLocalReviewContextReader(" ")).toBeNull(); }); - it("reads the owner-qualified review/CLAUDE.md + skills/*.md (sorted, .md only)", async () => { + it("reads the owner-qualified review/AGENTS.md + skills/*.md (sorted, .md only)", async () => { const dir = mkdtempSync(join(tmpdir(), "gt-review-")); const rev = join(dir, "jsonbored__gittensory", "review"); mkdirSync(join(rev, "skills"), { recursive: true }); - writeFileSync(join(rev, "CLAUDE.md"), "Review gittensory carefully.\n"); + writeFileSync(join(rev, "AGENTS.md"), "Review gittensory carefully.\n"); + writeFileSync(join(rev, "CLAUDE.md"), "Legacy guide should not win.\n"); writeFileSync(join(rev, "skills", "b-second.md"), "---\nname: second\nwhen: always\n---\nSecond.\n"); writeFileSync(join(rev, "skills", "a-first.md"), "First with no frontmatter.\n"); writeFileSync(join(rev, "skills", "notes.txt"), "ignored — not .md\n"); const reader = makeLocalReviewContextReader(dir)!; const ctx = await reader("JSONbored/gittensory"); expect(ctx.guide).toContain("Review gittensory carefully."); + expect(ctx.guide).not.toContain("Legacy guide should not win."); expect(ctx.skills.map((s) => s.name)).toEqual(["a-first", "second"]); // sorted by filename; .txt ignored }); - it("falls back to the bare repo-name folder; returns empty for a missing or invalid repo", async () => { + it("falls back to legacy CLAUDE.md in the bare repo-name folder; returns empty for a missing or invalid repo", async () => { const dir = mkdtempSync(join(tmpdir(), "gt-review-")); mkdirSync(join(dir, "metagraphed", "review"), { recursive: true }); writeFileSync(join(dir, "metagraphed", "review", "CLAUDE.md"), "Bare-folder guide.\n"); diff --git a/test/unit/selfhost-ai.test.ts b/test/unit/selfhost-ai.test.ts index c09275c8f3..b43bef3d1d 100644 --- a/test/unit/selfhost-ai.test.ts +++ b/test/unit/selfhost-ai.test.ts @@ -2,7 +2,8 @@ import { chmodSync, mkdtempSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { afterEach, describe, expect, it, vi } from "vitest"; -import { buildProvider, claudeErrorStatus, createAnthropicAi, createChainAi, createClaudeCodeAi, createCodexAi, createOpenAiCompatibleAi, createSelfHostAi, extractCliText, resolveAiReviewerPlan, resolveCliTimeoutMs, resolveEffort, resolveModel, resolveProviderNames, resolveRequiredCliProviders, redactSecrets, routeProviders, subscriptionCliEnv } from "../../src/selfhost/ai"; +import { buildProvider, claudeErrorStatus, createAnthropicAi, createChainAi, createClaudeCodeAi, createCodexAi, createOpenAiCompatibleAi, createSelfHostAi, extractCliText, extractCliUsage, resolveAiReviewerPlan, resolveCliTimeoutMs, resolveEffort, resolveModel, resolveProviderNames, resolveRequiredCliProviders, redactSecrets, routeProviders, subscriptionCliEnv } from "../../src/selfhost/ai"; +import { renderMetrics, resetMetrics } from "../../src/selfhost/metrics"; describe("resolveModel (#979 — never leak the Workers-AI default to a self-host backend)", () => { const WORKERS_DEFAULT = "@cf/meta/llama-3.1-8b-instruct-fp8-fast"; @@ -50,7 +51,10 @@ describe("resolveCliTimeoutMs (#selfhost — subprocess timeout scales with effo }); }); -afterEach(() => vi.unstubAllGlobals()); +afterEach(() => { + vi.unstubAllGlobals(); + resetMetrics(); +}); type SpawnResult = { stdout: string; code: number | null; stderr?: string }; type StubSpawn = ( @@ -257,7 +261,10 @@ describe("resolveProviderNames + resolveAiReviewerPlan (#dual-ai-combiner)", () }); describe("branch coverage — defaults + edge inputs", () => { - afterEach(() => vi.unstubAllGlobals()); + afterEach(() => { + vi.unstubAllGlobals(); + resetMetrics(); + }); it("chat with no apiKey + empty choices → empty response", async () => { vi.stubGlobal("fetch", vi.fn(async () => new Response(JSON.stringify({ choices: [] }), { status: 200 }))); @@ -275,6 +282,19 @@ describe("branch coverage — defaults + edge inputs", () => { expect(extractCliText(JSON.stringify({ result: 5 }))).toBe(""); expect(extractCliText(JSON.stringify({ text: "t" }))).toBe("t"); }); + it("extractCliUsage reads common JSON and JSONL token/cost fields", () => { + expect(extractCliUsage("")).toEqual({}); + expect(extractCliUsage("not json")).toEqual({}); + expect( + extractCliUsage( + [ + JSON.stringify({ usage: { input_tokens: 10, outputTokens: "5", total_tokens: 15 }, model: "gpt-5" }), + JSON.stringify({ tokenUsage: { prompt_tokens: 12, completion_tokens: 6, totalTokens: 18 }, total_cost_usd: "0.07" }), + JSON.stringify({ usage_metadata: { costUsd: 0.09 } }), + ].join("\n"), + ), + ).toEqual({ inputTokens: 12, outputTokens: 6, totalTokens: 18, costUsd: 0.09, model: "gpt-5" }); + }); it("claudeErrorStatus: subtype + unknown fallbacks", () => { expect(claudeErrorStatus(JSON.stringify({ is_error: true, subtype: "sub" }))).toBe("sub"); expect(claudeErrorStatus(JSON.stringify({ is_error: true }))).toBe("unknown"); @@ -321,6 +341,8 @@ describe("branch coverage — defaults + edge inputs", () => { it("extractCliText reads content + response fields", () => { expect(extractCliText(JSON.stringify({ content: "c" }))).toBe("c"); expect(extractCliText(JSON.stringify({ response: "r" }))).toBe("r"); + expect(extractCliText(JSON.stringify({ type: "item.completed", item: { type: "agent_message", text: "codex ok" } }))).toBe("codex ok"); + expect(extractCliText(JSON.stringify({ type: "item.completed", item: { content: [{ type: "output_text", text: "codex " }, { type: "output_text", text: "ok" }] } }))).toBe("codex ok"); }); it("chain wraps a non-Error throw", async () => { const p = { @@ -557,6 +579,20 @@ describe("subscription CLI helpers + fail-safe", () => { expect(extractCliText('not json\n{"result":"x"}')).toBe("x"); expect(extractCliText("not json\nstill not json")).toBe(""); }); + + it("records Codex CLI usage metrics from successful JSONL output", async () => { + const stdout = [ + JSON.stringify({ type: "token_count", usage: { input_tokens: 20, output_tokens: 7, total_tokens: 27 }, model: "gpt-5-codex" }), + JSON.stringify({ type: "result", result: "review" }), + ].join("\n"); + const ok: StubSpawn = async () => ({ stdout, code: 0 }); + await createCodexAi({ GITTENSORY_ENABLE_UNSAFE_CODEX_REVIEWER: "1", AI_EFFORT: "medium" }, ok).run("", { prompt: "x" }); + const metrics = await renderMetrics(); + expect(metrics).toContain('gittensory_ai_requests_total{effort="medium",model="gpt-5-codex",provider="codex"} 1'); + expect(metrics).toContain('gittensory_ai_input_tokens_total{effort="medium",kind="review",model="gpt-5-codex",provider="codex"} 20'); + expect(metrics).toContain('gittensory_ai_output_tokens_total{effort="medium",kind="review",model="gpt-5-codex",provider="codex"} 7'); + expect(metrics).toContain('gittensory_ai_total_tokens_total{effort="medium",model="gpt-5-codex",provider="codex"} 27'); + }); }); describe("redactSecrets — strip credentials from untrusted CLI stderr before it reaches logs/Sentry (#1605 sec)", () => { From e9d69b2f8ccafad11292e1d9402a59fd12e63c1c Mon Sep 17 00:00:00 2001 From: JSONbored <49853598+JSONbored@users.noreply.github.com> Date: Sun, 28 Jun 2026 06:01:23 -0700 Subject: [PATCH 02/68] fix(selfhost): make AI reviewer config explicit --- .env.example | 48 ++++-- docker-compose.yml | 14 +- docs/self-host/ai-providers.md | 23 ++- docs/self-host/configuration.md | 6 +- docs/self-host/troubleshooting.md | 13 +- docs/self-hosting.md | 40 ++--- src/env.d.ts | 21 ++- src/selfhost/ai.ts | 227 +++++++++++++++++++-------- src/selfhost/sentry.ts | 2 +- src/services/ai-review.ts | 27 +++- test/unit/ai-review-advisory.test.ts | 4 +- test/unit/selfhost-ai.test.ts | 87 +++++----- test/unit/selfhost-sentry.test.ts | 21 +++ 13 files changed, 363 insertions(+), 170 deletions(-) diff --git a/.env.example b/.env.example index f5fb491730..d50f4acfe5 100644 --- a/.env.example +++ b/.env.example @@ -142,22 +142,25 @@ GITTENSORY_REVIEW_DRAFT=false # CRON_INTERVAL_MS=120000 # maintain/sweep + sync cadence (default ~2 min) # --- Continuous backup (optional; the Litestream sidecar in docker-compose.yml) --- +# Blank is valid until --profile litestream is enabled. # LITESTREAM_ACCESS_KEY_ID= # LITESTREAM_SECRET_ACCESS_KEY= # LITESTREAM_ENDPOINT= # e.g. s3.us-west-002.backblazeb2.com (omit for AWS S3) # LITESTREAM_REGION=us-east-1 # --- Queue worker (#977/#1201) --- -# QUEUE_CONCURRENCY=1 # max concurrent job-processing loops per instance (default 1) +# QUEUE_CONCURRENCY=4 # max concurrent job-processing loops per instance (default 4; set 1 for strict serial processing) # --- Caddy HTTPS terminator (#1203; requires --profile caddy) --- # DOMAIN=gittensory.example.com # fully-qualified domain; Caddy auto-obtains a Let's Encrypt cert # --- Tailscale sidecar (#1204; requires --profile tailscale) --- +# Blank is valid until --profile tailscale is enabled. # TS_AUTHKEY= # Tailscale auth key (generate at tailscale.com/admin/settings/keys) # TS_EXTRA_ARGS= # extra tailscale up flags, e.g. --advertise-tags=tag:self-host # --- Self-hosted GitHub Actions runner (#1205; requires --profile runners) --- +# Blank tokens/URLs are valid until --profile runners is enabled. # RUNNER_TOKEN= # runner registration token (Settings → Actions → Runners → New) # RUNNER_REPO_URL=https://github.com/org/repo # RUNNER_ACCESS_TOKEN= # PAT with repo scope (alternative to RUNNER_TOKEN) @@ -202,21 +205,46 @@ GITTENSORY_REVIEW_DRAFT=false # # merged decision. single = one reviewer's verdict (auto when 1). # AI_ON_MERGE=either # synthesis merge rule: either (block if EITHER reviewer flags) | # # both (block only when both do). Ignored unless AI_COMBINE=synthesis. -# AI_BASE_URL=http://ollama:11434/v1 # OpenAI-compatible endpoint (Ollama default; or your provider's) -# AI_API_KEY= # generic key for the openai-compatible endpoint -# ANTHROPIC_API_KEY= # for AI_PROVIDER=anthropic (native Messages API, BYOK) -# OPENAI_API_KEY= # for AI_PROVIDER=openai -# CLAUDE_CODE_OAUTH_TOKEN= # for AI_PROVIDER=claude-code (subscription; from `claude setup-token`) +# Ollama reviewer (AI_PROVIDER=ollama). Defaults: OLLAMA_AI_BASE_URL=http://localhost:11434/v1, +# OLLAMA_AI_MODEL=llama3.1, no API key. Set the base URL to http://ollama:11434/v1 when using the compose +# --profile ollama service. +# OLLAMA_AI_BASE_URL=http://ollama:11434/v1 +# OLLAMA_AI_API_KEY= +# OLLAMA_AI_MODEL=llama3.1 +# +# Generic OpenAI-compatible reviewer (AI_PROVIDER=openai-compatible). Defaults: +# OPENAI_COMPATIBLE_AI_BASE_URL=http://localhost:11434/v1, OPENAI_COMPATIBLE_AI_MODEL=llama3.1. +# OPENAI_COMPATIBLE_AI_BASE_URL=http://localhost:11434/v1 +# OPENAI_COMPATIBLE_AI_API_KEY= +# OPENAI_COMPATIBLE_AI_MODEL=llama3.1 +# +# OpenAI API reviewer (AI_PROVIDER=openai). Defaults: OPENAI_AI_BASE_URL=https://api.openai.com/v1, +# OPENAI_AI_MODEL=llama3.1 unless set here. +# OPENAI_API_KEY= +# OPENAI_AI_BASE_URL=https://api.openai.com/v1 +# OPENAI_AI_MODEL=gpt-5.5 +# +# Anthropic API reviewer (AI_PROVIDER=anthropic). Defaults: ANTHROPIC_AI_BASE_URL=https://api.anthropic.com, +# ANTHROPIC_AI_MODEL=claude-sonnet-4-6 unless set here. +# ANTHROPIC_API_KEY= +# ANTHROPIC_AI_BASE_URL=https://api.anthropic.com +# ANTHROPIC_AI_MODEL=claude-sonnet-4-6 +# +# Claude Code subscription reviewer (AI_PROVIDER=claude-code). +# CLAUDE_CODE_OAUTH_TOKEN= # from `claude setup-token` +# CLAUDE_AI_MODEL=claude-sonnet-4-6 # any `claude` CLI model id/alias, e.g. sonnet | opus | claude-opus-4-8 +# CLAUDE_AI_EFFORT=high # low | medium | high | xhigh | max +# CLAUDE_AI_TIMEOUT_MS= # override CLI timeout in ms; unset scales by effort (low/medium 120s, high 240s, xhigh 360s, max 600s) # # Codex (ChatGPT subscription) reviewer is fail-closed by default for self-host PR review: `codex exec` stores its # OAuth credential in auth.json on the same filesystem that prompt-influenced reviews can read. Isolated maintainer # deployments can opt in explicitly after mounting auth at /data/codex (the image exposes it as ~/.codex). # GITTENSORY_ENABLE_UNSAFE_CODEX_REVIEWER=1 # Do NOT set CODEX_HOME for the app container; the provider rejects it so credentials are not advertised in env. -# AI_MODEL=llama3.1 # the model for your provider (e.g. llama3.1 for Ollama, sonnet -# # for claude-code, gpt-5 for codex). REQUIRED for non-Ollama: -# # without it the adapter falls back to a provider default, never -# # the Cloudflare Workers-AI id the core would otherwise pass. +# CODEX_AI_MODEL=gpt-5.5 # omit to let the Codex account default choose; set explicitly for repeatable reviews +# CODEX_AI_EFFORT=high # low | medium | high | xhigh. `max` is accepted and maps to xhigh. +# CODEX_AI_TIMEOUT_MS= # override CLI timeout in ms; unset scales by effort (low/medium 120s, high 240s, xhigh 360s) +# # Codex service speed is standard by default. No fast/priority tier is requested by this stack. # AI_EMBED_MODEL=bge-m3 # embedding model for RAG (openai-compatible /embeddings). MUST be # # 1024-dimensional (e.g. bge-m3 or mxbai-embed-large via Ollama). # # Used only when RAG is enabled (GITTENSORY_REVIEW_RAG + allowlist). diff --git a/docker-compose.yml b/docker-compose.yml index 19af1054c9..b1ff9cfc65 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -64,7 +64,7 @@ services: # QDRANT_URL: http://qdrant:6333 # Uncomment for Ollama AI (--profile ollama): # AI_PROVIDER: ollama - # AI_BASE_URL: http://ollama:11434/v1 + # OLLAMA_AI_BASE_URL: http://ollama:11434/v1 # BROKERED mode — use the central Gittensory Orb App instead of creating your own GitHub App. Install the # Orb App on your repos + set ORB_ENROLLMENT_SECRET in .env (loaded above); the engine then brokers # short-lived GitHub tokens from the Orb on demand (no own App private key). See .env.example. @@ -197,7 +197,7 @@ services: # ── Ollama (--profile ollama) ────────────────────────────────────────────── # After `docker compose --profile ollama up -d`, pull a model: # docker compose exec ollama ollama pull llama3.2 - # Then set AI_PROVIDER=ollama and AI_BASE_URL=http://ollama:11434/v1 in .env. + # Then set AI_PROVIDER=ollama and OLLAMA_AI_BASE_URL=http://ollama:11434/v1 in .env. ollama: image: ollama/ollama:0.30.10 restart: unless-stopped @@ -220,8 +220,8 @@ services: - gittensory-data:/data - ./litestream.yml:/etc/litestream.yml:ro environment: - LITESTREAM_ACCESS_KEY_ID: ${LITESTREAM_ACCESS_KEY_ID} - LITESTREAM_SECRET_ACCESS_KEY: ${LITESTREAM_SECRET_ACCESS_KEY} + LITESTREAM_ACCESS_KEY_ID: ${LITESTREAM_ACCESS_KEY_ID:-} + LITESTREAM_SECRET_ACCESS_KEY: ${LITESTREAM_SECRET_ACCESS_KEY:-} LITESTREAM_ENDPOINT: ${LITESTREAM_ENDPOINT:-} LITESTREAM_REGION: ${LITESTREAM_REGION:-us-east-1} @@ -401,7 +401,7 @@ services: - NET_ADMIN - SYS_MODULE environment: - TS_AUTHKEY: ${TS_AUTHKEY} + TS_AUTHKEY: ${TS_AUTHKEY:-} TS_STATE_DIR: /var/lib/tailscale TS_EXTRA_ARGS: ${TS_EXTRA_ARGS:-} volumes: @@ -421,8 +421,8 @@ services: profiles: ["runners"] environment: RUNNER_SCOPE: ${RUNNER_SCOPE:-repo} - REPO_URL: ${RUNNER_REPO_URL} - RUNNER_TOKEN: ${RUNNER_TOKEN} + REPO_URL: ${RUNNER_REPO_URL:-} + RUNNER_TOKEN: ${RUNNER_TOKEN:-} ACCESS_TOKEN: ${RUNNER_ACCESS_TOKEN:-} RUNNER_NAME: ${RUNNER_NAME:-gittensory-runner} LABELS: ${RUNNER_LABELS:-self-hosted,linux,x64} diff --git a/docs/self-host/ai-providers.md b/docs/self-host/ai-providers.md index 17cb29f364..46f55c1984 100644 --- a/docs/self-host/ai-providers.md +++ b/docs/self-host/ai-providers.md @@ -8,8 +8,8 @@ The reviewer is configured by `AI_PROVIDER`. Reviews degrade deterministically ( | ----------------------------------------- | ----------------------------------------------------------------------- | --------------------------------------------------------------------------------------- | | `claude-code` | Your **Claude** subscription via the `claude` CLI (read-only, headless) | `CLAUDE_CODE_OAUTH_TOKEN` (`claude setup-token`); CLI baked in (`INSTALL_AI_CLIS=true`) | | `codex` | Your **Codex** subscription via the `codex` CLI | local `codex` auth mounted at `/data/codex`, CLI baked in, explicit unsafe opt-in | -| `anthropic` | Native **Anthropic API** (BYOK, per-token billing — no weekly limit) | `ANTHROPIC_API_KEY`, `AI_MODEL` | -| `ollama` / `openai-compatible` / `openai` | Any OpenAI-compatible `/chat/completions` (+ `/embeddings`) | `AI_BASE_URL`, `AI_API_KEY`, `AI_MODEL` | +| `anthropic` | Native **Anthropic API** (BYOK, per-token billing — no weekly limit) | `ANTHROPIC_API_KEY`, `ANTHROPIC_AI_MODEL` | +| `ollama` / `openai-compatible` / `openai` | Any OpenAI-compatible `/chat/completions` (+ `/embeddings`) | provider-specific `*_AI_BASE_URL`, `*_AI_API_KEY`, `*_AI_MODEL` | **Chain / fallback:** `AI_PROVIDER` accepts a comma list, tried in order until one succeeds — e.g. `AI_PROVIDER=anthropic,ollama`. **Dual review:** two providers (`claude-code,codex`) run as independent reviewers @@ -21,11 +21,18 @@ combined per `AI_COMBINE` (`single`/`consensus`/`synthesis`). ## Model & effort (the intelligence dial) -| Var | Default | Notes | -| --------------- | ------------------ | ---------------------------------------------------------------------------------------------------------------------------------------------------- | -| `AI_MODEL` | provider default | e.g. `claude-sonnet-4-6`. **Leave unset on a `claude-code,codex` combo** — it's global and a Claude id breaks codex's account default. | -| `AI_EFFORT` | `high` | `low \| medium \| high \| xhigh \| max` → `claude --effort`. The engine wants substance, not speed. | -| `AI_TIMEOUT_MS` | scales with effort | Subprocess timeout. Unset ⇒ low/med 120s, high 240s, xhigh 360s, **max 600s** (so a big max-effort review isn't killed). Override clamped 30s–30min. | +| Var | Default | Notes | +| ---------------------------- | ------------------------------ | ------------------------------------------------------------------------------------- | +| `CLAUDE_AI_MODEL` | `claude-sonnet-4-6` | Any `claude` CLI model id/alias. | +| `CLAUDE_AI_EFFORT` | `high` | `low \| medium \| high \| xhigh \| max` -> `claude --effort`. | +| `CLAUDE_AI_TIMEOUT_MS` | scales with `CLAUDE_AI_EFFORT` | Unset -> low/med 120s, high 240s, xhigh 360s, max 600s. Override clamped 30s-30min. | +| `CODEX_AI_MODEL` | Codex account default | Set to `gpt-5.5` for repeatable Codex reviews. | +| `CODEX_AI_EFFORT` | `high` | `low \| medium \| high \| xhigh`; `max` maps to `xhigh`. | +| `CODEX_AI_TIMEOUT_MS` | scales with `CODEX_AI_EFFORT` | Unset -> low/med 120s, high 240s, xhigh 360s. Override clamped 30s-30min. | +| `OLLAMA_AI_MODEL` | `llama3.1` | Used only by `AI_PROVIDER=ollama`. | +| `OPENAI_COMPATIBLE_AI_MODEL` | `llama3.1` | Used only by `AI_PROVIDER=openai-compatible`. | +| `OPENAI_AI_MODEL` | `llama3.1` | Used only by `AI_PROVIDER=openai`; set to a real OpenAI model for API-backed reviews. | +| `ANTHROPIC_AI_MODEL` | `claude-sonnet-4-6` | Used only by `AI_PROVIDER=anthropic`. | ## Codex subscription reviewer @@ -35,6 +42,8 @@ Codex is intentionally disabled until the operator opts in with maintainer deployment, mount the Codex home at `/data/codex`; the image exposes that as the default `~/.codex` path for the `node` user. Do not set `CODEX_HOME` in the app environment. The provider rejects `CODEX_HOME` so the credential path is not advertised to the subprocess through env. +Set `CODEX_AI_MODEL=gpt-5.5` and `CODEX_AI_EFFORT=high` for the current recommended Codex reviewer. +The stack uses Codex standard speed by default; it does not request the fast/priority service tier. ## Cost & usage observability diff --git a/docs/self-host/configuration.md b/docs/self-host/configuration.md index c8c8cab882..b291487dc5 100644 --- a/docs/self-host/configuration.md +++ b/docs/self-host/configuration.md @@ -79,8 +79,10 @@ See [ai-providers.md](./ai-providers.md) for the full provider/model/effort/time | Var | Purpose | | -------------------------------------------- | ------------------------------------------------------------------------------------- | | `AI_PROVIDER` | `claude-code` / `codex` / `anthropic` / `ollama` / … (comma-list = chain/dual-review) | -| `AI_MODEL`, `AI_EFFORT` | Model id + intelligence dial (low…max, default high) | -| `AI_TIMEOUT_MS` | CLI subprocess timeout override (else scales with effort) | +| `CLAUDE_AI_*`, `CODEX_AI_*` | Subscription CLI model, effort, and timeout knobs | +| `OLLAMA_AI_*`, `OPENAI_COMPATIBLE_AI_*` | OpenAI-compatible base URL, optional key, and model knobs | +| `OPENAI_AI_MODEL`, `OPENAI_API_KEY` | OpenAI API reviewer model and key | +| `ANTHROPIC_AI_MODEL`, `ANTHROPIC_API_KEY` | Anthropic API reviewer model and key | | `CLAUDE_CODE_OAUTH_TOKEN` | Claude Code subscription token (`claude setup-token`) | | `AI_EMBED_BASE_URL` / `_MODEL` / `_PROVIDER` | Dedicated RAG embed provider | | `GITTENSORY_REPO_CONFIG_DIR` | Container-private per-repo config dir | diff --git a/docs/self-host/troubleshooting.md b/docs/self-host/troubleshooting.md index 2aa2ec3476..a8b5160960 100644 --- a/docs/self-host/troubleshooting.md +++ b/docs/self-host/troubleshooting.md @@ -37,11 +37,12 @@ docker exec gittensory-gittensory-1 sh -c 'which claude && claude --version' **Fix:** set `gate.aiReview.allAuthors: true` (or `settings.aiReviewAllAuthors: true`) in the repo's private `.gittensory.yml`. See [configuration.md](./configuration.md). -### A large `AI_EFFORT=max` review produces nothing +### A large high-effort CLI review produces nothing **Symptom:** big PRs silently get no review; small ones work. **Cause:** the CLI subprocess timed out. (Older builds hard-capped at 120s.) -**Fix:** the timeout now scales with `AI_EFFORT` (max → 600s); override with `AI_TIMEOUT_MS` (clamped 30s–30min). +**Fix:** the timeout now scales with `CLAUDE_AI_EFFORT` / `CODEX_AI_EFFORT`; override with +`CLAUDE_AI_TIMEOUT_MS` or `CODEX_AI_TIMEOUT_MS` (clamped 30s-30min). --- @@ -107,11 +108,11 @@ AI review entirely in advisory mode. If you still see repeats, check for an auto **Symptom:** `db:migrations:check` fails on a duplicate number after a rebase. **Fix:** renumber your migration to the next free `NNNN_*.sql` (the check prints `Next free:`). -### `codex` "model not supported when using Codex with a ChatGPT account" +### Codex model / effort is not what you expected -**Cause:** forcing `--model gpt-5-codex` on a ChatGPT-account login fails. -**Fix:** leave `AI_MODEL` unset for codex — it picks the account's own default. (Don't set a Claude model id on a -`claude-code,codex` combo: `AI_MODEL` is global and a Claude id breaks codex.) +**Cause:** Codex model and reasoning are provider-specific. Shared model settings are intentionally not used. +**Fix:** set `CODEX_AI_MODEL=gpt-5.5` and `CODEX_AI_EFFORT=high` for the current recommended self-host Codex +reviewer. Leave `CODEX_HOME` unset in the app environment. ### `codex_credential_isolation_required` diff --git a/docs/self-hosting.md b/docs/self-hosting.md index e2c06385c3..2225faf8b8 100644 --- a/docs/self-hosting.md +++ b/docs/self-hosting.md @@ -108,15 +108,16 @@ and only the AI **summary** degrades to "unavailable". To enable AI, set `AI_PRO | `AI_PROVIDER` | Backend | Extra config | | ----------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------- | -| `ollama` / `openai-compatible` / `openai` | any OpenAI-compatible `/chat/completions` endpoint (Ollama, OpenAI, Groq, Together, OpenRouter, vLLM, Gemini's OpenAI-compat endpoint, …) | `AI_BASE_URL`, `AI_API_KEY` (or `OPENAI_API_KEY`), `AI_MODEL` | -| `anthropic` | **native Anthropic Messages API** (BYOK — bills your API key) | `ANTHROPIC_API_KEY`, `AI_MODEL` (e.g. `claude-sonnet-4-6`) | -| `claude-code` | your **Claude** subscription via the `claude` CLI (read-only, headless) | `CLAUDE_CODE_OAUTH_TOKEN` (from `claude setup-token`), `AI_MODEL` (default `claude-sonnet-4-6`), `AI_EFFORT` (default `high`) | -| `codex` | your **Codex** subscription via the `codex` CLI | local `codex` auth, `AI_MODEL` (e.g. `gpt-5`) | +| `ollama` | local Ollama `/v1` endpoint | `OLLAMA_AI_BASE_URL`, `OLLAMA_AI_MODEL`, optional `OLLAMA_AI_API_KEY` | +| `openai-compatible` | any OpenAI-compatible `/chat/completions` endpoint (Groq, Together, OpenRouter, vLLM, Gemini's OpenAI-compat endpoint, …) | `OPENAI_COMPATIBLE_AI_BASE_URL`, `OPENAI_COMPATIBLE_AI_MODEL`, optional `OPENAI_COMPATIBLE_AI_API_KEY` | +| `openai` | OpenAI API `/v1` endpoint | `OPENAI_API_KEY`, `OPENAI_AI_MODEL`, optional `OPENAI_AI_BASE_URL` | +| `anthropic` | **native Anthropic Messages API** (BYOK — bills your API key) | `ANTHROPIC_API_KEY`, `ANTHROPIC_AI_MODEL`, optional `ANTHROPIC_AI_BASE_URL` | +| `claude-code` | your **Claude** subscription via the `claude` CLI (read-only, headless) | `CLAUDE_CODE_OAUTH_TOKEN`, `CLAUDE_AI_MODEL`, `CLAUDE_AI_EFFORT`, `CLAUDE_AI_TIMEOUT_MS` | +| `codex` | your **Codex** subscription via the `codex` CLI | local `codex` auth, `CODEX_AI_MODEL`, `CODEX_AI_EFFORT`, `CODEX_AI_TIMEOUT_MS` | -**Review timeout (`AI_TIMEOUT_MS`).** The `claude` / `codex` subprocess timeout. Left unset it **scales with -`AI_EFFORT`** (low/medium 120s, high 240s, xhigh 360s, max 600s) so a large `max`-effort review isn't SIGKILLed -mid-generation — the old fixed 120s cap silently dropped long reviews. Set `AI_TIMEOUT_MS` to override (clamped -30s–30min). +**Review timeout.** `CLAUDE_AI_TIMEOUT_MS` and `CODEX_AI_TIMEOUT_MS` override the subscription-CLI subprocess +timeout. Left unset, the timeout scales with the matching provider effort so large reviews are not SIGKILLed +mid-generation. Overrides are clamped to 30s-30min. **Fallback chain.** `AI_PROVIDER` accepts a comma-separated list and tries each in order until one succeeds — e.g. `AI_PROVIDER=anthropic,ollama` uses the Anthropic API first and falls back to a local Ollama model if it @@ -143,14 +144,13 @@ bake them in, then provide `CLAUDE_CODE_OAUTH_TOKEN` / codex auth at run time. N terminal — it's browser-interactive and prints the token; it has no headless mode). The provider forces the subscription token (it scrubs `ANTHROPIC_API_KEY`), so an API key won't be used here — use `AI_PROVIDER=anthropic` for API-key billing. The model defaults to `claude-sonnet-4-6` and the reasoning **effort** to `high` (a - substantive review, not a fast shallow one); override with `AI_MODEL` (any `claude`-CLI model id or alias — - `sonnet`, `opus`, `claude-opus-4-8`, …) and `AI_EFFORT` (`low`|`medium`|`high`|`xhigh`|`max`; the CLI clamps a - level above the model's own ceiling). -- **Codex (ChatGPT subscription).** The `codex` CLI is pre-baked, but self-hosted Codex reviews are fail-closed by - default because the CLI stores its OAuth refresh credential in `auth.json` on the same filesystem that the - prompt-influenced review sandbox can read. Do **not** copy `~/.codex/auth.json` into the app container or mount a - writable Codex home for PR review. Use `claude-code`, an API-key provider, or a local OpenAI-compatible endpoint for - automated reviews until Codex offers a credential-isolated non-interactive mode. + substantive review, not a fast shallow one); override with `CLAUDE_AI_MODEL` (any `claude`-CLI model id or alias, + e.g. `sonnet`, `opus`, `claude-opus-4-8`) and `CLAUDE_AI_EFFORT` (`low`|`medium`|`high`|`xhigh`|`max`; the CLI + clamps a level above the model's own ceiling). +- **Codex (ChatGPT subscription).** Mount the Codex home at `/data/codex`, leave `CODEX_HOME` unset, and opt in with + `GITTENSORY_ENABLE_UNSAFE_CODEX_REVIEWER=1` only on an isolated maintainer deployment. Set + `CODEX_AI_MODEL=gpt-5.5` and `CODEX_AI_EFFORT=high` for repeatable Codex reviews. The stack uses Codex standard + speed by default; it does not request the fast/priority service tier. **Local RAG (retrieval-augmented review).** Self-host ships a SQLite-backed vector store, so RAG works without Cloudflare Vectorize. Enable it with `GITTENSORY_REVIEW_RAG=true` + the repo in `GITTENSORY_REVIEW_REPOS`, and @@ -159,14 +159,8 @@ point at an **embedding-capable** OpenAI-compatible provider (Ollama) with a **1 SQLite DB (`_selfhost_vectors`) and queried by cosine similarity. Without an embedding model, RAG degrades to no-context (the review still runs). -> **Set `AI_MODEL`.** The core would otherwise hand the adapter a Cloudflare Workers-AI model id -> (`@cf/meta/...`) that Ollama / `claude` / `codex` can't use. The adapter ignores that id in favour of -> `AI_MODEL` (falling back to a provider default), so always set `AI_MODEL` to a real model for your provider. -> The `claude`/`codex` CLIs must be installed and authenticated in the runtime (a CLI-bearing image variant -> is a follow-up); without `AI_MODEL` + a working CLI, the call throws and the review degrades. - The local-AI default is Ollama: uncomment the `ollama` service in `docker-compose.yml`, set -`AI_PROVIDER=ollama` + `AI_BASE_URL=http://ollama:11434/v1`, then `docker compose exec ollama ollama pull +`AI_PROVIDER=ollama` + `OLLAMA_AI_BASE_URL=http://ollama:11434/v1`, then `docker compose exec ollama ollama pull `. **Subscription safety.** The CLI providers run as a read-only subprocess with billable API keys diff --git a/src/env.d.ts b/src/env.d.ts index 2627f8b309..fc255f5b50 100644 --- a/src/env.d.ts +++ b/src/env.d.ts @@ -42,13 +42,28 @@ declare global { AI_GATEWAY_ID?: string; /** Self-host AI provider selection + dual-review config (#dual-ai-combiner). `AI_PROVIDER` is a comma list of * providers (claude-code, codex, anthropic, ollama, …); `AI_COMBINE` picks single|consensus|synthesis (default - * synthesis for two); `AI_ON_MERGE` is the synthesis rule either|both. `AI_EFFORT` is the Claude Code - * intelligence dial (low|medium|high|xhigh|max, default high). `AI_REVIEW_PLAN` is the resolved plan + * synthesis for two); `AI_ON_MERGE` is the synthesis rule either|both. Provider-specific model/effort/timeout + * vars keep Claude/Codex/OpenAI/Ollama/Anthropic config explicit. `AI_REVIEW_PLAN` is the resolved plan * (computed from these at boot in server.ts and read at the review call site); undefined on cloud. */ AI_PROVIDER?: string; AI_COMBINE?: string; AI_ON_MERGE?: string; - AI_EFFORT?: string; + CLAUDE_AI_MODEL?: string; + CLAUDE_AI_EFFORT?: string; + CLAUDE_AI_TIMEOUT_MS?: string; + CODEX_AI_MODEL?: string; + CODEX_AI_EFFORT?: string; + CODEX_AI_TIMEOUT_MS?: string; + OLLAMA_AI_BASE_URL?: string; + OLLAMA_AI_API_KEY?: string; + OLLAMA_AI_MODEL?: string; + OPENAI_COMPATIBLE_AI_BASE_URL?: string; + OPENAI_COMPATIBLE_AI_API_KEY?: string; + OPENAI_COMPATIBLE_AI_MODEL?: string; + OPENAI_AI_BASE_URL?: string; + OPENAI_AI_MODEL?: string; + ANTHROPIC_AI_BASE_URL?: string; + ANTHROPIC_AI_MODEL?: string; AI_REVIEW_PLAN?: { reviewers: Array<{ model: string }>; combine: import("./services/ai-review").CombineStrategy; onMerge?: import("./services/ai-review").OnMerge | undefined }; ADMIN_GITHUB_LOGINS?: string; GITHUB_WEBHOOK_SECRET: string; diff --git a/src/selfhost/ai.ts b/src/selfhost/ai.ts index 5cd4ea1465..45c8db7816 100644 --- a/src/selfhost/ai.ts +++ b/src/selfhost/ai.ts @@ -29,40 +29,73 @@ function toMessages(options: AiRunOptions): Array<{ role: string; content: strin } /** The core passes a Workers-AI model id (e.g. "@cf/meta/llama-3.1-8b-instruct-fp8-fast") that is meaningless - * off-Workers — handing it to Ollama or `claude --model` fails. Prefer the operator-configured model - * (AI_MODEL / WORKERS_AI_SUMMARY_MODEL), then any non-Workers model the core passed, then a provider default. */ + * off-Workers — handing it to Ollama or `claude --model` fails. Prefer the provider-specific self-host model, + * then any non-Workers model the core passed, then a provider default. */ export function resolveModel(configured: string | undefined, passed: string, providerDefault: string): string { if (configured && configured.trim()) return configured.trim(); if (passed && !passed.startsWith("@cf/")) return passed; return providerDefault; } -function configuredModel(env: Record): string | undefined { - return env.AI_MODEL ?? env.WORKERS_AI_SUMMARY_MODEL; +function firstConfigured(...values: Array): string | undefined { + return values.find((value) => value !== undefined && value.trim() !== ""); } -const VALID_EFFORTS = new Set(["low", "medium", "high", "xhigh", "max"]); -/** Map `AI_EFFORT` (the operator's intelligence dial) to a `claude --effort` level. Defaults to "high" — the - * engine wants a substantive review, not a fast shallow one — and falls back to "high" for any unset or - * unrecognized value so a typo can't silently downgrade reviews. The CLI clamps a level above the model's - * own ceiling (e.g. xhigh on Sonnet) down on its own. */ +function configuredClaudeModel(env: Record): string | undefined { + return firstConfigured(env.CLAUDE_AI_MODEL); +} + +function configuredCodexModel(env: Record): string | undefined { + return firstConfigured(env.CODEX_AI_MODEL); +} + +function configuredAnthropicModel(env: Record): string | undefined { + return firstConfigured(env.ANTHROPIC_AI_MODEL); +} + +function configuredOpenAiCompatibleModel(name: string, env: Record): string | undefined { + if (name === "ollama") return firstConfigured(env.OLLAMA_AI_MODEL); + if (name === "openai") return firstConfigured(env.OPENAI_AI_MODEL); + return firstConfigured(env.OPENAI_COMPATIBLE_AI_MODEL); +} + +const VALID_CLAUDE_EFFORTS = new Set(["low", "medium", "high", "xhigh", "max"]); +const VALID_CODEX_EFFORTS = new Set(["low", "medium", "high", "xhigh"]); +/** Map `CLAUDE_AI_EFFORT` to a `claude --effort` level. Defaults to "high" — the engine wants a substantive + * review, not a fast shallow one — and falls back to "high" for any unset or unrecognized value so a typo can't + * silently downgrade reviews. The CLI clamps a level above the model's own ceiling (e.g. xhigh on Sonnet) down. */ export function resolveEffort(configured: string | undefined): string { const level = (configured ?? "").trim().toLowerCase(); - return VALID_EFFORTS.has(level) ? level : "high"; + return VALID_CLAUDE_EFFORTS.has(level) ? level : "high"; +} + +/** Map `CODEX_AI_EFFORT` to Codex reasoning effort. Codex currently supports xhigh as its top level, so a + * mistaken `max` preserves intent by resolving to xhigh instead of being dropped. */ +export function resolveCodexEffort(configured: string | undefined): string { + const level = (configured ?? "").trim().toLowerCase(); + if (VALID_CODEX_EFFORTS.has(level)) return level; + if (level === "max") return "xhigh"; + return "high"; } // Per-effort subprocess timeout (ms) for the subscription CLIs. A higher effort legitimately runs longer, so the // old fixed 120s cap silently SIGKILLed a large max-effort review mid-generation (the review then degrades to -// nothing). These scale the ceiling with the effort dial; AI_TIMEOUT_MS overrides them outright. +// nothing). These scale the ceiling with the provider-specific effort dial; provider-specific timeout vars override +// them outright. const EFFORT_TIMEOUT_MS: Record = { low: 120_000, medium: 120_000, high: 240_000, xhigh: 360_000, max: 600_000 }; -/** Resolve the subscription-CLI subprocess timeout (ms). An explicit `AI_TIMEOUT_MS` wins, clamped to a sane - * 30s–30min range so a typo can neither hang a worker nor cut a review off after a few seconds. Absent/invalid ⇒ - * it scales with the `AI_EFFORT` dial (resolveEffort always yields a known level, so the map lookup is total). */ -export function resolveCliTimeoutMs(env: Record): number { - const raw = Number(env.AI_TIMEOUT_MS); +function resolveCliTimeoutFrom(configured: string | undefined, effort: string): number { + const raw = Number(configured); if (Number.isFinite(raw) && raw > 0) return Math.min(1_800_000, Math.max(30_000, raw)); - return EFFORT_TIMEOUT_MS[resolveEffort(env.AI_EFFORT)]!; + return EFFORT_TIMEOUT_MS[effort]!; +} + +export function resolveClaudeCliTimeoutMs(env: Record): number { + return resolveCliTimeoutFrom(firstConfigured(env.CLAUDE_AI_TIMEOUT_MS), resolveEffort(firstConfigured(env.CLAUDE_AI_EFFORT))); +} + +export function resolveCodexCliTimeoutMs(env: Record): number { + return resolveCliTimeoutFrom(firstConfigured(env.CODEX_AI_TIMEOUT_MS), resolveCodexEffort(firstConfigured(env.CODEX_AI_EFFORT))); } /** OpenAI-compatible endpoint (Ollama's /v1, OpenAI, vLLM, LM Studio, …) — chat + embeddings. */ @@ -385,6 +418,32 @@ export function redactSecrets(text: string, knownSecrets: readonly string[] = [] return out; } +function errorMessage(error: unknown, knownSecrets: readonly string[] = []): string { + const message = error instanceof Error ? error.message : String(error); + return redactSecrets(message, knownSecrets).slice(0, 500); +} + +function logSelfHostAiProviderFailed(input: { + provider: string; + model: string; + effort?: string | undefined; + timeoutMs?: number | undefined; + error: unknown; + knownSecrets?: readonly string[] | undefined; +}): void { + console.error( + JSON.stringify({ + level: "error", + event: "selfhost_ai_provider_failed", + provider: input.provider, + model: input.model || "default", + ...(input.effort ? { effort: input.effort } : {}), + ...(input.timeoutMs ? { timeoutMs: input.timeoutMs } : {}), + error: errorMessage(input.error, input.knownSecrets), + }), + ); +} + /** Claude Code subscription (CLAUDE_CODE_OAUTH_TOKEN via `claude setup-token`). Headless, read-only, JSON. */ export function createClaudeCodeAi(parentEnv: Record, spawnImpl?: SpawnFn): SelfHostAi { return { @@ -394,29 +453,35 @@ export function createClaudeCodeAi(parentEnv: Record // claude's empty-prompt text answer as "success" and never reach the embed provider → RAG silently breaks. if (options.text) throw new Error("claude_code_no_embed"); const token = parentEnv.CLAUDE_CODE_OAUTH_TOKEN; - if (!token) throw new Error("claude_code_no_oauth_token"); - const env = subscriptionCliEnv(parentEnv, { CLAUDE_CODE_OAUTH_TOKEN: token }); - const prompt = toMessages(options).map((m) => m.content).join("\n\n"); - const spawn = spawnImpl ?? (await defaultSpawn()); - const claudeModel = resolveModel(configuredModel(parentEnv), model, "claude-sonnet-4-6"); - const effort = resolveEffort(parentEnv.AI_EFFORT); - const { stdout, code, stderr } = await spawn( - "claude", - ["--print", "--output-format", "json", "--model", claudeModel, "--permission-mode", "plan", "--effort", effort, "--disallowedTools", "Bash,Edit,Write,WebFetch,WebSearch"], - { env, input: prompt, timeoutMs: resolveCliTimeoutMs(parentEnv), cwd: await isolatedCliCwd() }, - ); - // Surface the STRUCTURED error envelope FIRST. `claude --output-format json` reports API/auth/model errors in its - // stdout JSON ({is_error,api_error_status}) on a NON-ZERO exit too — e.g. an unknown model exits 1 with the 404 - // envelope in stdout and EMPTY stderr. Checking it before the exit code turns an opaque `claude_code_exit_1: ` - // (the #1610 symptom) into a precise `claude_code_error_404` — the signal that makes a reviewer outage - // diagnosable in logs + Sentry instead of a dead end. - const errStatus = claudeErrorStatus(stdout); - if (errStatus) throw new Error(`claude_code_error_${errStatus}`); - if (code !== 0) throw new Error(`claude_code_exit_${code ?? "null"}: ${redactSecrets(stderr ?? "", [token]).slice(0, 500)}`); - const text = extractCliText(stdout); - if (!text) throw new Error("claude_code_empty_output"); - recordCliUsageMetrics("claude-code", claudeModel, effort, stdout); - return { response: text }; + const claudeModel = resolveModel(configuredClaudeModel(parentEnv), model, "claude-sonnet-4-6"); + const effort = resolveEffort(firstConfigured(parentEnv.CLAUDE_AI_EFFORT)); + const timeoutMs = resolveClaudeCliTimeoutMs(parentEnv); + try { + if (!token) throw new Error("claude_code_no_oauth_token"); + const env = subscriptionCliEnv(parentEnv, { CLAUDE_CODE_OAUTH_TOKEN: token }); + const prompt = toMessages(options).map((m) => m.content).join("\n\n"); + const spawn = spawnImpl ?? (await defaultSpawn()); + const { stdout, code, stderr } = await spawn( + "claude", + ["--print", "--output-format", "json", "--model", claudeModel, "--permission-mode", "plan", "--effort", effort, "--disallowedTools", "Bash,Edit,Write,WebFetch,WebSearch"], + { env, input: prompt, timeoutMs, cwd: await isolatedCliCwd() }, + ); + // Surface the STRUCTURED error envelope FIRST. `claude --output-format json` reports API/auth/model errors in its + // stdout JSON ({is_error,api_error_status}) on a NON-ZERO exit too — e.g. an unknown model exits 1 with the 404 + // envelope in stdout and EMPTY stderr. Checking it before the exit code turns an opaque `claude_code_exit_1: ` + // (the #1610 symptom) into a precise `claude_code_error_404` — the signal that makes a reviewer outage + // diagnosable in logs + Sentry instead of a dead end. + const errStatus = claudeErrorStatus(stdout); + if (errStatus) throw new Error(`claude_code_error_${errStatus}`); + if (code !== 0) throw new Error(`claude_code_exit_${code ?? "null"}: ${redactSecrets(stderr ?? "", [token]).slice(0, 500)}`); + const text = extractCliText(stdout); + if (!text) throw new Error("claude_code_empty_output"); + recordCliUsageMetrics("claude-code", claudeModel, effort, stdout); + return { response: text }; + } catch (error) { + logSelfHostAiProviderFailed({ provider: "claude-code", model: claudeModel, effort, timeoutMs, error, knownSecrets: token ? [token] : [] }); + throw error; + } }, }; } @@ -428,28 +493,36 @@ export function createCodexAi(parentEnv: Record, spa async run(model, options) { // Codex is chat-only here — reject embed requests so the chain routes them to an embed-capable provider. if (options.text) throw new Error("codex_no_embed"); - assertCodexCredentialIsolation(parentEnv); - const env = codexCliEnv(parentEnv); - const prompt = toMessages(options).map((m) => m.content).join("\n\n"); - const spawn = spawnImpl ?? (await defaultSpawn()); // codex 0.142+: `exec` is non-interactive — the old `--ask-for-approval` flag was REMOVED (passing it errors). // `--skip-git-repo-check` lets it run outside a git repo. Pass `--model` ONLY when one is explicitly - // configured: forcing a model (e.g. the old `gpt-5` default) fails on a ChatGPT-account login with "not - // supported", whose default model codex selects on its own. - const codexModel = resolveModel(configuredModel(parentEnv), model, ""); - const args = ["exec", "--json", "--skip-git-repo-check", "--sandbox", "read-only"]; - if (codexModel) args.push("--model", codexModel); - args.push("--", prompt); - const { stdout, code, stderr } = await spawn("codex", args, { - env, - timeoutMs: resolveCliTimeoutMs(parentEnv), - cwd: await isolatedCliCwd(), - }); - if (code !== 0) throw new Error(`codex_exit_${code ?? "null"}: ${redactSecrets(stderr ?? "").slice(0, 500)}`); - const text = extractCliText(stdout); - if (!text) throw new Error("codex_empty_output"); - recordCliUsageMetrics("codex", codexModel, resolveEffort(parentEnv.AI_EFFORT), stdout); - return { response: text }; + // configured: otherwise Codex selects the account default. + const codexModel = resolveModel(configuredCodexModel(parentEnv), model, ""); + const effort = resolveCodexEffort(firstConfigured(parentEnv.CODEX_AI_EFFORT)); + const timeoutMs = resolveCodexCliTimeoutMs(parentEnv); + try { + assertCodexCredentialIsolation(parentEnv); + const env = codexCliEnv(parentEnv); + const prompt = toMessages(options).map((m) => m.content).join("\n\n"); + const spawn = spawnImpl ?? (await defaultSpawn()); + const args = ["exec", "--json", "--skip-git-repo-check", "--sandbox", "read-only"]; + if (codexModel) args.push("--model", codexModel); + args.push("-c", `model_reasoning_effort="${effort}"`); + const { stdout, code, stderr } = await spawn("codex", args, { + env, + // `codex exec` reads stdin when no prompt argv is provided; keep PR prompts/diffs out of process listings. + input: prompt, + timeoutMs, + cwd: await isolatedCliCwd(), + }); + if (code !== 0) throw new Error(`codex_exit_${code ?? "null"}: ${redactSecrets(stderr ?? "").slice(0, 500)}`); + const text = extractCliText(stdout); + if (!text) throw new Error("codex_empty_output"); + recordCliUsageMetrics("codex", codexModel, effort, stdout); + return { response: text }; + } catch (error) { + logSelfHostAiProviderFailed({ provider: "codex", model: codexModel, effort, timeoutMs, error }); + throw error; + } }, }; } @@ -461,35 +534,53 @@ export function createChainAi(providers: Array<{ name: string; ai: SelfHostAi }> return { async run(model, options) { let lastError: unknown = new Error("no_ai_providers"); + const failures: Array<{ provider: string; error: string }> = []; for (const p of providers) { try { return await p.ai.run(model, options); } catch (error) { lastError = error; - console.error(JSON.stringify({ level: "warn", event: "selfhost_ai_provider_failed", provider: p.name, error: error instanceof Error ? error.message : "unknown" })); + failures.push({ provider: p.name, error: errorMessage(error) }); + console.error(JSON.stringify({ level: "warn", event: "selfhost_ai_provider_failed_in_chain", provider: p.name, error: errorMessage(error) })); } } + console.error( + JSON.stringify({ + level: "error", + event: "selfhost_ai_providers_exhausted", + provider: failures.length === 1 ? failures[0]?.provider : undefined, + model: model || "default", + providers: failures.map((failure) => failure.provider), + failures, + error: errorMessage(lastError), + }), + ); throw lastError instanceof Error ? lastError : new Error("all_ai_providers_failed"); }, }; } -/** Build one provider adapter by name (BYO credentials read from provider-specific env, then the generic - * AI_API_KEY). Returns undefined when its required credential is missing. */ +/** Build one provider adapter by name. Provider config stays explicit so dual-provider setups cannot accidentally + * reuse the wrong model/base/key across different backends. */ export function buildProvider(name: string, env: Record): SelfHostAi | undefined { switch (name) { case "ollama": case "openai-compatible": case "openai": return createOpenAiCompatibleAi({ - baseUrl: env.AI_BASE_URL ?? (name === "openai" ? "https://api.openai.com/v1" : "http://localhost:11434/v1"), - apiKey: env.AI_API_KEY ?? env.OPENAI_API_KEY, - model: configuredModel(env), + baseUrl: + name === "ollama" + ? (env.OLLAMA_AI_BASE_URL ?? "http://localhost:11434/v1") + : name === "openai" + ? (env.OPENAI_AI_BASE_URL ?? "https://api.openai.com/v1") + : (env.OPENAI_COMPATIBLE_AI_BASE_URL ?? "http://localhost:11434/v1"), + apiKey: name === "ollama" ? env.OLLAMA_AI_API_KEY : name === "openai" ? env.OPENAI_API_KEY : env.OPENAI_COMPATIBLE_AI_API_KEY, + model: configuredOpenAiCompatibleModel(name, env), embedModel: env.AI_EMBED_MODEL, }); case "anthropic": { - const apiKey = env.ANTHROPIC_API_KEY ?? env.AI_API_KEY; - return apiKey ? createAnthropicAi({ apiKey, model: configuredModel(env), baseUrl: env.AI_BASE_URL }) : undefined; + const apiKey = env.ANTHROPIC_API_KEY; + return apiKey ? createAnthropicAi({ apiKey, model: configuredAnthropicModel(env), baseUrl: env.ANTHROPIC_AI_BASE_URL }) : undefined; } case "claude-code": return createClaudeCodeAi(env); diff --git a/src/selfhost/sentry.ts b/src/selfhost/sentry.ts index 12d5af3221..2912c188c7 100644 --- a/src/selfhost/sentry.ts +++ b/src/selfhost/sentry.ts @@ -89,7 +89,7 @@ export function captureReviewFailure( // The structured-log fields worth indexing as Sentry tags — the dimensions operators filter + group by. Only // string|number values are tagged; everything else stays in the full "log" context. -const SENTRY_LOG_TAG_KEYS = ["repo", "repository", "installationId", "installation_id", "pull", "pullNumber", "pr", "project", "kind", "deliveryId"] as const; +const SENTRY_LOG_TAG_KEYS = ["repo", "repository", "installationId", "installation_id", "pull", "pullNumber", "pr", "project", "kind", "deliveryId", "provider", "model", "effort", "timeoutMs"] as const; /** A SHORT location suffix — " (repo#pr)" — for a no-message error title, so the issue list shows WHERE without * dumping every scalar field (which made titles unreadably long, e.g. trailing a full deliveryId). The complete diff --git a/src/services/ai-review.ts b/src/services/ai-review.ts index 610cf6d235..16e6ef7c44 100644 --- a/src/services/ai-review.ts +++ b/src/services/ai-review.ts @@ -1117,13 +1117,30 @@ export async function runGittensoryAiReview( }; } -/** The actual configured reviewer label for usage attribution (#1566): the self-host `AI_PROVIDER:AI_MODEL` when set, - * else the Worker dual-AI models. Without this, self-host claude-code reviews were mis-logged as the Workers-AI model - * ids (`@cf/openai/gpt-oss-120b+…`), which hid the silent claude-CLI-missing outage. */ +const SELF_HOST_REVIEWER_MODEL_ENV: Record = { + anthropic: "ANTHROPIC_AI_MODEL", + "claude-code": "CLAUDE_AI_MODEL", + codex: "CODEX_AI_MODEL", + ollama: "OLLAMA_AI_MODEL", + openai: "OPENAI_AI_MODEL", + "openai-compatible": "OPENAI_COMPATIBLE_AI_MODEL", +}; + +/** The actual configured reviewer label for usage attribution (#1566): the self-host provider plus its explicit + * provider-specific model when set, else the Worker dual-AI models. Without this, self-host claude-code reviews + * were mis-logged as the Workers-AI model ids (`@cf/openai/gpt-oss-120b+...`), which hid outages. */ function reviewerModelLabel(env: Env): string { - const e = env as unknown as { AI_PROVIDER?: string; AI_MODEL?: string }; + const e = env as unknown as Record; if (!e.AI_PROVIDER) return BEST_REVIEW_MODELS.join("+"); - return [e.AI_PROVIDER, e.AI_MODEL].filter(Boolean).join(":"); + return e.AI_PROVIDER.split(",") + .map((provider) => provider.trim().toLowerCase()) + .filter(Boolean) + .map((provider) => { + const modelEnv = SELF_HOST_REVIEWER_MODEL_ENV[provider]; + const model = modelEnv ? e[modelEnv]?.trim() : undefined; + return model ? `${provider}:${model}` : provider; + }) + .join("+"); } async function record( diff --git a/test/unit/ai-review-advisory.test.ts b/test/unit/ai-review-advisory.test.ts index de596bf3a7..e669163881 100644 --- a/test/unit/ai-review-advisory.test.ts +++ b/test/unit/ai-review-advisory.test.ts @@ -148,8 +148,8 @@ describe("runAiReviewForAdvisory", () => { // usage event must attribute the ACTUAL configured reviewer — not the hardcoded Workers-AI ids that hid the // silent outage. Exercises runWorkersOpinion's now-logging catch + reviewerModelLabel's provider arm. const env = aiEnv(async () => { throw new Error("claude CLI not found"); }); - (env as unknown as { AI_PROVIDER: string; AI_MODEL: string }).AI_PROVIDER = "claude-code"; - (env as unknown as { AI_PROVIDER: string; AI_MODEL: string }).AI_MODEL = "claude-sonnet-4-6"; + (env as unknown as { AI_PROVIDER: string; CLAUDE_AI_MODEL: string }).AI_PROVIDER = "claude-code"; + (env as unknown as { AI_PROVIDER: string; CLAUDE_AI_MODEL: string }).CLAUDE_AI_MODEL = "claude-sonnet-4-6"; const result = await runAiReviewForAdvisory(env, { settings: { aiReviewMode: "advisory" } as RepositorySettings, advisory: advisory(), repoFullName: "acme/widgets", pr, author: "alice", confirmedContributor: true }); expect(result).toBeUndefined(); // provider threw → no usable output, degraded not crashed const usage = await env.DB.prepare("SELECT model FROM ai_usage_events WHERE feature = 'ai_review_pr' ORDER BY created_at DESC LIMIT 1").first<{ model: string }>(); diff --git a/test/unit/selfhost-ai.test.ts b/test/unit/selfhost-ai.test.ts index b43bef3d1d..808502249b 100644 --- a/test/unit/selfhost-ai.test.ts +++ b/test/unit/selfhost-ai.test.ts @@ -2,7 +2,7 @@ import { chmodSync, mkdtempSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { afterEach, describe, expect, it, vi } from "vitest"; -import { buildProvider, claudeErrorStatus, createAnthropicAi, createChainAi, createClaudeCodeAi, createCodexAi, createOpenAiCompatibleAi, createSelfHostAi, extractCliText, extractCliUsage, resolveAiReviewerPlan, resolveCliTimeoutMs, resolveEffort, resolveModel, resolveProviderNames, resolveRequiredCliProviders, redactSecrets, routeProviders, subscriptionCliEnv } from "../../src/selfhost/ai"; +import { buildProvider, claudeErrorStatus, createAnthropicAi, createChainAi, createClaudeCodeAi, createCodexAi, createOpenAiCompatibleAi, createSelfHostAi, extractCliText, extractCliUsage, resolveAiReviewerPlan, resolveClaudeCliTimeoutMs, resolveCodexCliTimeoutMs, resolveCodexEffort, resolveEffort, resolveModel, resolveProviderNames, resolveRequiredCliProviders, redactSecrets, routeProviders, subscriptionCliEnv } from "../../src/selfhost/ai"; import { renderMetrics, resetMetrics } from "../../src/selfhost/metrics"; describe("resolveModel (#979 — never leak the Workers-AI default to a self-host backend)", () => { @@ -31,23 +31,34 @@ describe("resolveEffort (#selfhost-effort — Claude Code intelligence dial, def }); }); -describe("resolveCliTimeoutMs (#selfhost — subprocess timeout scales with effort, AI_TIMEOUT_MS overrides)", () => { - it("scales the default timeout with the AI_EFFORT dial (max needs far more than the old fixed 120s)", () => { - expect(resolveCliTimeoutMs({ AI_EFFORT: "low" })).toBe(120_000); - expect(resolveCliTimeoutMs({ AI_EFFORT: "medium" })).toBe(120_000); - expect(resolveCliTimeoutMs({ AI_EFFORT: "high" })).toBe(240_000); - expect(resolveCliTimeoutMs({ AI_EFFORT: "xhigh" })).toBe(360_000); - expect(resolveCliTimeoutMs({ AI_EFFORT: "max" })).toBe(600_000); - expect(resolveCliTimeoutMs({})).toBe(240_000); // unset effort → resolveEffort defaults to high +describe("resolveCodexEffort (#selfhost-effort — Codex reasoning effort, explicit provider var)", () => { + it("uses Codex-supported levels and maps max to xhigh", () => { + expect(resolveCodexEffort("low")).toBe("low"); + expect(resolveCodexEffort(" Medium ")).toBe("medium"); + expect(resolveCodexEffort("xhigh")).toBe("xhigh"); + expect(resolveCodexEffort("max")).toBe("xhigh"); + expect(resolveCodexEffort("ultra")).toBe("high"); }); - it("honors an explicit AI_TIMEOUT_MS, clamped to a sane 30s–30min range", () => { - expect(resolveCliTimeoutMs({ AI_TIMEOUT_MS: "300000", AI_EFFORT: "low" })).toBe(300_000); // in-range value wins over the effort scale - expect(resolveCliTimeoutMs({ AI_TIMEOUT_MS: "9999999" })).toBe(1_800_000); // clamped down to the 30min ceiling - expect(resolveCliTimeoutMs({ AI_TIMEOUT_MS: "1000" })).toBe(30_000); // clamped up to the 30s floor - }); - it("falls back to the effort scale on a non-positive or non-numeric AI_TIMEOUT_MS", () => { - expect(resolveCliTimeoutMs({ AI_TIMEOUT_MS: "0", AI_EFFORT: "max" })).toBe(600_000); // 0 is not > 0 → effort path - expect(resolveCliTimeoutMs({ AI_TIMEOUT_MS: "abc", AI_EFFORT: "high" })).toBe(240_000); // NaN → effort path +}); + +describe("provider-specific CLI timeouts (#selfhost — no shared timeout ambiguity)", () => { + it("scales Claude timeout from CLAUDE_AI_EFFORT and honors CLAUDE_AI_TIMEOUT_MS", () => { + expect(resolveClaudeCliTimeoutMs({ CLAUDE_AI_EFFORT: "low" })).toBe(120_000); + expect(resolveClaudeCliTimeoutMs({ CLAUDE_AI_EFFORT: "medium" })).toBe(120_000); + expect(resolveClaudeCliTimeoutMs({ CLAUDE_AI_EFFORT: "high" })).toBe(240_000); + expect(resolveClaudeCliTimeoutMs({ CLAUDE_AI_EFFORT: "xhigh" })).toBe(360_000); + expect(resolveClaudeCliTimeoutMs({ CLAUDE_AI_EFFORT: "max" })).toBe(600_000); + expect(resolveClaudeCliTimeoutMs({})).toBe(240_000); + expect(resolveClaudeCliTimeoutMs({ CLAUDE_AI_TIMEOUT_MS: "300000", CLAUDE_AI_EFFORT: "low" })).toBe(300_000); + }); + it("scales Codex timeout from CODEX_AI_EFFORT and honors CODEX_AI_TIMEOUT_MS", () => { + expect(resolveCodexCliTimeoutMs({ CODEX_AI_EFFORT: "low" })).toBe(120_000); + expect(resolveCodexCliTimeoutMs({ CODEX_AI_EFFORT: "medium" })).toBe(120_000); + expect(resolveCodexCliTimeoutMs({ CODEX_AI_EFFORT: "high" })).toBe(240_000); + expect(resolveCodexCliTimeoutMs({ CODEX_AI_EFFORT: "xhigh" })).toBe(360_000); + expect(resolveCodexCliTimeoutMs({ CODEX_AI_EFFORT: "max" })).toBe(360_000); + expect(resolveCodexCliTimeoutMs({ CODEX_AI_TIMEOUT_MS: "1000" })).toBe(30_000); + expect(resolveCodexCliTimeoutMs({ CODEX_AI_TIMEOUT_MS: "9999999" })).toBe(1_800_000); }); }); @@ -123,7 +134,7 @@ describe("createSelfHostAi — provider selection", () => { expect(createSelfHostAi({})).toBeUndefined(); }); it("maps ollama/openai-compatible/claude-code/codex to adapters", () => { - expect(typeof createSelfHostAi({ AI_PROVIDER: "ollama", AI_BASE_URL: "http://o/v1" })?.run).toBe("function"); + expect(typeof createSelfHostAi({ AI_PROVIDER: "ollama", OLLAMA_AI_BASE_URL: "http://o/v1" })?.run).toBe("function"); expect(typeof createSelfHostAi({ AI_PROVIDER: "claude-code" })?.run).toBe("function"); expect(typeof createSelfHostAi({ AI_PROVIDER: "codex" })?.run).toBe("function"); expect(createSelfHostAi({ AI_PROVIDER: "nonsense" })).toBeUndefined(); @@ -207,7 +218,7 @@ describe("routeProviders (#dual-ai-combiner — address one provider by name for }); it("createSelfHostAi wires routing for a 2+ provider AI_PROVIDER (addressable by name)", async () => { - const ai = createSelfHostAi({ AI_PROVIDER: "anthropic,ollama", ANTHROPIC_API_KEY: "sk-ant", AI_BASE_URL: "http://o/v1" }); + const ai = createSelfHostAi({ AI_PROVIDER: "anthropic,ollama", ANTHROPIC_API_KEY: "sk-ant", OLLAMA_AI_BASE_URL: "http://o/v1" }); expect(typeof ai?.run).toBe("function"); }); @@ -220,7 +231,7 @@ describe("routeProviders (#dual-ai-combiner — address one provider by name for sentModel = (JSON.parse(init.body) as { model: string }).model; return new Response(JSON.stringify({ choices: [{ message: { content: "ok" } }] }), { status: 200 }); })); - const ai = createSelfHostAi({ AI_PROVIDER: "openai-compatible", AI_BASE_URL: "http://o/v1" }); + const ai = createSelfHostAi({ AI_PROVIDER: "openai-compatible", OPENAI_COMPATIBLE_AI_BASE_URL: "http://o/v1" }); await ai?.run("openai-compatible", { prompt: "x" }); // the single-provider reviewer-plan address IS the provider name expect(sentModel).toBe("llama3.1"); // resolveModel(undefined, "", "llama3.1") — NOT the literal "openai-compatible" }); @@ -334,7 +345,7 @@ describe("branch coverage — defaults + edge inputs", () => { { role: "user", content: "follow-up" }, ]); }); - it("buildProvider uses provider-specific default base URLs when AI_BASE_URL is unset", () => { + it("buildProvider uses provider-specific default base URLs when provider base URLs are unset", () => { expect(typeof buildProvider("openai", {})?.run).toBe("function"); // defaults to https://api.openai.com/v1 expect(typeof buildProvider("ollama", {})?.run).toBe("function"); // defaults to http://localhost:11434/v1 }); @@ -403,7 +414,7 @@ describe("subscription CLI helpers + fail-safe", () => { expect(capturedEnv.CLAUDE_CODE_OAUTH_TOKEN).toBe("t"); }); - it("Claude Code pins the default model (claude-sonnet-4-6) + --effort high; AI_MODEL/AI_EFFORT override; timeout scales with effort", async () => { + it("Claude Code pins the default model (claude-sonnet-4-6) + --effort high; CLAUDE_AI_* overrides explicitly", async () => { let seen: string[] = []; let timeout = 0; const cap: StubSpawn = async (_c, a, o) => { @@ -411,13 +422,13 @@ describe("subscription CLI helpers + fail-safe", () => { timeout = o.timeoutMs; return { stdout: JSON.stringify({ type: "result", result: "ok" }), code: 0 }; }; - // empty model id (the dual-router default) + no AI_MODEL → pinned claude-sonnet-4-6; no AI_EFFORT → high + // Empty model id (the dual-router default) + no CLAUDE_AI_MODEL → pinned claude-sonnet-4-6; unset effort → high. await createClaudeCodeAi({ CLAUDE_CODE_OAUTH_TOKEN: "t" }, cap).run("", { prompt: "x" }); expect(seen[seen.indexOf("--model") + 1]).toBe("claude-sonnet-4-6"); expect(seen[seen.indexOf("--effort") + 1]).toBe("high"); expect(timeout).toBe(240_000); // high → 240s (not the old fixed 120s) - // operator overrides flow through to the argv + the timeout scale - await createClaudeCodeAi({ CLAUDE_CODE_OAUTH_TOKEN: "t", AI_MODEL: "claude-opus-4-8", AI_EFFORT: "max" }, cap).run("", { prompt: "x" }); + // Provider-specific overrides flow through to the argv + timeout scale. + await createClaudeCodeAi({ CLAUDE_CODE_OAUTH_TOKEN: "t", CLAUDE_AI_MODEL: "claude-opus-4-8", CLAUDE_AI_EFFORT: "max" }, cap).run("", { prompt: "x" }); expect(seen[seen.indexOf("--model") + 1]).toBe("claude-opus-4-8"); expect(seen[seen.indexOf("--effort") + 1]).toBe("max"); expect(timeout).toBe(600_000); // max → 600s, so a large max-effort review isn't SIGKILLed at 120s @@ -438,34 +449,38 @@ describe("subscription CLI helpers + fail-safe", () => { expect((await codexChain.run("bge-m3", { text: ["a"] })).data?.length).toBe(1); }); - it("Codex: 0.142+ exec flags (no --ask-for-approval, has --skip-git-repo-check); --model only when configured", async () => { + it("Codex: 0.142+ exec flags, stdin prompt, explicit CODEX_AI_* config", async () => { let seen: string[] = []; let capturedEnv: Record = {}; let capturedCwd = ""; + let capturedInput: string | undefined; let timeout = 0; const ok: StubSpawn = async (_cmd, args, opts) => { seen = args; capturedEnv = opts.env; capturedCwd = opts.cwd ?? ""; + capturedInput = opts.input; timeout = opts.timeoutMs; return { stdout: JSON.stringify({ type: "result", result: "codex review" }), code: 0 }; }; - // no configured model + the dual-router's empty model id → OMIT --model (codex picks the account default; - // forcing e.g. gpt-5 fails on a ChatGPT-account login). And the removed --ask-for-approval must never appear. + // No configured model + the dual-router's empty model id → omit --model (Codex picks the account default). expect( - (await createCodexAi({ PATH: "/bin", WORKER_ONLY_VALUE: "internal", OPENAI_API_KEY: "sk-bill", AI_TIMEOUT_MS: "300000", GITTENSORY_ENABLE_UNSAFE_CODEX_REVIEWER: "1" }, ok).run("", { + (await createCodexAi({ PATH: "/bin", WORKER_ONLY_VALUE: "internal", OPENAI_API_KEY: "sk-bill", CODEX_AI_TIMEOUT_MS: "300000", GITTENSORY_ENABLE_UNSAFE_CODEX_REVIEWER: "1" }, ok).run("", { prompt: "x", })).response, ).toBe("codex review"); - expect(seen).toEqual(["exec", "--json", "--skip-git-repo-check", "--sandbox", "read-only", "--", "x"]); + expect(seen).toEqual(["exec", "--json", "--skip-git-repo-check", "--sandbox", "read-only", "-c", 'model_reasoning_effort="high"']); expect(seen).not.toContain("--ask-for-approval"); + expect(seen).not.toContain("x"); + expect(capturedInput).toBe("x"); expect(capturedEnv).toEqual({ PATH: "/bin" }); expect(capturedCwd).toContain("gittensory-ai-"); - expect(timeout).toBe(300_000); // codex honors the same AI_TIMEOUT_MS override as Claude Code - // an explicit model (AI_MODEL, or a `codex:` reviewer id) IS passed through but not inherited as env. - await createCodexAi({ AI_MODEL: "o4-mini", GITTENSORY_ENABLE_UNSAFE_CODEX_REVIEWER: "1" }, ok).run("", { prompt: "x" }); - expect(seen.join(" ")).toContain("--model o4-mini"); - expect(capturedEnv.AI_MODEL).toBeUndefined(); + expect(timeout).toBe(300_000); + // Provider-specific model/effort are passed through. + await createCodexAi({ CODEX_AI_MODEL: "gpt-5.5", CODEX_AI_EFFORT: "high", GITTENSORY_ENABLE_UNSAFE_CODEX_REVIEWER: "1" }, ok).run("", { prompt: "x" }); + expect(seen.join(" ")).toContain("--model gpt-5.5"); + expect(seen.join(" ")).toContain('model_reasoning_effort="high"'); + expect(capturedEnv.CODEX_AI_MODEL).toBeUndefined(); const bad: StubSpawn = async () => ({ stdout: "", code: 1 }); await expect(createCodexAi({ GITTENSORY_ENABLE_UNSAFE_CODEX_REVIEWER: "1" }, bad).run("", { prompt: "x" })).rejects.toThrow(/codex_exit_1/); }); @@ -586,7 +601,7 @@ describe("subscription CLI helpers + fail-safe", () => { JSON.stringify({ type: "result", result: "review" }), ].join("\n"); const ok: StubSpawn = async () => ({ stdout, code: 0 }); - await createCodexAi({ GITTENSORY_ENABLE_UNSAFE_CODEX_REVIEWER: "1", AI_EFFORT: "medium" }, ok).run("", { prompt: "x" }); + await createCodexAi({ GITTENSORY_ENABLE_UNSAFE_CODEX_REVIEWER: "1", CODEX_AI_EFFORT: "medium" }, ok).run("", { prompt: "x" }); const metrics = await renderMetrics(); expect(metrics).toContain('gittensory_ai_requests_total{effort="medium",model="gpt-5-codex",provider="codex"} 1'); expect(metrics).toContain('gittensory_ai_input_tokens_total{effort="medium",kind="review",model="gpt-5-codex",provider="codex"} 20'); diff --git a/test/unit/selfhost-sentry.test.ts b/test/unit/selfhost-sentry.test.ts index 896d61a28a..09609ea258 100644 --- a/test/unit/selfhost-sentry.test.ts +++ b/test/unit/selfhost-sentry.test.ts @@ -235,6 +235,27 @@ describe("forwardStructuredLogToSentry — central console.log → Sentry error expect(mocks.scope.setFingerprint).toHaveBeenCalledWith(["gittensory-log", "orb_broker_unavailable"]); }); + it("indexes self-host AI provider dimensions as Sentry tags", async () => { + await initSentry({ SENTRY_DSN: "d" } as unknown as NodeJS.ProcessEnv); + forwardStructuredLogToSentry( + JSON.stringify({ + level: "error", + event: "selfhost_ai_provider_failed", + provider: "codex", + model: "gpt-5.5", + effort: "high", + timeoutMs: 240000, + error: "subscription_cli_timeout", + }), + ); + expect(lastCapturedError().name).toBe("selfhost_ai_provider_failed"); + expect(lastCapturedError().message).toBe("subscription_cli_timeout"); + expect(mocks.scope.setTag).toHaveBeenCalledWith("provider", "codex"); + expect(mocks.scope.setTag).toHaveBeenCalledWith("model", "gpt-5.5"); + expect(mocks.scope.setTag).toHaveBeenCalledWith("effort", "high"); + expect(mocks.scope.setTag).toHaveBeenCalledWith("timeoutMs", "240000"); + }); + it("forwards a level:fatal log titled by message (no event ⇒ no tag)", async () => { await initSentry({ SENTRY_DSN: "d" } as unknown as NodeJS.ProcessEnv); forwardStructuredLogToSentry( From df3ae59a5b47d66d13510d124150a05e9ffd3f31 Mon Sep 17 00:00:00 2001 From: JSONbored <49853598+JSONbored@users.noreply.github.com> Date: Sun, 28 Jun 2026 06:19:56 -0700 Subject: [PATCH 03/68] fix(observability): restore maintainer dashboard securely --- .env.example | 3 +- docker-compose.yml | 37 ++++- docs/self-hosting.md | 4 + grafana/dashboards/maintainer-reviews.json | 149 +++++++++++++++++--- grafana/dashboards/resource-hub.json | 2 +- grafana/provisioning/datasources/sqlite.yml | 6 +- scripts/export-grafana-reporting-db.sh | 111 +++++++++++++++ 7 files changed, 280 insertions(+), 32 deletions(-) create mode 100644 scripts/export-grafana-reporting-db.sh diff --git a/.env.example b/.env.example index d50f4acfe5..0c9da2274f 100644 --- a/.env.example +++ b/.env.example @@ -175,9 +175,10 @@ GITTENSORY_REVIEW_DRAFT=false # GRAFANA_ADMIN_PASSWORD=changeme # REQUIRED when using --profile observability; compose fails if unset # # Maintainer dashboards (in addition to the infra dashboard): -# • "Reviews & PRs (maintainer)" — per-repo + combined PR/review analytics (SQLite data source over the app DB). +# • "Reviews & PRs (maintainer)" — per-repo + combined PR/review analytics from a redacted reporting DB export. # • "Claude usage (OTEL)" — cost/tokens/model/effort from the review CLI's OpenTelemetry export (see below). # • "Resource hub" — links to every integrated service. +# GRAFANA_REPORTING_EXPORT_INTERVAL_SECONDS=30 # refresh cadence for the redacted reporting SQLite export # # Claude usage telemetry → OTEL collector → Prometheus → the Claude usage dashboard. OFF by default. # CLAUDE_CODE_ENABLE_TELEMETRY=1 # enable; needs --profile observability (starts the otel-collector) diff --git a/docker-compose.yml b/docker-compose.yml index b1ff9cfc65..26e7d27568 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -282,16 +282,21 @@ services: image: grafana/grafana:13.1.0 restart: unless-stopped profiles: ["observability"] - depends_on: [prometheus, loki] + depends_on: + prometheus: + condition: service_started + loki: + condition: service_started + reporting-exporter: + condition: service_healthy ports: - "3000:3000" volumes: - grafana-data:/var/lib/grafana - ./grafana/provisioning:/etc/grafana/provisioning:ro - ./grafana/dashboards:/var/lib/grafana/dashboards:ro - # Maintainer dashboards query aggregate review/AI-usage tables from the app DB. - # SQLite WAL readers need the shm file, so this cannot be read-only. - - gittensory-data:/appdb + # Maintainer dashboards query only the redacted reporting export, never the live app DB. + - grafana-reporting-data:/reporting:ro environment: GF_SECURITY_ADMIN_PASSWORD: ${GRAFANA_ADMIN_PASSWORD:?Set GRAFANA_ADMIN_PASSWORD in .env before using --profile observability} GF_USERS_ALLOW_SIGN_UP: "false" @@ -299,6 +304,29 @@ services: # Read-only fine-grained PAT for the GitHub data source provisioning ($GITHUB_TOKEN expansion). From .env. GITHUB_TOKEN: "${GITHUB_TOKEN:-}" + reporting-exporter: + image: alpine:3.20 + restart: unless-stopped + profiles: ["observability"] + depends_on: + gittensory: + condition: service_healthy + volumes: + # The exporter needs live DB read access to build the redacted snapshot. Grafana does not get this mount. + - gittensory-data:/appdb + - grafana-reporting-data:/reporting + - ./scripts/export-grafana-reporting-db.sh:/export-grafana-reporting-db.sh:ro + command: + - /bin/sh + - -c + - "apk add --no-cache sqlite >/dev/null 2>&1 && while true; do sh /export-grafana-reporting-db.sh || echo '[reporting] export failed'; sleep ${GRAFANA_REPORTING_EXPORT_INTERVAL_SECONDS:-30}; done" + healthcheck: + test: ["CMD-SHELL", "test -s /reporting/gittensory-reporting.sqlite && sqlite3 /reporting/gittensory-reporting.sqlite 'PRAGMA quick_check;' | grep -q '^ok$'"] + interval: 30s + timeout: 5s + start_period: 30s + retries: 3 + # ── Log pipeline (--profile observability) ───────────────────────────────── # Loki stores logs; Promtail discovers every container in this compose project via the read-only # docker-proxy (NOT a raw socket) and ships their logs to Loki. Browse in Grafana → Explore → @@ -464,6 +492,7 @@ volumes: prometheus-data: alertmanager-data: grafana-data: + grafana-reporting-data: loki-data: promtail-data: tailscale-state: diff --git a/docs/self-hosting.md b/docs/self-hosting.md index 2225faf8b8..b47611970c 100644 --- a/docs/self-hosting.md +++ b/docs/self-hosting.md @@ -244,6 +244,10 @@ content-lane are not yet per-repo toggleable and stay on the allowlist.) For **continuous, point-in-time backup**, enable the optional [Litestream](https://litestream.io) sidecar in `docker-compose.yml` (copy `litestream.yml.example` → `litestream.yml`, set your bucket + credentials); it streams every change to S3/B2/MinIO/R2. +- **Maintainer Grafana dashboards.** Grafana does **not** mount the live app database. The observability profile + starts `reporting-exporter`, which copies only the dashboard-safe `review_targets` and `ai_usage_events` + columns into `/reporting/gittensory-reporting.sqlite` every `GRAFANA_REPORTING_EXPORT_INTERVAL_SECONDS` seconds + (default 30). The SQLite datasource points at that redacted reporting DB. - **App-level metrics.** Enable `GITTENSORY_REVIEW_OPS=true` for the read-only gate-block anomaly scan and the bearer-gated `GET /v1/internal/ops/stats` aggregate. diff --git a/grafana/dashboards/maintainer-reviews.json b/grafana/dashboards/maintainer-reviews.json index d8a12c1638..6b65bd4085 100644 --- a/grafana/dashboards/maintainer-reviews.json +++ b/grafana/dashboards/maintainer-reviews.json @@ -1,35 +1,138 @@ { "uid": "gittensory-maintainer", "title": "Gittensory — Reviews & PRs (maintainer)", - "tags": [ - "gittensory", - "maintainer" - ], + "tags": ["gittensory", "maintainer"], "timezone": "browser", "schemaVersion": 39, - "version": 2, + "version": 3, + "editable": false, + "graphTooltip": 1, "refresh": "1m", - "time": { - "from": "now-90d", - "to": "now" - }, - "templating": { - "list": [] - }, + "time": { "from": "now-90d", "to": "now" }, + "templating": { "list": [] }, + "annotations": { "list": [] }, + "links": [], "panels": [ { - "type": "text", - "title": "Maintainer review dashboard disabled", - "gridPos": { - "h": 8, - "w": 24, - "x": 0, - "y": 0 + "type": "row", + "id": 1, + "title": "Reviews & PRs (maintainer)", + "collapsed": false, + "gridPos": { "h": 1, "w": 24, "x": 0, "y": 0 } + }, + { + "type": "stat", + "id": 2, + "title": "PRs tracked", + "datasource": { "type": "frser-sqlite-datasource", "uid": "gittensory-db" }, + "gridPos": { "h": 4, "w": 4, "x": 0, "y": 1 }, + "fieldConfig": { "defaults": { "color": { "mode": "fixed", "fixedColor": "blue" }, "unit": "short" }, "overrides": [] }, + "options": { "colorMode": "background", "graphMode": "none", "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, "textMode": "value" }, + "targets": [{ "datasource": { "type": "frser-sqlite-datasource", "uid": "gittensory-db" }, "refId": "A", "queryType": "table", "queryText": "SELECT count(*) AS prs FROM review_targets", "rawQueryText": "SELECT count(*) AS prs FROM review_targets" }] + }, + { + "type": "stat", + "id": 3, + "title": "Merged", + "datasource": { "type": "frser-sqlite-datasource", "uid": "gittensory-db" }, + "gridPos": { "h": 4, "w": 4, "x": 4, "y": 1 }, + "fieldConfig": { "defaults": { "color": { "mode": "fixed", "fixedColor": "green" }, "unit": "short" }, "overrides": [] }, + "options": { "colorMode": "background", "graphMode": "none", "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, "textMode": "value" }, + "targets": [{ "datasource": { "type": "frser-sqlite-datasource", "uid": "gittensory-db" }, "refId": "A", "queryType": "table", "queryText": "SELECT count(*) AS merged FROM review_targets WHERE status='merged'", "rawQueryText": "SELECT count(*) AS merged FROM review_targets WHERE status='merged'" }] + }, + { + "type": "stat", + "id": 4, + "title": "Closed", + "datasource": { "type": "frser-sqlite-datasource", "uid": "gittensory-db" }, + "gridPos": { "h": 4, "w": 4, "x": 8, "y": 1 }, + "fieldConfig": { "defaults": { "color": { "mode": "fixed", "fixedColor": "red" }, "unit": "short" }, "overrides": [] }, + "options": { "colorMode": "background", "graphMode": "none", "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, "textMode": "value" }, + "targets": [{ "datasource": { "type": "frser-sqlite-datasource", "uid": "gittensory-db" }, "refId": "A", "queryType": "table", "queryText": "SELECT count(*) AS closed FROM review_targets WHERE status='closed'", "rawQueryText": "SELECT count(*) AS closed FROM review_targets WHERE status='closed'" }] + }, + { + "type": "stat", + "id": 5, + "title": "Manual review", + "datasource": { "type": "frser-sqlite-datasource", "uid": "gittensory-db" }, + "gridPos": { "h": 4, "w": 4, "x": 12, "y": 1 }, + "fieldConfig": { "defaults": { "color": { "mode": "fixed", "fixedColor": "orange" }, "unit": "short" }, "overrides": [] }, + "options": { "colorMode": "background", "graphMode": "none", "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, "textMode": "value" }, + "targets": [{ "datasource": { "type": "frser-sqlite-datasource", "uid": "gittensory-db" }, "refId": "A", "queryType": "table", "queryText": "SELECT count(*) AS manual FROM review_targets WHERE status='manual' OR verdict='manual'", "rawQueryText": "SELECT count(*) AS manual FROM review_targets WHERE status='manual' OR verdict='manual'" }] + }, + { + "type": "stat", + "id": 6, + "title": "Commented (advisory)", + "datasource": { "type": "frser-sqlite-datasource", "uid": "gittensory-db" }, + "gridPos": { "h": 4, "w": 4, "x": 16, "y": 1 }, + "fieldConfig": { "defaults": { "color": { "mode": "fixed", "fixedColor": "blue" }, "unit": "short" }, "overrides": [] }, + "options": { "colorMode": "background", "graphMode": "none", "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, "textMode": "value" }, + "targets": [{ "datasource": { "type": "frser-sqlite-datasource", "uid": "gittensory-db" }, "refId": "A", "queryType": "table", "queryText": "SELECT count(*) AS commented FROM review_targets WHERE status='commented'", "rawQueryText": "SELECT count(*) AS commented FROM review_targets WHERE status='commented'" }] + }, + { + "type": "stat", + "id": 7, + "title": "Ignored", + "datasource": { "type": "frser-sqlite-datasource", "uid": "gittensory-db" }, + "gridPos": { "h": 4, "w": 4, "x": 20, "y": 1 }, + "fieldConfig": { "defaults": { "color": { "mode": "fixed", "fixedColor": "purple" }, "unit": "short" }, "overrides": [] }, + "options": { "colorMode": "background", "graphMode": "none", "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, "textMode": "value" }, + "targets": [{ "datasource": { "type": "frser-sqlite-datasource", "uid": "gittensory-db" }, "refId": "A", "queryType": "table", "queryText": "SELECT count(*) AS ignored FROM review_targets WHERE status='ignored'", "rawQueryText": "SELECT count(*) AS ignored FROM review_targets WHERE status='ignored'" }] + }, + { + "type": "table", + "id": 8, + "title": "Pull requests (latest 1000)", + "datasource": { "type": "frser-sqlite-datasource", "uid": "gittensory-db" }, + "gridPos": { "h": 16, "w": 24, "x": 0, "y": 5 }, + "fieldConfig": { + "defaults": { "custom": { "align": "auto", "cellOptions": { "type": "auto" }, "filterable": true, "inspect": false } }, + "overrides": [ + { "matcher": { "id": "byName", "options": "number" }, "properties": [ + { "id": "links", "value": [{ "title": "Open #${__data.fields.number} on GitHub", "url": "https://github.com/${__data.fields.repo}/pull/${__data.fields.number}", "targetBlank": true }] }, + { "id": "custom.width", "value": 80 } + ]}, + { "matcher": { "id": "byName", "options": "author" }, "properties": [ + { "id": "links", "value": [{ "title": "@${__data.fields.author} on GitHub", "url": "https://github.com/${__data.fields.author}", "targetBlank": true }] }, + { "id": "custom.width", "value": 160 } + ]}, + { "matcher": { "id": "byName", "options": "repo" }, "properties": [{ "id": "custom.width", "value": 200 }] }, + { "matcher": { "id": "byName", "options": "status" }, "properties": [ + { "id": "custom.cellOptions", "value": { "type": "color-background", "mode": "basic" } }, + { "id": "custom.width", "value": 110 }, + { "id": "mappings", "value": [{ "type": "value", "options": { "merged": { "color": "green", "index": 0 }, "closed": { "color": "red", "index": 1 }, "manual": { "color": "orange", "index": 2 }, "commented": { "color": "blue", "index": 3 }, "ignored": { "color": "purple", "index": 4 } } }] } + ]}, + { "matcher": { "id": "byName", "options": "verdict" }, "properties": [ + { "id": "custom.cellOptions", "value": { "type": "color-background", "mode": "basic" } }, + { "id": "custom.width", "value": 100 }, + { "id": "mappings", "value": [{ "type": "value", "options": { "merge": { "color": "green", "index": 0 }, "close": { "color": "red", "index": 1 }, "manual": { "color": "orange", "index": 2 }, "comment": { "color": "blue", "index": 3 }, "ignore": { "color": "purple", "index": 4 } } }] } + ]}, + { "matcher": { "id": "byName", "options": "updated_at" }, "properties": [{ "id": "custom.width", "value": 200 }] } + ] }, - "options": { - "mode": "markdown", - "content": "## Maintainer review dashboard disabled\n\nThis dashboard does not query the live Gittensory SQLite database from Grafana. Direct datasource access would bypass application authorization and expose private application tables. Use the application API or a separately redacted reporting database for maintainer review analytics." - } + "options": { "showHeader": true, "cellHeight": "sm", "footer": { "show": false }, "sortBy": [{ "displayName": "updated_at", "desc": true }] }, + "targets": [{ "datasource": { "type": "frser-sqlite-datasource", "uid": "gittensory-db" }, "refId": "A", "queryType": "table", "queryText": "SELECT repo, number, submitter AS author, status, verdict, title, updated_at FROM review_targets ORDER BY updated_at DESC LIMIT 1000", "rawQueryText": "SELECT repo, number, submitter AS author, status, verdict, title, updated_at FROM review_targets ORDER BY updated_at DESC LIMIT 1000" }] + }, + { + "type": "timeseries", + "id": 9, + "title": "Reviews per day", + "datasource": { "type": "frser-sqlite-datasource", "uid": "gittensory-db" }, + "gridPos": { "h": 8, "w": 12, "x": 0, "y": 21 }, + "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "drawStyle": "bars", "fillOpacity": 60, "lineWidth": 1, "showPoints": "never", "stacking": { "mode": "none" } }, "unit": "short" }, "overrides": [] }, + "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": false }, "tooltip": { "mode": "single", "sort": "none" } }, + "targets": [{ "datasource": { "type": "frser-sqlite-datasource", "uid": "gittensory-db" }, "refId": "A", "queryType": "time series", "timeColumns": ["time"], "queryText": "SELECT date(created_at) AS time, count(*) AS reviews FROM review_targets GROUP BY date(created_at) ORDER BY time", "rawQueryText": "SELECT date(created_at) AS time, count(*) AS reviews FROM review_targets GROUP BY date(created_at) ORDER BY time" }] + }, + { + "type": "piechart", + "id": 10, + "title": "By verdict", + "datasource": { "type": "frser-sqlite-datasource", "uid": "gittensory-db" }, + "gridPos": { "h": 8, "w": 12, "x": 12, "y": 21 }, + "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "unit": "short" }, "overrides": [] }, + "options": { "legend": { "displayMode": "list", "placement": "right", "showLegend": true, "values": ["value", "percent"] }, "pieType": "donut", "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": true }, "tooltip": { "mode": "single", "sort": "none" } }, + "targets": [{ "datasource": { "type": "frser-sqlite-datasource", "uid": "gittensory-db" }, "refId": "A", "queryType": "table", "queryText": "SELECT verdict, count(*) AS c FROM review_targets WHERE verdict IS NOT NULL GROUP BY verdict ORDER BY c DESC", "rawQueryText": "SELECT verdict, count(*) AS c FROM review_targets WHERE verdict IS NOT NULL GROUP BY verdict ORDER BY c DESC" }] } ] } diff --git a/grafana/dashboards/resource-hub.json b/grafana/dashboards/resource-hub.json index 593a382619..ff7cd07c95 100644 --- a/grafana/dashboards/resource-hub.json +++ b/grafana/dashboards/resource-hub.json @@ -47,7 +47,7 @@ "gridPos": { "h": 13, "w": 12, "x": 0, "y": 0 }, "options": { "mode": "markdown", - "content": "## 🧠 AI providers\n- **Claude Code / Codex** — subscription CLIs, the review brain. Health: app boot log `selfhost_ai_provider`.\n- **Ollama** — embeddings (`bge-m3`) for RAG · [API root](http://localhost:11434) · `docker exec gittensory-ollama-1 ollama list`\n\n## 🔎 Vector store (RAG)\n- **Qdrant** — [Dashboard](http://localhost:6333/dashboard) · [Collections API](http://localhost:6333/collections)\n- Embeddings indexed per repo; see the *RAG indexing* doc.\n\n## ⚙️ Engine / API\n- **gittensory app** — [/ ](http://localhost:8787/) · [/ready](http://localhost:8787/ready) · [/metrics](http://localhost:8787/metrics)\n- Internal jobs: `POST /v1/internal/jobs/rag-index` (bearer `INTERNAL_JOB_TOKEN`)\n\n## 💾 Data\n- **SQLite** (default) at `/data/gittensory.sqlite` — not mounted into Grafana; use an explicitly redacted reporting database for dashboard analytics.\n- **Postgres/pgvector** (optional `--profile postgres`).\n\n> Links assume the default published ports on the Docker host. If you run Grafana on a remote host, replace `localhost` with that host, and publish the service `ports:` you want to reach." + "content": "## 🧠 AI providers\n- **Claude Code / Codex** — subscription CLIs, the review brain. Health: app boot log `selfhost_ai_provider`.\n- **Ollama** — embeddings (`bge-m3`) for RAG · [API root](http://localhost:11434) · `docker exec gittensory-ollama-1 ollama list`\n\n## 🔎 Vector store (RAG)\n- **Qdrant** — [Dashboard](http://localhost:6333/dashboard) · [Collections API](http://localhost:6333/collections)\n- Embeddings indexed per repo; see the *RAG indexing* doc.\n\n## ⚙️ Engine / API\n- **gittensory app** — [/ ](http://localhost:8787/) · [/ready](http://localhost:8787/ready) · [/metrics](http://localhost:8787/metrics)\n- Internal jobs: `POST /v1/internal/jobs/rag-index` (bearer `INTERNAL_JOB_TOKEN`)\n\n## 💾 Data\n- **SQLite** (default) at `/data/gittensory.sqlite` — private to the app/exporter. Grafana reads `/reporting/gittensory-reporting.sqlite`, a redacted reporting snapshot.\n- **Postgres/pgvector** (optional `--profile postgres`).\n\n> Links assume the default published ports on the Docker host. If you run Grafana on a remote host, replace `localhost` with that host, and publish the service `ports:` you want to reach." } }, diff --git a/grafana/provisioning/datasources/sqlite.yml b/grafana/provisioning/datasources/sqlite.yml index ccc205ec75..ce2e7c8fc2 100644 --- a/grafana/provisioning/datasources/sqlite.yml +++ b/grafana/provisioning/datasources/sqlite.yml @@ -1,5 +1,5 @@ -# Maintainer-only SQLite datasource for local operator dashboards. The compose stack mounts the app -# database at /appdb for Grafana, and these dashboards run aggregate SELECTs over internal tables. +# Maintainer-only SQLite datasource for local operator dashboards. Grafana reads a redacted reporting +# database produced by reporting-exporter; it never mounts the live application database. apiVersion: 1 datasources: - name: GittensoryDB @@ -8,4 +8,4 @@ datasources: access: proxy editable: false jsonData: - path: /appdb/gittensory.sqlite + path: /reporting/gittensory-reporting.sqlite diff --git a/scripts/export-grafana-reporting-db.sh b/scripts/export-grafana-reporting-db.sh new file mode 100644 index 0000000000..ab26563cba --- /dev/null +++ b/scripts/export-grafana-reporting-db.sh @@ -0,0 +1,111 @@ +#!/bin/sh +set -eu + +APP_DB="${GITTENSORY_REPORTING_SOURCE_DB:-/appdb/gittensory.sqlite}" +OUT_DIR="${GITTENSORY_REPORTING_DIR:-/reporting}" +OUT_DB="${GITTENSORY_REPORTING_DB:-$OUT_DIR/gittensory-reporting.sqlite}" +TMP_DB="${OUT_DB}.tmp" + +mkdir -p "$OUT_DIR" + +if [ ! -s "$APP_DB" ]; then + echo "reporting export skipped: source database missing at $APP_DB" >&2 + exit 0 +fi + +rm -f "$TMP_DB" "$TMP_DB-wal" "$TMP_DB-shm" + +sqlite3 "$TMP_DB" <<'SQL' +PRAGMA synchronous=NORMAL; + +CREATE TABLE review_targets ( + repo TEXT NOT NULL, + number INTEGER NOT NULL, + submitter TEXT, + status TEXT NOT NULL, + verdict TEXT, + title TEXT, + created_at TEXT NOT NULL, + updated_at TEXT NOT NULL +); +CREATE INDEX review_targets_updated_idx ON review_targets(updated_at); +CREATE INDEX review_targets_status_idx ON review_targets(status); +CREATE INDEX review_targets_verdict_idx ON review_targets(verdict); + +CREATE TABLE ai_usage_events ( + feature TEXT NOT NULL, + model TEXT NOT NULL, + status TEXT NOT NULL, + estimated_neurons INTEGER NOT NULL DEFAULT 0, + detail TEXT, + metadata_json TEXT NOT NULL DEFAULT '{}', + created_at TEXT NOT NULL +); +CREATE INDEX ai_usage_events_feature_created_idx ON ai_usage_events(feature, created_at); +CREATE INDEX ai_usage_events_model_created_idx ON ai_usage_events(model, created_at); +SQL + +if sqlite3 "$APP_DB" "SELECT 1 FROM sqlite_master WHERE type='table' AND name='review_targets' LIMIT 1" | grep -q 1; then + sqlite3 "$APP_DB" </dev/null +echo "reporting export complete: $OUT_DB" From d4913cecbf780d48d3c03a28459404445dbff4f3 Mon Sep 17 00:00:00 2001 From: JSONbored <49853598+JSONbored@users.noreply.github.com> Date: Sun, 28 Jun 2026 06:59:26 -0700 Subject: [PATCH 04/68] build(selfhost): automate sentry release source maps --- .env.example | 9 +++ .github/workflows/release-selfhost.yml | 96 +++++++++++++++++++++++++- Dockerfile | 33 +++++++-- docs/self-host/configuration.md | 17 +++++ docs/self-host/troubleshooting.md | 9 +++ docs/self-hosting.md | 8 ++- grafana/dashboards/resource-hub.json | 10 ++- scripts/build-selfhost.mjs | 5 ++ src/selfhost/sentry.ts | 12 +++- test/unit/selfhost-sentry.test.ts | 29 ++++++++ 10 files changed, 215 insertions(+), 13 deletions(-) diff --git a/.env.example b/.env.example index 0c9da2274f..3e78bc09e3 100644 --- a/.env.example +++ b/.env.example @@ -193,6 +193,15 @@ GITTENSORY_REVIEW_DRAFT=false # The ENGINE posts a per-repo review summary when it publishes a review — set a per-repo map and/or a global fallback: # DISCORD_WEBHOOK_URL=https://discord.com/api/webhooks/... # global fallback for any repo without its own # DISCORD_REPO_WEBHOOKS={"owner/repoA":"https://discord.com/api/webhooks/...","owner/repoB":"https://..."} # per-repo +# +# Sentry error tracking. OFF when SENTRY_DSN is unset. Official self-host release images bake +# GITTENSORY_VERSION=gittensory-selfhost@; initSentry uses that as the release id unless +# SENTRY_RELEASE is set explicitly (useful for custom/local images). +# SENTRY_DSN= +# SENTRY_DSN_FILE= # optional mounted secret file; existing *_FILE loader reads it +# SENTRY_ENVIRONMENT=selfhost +# SENTRY_RELEASE= +# SENTRY_TRACES_SAMPLE_RATE=0 # --- AI review backend (optional; without it reviews run deterministically) --- # AI_SUMMARIES_ENABLED=true diff --git a/.github/workflows/release-selfhost.yml b/.github/workflows/release-selfhost.yml index 9b4a7f3a66..2ede23cd27 100644 --- a/.github/workflows/release-selfhost.yml +++ b/.github/workflows/release-selfhost.yml @@ -20,27 +20,100 @@ permissions: contents: write # create the GitHub Release packages: write # push to GHCR +concurrency: + group: release-selfhost-${{ github.ref_name }} + cancel-in-progress: false + jobs: release: runs-on: ubuntu-latest timeout-minutes: 40 # Environment gate — requires reviewer approval before a release runs (configure under repo Settings > Environments). environment: release + env: + SENTRY_ORG: jsonbored + SENTRY_PROJECT: gittensory steps: - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 with: + fetch-depth: 0 persist-credentials: false + - name: Verify release commit is on main + env: + RELEASE_SHA: ${{ github.sha }} + run: | + git fetch --no-tags origin main + if ! git merge-base --is-ancestor "$RELEASE_SHA" origin/main; then + echo "::error::Self-host releases must be cut from a commit reachable from main." + exit 1 + fi + - name: Resolve version id: version env: + EVENT_NAME: ${{ github.event_name }} INPUT_VERSION: ${{ github.event.inputs.version }} + REF_NAME: ${{ github.ref_name }} run: | - if [ "${{ github.event_name }}" = "workflow_dispatch" ]; then - echo "v=${INPUT_VERSION}" >> "$GITHUB_OUTPUT" + if [ "$EVENT_NAME" = "workflow_dispatch" ]; then + version="$INPUT_VERSION" else - echo "v=${GITHUB_REF_NAME#selfhost-v}" >> "$GITHUB_OUTPUT" + case "$REF_NAME" in + selfhost-v*) version="${REF_NAME#selfhost-v}" ;; + *) echo "::error::Invalid self-host release tag: $REF_NAME"; exit 1 ;; + esac + fi + if ! [[ "$version" =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]]; then + echo "::error::Version must be plain semver (example: 0.1.0)" + exit 1 fi + release="gittensory-selfhost@$version" + echo "v=$version" >> "$GITHUB_OUTPUT" + echo "release=$release" >> "$GITHUB_OUTPUT" + + - uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6 + with: + node-version: "24" + cache: "npm" + + - name: Install deps + run: npm ci --ignore-scripts + + - name: Build self-host bundle for release + run: SELFHOST_SOURCEMAP=1 SELFHOST_BUNDLE_ALL=1 node scripts/build-selfhost.mjs --all --sourcemap + + - name: Detect Sentry release token + id: sentry + env: + SENTRY_AUTH_TOKEN: ${{ secrets.SENTRY_AUTH_TOKEN }} + run: | + if [ -n "$SENTRY_AUTH_TOKEN" ]; then + echo "enabled=true" >> "$GITHUB_OUTPUT" + else + echo "enabled=false" >> "$GITHUB_OUTPUT" + fi + + - name: Require Sentry token for official release + if: github.repository == 'JSONbored/gittensory' && steps.sentry.outputs.enabled != 'true' + run: | + echo "::error::Configure SENTRY_AUTH_TOKEN in the release environment before publishing official self-host images." + exit 1 + + - name: Prepare Sentry release + source maps + if: steps.sentry.outputs.enabled == 'true' + uses: getsentry/action-release@ff07929a6537bac57790c3451cf4d364aca38528 # v3 + env: + SENTRY_AUTH_TOKEN: ${{ secrets.SENTRY_AUTH_TOKEN }} + with: + release: ${{ steps.version.outputs.release }} + set_commits: auto + ignore_missing: true + ignore_empty: true + sourcemaps: dist + url_prefix: /app/dist + strip_common_prefix: true + finalize: false - uses: docker/setup-qemu-action@c7c53464625b32c7a7e944ae62b3e17d2b600130 # v3 - uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3 @@ -65,13 +138,19 @@ jobs: org.opencontainers.image.title=gittensory-selfhost org.opencontainers.image.description=Self-hostable Gittensory review engine org.opencontainers.image.version=${{ steps.version.outputs.v }} + org.opencontainers.image.revision=${{ github.sha }} - name: Build + push (linux/amd64 + linux/arm64) uses: docker/build-push-action@10e90e3645eae34f1e60eeb005ba3a3d33f178e8 # v6 with: context: . + target: runtime-prebuilt + build-contexts: | + selfhost_dist=./dist platforms: linux/amd64,linux/arm64 push: true + build-args: | + GITTENSORY_VERSION=${{ steps.version.outputs.release }} tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} provenance: true @@ -92,3 +171,14 @@ jobs: Multi-arch (linux/amd64 + linux/arm64). See [docs/self-hosting.md](docs/self-hosting.md) for setup. To include the Claude Code / Codex subscription CLIs, build locally with `--build-arg INSTALL_AI_CLIS=true`. + Sentry release id baked into the image: `${{ steps.version.outputs.release }}`. + + - name: Finalize Sentry release + if: steps.sentry.outputs.enabled == 'true' + uses: getsentry/action-release@ff07929a6537bac57790c3451cf4d364aca38528 # v3 + env: + SENTRY_AUTH_TOKEN: ${{ secrets.SENTRY_AUTH_TOKEN }} + with: + release: ${{ steps.version.outputs.release }} + environment: selfhost + set_commits: skip diff --git a/Dockerfile b/Dockerfile index 0092349035..2cc0da505f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -3,10 +3,13 @@ # Cloudflare Worker (wrangler) deploy is unaffected. SECRETS ARE NEVER BAKED: supply them at run time via # the .env file or mounted *_FILE secrets (see docker-compose.yml + .env.example). +ARG GITTENSORY_VERSION= + # --- build: install deps + bundle the Node entry -------------------------------------------------------- # ECR Public Gallery mirrors Docker Official Images with no rate limits and no auth. FROM public.ecr.aws/docker/library/node:24-slim AS build WORKDIR /app +ARG SELFHOST_SOURCEMAP=false COPY package*.json ./ # --ignore-scripts: no native builds are needed (SQLite is the built-in node:sqlite; @hono/node-server is # pure JS; esbuild ships its binary as an optional dependency, not a script). @@ -14,19 +17,21 @@ RUN npm ci --ignore-scripts COPY . . # --all: bundle every dependency into one self-contained dist/server.mjs, so the runtime image needs no # node_modules (≈10× smaller). The bundle has zero `cloudflare:*` imports (stubbed at build), so no loader. -RUN node scripts/build-selfhost.mjs --all +# Release builds set SELFHOST_SOURCEMAP=true so Sentry can unminify the production bundle. +RUN if [ "$SELFHOST_SOURCEMAP" = "true" ]; then SELFHOST_SOURCEMAP=1 node scripts/build-selfhost.mjs --all; else node scripts/build-selfhost.mjs --all; fi -# --- runtime: slim, non-root ---------------------------------------------------------------------------- -FROM public.ecr.aws/docker/library/node:24-slim AS runtime +# --- runtime base: slim, non-root ----------------------------------------------------------------------- +FROM public.ecr.aws/docker/library/node:24-slim AS runtime-base WORKDIR /app +ARG GITTENSORY_VERSION ENV NODE_ENV=production \ PLATFORM=self-hosted \ PORT=8787 \ DATABASE_PATH=/data/gittensory.sqlite \ MIGRATIONS_DIR=/app/migrations \ - NPM_CONFIG_PREFIX=/home/node/.npm-global -COPY --from=build /app/dist ./dist -COPY --from=build /app/migrations ./migrations + NPM_CONFIG_PREFIX=/home/node/.npm-global \ + GITTENSORY_VERSION=${GITTENSORY_VERSION} \ + NODE_OPTIONS=--enable-source-maps # Optional: bake the Claude Code / Codex CLIs so the `claude-code` / `codex` subscription providers (#979) # work in-image. Build with `--build-arg INSTALL_AI_CLIS=true`. No credentials are baked — operators mint # CLAUDE_CODE_OAUTH_TOKEN (`claude setup-token`) / codex auth at run time and pass it via the env. @@ -47,7 +52,7 @@ USER root # Optional: enable visual review via an external Chrome sidecar (e.g. `browserless/chrome:latest`). # Build with `--build-arg INSTALL_VISUAL_REVIEW=true` then set BROWSER_WS_ENDPOINT= at runtime. ARG INSTALL_VISUAL_REVIEW=false -COPY --from=build /app/package*.json ./ +COPY package*.json ./ RUN if [ "$INSTALL_VISUAL_REVIEW" = "true" ]; then npm install puppeteer-core@22.13.1 --ignore-scripts; fi # Data dir (the SQLite file) — owned by the unprivileged node user; mount a volume here to persist. RUN mkdir -p /data && chown -R node:node /data /app @@ -62,3 +67,17 @@ EXPOSE 8787 HEALTHCHECK --interval=30s --timeout=5s --start-period=60s --retries=3 \ CMD node -e "fetch('http://127.0.0.1:'+(process.env.PORT||8787)+'/ready').then(r=>process.exit(r.ok?0:1)).catch(()=>process.exit(1))" CMD ["node", "dist/server.mjs"] + +# Release target: consume the already-built, Sentry-injected dist/ from a named BuildKit context: +# docker buildx build --target runtime-prebuilt --build-context selfhost_dist=./dist ... +# This keeps the uploaded source maps and the deployed bundle byte-identical. +FROM runtime-base AS runtime-prebuilt +COPY --chown=node:node --from=selfhost_dist / ./dist +COPY --chown=node:node migrations ./migrations +CMD ["node", "dist/server.mjs"] + +# Default target: normal local/CI image builds still compile inside Docker. +FROM runtime-base AS runtime +COPY --chown=node:node --from=build /app/dist ./dist +COPY --chown=node:node --from=build /app/migrations ./migrations +CMD ["node", "dist/server.mjs"] diff --git a/docs/self-host/configuration.md b/docs/self-host/configuration.md index b291487dc5..7c012206b8 100644 --- a/docs/self-host/configuration.md +++ b/docs/self-host/configuration.md @@ -87,6 +87,23 @@ See [ai-providers.md](./ai-providers.md) for the full provider/model/effort/time | `AI_EMBED_BASE_URL` / `_MODEL` / `_PROVIDER` | Dedicated RAG embed provider | | `GITTENSORY_REPO_CONFIG_DIR` | Container-private per-repo config dir | +## Sentry environment variables + +Sentry is off unless `SENTRY_DSN` is set. Official self-host release images bake +`GITTENSORY_VERSION=gittensory-selfhost@`; `initSentry()` uses that as the release id unless +`SENTRY_RELEASE` is explicitly set. + +| Var | Purpose | +| --------------------------- | ----------------------------------------------------------------------- | +| `SENTRY_DSN` | Enables Sentry error reporting when set | +| `SENTRY_DSN_FILE` | Optional mounted-file form, handled by the existing `*_FILE` loader | +| `SENTRY_ENVIRONMENT` | Runtime environment; use `selfhost` for Docker-stack installs | +| `SENTRY_RELEASE` | Optional override for custom images; leave empty on official images | +| `SENTRY_TRACES_SAMPLE_RATE` | Optional tracing sample rate; default `0` keeps tracing off | +| `GITTENSORY_VERSION` | Image-baked release id used as Sentry fallback | + +For Sentry's GitHub code mapping, use **Stack Trace Root** `/app` and **Source Code Root** `.`. + ## Secrets — never commit them `CLAUDE_CODE_OAUTH_TOKEN`, `ANTHROPIC_API_KEY`, `INTERNAL_JOB_TOKEN`, `TOKEN_ENCRYPTION_SECRET`, the App private diff --git a/docs/self-host/troubleshooting.md b/docs/self-host/troubleshooting.md index a8b5160960..238d7897b1 100644 --- a/docs/self-host/troubleshooting.md +++ b/docs/self-host/troubleshooting.md @@ -114,6 +114,15 @@ AI review entirely in advisory mode. If you still see repeats, check for an auto **Fix:** set `CODEX_AI_MODEL=gpt-5.5` and `CODEX_AI_EFFORT=high` for the current recommended self-host Codex reviewer. Leave `CODEX_HOME` unset in the app environment. +### Sentry issues still point at `dist/server.mjs` + +**Cause:** Sentry is enabled, but the running image's release id does not match an uploaded self-host release +artifact, or the GitHub code mapping is pointed at generated `dist/` output. +**Fix:** official images set `GITTENSORY_VERSION=gittensory-selfhost@` and the release workflow uploads +the matching source maps automatically. Leave `SENTRY_RELEASE` empty unless you are running a custom image with a +custom uploaded release. In Sentry's GitHub integration, set **Stack Trace Root** to `/app` and **Source Code +Root** to `.`. + ### `codex_credential_isolation_required` **Cause:** the Codex subscription reviewer is fail-closed unless you explicitly opt into mounted diff --git a/docs/self-hosting.md b/docs/self-hosting.md index b47611970c..484798c4c0 100644 --- a/docs/self-hosting.md +++ b/docs/self-hosting.md @@ -55,7 +55,8 @@ node --import ./scripts/register-selfhost.mjs dist/server.mjs Releases are cut by pushing a `selfhost-v` tag (e.g. `selfhost-v0.1.0`): CI builds the multi-arch image, pushes it to GHCR with `:`, `:latest`, and `:sha-…` tags (with provenance + SBOM), and opens a -GitHub Release. +GitHub Release. Official release images also bake `GITTENSORY_VERSION=gittensory-selfhost@` so Sentry +events can be matched to the release/source maps uploaded by the release workflow. --- @@ -239,6 +240,11 @@ content-lane are not yet per-repo toggleable and stay on the allowlist.) in-flight job, checkpoints the WAL, and closes the DB before exiting. - **Logs** are structured JSON (`selfhost_listening`, `selfhost_migrations_applied`, `selfhost_ai_provider`, `selfhost_queue_recovered`, `selfhost_job_dead`, `selfhost_cron_error`, `selfhost_shutdown`, …). +- **Sentry.** Set `SENTRY_DSN` (or mount `SENTRY_DSN_FILE`) to enable error reporting. Keep + `SENTRY_ENVIRONMENT=selfhost`; leave `SENTRY_RELEASE` empty on official images so the baked + `GITTENSORY_VERSION` is used. For custom images, set `SENTRY_RELEASE` to the exact release id whose source maps + you uploaded. In Sentry's GitHub integration, map stack traces with **Stack Trace Root** `/app` and + **Source Code Root** `.`. - **Data + backup.** Everything is the single SQLite file on the `gittensory-data` volume (WAL mode). Back up by snapshotting the volume or copying the `.sqlite` file. Migrations are idempotent and re-checked at boot. For **continuous, point-in-time backup**, enable the optional [Litestream](https://litestream.io) sidecar in diff --git a/grafana/dashboards/resource-hub.json b/grafana/dashboards/resource-hub.json index ff7cd07c95..972c5b65a2 100644 --- a/grafana/dashboards/resource-hub.json +++ b/grafana/dashboards/resource-hub.json @@ -37,6 +37,14 @@ "url": "http://localhost:8787/metrics", "icon": "external link", "targetBlank": true + }, + { + "title": "Sentry — errors", + "type": "link", + "url": "https://jsonbored.sentry.io/projects/gittensory/", + "icon": "bolt", + "targetBlank": true, + "tooltip": "Crash & error tracking (uncaught, dead-letter jobs, review failures)" } ], "panels": [ @@ -58,7 +66,7 @@ "gridPos": { "h": 13, "w": 12, "x": 12, "y": 0 }, "options": { "mode": "markdown", - "content": "## 📊 Dashboards\n- **[Upstream PRs & issues (GitHub)](/d/gittensory-github)** — live, accurate census + open-PR triage (GitHub API).\n- **[Reviews & PRs (maintainer)](/d/gittensory-maintainer)** — gittensory's own review activity + reviewed-PR log.\n- **[Claude usage (OTEL)](/d/gittensory-claude)** — cost / tokens / model / effort from the review CLI.\n- **[Codex usage (self-host)](/d/gittensory-codex)** — Codex review counts, token counters, and durable AI usage rows.\n- **[Gittensory (infra)](/d/gittensory)** — AI usage & cost, queue, jobs, HTTP.\n\n## 📈 Metrics & logs\n- **Prometheus** — [targets](http://localhost:9090/targets) · [graph](http://localhost:9090)\n- **Alertmanager** — [alerts](http://localhost:9093)\n- **Loki** — query in [Explore](/explore) (pick the *Loki* datasource), e.g. `{compose_service=\"gittensory\"}`\n\n## 🩺 Quick health checks\n| What | Where |\n|---|---|\n| App serving | `GET /ready` → 200 |\n| AI wired | boot log `selfhost_ai_provider` |\n| Embeds wired | boot log `selfhost_embed_provider` |\n| Vectors wired | boot log `selfhost_vectorize` |\n| Token spend | infra dashboard → *AI Usage & Cost* |\n\n## 📚 Docs\n- `docs/self-host/` — configuration, ai-providers, rag-indexing, review-modes, troubleshooting." + "content": "## 📊 Dashboards\n- **[Upstream PRs & issues (GitHub)](/d/gittensory-github)** — live, accurate census + open-PR triage (GitHub API).\n- **[Reviews & PRs (maintainer)](/d/gittensory-maintainer)** — gittensory's own review activity + reviewed-PR log.\n- **[Claude usage (OTEL)](/d/gittensory-claude)** — cost / tokens / model / effort from the review CLI.\n- **[Codex usage (self-host)](/d/gittensory-codex)** — Codex review counts, token counters, and durable AI usage rows.\n- **[Gittensory (infra)](/d/gittensory)** — AI usage & cost, queue, jobs, HTTP.\n\n## 📈 Metrics & logs\n- **Prometheus** — [targets](http://localhost:9090/targets) · [graph](http://localhost:9090)\n- **Alertmanager** — [alerts](http://localhost:9093)\n- **Loki** — query in [Explore](/explore) (pick the *Loki* datasource), e.g. `{compose_service=\"gittensory\"}`\n- **Sentry** — release/source-map enriched errors. Edit the dashboard link if your project URL differs.\n\n## 🩺 Quick health checks\n| What | Where |\n|---|---|\n| App serving | `GET /ready` → 200 |\n| AI wired | boot log `selfhost_ai_provider` |\n| Embeds wired | boot log `selfhost_embed_provider` |\n| Vectors wired | boot log `selfhost_vectorize` |\n| Token spend | infra dashboard → *AI Usage & Cost* |\n\n## 📚 Docs\n- `docs/self-host/` — configuration, ai-providers, rag-indexing, review-modes, troubleshooting." } } ] diff --git a/scripts/build-selfhost.mjs b/scripts/build-selfhost.mjs index a8c9ee1c4f..6d8f8b06b4 100644 --- a/scripts/build-selfhost.mjs +++ b/scripts/build-selfhost.mjs @@ -2,6 +2,7 @@ // default → node_modules stay external (resolved at runtime; fast local dev rebuilds). // --all / SELFHOST_BUNDLE_ALL=1 → bundle EVERYTHING into one self-contained file (the Docker image needs no // node_modules → a ~10× smaller image). node: builtins stay external (platform:node). +// --sourcemap / SELFHOST_SOURCEMAP=1 → emit dist/server.mjs.map for release/Sentry builds. // In both modes the Cloudflare-only specifiers resolve to Node stubs via the plugin (precedence over external), // so the bundle has zero `cloudflare:*` imports. import { dirname, resolve } from "node:path"; @@ -10,6 +11,7 @@ import esbuild from "esbuild"; const root = resolve(dirname(fileURLToPath(import.meta.url)), ".."); const bundleAll = process.env.SELFHOST_BUNDLE_ALL === "1" || process.argv.includes("--all"); +const sourcemap = process.env.SELFHOST_SOURCEMAP === "1" || process.argv.includes("--sourcemap"); await esbuild.build({ entryPoints: [resolve(root, "src/server.ts")], @@ -18,6 +20,9 @@ await esbuild.build({ format: "esm", target: "node22", outfile: resolve(root, "dist/server.mjs"), + sourcemap, + sourcesContent: true, + sourceRoot: process.env.SELFHOST_SOURCE_ROOT?.trim() || "/app/dist", // External: nothing (bundle all) vs every package (external). node: builtins are always external on node. ...(bundleAll ? {} : { packages: "external" }), // Bundling CJS deps into an ESM output needs require/__dirname/__filename shimmed (some deps call them). diff --git a/src/selfhost/sentry.ts b/src/selfhost/sentry.ts index 2912c188c7..d38e701799 100644 --- a/src/selfhost/sentry.ts +++ b/src/selfhost/sentry.ts @@ -9,6 +9,15 @@ let active = false; const SECRET_KEY = /(token|secret|key|password|passwd|authorization|auth|dsn|cookie|bearer|credential|private)/i; +function firstNonBlank(...values: Array): string | undefined { + return values.map((value) => value?.trim()).find((value): value is string => Boolean(value)); +} + +/** Resolve the Sentry release id from explicit override first, then the image-baked self-host version. */ +export function resolveSentryRelease(env: NodeJS.ProcessEnv): string | undefined { + return firstNonBlank(env.SENTRY_RELEASE, env.GITTENSORY_VERSION); +} + /** beforeSend scrubber — redact anything token/secret-like before an event leaves the box (privacy boundary). */ export function scrubEvent(event: T): T { const redact = (obj: unknown, depth: number): void => { @@ -38,10 +47,11 @@ export function scrubEvent(event: T): T { export async function initSentry(env: NodeJS.ProcessEnv): Promise { if (!env.SENTRY_DSN) return false; Sentry = await import("@sentry/node"); + const release = resolveSentryRelease(env); Sentry.init({ dsn: env.SENTRY_DSN, environment: env.SENTRY_ENVIRONMENT ?? "production", - release: env.SENTRY_RELEASE ?? env.GITTENSORY_VERSION, + ...(release ? { release } : {}), tracesSampleRate: Number(env.SENTRY_TRACES_SAMPLE_RATE ?? "0"), serverName: env.PUBLIC_API_ORIGIN, beforeSend: (e) => scrubEvent(e), diff --git a/test/unit/selfhost-sentry.test.ts b/test/unit/selfhost-sentry.test.ts index 09609ea258..496560c6c7 100644 --- a/test/unit/selfhost-sentry.test.ts +++ b/test/unit/selfhost-sentry.test.ts @@ -29,6 +29,7 @@ import { installStructuredLogForwarding, scrubEvent, resetSentryForTest, + resolveSentryRelease, } from "../../src/selfhost/sentry"; beforeEach(() => { @@ -96,6 +97,7 @@ describe("enabled when SENTRY_DSN is set", () => { expect(mocks.init).toHaveBeenCalledTimes(1); const opts = mocks.init.mock.calls[0]![0]; expect(opts.environment).toBe("production"); + expect(opts.release).toBeUndefined(); expect(opts.tracesSampleRate).toBe(0); expect( opts.beforeSend({ extra: { sessionToken: "s" } }).extra.sessionToken, @@ -117,6 +119,33 @@ describe("enabled when SENTRY_DSN is set", () => { expect(opts.serverName).toBe("https://self.host"); }); + it("uses the image-baked version as the release fallback and ignores blank overrides", async () => { + expect( + resolveSentryRelease({ + SENTRY_RELEASE: " ", + GITTENSORY_VERSION: " gittensory-selfhost@0.1.0 ", + } as unknown as NodeJS.ProcessEnv), + ).toBe("gittensory-selfhost@0.1.0"); + + await initSentry({ + SENTRY_DSN: "d", + SENTRY_RELEASE: "", + GITTENSORY_VERSION: "gittensory-selfhost@0.1.0", + } as unknown as NodeJS.ProcessEnv); + expect(mocks.init.mock.calls[0]![0].release).toBe( + "gittensory-selfhost@0.1.0", + ); + }); + + it("prefers an explicit nonblank SENTRY_RELEASE over GITTENSORY_VERSION", () => { + expect( + resolveSentryRelease({ + SENTRY_RELEASE: "custom@sha", + GITTENSORY_VERSION: "gittensory-selfhost@0.1.0", + } as unknown as NodeJS.ProcessEnv), + ).toBe("custom@sha"); + }); + it("captureError sends with context, and without context skips setContext", async () => { await initSentry({ SENTRY_DSN: "d" } as unknown as NodeJS.ProcessEnv); captureError(new Error("boom"), { kind: "job_dead" }); From 5f498188fb68fab837c2b4db0abe270708d85fd0 Mon Sep 17 00:00:00 2001 From: JSONbored <49853598+JSONbored@users.noreply.github.com> Date: Sun, 28 Jun 2026 07:08:46 -0700 Subject: [PATCH 05/68] ci(selfhost): disable release workflow dependency cache --- .github/workflows/release-selfhost.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/release-selfhost.yml b/.github/workflows/release-selfhost.yml index 2ede23cd27..aba01dcbc3 100644 --- a/.github/workflows/release-selfhost.yml +++ b/.github/workflows/release-selfhost.yml @@ -72,10 +72,10 @@ jobs: echo "v=$version" >> "$GITHUB_OUTPUT" echo "release=$release" >> "$GITHUB_OUTPUT" + # Release jobs receive publishing/Sentry credentials, so avoid shared dependency caches here. - uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6 with: node-version: "24" - cache: "npm" - name: Install deps run: npm ci --ignore-scripts From 0f465aab37aab1c14008600f62cfc4a37d0e401f Mon Sep 17 00:00:00 2001 From: JSONbored <49853598+JSONbored@users.noreply.github.com> Date: Sun, 28 Jun 2026 07:24:52 -0700 Subject: [PATCH 06/68] test(selfhost): restore reviewer diagnostics patch coverage --- src/selfhost/ai.ts | 4 ++-- test/unit/ai-review-advisory.test.ts | 9 +++++++++ test/unit/selfhost-ai.test.ts | 17 +++++++++++++++++ 3 files changed, 28 insertions(+), 2 deletions(-) diff --git a/src/selfhost/ai.ts b/src/selfhost/ai.ts index 45c8db7816..62ebd53c6a 100644 --- a/src/selfhost/ai.ts +++ b/src/selfhost/ai.ts @@ -437,8 +437,8 @@ function logSelfHostAiProviderFailed(input: { event: "selfhost_ai_provider_failed", provider: input.provider, model: input.model || "default", - ...(input.effort ? { effort: input.effort } : {}), - ...(input.timeoutMs ? { timeoutMs: input.timeoutMs } : {}), + effort: input.effort, + timeoutMs: input.timeoutMs, error: errorMessage(input.error, input.knownSecrets), }), ); diff --git a/test/unit/ai-review-advisory.test.ts b/test/unit/ai-review-advisory.test.ts index e669163881..0adfadc77e 100644 --- a/test/unit/ai-review-advisory.test.ts +++ b/test/unit/ai-review-advisory.test.ts @@ -156,6 +156,15 @@ describe("runAiReviewForAdvisory", () => { expect(usage?.model).toBe("claude-code:claude-sonnet-4-6"); }); + it("records explicit self-host reviewer labels when models are omitted or providers are unknown", async () => { + const env = aiEnv(async () => { throw new Error("codex unavailable"); }); + (env as unknown as { AI_PROVIDER: string }).AI_PROVIDER = " CODEX , unknown-provider "; + const result = await runAiReviewForAdvisory(env, { settings: { aiReviewMode: "advisory" } as RepositorySettings, advisory: advisory(), repoFullName: "acme/widgets", pr, author: "alice", confirmedContributor: true }); + expect(result).toBeUndefined(); + const usage = await env.DB.prepare("SELECT model FROM ai_usage_events WHERE feature = 'ai_review_pr' ORDER BY created_at DESC LIMIT 1").first<{ model: string }>(); + expect(usage?.model).toBe("codex+unknown-provider"); + }); + it("no-ops for a non-confirmed contributor under the gittensor pack and when there is no head SHA", async () => { const env = aiEnv(async () => ({ response: defectJson() })); const base = { settings: { aiReviewMode: "block", gatePack: "gittensor" } as RepositorySettings, repoFullName: "acme/widgets", pr, author: "alice" }; diff --git a/test/unit/selfhost-ai.test.ts b/test/unit/selfhost-ai.test.ts index 808502249b..bc50904a84 100644 --- a/test/unit/selfhost-ai.test.ts +++ b/test/unit/selfhost-ai.test.ts @@ -300,6 +300,7 @@ describe("branch coverage — defaults + edge inputs", () => { extractCliUsage( [ JSON.stringify({ usage: { input_tokens: 10, outputTokens: "5", total_tokens: 15 }, model: "gpt-5" }), + "", JSON.stringify({ tokenUsage: { prompt_tokens: 12, completion_tokens: 6, totalTokens: 18 }, total_cost_usd: "0.07" }), JSON.stringify({ usage_metadata: { costUsd: 0.09 } }), ].join("\n"), @@ -348,6 +349,7 @@ describe("branch coverage — defaults + edge inputs", () => { it("buildProvider uses provider-specific default base URLs when provider base URLs are unset", () => { expect(typeof buildProvider("openai", {})?.run).toBe("function"); // defaults to https://api.openai.com/v1 expect(typeof buildProvider("ollama", {})?.run).toBe("function"); // defaults to http://localhost:11434/v1 + expect(typeof buildProvider("openai-compatible", {})?.run).toBe("function"); // defaults to http://localhost:11434/v1 }); it("extractCliText reads content + response fields", () => { expect(extractCliText(JSON.stringify({ content: "c" }))).toBe("c"); @@ -365,6 +367,7 @@ describe("branch coverage — defaults + edge inputs", () => { }, }; await expect(createChainAi([p]).run("m", { prompt: "x" })).rejects.toThrow(/all_ai_providers_failed/); + await expect(createChainAi([p]).run("", { prompt: "x" })).rejects.toThrow(/all_ai_providers_failed/); }); }); @@ -501,6 +504,20 @@ describe("subscription CLI helpers + fail-safe", () => { } }); + it("drives the REAL subprocess (defaultSpawn) against a fake `codex` on PATH", async () => { + const dir = mkdtempSync(join(tmpdir(), "fakecli-")); + const fake = join(dir, "codex"); + writeFileSync(fake, "#!/usr/bin/env node\nlet i='';process.stdin.on('data',d=>i+=d);process.stdin.on('end',()=>process.stdout.write(JSON.stringify({type:'result',result:'OK:'+i.trim()})));\n"); + chmodSync(fake, 0o755); + const origPath = process.env.PATH; + try { + const out = await createCodexAi({ PATH: `${dir}:${origPath ?? ""}`, HOME: dir, GITTENSORY_ENABLE_UNSAFE_CODEX_REVIEWER: "1" }).run("", { prompt: "hello" }); + expect(out.response).toBe("OK:hello"); + } finally { + process.env.PATH = origPath; + } + }); + it("Claude Code throws on no-token / non-zero exit / empty output", async () => { await expect(createClaudeCodeAi({}).run("m", { prompt: "x" })).rejects.toThrow(/claude_code_no_oauth_token/); const exit1: StubSpawn = async () => ({ stdout: "", code: 1 }); From 88856ea8e5bc1fa01417e8cfed8689b0af99cd20 Mon Sep 17 00:00:00 2001 From: JSONbored <49853598+JSONbored@users.noreply.github.com> Date: Sun, 28 Jun 2026 07:35:52 -0700 Subject: [PATCH 07/68] fix(observability): include dual Codex review records --- grafana/dashboards/codex-usage.json | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/grafana/dashboards/codex-usage.json b/grafana/dashboards/codex-usage.json index 67d70a109b..c05b1182b8 100644 --- a/grafana/dashboards/codex-usage.json +++ b/grafana/dashboards/codex-usage.json @@ -58,8 +58,8 @@ { "refId": "A", "queryType": "table", - "queryText": "SELECT count(*) AS reviews FROM ai_usage_events WHERE feature = 'ai_review_pr' AND model LIKE 'codex%'", - "rawQueryText": "SELECT count(*) AS reviews FROM ai_usage_events WHERE feature = 'ai_review_pr' AND model LIKE 'codex%'" + "queryText": "SELECT count(*) AS reviews FROM ai_usage_events WHERE feature = 'ai_review_pr' AND (('+' || model || '+') LIKE '%+codex+%' OR ('+' || model || '+') LIKE '%+codex:%')", + "rawQueryText": "SELECT count(*) AS reviews FROM ai_usage_events WHERE feature = 'ai_review_pr' AND (('+' || model || '+') LIKE '%+codex+%' OR ('+' || model || '+') LIKE '%+codex:%')" } ] }, @@ -125,8 +125,8 @@ "refId": "A", "queryType": "time series", "timeColumns": ["time"], - "queryText": "SELECT date(created_at) AS time, sum(estimated_neurons) AS estimated_neurons FROM ai_usage_events WHERE feature = 'ai_review_pr' AND model LIKE 'codex%' GROUP BY date(created_at) ORDER BY time", - "rawQueryText": "SELECT date(created_at) AS time, sum(estimated_neurons) AS estimated_neurons FROM ai_usage_events WHERE feature = 'ai_review_pr' AND model LIKE 'codex%' GROUP BY date(created_at) ORDER BY time" + "queryText": "SELECT date(created_at) AS time, sum(estimated_neurons) AS estimated_neurons FROM ai_usage_events WHERE feature = 'ai_review_pr' AND (('+' || model || '+') LIKE '%+codex+%' OR ('+' || model || '+') LIKE '%+codex:%') GROUP BY date(created_at) ORDER BY time", + "rawQueryText": "SELECT date(created_at) AS time, sum(estimated_neurons) AS estimated_neurons FROM ai_usage_events WHERE feature = 'ai_review_pr' AND (('+' || model || '+') LIKE '%+codex+%' OR ('+' || model || '+') LIKE '%+codex:%') GROUP BY date(created_at) ORDER BY time" } ] }, @@ -142,8 +142,8 @@ { "refId": "A", "queryType": "table", - "queryText": "SELECT status, count(*) AS count FROM ai_usage_events WHERE feature = 'ai_review_pr' AND model LIKE 'codex%' GROUP BY status ORDER BY count DESC", - "rawQueryText": "SELECT status, count(*) AS count FROM ai_usage_events WHERE feature = 'ai_review_pr' AND model LIKE 'codex%' GROUP BY status ORDER BY count DESC" + "queryText": "SELECT status, count(*) AS count FROM ai_usage_events WHERE feature = 'ai_review_pr' AND (('+' || model || '+') LIKE '%+codex+%' OR ('+' || model || '+') LIKE '%+codex:%') GROUP BY status ORDER BY count DESC", + "rawQueryText": "SELECT status, count(*) AS count FROM ai_usage_events WHERE feature = 'ai_review_pr' AND (('+' || model || '+') LIKE '%+codex+%' OR ('+' || model || '+') LIKE '%+codex:%') GROUP BY status ORDER BY count DESC" } ] }, @@ -159,8 +159,8 @@ { "refId": "A", "queryType": "table", - "queryText": "SELECT created_at, model, status, estimated_neurons, detail, json_extract(metadata_json, '$.repoFullName') AS repo, json_extract(metadata_json, '$.pullNumber') AS pr FROM ai_usage_events WHERE feature = 'ai_review_pr' AND model LIKE 'codex%' ORDER BY created_at DESC LIMIT 100", - "rawQueryText": "SELECT created_at, model, status, estimated_neurons, detail, json_extract(metadata_json, '$.repoFullName') AS repo, json_extract(metadata_json, '$.pullNumber') AS pr FROM ai_usage_events WHERE feature = 'ai_review_pr' AND model LIKE 'codex%' ORDER BY created_at DESC LIMIT 100" + "queryText": "SELECT created_at, model, status, estimated_neurons, detail, json_extract(metadata_json, '$.repoFullName') AS repo, json_extract(metadata_json, '$.pullNumber') AS pr FROM ai_usage_events WHERE feature = 'ai_review_pr' AND (('+' || model || '+') LIKE '%+codex+%' OR ('+' || model || '+') LIKE '%+codex:%') ORDER BY created_at DESC LIMIT 100", + "rawQueryText": "SELECT created_at, model, status, estimated_neurons, detail, json_extract(metadata_json, '$.repoFullName') AS repo, json_extract(metadata_json, '$.pullNumber') AS pr FROM ai_usage_events WHERE feature = 'ai_review_pr' AND (('+' || model || '+') LIKE '%+codex+%' OR ('+' || model || '+') LIKE '%+codex:%') ORDER BY created_at DESC LIMIT 100" } ] } From 463026ad3987cba307d87d444499df813ed3d222 Mon Sep 17 00:00:00 2001 From: JSONbored <49853598+JSONbored@users.noreply.github.com> Date: Sun, 28 Jun 2026 07:46:03 -0700 Subject: [PATCH 08/68] fix(selfhost): default openai reviewer to openai model --- .env.example | 2 +- docs/self-host/ai-providers.md | 2 +- src/selfhost/ai.ts | 26 ++++++++++++++++++++++++-- test/unit/selfhost-ai.test.ts | 11 +++++++++++ 4 files changed, 37 insertions(+), 4 deletions(-) diff --git a/.env.example b/.env.example index 3e78bc09e3..8fed437124 100644 --- a/.env.example +++ b/.env.example @@ -229,7 +229,7 @@ GITTENSORY_REVIEW_DRAFT=false # OPENAI_COMPATIBLE_AI_MODEL=llama3.1 # # OpenAI API reviewer (AI_PROVIDER=openai). Defaults: OPENAI_AI_BASE_URL=https://api.openai.com/v1, -# OPENAI_AI_MODEL=llama3.1 unless set here. +# OPENAI_AI_MODEL=gpt-5.5 unless set here. # OPENAI_API_KEY= # OPENAI_AI_BASE_URL=https://api.openai.com/v1 # OPENAI_AI_MODEL=gpt-5.5 diff --git a/docs/self-host/ai-providers.md b/docs/self-host/ai-providers.md index 46f55c1984..bcbdbb8c17 100644 --- a/docs/self-host/ai-providers.md +++ b/docs/self-host/ai-providers.md @@ -31,7 +31,7 @@ combined per `AI_COMBINE` (`single`/`consensus`/`synthesis`). | `CODEX_AI_TIMEOUT_MS` | scales with `CODEX_AI_EFFORT` | Unset -> low/med 120s, high 240s, xhigh 360s. Override clamped 30s-30min. | | `OLLAMA_AI_MODEL` | `llama3.1` | Used only by `AI_PROVIDER=ollama`. | | `OPENAI_COMPATIBLE_AI_MODEL` | `llama3.1` | Used only by `AI_PROVIDER=openai-compatible`. | -| `OPENAI_AI_MODEL` | `llama3.1` | Used only by `AI_PROVIDER=openai`; set to a real OpenAI model for API-backed reviews. | +| `OPENAI_AI_MODEL` | `gpt-5.5` | Used only by `AI_PROVIDER=openai`; override for a different OpenAI model. | | `ANTHROPIC_AI_MODEL` | `claude-sonnet-4-6` | Used only by `AI_PROVIDER=anthropic`. | ## Codex subscription reviewer diff --git a/src/selfhost/ai.ts b/src/selfhost/ai.ts index 62ebd53c6a..90fcdd9a37 100644 --- a/src/selfhost/ai.ts +++ b/src/selfhost/ai.ts @@ -59,6 +59,16 @@ function configuredOpenAiCompatibleModel(name: string, env: Record } /** OpenAI-compatible endpoint (Ollama's /v1, OpenAI, vLLM, LM Studio, …) — chat + embeddings. */ -export function createOpenAiCompatibleAi(opts: { baseUrl: string; apiKey?: string | undefined; model?: string | undefined; embedModel?: string | undefined }): SelfHostAi { +export function createOpenAiCompatibleAi(opts: { + baseUrl: string; + apiKey?: string | undefined; + model?: string | undefined; + defaultModel?: string | undefined; + embedModel?: string | undefined; +}): SelfHostAi { const base = opts.baseUrl.replace(/\/+$/, ""); const headers = (): Record => ({ "content-type": "application/json", ...(opts.apiKey ? { authorization: `Bearer ${opts.apiKey}` } : {}) }); return { @@ -120,7 +136,12 @@ export function createOpenAiCompatibleAi(opts: { baseUrl: string; apiKey?: strin const res = await fetch(`${base}/chat/completions`, { method: "POST", headers: headers(), - body: JSON.stringify({ model: resolveModel(opts.model, model, "llama3.1"), messages: toMessages(options), max_tokens: options.max_tokens, temperature: options.temperature }), + body: JSON.stringify({ + model: resolveModel(opts.model, model, opts.defaultModel ?? DEFAULT_OPENAI_COMPATIBLE_CHAT_MODEL), + messages: toMessages(options), + max_tokens: options.max_tokens, + temperature: options.temperature, + }), signal: AbortSignal.timeout(120_000), }); if (!res.ok) throw new Error(`ai_http_${res.status}`); @@ -576,6 +597,7 @@ export function buildProvider(name: string, env: Record { + let sentModel = ""; + vi.stubGlobal("fetch", vi.fn(async (_u: string, init: { body: string }) => { + sentModel = (JSON.parse(init.body) as { model: string }).model; + return new Response(JSON.stringify({ choices: [{ message: { content: "ok" } }] }), { status: 200 }); + })); + const ai = createSelfHostAi({ AI_PROVIDER: "openai", OPENAI_API_KEY: "sk-test" }); + await ai?.run("openai", { prompt: "x" }); + expect(sentModel).toBe("gpt-5.5"); + }); }); describe("resolveProviderNames + resolveAiReviewerPlan (#dual-ai-combiner)", () => { From 601ab7ae54092bc28188f35d1580d14fd607b310 Mon Sep 17 00:00:00 2001 From: JSONbored <49853598+JSONbored@users.noreply.github.com> Date: Sun, 28 Jun 2026 07:55:46 -0700 Subject: [PATCH 09/68] fix(signals): keep maintainer work out of contributor lane holds --- src/signals/engine.ts | 14 ++++++++------ test/unit/signals-coverage.test.ts | 23 +++++++++++++++++++++++ 2 files changed, 31 insertions(+), 6 deletions(-) diff --git a/src/signals/engine.ts b/src/signals/engine.ts index eddf1a29f7..ecb9b138d7 100644 --- a/src/signals/engine.ts +++ b/src/signals/engine.ts @@ -2465,13 +2465,15 @@ export function buildPreflightResult( }), ); const findings: SignalFinding[] = []; - if (lane.lane === "unknown" || lane.lane === "inactive") { + const laneUnavailable = lane.lane === "unknown" || lane.lane === "inactive"; + const maintainerAuthored = isMaintainerAssociation(input.authorAssociation); + if (laneUnavailable) { findings.push({ code: "lane_not_recommended", - severity: "warning", - title: "Repo lane is not ready for a confident recommendation", - detail: lane.summary, - action: "Refresh registry data or choose a registered active repo.", + severity: maintainerAuthored ? "info" : "warning", + title: maintainerAuthored ? "Repo lane unavailable for contributor scoring" : "Repo lane is not ready for a confident recommendation", + detail: maintainerAuthored ? `${lane.summary} Maintainer-authored work is treated as repo stewardship, not contributor-lane eligibility.` : lane.summary, + action: maintainerAuthored ? "No action." : "Refresh registry data or choose a registered active repo.", }); } if (linkedIssues.length === 0 && lane.lane !== "issue_discovery") { @@ -2534,7 +2536,7 @@ export function buildPreflightResult( return { repoFullName: input.repoFullName, generatedAt: nowIso(), - status: lane.lane === "unknown" || lane.lane === "inactive" ? "hold" : hasWarning ? "needs_work" : "ready", + status: laneUnavailable && !maintainerAuthored ? "hold" : hasWarning ? "needs_work" : "ready", lane, reviewBurden, linkedIssues, diff --git a/test/unit/signals-coverage.test.ts b/test/unit/signals-coverage.test.ts index ca9598329a..ed3544e93f 100644 --- a/test/unit/signals-coverage.test.ts +++ b/test/unit/signals-coverage.test.ts @@ -248,12 +248,35 @@ describe("signal coverage edge cases", () => { [], [], ); + const missingRepo = { ...directRepo, isRegistered: false, registryConfig: null }; + const outsideUnknownLane = buildPreflightResult( + { repoFullName: missingRepo.fullName, title: "Fix cache invalidation", body: "Fixes #1", linkedIssues: [1], authorAssociation: "CONTRIBUTOR" }, + missingRepo, + [issue(missingRepo.fullName, 1, "Cache invalidation")], + [], + ); + const ownerUnknownLane = buildPreflightResult( + { repoFullName: missingRepo.fullName, title: "Fix cache invalidation", body: "Fixes #1", linkedIssues: [1], authorAssociation: "OWNER" }, + missingRepo, + [issue(missingRepo.fullName, 1, "Cache invalidation")], + [], + ); expect(cleanPacket.pullRequestPackets[0]).toMatchObject({ reviewPriority: "review", reasons: ["No obvious queue hygiene issue detected in cached metadata."] }); expect(cleanPacket.suggestedActions).toEqual(["Queue looks manageable from cached Gittensory signals."]); expect(local.status).toBe("ready"); expect(local.localDiff).toMatchObject({ codeFileCount: 1, testFileCount: 1, inferredLinkedIssues: [1] }); expect(directNoIssue.findings.map((finding) => finding.code)).toContain("missing_linked_issue"); + expect(outsideUnknownLane).toMatchObject({ status: "hold" }); + expect(outsideUnknownLane.findings.find((finding) => finding.code === "lane_not_recommended")).toMatchObject({ + severity: "warning", + action: "Refresh registry data or choose a registered active repo.", + }); + expect(ownerUnknownLane).toMatchObject({ status: "ready" }); + expect(ownerUnknownLane.findings.find((finding) => finding.code === "lane_not_recommended")).toMatchObject({ + severity: "info", + action: "No action.", + }); }); it("covers issue quality, burden, bounties, noise, and reviewability edge decisions", () => { From 29124590ba415dd6850cf8f89d5e81cccd6058d8 Mon Sep 17 00:00:00 2001 From: JSONbored <49853598+JSONbored@users.noreply.github.com> Date: Sun, 28 Jun 2026 08:02:38 -0700 Subject: [PATCH 10/68] ci(selfhost): verify release source map context --- .github/workflows/release-selfhost.yml | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/.github/workflows/release-selfhost.yml b/.github/workflows/release-selfhost.yml index aba01dcbc3..821312c999 100644 --- a/.github/workflows/release-selfhost.yml +++ b/.github/workflows/release-selfhost.yml @@ -83,6 +83,23 @@ jobs: - name: Build self-host bundle for release run: SELFHOST_SOURCEMAP=1 SELFHOST_BUNDLE_ALL=1 node scripts/build-selfhost.mjs --all --sourcemap + - name: Verify release source maps are in the image context + run: | + test -s dist/server.mjs + test -s dist/server.mjs.map + grep -Fq 'sourceMappingURL=server.mjs.map' dist/server.mjs + node --input-type=module <<'NODE' + import { readFileSync } from "node:fs"; + + const sourceMap = JSON.parse(readFileSync("dist/server.mjs.map", "utf8")); + if (sourceMap.sourceRoot !== "/app/dist") { + throw new Error(`Expected sourceRoot /app/dist, got ${sourceMap.sourceRoot ?? ""}`); + } + if (!Array.isArray(sourceMap.sources) || sourceMap.sources.length === 0) { + throw new Error("Expected release source map to include original sources"); + } + NODE + - name: Detect Sentry release token id: sentry env: From e1e4e322b52db2ef74baa8c12d86b8083435b65a Mon Sep 17 00:00:00 2001 From: JSONbored <49853598+JSONbored@users.noreply.github.com> Date: Sun, 28 Jun 2026 08:17:59 -0700 Subject: [PATCH 11/68] fix(observability): harden reporting export swap --- scripts/export-grafana-reporting-db.sh | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/scripts/export-grafana-reporting-db.sh b/scripts/export-grafana-reporting-db.sh index ab26563cba..39b854e353 100644 --- a/scripts/export-grafana-reporting-db.sh +++ b/scripts/export-grafana-reporting-db.sh @@ -6,6 +6,10 @@ OUT_DIR="${GITTENSORY_REPORTING_DIR:-/reporting}" OUT_DB="${GITTENSORY_REPORTING_DB:-$OUT_DIR/gittensory-reporting.sqlite}" TMP_DB="${OUT_DB}.tmp" +sql_string() { + printf "%s" "$1" | sed "s/'/''/g" +} + mkdir -p "$OUT_DIR" if [ ! -s "$APP_DB" ]; then @@ -14,6 +18,7 @@ if [ ! -s "$APP_DB" ]; then fi rm -f "$TMP_DB" "$TMP_DB-wal" "$TMP_DB-shm" +TMP_DB_SQL="$(sql_string "$TMP_DB")" sqlite3 "$TMP_DB" <<'SQL' PRAGMA synchronous=NORMAL; @@ -48,7 +53,7 @@ SQL if sqlite3 "$APP_DB" "SELECT 1 FROM sqlite_master WHERE type='table' AND name='review_targets' LIMIT 1" | grep -q 1; then sqlite3 "$APP_DB" </dev/null echo "reporting export complete: $OUT_DB" From 1284cb7ace04e6b831fa4b5ff6aa2dc75a602002 Mon Sep 17 00:00:00 2001 From: JSONbored <49853598+JSONbored@users.noreply.github.com> Date: Sun, 28 Jun 2026 08:26:02 -0700 Subject: [PATCH 12/68] fix(observability): keep reporting exporter healthy --- .env.example | 1 + docker-compose.yml | 13 ++++++++++++- docs/self-host/configuration.md | 1 + docs/self-hosting.md | 6 +++++- scripts/export-grafana-reporting-db.sh | 13 ++++++++----- 5 files changed, 27 insertions(+), 7 deletions(-) diff --git a/.env.example b/.env.example index 8fed437124..c10d3d6bfb 100644 --- a/.env.example +++ b/.env.example @@ -179,6 +179,7 @@ GITTENSORY_REVIEW_DRAFT=false # • "Claude usage (OTEL)" — cost/tokens/model/effort from the review CLI's OpenTelemetry export (see below). # • "Resource hub" — links to every integrated service. # GRAFANA_REPORTING_EXPORT_INTERVAL_SECONDS=30 # refresh cadence for the redacted reporting SQLite export +# GITTENSORY_REPORTING_SOURCE_DB=/appdb/gittensory.sqlite # if DATABASE_PATH=/data/custom.sqlite, set /appdb/custom.sqlite # # Claude usage telemetry → OTEL collector → Prometheus → the Claude usage dashboard. OFF by default. # CLAUDE_CODE_ENABLE_TELEMETRY=1 # enable; needs --profile observability (starts the otel-collector) diff --git a/docker-compose.yml b/docker-compose.yml index 26e7d27568..db0ae8c323 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -316,12 +316,23 @@ services: - gittensory-data:/appdb - grafana-reporting-data:/reporting - ./scripts/export-grafana-reporting-db.sh:/export-grafana-reporting-db.sh:ro + environment: + # Default SQLite app DB path maps app /data/gittensory.sqlite to exporter /appdb/gittensory.sqlite. + # If you override DATABASE_PATH, set this to the matching /appdb/ path. DATABASE_URL/Postgres + # deployments export an empty dashboard-safe DB so Grafana can still start until a SQL exporter is added. + GITTENSORY_REPORTING_SOURCE_DB: "${GITTENSORY_REPORTING_SOURCE_DB:-/appdb/gittensory.sqlite}" + GITTENSORY_REPORTING_DIR: /reporting + GITTENSORY_REPORTING_DB: "${GITTENSORY_REPORTING_DB:-/reporting/gittensory-reporting.sqlite}" command: - /bin/sh - -c - "apk add --no-cache sqlite >/dev/null 2>&1 && while true; do sh /export-grafana-reporting-db.sh || echo '[reporting] export failed'; sleep ${GRAFANA_REPORTING_EXPORT_INTERVAL_SECONDS:-30}; done" healthcheck: - test: ["CMD-SHELL", "test -s /reporting/gittensory-reporting.sqlite && sqlite3 /reporting/gittensory-reporting.sqlite 'PRAGMA quick_check;' | grep -q '^ok$'"] + test: + [ + "CMD-SHELL", + "db=\"$${GITTENSORY_REPORTING_DB:-/reporting/gittensory-reporting.sqlite}\"; test -s \"$$db\" && sqlite3 \"$$db\" 'PRAGMA quick_check;' | grep -q '^ok$'", + ] interval: 30s timeout: 5s start_period: 30s diff --git a/docs/self-host/configuration.md b/docs/self-host/configuration.md index 7c012206b8..f42239b1d4 100644 --- a/docs/self-host/configuration.md +++ b/docs/self-host/configuration.md @@ -86,6 +86,7 @@ See [ai-providers.md](./ai-providers.md) for the full provider/model/effort/time | `CLAUDE_CODE_OAUTH_TOKEN` | Claude Code subscription token (`claude setup-token`) | | `AI_EMBED_BASE_URL` / `_MODEL` / `_PROVIDER` | Dedicated RAG embed provider | | `GITTENSORY_REPO_CONFIG_DIR` | Container-private per-repo config dir | +| `GITTENSORY_REPORTING_SOURCE_DB` | Optional reporting-exporter source path for non-default SQLite `DATABASE_PATH` | ## Sentry environment variables diff --git a/docs/self-hosting.md b/docs/self-hosting.md index 484798c4c0..7069361fa0 100644 --- a/docs/self-hosting.md +++ b/docs/self-hosting.md @@ -253,7 +253,11 @@ content-lane are not yet per-repo toggleable and stay on the allowlist.) - **Maintainer Grafana dashboards.** Grafana does **not** mount the live app database. The observability profile starts `reporting-exporter`, which copies only the dashboard-safe `review_targets` and `ai_usage_events` columns into `/reporting/gittensory-reporting.sqlite` every `GRAFANA_REPORTING_EXPORT_INTERVAL_SECONDS` seconds - (default 30). The SQLite datasource points at that redacted reporting DB. + (default 30). The SQLite datasource points at that redacted reporting DB. If you override the app SQLite + `DATABASE_PATH`, set `GITTENSORY_REPORTING_SOURCE_DB` to the matching exporter mount path, for example + `/appdb/custom.sqlite` for `DATABASE_PATH=/data/custom.sqlite`. `DATABASE_URL`/Postgres deployments currently + export an empty dashboard-safe DB so Grafana can start; Postgres-backed maintainer analytics need a dedicated SQL + exporter. - **App-level metrics.** Enable `GITTENSORY_REVIEW_OPS=true` for the read-only gate-block anomaly scan and the bearer-gated `GET /v1/internal/ops/stats` aggregate. diff --git a/scripts/export-grafana-reporting-db.sh b/scripts/export-grafana-reporting-db.sh index 39b854e353..fc8a78ed32 100644 --- a/scripts/export-grafana-reporting-db.sh +++ b/scripts/export-grafana-reporting-db.sh @@ -12,11 +12,6 @@ sql_string() { mkdir -p "$OUT_DIR" -if [ ! -s "$APP_DB" ]; then - echo "reporting export skipped: source database missing at $APP_DB" >&2 - exit 0 -fi - rm -f "$TMP_DB" "$TMP_DB-wal" "$TMP_DB-shm" TMP_DB_SQL="$(sql_string "$TMP_DB")" @@ -50,6 +45,14 @@ CREATE INDEX ai_usage_events_feature_created_idx ON ai_usage_events(feature, cre CREATE INDEX ai_usage_events_model_created_idx ON ai_usage_events(model, created_at); SQL +if [ ! -s "$APP_DB" ]; then + sqlite3 "$TMP_DB" "PRAGMA quick_check;" | grep -qx "ok" + mv "$TMP_DB" "$OUT_DB" + rm -f "$TMP_DB-wal" "$TMP_DB-shm" + echo "reporting export empty: source database missing at $APP_DB" >&2 + exit 0 +fi + if sqlite3 "$APP_DB" "SELECT 1 FROM sqlite_master WHERE type='table' AND name='review_targets' LIMIT 1" | grep -q 1; then sqlite3 "$APP_DB" < Date: Sun, 28 Jun 2026 08:31:27 -0700 Subject: [PATCH 13/68] ci(selfhost): cover visual release image --- .github/workflows/selfhost.yml | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/.github/workflows/selfhost.yml b/.github/workflows/selfhost.yml index aa791e38a4..55d2e4162a 100644 --- a/.github/workflows/selfhost.yml +++ b/.github/workflows/selfhost.yml @@ -80,6 +80,17 @@ jobs: --load \ -t gittensory:selfhost-ci . + - name: Build release target with visual review deps + run: | + docker buildx build \ + --target runtime-prebuilt \ + --build-context selfhost_dist=./dist \ + --build-arg INSTALL_VISUAL_REVIEW=true \ + --load \ + -t gittensory:selfhost-prebuilt-visual-ci . + docker run --rm --entrypoint node gittensory:selfhost-prebuilt-visual-ci \ + -e "import('puppeteer-core').then(() => console.log('puppeteer-core ok'))" + - name: Boot the container + smoke-test /health, /ready, /metrics, migrations run: | docker run -d --name gt -p 8787:8787 gittensory:selfhost-ci From 90cc4671650667af2f6dd7f72afa4c583d763df1 Mon Sep 17 00:00:00 2001 From: JSONbored <49853598+JSONbored@users.noreply.github.com> Date: Sun, 28 Jun 2026 08:39:01 -0700 Subject: [PATCH 14/68] fix(observability): quote reporting exporter shell inputs --- docker-compose.yml | 10 +++++++++- scripts/export-grafana-reporting-db.sh | 10 ++++------ 2 files changed, 13 insertions(+), 7 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index db0ae8c323..76dc536b98 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -323,10 +323,18 @@ services: GITTENSORY_REPORTING_SOURCE_DB: "${GITTENSORY_REPORTING_SOURCE_DB:-/appdb/gittensory.sqlite}" GITTENSORY_REPORTING_DIR: /reporting GITTENSORY_REPORTING_DB: "${GITTENSORY_REPORTING_DB:-/reporting/gittensory-reporting.sqlite}" + GRAFANA_REPORTING_EXPORT_INTERVAL_SECONDS: "${GRAFANA_REPORTING_EXPORT_INTERVAL_SECONDS:-30}" command: - /bin/sh - -c - - "apk add --no-cache sqlite >/dev/null 2>&1 && while true; do sh /export-grafana-reporting-db.sh || echo '[reporting] export failed'; sleep ${GRAFANA_REPORTING_EXPORT_INTERVAL_SECONDS:-30}; done" + - >- + apk add --no-cache sqlite >/dev/null 2>&1 && + while true; do + sh /export-grafana-reporting-db.sh || echo '[reporting] export failed'; + interval="$${GRAFANA_REPORTING_EXPORT_INTERVAL_SECONDS:-30}"; + case "$$interval" in ''|*[!0-9]*) interval=30;; esac; + sleep "$$interval"; + done healthcheck: test: [ diff --git a/scripts/export-grafana-reporting-db.sh b/scripts/export-grafana-reporting-db.sh index fc8a78ed32..039850bdb0 100644 --- a/scripts/export-grafana-reporting-db.sh +++ b/scripts/export-grafana-reporting-db.sh @@ -54,8 +54,7 @@ if [ ! -s "$APP_DB" ]; then fi if sqlite3 "$APP_DB" "SELECT 1 FROM sqlite_master WHERE type='table' AND name='review_targets' LIMIT 1" | grep -q 1; then - sqlite3 "$APP_DB" < Date: Sun, 28 Jun 2026 09:03:47 -0700 Subject: [PATCH 15/68] fix(selfhost): reject ambiguous AI provider config Fail startup when deprecated shared AI config knobs are set, derive self-host AI usage labels from the resolved reviewer plan, require OpenAI credentials before adding that provider, and mount the reporting source data read-only. --- .env.example | 5 ++ docker-compose.yml | 2 +- docs/self-host/ai-providers.md | 4 + src/selfhost/ai-config.ts | 107 +++++++++++++++++++++++++++ src/selfhost/ai.ts | 10 +-- src/services/ai-review.ts | 29 ++------ test/unit/ai-review-advisory.test.ts | 14 +++- test/unit/selfhost-ai.test.ts | 15 +++- 8 files changed, 156 insertions(+), 30 deletions(-) create mode 100644 src/selfhost/ai-config.ts diff --git a/.env.example b/.env.example index c10d3d6bfb..c1acc7c9c6 100644 --- a/.env.example +++ b/.env.example @@ -206,6 +206,11 @@ GITTENSORY_REVIEW_DRAFT=false # --- AI review backend (optional; without it reviews run deterministically) --- # AI_SUMMARIES_ENABLED=true +# Deprecated shared AI_* knobs are intentionally rejected at startup: +# AI_BASE_URL, AI_API_KEY, AI_MODEL, AI_EFFORT, AI_TIMEOUT_MS. Use the explicit +# provider-specific variables below so Claude, Codex, Ollama, OpenAI, and +# Anthropic settings cannot be mixed up. +# # AI_PROVIDER=ollama # ollama | openai-compatible | openai | anthropic | claude-code | # # codex. A COMMA-LIST of TWO providers is a DUAL reviewer pair # # (e.g. "claude-code,codex") combined per AI_COMBINE below; for a diff --git a/docker-compose.yml b/docker-compose.yml index 76dc536b98..dd4c14bae4 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -313,7 +313,7 @@ services: condition: service_healthy volumes: # The exporter needs live DB read access to build the redacted snapshot. Grafana does not get this mount. - - gittensory-data:/appdb + - gittensory-data:/appdb:ro - grafana-reporting-data:/reporting - ./scripts/export-grafana-reporting-db.sh:/export-grafana-reporting-db.sh:ro environment: diff --git a/docs/self-host/ai-providers.md b/docs/self-host/ai-providers.md index bcbdbb8c17..744309be78 100644 --- a/docs/self-host/ai-providers.md +++ b/docs/self-host/ai-providers.md @@ -2,6 +2,10 @@ The reviewer is configured by `AI_PROVIDER`. Reviews degrade deterministically (no AI) if it's unset. +Deprecated shared knobs (`AI_BASE_URL`, `AI_API_KEY`, `AI_MODEL`, `AI_EFFORT`, `AI_TIMEOUT_MS`) fail startup with a +clear migration error. Use the provider-specific variables below so dual-review setups cannot accidentally reuse the +wrong model, base URL, key, effort, or timeout. + ## Providers | `AI_PROVIDER` | Backend | Needs | diff --git a/src/selfhost/ai-config.ts b/src/selfhost/ai-config.ts new file mode 100644 index 0000000000..6281e82b9d --- /dev/null +++ b/src/selfhost/ai-config.ts @@ -0,0 +1,107 @@ +const LEGACY_SHARED_AI_ENV = [ + "AI_BASE_URL", + "AI_API_KEY", + "AI_MODEL", + "AI_EFFORT", + "AI_TIMEOUT_MS", +] as const; + +const LEGACY_SHARED_AI_REPLACEMENTS = + "Use provider-specific settings instead: OLLAMA_AI_BASE_URL/OLLAMA_AI_MODEL/OLLAMA_AI_API_KEY, " + + "OPENAI_COMPATIBLE_AI_BASE_URL/OPENAI_COMPATIBLE_AI_MODEL/OPENAI_COMPATIBLE_AI_API_KEY, " + + "OPENAI_AI_BASE_URL/OPENAI_AI_MODEL/OPENAI_API_KEY, ANTHROPIC_AI_BASE_URL/ANTHROPIC_AI_MODEL/ANTHROPIC_API_KEY, " + + "CLAUDE_AI_MODEL/CLAUDE_AI_EFFORT/CLAUDE_AI_TIMEOUT_MS, or CODEX_AI_MODEL/CODEX_AI_EFFORT/CODEX_AI_TIMEOUT_MS."; + +export const SELF_HOST_REVIEWER_MODEL_ENV: Record = { + anthropic: "ANTHROPIC_AI_MODEL", + "claude-code": "CLAUDE_AI_MODEL", + codex: "CODEX_AI_MODEL", + ollama: "OLLAMA_AI_MODEL", + openai: "OPENAI_AI_MODEL", + "openai-compatible": "OPENAI_COMPATIBLE_AI_MODEL", +}; + +function configured( + env: Record, + key: string, +): boolean { + return env[key] !== undefined && env[key]?.trim() !== ""; +} + +export function assertNoLegacySharedAiEnv( + env: Record, +): void { + const legacy = LEGACY_SHARED_AI_ENV.filter((key) => configured(env, key)); + if (legacy.length === 0) return; + throw new Error( + `legacy_shared_ai_config_unsupported: ${legacy.join(", ")} are no longer supported. ${LEGACY_SHARED_AI_REPLACEMENTS}`, + ); +} + +function parseProviderNames(env: Record): string[] { + assertNoLegacySharedAiEnv(env); + return (env.AI_PROVIDER ?? "") + .split(",") + .map((s) => s.trim().toLowerCase()) + .filter(Boolean); +} + +export function isConfiguredSelfHostProvider( + name: string, + env: Record, +): boolean { + switch (name) { + case "anthropic": + return configured(env, "ANTHROPIC_API_KEY"); + case "claude-code": + case "codex": + case "ollama": + case "openai-compatible": + return true; + case "openai": + return configured(env, "OPENAI_API_KEY"); + default: + return false; + } +} + +export function resolveConfiguredProviderNames( + env: Record, +): string[] { + return parseProviderNames(env).filter((name) => + isConfiguredSelfHostProvider(name, env), + ); +} + +export function labelSelfHostReviewerModel( + model: string, + env: Record, +): string { + const trimmed = model.trim(); + const colon = trimmed.indexOf(":"); + const provider = ( + colon < 0 ? trimmed : trimmed.slice(0, colon) + ).toLowerCase(); + const modelEnv = SELF_HOST_REVIEWER_MODEL_ENV[provider]; + if (!modelEnv) return trimmed; + if (colon >= 0 && trimmed.slice(colon + 1).trim()) + return `${provider}:${trimmed.slice(colon + 1).trim()}`; + const configuredModel = env[modelEnv]?.trim(); + return configuredModel ? `${provider}:${configuredModel}` : provider; +} + +export function labelSelfHostReviewerModels( + reviewers: ReadonlyArray<{ model: string }>, + env: Record, +): string { + return reviewers + .map((reviewer) => labelSelfHostReviewerModel(reviewer.model, env)) + .join("+"); +} + +export function labelSelfHostReviewerNames( + names: readonly string[], + env: Record, +): string { + return names.map((name) => labelSelfHostReviewerModel(name, env)).join("+"); +} diff --git a/src/selfhost/ai.ts b/src/selfhost/ai.ts index 90fcdd9a37..1cfe7b065e 100644 --- a/src/selfhost/ai.ts +++ b/src/selfhost/ai.ts @@ -7,6 +7,8 @@ // records an error and degrades — never a silent wrong answer). import type { CombineStrategy, OnMerge } from "../services/ai-review"; +import { isConfiguredSelfHostProvider, resolveConfiguredProviderNames } from "./ai-config"; +export { assertNoLegacySharedAiEnv } from "./ai-config"; import { incr } from "./metrics"; interface AiRunOptions { @@ -584,6 +586,7 @@ export function createChainAi(providers: Array<{ name: string; ai: SelfHostAi }> /** Build one provider adapter by name. Provider config stays explicit so dual-provider setups cannot accidentally * reuse the wrong model/base/key across different backends. */ export function buildProvider(name: string, env: Record): SelfHostAi | undefined { + if (!isConfiguredSelfHostProvider(name, env)) return undefined; switch (name) { case "ollama": case "openai-compatible": @@ -639,17 +642,14 @@ export function routeProviders(providers: Array<{ name: string; ai: SelfHostAi } * order, lowercased. Shared by the adapter and the dual-review plan so they never disagree about which providers * exist (e.g. an uncredentialed entry can't become a "reviewer" the router would then miss). */ function buildProviders(env: Record): Array<{ name: string; ai: SelfHostAi }> { - return (env.AI_PROVIDER ?? "") - .split(",") - .map((s) => s.trim().toLowerCase()) - .filter(Boolean) + return resolveConfiguredProviderNames(env) .map((name) => ({ name, ai: buildProvider(name, env) })) .filter((p): p is { name: string; ai: SelfHostAi } => Boolean(p.ai)); } /** The credentialed self-host provider names from AI_PROVIDER, in order. Empty when unconfigured. */ export function resolveProviderNames(env: Record): string[] { - return buildProviders(env).map((p) => p.name); + return resolveConfiguredProviderNames(env); } /** CLI-subscription providers need their binary present on PATH; keep boot preflight parsing identical to AI_PROVIDER. */ diff --git a/src/services/ai-review.ts b/src/services/ai-review.ts index 16e6ef7c44..806590e0df 100644 --- a/src/services/ai-review.ts +++ b/src/services/ai-review.ts @@ -24,6 +24,7 @@ import { import { sanitizePublicComment } from "../queue-intelligence"; import { defangReviewInput } from "../review/safety"; import { convergedFeatureActive } from "../review/feature-activation"; +import { labelSelfHostReviewerModels, labelSelfHostReviewerNames, resolveConfiguredProviderNames } from "../selfhost/ai-config"; import { errorMessage } from "../utils/json"; import type { ReviewProfile } from "../signals/focus-manifest"; @@ -1117,30 +1118,16 @@ export async function runGittensoryAiReview( }; } -const SELF_HOST_REVIEWER_MODEL_ENV: Record = { - anthropic: "ANTHROPIC_AI_MODEL", - "claude-code": "CLAUDE_AI_MODEL", - codex: "CODEX_AI_MODEL", - ollama: "OLLAMA_AI_MODEL", - openai: "OPENAI_AI_MODEL", - "openai-compatible": "OPENAI_COMPATIBLE_AI_MODEL", -}; - /** The actual configured reviewer label for usage attribution (#1566): the self-host provider plus its explicit * provider-specific model when set, else the Worker dual-AI models. Without this, self-host claude-code reviews * were mis-logged as the Workers-AI model ids (`@cf/openai/gpt-oss-120b+...`), which hid outages. */ -function reviewerModelLabel(env: Env): string { +function reviewerModelLabel(env: Env, input: GittensoryAiReviewInput): string { const e = env as unknown as Record; - if (!e.AI_PROVIDER) return BEST_REVIEW_MODELS.join("+"); - return e.AI_PROVIDER.split(",") - .map((provider) => provider.trim().toLowerCase()) - .filter(Boolean) - .map((provider) => { - const modelEnv = SELF_HOST_REVIEWER_MODEL_ENV[provider]; - const model = modelEnv ? e[modelEnv]?.trim() : undefined; - return model ? `${provider}:${model}` : provider; - }) - .join("+"); + const reviewers = (input.reviewers?.length ? input.reviewers : env.AI_REVIEW_PLAN?.reviewers) ?? null; + if (reviewers?.length) return labelSelfHostReviewerModels(reviewers, e); + const providers = resolveConfiguredProviderNames(e); + if (providers.length > 0) return labelSelfHostReviewerNames(providers, e); + return BEST_REVIEW_MODELS.join("+"); } async function record( @@ -1158,7 +1145,7 @@ async function record( route: "github_app.ai_review", model: input.providerKey ? `byok:${input.providerKey.provider}` - : reviewerModelLabel(env), + : reviewerModelLabel(env, input), status, estimatedNeurons, detail, diff --git a/test/unit/ai-review-advisory.test.ts b/test/unit/ai-review-advisory.test.ts index 0adfadc77e..e1d356ef7f 100644 --- a/test/unit/ai-review-advisory.test.ts +++ b/test/unit/ai-review-advisory.test.ts @@ -162,7 +162,19 @@ describe("runAiReviewForAdvisory", () => { const result = await runAiReviewForAdvisory(env, { settings: { aiReviewMode: "advisory" } as RepositorySettings, advisory: advisory(), repoFullName: "acme/widgets", pr, author: "alice", confirmedContributor: true }); expect(result).toBeUndefined(); const usage = await env.DB.prepare("SELECT model FROM ai_usage_events WHERE feature = 'ai_review_pr' ORDER BY created_at DESC LIMIT 1").first<{ model: string }>(); - expect(usage?.model).toBe("codex+unknown-provider"); + expect(usage?.model).toBe("codex"); + }); + + it("records the resolved self-host reviewer plan instead of raw AI_PROVIDER entries", async () => { + const env = aiEnv(async () => { throw new Error("ollama unavailable"); }); + Object.assign(env as unknown as Record, { + AI_PROVIDER: "anthropic,ollama", + AI_REVIEW_PLAN: { reviewers: [{ model: "ollama" }], combine: "single" }, + }); + const result = await runAiReviewForAdvisory(env, { settings: { aiReviewMode: "advisory" } as RepositorySettings, advisory: advisory(), repoFullName: "acme/widgets", pr, author: "alice", confirmedContributor: true }); + expect(result).toBeUndefined(); + const usage = await env.DB.prepare("SELECT model FROM ai_usage_events WHERE feature = 'ai_review_pr' ORDER BY created_at DESC LIMIT 1").first<{ model: string }>(); + expect(usage?.model).toBe("ollama"); }); it("no-ops for a non-confirmed contributor under the gittensor pack and when there is no head SHA", async () => { diff --git a/test/unit/selfhost-ai.test.ts b/test/unit/selfhost-ai.test.ts index 0f4a557699..10abcc24b7 100644 --- a/test/unit/selfhost-ai.test.ts +++ b/test/unit/selfhost-ai.test.ts @@ -2,7 +2,8 @@ import { chmodSync, mkdtempSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { afterEach, describe, expect, it, vi } from "vitest"; -import { buildProvider, claudeErrorStatus, createAnthropicAi, createChainAi, createClaudeCodeAi, createCodexAi, createOpenAiCompatibleAi, createSelfHostAi, extractCliText, extractCliUsage, resolveAiReviewerPlan, resolveClaudeCliTimeoutMs, resolveCodexCliTimeoutMs, resolveCodexEffort, resolveEffort, resolveModel, resolveProviderNames, resolveRequiredCliProviders, redactSecrets, routeProviders, subscriptionCliEnv } from "../../src/selfhost/ai"; +import { assertNoLegacySharedAiEnv, buildProvider, claudeErrorStatus, createAnthropicAi, createChainAi, createClaudeCodeAi, createCodexAi, createOpenAiCompatibleAi, createSelfHostAi, extractCliText, extractCliUsage, resolveAiReviewerPlan, resolveClaudeCliTimeoutMs, resolveCodexCliTimeoutMs, resolveCodexEffort, resolveEffort, resolveModel, resolveProviderNames, resolveRequiredCliProviders, redactSecrets, routeProviders, subscriptionCliEnv } from "../../src/selfhost/ai"; +import { labelSelfHostReviewerModel } from "../../src/selfhost/ai-config"; import { renderMetrics, resetMetrics } from "../../src/selfhost/metrics"; describe("resolveModel (#979 — never leak the Workers-AI default to a self-host backend)", () => { @@ -145,6 +146,10 @@ describe("createSelfHostAi — provider selection", () => { // "anthropic,ollama" with a key → both build → a chain (a runnable adapter) expect(typeof createSelfHostAi({ AI_PROVIDER: "anthropic,ollama", ANTHROPIC_API_KEY: "sk-ant" })?.run).toBe("function"); }); + it("fails loudly when deprecated shared AI env knobs are present", () => { + expect(() => assertNoLegacySharedAiEnv({ AI_PROVIDER: "ollama", AI_BASE_URL: "http://ollama:11434/v1", AI_MODEL: "llama3.1" })).toThrow(/legacy_shared_ai_config_unsupported: AI_BASE_URL, AI_MODEL/); + expect(() => createSelfHostAi({ AI_PROVIDER: "ollama", AI_EFFORT: "high" })).toThrow(/CLAUDE_AI_EFFORT\/CLAUDE_AI_TIMEOUT_MS/); + }); }); describe("createAnthropicAi (#979 native BYOK)", () => { @@ -254,6 +259,7 @@ describe("resolveProviderNames + resolveAiReviewerPlan (#dual-ai-combiner)", () expect(resolveProviderNames({ AI_PROVIDER: " Claude-Code , CODEX " })).toEqual(["claude-code", "codex"]); // CLI providers always credentialed expect(resolveProviderNames({ AI_PROVIDER: "anthropic,ollama" })).toEqual(["ollama"]); // anthropic dropped (no key); ollama needs none expect(resolveProviderNames({ AI_PROVIDER: "anthropic,ollama", ANTHROPIC_API_KEY: "sk-ant" })).toEqual(["anthropic", "ollama"]); + expect(resolveProviderNames({ AI_PROVIDER: "openai,ollama" })).toEqual(["ollama"]); // openai requires OPENAI_API_KEY }); it("resolveRequiredCliProviders mirrors comma-list AI_PROVIDER parsing for boot preflight", () => { @@ -280,6 +286,10 @@ describe("resolveProviderNames + resolveAiReviewerPlan (#dual-ai-combiner)", () expect(resolveAiReviewerPlan({ AI_PROVIDER: "claude-code,codex", AI_COMBINE: "garbage", AI_ON_MERGE: "nonsense" })).toMatchObject({ combine: "synthesis", onMerge: undefined }); // invalid → defaults expect(resolveAiReviewerPlan({ AI_PROVIDER: "claude-code,codex,ollama" })?.reviewers).toEqual([{ model: "claude-code" }, { model: "codex" }]); // first two }); + + it("labels explicit provider:model reviewer ids without consulting env defaults", () => { + expect(labelSelfHostReviewerModel(" CODEX:gpt-5.5 ", { CODEX_AI_MODEL: "ignored" })).toBe("codex:gpt-5.5"); + }); }); describe("branch coverage — defaults + edge inputs", () => { @@ -358,7 +368,8 @@ describe("branch coverage — defaults + edge inputs", () => { ]); }); it("buildProvider uses provider-specific default base URLs when provider base URLs are unset", () => { - expect(typeof buildProvider("openai", {})?.run).toBe("function"); // defaults to https://api.openai.com/v1 + expect(buildProvider("openai", {})).toBeUndefined(); // openai is credentialed and requires OPENAI_API_KEY + expect(typeof buildProvider("openai", { OPENAI_API_KEY: "sk-test" })?.run).toBe("function"); // defaults to https://api.openai.com/v1 expect(typeof buildProvider("ollama", {})?.run).toBe("function"); // defaults to http://localhost:11434/v1 expect(typeof buildProvider("openai-compatible", {})?.run).toBe("function"); // defaults to http://localhost:11434/v1 }); From 6b50fedd64fe4d37e02e6b1e8b0dbb0bdc961bb1 Mon Sep 17 00:00:00 2001 From: JSONbored <49853598+JSONbored@users.noreply.github.com> Date: Sun, 28 Jun 2026 09:12:09 -0700 Subject: [PATCH 16/68] fix(observability): tolerate reporting usage schema drift Keep the Grafana reporting export dashboard-safe when older source databases do not have the AI usage estimate column, and cover current plus missing-column exports in unit tests. --- scripts/export-grafana-reporting-db.sh | 11 ++- test/unit/selfhost-grafana-reporting.test.ts | 86 ++++++++++++++++++++ 2 files changed, 96 insertions(+), 1 deletion(-) create mode 100644 test/unit/selfhost-grafana-reporting.test.ts diff --git a/scripts/export-grafana-reporting-db.sh b/scripts/export-grafana-reporting-db.sh index 039850bdb0..d0a4679404 100644 --- a/scripts/export-grafana-reporting-db.sh +++ b/scripts/export-grafana-reporting-db.sh @@ -10,6 +10,10 @@ sql_string() { printf "%s" "$1" | sed "s/'/''/g" } +source_column_exists() { + sqlite3 "$APP_DB" "SELECT 1 FROM pragma_table_info('$1') WHERE name = '$2' LIMIT 1" | grep -q 1 +} + mkdir -p "$OUT_DIR" rm -f "$TMP_DB" "$TMP_DB-wal" "$TMP_DB-shm" @@ -82,6 +86,11 @@ DETACH report; fi if sqlite3 "$APP_DB" "SELECT 1 FROM sqlite_master WHERE type='table' AND name='ai_usage_events' LIMIT 1" | grep -q 1; then + ESTIMATED_NEURONS_EXPR=0 + if source_column_exists "ai_usage_events" "estimated_neurons"; then + ESTIMATED_NEURONS_EXPR="estimated_neurons" + fi + sqlite3 -cmd ".timeout 5000" "$APP_DB" " ATTACH '$TMP_DB_SQL' AS report; INSERT INTO report.ai_usage_events ( @@ -97,7 +106,7 @@ SELECT feature, model, status, - estimated_neurons, + COALESCE($ESTIMATED_NEURONS_EXPR, 0), detail, json_object( 'repoFullName', json_extract(metadata_json, '$.repoFullName'), diff --git a/test/unit/selfhost-grafana-reporting.test.ts b/test/unit/selfhost-grafana-reporting.test.ts new file mode 100644 index 0000000000..d6ef7c8305 --- /dev/null +++ b/test/unit/selfhost-grafana-reporting.test.ts @@ -0,0 +1,86 @@ +import { execFileSync } from "node:child_process"; +import { mkdtempSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { afterEach, describe, expect, it } from "vitest"; + +const tmpRoots: string[] = []; + +function tmpRoot(): string { + const dir = mkdtempSync(join(tmpdir(), "gittensory-reporting-")); + tmpRoots.push(dir); + return dir; +} + +afterEach(() => { + for (const dir of tmpRoots.splice(0)) rmSync(dir, { force: true, recursive: true }); +}); + +function sqlite(db: string, sql: string): string { + return execFileSync("sqlite3", [db, sql], { encoding: "utf8" }).trim(); +} + +function runExporter(root: string, sourceDb: string, outDb: string): void { + execFileSync("sh", ["scripts/export-grafana-reporting-db.sh"], { + cwd: process.cwd(), + env: { + ...process.env, + GITTENSORY_REPORTING_SOURCE_DB: sourceDb, + GITTENSORY_REPORTING_DIR: root, + GITTENSORY_REPORTING_DB: outDb, + }, + stdio: "pipe", + }); +} + +describe("Grafana reporting exporter", () => { + it("copies durable AI usage estimate rows into the redacted reporting database", () => { + const root = tmpRoot(); + const appDb = join(root, "app.sqlite"); + const outDb = join(root, "reporting.sqlite"); + sqlite(appDb, ` + CREATE TABLE ai_usage_events ( + feature TEXT NOT NULL, + model TEXT NOT NULL, + status TEXT NOT NULL, + estimated_neurons INTEGER NOT NULL DEFAULT 0, + detail TEXT, + metadata_json TEXT NOT NULL DEFAULT '{}', + created_at TEXT NOT NULL + ); + INSERT INTO ai_usage_events (feature, model, status, estimated_neurons, detail, metadata_json, created_at) + VALUES ('ai_review_pr', 'codex:gpt-5.5', 'ok', 42, 'done', '{"repoFullName":"JSONbored/gittensory","pullNumber":1678,"private":"drop"}', '2026-06-28T00:00:00Z'); + `); + + runExporter(root, appDb, outDb); + + expect(sqlite(outDb, "PRAGMA quick_check;")).toBe("ok"); + expect(sqlite(outDb, "SELECT estimated_neurons FROM ai_usage_events;")).toBe("42"); + expect(sqlite(outDb, "SELECT json_extract(metadata_json, '$.repoFullName') FROM ai_usage_events;")).toBe("JSONbored/gittensory"); + expect(sqlite(outDb, "SELECT json_extract(metadata_json, '$.private') IS NULL FROM ai_usage_events;")).toBe("1"); + expect(sqlite(outDb, "SELECT sum(estimated_neurons) FROM ai_usage_events WHERE feature = 'ai_review_pr' AND (('+' || model || '+') LIKE '%+codex+%' OR ('+' || model || '+') LIKE '%+codex:%');")).toBe("42"); + }); + + it("keeps the dashboard schema valid when an older source DB has no estimate column", () => { + const root = tmpRoot(); + const appDb = join(root, "app.sqlite"); + const outDb = join(root, "reporting.sqlite"); + sqlite(appDb, ` + CREATE TABLE ai_usage_events ( + feature TEXT NOT NULL, + model TEXT NOT NULL, + status TEXT NOT NULL, + detail TEXT, + metadata_json TEXT NOT NULL DEFAULT '{}', + created_at TEXT NOT NULL + ); + INSERT INTO ai_usage_events (feature, model, status, detail, metadata_json, created_at) + VALUES ('ai_review_pr', 'codex', 'error', 'failed', '{"repoFullName":"JSONbored/gittensory","pullNumber":1678}', '2026-06-28T00:00:00Z'); + `); + + runExporter(root, appDb, outDb); + + expect(sqlite(outDb, "PRAGMA quick_check;")).toBe("ok"); + expect(sqlite(outDb, "SELECT estimated_neurons FROM ai_usage_events;")).toBe("0"); + }); +}); From 7161a9bb76e8085ca612060f8c23f56660c82feb Mon Sep 17 00:00:00 2001 From: JSONbored <49853598+JSONbored@users.noreply.github.com> Date: Sun, 28 Jun 2026 14:09:29 -0700 Subject: [PATCH 17/68] test(observability): skip exporter smoke without sqlite cli Keep the reporting exporter smoke useful where the sqlite CLI is installed, while allowing the normal GitHub test runner to pass when that external binary is unavailable. --- test/unit/selfhost-grafana-reporting.test.ts | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/test/unit/selfhost-grafana-reporting.test.ts b/test/unit/selfhost-grafana-reporting.test.ts index d6ef7c8305..eca3cb1e81 100644 --- a/test/unit/selfhost-grafana-reporting.test.ts +++ b/test/unit/selfhost-grafana-reporting.test.ts @@ -5,6 +5,14 @@ import { join } from "node:path"; import { afterEach, describe, expect, it } from "vitest"; const tmpRoots: string[] = []; +const sqliteCliAvailable = (() => { + try { + execFileSync("sqlite3", ["--version"], { stdio: "ignore" }); + return true; + } catch { + return false; + } +})(); function tmpRoot(): string { const dir = mkdtempSync(join(tmpdir(), "gittensory-reporting-")); @@ -33,7 +41,7 @@ function runExporter(root: string, sourceDb: string, outDb: string): void { }); } -describe("Grafana reporting exporter", () => { +(sqliteCliAvailable ? describe : describe.skip)("Grafana reporting exporter", () => { it("copies durable AI usage estimate rows into the redacted reporting database", () => { const root = tmpRoot(); const appDb = join(root, "app.sqlite"); From 724ccace96786ab06d62230dff4bf16ec77eb052 Mon Sep 17 00:00:00 2001 From: JSONbored <49853598+JSONbored@users.noreply.github.com> Date: Sun, 28 Jun 2026 14:18:36 -0700 Subject: [PATCH 18/68] fix(selfhost): protect private Codex runtime config Expose self-host build args through Compose, keep private review config and Codex auth out of Git and Docker build contexts, and make the Codex dashboard distinguish CLI attempts from successful durable review records. --- .dockerignore | 6 +++++ .env.example | 1 + .gitignore | 8 +++++++ docker-compose.yml | 3 +++ grafana/dashboards/codex-usage.json | 35 ++++++++++++++++------------- 5 files changed, 37 insertions(+), 16 deletions(-) diff --git a/.dockerignore b/.dockerignore index 89cf858992..c583bb5a69 100644 --- a/.dockerignore +++ b/.dockerignore @@ -11,6 +11,12 @@ dist-ssr coverage .git .claude +# Runtime-only private review config and subscription CLI auth must never enter image layers. +gittensory-config +**/gittensory-config +**/.codex +auth.json +**/auth.json # The review-enrichment service (REES) is a separate Railway service with its own Dockerfile — keep it out of the engine image. review-enrichment .DS_Store diff --git a/.env.example b/.env.example index c1acc7c9c6..e6b48d8564 100644 --- a/.env.example +++ b/.env.example @@ -255,6 +255,7 @@ GITTENSORY_REVIEW_DRAFT=false # Codex (ChatGPT subscription) reviewer is fail-closed by default for self-host PR review: `codex exec` stores its # OAuth credential in auth.json on the same filesystem that prompt-influenced reviews can read. Isolated maintainer # deployments can opt in explicitly after mounting auth at /data/codex (the image exposes it as ~/.codex). +# INSTALL_AI_CLIS=true # compose build arg; bake claude/codex CLIs into the app image # GITTENSORY_ENABLE_UNSAFE_CODEX_REVIEWER=1 # Do NOT set CODEX_HOME for the app container; the provider rejects it so credentials are not advertised in env. # CODEX_AI_MODEL=gpt-5.5 # omit to let the Codex account default choose; set explicitly for repeatable reviews diff --git a/.gitignore b/.gitignore index 2ffd37d7f8..fb0cbb8299 100644 --- a/.gitignore +++ b/.gitignore @@ -13,6 +13,14 @@ output/ .env.* !.env.example *.local +# Private self-host operator config. Root AGENTS.md/CLAUDE.md are public project docs; +# repo-scoped review instructions live under this ignored mount. +gittensory-config/ +# Codex/CLI auth state must only live in runtime volumes or operator home dirs. +.codex/ +**/.codex/ +auth.json +**/auth.json .DS_Store coverage/ *.tsbuildinfo diff --git a/docker-compose.yml b/docker-compose.yml index dd4c14bae4..85c17090ee 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -28,6 +28,9 @@ services: gittensory: build: context: . + args: + INSTALL_AI_CLIS: "${INSTALL_AI_CLIS:-false}" + INSTALL_VISUAL_REVIEW: "${INSTALL_VISUAL_REVIEW:-false}" restart: unless-stopped ports: # Remove this when using the caddy profile — Caddy becomes the public listener. diff --git a/grafana/dashboards/codex-usage.json b/grafana/dashboards/codex-usage.json index c05b1182b8..c314967c8e 100644 --- a/grafana/dashboards/codex-usage.json +++ b/grafana/dashboards/codex-usage.json @@ -7,7 +7,7 @@ "version": 1, "refresh": "30s", "time": { "from": "now-7d", "to": "now" }, - "description": "Codex review usage for the self-hosted stack. Live counters come from the app's subscription-CLI adapter; durable review records come from ai_usage_events. USD cost appears only when the Codex CLI emits a cost field.", + "description": "Codex review observability for the self-hosted stack. Prometheus panels are subscription-CLI attempt counters for the selected range; durable review records come from ai_usage_events and must be read with their status. USD cost appears only when the Codex CLI emits a cost field.", "panels": [ { "id": 1, @@ -19,47 +19,50 @@ "id": 2, "type": "stat", "title": "Reported cost", - "description": "USD cost reported by Codex JSON/JSONL output, if available. Subscription CLIs may not emit this.", + "description": "USD cost reported by Codex JSON/JSONL output in the selected range, if available. Subscription CLIs may not emit this.", "gridPos": { "h": 5, "w": 6, "x": 0, "y": 1 }, "datasource": { "type": "prometheus", "uid": "prometheus" }, "fieldConfig": { "defaults": { "unit": "currencyUSD", "decimals": 4, "color": { "mode": "fixed", "fixedColor": "green" } } }, "options": { "reduceOptions": { "calcs": ["lastNotNull"] }, "colorMode": "value", "graphMode": "area", "justifyMode": "center" }, - "targets": [{ "refId": "A", "instant": true, "expr": "sum(gittensory_ai_cost_usd_total{provider=\"codex\"}) or vector(0)" }] + "targets": [{ "refId": "A", "instant": true, "expr": "sum(increase(gittensory_ai_cost_usd_total{provider=\"codex\"}[$__range])) or vector(0)" }] }, { "id": 3, "type": "stat", - "title": "CLI requests", + "title": "CLI attempts", + "description": "Codex CLI attempts in the selected range. Failed or timed-out attempts can still appear here.", "gridPos": { "h": 5, "w": 6, "x": 6, "y": 1 }, "datasource": { "type": "prometheus", "uid": "prometheus" }, "fieldConfig": { "defaults": { "unit": "short", "color": { "mode": "fixed", "fixedColor": "blue" } } }, "options": { "reduceOptions": { "calcs": ["lastNotNull"] }, "colorMode": "value", "graphMode": "area", "justifyMode": "center" }, - "targets": [{ "refId": "A", "instant": true, "expr": "sum(gittensory_ai_requests_total{provider=\"codex\"}) or vector(0)" }] + "targets": [{ "refId": "A", "instant": true, "expr": "sum(increase(gittensory_ai_requests_total{provider=\"codex\"}[$__range])) or vector(0)" }] }, { "id": 4, "type": "stat", - "title": "CLI tokens", + "title": "Reported CLI tokens", + "description": "Tokens reported by Codex CLI output in the selected range. This is not billing-grade cost accounting.", "gridPos": { "h": 5, "w": 6, "x": 12, "y": 1 }, "datasource": { "type": "prometheus", "uid": "prometheus" }, "fieldConfig": { "defaults": { "unit": "short", "color": { "mode": "fixed", "fixedColor": "purple" } } }, "options": { "reduceOptions": { "calcs": ["lastNotNull"] }, "colorMode": "value", "graphMode": "area", "justifyMode": "center" }, - "targets": [{ "refId": "A", "instant": true, "expr": "sum(gittensory_ai_total_tokens_total{provider=\"codex\"}) or (sum(gittensory_ai_input_tokens_total{provider=\"codex\"}) + sum(gittensory_ai_output_tokens_total{provider=\"codex\"})) or vector(0)" }] + "targets": [{ "refId": "A", "instant": true, "expr": "sum(increase(gittensory_ai_total_tokens_total{provider=\"codex\"}[$__range])) or (sum(increase(gittensory_ai_input_tokens_total{provider=\"codex\"}[$__range])) + sum(increase(gittensory_ai_output_tokens_total{provider=\"codex\"}[$__range]))) or vector(0)" }] }, { "id": 5, "type": "stat", - "title": "Review records", + "title": "Successful review records", + "description": "Durable Codex-attributed ai_review_pr records with status=ok.", "gridPos": { "h": 5, "w": 6, "x": 18, "y": 1 }, "datasource": { "type": "frser-sqlite-datasource", "uid": "gittensory-db" }, - "fieldConfig": { "defaults": { "unit": "short", "color": { "mode": "fixed", "fixedColor": "orange" } } }, + "fieldConfig": { "defaults": { "unit": "short", "color": { "mode": "fixed", "fixedColor": "green" } } }, "options": { "reduceOptions": { "calcs": ["lastNotNull"] }, "colorMode": "value", "graphMode": "none", "justifyMode": "center" }, "targets": [ { "refId": "A", "queryType": "table", - "queryText": "SELECT count(*) AS reviews FROM ai_usage_events WHERE feature = 'ai_review_pr' AND (('+' || model || '+') LIKE '%+codex+%' OR ('+' || model || '+') LIKE '%+codex:%')", - "rawQueryText": "SELECT count(*) AS reviews FROM ai_usage_events WHERE feature = 'ai_review_pr' AND (('+' || model || '+') LIKE '%+codex+%' OR ('+' || model || '+') LIKE '%+codex:%')" + "queryText": "SELECT count(*) AS reviews FROM ai_usage_events WHERE feature = 'ai_review_pr' AND status = 'ok' AND (('+' || model || '+') LIKE '%+codex+%' OR ('+' || model || '+') LIKE '%+codex:%')", + "rawQueryText": "SELECT count(*) AS reviews FROM ai_usage_events WHERE feature = 'ai_review_pr' AND status = 'ok' AND (('+' || model || '+') LIKE '%+codex+%' OR ('+' || model || '+') LIKE '%+codex:%')" } ] }, @@ -72,7 +75,7 @@ { "id": 7, "type": "timeseries", - "title": "Requests by model and effort", + "title": "CLI attempts by model and effort", "gridPos": { "h": 8, "w": 12, "x": 0, "y": 7 }, "datasource": { "type": "prometheus", "uid": "prometheus" }, "fieldConfig": { "defaults": { "unit": "short", "custom": { "drawStyle": "bars", "fillOpacity": 70, "lineWidth": 1, "stacking": { "mode": "none" } } } }, @@ -88,7 +91,7 @@ { "id": 8, "type": "timeseries", - "title": "Tokens by direction", + "title": "Reported tokens by direction", "gridPos": { "h": 8, "w": 12, "x": 12, "y": 7 }, "datasource": { "type": "prometheus", "uid": "prometheus" }, "fieldConfig": { "defaults": { "unit": "short", "custom": { "drawStyle": "bars", "fillOpacity": 70, "lineWidth": 1, "stacking": { "mode": "normal" } } } }, @@ -109,7 +112,7 @@ { "id": 9, "type": "row", - "title": "Durable review records", + "title": "Durable review records (status-aware)", "gridPos": { "h": 1, "w": 24, "x": 0, "y": 15 } }, { @@ -133,7 +136,7 @@ { "id": 11, "type": "piechart", - "title": "Review status", + "title": "Review record status", "gridPos": { "h": 8, "w": 12, "x": 12, "y": 16 }, "datasource": { "type": "frser-sqlite-datasource", "uid": "gittensory-db" }, "fieldConfig": { "defaults": { "unit": "short" } }, @@ -150,7 +153,7 @@ { "id": 12, "type": "table", - "title": "Recent Codex review events", + "title": "Recent Codex-attributed review events", "gridPos": { "h": 10, "w": 24, "x": 0, "y": 24 }, "datasource": { "type": "frser-sqlite-datasource", "uid": "gittensory-db" }, "options": { "showHeader": true, "cellHeight": "sm", "sortBy": [{ "displayName": "created_at", "desc": true }] }, From ec3be77fe30205ec74ca6999c74f2beef22aa2e3 Mon Sep 17 00:00:00 2001 From: JSONbored <49853598+JSONbored@users.noreply.github.com> Date: Sun, 28 Jun 2026 14:35:42 -0700 Subject: [PATCH 19/68] fix(gate): keep readiness advisory --- .gittensory.yml | 2 +- docs/review-configuration.md | 4 +-- src/config/gittensory-repo-focus-manifest.ts | 2 +- src/rules/advisory.ts | 35 +++++++++++--------- test/unit/gate-check-policy.test.ts | 8 +++++ test/unit/rules.test.ts | 24 +++++++------- 6 files changed, 43 insertions(+), 32 deletions(-) diff --git a/.gittensory.yml b/.gittensory.yml index 8c9aa0954f..7518084986 100644 --- a/.gittensory.yml +++ b/.gittensory.yml @@ -40,7 +40,7 @@ gate: linkedIssue: advisory # block | advisory | off — issues aren't always available; advise, don't block duplicates: block # block | advisory | off — block obvious duplicate PRs readiness: - mode: advisory # block | advisory | off — readiness-score floor + mode: advisory # advisory | off — readiness score is informational and never blocks the Gate minScore: 60 # aiReview: # opt-in AI maintainer review (off by default; needs the AI flags enabled) # mode: advisory # block | advisory | off — block only blocks on a dual-model consensus defect diff --git a/docs/review-configuration.md b/docs/review-configuration.md index 940ee2f95e..684c6f6be2 100644 --- a/docs/review-configuration.md +++ b/docs/review-configuration.md @@ -99,8 +99,8 @@ already-enabled gate. | Policy pack | `gate.pack` | `gatePack` | `gittensor` / `oss-anti-slop` | `gittensor` | `gittensor` = confirmed-contributor-gated, registry-aware. `oss-anti-slop` runs the deterministic rules against any author on any repo. | | Linked-issue gate | `gate.linkedIssue` | `linkedIssueGateMode` | `off`/`advisory`/`block` | `advisory` | If the dashboard "Require linked issue" toggle (`requireLinkedIssue`) is on but this is `off`, it is auto-promoted to `block`. | | Duplicate-PR gate | `gate.duplicates` | `duplicatePrGateMode` | `off`/`advisory`/`block` | `block` | Detects duplicate/superseding PRs. | -| Quality / merge-readiness score gate | `gate.readiness.mode` | `qualityGateMode` | `off`/`advisory`/`block` | `advisory` | The PR-quality score gate. | -| Quality min score | `gate.readiness.minScore` | `qualityGateMinScore` | number 0–100 (nullable) | `null` | At/above this score the quality dimension passes; `null` = engine default band. | +| Quality / merge-readiness score signal | `gate.readiness.mode` | `qualityGateMode` | `off`/`advisory`/`block` | `advisory` | Advisory/informational only. `block` is accepted for older configs but does not fail the Gate check. | +| Quality min score | `gate.readiness.minScore` | `qualityGateMinScore` | number 0–100 (nullable) | `null` | Advisory warning threshold for the readiness signal; `null` disables the threshold. | | Slop gate | `gate.slop.mode` | `slopGateMode` | `off`/`advisory`/`block` | `off` | Deterministic anti-slop signal. `advisory` surfaces the slop score + warnings; `block` also hard-blocks at/above the min score. Opt-in. | | Slop min score | `gate.slop.minScore` | `slopGateMinScore` | number 0–100 (nullable) | `null` (engine uses `60`, the "high" band) | The slop-risk threshold at/above which `slop block` blocks. | | Slop AI advisory | `gate.slop.aiAdvisory` | `slopAiAdvisory` | bool | `false` | When `true` **and** slop is not `off`, a free Workers-AI pass adds an **advisory-only** `ai_slop_advisory` finding. Never feeds the slop score or the gate. | diff --git a/src/config/gittensory-repo-focus-manifest.ts b/src/config/gittensory-repo-focus-manifest.ts index 96a200c0d0..a65403cdd4 100644 --- a/src/config/gittensory-repo-focus-manifest.ts +++ b/src/config/gittensory-repo-focus-manifest.ts @@ -44,7 +44,7 @@ gate: linkedIssue: advisory # block | advisory | off — issues aren't always available; advise, don't block duplicates: block # block | advisory | off — block obvious duplicate PRs readiness: - mode: advisory # block | advisory | off — readiness-score floor + mode: advisory # advisory | off — readiness score is informational and never blocks the Gate minScore: 60 # aiReview: # opt-in AI maintainer review (off by default; needs the AI flags enabled) # mode: advisory # block | advisory | off — block only blocks on a dual-model consensus defect diff --git a/src/rules/advisory.ts b/src/rules/advisory.ts index ee066faa1c..7e3d4ce607 100644 --- a/src/rules/advisory.ts +++ b/src/rules/advisory.ts @@ -18,6 +18,8 @@ export type GateCheckConclusion = "success" | "failure" | "action_required" | "n export type GateCheckPolicy = { linkedIssueGateMode?: GateRuleMode | undefined; duplicatePrGateMode?: GateRuleMode | undefined; + /** Historical readiness-score mode. Retained for config compatibility, but readiness is informational only: + * a low readiness score may be surfaced as an advisory warning and must never fail the Gate check. */ qualityGateMode?: GateRuleMode | undefined; qualityGateMinScore?: number | null | undefined; /** When `block`, a dual-model AI consensus defect (`ai_consensus_defect` finding) becomes a hard @@ -480,13 +482,14 @@ function evaluateGateCheckCore(advisoryResult: Advisory, policy: GateCheckPolicy warnings, }; } - // Merge-readiness composite (#551): when set, escalate every sub-gate to its mode so they roll into one - // pass/fail. When off, this is a no-op and each sub-gate keeps its own mode. + // Merge-readiness composite (#551): when set, escalate enforceable sub-gates to its mode so they roll into one + // pass/fail. Readiness/quality stays advisory-only. const effective = applyMergeReadinessGate(policy); const configuredBlockers = advisoryResult.findings.filter((finding) => isConfiguredGateBlocker(finding, effective)); - const qualityBlocker = buildQualityGateBlocker(effective); + const qualityWarning = buildQualityGateWarning(effective); const slopBlocker = buildSlopGateBlocker(effective); - const blockers = [...configuredBlockers, ...(qualityBlocker ? [qualityBlocker] : []), ...(slopBlocker ? [slopBlocker] : [])]; + const blockers = [...configuredBlockers, ...(slopBlocker ? [slopBlocker] : [])]; + const gateWarnings = qualityWarning ? [...warnings, qualityWarning] : warnings; const lowConfidenceAiHolds = advisoryResult.findings.filter((finding) => isLowConfidenceAiReviewHold(finding, effective)); // Non-confirmed contributors are gated NORMALLY (real blockers → failure → one-shot close; clean → success → // merge), the SAME as confirmed contributors: the review + CI + guardrail vet every PR, and confirmed-status @@ -509,7 +512,7 @@ function evaluateGateCheckCore(advisoryResult: Advisory, policy: GateCheckPolicy title: "Gittensory Gate — first-contribution grace", summary: "This is a first-time contribution to this repo, so the gate stays advisory rather than blocking. The findings remain visible, and the gate will apply normally once this author has merge history here.", blockers: [], - warnings, + warnings: gateWarnings, }; } if (blockers.length === 0) { @@ -520,7 +523,7 @@ function evaluateGateCheckCore(advisoryResult: Advisory, policy: GateCheckPolicy title: "Gittensory Gate — held for human review", summary: "The AI review flagged a possible must-fix defect below the automatic close-confidence floor, so the gate is held for a human reviewer instead of passed automatically.", blockers: [], - warnings: [...warnings, ...lowConfidenceAiHolds], + warnings: [...gateWarnings, ...lowConfidenceAiHolds], }; } // Fail-CLOSED AI hold (#ai-fail-closed, #audit-3.5): with NO deterministic blocker, a block-mode AI review @@ -535,7 +538,7 @@ function evaluateGateCheckCore(advisoryResult: Advisory, policy: GateCheckPolicy title: "Gittensory Gate — held for human review", summary: "The AI review could not be completed for this change, so the gate is held for a human reviewer rather than passed automatically. It re-evaluates on the next update.", blockers: [], - warnings, + warnings: gateWarnings, }; } // Manual-review HOLD (#gate-size / #gate-guardrail): a PR that would otherwise PASS but is oversized or touches @@ -553,7 +556,7 @@ function evaluateGateCheckCore(advisoryResult: Advisory, policy: GateCheckPolicy title: "Gittensory Gate — held for manual review", summary: holds.map((h) => sanitizeForCheckRun(h.title)).join("; "), blockers: [], - warnings: [...warnings, ...holds], + warnings: [...gateWarnings, ...holds], }; } return { @@ -562,7 +565,7 @@ function evaluateGateCheckCore(advisoryResult: Advisory, policy: GateCheckPolicy title: "Gittensory Gate passed", summary: "No configured hard blocker was found. Advisory findings, if any, stay advisory.", blockers, - warnings, + warnings: gateWarnings, }; } // Name the exact blocker(s) + fix in the title so the contributor sees WHY at a glance. @@ -576,7 +579,7 @@ function evaluateGateCheckCore(advisoryResult: Advisory, policy: GateCheckPolicy .map((finding) => `${sanitizeForCheckRun(finding.title)}${finding.action ? ` — ${sanitizeForCheckRun(finding.action)}` : ""}`) .join("; "), blockers, - warnings: advisoryResult.findings.filter((finding) => finding.severity === "warning" && !blockers.includes(finding)), + warnings: [...advisoryResult.findings.filter((finding) => finding.severity === "warning" && !blockers.includes(finding)), ...(qualityWarning ? [qualityWarning] : [])], }; } @@ -879,8 +882,8 @@ function isLowConfidenceAiReviewHold(finding: AdvisoryFinding, policy: GateCheck return confidence < (policy.aiReviewCloseConfidence ?? DEFAULT_AI_REVIEW_CLOSE_CONFIDENCE); } -function buildQualityGateBlocker(policy: GateCheckPolicy): AdvisoryFinding | null { - if (gateMode(policy.qualityGateMode) !== "block") return null; +function buildQualityGateWarning(policy: GateCheckPolicy): AdvisoryFinding | null { + if (gateMode(policy.qualityGateMode) === "off") return null; const score = normalizeScore(policy.readinessScore); const minScore = normalizeScore(policy.qualityGateMinScore); if (score === null || minScore === null || score >= minScore) return null; @@ -889,7 +892,7 @@ function buildQualityGateBlocker(policy: GateCheckPolicy): AdvisoryFinding | nul severity: "warning", title: "Readiness score is below the configured threshold", detail: `The public readiness score is ${score}/100, below the repository threshold of ${minScore}/100.`, - action: "Address the short explicit PR panel actions, then re-run the gate.", + action: "Use the readiness panel as advisory maintainer context; the score does not block this PR.", }; } @@ -916,8 +919,9 @@ function gateMode(value: GateRuleMode | null | undefined): GateRuleMode { } // #551: the master merge-readiness composite. When mergeReadinessGateMode is set (advisory/block) it -// OVERRIDES the four sub-gates to its mode so they roll into one pass/fail; when off, the policy is returned -// unchanged and each sub-gate keeps its own mode. +// OVERRIDES the enforceable sub-gates to its mode so they roll into one pass/fail; when off, the policy is +// returned unchanged and each sub-gate keeps its own mode. Readiness/quality is intentionally excluded: +// readiness is always advisory/informational, even if an older config still says `readiness: block`. function applyMergeReadinessGate(policy: GateCheckPolicy): GateCheckPolicy { const composite = gateMode(policy.mergeReadinessGateMode ?? "off"); if (composite === "off") return policy; @@ -925,7 +929,6 @@ function applyMergeReadinessGate(policy: GateCheckPolicy): GateCheckPolicy { ...policy, linkedIssueGateMode: composite, duplicatePrGateMode: composite, - qualityGateMode: composite, slopGateMode: composite, }; } diff --git a/test/unit/gate-check-policy.test.ts b/test/unit/gate-check-policy.test.ts index 24dc4a1d04..421a906513 100644 --- a/test/unit/gate-check-policy.test.ts +++ b/test/unit/gate-check-policy.test.ts @@ -395,6 +395,14 @@ describe("merge-readiness composite gate (#551)", () => { expect(result.summary).toContain("No linked issue detected"); expect(result.summary).toContain("Possible duplicate PR"); }); + + it("does not escalate readiness score into a blocker", () => { + const eff = resolveEffectiveSettings(settings({ mergeReadinessGateMode: "block", qualityGateMinScore: 90 }), parseFocusManifest(null)); + const result = evaluateGateCheck({ ...missingIssueAdvisory(), findings: [] }, gateCheckPolicy(eff, 42, true)); + expect(result.conclusion).toBe("success"); + expect(result.blockers).toEqual([]); + expect(result.warnings.map((finding) => finding.code)).toEqual(["readiness_score_below_threshold"]); + }); }); describe("first-time-contributor grace (#552)", () => { diff --git a/test/unit/rules.test.ts b/test/unit/rules.test.ts index 433876d8d4..06fdda59e0 100644 --- a/test/unit/rules.test.ts +++ b/test/unit/rules.test.ts @@ -311,7 +311,7 @@ describe("advisory rules", () => { expect(evaluateGateCheck(splitAdvisory).conclusion).toBe("success"); }); - it("only enforces readiness score when quality gate mode is block", () => { + it("keeps readiness score advisory even when legacy config says block", () => { const advisory = buildPullRequestAdvisory(repo, { repoFullName: repo.fullName, number: 24, @@ -330,17 +330,17 @@ describe("advisory rules", () => { expect(evaluateGateCheck(advisory, { qualityGateMode: "block", qualityGateMinScore: 90, readinessScore: null }).conclusion).toBe("success"); expect(evaluateGateCheck(advisory, { qualityGateMode: "block", qualityGateMinScore: 90, readinessScore: 90 }).conclusion).toBe("success"); - const failingGate = evaluateGateCheck(advisory, { qualityGateMode: "block", qualityGateMinScore: 90, readinessScore: 89.4 }); - const output = formatGateCheckOutput(failingGate); + const advisoryGate = evaluateGateCheck(advisory, { qualityGateMode: "block", qualityGateMinScore: 90, readinessScore: 89.4 }); - expect(failingGate.conclusion).toBe("failure"); - expect(failingGate.blockers.map((finding) => finding.code)).toEqual(["readiness_score_below_threshold"]); - expect(output.text).toContain("Readiness score is below the configured threshold"); - expect(output.text).toContain("Action: Address the short explicit PR panel actions"); + expect(advisoryGate.conclusion).toBe("success"); + expect(advisoryGate.blockers).toEqual([]); + expect(advisoryGate.warnings.map((finding) => finding.code)).toEqual(["readiness_score_below_threshold"]); + expect(formatGateCheckOutput(advisoryGate).text).not.toContain("Readiness score is below the configured threshold"); - expect(evaluateGateCheck(advisory, { qualityGateMode: "block", qualityGateMinScore: 101, readinessScore: -5 }).blockers[0]?.detail).toContain("0/100"); + expect(evaluateGateCheck(advisory, { qualityGateMode: "block", qualityGateMinScore: 101, readinessScore: -5 }).warnings[0]?.detail).toContain("0/100"); expect(evaluateGateCheck(advisory, { qualityGateMode: "block", qualityGateMinScore: 99, readinessScore: 102 }).conclusion).toBe("success"); expect(evaluateGateCheck(advisory, { qualityGateMode: "block", qualityGateMinScore: Number.NaN, readinessScore: 10 }).conclusion).toBe("success"); + expect(evaluateGateCheck(advisory, { mergeReadinessGateMode: "block", qualityGateMinScore: 90, readinessScore: 10 }).conclusion).toBe("success"); }); it("summarizes multiple configured hard blockers without swallowing advisory warnings", () => { @@ -358,12 +358,12 @@ describe("advisory rules", () => { expect(gate.conclusion).toBe("failure"); // Title names the blocker count; summary enumerates every active blocker with its fix. - expect(gate.title).toBe("Gittensory Gate: 3 blockers"); + expect(gate.title).toBe("Gittensory Gate: 2 blockers"); expect(gate.summary).toContain("No linked issue detected"); expect(gate.summary).toContain("Linked issue overlaps another open PR"); - expect(gate.summary).toContain("Readiness score is below the configured threshold — Address the short explicit PR panel actions"); - expect(gate.blockers.map((finding) => finding.code)).toEqual(["missing_linked_issue", "duplicate_pr_risk", "readiness_score_below_threshold"]); - expect(gate.warnings.map((finding) => finding.code)).toEqual(["busy_pr_queue"]); + expect(gate.summary).not.toContain("Readiness score is below the configured threshold"); + expect(gate.blockers.map((finding) => finding.code)).toEqual(["missing_linked_issue", "duplicate_pr_risk"]); + expect(gate.warnings.map((finding) => finding.code)).toEqual(["busy_pr_queue", "readiness_score_below_threshold"]); }); it("gates NON-confirmed contributors normally — a real blocker closes them like a confirmed author (#gate-nonconfirmed)", () => { From 26aa2b7a2e63b5d52579935e401090765b3e1910 Mon Sep 17 00:00:00 2001 From: JSONbored <49853598+JSONbored@users.noreply.github.com> Date: Sun, 28 Jun 2026 14:50:09 -0700 Subject: [PATCH 20/68] fix(gate): avoid stale pending check conclusions --- src/github/app.ts | 12 ++++++-- test/unit/github-app.test.ts | 60 +++++++++++++++++++++++++++++++++++- 2 files changed, 69 insertions(+), 3 deletions(-) diff --git a/src/github/app.ts b/src/github/app.ts index 098da909ca..39b030e204 100644 --- a/src/github/app.ts +++ b/src/github/app.ts @@ -29,6 +29,8 @@ type CheckRunListResponse = { id: number; html_url?: string; name?: string; + status?: GitHubCheckStatus | string | null; + conclusion?: string | null; }>; }; @@ -498,6 +500,7 @@ export async function createOrUpdatePendingGateCheckRun( "Gittensory is running deterministic public PR hygiene checks.", text: "The Gate blocks every author on the repo's configured hard blockers (duplicate PRs by default); on everything else, and while state is still syncing, it stays advisory.", }, + updateExisting: "in_progress_only", mode, }, ); @@ -612,6 +615,7 @@ async function createOrUpdateNamedCheckRun( conclusion?: GitHubCheckConclusion | undefined; output: CheckRunOutput; checkRunId?: number | undefined; + updateExisting?: "any" | "in_progress_only" | "never" | undefined; mode?: AgentActionMode | undefined; }, ): Promise { @@ -685,7 +689,7 @@ async function createOrUpdateNamedCheckRun( if (check.checkRunId) { const out = await patchCheckRun(check.checkRunId); if (out) return out; - } else { + } else if (check.updateExisting !== "never") { const existing = await octokit.request( "GET /repos/{owner}/{repo}/commits/{ref}/check-runs", { @@ -699,7 +703,11 @@ async function createOrUpdateNamedCheckRun( ); const existingCheckRun = (existing.data as CheckRunListResponse) .check_runs?.[0]; - if (existingCheckRun) { + if ( + existingCheckRun && + (check.updateExisting !== "in_progress_only" || + (existingCheckRun.status ?? "").toLowerCase() !== "completed") + ) { const out = await patchCheckRun(existingCheckRun.id); if (out) return out; } diff --git a/test/unit/github-app.test.ts b/test/unit/github-app.test.ts index 31beb1194c..90d5f7a507 100644 --- a/test/unit/github-app.test.ts +++ b/test/unit/github-app.test.ts @@ -834,7 +834,7 @@ describe("GitHub check runs", () => { if (url.includes("/commits/pending-existing/check-runs")) { return Response.json({ total_count: 1, - check_runs: [{ id: 333, name: "Gittensory Gate" }], + check_runs: [{ id: 333, name: "Gittensory Gate", status: "in_progress" }], }); } if (url.includes("/check-runs/333")) { @@ -864,6 +864,64 @@ describe("GitHub check runs", () => { expect(capturedBody).not.toHaveProperty("conclusion"); }); + it("posts a fresh pending Gate check instead of patching a completed run", async () => { + const privateKey = await generatePrivateKeyPem(); + const calls: string[] = []; + let capturedBody: { status?: string; conclusion?: string; output?: { title?: string } } = {}; + vi.stubGlobal( + "fetch", + async (input: RequestInfo | URL, init?: RequestInit) => { + const url = input.toString(); + const method = init?.method ?? "GET"; + calls.push(`${method} ${url}`); + if (url.includes("/access_tokens")) + return Response.json({ token: "installation-token" }); + if (url.includes("/commits/pending-after-failure/check-runs")) { + return Response.json({ + total_count: 1, + check_runs: [ + { + id: 444, + name: "Gittensory Gate", + status: "completed", + conclusion: "failure", + }, + ], + }); + } + if (url.includes("/check-runs/444")) + throw new Error("must not patch completed Gate run"); + if (url.includes("/check-runs") && method === "POST") { + capturedBody = JSON.parse(String(init?.body)) as typeof capturedBody; + return Response.json({ + id: 445, + html_url: "https://github.com/checks/445", + }, { status: 201 }); + } + return new Response("not found", { status: 404 }); + }, + ); + + const result = await createOrUpdatePendingGateCheckRun( + createTestEnv({ GITHUB_APP_PRIVATE_KEY: privateKey }), + 123, + "JSONbored/gittensory", + gateAdvisory("pending-after-failure"), + ); + + expect(result).toMatchObject({ + kind: "published", + id: 445, + html_url: "https://github.com/checks/445", + }); + expect(calls.some((call) => call.includes("/check-runs/444"))).toBe(false); + expect(capturedBody).toMatchObject({ + status: "in_progress", + output: { title: "Gittensory Gate is evaluating" }, + }); + expect(capturedBody).not.toHaveProperty("conclusion"); + }); + it("publishes a skipped Gate check for closed PR races", async () => { const privateKey = await generatePrivateKeyPem(); let capturedBody: { From 14b0f4758d99ce77d003d9decd2f5f48a8817631 Mon Sep 17 00:00:00 2001 From: JSONbored <49853598+JSONbored@users.noreply.github.com> Date: Sun, 28 Jun 2026 14:59:51 -0700 Subject: [PATCH 21/68] fix(observability): refresh maintainer review export --- docs/self-hosting.md | 15 ++-- scripts/export-grafana-reporting-db.sh | 90 +++++++++++++++++++- test/unit/selfhost-grafana-reporting.test.ts | 90 ++++++++++++++++++++ 3 files changed, 184 insertions(+), 11 deletions(-) diff --git a/docs/self-hosting.md b/docs/self-hosting.md index 7069361fa0..c0fc4bdd79 100644 --- a/docs/self-hosting.md +++ b/docs/self-hosting.md @@ -251,13 +251,14 @@ content-lane are not yet per-repo toggleable and stay on the allowlist.) `docker-compose.yml` (copy `litestream.yml.example` → `litestream.yml`, set your bucket + credentials); it streams every change to S3/B2/MinIO/R2. - **Maintainer Grafana dashboards.** Grafana does **not** mount the live app database. The observability profile - starts `reporting-exporter`, which copies only the dashboard-safe `review_targets` and `ai_usage_events` - columns into `/reporting/gittensory-reporting.sqlite` every `GRAFANA_REPORTING_EXPORT_INTERVAL_SECONDS` seconds - (default 30). The SQLite datasource points at that redacted reporting DB. If you override the app SQLite - `DATABASE_PATH`, set `GITTENSORY_REPORTING_SOURCE_DB` to the matching exporter mount path, for example - `/appdb/custom.sqlite` for `DATABASE_PATH=/data/custom.sqlite`. `DATABASE_URL`/Postgres deployments currently - export an empty dashboard-safe DB so Grafana can start; Postgres-backed maintainer analytics need a dedicated SQL - exporter. + starts `reporting-exporter`, which projects the active `pull_requests` + latest `advisories` rows into a + dashboard-safe `review_targets` snapshot, preserves older non-overlapping legacy `review_targets`, and copies + redacted `ai_usage_events` rows into `/reporting/gittensory-reporting.sqlite` every + `GRAFANA_REPORTING_EXPORT_INTERVAL_SECONDS` seconds (default 30). The SQLite datasource points at that redacted + reporting DB. If you override the app SQLite `DATABASE_PATH`, set `GITTENSORY_REPORTING_SOURCE_DB` to the + matching exporter mount path, for example `/appdb/custom.sqlite` for `DATABASE_PATH=/data/custom.sqlite`. + `DATABASE_URL`/Postgres deployments currently export an empty dashboard-safe DB so Grafana can start; + Postgres-backed maintainer analytics need a dedicated SQL exporter. - **App-level metrics.** Enable `GITTENSORY_REVIEW_OPS=true` for the read-only gate-block anomaly scan and the bearer-gated `GET /v1/internal/ops/stats` aggregate. diff --git a/scripts/export-grafana-reporting-db.sh b/scripts/export-grafana-reporting-db.sh index d0a4679404..8e6a8cc905 100644 --- a/scripts/export-grafana-reporting-db.sh +++ b/scripts/export-grafana-reporting-db.sh @@ -14,6 +14,10 @@ source_column_exists() { sqlite3 "$APP_DB" "SELECT 1 FROM pragma_table_info('$1') WHERE name = '$2' LIMIT 1" | grep -q 1 } +source_table_exists() { + sqlite3 "$APP_DB" "SELECT 1 FROM sqlite_master WHERE type='table' AND name='$1' LIMIT 1" | grep -q 1 +} + mkdir -p "$OUT_DIR" rm -f "$TMP_DB" "$TMP_DB-wal" "$TMP_DB-shm" @@ -57,9 +61,54 @@ if [ ! -s "$APP_DB" ]; then exit 0 fi -if sqlite3 "$APP_DB" "SELECT 1 FROM sqlite_master WHERE type='table' AND name='review_targets' LIMIT 1" | grep -q 1; then +if source_table_exists "pull_requests" && source_table_exists "advisories"; then sqlite3 -cmd ".timeout 5000" "$APP_DB" " ATTACH '$TMP_DB_SQL' AS report; +WITH latest_advisories AS ( + SELECT + repo_full_name, + pull_number, + conclusion, + updated_at, + ROW_NUMBER() OVER ( + PARTITION BY repo_full_name, pull_number + ORDER BY updated_at DESC, rowid DESC + ) AS rn + FROM main.advisories + WHERE pull_number IS NOT NULL +), +current_pull_requests AS ( + SELECT + p.repo_full_name AS repo, + p.number AS number, + p.author_login AS submitter, + CASE + WHEN lower(p.state) = 'closed' AND p.merged_at IS NOT NULL THEN 'merged' + WHEN lower(p.state) = 'closed' THEN 'closed' + WHEN a.conclusion IN ('failure', 'action_required') THEN 'manual' + WHEN a.conclusion IS NOT NULL THEN 'commented' + ELSE 'manual' + END AS status, + CASE a.conclusion + WHEN 'success' THEN 'merge' + WHEN 'failure' THEN 'close' + WHEN 'action_required' THEN 'manual' + WHEN 'neutral' THEN 'comment' + WHEN 'skipped' THEN 'ignore' + ELSE NULL + END AS verdict, + p.title AS title, + p.created_at AS created_at, + CASE + WHEN a.updated_at IS NOT NULL AND a.updated_at > p.updated_at THEN a.updated_at + ELSE p.updated_at + END AS updated_at + FROM main.pull_requests p + LEFT JOIN latest_advisories a + ON a.repo_full_name = p.repo_full_name + AND a.pull_number = p.number + AND a.rn = 1 +) INSERT INTO report.review_targets ( repo, number, @@ -79,13 +128,46 @@ SELECT title, created_at, updated_at -FROM main.review_targets -WHERE kind = 'pull_request'; +FROM current_pull_requests; +DETACH report; +" +fi + +if source_table_exists "review_targets"; then + sqlite3 -cmd ".timeout 5000" "$APP_DB" " +ATTACH '$TMP_DB_SQL' AS report; +INSERT INTO report.review_targets ( + repo, + number, + submitter, + status, + verdict, + title, + created_at, + updated_at +) +SELECT + t.repo, + t.number, + t.submitter, + t.status, + t.verdict, + t.title, + t.created_at, + t.updated_at +FROM main.review_targets t +WHERE t.kind = 'pull_request' + AND NOT EXISTS ( + SELECT 1 + FROM report.review_targets r + WHERE r.repo = t.repo + AND r.number = t.number + ); DETACH report; " fi -if sqlite3 "$APP_DB" "SELECT 1 FROM sqlite_master WHERE type='table' AND name='ai_usage_events' LIMIT 1" | grep -q 1; then +if source_table_exists "ai_usage_events"; then ESTIMATED_NEURONS_EXPR=0 if source_column_exists "ai_usage_events" "estimated_neurons"; then ESTIMATED_NEURONS_EXPR="estimated_neurons" diff --git a/test/unit/selfhost-grafana-reporting.test.ts b/test/unit/selfhost-grafana-reporting.test.ts index eca3cb1e81..13df7b3d99 100644 --- a/test/unit/selfhost-grafana-reporting.test.ts +++ b/test/unit/selfhost-grafana-reporting.test.ts @@ -42,6 +42,96 @@ function runExporter(root: string, sourceDb: string, outDb: string): void { } (sqliteCliAvailable ? describe : describe.skip)("Grafana reporting exporter", () => { + it("prefers current pull request rows while preserving non-overlapping legacy review history", () => { + const root = tmpRoot(); + const appDb = join(root, "app.sqlite"); + const outDb = join(root, "reporting.sqlite"); + sqlite(appDb, ` + CREATE TABLE review_targets ( + kind TEXT NOT NULL, + repo TEXT NOT NULL, + number INTEGER NOT NULL, + submitter TEXT, + status TEXT NOT NULL, + verdict TEXT, + title TEXT, + created_at TEXT NOT NULL, + updated_at TEXT NOT NULL + ); + INSERT INTO review_targets (kind, repo, number, submitter, status, verdict, title, created_at, updated_at) + VALUES + ('pull_request', 'JSONbored/gittensory', 1690, 'stale', 'closed', 'close', 'stale current PR', '2026-06-22T17:00:00Z', '2026-06-22T17:00:00Z'), + ('pull_request', 'JSONbored/gittensory', 1049, 'bohdansolovie', 'closed', 'close', 'historical PR', '2026-06-22T17:28:56Z', '2026-06-22T17:28:56Z'); + + CREATE TABLE pull_requests ( + repo_full_name TEXT NOT NULL, + number INTEGER NOT NULL, + title TEXT NOT NULL, + state TEXT NOT NULL, + author_login TEXT, + merged_at TEXT, + created_at TEXT NOT NULL, + updated_at TEXT NOT NULL + ); + INSERT INTO pull_requests (repo_full_name, number, title, state, author_login, merged_at, created_at, updated_at) + VALUES + ('JSONbored/gittensory', 1690, 'fresh advisory PR', 'open', 'JSONbored', NULL, '2026-06-28T21:00:00Z', '2026-06-28T21:39:58Z'), + ('JSONbored/gittensory', 1691, 'fresh merged PR', 'closed', 'tmimmanuel', '2026-06-28T21:46:51Z', '2026-06-28T21:30:00Z', '2026-06-28T21:47:36Z'); + + CREATE TABLE advisories ( + repo_full_name TEXT NOT NULL, + pull_number INTEGER, + conclusion TEXT NOT NULL, + updated_at TEXT NOT NULL + ); + INSERT INTO advisories (repo_full_name, pull_number, conclusion, updated_at) + VALUES + ('JSONbored/gittensory', 1690, 'failure', '2026-06-28T21:25:00Z'), + ('JSONbored/gittensory', 1690, 'neutral', '2026-06-28T21:40:00Z'), + ('JSONbored/gittensory', 1691, 'success', '2026-06-28T21:47:40Z'); + `); + + runExporter(root, appDb, outDb); + + expect(sqlite(outDb, "PRAGMA quick_check;")).toBe("ok"); + expect(sqlite(outDb, "SELECT count(*) FROM review_targets;")).toBe("3"); + expect(sqlite(outDb, "SELECT submitter || '|' || status || '|' || verdict || '|' || updated_at FROM review_targets WHERE repo='JSONbored/gittensory' AND number=1690;")).toBe( + "JSONbored|commented|comment|2026-06-28T21:40:00Z", + ); + expect(sqlite(outDb, "SELECT status || '|' || verdict || '|' || updated_at FROM review_targets WHERE repo='JSONbored/gittensory' AND number=1691;")).toBe( + "merged|merge|2026-06-28T21:47:40Z", + ); + expect(sqlite(outDb, "SELECT title FROM review_targets WHERE repo='JSONbored/gittensory' AND number=1049;")).toBe("historical PR"); + }); + + it("falls back to legacy review_targets when the current PR cache is absent", () => { + const root = tmpRoot(); + const appDb = join(root, "app.sqlite"); + const outDb = join(root, "reporting.sqlite"); + sqlite(appDb, ` + CREATE TABLE review_targets ( + kind TEXT NOT NULL, + repo TEXT NOT NULL, + number INTEGER NOT NULL, + submitter TEXT, + status TEXT NOT NULL, + verdict TEXT, + title TEXT, + created_at TEXT NOT NULL, + updated_at TEXT NOT NULL + ); + INSERT INTO review_targets (kind, repo, number, submitter, status, verdict, title, created_at, updated_at) + VALUES ('pull_request', 'JSONbored/gittensory', 1049, 'bohdansolovie', 'closed', 'close', 'legacy PR', '2026-06-22T17:28:56Z', '2026-06-22T17:28:56Z'); + `); + + runExporter(root, appDb, outDb); + + expect(sqlite(outDb, "PRAGMA quick_check;")).toBe("ok"); + expect(sqlite(outDb, "SELECT repo || '#' || number || '|' || status || '|' || verdict FROM review_targets;")).toBe( + "JSONbored/gittensory#1049|closed|close", + ); + }); + it("copies durable AI usage estimate rows into the redacted reporting database", () => { const root = tmpRoot(); const appDb = join(root, "app.sqlite"); From 6f8b04aead3dc663e118cbf284fec14526ba6fc5 Mon Sep 17 00:00:00 2001 From: JSONbored <49853598+JSONbored@users.noreply.github.com> Date: Sun, 28 Jun 2026 15:12:56 -0700 Subject: [PATCH 22/68] fix(github): refresh rejected installation tokens --- src/github/app.ts | 248 ++++++++++++++++++------------ src/github/comments.ts | 69 +++++---- test/unit/github-app.test.ts | 41 +++++ test/unit/github-comments.test.ts | 42 +++++ 4 files changed, 267 insertions(+), 133 deletions(-) diff --git a/src/github/app.ts b/src/github/app.ts index 39b030e204..120b172efa 100644 --- a/src/github/app.ts +++ b/src/github/app.ts @@ -238,6 +238,55 @@ export async function createInstallationToken( return mint; } +function githubErrorStatus(error: unknown): number | null { + const err = error as { + status?: number; + response?: { status?: number } | null; + }; + return err.status ?? err.response?.status ?? null; +} + +export function isGitHubBadCredentialsError(error: unknown): boolean { + const status = githubErrorStatus(error); + return status === 401 || /bad credentials/i.test(errorMessage(error)); +} + +async function expireCachedInstallationToken( + installationId: number, + rejectedToken: string, +): Promise { + const cached = await readCachedToken(installationId).catch(() => null); + if (cached && cached.token !== rejectedToken) return; + await writeCachedToken(installationId, { token: "", expiresAtMs: 0 }); +} + +export async function withInstallationTokenRetry( + env: Env, + installationId: number, + operation: (token: string) => Promise, +): Promise { + const token = await createInstallationToken(env, installationId); + try { + return await operation(token); + } catch (error) { + if (!isGitHubBadCredentialsError(error)) throw error; + await expireCachedInstallationToken(installationId, token).catch( + () => undefined, + ); + console.warn( + JSON.stringify({ + level: "warn", + event: "github_installation_token_rejected", + installationId, + status: githubErrorStatus(error), + message: errorMessage(error).slice(0, 200), + }), + ); + const freshToken = await createInstallationToken(env, installationId); + return await operation(freshToken); + } +} + /** Mint a fresh installation token (broker or local App-JWT) and cache it. `cached` is the expired/absent prior * entry, consulted only for the brokered stale-token grace. Extracted from createInstallationToken so that * function can single-flight concurrent cold-cache callers onto one mint (see inFlightMints). */ @@ -626,120 +675,121 @@ async function createOrUpdateNamedCheckRun( if (!owner || !repo) throw new Error(`Invalid repository full name: ${repoFullName}`); - const token = await createInstallationToken(env, installationId); - // makeInstallationOctokit injects the shared per-request timeout (a stalled PATCH can never orphan the - // in_progress check) AND suppresses the check-run writes under a non-live mode (dry-run / pause / freeze). - const octokit = makeInstallationOctokit(env, token, check.mode); - // Point the merge-box "Details" link at the repo's Gittensory maintainer panel instead of GitHub's generic - // check page. Spread conditionally so a URL-construction failure (null) just omits it. (#audit-details-url) - const detailsUrl = maintainerControlPanelUrl(env, repoFullName); - const detailsUrlBody = detailsUrl ? { details_url: detailsUrl } : {}; - - // POST a fresh check-run THIS App owns. Used for a brand-new run AND as the cross-app fallback below. - const postNewCheckRun = async (): Promise => { - const response = await octokit.request( - "POST /repos/{owner}/{repo}/check-runs", - { - owner, - repo, - name: check.name, - head_sha: headSha, - status: check.status ?? "completed", - ...(check.conclusion ? { conclusion: check.conclusion } : {}), - output: check.output, - ...detailsUrlBody, - }, - ); - return publishedOutcome(response.data as CheckRunResponse); - }; - // PATCH an existing run by id. If that run was created by a PRIOR GitHub App (install migrated / reinstalled under a - // new app_id), GitHub 403s "can only be modified by the GitHub App that created it" — that stale run is unreachable, - // so fall through (null) to POST a fresh one this App owns instead of failing the gate forever. (#cross-app-checkrun) - const patchCheckRun = async (id: number): Promise => { - try { + return await withInstallationTokenRetry(env, installationId, async (token) => { + // makeInstallationOctokit injects the shared per-request timeout (a stalled PATCH can never orphan the + // in_progress check) AND suppresses the check-run writes under a non-live mode (dry-run / pause / freeze). + const octokit = makeInstallationOctokit(env, token, check.mode); + // Point the merge-box "Details" link at the repo's Gittensory maintainer panel instead of GitHub's generic + // check page. Spread conditionally so a URL-construction failure (null) just omits it. (#audit-details-url) + const detailsUrl = maintainerControlPanelUrl(env, repoFullName); + const detailsUrlBody = detailsUrl ? { details_url: detailsUrl } : {}; + + // POST a fresh check-run THIS App owns. Used for a brand-new run AND as the cross-app fallback below. + const postNewCheckRun = async (): Promise => { const response = await octokit.request( - "PATCH /repos/{owner}/{repo}/check-runs/{check_run_id}", + "POST /repos/{owner}/{repo}/check-runs", { owner, repo, - check_run_id: id, name: check.name, + head_sha: headSha, status: check.status ?? "completed", ...(check.conclusion ? { conclusion: check.conclusion } : {}), - output: outputForCheckRunUpdate(check.output), + output: check.output, ...detailsUrlBody, }, ); return publishedOutcome(response.data as CheckRunResponse); - } catch (error) { - if (!isCrossAppCheckRunError(error)) throw error; - console.log( - JSON.stringify({ - level: "info", - event: "check_run_cross_app_repost", - repository: `${owner}/${repo}`, - staleCheckRunId: id, - }), - ); - return null; - } - }; + }; + // PATCH an existing run by id. If that run was created by a PRIOR GitHub App (install migrated / reinstalled under a + // new app_id), GitHub 403s "can only be modified by the GitHub App that created it" — that stale run is unreachable, + // so fall through (null) to POST a fresh one this App owns instead of failing the gate forever. (#cross-app-checkrun) + const patchCheckRun = async (id: number): Promise => { + try { + const response = await octokit.request( + "PATCH /repos/{owner}/{repo}/check-runs/{check_run_id}", + { + owner, + repo, + check_run_id: id, + name: check.name, + status: check.status ?? "completed", + ...(check.conclusion ? { conclusion: check.conclusion } : {}), + output: outputForCheckRunUpdate(check.output), + ...detailsUrlBody, + }, + ); + return publishedOutcome(response.data as CheckRunResponse); + } catch (error) { + if (!isCrossAppCheckRunError(error)) throw error; + console.log( + JSON.stringify({ + level: "info", + event: "check_run_cross_app_repost", + repository: `${owner}/${repo}`, + staleCheckRunId: id, + }), + ); + return null; + } + }; - try { - if (check.checkRunId) { - const out = await patchCheckRun(check.checkRunId); - if (out) return out; - } else if (check.updateExisting !== "never") { - const existing = await octokit.request( - "GET /repos/{owner}/{repo}/commits/{ref}/check-runs", - { - owner, - repo, - ref: advisory.headSha, - check_name: check.name, - filter: "latest", - per_page: 1, - }, - ); - const existingCheckRun = (existing.data as CheckRunListResponse) - .check_runs?.[0]; - if ( - existingCheckRun && - (check.updateExisting !== "in_progress_only" || - (existingCheckRun.status ?? "").toLowerCase() !== "completed") - ) { - const out = await patchCheckRun(existingCheckRun.id); + try { + if (check.checkRunId) { + const out = await patchCheckRun(check.checkRunId); if (out) return out; + } else if (check.updateExisting !== "never") { + const existing = await octokit.request( + "GET /repos/{owner}/{repo}/commits/{ref}/check-runs", + { + owner, + repo, + ref: headSha, + check_name: check.name, + filter: "latest", + per_page: 1, + }, + ); + const existingCheckRun = (existing.data as CheckRunListResponse) + .check_runs?.[0]; + if ( + existingCheckRun && + (check.updateExisting !== "in_progress_only" || + (existingCheckRun.status ?? "").toLowerCase() !== "completed") + ) { + const out = await patchCheckRun(existingCheckRun.id); + if (out) return out; + } } + return await postNewCheckRun(); + } catch (error) { + if (isCheckRunPermissionError(error)) { + // Capture the ACTUAL response (status + body). A 403 here is often NOT a real permission gap (the App has + // Checks:write) — it can be a per-PR access quirk (e.g. a fork-head commit the App can't write to) — and this + // log is the only way to tell why, instead of an opaque "permission missing". Surfaces to Sentry with a real + // message via console.error (#review-403-context). + const e = error as { status?: number; message?: string }; + console.error( + JSON.stringify({ + level: "error", + event: "check_run_post_denied", + repository: `${owner}/${repo}`, + status: e.status ?? null, + message: (e.message ?? "Resource not accessible by integration").slice( + 0, + 300, + ), + }), + ); + return { + kind: "permission_missing", + warning: + "GitHub App Checks: write permission is missing. Enable it in the GitHub App settings and re-approve the installation.", + }; + } + throw error; } - return await postNewCheckRun(); - } catch (error) { - if (isCheckRunPermissionError(error)) { - // Capture the ACTUAL response (status + body). A 403 here is often NOT a real permission gap (the App has - // Checks:write) — it can be a per-PR access quirk (e.g. a fork-head commit the App can't write to) — and this - // log is the only way to tell why, instead of an opaque "permission missing". Surfaces to Sentry with a real - // message via console.error (#review-403-context). - const e = error as { status?: number; message?: string }; - console.error( - JSON.stringify({ - level: "error", - event: "check_run_post_denied", - repository: `${owner}/${repo}`, - status: e.status ?? null, - message: (e.message ?? "Resource not accessible by integration").slice( - 0, - 300, - ), - }), - ); - return { - kind: "permission_missing", - warning: - "GitHub App Checks: write permission is missing. Enable it in the GitHub App settings and re-approve the installation.", - }; - } - throw error; - } + }); } function outputForCheckRunUpdate(output: CheckRunOutput): CheckRunOutput { diff --git a/src/github/comments.ts b/src/github/comments.ts index f31a2d7bb2..3c2c1f96bb 100644 --- a/src/github/comments.ts +++ b/src/github/comments.ts @@ -1,4 +1,4 @@ -import { createInstallationToken } from "./app"; +import { withInstallationTokenRetry } from "./app"; import { makeInstallationOctokit } from "./client"; import type { AgentActionMode } from "../settings/agent-execution"; @@ -53,46 +53,47 @@ async function createOrUpdateIssueCommentWithMarker( const [owner, repo] = repoFullName.split("/"); if (!owner || !repo) throw new Error(`Invalid repository full name: ${repoFullName}`); - const token = await createInstallationToken(env, installationId); - // Non-live mode suppresses the comment create/update writes; the GET marker-search probe below still runs. - const octokit = makeInstallationOctokit(env, token, options.mode ?? "live"); - const botLogin = `${env.GITHUB_APP_SLUG}[bot]`; - const markers = markerAliases(marker); - let existing: IssueComment | undefined; - for (let page = 1; !existing && page <= COMMENT_SEARCH_PAGE_LIMIT; page += 1) { - const response = await octokit.request("GET /repos/{owner}/{repo}/issues/{issue_number}/comments", { + return await withInstallationTokenRetry(env, installationId, async (token) => { + // Non-live mode suppresses the comment create/update writes; the GET marker-search probe below still runs. + const octokit = makeInstallationOctokit(env, token, options.mode ?? "live"); + const botLogin = `${env.GITHUB_APP_SLUG}[bot]`; + const markers = markerAliases(marker); + let existing: IssueComment | undefined; + for (let page = 1; !existing && page <= COMMENT_SEARCH_PAGE_LIMIT; page += 1) { + const response = await octokit.request("GET /repos/{owner}/{repo}/issues/{issue_number}/comments", { + owner, + repo, + issue_number: issueNumber, + per_page: 100, + page, + }); + const batch = response.data as IssueComment[]; + existing = batch.find((comment) => isGittensoryBotComment(comment, botLogin) && markers.some((candidate) => comment.body?.includes(candidate))); + if (batch.length < 100) break; + } + if (existing) { + // Idempotency (#4): skip the PATCH when the rendered body is byte-identical to what's already posted. The + // re-gate sweep re-renders the same surface every cycle for an unchanged PR; without this, every cycle PATCHes + // GitHub (a write + rate-limit cost) for no visible change. Defense-in-depth alongside the head_sha publish + // marker — also collapses a duplicate webhook delivery for the same commit. + if (existing.body === body) return { id: existing.id, ...(existing.html_url !== undefined ? { html_url: existing.html_url } : {}) }; + const response = await octokit.request("PATCH /repos/{owner}/{repo}/issues/comments/{comment_id}", { + owner, + repo, + comment_id: existing.id, + body, + }); + return response.data as { id: number; html_url?: string }; + } + if (options.createIfMissing === false) return null; + const response = await octokit.request("POST /repos/{owner}/{repo}/issues/{issue_number}/comments", { owner, repo, issue_number: issueNumber, - per_page: 100, - page, - }); - const batch = response.data as IssueComment[]; - existing = batch.find((comment) => isGittensoryBotComment(comment, botLogin) && markers.some((candidate) => comment.body?.includes(candidate))); - if (batch.length < 100) break; - } - if (existing) { - // Idempotency (#4): skip the PATCH when the rendered body is byte-identical to what's already posted. The - // re-gate sweep re-renders the same surface every cycle for an unchanged PR; without this, every cycle PATCHes - // GitHub (a write + rate-limit cost) for no visible change. Defense-in-depth alongside the head_sha publish - // marker — also collapses a duplicate webhook delivery for the same commit. - if (existing.body === body) return { id: existing.id, ...(existing.html_url !== undefined ? { html_url: existing.html_url } : {}) }; - const response = await octokit.request("PATCH /repos/{owner}/{repo}/issues/comments/{comment_id}", { - owner, - repo, - comment_id: existing.id, body, }); return response.data as { id: number; html_url?: string }; - } - if (options.createIfMissing === false) return null; - const response = await octokit.request("POST /repos/{owner}/{repo}/issues/{issue_number}/comments", { - owner, - repo, - issue_number: issueNumber, - body, }); - return response.data as { id: number; html_url?: string }; } function isGittensoryBotComment(comment: IssueComment, botLogin: string): boolean { diff --git a/test/unit/github-app.test.ts b/test/unit/github-app.test.ts index 90d5f7a507..047ce05ebd 100644 --- a/test/unit/github-app.test.ts +++ b/test/unit/github-app.test.ts @@ -146,6 +146,47 @@ describe("GitHub check runs", () => { expect(mints).toBe(1); }); + it("expires a rejected cached installation token and retries check-run publication once", async () => { + const privateKey = await generatePrivateKeyPem(); + let mints = 0; + let rejectedReads = 0; + vi.stubGlobal("fetch", async (input: RequestInfo | URL, init?: RequestInit) => { + const url = input.toString(); + if (url.includes("/access_tokens")) { + mints += 1; + return Response.json({ + token: mints === 1 ? "stale-token" : "fresh-token", + expires_at: new Date(Date.now() + 60 * 60_000).toISOString(), + }); + } + const auth = new Headers(init?.headers).get("authorization") ?? ""; + if (url.includes("/commits/stale-head/check-runs") && auth.includes("stale-token")) { + rejectedReads += 1; + return Response.json({ message: "Bad credentials" }, { status: 401 }); + } + if (url.includes("/commits/stale-head/check-runs")) { + expect(auth).toContain("fresh-token"); + return Response.json({ total_count: 0, check_runs: [] }); + } + if (url.includes("/check-runs") && init?.method === "POST") { + expect(auth).toContain("fresh-token"); + return Response.json({ id: 556, html_url: "https://github.com/checks/556" }, { status: 201 }); + } + return new Response("not found", { status: 404 }); + }); + + const result = await createOrUpdatePendingGateCheckRun( + createTestEnv({ GITHUB_APP_PRIVATE_KEY: privateKey }), + 556, + "JSONbored/gittensory", + gateAdvisory("stale-head"), + ); + + expect(result).toMatchObject({ kind: "published", id: 556 }); + expect(mints).toBe(2); + expect(rejectedReads).toBe(1); + }); + it("single-flights concurrent cold-cache mints for one install (no thundering herd)", async () => { const privateKey = await generatePrivateKeyPem(); let mints = 0; diff --git a/test/unit/github-comments.test.ts b/test/unit/github-comments.test.ts index 483e2b7760..49b425cb54 100644 --- a/test/unit/github-comments.test.ts +++ b/test/unit/github-comments.test.ts @@ -35,6 +35,48 @@ describe("GitHub PR intelligence comments", () => { expect(calls.some((call) => call.startsWith("POST ") && call.includes("/issues/12/comments"))).toBe(true); }); + it("expires a rejected cached installation token and retries PR panel publication once", async () => { + const privateKey = await generatePrivateKeyPem(); + let mints = 0; + let rejectedReads = 0; + vi.stubGlobal("fetch", async (input: RequestInfo | URL, init?: RequestInit) => { + const url = input.toString(); + if (url.includes("/access_tokens")) { + mints += 1; + return Response.json({ + token: mints === 1 ? "stale-token" : "fresh-token", + expires_at: new Date(Date.now() + 60 * 60_000).toISOString(), + }); + } + const auth = new Headers(init?.headers).get("authorization") ?? ""; + if (url.includes("/issues/12/comments") && auth.includes("stale-token")) { + rejectedReads += 1; + return Response.json({ message: "Bad credentials" }, { status: 401 }); + } + if (url.includes("/issues/12/comments") && (init?.method ?? "GET") === "GET") { + expect(auth).toContain("fresh-token"); + return Response.json([]); + } + if (url.includes("/issues/12/comments") && init?.method === "POST") { + expect(auth).toContain("fresh-token"); + return Response.json({ id: 515, html_url: "https://github.com/comment/515" }); + } + return new Response("not found", { status: 404 }); + }); + + const result = await createOrUpdatePrIntelligenceComment( + createTestEnv({ GITHUB_APP_PRIVATE_KEY: privateKey }), + 9988, + "JSONbored/gittensory", + 12, + `${PR_INTELLIGENCE_COMMENT_MARKER}\nbody`, + ); + + expect(result?.id).toBe(515); + expect(mints).toBe(2); + expect(rejectedReads).toBe(1); + }); + it("updates an existing sticky comment instead of duplicating it", async () => { const privateKey = await generatePrivateKeyPem(); const calls: string[] = []; From 3ae58f09e6d5893896ab9b8e3b656ac0b349a7c0 Mon Sep 17 00:00:00 2001 From: JSONbored <49853598+JSONbored@users.noreply.github.com> Date: Sun, 28 Jun 2026 15:45:13 -0700 Subject: [PATCH 23/68] fix(review): refresh stale surfaces after CI settles --- src/db/repositories.ts | 9 ++--- src/db/schema.ts | 9 ++--- src/github/backfill.ts | 30 +++++++++++---- src/queue/processors.ts | 40 ++++++++------------ src/selfhost/pg-queue.ts | 37 ++++++++++++++++-- src/selfhost/sqlite-queue.ts | 39 +++++++++++++++++-- src/types.ts | 6 +-- test/unit/backfill.test.ts | 24 ++++++++++++ test/unit/queue.test.ts | 22 +++++------ test/unit/selfhost-pg-queue.test.ts | 19 +++++++++- test/unit/selfhost-sqlite-queue.test.ts | 50 +++++++++++++++++++++++-- 11 files changed, 216 insertions(+), 69 deletions(-) diff --git a/src/db/repositories.ts b/src/db/repositories.ts index 5b791437c1..374317d028 100644 --- a/src/db/repositories.ts +++ b/src/db/repositories.ts @@ -2658,11 +2658,10 @@ export async function markPullRequestApproved(env: Env, fullName: string, number .where(and(eq(pullRequests.repoFullName, fullName), eq(pullRequests.number, number), eq(pullRequests.headSha, headSha))); } -/** Over-publish dedup (#4): record the head SHA at which the PR's public surface was just published. The scheduled - * re-gate sweep skips re-reviewing while last_published_surface_sha == headSha. Scoped to headSha so a later commit - * (push/rebase/force-push — the live head no longer matches) re-publishes the new code without any manual reset. - * The eq(headSha) in the WHERE is load-bearing: if the live head advanced between review and this write, the UPDATE - * no-ops (never stamps a stale head) → the next sweep correctly re-reviews. Mirrors markPullRequestApproved. */ +/** Public-surface marker: record the head SHA at which the PR's public surface was just published. This is + * reporting/diagnostic state, not a hard scheduled-sweep skip; GitHub comments/checks can be stale or partial even + * when the marker matches the current head. The eq(headSha) in the WHERE is load-bearing: if the live head advanced + * between review and this write, the UPDATE no-ops (never stamps a stale head). Mirrors markPullRequestApproved. */ export async function markPullRequestSurfacePublished(env: Env, fullName: string, number: number, headSha: string | null | undefined): Promise { if (!headSha) return; // no head to key the marker on → nothing to stamp (the caller's advisory had no head SHA) const db = getDb(env.DB); diff --git a/src/db/schema.ts b/src/db/schema.ts index 396992ab51..ac475f4d7c 100644 --- a/src/db/schema.ts +++ b/src/db/schema.ts @@ -304,11 +304,10 @@ export const pullRequests = sqliteTable( // review WRITE that would bump updated_at is suppressed (dry-run / paused). gittensory-computed (sweep-written), // omitted from the GitHub-sync SET clause so a later sync cannot clobber it. (Mirrors approved_head_sha.) lastRegatedAt: text("last_regated_at"), - // Over-publish dedup: the head SHA at which the public surface (comment/label/check-run) was LAST published. - // The sweep skips re-reviewing + re-publishing a PR while last_published_surface_sha === headSha (already - // current). Keyed to head SHA → a push/rebase/force-push (new head) clears the match and the next sweep - // re-reviews + re-publishes. gittensory-computed (publish-written), omitted from the GitHub-sync SET clause so - // a later sync cannot clobber it. (Mirrors approved_head_sha.) + // Public-surface marker: the head SHA at which the public surface (comment/label/check-run) was LAST published. + // Used for reporting and stale-surface diagnostics, not as a hard sweep skip; GitHub comments/checks can still + // be stale or partial while this marker matches headSha. gittensory-computed (publish-written), omitted from + // the GitHub-sync SET clause so a later sync cannot clobber it. (Mirrors approved_head_sha.) lastPublishedSurfaceSha: text("last_published_surface_sha"), createdAt: text("created_at").notNull().$defaultFn(() => nowIso()), updatedAt: text("updated_at").notNull().$defaultFn(() => nowIso()), diff --git a/src/github/backfill.ts b/src/github/backfill.ts index a4fcfbaf47..516a804e16 100644 --- a/src/github/backfill.ts +++ b/src/github/backfill.ts @@ -1930,6 +1930,10 @@ function isOwnGitHubAppCheckRun(env: Env, run: GitHubCheckRunPayload): boolean { export type LiveCiAggregate = { ciState: "passed" | "failed" | "pending" | "unverified"; + // Any non-bot CI source that is still pending, inferred missing, or unreadable. This is deliberately broader + // than ciState: a non-required pending check must not fail the gate, but review execution should still wait + // until every visible CI signal has settled. + hasPending: boolean; // Checks that FAIL the gate: every failing check when required contexts are unknown, else only the failing // REQUIRED contexts. These drive ciState === "failed" and the disposition (no-merge / close / request-changes). failingDetails: Array<{ name: string; summary?: string; detailsUrl?: string }>; @@ -1988,13 +1992,14 @@ export async function fetchLiveCiAggregate( // back to gating on all contexts to avoid silently passing an unknown required failure. requiredContexts?: ReadonlySet | null, ): Promise { - if (!headSha) return { ciState: "unverified", failingDetails: [], nonRequiredFailingDetails: [] }; + if (!headSha) return { ciState: "unverified", hasPending: false, failingDetails: [], nonRequiredFailingDetails: [] }; const enforceRequiredOnly = requiredContexts != null && requiredContexts.size > 0; const isRequired = (name: string): boolean => !enforceRequiredOnly || requiredContexts.has(name); const failingDetails: LiveCiAggregate["failingDetails"] = []; const nonRequiredFailingDetails: LiveCiAggregate["nonRequiredFailingDetails"] = []; let total = 0; let anyPending = false; + let anyVisiblePending = false; // CI visibility flags: a failed/short read of either source means we did NOT enumerate the commit's full check // set, so we must not certify it "passed". They drive the fail-CLOSED degrade below. In enforce-required mode // the absent-context guard already catches this; this additionally closes the fold-all (unknown-required) seam @@ -2036,8 +2041,9 @@ export async function fetchLiveCiAggregate( (isRequired(run.name) ? failingDetails : nonRequiredFailingDetails).push(detail); } else if (conclusion ? CI_PASSING_CONCLUSIONS.has(conclusion) : status === "completed") { // concluded and not failing → passing - } else if (isRequired(run.name)) { - anyPending = true; // queued / in_progress / not yet concluded — only a REQUIRED check holds the gate + } else { + anyVisiblePending = true; + if (isRequired(run.name)) anyPending = true; // queued / in_progress / not yet concluded — only a REQUIRED check holds the gate } } if (!hasNextPage(result.link)) break; @@ -2074,8 +2080,9 @@ export async function fetchLiveCiAggregate( (isRequired(name) ? failingDetails : nonRequiredFailingDetails).push(detail); } else if (state === "success") { // passing - } else if (isRequired(name)) { - anyPending = true; // pending — only a REQUIRED context holds the gate + } else { + anyVisiblePending = true; + if (isRequired(name)) anyPending = true; // pending — only a REQUIRED context holds the gate } } @@ -2084,7 +2091,10 @@ export async function fetchLiveCiAggregate( // trigger on forks, or a check that was skipped). if (seenContextNames) { for (const ctx of requiredContexts!) { - if (!seenContextNames.has(ctx)) anyPending = true; + if (!seenContextNames.has(ctx)) { + anyPending = true; + anyVisiblePending = true; + } } } @@ -2125,7 +2135,13 @@ export async function fetchLiveCiAggregate( // commit as passed/clean — hold (pending) so the gate waits and re-evaluates on the next sweep instead of // auto-merging on partial data. An OBSERVED failure ("failed") is authoritative and preserved. if ((checkRunsIncomplete || statusIncomplete) && ciState !== "failed") ciState = "pending"; - return { ciState, failingDetails, nonRequiredFailingDetails }; + const hasPending = + anyVisiblePending || + anyPending || + checkRunsIncomplete || + statusIncomplete || + ciState === "pending"; + return { ciState, hasPending, failingDetails, nonRequiredFailingDetails }; } /** diff --git a/src/queue/processors.ts b/src/queue/processors.ts index 742c1f3d58..9ea9f40019 100644 --- a/src/queue/processors.ts +++ b/src/queue/processors.ts @@ -1107,7 +1107,9 @@ async function sweepRepoRegate( // dry-run between fan-out and processing stays inert. Self-contained: resolves the repo settings to mirror the // sweep's skipAiReview policy. The convergence marker is NOT stamped here — the sweep already stamped every // candidate at dispatch (#audit-sweep-dispatch-stamp), so the in-flight guard does not wait on this job and a -// deferred/failed re-review never stalls convergence (the next sweep after the window re-claims the PR). +// deferred/failed re-review never stalls convergence (the next sweep after the window re-claims the PR). The public +// surface marker is observability only; it cannot prove GitHub still shows a complete current review panel, so the +// per-PR job always re-evaluates the head. async function regatePullRequest( env: Env, repoFullName: string, @@ -1149,7 +1151,6 @@ async function regatePullRequest( // re-spend, so an advisory PR gets a posted review without burning a token every sweep tick. { skipAiReview: settings.aiReviewMode === "off", - skipWhenSurfaceCurrent: true, }, ).catch((error) => { console.error( @@ -1480,7 +1481,7 @@ async function reReviewStoredPullRequest( repoFullName: string, prNumber: number, previewPollAttempt?: number, - options: { skipAiReview?: boolean; skipWhenSurfaceCurrent?: boolean } = {}, + options: { skipAiReview?: boolean } = {}, ): Promise { const [repo, settings] = await Promise.all([ getRepository(env, repoFullName), @@ -1513,16 +1514,6 @@ async function reReviewStoredPullRequest( /* v8 ignore next -- the row was just upserted above, so the re-read always returns it; `?? pr` is belt-and-suspenders fail-open. */ pr = (await getPullRequest(env, repoFullName, prNumber)) ?? pr; } - // Over-publish dedup (#4): only scheduled sweeps opt into the head-only shortcut. Event-driven - // re-reviews (CI completion, deployment/preview refreshes) must re-evaluate dynamic same-head state. - if ( - options.skipWhenSurfaceCurrent && - pr.lastPublishedSurfaceSha && - pr.lastPublishedSurfaceSha === pr.headSha - ) { - console.log(JSON.stringify({ level: "info", event: "rereview_skipped_surface_current", deliveryId, repository: repoFullName, pullNumber: prNumber, headSha: pr.headSha })); - return; - } // Operator review flow: rebase-if-behind → wait for ALL CI to finish → only THEN review. Defers (returns) when // a rebase fired a synchronize, or CI is still running — the synchronize / CI-completion webhook re-triggers // once the head is current and CI has settled (the sweep backstops a missed event). REST-budget dedup @@ -1629,10 +1620,9 @@ async function reReviewStoredPullRequest( * Operator per-PR review flow (rebase → wait for ALL CI → review once). Returns TRUE to review NOW, FALSE to * DEFER: * - BEHIND base → issue update-branch; the resulting `synchronize` re-triggers on the rebased head. - * - CI still RUNNING (any check pending, none failed yet → ciState "pending") → wait; the check_run/check_suite - * `completed` webhook re-triggers once CI settles (the sweep backstops a missed event). A RED check - * (ciState "failed") does NOT defer — a bad PR is reviewed + closed promptly; only a green-so-far-but-still- - * running PR waits, so we never merge before every check is green. + * - CI still RUNNING (any non-bot check/status pending, regardless of whether it is branch-protection-required) + * → wait; the check_run/check_suite `completed` webhook re-triggers once CI settles (the sweep backstops a + * missed event). Once settled, only the gate disposition can block/close; readiness remains advisory. * Agent-OFF / draft / no-head PRs are never gated (reviewed as before). Fail-OPEN on a token/API hiccup (review * rather than stall a PR forever). */ @@ -1702,7 +1692,8 @@ async function prReadyForReview( } // Not authorized, staged, dry-run, or failed (conflict/transient) → fall through and review without mutating. } - // 2) wait for trusted required CI to finish. Non-required checks are advisory and must not stall review. + // 2) wait for CI to finish before running the Gittensory review. Required contexts still define which failures + // block/close, but hasPending tracks any visible non-bot CI that is not settled yet. const requiredContexts = await fetchRequiredStatusContexts( env, repoFullName, @@ -1716,7 +1707,7 @@ async function prReadyForReview( token, requiredContexts, ).catch(() => undefined); - if (ci?.ciState === "pending") { + if (ci?.hasPending) { // Staleness cap: genuinely-running CI settles in minutes. A required check that stays pending far longer // (an orphaned / never-completing check — e.g. a fork check that never reports back) would otherwise make us // defer FOREVER → the PR is silently stuck and never surfaces (the dominant metagraphed stall). Past @@ -1742,7 +1733,7 @@ async function prReadyForReview( targetKey: `${repoFullName}#${pr.number}`, outcome: "completed", detail: - "required CI stuck pending past the staleness cap — finalizing so the PR is surfaced, not silently deferred forever", + "CI stuck pending past the staleness cap — finalizing so the PR is surfaced, not silently deferred forever", metadata: { deliveryId, repoFullName }, }).catch(() => undefined); // fall through → return true → the gate finalizes + the PR is disposed/held, never silently stuck. @@ -5365,10 +5356,11 @@ async function maybePublishPrPublicSurface( failedOutputs, }, }); - // Over-publish dedup (#4): stamp the head SHA we just published at, so the scheduled sweep skips re-reviewing + - // re-publishing this PR until its head changes (see the guard in reReviewStoredPullRequest). Reached only when at - // least one surface output actually published (the zero-output early-return above covers the suppressed/dry-run - // case). The helper no-ops on a null head, and its WHERE pins head_sha so a head that advanced mid-pass won't stamp. + // Stamp the head SHA we just published at for reporting and stale-surface diagnostics. This is not a hard + // re-review skip: GitHub comments/checks can be stale or incomplete even when this marker matches the current + // head. Reached only when at least one surface output actually published (the zero-output early-return above + // covers the suppressed/dry-run case). The helper no-ops on a null head, and its WHERE pins head_sha so a head + // that advanced mid-pass won't stamp. await markPullRequestSurfacePublished(env, repoFullName, pr.number, advisory.headSha).catch((error) => { console.error(JSON.stringify({ level: "warn", event: "surface_published_mark_failed", repoFullName, pullNumber: pr.number, error: errorMessage(error) })); }); diff --git a/src/selfhost/pg-queue.ts b/src/selfhost/pg-queue.ts index a82b66e2ac..2e0ff22097 100644 --- a/src/selfhost/pg-queue.ts +++ b/src/selfhost/pg-queue.ts @@ -24,11 +24,16 @@ ALTER TABLE ${TABLE} ADD COLUMN IF NOT EXISTS priority INTEGER NOT NULL DEFAULT CREATE INDEX IF NOT EXISTS ${TABLE}_claim ON ${TABLE}(status, run_after, priority);`; // Webhook-driven work (a fresh PR → its review) jumps ahead of heavy background jobs (rag-index, the regate sweep) -// so a NEW PR is reviewed promptly instead of waiting behind them in the shared FIFO queue. Additive: all other -// jobs stay priority 0 (today's FIFO order). Mirrors sqlite-queue. (#review-latency) -const HIGH_PRIORITY_TYPES = new Set(["github-webhook"]); +// so a NEW PR is reviewed promptly instead of waiting behind them in the shared FIFO queue. Per-PR review refreshes +// sit just below webhooks: they repair stale GitHub surfaces quickly without starving fresh webhook work. Additive: +// all other jobs stay priority 0 (today's FIFO order). Mirrors sqlite-queue. (#review-latency) +const PRIORITY_BY_TYPE = new Map([ + ["github-webhook", 10], + ["agent-regate-pr", 9], + ["recapture-preview", 9], +]); function jobPriority(payload: string): number { - return HIGH_PRIORITY_TYPES.has(extractPayloadType(payload) ?? "") ? 10 : 0; + return PRIORITY_BY_TYPE.get(extractPayloadType(payload) ?? "") ?? 0; } export interface PgDurableQueue { @@ -77,6 +82,30 @@ export function createPgQueue( async function init(): Promise { await pool.query(DDL); + const priorityBackfilled = + ( + await pool.query( + `UPDATE ${TABLE} + SET priority = CASE + WHEN payload ~ '"type"[[:space:]]*:[[:space:]]*"github-webhook"' THEN 10 + WHEN payload ~ '"type"[[:space:]]*:[[:space:]]*"(agent-regate-pr|recapture-preview)"' THEN 9 + ELSE 0 + END + WHERE status IN ('pending', 'processing') + AND priority IS DISTINCT FROM CASE + WHEN payload ~ '"type"[[:space:]]*:[[:space:]]*"github-webhook"' THEN 10 + WHEN payload ~ '"type"[[:space:]]*:[[:space:]]*"(agent-regate-pr|recapture-preview)"' THEN 9 + ELSE 0 + END`, + ) + ).rowCount ?? 0; + if (priorityBackfilled) + console.log( + JSON.stringify({ + event: "selfhost_queue_priority_backfilled", + count: priorityBackfilled, + }), + ); const recovered = ( await pool.query( diff --git a/src/selfhost/sqlite-queue.ts b/src/selfhost/sqlite-queue.ts index 3a79ac0c0f..006e778104 100644 --- a/src/selfhost/sqlite-queue.ts +++ b/src/selfhost/sqlite-queue.ts @@ -26,11 +26,16 @@ DROP INDEX IF EXISTS ${TABLE}_claim; CREATE INDEX ${TABLE}_claim ON ${TABLE}(status, run_after, priority);`; // Webhook-driven work (a fresh PR → its review) jumps ahead of heavy background jobs (rag-index ~4min, the regate -// sweep) so a NEW PR is reviewed promptly instead of waiting behind them in the shared FIFO queue. Additive: every -// other job stays priority 0 (today's FIFO order), so only github-webhook moves. (#review-latency) -const HIGH_PRIORITY_TYPES = new Set(["github-webhook"]); +// sweep) so a NEW PR is reviewed promptly instead of waiting behind them in the shared FIFO queue. Per-PR review +// refreshes sit just below webhooks: they repair stale GitHub surfaces quickly without starving fresh webhook work. +// Additive: every other job stays priority 0 (today's FIFO order). (#review-latency) +const PRIORITY_BY_TYPE = new Map([ + ["github-webhook", 10], + ["agent-regate-pr", 9], + ["recapture-preview", 9], +]); function jobPriority(payload: string): number { - return HIGH_PRIORITY_TYPES.has(extractPayloadType(payload) ?? "") ? 10 : 0; + return PRIORITY_BY_TYPE.get(extractPayloadType(payload) ?? "") ?? 0; } export interface DurableQueue { @@ -83,6 +88,14 @@ export function createSqliteQueue( /* column already present */ } driver.exec(CLAIM_INDEX_DDL); + const priorityBackfilled = backfillJobPriorities(driver); + if (priorityBackfilled) + console.log( + JSON.stringify({ + event: "selfhost_queue_priority_backfilled", + count: priorityBackfilled, + }), + ); // Recover jobs a crashed previous run left mid-flight → make them claimable again. const recovered = driver.query( `UPDATE ${TABLE} SET status='pending' WHERE status='processing'`, @@ -292,3 +305,21 @@ export function createSqliteQueue( }, }; } + +function backfillJobPriorities(driver: SqliteDriver): number { + const { rows } = driver.query( + `SELECT id, payload, priority FROM ${TABLE} WHERE status IN ('pending', 'processing')`, + [], + ); + let changed = 0; + for (const row of rows as Array<{ id: number; payload: string; priority: number }>) { + const priority = jobPriority(row.payload); + if (priority === Number(row.priority ?? 0)) continue; + driver.query(`UPDATE ${TABLE} SET priority=? WHERE id=?`, [ + priority, + row.id, + ]); + changed += 1; + } + return changed; +} diff --git a/src/types.ts b/src/types.ts index eea8fab367..9a2793fe5a 100644 --- a/src/types.ts +++ b/src/types.ts @@ -453,9 +453,9 @@ export type PullRequestRecord = { * review write that would bump updatedAt is suppressed (dry-run / paused). Sweep-written; read straight from * the row (never the GitHub payload). */ lastRegatedAt?: string | null | undefined; - /** Over-publish dedup: the head SHA at which the public surface was last published. The re-gate sweep skips - * re-reviewing + re-publishing while lastPublishedSurfaceSha === headSha; a new commit (push/rebase/force-push) - * clears the match so the surface re-publishes the new code. Publish-written; read straight from the row. */ + /** Public-surface marker: the head SHA at which the public surface was last published. Used for reporting and + * stale-surface diagnostics, not as a hard re-review skip: GitHub comments/checks can still be stale or partial + * while this marker matches headSha. Publish-written; read straight from the row. */ lastPublishedSurfaceSha?: string | null | undefined; }; diff --git a/test/unit/backfill.test.ts b/test/unit/backfill.test.ts index 65dd97559f..2ab64a3d20 100644 --- a/test/unit/backfill.test.ts +++ b/test/unit/backfill.test.ts @@ -2817,10 +2817,34 @@ describe("GitHub backfill", () => { const aggregate = await fetchLiveCiAggregate(env, "JSONbored/gittensory", "abc123", "public-token", new Set(["trusted-required-ci"])); expect(aggregate.ciState).toBe("passed"); + expect(aggregate.hasPending).toBe(true); expect(aggregate.failingDetails).toEqual([]); expect(aggregate.nonRequiredFailingDetails.map((detail) => detail.name).sort()).toEqual(["attacker/non-required-check", "attacker/non-required-status"]); }); + it("keeps an observed failure failed while still reporting pending CI separately", async () => { + const env = createTestEnv({ GITHUB_PUBLIC_TOKEN: "public-token" }); + vi.stubGlobal("fetch", async (input: RequestInfo | URL) => { + const url = input.toString(); + if (url.includes("/check-runs?")) { + return Response.json({ + check_runs: [ + { name: "test", status: "completed", conclusion: "failure", output: { title: "Test failed" } }, + { name: "coverage", status: "in_progress", conclusion: null }, + ], + }); + } + if (url.includes("/status?")) return Response.json({ statuses: [] }); + return new Response("not found", { status: 404 }); + }); + + const aggregate = await fetchLiveCiAggregate(env, "JSONbored/gittensory", "abc123", "public-token", null); + + expect(aggregate.ciState).toBe("failed"); + expect(aggregate.hasPending).toBe(true); + expect(aggregate.failingDetails).toEqual([expect.objectContaining({ name: "test" })]); + }); + it("falls back to gating all contexts when required contexts are unavailable", async () => { const env = createTestEnv({ GITHUB_PUBLIC_TOKEN: "public-token" }); vi.stubGlobal("fetch", async (input: RequestInfo | URL) => { diff --git a/test/unit/queue.test.ts b/test/unit/queue.test.ts index dba7b140c6..310ac36329 100644 --- a/test/unit/queue.test.ts +++ b/test/unit/queue.test.ts @@ -865,13 +865,13 @@ describe("queue processors", () => { resyncUpsertSpy.mockRestore(); }); - it("#4 over-publish dedup: the sweep SKIPS re-review when the surface was already published at the current head", async () => { + it("#4 stale-surface repair: the sweep re-reviews even when the local surface marker already matches the current head", async () => { const env = createTestEnv({ GITHUB_APP_PRIVATE_KEY: await generatePrivateKeyPem() }); await upsertInstallation(env, { action: "created", installation: { id: 9001, account: { login: "owner", id: 1, type: "Organization" }, target_type: "Organization", repository_selection: "selected", permissions: {}, events: [] } }); await upsertRepositoryFromGitHub(env, { name: "agent-repo", full_name: "owner/agent-repo", private: false, owner: { login: "owner" } }, 9001); await upsertRepositorySettings(env, { repoFullName: "owner/agent-repo", autonomy: { merge: "auto" }, aiReviewMode: "off", gatePack: "oss-anti-slop", gateCheckMode: "enabled", checkRunMode: "off", commentMode: "off", publicSurface: "off" }); await upsertPullRequestFromGitHub(env, "owner/agent-repo", { number: 7, title: "Current PR", state: "open", user: { login: "contributor" }, head: { sha: "a7" }, labels: [], body: "Closes #1" }); - await repositoriesModule.markPullRequestSurfacePublished(env, "owner/agent-repo", 7, "a7"); // already published at the live head + await repositoriesModule.markPullRequestSurfacePublished(env, "owner/agent-repo", 7, "a7"); // marker says current, but GitHub may still show a stale/partial panel let checkRunsFetched = false; vi.stubGlobal("fetch", async (input: RequestInfo | URL) => { const url = input.toString(); @@ -883,13 +883,14 @@ describe("queue processors", () => { }); vi.setSystemTime(new Date("2026-05-28T02:00:00.000Z")); - await processJob(env, { type: "agent-regate-pr", deliveryId: "skip-current", repoFullName: "owner/agent-repo", prNumber: 7, installationId: 9001 }); + await processJob(env, { type: "agent-regate-pr", deliveryId: "repair-current", repoFullName: "owner/agent-repo", prNumber: 7, installationId: 9001 }); - // The dedup guard returned BEFORE prReadyForReview → no CI check-runs were fetched (the re-review never ran). - expect(checkRunsFetched).toBe(false); + // The marker is not authoritative enough to skip: re-review still reaches prReadyForReview and can repair + // stale legacy/placeholder GitHub surfaces at the same head. + expect(checkRunsFetched).toBe(true); }); - it("#4 over-publish dedup: same-head CI completions bypass the sweep-only surface-current shortcut", async () => { + it("#4 stale-surface repair: same-head CI completions also re-run review when the marker is current", async () => { const env = createTestEnv({ GITHUB_APP_PRIVATE_KEY: await generatePrivateKeyPem(), GITTENSORY_REVIEW_REPOS: "owner/agent-repo" }); await upsertInstallation(env, { action: "created", installation: { id: 9001, account: { login: "owner", id: 1, type: "Organization" }, target_type: "Organization", repository_selection: "selected", permissions: {}, events: [] } }); await upsertRepositoryFromGitHub(env, { name: "agent-repo", full_name: "owner/agent-repo", private: false, owner: { login: "owner" } }, 9001); @@ -921,11 +922,11 @@ describe("queue processors", () => { }); // CI completion is event-driven dynamic state, so it must re-run prReadyForReview even when the last surface - // publish marker already matches this head SHA. Only the scheduled sweep may use the head-only shortcut. + // publish marker already matches this head SHA. expect(checkRunsFetched).toBe(true); }); - it("#4 over-publish dedup: a rebased PR (marker != live head) is NOT skipped — it resyncs + re-reviews at the new head, and the marker survives the resync", async () => { + it("#4 stale-surface repair: a rebased PR resyncs + re-reviews at the new head, and the marker survives the resync", async () => { const env = createTestEnv({ GITHUB_APP_PRIVATE_KEY: await generatePrivateKeyPem() }); await upsertInstallation(env, { action: "created", installation: { id: 9001, account: { login: "owner", id: 1, type: "Organization" }, target_type: "Organization", repository_selection: "selected", permissions: {}, events: [] } }); await upsertRepositoryFromGitHub(env, { name: "agent-repo", full_name: "owner/agent-repo", private: false, owner: { login: "owner" } }, 9001); @@ -948,9 +949,8 @@ describe("queue processors", () => { await processJob(env, { type: "agent-regate-pr", deliveryId: "rebase-rereview", repoFullName: "owner/agent-repo", prNumber: 7, installationId: 9001 }); - // The old-head marker (a7) != the live rebased head (b8) → the guard fell THROUGH → the PR was resynced to b8 and - // re-reviewed at the new head (check-runs fetched at b8). The marker is NOT in the GitHub-sync SET clause, so the - // resync upsert preserved it (still a7) until a fresh publish advances it — proving rebases are never skipped. + // The PR was resynced to b8 and re-reviewed at the new head (check-runs fetched at b8). The marker is NOT in the + // GitHub-sync SET clause, so the resync upsert preserved it (still a7) until a fresh publish advances it. expect(checkRunsFetchedAtNewHead).toBe(true); const stored = await getPullRequest(env, "owner/agent-repo", 7); expect(stored?.headSha).toBe("b8"); diff --git a/test/unit/selfhost-pg-queue.test.ts b/test/unit/selfhost-pg-queue.test.ts index d23a441d26..a9fdc77326 100644 --- a/test/unit/selfhost-pg-queue.test.ts +++ b/test/unit/selfhost-pg-queue.test.ts @@ -51,20 +51,35 @@ describe("createPgQueue (durable #977)", () => { it("init() creates the table and recovers stuck-processing jobs", async () => { const m = makePool(); m.fn.mockResolvedValueOnce({ rows: [], rowCount: 0 }); // DDL + m.fn.mockResolvedValueOnce({ rows: [], rowCount: 0 }); // priority backfill m.fn.mockResolvedValueOnce({ rows: [], rowCount: 2 }); // recovery UPDATE const q = createPgQueue(m.pool, async () => undefined); await q.init(); - expect(m.pool.query).toHaveBeenCalledTimes(2); + expect(m.pool.query).toHaveBeenCalledTimes(3); }); it("init() handles null rowCount from the recovery query (rowCount ?? 0 nullish arm)", async () => { const m = makePool(); m.fn.mockResolvedValueOnce({ rows: [], rowCount: 0 }); // DDL + m.fn.mockResolvedValueOnce({ rows: [], rowCount: 0 }); // priority backfill // pg driver can return null for rowCount on some UPDATE results m.fn.mockResolvedValueOnce({ rows: [], rowCount: null }); const q = createPgQueue(m.pool, async () => undefined); await q.init(); // rowCount=null → ?? 0 → 0 → no recovery log emitted - expect(m.pool.query).toHaveBeenCalledTimes(2); + expect(m.pool.query).toHaveBeenCalledTimes(3); + }); + + it("init() backfills review-refresh priorities without parsing payload JSON in SQL", async () => { + const m = makePool(); + m.fn.mockResolvedValueOnce({ rows: [], rowCount: 0 }); // DDL + m.fn.mockResolvedValueOnce({ rows: [], rowCount: 3 }); // priority backfill + m.fn.mockResolvedValueOnce({ rows: [], rowCount: 0 }); // recovery UPDATE + const q = createPgQueue(m.pool, async () => undefined); + await q.init(); + expect(m.pool.query).toHaveBeenNthCalledWith( + 2, + expect.stringContaining("agent-regate-pr|recapture-preview"), + ); }); it("processes a job successfully (job_complete audit emitted)", async () => { diff --git a/test/unit/selfhost-sqlite-queue.test.ts b/test/unit/selfhost-sqlite-queue.test.ts index 8ff6cd924e..4fbd93df70 100644 --- a/test/unit/selfhost-sqlite-queue.test.ts +++ b/test/unit/selfhost-sqlite-queue.test.ts @@ -26,11 +26,13 @@ describe("createSqliteQueue (durable #980)", () => { expect(q.size()).toBe(0); }); - it("tags github-webhook with priority 10, other jobs 0, untyped 0 (#review-latency)", async () => { + it("tags webhook and PR review refresh jobs with elevated priorities (#review-latency)", async () => { const driver = makeDriver(); const q = createSqliteQueue(driver, async () => undefined); // delaySeconds keeps them pending (not claimed) so we can read the stored priority. await q.binding.send(msg("github-webhook"), { delaySeconds: 60 }); + await q.binding.send(msg("agent-regate-pr"), { delaySeconds: 60 }); + await q.binding.send(msg("recapture-preview"), { delaySeconds: 60 }); await q.binding.send(msg("rag-index-repo"), { delaySeconds: 60 }); await q.binding.send({} as unknown as JobMessage, { delaySeconds: 60 }); // no type → priority 0 fallback const { rows } = driver.query( @@ -42,10 +44,47 @@ describe("createSqliteQueue (durable #980)", () => { (r) => r.payload === p, )?.priority; expect(prio(JSON.stringify(msg("github-webhook")))).toBe(10); + expect(prio(JSON.stringify(msg("agent-regate-pr")))).toBe(9); + expect(prio(JSON.stringify(msg("recapture-preview")))).toBe(9); expect(prio(JSON.stringify(msg("rag-index-repo")))).toBe(0); expect(prio("{}")).toBe(0); }); + it("backfills stale priorities on startup so existing regate jobs are not buried", async () => { + const driver = makeDriver(); + driver.exec(` + CREATE TABLE _selfhost_jobs ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + payload TEXT NOT NULL, + status TEXT NOT NULL DEFAULT 'pending', + attempts INTEGER NOT NULL DEFAULT 0, + run_after INTEGER NOT NULL DEFAULT 0, + created_at INTEGER NOT NULL, + last_error TEXT, + priority INTEGER NOT NULL DEFAULT 0 + ); + `); + driver.query( + "INSERT INTO _selfhost_jobs (payload, status, attempts, run_after, created_at, priority) VALUES (?, 'pending', 0, 0, 0, 0)", + [JSON.stringify(msg("agent-regate-pr"))], + ); + driver.query( + "INSERT INTO _selfhost_jobs (payload, status, attempts, run_after, created_at, priority) VALUES (?, 'pending', 0, 0, 0, 0)", + [JSON.stringify(msg("github-webhook"))], + ); + + createSqliteQueue(driver, async () => undefined); + + const { rows } = driver.query( + "SELECT payload, priority FROM _selfhost_jobs ORDER BY id", + [], + ); + expect(rows.map((row) => row as { payload: string; priority: number })).toEqual([ + { payload: JSON.stringify(msg("agent-regate-pr")), priority: 9 }, + { payload: JSON.stringify(msg("github-webhook")), priority: 10 }, + ]); + }); + it("migrates an old queue table without a priority column before creating the claim index", async () => { const driver = makeDriver(); driver.exec(` @@ -97,7 +136,7 @@ describe("createSqliteQueue (durable #980)", () => { ).toEqual(["status", "run_after", "priority"]); }); - it("claims a high-priority (github-webhook) job before an earlier low-priority one", async () => { + it("claims webhook work before regate work, and regate work before earlier background jobs", async () => { const driver = makeDriver(); const seen: string[] = []; const q = createSqliteQueue(driver, async (m) => void seen.push(typeOf(m)), { @@ -108,13 +147,16 @@ describe("createSqliteQueue (durable #980)", () => { "INSERT INTO _selfhost_jobs (payload, status, attempts, run_after, created_at, priority) VALUES (?, 'pending', 0, 0, 0, 0)", [JSON.stringify(msg("rag-index-repo"))], ); + driver.query( + "INSERT INTO _selfhost_jobs (payload, status, attempts, run_after, created_at, priority) VALUES (?, 'pending', 0, 0, 0, 9)", + [JSON.stringify(msg("agent-regate-pr"))], + ); driver.query( "INSERT INTO _selfhost_jobs (payload, status, attempts, run_after, created_at, priority) VALUES (?, 'pending', 0, 0, 0, 10)", [JSON.stringify(msg("github-webhook"))], ); await q.drain(); - // github-webhook (priority 10, inserted LATER) is processed BEFORE the earlier rag-index-repo (priority 0). - expect(seen).toEqual(["github-webhook", "rag-index-repo"]); + expect(seen).toEqual(["github-webhook", "agent-regate-pr", "rag-index-repo"]); }); it("retries then dead-letters after maxRetries", async () => { From 19374beb63738b4b62f435ddd471dbaaf641cdb7 Mon Sep 17 00:00:00 2001 From: JSONbored <49853598+JSONbored@users.noreply.github.com> Date: Sun, 28 Jun 2026 15:45:58 -0700 Subject: [PATCH 24/68] fix(observability): count failed AI CLI attempts --- grafana/dashboards/maintainer-reviews.json | 2 +- src/selfhost/ai.ts | 14 ++++++++++++-- test/unit/selfhost-ai.test.ts | 8 ++++++++ 3 files changed, 21 insertions(+), 3 deletions(-) diff --git a/grafana/dashboards/maintainer-reviews.json b/grafana/dashboards/maintainer-reviews.json index 6b65bd4085..57397c6f32 100644 --- a/grafana/dashboards/maintainer-reviews.json +++ b/grafana/dashboards/maintainer-reviews.json @@ -78,7 +78,7 @@ "gridPos": { "h": 4, "w": 4, "x": 20, "y": 1 }, "fieldConfig": { "defaults": { "color": { "mode": "fixed", "fixedColor": "purple" }, "unit": "short" }, "overrides": [] }, "options": { "colorMode": "background", "graphMode": "none", "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, "textMode": "value" }, - "targets": [{ "datasource": { "type": "frser-sqlite-datasource", "uid": "gittensory-db" }, "refId": "A", "queryType": "table", "queryText": "SELECT count(*) AS ignored FROM review_targets WHERE status='ignored'", "rawQueryText": "SELECT count(*) AS ignored FROM review_targets WHERE status='ignored'" }] + "targets": [{ "datasource": { "type": "frser-sqlite-datasource", "uid": "gittensory-db" }, "refId": "A", "queryType": "table", "queryText": "SELECT count(*) AS ignored FROM review_targets WHERE status='ignored' OR verdict='ignore'", "rawQueryText": "SELECT count(*) AS ignored FROM review_targets WHERE status='ignored' OR verdict='ignore'" }] }, { "type": "table", diff --git a/src/selfhost/ai.ts b/src/selfhost/ai.ts index 1cfe7b065e..b1daba2a45 100644 --- a/src/selfhost/ai.ts +++ b/src/selfhost/ai.ts @@ -479,16 +479,20 @@ export function createClaudeCodeAi(parentEnv: Record const claudeModel = resolveModel(configuredClaudeModel(parentEnv), model, "claude-sonnet-4-6"); const effort = resolveEffort(firstConfigured(parentEnv.CLAUDE_AI_EFFORT)); const timeoutMs = resolveClaudeCliTimeoutMs(parentEnv); + let attempted = false; + let stdoutForMetrics = ""; try { if (!token) throw new Error("claude_code_no_oauth_token"); const env = subscriptionCliEnv(parentEnv, { CLAUDE_CODE_OAUTH_TOKEN: token }); const prompt = toMessages(options).map((m) => m.content).join("\n\n"); const spawn = spawnImpl ?? (await defaultSpawn()); + attempted = true; const { stdout, code, stderr } = await spawn( "claude", ["--print", "--output-format", "json", "--model", claudeModel, "--permission-mode", "plan", "--effort", effort, "--disallowedTools", "Bash,Edit,Write,WebFetch,WebSearch"], { env, input: prompt, timeoutMs, cwd: await isolatedCliCwd() }, ); + stdoutForMetrics = stdout; // Surface the STRUCTURED error envelope FIRST. `claude --output-format json` reports API/auth/model errors in its // stdout JSON ({is_error,api_error_status}) on a NON-ZERO exit too — e.g. an unknown model exits 1 with the 404 // envelope in stdout and EMPTY stderr. Checking it before the exit code turns an opaque `claude_code_exit_1: ` @@ -499,11 +503,12 @@ export function createClaudeCodeAi(parentEnv: Record if (code !== 0) throw new Error(`claude_code_exit_${code ?? "null"}: ${redactSecrets(stderr ?? "", [token]).slice(0, 500)}`); const text = extractCliText(stdout); if (!text) throw new Error("claude_code_empty_output"); - recordCliUsageMetrics("claude-code", claudeModel, effort, stdout); return { response: text }; } catch (error) { logSelfHostAiProviderFailed({ provider: "claude-code", model: claudeModel, effort, timeoutMs, error, knownSecrets: token ? [token] : [] }); throw error; + } finally { + if (attempted) recordCliUsageMetrics("claude-code", claudeModel, effort, stdoutForMetrics); } }, }; @@ -522,6 +527,8 @@ export function createCodexAi(parentEnv: Record, spa const codexModel = resolveModel(configuredCodexModel(parentEnv), model, ""); const effort = resolveCodexEffort(firstConfigured(parentEnv.CODEX_AI_EFFORT)); const timeoutMs = resolveCodexCliTimeoutMs(parentEnv); + let attempted = false; + let stdoutForMetrics = ""; try { assertCodexCredentialIsolation(parentEnv); const env = codexCliEnv(parentEnv); @@ -530,6 +537,7 @@ export function createCodexAi(parentEnv: Record, spa const args = ["exec", "--json", "--skip-git-repo-check", "--sandbox", "read-only"]; if (codexModel) args.push("--model", codexModel); args.push("-c", `model_reasoning_effort="${effort}"`); + attempted = true; const { stdout, code, stderr } = await spawn("codex", args, { env, // `codex exec` reads stdin when no prompt argv is provided; keep PR prompts/diffs out of process listings. @@ -537,14 +545,16 @@ export function createCodexAi(parentEnv: Record, spa timeoutMs, cwd: await isolatedCliCwd(), }); + stdoutForMetrics = stdout; if (code !== 0) throw new Error(`codex_exit_${code ?? "null"}: ${redactSecrets(stderr ?? "").slice(0, 500)}`); const text = extractCliText(stdout); if (!text) throw new Error("codex_empty_output"); - recordCliUsageMetrics("codex", codexModel, effort, stdout); return { response: text }; } catch (error) { logSelfHostAiProviderFailed({ provider: "codex", model: codexModel, effort, timeoutMs, error }); throw error; + } finally { + if (attempted) recordCliUsageMetrics("codex", codexModel, effort, stdoutForMetrics); } }, }; diff --git a/test/unit/selfhost-ai.test.ts b/test/unit/selfhost-ai.test.ts index 10abcc24b7..fa8ce19412 100644 --- a/test/unit/selfhost-ai.test.ts +++ b/test/unit/selfhost-ai.test.ts @@ -546,6 +546,8 @@ describe("subscription CLI helpers + fail-safe", () => { await expect(createClaudeCodeAi({ CLAUDE_CODE_OAUTH_TOKEN: "t" }, exit1).run("m", { prompt: "x" })).rejects.toThrow(/claude_code_exit_1/); const empty: StubSpawn = async () => ({ stdout: "", code: 0 }); await expect(createClaudeCodeAi({ CLAUDE_CODE_OAUTH_TOKEN: "t" }, empty).run("m", { prompt: "x" })).rejects.toThrow(/claude_code_empty_output/); + const metrics = await renderMetrics(); + expect(metrics).toContain('gittensory_ai_requests_total{effort="high",model="m",provider="claude-code"} 2'); }); it("Codex throws on empty output", async () => { @@ -553,6 +555,8 @@ describe("subscription CLI helpers + fail-safe", () => { await expect( createCodexAi({ GITTENSORY_ENABLE_UNSAFE_CODEX_REVIEWER: "1" }, empty).run("gpt-5", { prompt: "x" }), ).rejects.toThrow(/codex_empty_output/); + const metrics = await renderMetrics(); + expect(metrics).toContain('gittensory_ai_requests_total{effort="high",model="gpt-5",provider="codex"} 1'); }); it("Codex fails closed when a mounted OAuth home would be exposed to the review sandbox", async () => { @@ -570,6 +574,8 @@ describe("subscription CLI helpers + fail-safe", () => { await expect(createCodexAi({}, shouldNotSpawn).run("gpt-5", { prompt: "x" })).rejects.toThrow( /codex_credential_isolation_required/, ); + const metrics = await renderMetrics(); + expect(metrics).not.toContain("gittensory_ai_requests_total"); }); it("surfaces the CLI's stderr in the non-zero-exit error (diagnosable failures, #26)", async () => { @@ -583,6 +589,8 @@ describe("subscription CLI helpers + fail-safe", () => { await expect(createCodexAi({ GITTENSORY_ENABLE_UNSAFE_CODEX_REVIEWER: "1" }, codexErr).run("m", { prompt: "x" })).rejects.toThrow( /codex_exit_1: stream error: rate limit reached/, ); + const metrics = await renderMetrics(); + expect(metrics).toContain('gittensory_ai_requests_total{effort="high",model="m",provider="codex"} 1'); }); it("redacts the OAuth token and key-shaped tokens from claude stderr before they reach the error (#1605 sec)", async () => { From ef90bc9b4807fdd27f701da59fea1d1ffe2fee31 Mon Sep 17 00:00:00 2001 From: JSONbored <49853598+JSONbored@users.noreply.github.com> Date: Sun, 28 Jun 2026 15:58:15 -0700 Subject: [PATCH 25/68] fix(gate): defer manual reruns until CI settles --- src/queue/processors.ts | 20 +++++++++++ test/unit/queue.test.ts | 75 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 95 insertions(+) diff --git a/src/queue/processors.ts b/src/queue/processors.ts index 9ea9f40019..2e74895831 100644 --- a/src/queue/processors.ts +++ b/src/queue/processors.ts @@ -5973,6 +5973,26 @@ async function maybeProcessPrPanelRetrigger( ) { await refreshPullRequestDetails(env, repoFullName, pr.number); } + if ( + !(await prReadyForReview( + env, + installationId, + repoFullName, + pr, + settings, + deliveryId, + )) + ) { + await recordAuditEvent(env, { + eventType: "github_app.pr_panel_retrigger_deferred", + actor, + targetKey: `${repoFullName}#${pr.number}`, + outcome: "queued", + detail: "manual panel retrigger deferred until CI finishes", + metadata: { deliveryId, repoFullName, commentId: comment.id }, + }).catch(() => undefined); + return true; + } await maybePublishPrPublicSurface( env, installationId, diff --git a/test/unit/queue.test.ts b/test/unit/queue.test.ts index 310ac36329..cba4332898 100644 --- a/test/unit/queue.test.ts +++ b/test/unit/queue.test.ts @@ -3499,6 +3499,81 @@ describe("queue processors", () => { expect(usageEvents).toEqual(expect.arrayContaining([expect.objectContaining({ surface: "github_app", eventName: "pr_panel_retriggered", outcome: "completed" })])); }); + it("defers a manual panel rerun while CI is still running", async () => { + const env = createTestEnv({ GITHUB_APP_PRIVATE_KEY: await generatePrivateKeyPem() }); + await upsertRepositoryFromGitHub(env, { name: "gittensory", full_name: "JSONbored/gittensory", private: false, owner: { login: "JSONbored" } }, 123); + await upsertRepositorySettings(env, { + repoFullName: "JSONbored/gittensory", + commentMode: "all_prs", + publicSurface: "comment_only", + autoLabelEnabled: false, + checkRunMode: "off", + gateCheckMode: "enabled", + includeMaintainerAuthors: true, + autonomy: { merge: "auto" }, + commandAuthorization: { default: ["maintainer"], commands: { "review-now": ["maintainer"] } }, + }); + await upsertPullRequestFromGitHub(env, "JSONbored/gittensory", { + number: 46, + title: "Pending CI rerun", + state: "open", + user: { login: "contributor" }, + author_association: "CONTRIBUTOR", + head: { sha: "pendingci" }, + base: { ref: "main" }, + labels: [], + body: "Validation: npm test", + }); + const checkedPanel = [ + "", + "", + "- [x] Re-run Gittensory review", + ].join("\n"); + let commentPatches = 0; + vi.stubGlobal("fetch", async (input: RequestInfo | URL, init?: RequestInit) => { + const url = input.toString(); + const method = init?.method ?? "GET"; + if (url === "https://api.gittensor.io/miners") return Response.json([]); + if (url.includes("/access_tokens")) return Response.json({ token: "installation-token" }); + if (url.includes("/collaborators/maintainer/permission")) return Response.json({ permission: "maintain" }); + if (/\/pulls\/46(?:\?|$)/.test(url)) return Response.json({ number: 46, mergeable_state: "clean" }); + if (url.includes("/commits/pendingci/check-runs")) { + return Response.json({ check_runs: [{ name: "test", status: "in_progress", conclusion: null, app: { slug: "github-actions" } }] }); + } + if (url.includes("/commits/pendingci/status")) return Response.json({ statuses: [] }); + if (url.includes("/issues/comments/778") && method === "PATCH") { + commentPatches += 1; + return Response.json({ id: 778 }); + } + return new Response("not found", { status: 404 }); + }); + + await processJob(env, { + type: "github-webhook", + deliveryId: "panel-retrigger-ci-pending", + eventName: "issue_comment", + payload: { + action: "edited", + installation: { id: 123, account: { login: "JSONbored", id: 1, type: "User" } }, + repository: { name: "gittensory", full_name: "JSONbored/gittensory", private: false, owner: { login: "JSONbored" } }, + issue: { number: 46, title: "Pending CI rerun", state: "open", user: { login: "contributor" }, pull_request: {} }, + comment: { id: 778, body: checkedPanel, user: { login: "gittensory[bot]", type: "Bot" } }, + sender: { login: "maintainer", type: "User" }, + }, + }); + + expect(commentPatches).toBe(0); + const audit = await env.DB.prepare("select event_type, actor, target_key, outcome from audit_events where event_type = ?") + .bind("github_app.pr_panel_retrigger_deferred") + .first<{ event_type: string; actor: string; target_key: string; outcome: string }>(); + expect(audit).toMatchObject({ + event_type: "github_app.pr_panel_retrigger_deferred", + actor: "maintainer", + target_key: "JSONbored/gittensory#46", + outcome: "queued", + }); + }); + it("refreshes the PR's files on a manual rerun so the slop/manifest gate evaluates the current diff", async () => { const env = createTestEnv({ GITHUB_APP_PRIVATE_KEY: await generatePrivateKeyPem() }); await upsertRepositoryFromGitHub(env, { name: "gittensory", full_name: "JSONbored/gittensory", private: false, owner: { login: "JSONbored" } }, 123); From 123795bc2dc91846a739262482224970cd6c07a9 Mon Sep 17 00:00:00 2001 From: JSONbored <49853598+JSONbored@users.noreply.github.com> Date: Sun, 28 Jun 2026 16:17:36 -0700 Subject: [PATCH 26/68] fix(queue): preserve review jobs during rate limits --- src/selfhost/audit.ts | 7 +- src/selfhost/pg-queue.ts | 68 +++++++------- src/selfhost/queue-common.ts | 115 ++++++++++++++++++++++++ src/selfhost/sqlite-queue.ts | 33 ++++--- test/unit/selfhost-pg-queue.test.ts | 58 ++++++++++-- test/unit/selfhost-queue-common.test.ts | 78 ++++++++++++++++ test/unit/selfhost-sqlite-queue.test.ts | 60 ++++++++++++- 7 files changed, 366 insertions(+), 53 deletions(-) create mode 100644 src/selfhost/queue-common.ts create mode 100644 test/unit/selfhost-queue-common.test.ts diff --git a/src/selfhost/audit.ts b/src/selfhost/audit.ts index 096abea0fa..e5fb987839 100644 --- a/src/selfhost/audit.ts +++ b/src/selfhost/audit.ts @@ -3,7 +3,11 @@ // setup. Written to process.stdout so it is captured by Docker's default json-file log driver and is // accessible via `docker compose logs gittensory`. -export type AuditEventType = "job_complete" | "job_dead" | "job_error"; +export type AuditEventType = + | "job_complete" + | "job_dead" + | "job_error" + | "job_rate_limited"; export interface AuditEvent { event: AuditEventType; @@ -13,6 +17,7 @@ export interface AuditEvent { latency_ms: number; // wall time from claim to completion/failure attempts: number; // total attempts consumed (1 = first-try success) error?: string; // last error message, present for job_dead / job_error + retry_after_ms?: number; // next retry delay for job_rate_limited } /** Emit a single audit event as a JSON line on stdout. */ diff --git a/src/selfhost/pg-queue.ts b/src/selfhost/pg-queue.ts index 2e0ff22097..a7ccca9026 100644 --- a/src/selfhost/pg-queue.ts +++ b/src/selfhost/pg-queue.ts @@ -6,6 +6,7 @@ import type { Pool } from "pg"; import { logAudit, extractPayloadType } from "./audit"; import { incr } from "./metrics"; import { captureError } from "./sentry"; +import { githubRateLimitRetryDelayMs, jobPriority } from "./queue-common"; import type { JobMessage } from "../types"; const TABLE = "_selfhost_jobs"; @@ -23,19 +24,6 @@ CREATE TABLE IF NOT EXISTS ${TABLE} ( ALTER TABLE ${TABLE} ADD COLUMN IF NOT EXISTS priority INTEGER NOT NULL DEFAULT 0; CREATE INDEX IF NOT EXISTS ${TABLE}_claim ON ${TABLE}(status, run_after, priority);`; -// Webhook-driven work (a fresh PR → its review) jumps ahead of heavy background jobs (rag-index, the regate sweep) -// so a NEW PR is reviewed promptly instead of waiting behind them in the shared FIFO queue. Per-PR review refreshes -// sit just below webhooks: they repair stale GitHub surfaces quickly without starving fresh webhook work. Additive: -// all other jobs stay priority 0 (today's FIFO order). Mirrors sqlite-queue. (#review-latency) -const PRIORITY_BY_TYPE = new Map([ - ["github-webhook", 10], - ["agent-regate-pr", 9], - ["recapture-preview", 9], -]); -function jobPriority(payload: string): number { - return PRIORITY_BY_TYPE.get(extractPayloadType(payload) ?? "") ?? 0; -} - export interface PgDurableQueue { binding: Queue; init(): Promise; @@ -82,23 +70,7 @@ export function createPgQueue( async function init(): Promise { await pool.query(DDL); - const priorityBackfilled = - ( - await pool.query( - `UPDATE ${TABLE} - SET priority = CASE - WHEN payload ~ '"type"[[:space:]]*:[[:space:]]*"github-webhook"' THEN 10 - WHEN payload ~ '"type"[[:space:]]*:[[:space:]]*"(agent-regate-pr|recapture-preview)"' THEN 9 - ELSE 0 - END - WHERE status IN ('pending', 'processing') - AND priority IS DISTINCT FROM CASE - WHEN payload ~ '"type"[[:space:]]*:[[:space:]]*"github-webhook"' THEN 10 - WHEN payload ~ '"type"[[:space:]]*:[[:space:]]*"(agent-regate-pr|recapture-preview)"' THEN 9 - ELSE 0 - END`, - ) - ).rowCount ?? 0; + const priorityBackfilled = await backfillJobPriorities(); if (priorityBackfilled) console.log( JSON.stringify({ @@ -118,6 +90,23 @@ export function createPgQueue( ); } + async function backfillJobPriorities(): Promise { + const res = await pool.query( + `SELECT id, payload, priority FROM ${TABLE} WHERE status IN ('pending', 'processing')`, + ); + let changed = 0; + for (const row of res.rows as Array<{ id: string; payload: string; priority: number | string }>) { + const priority = jobPriority(row.payload); + if (priority === Number(row.priority ?? 0)) continue; + await pool.query(`UPDATE ${TABLE} SET priority=$1 WHERE id=$2`, [ + priority, + row.id, + ]); + changed += 1; + } + return changed; + } + async function enqueue( message: JobMessage, delaySeconds: number, @@ -186,6 +175,25 @@ export function createPgQueue( } catch (error) { const attempts = Number(job.attempts) + 1; const errMsg = error instanceof Error ? error.message : "unknown error"; + const rateLimitDelayMs = githubRateLimitRetryDelayMs(error); + if (rateLimitDelayMs !== null) { + await pool.query( + `UPDATE ${TABLE} SET status='pending', run_after=$1, last_error=$2 WHERE id=$3`, + [Date.now() + rateLimitDelayMs, errMsg, job.id], + ); + incr("gittensory_jobs_rate_limited_total"); + logAudit({ + event: "job_rate_limited", + ts: Date.now(), + job_id: job.id, + payload_type: extractPayloadType(job.payload), + latency_ms: Date.now() - claimedAt, + attempts, + retry_after_ms: rateLimitDelayMs, + error: errMsg, + }); + return true; + } incr("gittensory_jobs_failed_total"); if (attempts >= maxRetries) { await pool.query( diff --git a/src/selfhost/queue-common.ts b/src/selfhost/queue-common.ts new file mode 100644 index 0000000000..d9037b6cb5 --- /dev/null +++ b/src/selfhost/queue-common.ts @@ -0,0 +1,115 @@ +import { extractPayloadType } from "./audit"; + +// Webhook-driven work (a fresh PR -> its review) jumps ahead of heavy background jobs. Per-PR review refreshes +// sit just below real webhooks, and sweep fan-out sits below those so stale surfaces are repaired during bursts. +// Bot-generated comment edits are background noise; keeping them with real webhooks lets panel edits starve repair. +const PRIORITY_BY_TYPE = new Map([ + ["agent-regate-pr", 9], + ["recapture-preview", 9], + ["agent-regate-sweep", 8], +]); + +export function jobPriority(payload: string): number { + const type = extractPayloadType(payload) ?? ""; + if (type === "github-webhook") return githubWebhookPriority(payload); + return PRIORITY_BY_TYPE.get(type) ?? 0; +} + +function githubWebhookPriority(payload: string): number { + try { + const message = JSON.parse(payload) as { + eventName?: unknown; + payload?: { + action?: unknown; + sender?: { login?: unknown; type?: unknown } | null; + } | null; + }; + const eventName = typeof message.eventName === "string" ? message.eventName : ""; + const action = typeof message.payload?.action === "string" ? message.payload.action : ""; + const senderLogin = + typeof message.payload?.sender?.login === "string" + ? message.payload.sender.login.toLowerCase() + : ""; + const senderType = + typeof message.payload?.sender?.type === "string" + ? message.payload.sender.type.toLowerCase() + : ""; + if ( + eventName === "issue_comment" && + action === "edited" && + (senderType === "bot" || senderLogin.endsWith("[bot]")) + ) + return 0; + } catch { + return 0; + } + return 10; +} + +const DEFAULT_GITHUB_RATE_LIMIT_RETRY_MS = 5 * 60_000; +const MAX_GITHUB_RATE_LIMIT_RETRY_MS = 65 * 60_000; + +export function githubRateLimitRetryDelayMs( + error: unknown, + nowMs = Date.now(), +): number | null { + if (typeof error !== "object" || error === null) return null; + const err = error as { + status?: unknown; + message?: unknown; + response?: { headers?: Headers | Record | null } | null; + }; + const status = typeof err.status === "number" ? err.status : null; + const message = typeof err.message === "string" ? err.message : ""; + const headers = err.response?.headers ?? null; + const retryAfter = numberHeader(headers, "retry-after"); + if (retryAfter !== null) + return clampRetryDelay(retryAfter * 1000); + + const remaining = stringHeader(headers, "x-ratelimit-remaining"); + const reset = numberHeader(headers, "x-ratelimit-reset"); + if (remaining === "0" && reset !== null) { + const delay = reset * 1000 - nowMs + 5_000; + return clampRetryDelay(delay); + } + + if ( + (status === 403 || status === 429 || status === null) && + /secondary rate limit|\babuse\b|api rate limit exceeded|rate limit/i.test( + message, + ) + ) + return DEFAULT_GITHUB_RATE_LIMIT_RETRY_MS; + + return null; +} + +function clampRetryDelay(delayMs: number): number { + if (!Number.isFinite(delayMs) || delayMs <= 0) return DEFAULT_GITHUB_RATE_LIMIT_RETRY_MS; + return Math.min(Math.ceil(delayMs), MAX_GITHUB_RATE_LIMIT_RETRY_MS); +} + +function numberHeader( + headers: Headers | Record | null, + key: string, +): number | null { + const raw = stringHeader(headers, key); + if (raw === null) return null; + const parsed = Number(raw); + return Number.isFinite(parsed) ? parsed : null; +} + +function stringHeader( + headers: Headers | Record | null, + key: string, +): string | null { + if (!headers) return null; + if (typeof (headers as Headers).get === "function") { + const value = (headers as Headers).get(key); + return value === null ? null : String(value); + } + const value = + (headers as Record)[key] ?? + (headers as Record)[key.toLowerCase()]; + return value == null ? null : String(value); +} diff --git a/src/selfhost/sqlite-queue.ts b/src/selfhost/sqlite-queue.ts index 006e778104..b5ed66563f 100644 --- a/src/selfhost/sqlite-queue.ts +++ b/src/selfhost/sqlite-queue.ts @@ -7,6 +7,7 @@ import type { SqliteDriver } from "./d1-adapter"; import { logAudit, extractPayloadType } from "./audit"; import { incr } from "./metrics"; import { captureError } from "./sentry"; +import { githubRateLimitRetryDelayMs, jobPriority } from "./queue-common"; import type { JobMessage } from "../types"; const TABLE = "_selfhost_jobs"; @@ -25,19 +26,6 @@ const CLAIM_INDEX_DDL = ` DROP INDEX IF EXISTS ${TABLE}_claim; CREATE INDEX ${TABLE}_claim ON ${TABLE}(status, run_after, priority);`; -// Webhook-driven work (a fresh PR → its review) jumps ahead of heavy background jobs (rag-index ~4min, the regate -// sweep) so a NEW PR is reviewed promptly instead of waiting behind them in the shared FIFO queue. Per-PR review -// refreshes sit just below webhooks: they repair stale GitHub surfaces quickly without starving fresh webhook work. -// Additive: every other job stays priority 0 (today's FIFO order). (#review-latency) -const PRIORITY_BY_TYPE = new Map([ - ["github-webhook", 10], - ["agent-regate-pr", 9], - ["recapture-preview", 9], -]); -function jobPriority(payload: string): number { - return PRIORITY_BY_TYPE.get(extractPayloadType(payload) ?? "") ?? 0; -} - export interface DurableQueue { binding: Queue; start(): void; @@ -179,6 +167,25 @@ export function createSqliteQueue( } catch (error) { const attempts = job.attempts + 1; const errMsg = error instanceof Error ? error.message : "unknown error"; + const rateLimitDelayMs = githubRateLimitRetryDelayMs(error); + if (rateLimitDelayMs !== null) { + driver.query( + `UPDATE ${TABLE} SET status='pending', run_after=?, last_error=? WHERE id=?`, + [Date.now() + rateLimitDelayMs, errMsg, job.id], + ); + incr("gittensory_jobs_rate_limited_total"); + logAudit({ + event: "job_rate_limited", + ts: Date.now(), + job_id: job.id, + payload_type: extractPayloadType(job.payload), + latency_ms: Date.now() - claimedAt, + attempts, + retry_after_ms: rateLimitDelayMs, + error: errMsg, + }); + return true; + } incr("gittensory_jobs_failed_total"); if (attempts >= maxRetries) { driver.query( diff --git a/test/unit/selfhost-pg-queue.test.ts b/test/unit/selfhost-pg-queue.test.ts index a9fdc77326..e2b3dbdfc9 100644 --- a/test/unit/selfhost-pg-queue.test.ts +++ b/test/unit/selfhost-pg-queue.test.ts @@ -6,6 +6,12 @@ import { createPgQueue } from "../../src/selfhost/pg-queue"; import type { JobMessage } from "../../src/types"; const msg = (t: string): JobMessage => ({ type: t }) as unknown as JobMessage; +const webhook = (sender: { login: string; type: string }, eventName = "issue_comment", action = "edited"): JobMessage => + ({ + type: "github-webhook", + eventName, + payload: { action, sender }, + }) as unknown as JobMessage; const typeOf = (m: JobMessage): string => (m as unknown as { type: string }).type; type MockFn = { mockResolvedValueOnce(v: unknown): void }; @@ -51,7 +57,7 @@ describe("createPgQueue (durable #977)", () => { it("init() creates the table and recovers stuck-processing jobs", async () => { const m = makePool(); m.fn.mockResolvedValueOnce({ rows: [], rowCount: 0 }); // DDL - m.fn.mockResolvedValueOnce({ rows: [], rowCount: 0 }); // priority backfill + m.fn.mockResolvedValueOnce({ rows: [], rowCount: 0 }); // priority backfill SELECT m.fn.mockResolvedValueOnce({ rows: [], rowCount: 2 }); // recovery UPDATE const q = createPgQueue(m.pool, async () => undefined); await q.init(); @@ -61,7 +67,7 @@ describe("createPgQueue (durable #977)", () => { it("init() handles null rowCount from the recovery query (rowCount ?? 0 nullish arm)", async () => { const m = makePool(); m.fn.mockResolvedValueOnce({ rows: [], rowCount: 0 }); // DDL - m.fn.mockResolvedValueOnce({ rows: [], rowCount: 0 }); // priority backfill + m.fn.mockResolvedValueOnce({ rows: [], rowCount: 0 }); // priority backfill SELECT // pg driver can return null for rowCount on some UPDATE results m.fn.mockResolvedValueOnce({ rows: [], rowCount: null }); const q = createPgQueue(m.pool, async () => undefined); @@ -69,17 +75,26 @@ describe("createPgQueue (durable #977)", () => { expect(m.pool.query).toHaveBeenCalledTimes(3); }); - it("init() backfills review-refresh priorities without parsing payload JSON in SQL", async () => { + it("init() backfills event-aware priorities with the shared classifier", async () => { const m = makePool(); m.fn.mockResolvedValueOnce({ rows: [], rowCount: 0 }); // DDL - m.fn.mockResolvedValueOnce({ rows: [], rowCount: 3 }); // priority backfill + m.fn.mockResolvedValueOnce({ + rows: [ + { id: "a", payload: JSON.stringify(msg("agent-regate-pr")), priority: 0 }, + { id: "b", payload: JSON.stringify(webhook({ login: "gittensory-orb[bot]", type: "Bot" })), priority: 10 }, + { id: "c", payload: JSON.stringify(msg("agent-regate-sweep")), priority: 0 }, + ], + rowCount: 3, + }); // priority backfill SELECT + m.fn.mockResolvedValueOnce({ rows: [], rowCount: 1 }); // update a + m.fn.mockResolvedValueOnce({ rows: [], rowCount: 1 }); // update b + m.fn.mockResolvedValueOnce({ rows: [], rowCount: 1 }); // update c m.fn.mockResolvedValueOnce({ rows: [], rowCount: 0 }); // recovery UPDATE const q = createPgQueue(m.pool, async () => undefined); await q.init(); - expect(m.pool.query).toHaveBeenNthCalledWith( - 2, - expect.stringContaining("agent-regate-pr|recapture-preview"), - ); + expect(m.pool.query).toHaveBeenCalledWith(expect.stringContaining("UPDATE _selfhost_jobs SET priority=$1"), [9, "a"]); + expect(m.pool.query).toHaveBeenCalledWith(expect.stringContaining("UPDATE _selfhost_jobs SET priority=$1"), [0, "b"]); + expect(m.pool.query).toHaveBeenCalledWith(expect.stringContaining("UPDATE _selfhost_jobs SET priority=$1"), [8, "c"]); }); it("processes a job successfully (job_complete audit emitted)", async () => { @@ -116,6 +131,33 @@ describe("createPgQueue (durable #977)", () => { expect(calls).toBe(2); }); + it("reschedules GitHub rate-limit failures without consuming the dead-letter budget", async () => { + const m = makePool(); + m.enqueueJob("1", { type: "github-webhook" }, 4); + const rateLimit = new Error("API rate limit exceeded for installation ID 123"); + Object.assign(rateLimit, { + status: 403, + response: { headers: { "retry-after": "120" } }, + }); + const q = createPgQueue( + m.pool, + async () => { + throw rateLimit; + }, + { maxRetries: 1, backoffMs: () => 0 }, + ); + await q.init(); + await q.drain(); + expect(m.pool.query).toHaveBeenCalledWith( + expect.stringContaining("SET status='pending', run_after=$1"), + expect.arrayContaining([expect.any(Number), "API rate limit exceeded for installation ID 123", "1"]), + ); + expect(m.pool.query).not.toHaveBeenCalledWith( + expect.stringContaining("status='dead'"), + expect.anything(), + ); + }); + it("records 'unknown error' when consumer throws a non-Error", async () => { const m = makePool(); m.enqueueJob("1", { type: "t" }, 0); diff --git a/test/unit/selfhost-queue-common.test.ts b/test/unit/selfhost-queue-common.test.ts new file mode 100644 index 0000000000..85fea549af --- /dev/null +++ b/test/unit/selfhost-queue-common.test.ts @@ -0,0 +1,78 @@ +import { describe, expect, it } from "vitest"; +import { + githubRateLimitRetryDelayMs, + jobPriority, +} from "../../src/selfhost/queue-common"; + +const payload = (value: unknown): string => JSON.stringify(value); + +describe("self-host queue common helpers", () => { + it("classifies job priority by job type and webhook sender", () => { + expect(jobPriority(payload({ type: "github-webhook" }))).toBe(10); + expect(jobPriority(payload({ type: "agent-regate-pr" }))).toBe(9); + expect(jobPriority(payload({ type: "recapture-preview" }))).toBe(9); + expect(jobPriority(payload({ type: "agent-regate-sweep" }))).toBe(8); + expect(jobPriority(payload({ type: "rag-index-repo" }))).toBe(0); + expect(jobPriority("{}")).toBe(0); + expect(jobPriority("not-json")).toBe(0); + }); + + it("demotes bot-authored issue-comment edit webhooks without demoting human reruns", () => { + const issueCommentEdit = (sender: { login?: string; type?: string }) => + payload({ + type: "github-webhook", + eventName: "issue_comment", + payload: { action: "edited", sender }, + }); + expect( + jobPriority(issueCommentEdit({ login: "gittensory-orb[bot]", type: "Bot" })), + ).toBe(0); + expect( + jobPriority(issueCommentEdit({ login: "codecov[bot]", type: "User" })), + ).toBe(0); + expect( + jobPriority(issueCommentEdit({ login: "jsonbored", type: "User" })), + ).toBe(10); + expect( + jobPriority( + payload({ + type: "github-webhook", + eventName: "issue_comment", + payload: { action: "created", sender: { login: "codecov[bot]" } }, + }), + ), + ).toBe(10); + }); + + it("extracts retry delays from GitHub rate-limit errors", () => { + expect(githubRateLimitRetryDelayMs(null)).toBeNull(); + expect(githubRateLimitRetryDelayMs({ status: 403, message: "Forbidden" })).toBeNull(); + + expect( + githubRateLimitRetryDelayMs({ + status: 403, + message: "secondary rate limit", + }), + ).toBe(300_000); + expect( + githubRateLimitRetryDelayMs({ + status: 429, + response: { headers: new Headers({ "retry-after": "2" }) }, + }), + ).toBe(2_000); + expect( + githubRateLimitRetryDelayMs( + { + status: 403, + response: { + headers: { + "x-ratelimit-remaining": "0", + "x-ratelimit-reset": "1003", + }, + }, + }, + 1_000_000, + ), + ).toBe(8_000); + }); +}); diff --git a/test/unit/selfhost-sqlite-queue.test.ts b/test/unit/selfhost-sqlite-queue.test.ts index 4fbd93df70..4c27a7458d 100644 --- a/test/unit/selfhost-sqlite-queue.test.ts +++ b/test/unit/selfhost-sqlite-queue.test.ts @@ -8,6 +8,12 @@ function makeDriver(): ReturnType { return nodeSqliteDriver(new DatabaseSync(":memory:") as never); } const msg = (t: string): JobMessage => ({ type: t }) as unknown as JobMessage; +const webhook = (sender: { login: string; type: string }, eventName = "issue_comment", action = "edited"): JobMessage => + ({ + type: "github-webhook", + eventName, + payload: { action, sender }, + }) as unknown as JobMessage; const typeOf = (m: JobMessage): string => (m as unknown as { type: string }).type; describe("createSqliteQueue (durable #980)", () => { @@ -33,6 +39,9 @@ describe("createSqliteQueue (durable #980)", () => { await q.binding.send(msg("github-webhook"), { delaySeconds: 60 }); await q.binding.send(msg("agent-regate-pr"), { delaySeconds: 60 }); await q.binding.send(msg("recapture-preview"), { delaySeconds: 60 }); + await q.binding.send(msg("agent-regate-sweep"), { delaySeconds: 60 }); + await q.binding.send(webhook({ login: "gittensory-orb[bot]", type: "Bot" }), { delaySeconds: 60 }); + await q.binding.send(webhook({ login: "maintainer", type: "User" }), { delaySeconds: 60 }); await q.binding.send(msg("rag-index-repo"), { delaySeconds: 60 }); await q.binding.send({} as unknown as JobMessage, { delaySeconds: 60 }); // no type → priority 0 fallback const { rows } = driver.query( @@ -46,6 +55,9 @@ describe("createSqliteQueue (durable #980)", () => { expect(prio(JSON.stringify(msg("github-webhook")))).toBe(10); expect(prio(JSON.stringify(msg("agent-regate-pr")))).toBe(9); expect(prio(JSON.stringify(msg("recapture-preview")))).toBe(9); + expect(prio(JSON.stringify(msg("agent-regate-sweep")))).toBe(8); + expect(prio(JSON.stringify(webhook({ login: "maintainer", type: "User" })))).toBe(10); + expect(prio(JSON.stringify(webhook({ login: "gittensory-orb[bot]", type: "Bot" })))).toBe(0); expect(prio(JSON.stringify(msg("rag-index-repo")))).toBe(0); expect(prio("{}")).toBe(0); }); @@ -72,6 +84,10 @@ describe("createSqliteQueue (durable #980)", () => { "INSERT INTO _selfhost_jobs (payload, status, attempts, run_after, created_at, priority) VALUES (?, 'pending', 0, 0, 0, 0)", [JSON.stringify(msg("github-webhook"))], ); + driver.query( + "INSERT INTO _selfhost_jobs (payload, status, attempts, run_after, created_at, priority) VALUES (?, 'pending', 0, 0, 0, 10)", + [JSON.stringify(webhook({ login: "gittensory-orb[bot]", type: "Bot" }))], + ); createSqliteQueue(driver, async () => undefined); @@ -82,6 +98,7 @@ describe("createSqliteQueue (durable #980)", () => { expect(rows.map((row) => row as { payload: string; priority: number })).toEqual([ { payload: JSON.stringify(msg("agent-regate-pr")), priority: 9 }, { payload: JSON.stringify(msg("github-webhook")), priority: 10 }, + { payload: JSON.stringify(webhook({ login: "gittensory-orb[bot]", type: "Bot" })), priority: 0 }, ]); }); @@ -151,12 +168,20 @@ describe("createSqliteQueue (durable #980)", () => { "INSERT INTO _selfhost_jobs (payload, status, attempts, run_after, created_at, priority) VALUES (?, 'pending', 0, 0, 0, 9)", [JSON.stringify(msg("agent-regate-pr"))], ); + driver.query( + "INSERT INTO _selfhost_jobs (payload, status, attempts, run_after, created_at, priority) VALUES (?, 'pending', 0, 0, 0, 8)", + [JSON.stringify(msg("agent-regate-sweep"))], + ); + driver.query( + "INSERT INTO _selfhost_jobs (payload, status, attempts, run_after, created_at, priority) VALUES (?, 'pending', 0, 0, 0, 0)", + [JSON.stringify(webhook({ login: "gittensory-orb[bot]", type: "Bot" }))], + ); driver.query( "INSERT INTO _selfhost_jobs (payload, status, attempts, run_after, created_at, priority) VALUES (?, 'pending', 0, 0, 0, 10)", [JSON.stringify(msg("github-webhook"))], ); await q.drain(); - expect(seen).toEqual(["github-webhook", "agent-regate-pr", "rag-index-repo"]); + expect(seen).toEqual(["github-webhook", "agent-regate-pr", "agent-regate-sweep", "rag-index-repo", "github-webhook"]); }); it("retries then dead-letters after maxRetries", async () => { @@ -177,6 +202,39 @@ describe("createSqliteQueue (durable #980)", () => { expect(q.size()).toBe(0); }); + it("reschedules GitHub rate-limit failures without consuming the dead-letter budget", async () => { + const driver = makeDriver(); + let calls = 0; + const rateLimit = new Error("API rate limit exceeded for installation ID 123"); + Object.assign(rateLimit, { status: 403 }); + const q = createSqliteQueue( + driver, + async () => { + calls += 1; + throw rateLimit; + }, + { maxRetries: 1, backoffMs: () => 0 }, + ); + await q.binding.send(msg("github-webhook")); + await q.drain(); + const { rows } = driver.query( + "SELECT status, attempts, run_after, last_error FROM _selfhost_jobs", + [], + ); + const row = rows[0] as { + status: string; + attempts: number; + run_after: number; + last_error: string; + }; + expect(calls).toBe(1); + expect(q.deadCount()).toBe(0); + expect(row.status).toBe("pending"); + expect(row.attempts).toBe(0); + expect(row.run_after).toBeGreaterThan(Date.now()); + expect(row.last_error).toContain("API rate limit exceeded"); + }); + it("SURVIVES A RESTART: a fresh queue over the same DB processes a persisted pending job", async () => { const driver = makeDriver(); const seen: string[] = []; From 6953b4d785da006c0af83f0d854b0b0c6ac6c0b3 Mon Sep 17 00:00:00 2001 From: JSONbored <49853598+JSONbored@users.noreply.github.com> Date: Sun, 28 Jun 2026 16:40:44 -0700 Subject: [PATCH 27/68] fix(review): retry rate-limited public surface refreshes --- src/github/app.ts | 17 ++++ src/queue/processors.ts | 26 ++++-- src/review/unified-comment.ts | 4 +- test/unit/queue.test.ts | 132 +++++++++++++++++++++++------- test/unit/unified-comment.test.ts | 14 ++-- 5 files changed, 145 insertions(+), 48 deletions(-) diff --git a/src/github/app.ts b/src/github/app.ts index 120b172efa..041846b56c 100644 --- a/src/github/app.ts +++ b/src/github/app.ts @@ -839,6 +839,23 @@ function isRateLimitedError(error: { ); } +export function isGitHubRateLimitedError(error: unknown): boolean { + if (typeof error !== "object" || error === null) return false; + const e = error as { + status?: number; + message?: string; + response?: { headers?: Record }; + }; + if (isRateLimitedError(e)) return true; + return ( + e.status === undefined && + typeof e.message === "string" && + /secondary rate limit|\babuse\b|api rate limit exceeded|rate limit/i.test( + e.message, + ) + ); +} + /** Exported for tests. */ export function isCheckRunPermissionError(error: unknown): boolean { /* v8 ignore next -- Octokit wraps thrown fetch values in HttpError objects before this helper sees them. */ diff --git a/src/queue/processors.ts b/src/queue/processors.ts index 2e74895831..8444c7c5aa 100644 --- a/src/queue/processors.ts +++ b/src/queue/processors.ts @@ -103,6 +103,7 @@ import { createOrUpdateSkippedGateCheckRun, getInstallationId, getRepositoryCollaboratorPermission, + isGitHubRateLimitedError, isForeignAppInstallation, } from "../github/app"; import { @@ -1153,6 +1154,7 @@ async function regatePullRequest( skipAiReview: settings.aiReviewMode === "off", }, ).catch((error) => { + if (isGitHubRateLimitedError(error)) throw error; console.error( JSON.stringify({ level: "warn", @@ -1581,6 +1583,7 @@ async function reReviewStoredPullRequest( ...(options.skipAiReview ? { skipAiReview: true } : {}), }, ).catch((error) => { + if (isGitHubRateLimitedError(error)) throw error; console.error( JSON.stringify({ level: "warn", @@ -3049,6 +3052,7 @@ async function processGitHubWebhook( action: payload.action, }, ).catch((error) => { + if (isGitHubRateLimitedError(error)) throw error; console.error( JSON.stringify({ level: "warn", @@ -4527,12 +4531,12 @@ async function maybePublishPrPublicSurface( confirmedContributor, skipAiReview: webhook.skipAiReview, })); - // Post a transient "🟪 reviewing…" placeholder BEFORE the AI runs so contributors see the bot - // is actively working rather than silent. In-place upsert: once the final verdict is ready it - // overwrites this comment. Best-effort — a failed post never aborts the review. (#reviewing-placeholder) + // Post a transient "🟪 reviewing…" placeholder BEFORE the review refresh runs so contributors never see a + // stale green/yellow/red verdict while the current head is being recomputed. In-place upsert: once the final + // verdict is ready it overwrites this comment. Best-effort — a failed post never aborts the review. if ( shouldPostReviewingPlaceholder({ - aiReviewWillRun, + reviewWillRun: true, mode, willComment: decision.willComment, }) @@ -4754,11 +4758,10 @@ async function maybePublishPrPublicSurface( webhook.deliveryId, gateCheckResult.warning, ); - // A 403 on the COMPLETION call is classified as permission_missing and does NOT throw, so the catch - // below never runs and the pending in_progress check would be orphaned. But the pending check already - // posted (pendingGateCheckRunId is set), proving the App had Checks:write — so a 403 here is almost - // always a transient secondary-rate-limit, not a real revocation. Finalize the pending check to - // neutral (mirrors the catch); if it were a genuine revocation this PATCH also 403s and is swallowed. + // A permission_missing completion result does NOT throw, so the catch below never runs and the pending + // in_progress check would be orphaned. But the pending check already posted (pendingGateCheckRunId is + // set), proving the App could write checks for this head at least once. Finalize the pending check to + // neutral (mirrors the catch); if access was truly revoked this PATCH also fails and is swallowed. if (pendingGateCheckRunId !== undefined && !gateFinalized) { await createOrUpdateErroredGateCheckRun( env, @@ -4772,6 +4775,7 @@ async function maybePublishPrPublicSurface( } } } catch (checkError) { + if (isGitHubRateLimitedError(checkError)) throw checkError; // CRITICAL: a check-run API failure (e.g. a 422 from an over-long output.title) must NEVER abort the // review. The outer catch re-throws → the comment, the audit row, and the auto-action (merge/close) // would all be skipped and the review dead-lettered. That is exactly why red-CI PRs (whose gate title @@ -4799,6 +4803,7 @@ async function maybePublishPrPublicSurface( } } } catch (error) { + if (isGitHubRateLimitedError(error)) throw error; // The pending Gate check was posted but evaluation could not finish. Finalize it to a neutral // (non-blocking) terminal state so it never hangs in_progress; it re-runs on the next push. Only when // the gate was enabled, a pending check id exists, and a real conclusion was not already published. @@ -4914,6 +4919,7 @@ async function maybePublishPrPublicSurface( webhook.deliveryId, message, ); + if (isGitHubRateLimitedError(error)) throw error; } } @@ -5204,6 +5210,7 @@ async function maybePublishPrPublicSurface( webhook.deliveryId, message, ); + if (isGitHubRateLimitedError(error)) throw error; } // Quiet inline review comments (#inline-comments): layer the AI's line-anchored findings on top of the // summary just posted, as a NON-BLOCKING COMMENT review. A no-op (no extra work) unless this is a fresh @@ -5246,6 +5253,7 @@ async function maybePublishPrPublicSurface( webhook.deliveryId, message, ); + if (isGitHubRateLimitedError(error)) throw error; } // Per-PR TYPE label (reviewbot auto-label parity): exactly ONE of gittensor:bug/feature/priority by the PR // title + changed paths. Review-time + neutral, BEST-EFFORT + independent of the context label above so a diff --git a/src/review/unified-comment.ts b/src/review/unified-comment.ts index 4d9d24e198..92eb078d85 100644 --- a/src/review/unified-comment.ts +++ b/src/review/unified-comment.ts @@ -537,6 +537,6 @@ export function renderReviewingPlaceholder(ctx: { brand?: string } = {}): string /** Returns true when the reviewing placeholder should be posted before the AI review runs. * Pure helper so both branches are testable without async setup. */ -export function shouldPostReviewingPlaceholder(args: { aiReviewWillRun: boolean; mode: string; willComment: boolean }): boolean { - return args.aiReviewWillRun && args.mode === "live" && args.willComment; +export function shouldPostReviewingPlaceholder(args: { reviewWillRun: boolean; mode: string; willComment: boolean }): boolean { + return args.reviewWillRun && args.mode === "live" && args.willComment; } diff --git a/test/unit/queue.test.ts b/test/unit/queue.test.ts index cba4332898..265fd6f2f0 100644 --- a/test/unit/queue.test.ts +++ b/test/unit/queue.test.ts @@ -1114,7 +1114,7 @@ describe("queue processors", () => { expect(commentBodies.some((body) => !body.includes("is reviewing"))).toBe(true); }); - it("does not post the 🟪 reviewing placeholder when public AI comments are disabled (regression)", async () => { + it("posts the 🟪 reviewing placeholder for non-AI comment refreshes, then overwrites it with the verdict", async () => { let aiCalls = 0; const env = createTestEnv({ GITHUB_APP_PRIVATE_KEY: await generatePrivateKeyPem(), @@ -1158,9 +1158,82 @@ describe("queue processors", () => { }); expect(aiCalls).toBe(0); - expect(commentBodies.length).toBe(1); - expect(commentBodies[0]).not.toContain("is reviewing"); - expect(commentBodies[0]).not.toContain("🟪"); + expect(commentBodies.length).toBeGreaterThanOrEqual(2); + expect(commentBodies[0]).toContain("is reviewing"); + expect(commentBodies[0]).toContain("🟪"); + expect(commentBodies.some((body) => !body.includes("is reviewing"))).toBe(true); + }); + + it("keeps the PR comment in 🟪 reviewing state and retries when the final comment update is rate-limited", async () => { + const env = createTestEnv({ + GITHUB_APP_PRIVATE_KEY: await generatePrivateKeyPem(), + AI_SUMMARIES_ENABLED: "true", + AI_PUBLIC_COMMENTS_ENABLED: "false", + }); + await persistRegistrySnapshot( + env, + normalizeRegistryPayload( + { "JSONbored/gittensory": { emission_share: 0.01, issue_discovery_share: 0 } }, + { kind: "raw-github", url: "https://example.test" }, + "2026-05-23T00:00:00.000Z", + ), + ); + await upsertRepositoryFromGitHub(env, { name: "gittensory", full_name: "JSONbored/gittensory", private: false, owner: { login: "JSONbored" } }, 123); + await upsertRepositorySettings(env, { + repoFullName: "JSONbored/gittensory", + commentMode: "all_prs", + publicSurface: "comment_only", + autoLabelEnabled: false, + checkRunMode: "off", + gateCheckMode: "off", + aiReviewMode: "advisory", + }); + const postedBodies: string[] = []; + let finalCommentAttempted = false; + vi.stubGlobal("fetch", async (input: RequestInfo | URL, init?: RequestInit) => { + const url = input.toString(); + const method = init?.method ?? "GET"; + if (url.includes("/access_tokens")) return Response.json({ token: "installation-token" }); + if (url.includes("/pulls/9/files")) return Response.json([{ filename: "src/a.ts", status: "modified", additions: 1, deletions: 0, changes: 1, patch: "@@\n+export const ok = true;" }]); + if (url.endsWith("/pulls/9")) return Response.json({ number: 9, title: "Clean PR", state: "open", user: { login: "contributor" }, head: { sha: "a9" }, labels: [], body: "Closes #1", mergeable_state: "clean" }); + if (url.includes("/commits/a9/check-runs")) return Response.json({ total_count: 0, check_runs: [] }); + if (url.includes("/commits/a9/status")) return Response.json({ state: "success", statuses: [] }); + if (url.includes("/issues/1")) return Response.json({ number: 1, title: "Issue", state: "open", labels: [], user: { login: "reporter" } }); + if (url.includes("/issues/9/comments") && method === "GET") return Response.json([]); + if (url.includes("/issues/9/comments") && method === "POST") { + const body = String((JSON.parse(String(init?.body ?? "{}")) as { body?: string }).body ?? ""); + if (postedBodies.length === 0) { + postedBodies.push(body); + return Response.json({ id: 1 }, { status: 201 }); + } + finalCommentAttempted = true; + return new Response(JSON.stringify({ message: "API rate limit exceeded" }), { + status: 403, + headers: { "x-ratelimit-remaining": "0" }, + }); + } + if (url.includes("/branches/")) return Response.json({ protected: false, protection: { required_status_checks: { contexts: [] } } }); + return Response.json({}); + }); + + await expect( + processJob(env, { + type: "github-webhook", + deliveryId: "reviewing-placeholder-comment-ratelimit", + eventName: "pull_request", + payload: { + action: "opened", + installation: { id: 123, account: { login: "JSONbored", id: 1, type: "User" } }, + repository: { name: "gittensory", full_name: "JSONbored/gittensory", private: false, owner: { login: "JSONbored" } }, + pull_request: { number: 9, title: "Clean PR", state: "open", user: { login: "contributor" }, head: { sha: "a9" }, labels: [], body: "Closes #1" }, + }, + }), + ).rejects.toThrow(/rate limit/i); + + expect(finalCommentAttempted).toBe(true); + expect(postedBodies).toHaveLength(1); + expect(postedBodies[0]).toContain("is reviewing"); + expect(postedBodies[0]).toContain("🟪"); }); it("agent re-gate sweep re-reviews each stale open PR (installation id) and swallows a failing re-review", async () => { @@ -3116,7 +3189,7 @@ describe("queue processors", () => { expect(audit?.outcome).toBe("error"); }); - it("finalizes the Gate to neutral when the completion call returns a transient 403 (not left in_progress)", async () => { + it("propagates a rate-limited Gate completion so the queue retries and the pending Gate stays reviewing", async () => { const env = createTestEnv({ GITHUB_APP_PRIVATE_KEY: await generatePrivateKeyPem() }); await persistRegistrySnapshot( env, @@ -3147,33 +3220,30 @@ describe("queue processors", () => { if (url.includes("/check-runs/971") && method === "PATCH") { const body = JSON.parse(String(init?.body ?? "{}")) as { status?: string; conclusion?: string; output?: { title?: string } }; patchBodies.push(body); - // First PATCH = the gate completion; a transient 403 (e.g. secondary rate limit) is classified as - // permission_missing and does NOT throw — the fix must still finalize the already-posted pending check. + // First PATCH = the gate completion. A rate-limit 403 must propagate to the queue instead of being + // swallowed as nonfatal; the pending check remains in_progress while the queue backs off and retries. if (patchBodies.length === 1) return new Response(JSON.stringify({ message: "You have exceeded a secondary rate limit" }), { status: 403 }); return Response.json({ id: 971 }); } return new Response("not found", { status: 404 }); }); - await processJob(env, { - type: "github-webhook", - deliveryId: "gate-finalize-on-403", - eventName: "pull_request", - payload: { - action: "opened", - installation: { id: 123, account: { login: "JSONbored", id: 1, type: "User" } }, - repository: { name: "gittensory", full_name: "JSONbored/gittensory", private: false, owner: { login: "JSONbored" } }, - pull_request: { number: 81, title: "Some change", state: "open", user: { login: "contributor" }, head: { sha: "forbidden403" }, labels: [], body: "No issue link." }, - }, - }); + await expect( + processJob(env, { + type: "github-webhook", + deliveryId: "gate-finalize-on-403", + eventName: "pull_request", + payload: { + action: "opened", + installation: { id: 123, account: { login: "JSONbored", id: 1, type: "User" } }, + repository: { name: "gittensory", full_name: "JSONbored/gittensory", private: false, owner: { login: "JSONbored" } }, + pull_request: { number: 81, title: "Some change", state: "open", user: { login: "contributor" }, head: { sha: "forbidden403" }, labels: [], body: "No issue link." }, + }, + }), + ).rejects.toThrow(/rate limit/i); - // The completion PATCH 403'd (permission_missing, no throw), so the fix finalized the SAME check (id 971) - // to a neutral, non-blocking terminal state instead of orphaning it in_progress. - expect(patchBodies.length).toBe(2); - const finalize = patchBodies[1]; - expect(finalize?.status).toBe("completed"); - expect(finalize?.conclusion).toBe("neutral"); - expect(finalize?.output?.title).toBe("Gittensory Gate — could not finish evaluating"); + expect(patchBodies).toHaveLength(1); + expect(patchBodies[0]?.status).toBe("completed"); }); it("disables the gate from .gittensory.yml (gate.enabled: false) even when repo settings enable it", async () => { @@ -3481,7 +3551,8 @@ describe("queue processors", () => { }); // token: 1 — the installation token is now cached + reused within the request (was 2: main + permission check). - expect(calls).toEqual({ token: 1, permission: 1, minerList: 1, commentGets: 1, commentPatches: 1, checkRuns: 0 }); + // commentGets/commentPatches: 2 — first the purple reviewing placeholder, then the final refreshed panel. + expect(calls).toEqual({ token: 1, permission: 1, minerList: 1, commentGets: 2, commentPatches: 2, checkRuns: 0 }); expect(patchedBody).toContain(""); expect(patchedBody).toContain("Readiness score:"); expect(patchedBody).toContain("- [ ] Re-run Gittensory review"); @@ -3719,7 +3790,7 @@ describe("queue processors", () => { // The confirmed-miner detection WAS fetched (the #824 helper's miner-detection path) and the panel retriggered. expect(calls.minerList).toBeGreaterThanOrEqual(1); expect(calls.permission).toBe(1); - expect(calls.commentPatches).toBe(1); + expect(calls.commentPatches).toBe(2); const audit = await env.DB.prepare("select actor, outcome from audit_events where event_type = ? and target_key = ?") .bind("github_app.pr_panel_retriggered", "JSONbored/gittensory#48") .first<{ actor: string; outcome: string }>(); @@ -3866,7 +3937,8 @@ describe("queue processors", () => { }); // token: 1 — the installation token is now cached + reused within the request (was 2: main + permission check). - expect(calls).toEqual({ token: 1, permission: 1, minerList: 1, commentGets: 1, commentPatches: 1 }); + // commentGets/commentPatches: 2 — first the purple reviewing placeholder, then the final refreshed panel. + expect(calls).toEqual({ token: 1, permission: 1, minerList: 1, commentGets: 2, commentPatches: 2 }); }); it("skips PR panel reruns when the editing actor and PR author are unavailable", async () => { @@ -4451,7 +4523,7 @@ describe("queue processors", () => { }, }); - expect(calls.comments).toBe(1); + expect(calls.comments).toBe(2); // Still leads with the panel marker → the upsert updates the SAME sticky comment in place (no duplicate). expect(postedBody).toContain(""); // The UNIFIED shape, which the legacy body never emits: a full-comment GitHub alert wrapper… @@ -5333,7 +5405,7 @@ describe("queue processors", () => { }, }); - expect(calls).toEqual({ minerList: 2, comments: 1 }); + expect(calls).toEqual({ minerList: 2, comments: 2 }); }); it("fails closed when official miner detection is unavailable", async () => { diff --git a/test/unit/unified-comment.test.ts b/test/unit/unified-comment.test.ts index 9c7d2a312b..59a1fd5a07 100644 --- a/test/unit/unified-comment.test.ts +++ b/test/unit/unified-comment.test.ts @@ -437,23 +437,23 @@ describe("renderReviewingPlaceholder", () => { }); describe("shouldPostReviewingPlaceholder", () => { - it("returns true when AI will run, mode is live, and a comment will be posted", () => { - expect(shouldPostReviewingPlaceholder({ aiReviewWillRun: true, mode: "live", willComment: true })).toBe(true); + it("returns true when a live review refresh will post a comment", () => { + expect(shouldPostReviewingPlaceholder({ reviewWillRun: true, mode: "live", willComment: true })).toBe(true); }); - it("returns false when AI review will not run", () => { - expect(shouldPostReviewingPlaceholder({ aiReviewWillRun: false, mode: "live", willComment: true })).toBe(false); + it("returns false when no review refresh is running", () => { + expect(shouldPostReviewingPlaceholder({ reviewWillRun: false, mode: "live", willComment: true })).toBe(false); }); it("returns false in dry-run mode — placeholder must never write to GitHub in non-live mode", () => { - expect(shouldPostReviewingPlaceholder({ aiReviewWillRun: true, mode: "dry_run", willComment: true })).toBe(false); + expect(shouldPostReviewingPlaceholder({ reviewWillRun: true, mode: "dry_run", willComment: true })).toBe(false); }); it("returns false in paused mode", () => { - expect(shouldPostReviewingPlaceholder({ aiReviewWillRun: true, mode: "paused", willComment: true })).toBe(false); + expect(shouldPostReviewingPlaceholder({ reviewWillRun: true, mode: "paused", willComment: true })).toBe(false); }); it("returns false when no comment will be posted — avoids a permanent orphaned purple comment", () => { - expect(shouldPostReviewingPlaceholder({ aiReviewWillRun: true, mode: "live", willComment: false })).toBe(false); + expect(shouldPostReviewingPlaceholder({ reviewWillRun: true, mode: "live", willComment: false })).toBe(false); }); }); From 9d07a9ba898de5d9a5e0f98c88689463e1a77f73 Mon Sep 17 00:00:00 2001 From: JSONbored <49853598+JSONbored@users.noreply.github.com> Date: Sun, 28 Jun 2026 17:35:15 -0700 Subject: [PATCH 28/68] fix(review): coalesce self-host review retries --- src/env.d.ts | 6 + src/github/webhook.ts | 24 +++- src/queue/processors.ts | 121 +++++++++++++++----- src/queue/retryable.ts | 39 +++++++ src/review/unified-comment.ts | 10 +- src/selfhost/pg-queue.ts | 12 +- src/selfhost/queue-common.ts | 5 + src/selfhost/sqlite-queue.ts | 12 +- src/server.ts | 1 + test/unit/ci-completion-fork-resume.test.ts | 16 +-- test/unit/queue.test.ts | 80 +++++++++++++ test/unit/selfhost-pg-queue.test.ts | 27 +++++ test/unit/selfhost-queue-common.test.ts | 14 +++ test/unit/selfhost-sqlite-queue.test.ts | 36 ++++++ test/unit/unified-comment.test.ts | 7 ++ test/unit/webhook.test.ts | 40 +++++++ 16 files changed, 399 insertions(+), 51 deletions(-) create mode 100644 src/queue/retryable.ts diff --git a/src/env.d.ts b/src/env.d.ts index fc255f5b50..819e251a64 100644 --- a/src/env.d.ts +++ b/src/env.d.ts @@ -25,6 +25,12 @@ declare global { * force MANUAL review (no auto-merge / auto-close). Optional — absent ⇒ the conservative * DEFAULT_CRUCIAL_GUARDRAIL_GLOBS fallback applies (CI workflows + scripts still guarded). */ REVIEW_CONFIG?: KVNamespace; + /** Self-host transient cache for short-lived coalescing/backpressure keys. */ + SELFHOST_TRANSIENT_CACHE?: { + get(key: string): Promise; + set(key: string, value: string, ttlSeconds: number): Promise; + del?(key: string): Promise; + }; /** TODO (convergence follow-up): a per-PR LOCK Durable Object (`SubmissionLock` mutex) is a separate, * more-involved sub-task — it needs the ported DO class + its own migration tag, not just a binding here. * Deliberately NOT declared in this chunk; the review path keeps its current concurrency behavior. */ diff --git a/src/github/webhook.ts b/src/github/webhook.ts index 380c4c93f0..08bef7ad1f 100644 --- a/src/github/webhook.ts +++ b/src/github/webhook.ts @@ -37,6 +37,8 @@ export async function handleGitHubWebhook(c: Context<{ Bindings: Env }>): Promis export async function enqueueVerifiedWebhook(c: Context<{ Bindings: Env }>, deliveryId: string, eventName: string, rawBody: string): Promise { const result = await enqueueWebhookByEnv(c.env, deliveryId, eventName, rawBody); switch (result) { + case "ignored": + return c.json({ ok: true, deliveryId, eventName, status: "ignored" }, 202); case "invalid_json": return c.json({ error: "invalid_json" }, 400); case "duplicate": @@ -48,7 +50,7 @@ export async function enqueueVerifiedWebhook(c: Context<{ Bindings: Env }>, deli } } -export type EnqueueWebhookResult = "queued" | "duplicate" | "invalid_json" | "enqueue_failed"; +export type EnqueueWebhookResult = "queued" | "duplicate" | "ignored" | "invalid_json" | "enqueue_failed"; /** Env-based core of the webhook enqueue (parse → dedup → record → WEBHOOKS lane), with NO Hono Context. Shared by * the request-context receiver above AND the pull-mode relay drain loop (server.ts), which has no Context. Returns @@ -79,6 +81,10 @@ export async function enqueueWebhookByEnv(env: Env, deliveryId: string, eventNam repositoryFullName: payload.repository?.full_name, payloadHash, }; + if (isSelfAuthoredAppCommentWebhook(env, eventName, payload)) { + await recordWebhookEvent(env, { ...eventRow, status: "processed" }); + return "ignored"; + } await recordWebhookEvent(env, { ...eventRow, status: "queued" }); const message: JobMessage = { type: "github-webhook", deliveryId, eventName, payload }; @@ -97,6 +103,22 @@ export async function enqueueWebhookByEnv(env: Env, deliveryId: string, eventNam return "queued"; } +function isSelfAuthoredAppCommentWebhook( + env: Env, + eventName: string, + payload: GitHubWebhookPayload, +): boolean { + if (eventName !== "issue_comment") return false; + if (payload.action !== "created" && payload.action !== "edited") return false; + const botLogin = `${env.GITHUB_APP_SLUG}[bot]`.toLowerCase(); + return ( + payload.sender?.type === "Bot" && + payload.sender.login?.toLowerCase() === botLogin && + payload.comment?.user?.type === "Bot" && + payload.comment.user.login?.toLowerCase() === botLogin + ); +} + /** The brokered self-host's relay RECEIVER. The central Orb forwards an event here, HMAC-signed (x-orb-signature- * 256) with THIS container's enrollment secret. We verify with our own ORB_ENROLLMENT_SECRET, then enqueue the * event exactly like a GitHub webhook (the body IS a GitHub webhook payload; only the transport differs). */ diff --git a/src/queue/processors.ts b/src/queue/processors.ts index 8444c7c5aa..2e2d8ae398 100644 --- a/src/queue/processors.ts +++ b/src/queue/processors.ts @@ -264,6 +264,7 @@ import { import { isDuplicateClusterWinner } from "../signals/duplicate-winner"; import { buildUnifiedReviewDiff } from "../review/review-diff"; import { buildUnifiedCommentBody } from "../review/unified-comment-bridge"; +import { isRetryableJobError, RetryableJobError } from "./retryable"; import { screenshotsAllowed } from "../review/visual-wire"; import { isVisualPath } from "../review/visual/paths"; import { buildCapture, type CaptureRoute } from "../review/visual/capture"; @@ -398,6 +399,7 @@ const PR_PUBLIC_SURFACE_ACTIONS = new Set([ ]); const PR_GATE_CLOSED_ACTIONS = new Set(["closed"]); const ISSUE_PLAN_COOLDOWN_MS = 10 * 60 * 1000; +const AI_REVIEW_INCOMPLETE_RETRY_MS = 5 * 60 * 1000; /** * Run (or dry-run) the data-retention prune across the configured log/snapshot tables and audit the @@ -1154,7 +1156,7 @@ async function regatePullRequest( skipAiReview: settings.aiReviewMode === "off", }, ).catch((error) => { - if (isGitHubRateLimitedError(error)) throw error; + if (isGitHubRateLimitedError(error) || isRetryableJobError(error)) throw error; console.error( JSON.stringify({ level: "warn", @@ -1583,7 +1585,7 @@ async function reReviewStoredPullRequest( ...(options.skipAiReview ? { skipAiReview: true } : {}), }, ).catch((error) => { - if (isGitHubRateLimitedError(error)) throw error; + if (isGitHubRateLimitedError(error) || isRetryableJobError(error)) throw error; console.error( JSON.stringify({ level: "warn", @@ -1749,10 +1751,34 @@ async function prReadyForReview( // (held / needs-human) instead of deferring forever. Generous so a genuinely-slow CI is never cut off early. const STUCK_CI_DEFER_MS = 30 * 60 * 1000; +async function getTransientKey(env: Env, key: string): Promise { + if (!env.SELFHOST_TRANSIENT_CACHE) return null; + try { + return await env.SELFHOST_TRANSIENT_CACHE.get(key); + } catch { + return null; + } +} + +async function putTransientKey( + env: Env, + key: string, + value: string, + ttlSeconds: number, +): Promise { + if (!env.SELFHOST_TRANSIENT_CACHE) return; + try { + await env.SELFHOST_TRANSIENT_CACHE.set(key, value, ttlSeconds); + } catch { + // best-effort coalescing only + } +} + /** - * True when CI for this PR+headSha has been pending past STUCK_CI_DEFER_MS. Stamps the first-seen time in KV - * (REVIEW_CONFIG) keyed by repo#pr:headSha — a new push is a new SHA, so the window resets per commit. A missing - * KV / KV hiccup degrades to `false` (never force-finalize → keeps the safe old defer rather than acting early). + * True when CI for this PR+headSha has been pending past STUCK_CI_DEFER_MS. Stamps the first-seen time in a + * transient cache keyed by repo#pr:headSha — a new push is a new SHA, so the window resets per commit. A missing + * cache / cache hiccup degrades to `false` (never force-finalize → keeps the safe old defer rather than acting + * early). */ async function ciPendingDeferStuck( env: Env, @@ -1760,14 +1786,12 @@ async function ciPendingDeferStuck( prNumber: number, headSha: string | null | undefined, ): Promise { - if (!env.REVIEW_CONFIG || !headSha) return false; + if (!headSha) return false; const key = `ci-pending-first-seen:${repoFullName.toLowerCase()}#${prNumber}:${headSha}`; try { - const first = await env.REVIEW_CONFIG.get(key); + const first = await getTransientKey(env, key); if (!first) { - await env.REVIEW_CONFIG.put(key, String(Date.now()), { - expirationTtl: 7 * 24 * 3600, - }); + await putTransientKey(env, key, String(Date.now()), 7 * 24 * 3600); return false; } const firstMs = Number(first); @@ -1792,16 +1816,13 @@ const MAX_PREVIEW_POLLS = 5; /** * Coalesce CI-completion re-reviews: claims a per-PR window and returns true if this PR was already re-reviewed - * within CI_COALESCE_WINDOW_SECONDS (caller skips). KV-backed (REVIEW_CONFIG); a missing KV or a KV hiccup - * degrades to NO coalescing (returns false — never blocks a re-review, never throws). + * within CI_COALESCE_WINDOW_SECONDS (caller skips). Self-host uses the transient Redis cache. A missing cache or + * cache hiccup degrades to NO coalescing (returns false — never blocks a re-review, never throws). */ async function ciCompletionCoalesced(env: Env, key: string): Promise { - if (!env.REVIEW_CONFIG) return false; try { - if (await env.REVIEW_CONFIG.get(key)) return true; // already handled within the window → skip this event - await env.REVIEW_CONFIG.put(key, "1", { - expirationTtl: CI_COALESCE_WINDOW_SECONDS, - }); // claim the window + if (await getTransientKey(env, key)) return true; // already handled within the window → skip this event + await putTransientKey(env, key, "1", CI_COALESCE_WINDOW_SECONDS); // claim the window return false; } catch { return false; @@ -3052,7 +3073,7 @@ async function processGitHubWebhook( action: payload.action, }, ).catch((error) => { - if (isGitHubRateLimitedError(error)) throw error; + if (isGitHubRateLimitedError(error) || isRetryableJobError(error)) throw error; console.error( JSON.stringify({ level: "warn", @@ -4307,6 +4328,7 @@ async function maybePublishPrPublicSurface( } | undefined; let inlineCommentsEnabledForReview = false; + let aiReviewExpected = false; let gateFinalized = false; // The PR's changed files are needed by the slop/manifest gates, the AI review + grounding + RAG, the secret // scan, the check-run, and the unified comment. Resolve them AT MOST ONCE per review and share across the @@ -4531,9 +4553,11 @@ async function maybePublishPrPublicSurface( confirmedContributor, skipAiReview: webhook.skipAiReview, })); + aiReviewExpected = aiReviewWillRun; // Post a transient "🟪 reviewing…" placeholder BEFORE the review refresh runs so contributors never see a // stale green/yellow/red verdict while the current head is being recomputed. In-place upsert: once the final - // verdict is ready it overwrites this comment. Best-effort — a failed post never aborts the review. + // verdict is ready it overwrites this comment. GitHub rate-limits still abort so the queue can retry instead + // of leaving a stale public surface visible. if ( shouldPostReviewingPlaceholder({ reviewWillRun: true, @@ -4542,14 +4566,26 @@ async function maybePublishPrPublicSurface( }) ) { const placeholderBody = `${PR_PANEL_COMMENT_MARKER}\n\n${renderReviewingPlaceholder()}`; - await createOrUpdatePrIntelligenceComment( - env, - installationId, - repoFullName, - pr.number, - placeholderBody, - { mode }, - ).catch(() => undefined); + try { + await createOrUpdatePrIntelligenceComment( + env, + installationId, + repoFullName, + pr.number, + placeholderBody, + { mode }, + ); + } catch (error) { + if (isGitHubRateLimitedError(error)) throw error; + await recordAuditEvent(env, { + eventType: "github_app.reviewing_placeholder_failed", + actor: author, + targetKey: `${repoFullName}#${pr.number}`, + outcome: "error", + detail: errorMessage(error), + metadata: { deliveryId: webhook.deliveryId, repoFullName }, + }).catch(() => undefined); + } } if (aiReviewWillRun) { // #1 self-host AI-review cache: the LLM output for a PR changes only when the code (head SHA) or the review @@ -4627,6 +4663,35 @@ async function maybePublishPrPublicSurface( ).catch(() => undefined); } } + if (aiReviewExpected && !aiReview?.notes?.trim()) { + const retryError = new RetryableJobError( + "AI review did not produce a public summary yet; keeping PR surface in reviewing state", + { + retryAfterMs: AI_REVIEW_INCOMPLETE_RETRY_MS, + retryKind: "ai_review_public_summary_missing", + }, + ); + await recordAuditEvent(env, { + eventType: "github_app.ai_review_public_summary_missing", + actor: author, + targetKey: `${repoFullName}#${pr.number}`, + outcome: "error", + detail: retryError.message, + metadata: { + deliveryId: webhook.deliveryId, + repoFullName, + retryAfterMs: AI_REVIEW_INCOMPLETE_RETRY_MS, + }, + }).catch(() => undefined); + captureReviewFailure(retryError, { + kind: "review", + reason: "ai_review_public_summary_missing", + repo: repoFullName, + pr: pr.number, + head_sha: advisory.headSha, + }); + throw retryError; + } // Secrets-scan (#audit-3.4): always scans the REAL resolved diff and, on a CONCRETE credential hit, appends a // critical `secret_leak` hard blocker BEFORE the gate evaluates — unconditionally, since a committed token is @@ -4803,7 +4868,7 @@ async function maybePublishPrPublicSurface( } } } catch (error) { - if (isGitHubRateLimitedError(error)) throw error; + if (isGitHubRateLimitedError(error) || isRetryableJobError(error)) throw error; // The pending Gate check was posted but evaluation could not finish. Finalize it to a neutral // (non-blocking) terminal state so it never hangs in_progress; it re-runs on the next push. Only when // the gate was enabled, a pending check id exists, and a real conclusion was not already published. diff --git a/src/queue/retryable.ts b/src/queue/retryable.ts new file mode 100644 index 0000000000..a514dbbbed --- /dev/null +++ b/src/queue/retryable.ts @@ -0,0 +1,39 @@ +const DEFAULT_RETRY_AFTER_MS = 5 * 60 * 1000; +const MIN_RETRY_AFTER_MS = 1_000; +const MAX_RETRY_AFTER_MS = 60 * 60 * 1000; + +function clampRetryAfterMs(value: number): number { + if (!Number.isFinite(value)) return DEFAULT_RETRY_AFTER_MS; + return Math.min( + MAX_RETRY_AFTER_MS, + Math.max(MIN_RETRY_AFTER_MS, Math.round(value)), + ); +} + +export class RetryableJobError extends Error { + readonly retryAfterMs: number; + readonly retryKind: string; + + constructor( + message: string, + opts: { retryAfterMs?: number | undefined; retryKind: string }, + ) { + super(message); + this.name = "RetryableJobError"; + this.retryAfterMs = clampRetryAfterMs( + opts.retryAfterMs ?? DEFAULT_RETRY_AFTER_MS, + ); + this.retryKind = opts.retryKind; + } +} + +export function isRetryableJobError( + error: unknown, +): error is RetryableJobError { + return error instanceof RetryableJobError; +} + +export function retryableJobDelayMs(error: unknown): number | null { + if (!isRetryableJobError(error)) return null; + return error.retryAfterMs; +} diff --git a/src/review/unified-comment.ts b/src/review/unified-comment.ts index 92eb078d85..9abc3372e1 100644 --- a/src/review/unified-comment.ts +++ b/src/review/unified-comment.ts @@ -303,7 +303,7 @@ function plural(n: number, one: string): string { function statusChips(input: UnifiedReviewInput, ctx: UnifiedCommentContext): string { const chips: string[] = [`\`${plural(input.changedFiles, "file")}\``]; - if (input.reviewerCount > 0) chips.push(`\`${input.reviewerCount} AI reviewers\``); + if (input.reviewerCount > 0) chips.push(`\`${plural(input.reviewerCount, "AI reviewer")}\``); const blockerCount = (input.blockers ?? []).length; chips.push(blockerCount ? `\`${plural(blockerCount, "blocker")}\`` : "`no blockers`"); if (typeof ctx.readinessScore === "number") chips.push(`\`readiness ${Math.round(ctx.readinessScore)}/100\``); @@ -385,11 +385,17 @@ function failingChecksBlock(readiness: MergeReadiness | undefined): string { function signalTable(input: UnifiedReviewInput, ctx: UnifiedCommentContext): string { const blockerCount = (input.blockers ?? []).length; + const reviewerEvidence = + input.reviewerCount > 1 + ? `${input.reviewerCount} reviewers, synthesized` + : input.reviewerCount === 1 + ? "1 reviewer" + : "No AI review summary"; const codeRow: UnifiedSignalRow = { label: "Code review", state: blockerCount ? "fail" : "ok", result: blockerCount ? plural(blockerCount, "blocker") : "No blockers", - evidence: input.reviewerCount > 0 ? `${input.reviewerCount} reviewers, synthesized` : "synthesized", + evidence: reviewerEvidence, }; const rows = [codeRow, ...(ctx.signals ?? [])]; const lines = rows.map((r, i) => { diff --git a/src/selfhost/pg-queue.ts b/src/selfhost/pg-queue.ts index a7ccca9026..641e017af8 100644 --- a/src/selfhost/pg-queue.ts +++ b/src/selfhost/pg-queue.ts @@ -6,7 +6,7 @@ import type { Pool } from "pg"; import { logAudit, extractPayloadType } from "./audit"; import { incr } from "./metrics"; import { captureError } from "./sentry"; -import { githubRateLimitRetryDelayMs, jobPriority } from "./queue-common"; +import { githubRateLimitRetryDelayMs, jobPriority, nonConsumingRetryDelayMs } from "./queue-common"; import type { JobMessage } from "../types"; const TABLE = "_selfhost_jobs"; @@ -175,13 +175,13 @@ export function createPgQueue( } catch (error) { const attempts = Number(job.attempts) + 1; const errMsg = error instanceof Error ? error.message : "unknown error"; - const rateLimitDelayMs = githubRateLimitRetryDelayMs(error); - if (rateLimitDelayMs !== null) { + const nonConsumingDelayMs = nonConsumingRetryDelayMs(error); + if (nonConsumingDelayMs !== null) { await pool.query( `UPDATE ${TABLE} SET status='pending', run_after=$1, last_error=$2 WHERE id=$3`, - [Date.now() + rateLimitDelayMs, errMsg, job.id], + [Date.now() + nonConsumingDelayMs, errMsg, job.id], ); - incr("gittensory_jobs_rate_limited_total"); + incr(githubRateLimitRetryDelayMs(error) !== null ? "gittensory_jobs_rate_limited_total" : "gittensory_jobs_deferred_total"); logAudit({ event: "job_rate_limited", ts: Date.now(), @@ -189,7 +189,7 @@ export function createPgQueue( payload_type: extractPayloadType(job.payload), latency_ms: Date.now() - claimedAt, attempts, - retry_after_ms: rateLimitDelayMs, + retry_after_ms: nonConsumingDelayMs, error: errMsg, }); return true; diff --git a/src/selfhost/queue-common.ts b/src/selfhost/queue-common.ts index d9037b6cb5..59abbd331b 100644 --- a/src/selfhost/queue-common.ts +++ b/src/selfhost/queue-common.ts @@ -1,3 +1,4 @@ +import { retryableJobDelayMs } from "../queue/retryable"; import { extractPayloadType } from "./audit"; // Webhook-driven work (a fresh PR -> its review) jumps ahead of heavy background jobs. Per-PR review refreshes @@ -84,6 +85,10 @@ export function githubRateLimitRetryDelayMs( return null; } +export function nonConsumingRetryDelayMs(error: unknown): number | null { + return githubRateLimitRetryDelayMs(error) ?? retryableJobDelayMs(error); +} + function clampRetryDelay(delayMs: number): number { if (!Number.isFinite(delayMs) || delayMs <= 0) return DEFAULT_GITHUB_RATE_LIMIT_RETRY_MS; return Math.min(Math.ceil(delayMs), MAX_GITHUB_RATE_LIMIT_RETRY_MS); diff --git a/src/selfhost/sqlite-queue.ts b/src/selfhost/sqlite-queue.ts index b5ed66563f..6df0ac6a43 100644 --- a/src/selfhost/sqlite-queue.ts +++ b/src/selfhost/sqlite-queue.ts @@ -7,7 +7,7 @@ import type { SqliteDriver } from "./d1-adapter"; import { logAudit, extractPayloadType } from "./audit"; import { incr } from "./metrics"; import { captureError } from "./sentry"; -import { githubRateLimitRetryDelayMs, jobPriority } from "./queue-common"; +import { githubRateLimitRetryDelayMs, jobPriority, nonConsumingRetryDelayMs } from "./queue-common"; import type { JobMessage } from "../types"; const TABLE = "_selfhost_jobs"; @@ -167,13 +167,13 @@ export function createSqliteQueue( } catch (error) { const attempts = job.attempts + 1; const errMsg = error instanceof Error ? error.message : "unknown error"; - const rateLimitDelayMs = githubRateLimitRetryDelayMs(error); - if (rateLimitDelayMs !== null) { + const nonConsumingDelayMs = nonConsumingRetryDelayMs(error); + if (nonConsumingDelayMs !== null) { driver.query( `UPDATE ${TABLE} SET status='pending', run_after=?, last_error=? WHERE id=?`, - [Date.now() + rateLimitDelayMs, errMsg, job.id], + [Date.now() + nonConsumingDelayMs, errMsg, job.id], ); - incr("gittensory_jobs_rate_limited_total"); + incr(githubRateLimitRetryDelayMs(error) !== null ? "gittensory_jobs_rate_limited_total" : "gittensory_jobs_deferred_total"); logAudit({ event: "job_rate_limited", ts: Date.now(), @@ -181,7 +181,7 @@ export function createSqliteQueue( payload_type: extractPayloadType(job.payload), latency_ms: Date.now() - claimedAt, attempts, - retry_after_ms: rateLimitDelayMs, + retry_after_ms: nonConsumingDelayMs, error: errMsg, }); return true; diff --git a/src/server.ts b/src/server.ts index 683232b418..16cc4ac14c 100644 --- a/src/server.ts +++ b/src/server.ts @@ -465,6 +465,7 @@ async function main(): Promise { AI: ai, ...(embedAi ? { AI_EMBED: embedAi as unknown as Ai } : {}), ...(aiReviewPlan ? { AI_REVIEW_PLAN: aiReviewPlan } : {}), + ...(webhookCache ? { SELFHOST_TRANSIENT_CACHE: webhookCache } : {}), // Qdrant takes priority; falls back to the backend's built-in vectorize (pgvector or sqlite-vec) ...(vectorizeOverride ? { VECTORIZE: vectorizeOverride } diff --git a/test/unit/ci-completion-fork-resume.test.ts b/test/unit/ci-completion-fork-resume.test.ts index 8cc39b8751..c7ebb2ba72 100644 --- a/test/unit/ci-completion-fork-resume.test.ts +++ b/test/unit/ci-completion-fork-resume.test.ts @@ -10,18 +10,18 @@ import type { GitHubWebhookPayload, JobMessage } from "../../src/types"; const FORK_SHA = "deadbeefcafe1234deadbeefcafe1234deadbeef"; -class MemoryKv { +class MemoryTransientCache { readonly values = new Map(); getCalls = 0; - putCalls = 0; + setCalls = 0; async get(key: string): Promise { this.getCalls += 1; return this.values.get(key) ?? null; } - async put(key: string, value: string): Promise { - this.putCalls += 1; + async set(key: string, value: string): Promise { + this.setCalls += 1; this.values.set(key, value); } } @@ -154,8 +154,8 @@ describe("CI-completion fork PR resume (head-SHA fallback)", () => { }); it("dispatch: duplicate empty-pull_requests fork completions coalesce before head-SHA resolution", async () => { - const kv = new MemoryKv(); - const env = createTestEnv({ GITHUB_PUBLIC_TOKEN: "public-token", REVIEW_CONFIG: kv as unknown as KVNamespace }); + const cache = new MemoryTransientCache(); + const env = createTestEnv({ GITHUB_PUBLIC_TOKEN: "public-token", SELFHOST_TRANSIENT_CACHE: cache }); await upsertRepositoryFromGitHub(env, { name: "gittensory", full_name: "JSONbored/gittensory", private: false, owner: { login: "JSONbored" } }, 5001); let commitPullsCalls = 0; @@ -174,8 +174,8 @@ describe("CI-completion fork PR resume (head-SHA fallback)", () => { } expect(commitPullsCalls).toBe(1); - expect(kv.values.has(`ci-head-sha-resolve:jsonbored/gittensory@${FORK_SHA}`)).toBe(true); - expect(kv.putCalls).toBe(2); // one head-SHA resolution claim + one per-PR re-review claim + expect(cache.values.has(`ci-head-sha-resolve:jsonbored/gittensory@${FORK_SHA}`)).toBe(true); + expect(cache.setCalls).toBe(2); // one head-SHA resolution claim + one per-PR re-review claim const audits = await env.DB.prepare("select count(*) as n from audit_events where event_type = ?") .bind("github_app.ci_completion_fork_resume") diff --git a/test/unit/queue.test.ts b/test/unit/queue.test.ts index 265fd6f2f0..af580e3889 100644 --- a/test/unit/queue.test.ts +++ b/test/unit/queue.test.ts @@ -1236,6 +1236,86 @@ describe("queue processors", () => { expect(postedBodies[0]).toContain("🟪"); }); + it("keeps the PR comment and Gate in 🟪 reviewing state when AI review produces no public summary", async () => { + const env = createTestEnv({ + GITHUB_APP_PRIVATE_KEY: await generatePrivateKeyPem(), + AI: { + run: async () => ({ response: "not-json" }), + } as unknown as Ai, + AI_SUMMARIES_ENABLED: "true", + AI_PUBLIC_COMMENTS_ENABLED: "true", + AI_DAILY_NEURON_BUDGET: "100000", + }); + await persistRegistrySnapshot( + env, + normalizeRegistryPayload( + { "JSONbored/gittensory": { emission_share: 0.01, issue_discovery_share: 0 } }, + { kind: "raw-github", url: "https://example.test" }, + "2026-05-23T00:00:00.000Z", + ), + ); + await upsertRepositoryFromGitHub(env, { name: "gittensory", full_name: "JSONbored/gittensory", private: false, owner: { login: "JSONbored" } }, 123); + await upsertRepositorySettings(env, { + repoFullName: "JSONbored/gittensory", + commentMode: "all_prs", + publicSurface: "comment_only", + autoLabelEnabled: false, + checkRunMode: "off", + gateCheckMode: "enabled", + aiReviewMode: "block", + gatePack: "oss-anti-slop", + }); + const commentBodies: string[] = []; + const checkPatches: Array<{ status?: string; conclusion?: string }> = []; + vi.stubGlobal("fetch", async (input: RequestInfo | URL, init?: RequestInit) => { + const url = input.toString(); + const method = init?.method ?? "GET"; + if (url.includes("/access_tokens")) return Response.json({ token: "installation-token" }); + if (url.includes("/pulls/10/files")) return Response.json([{ filename: "src/a.ts", status: "modified", additions: 1, deletions: 0, changes: 1, patch: "@@\n+export const ok = true;" }]); + if (url.endsWith("/pulls/10")) return Response.json({ number: 10, title: "Clean PR", state: "open", user: { login: "contributor" }, head: { sha: "a10" }, labels: [], body: "Closes #1", mergeable_state: "clean" }); + if (url.includes("/commits/a10/check-runs")) return Response.json({ total_count: 0, check_runs: [] }); + if (url.includes("/commits/a10/status")) return Response.json({ state: "success", statuses: [] }); + if (url.includes("/issues/10/comments") && method === "GET") return Response.json([]); + if (url.includes("/issues/10/comments") && method === "POST") { + commentBodies.push(String((JSON.parse(String(init?.body ?? "{}")) as { body?: string }).body ?? "")); + return Response.json({ id: 1 }, { status: 201 }); + } + if (url.includes("/issues/1")) return Response.json({ number: 1, title: "Issue", state: "open", labels: [], user: { login: "reporter" } }); + if (url.includes("/check-runs") && method === "POST") return Response.json({ id: 971 }, { status: 201 }); + if (url.includes("/check-runs/971") && method === "PATCH") { + checkPatches.push(JSON.parse(String(init?.body ?? "{}")) as { status?: string; conclusion?: string }); + return Response.json({ id: 971 }); + } + if (url.includes("/branches/")) return Response.json({ protected: false, protection: { required_status_checks: { contexts: [] } } }); + return Response.json({}); + }); + + await expect( + processJob(env, { + type: "github-webhook", + deliveryId: "reviewing-placeholder-ai-summary-missing", + eventName: "pull_request", + payload: { + action: "opened", + installation: { id: 123, account: { login: "JSONbored", id: 1, type: "User" } }, + repository: { name: "gittensory", full_name: "JSONbored/gittensory", private: false, owner: { login: "JSONbored" } }, + pull_request: { number: 10, title: "Clean PR", state: "open", user: { login: "contributor" }, head: { sha: "a10" }, labels: [], body: "Closes #1" }, + }, + }), + ).rejects.toThrow(/public summary/i); + + expect(commentBodies).toHaveLength(1); + expect(commentBodies[0]).toContain("is reviewing"); + expect(commentBodies[0]).toContain("🟪"); + expect(commentBodies[0]).not.toContain("held for maintainer review"); + expect(commentBodies[0]).not.toContain("Review summary"); + expect(checkPatches).toHaveLength(0); + const audit = await env.DB.prepare("select count(*) as n from audit_events where event_type = ?") + .bind("github_app.ai_review_public_summary_missing") + .first<{ n: number }>(); + expect(audit?.n).toBe(1); + }); + it("agent re-gate sweep re-reviews each stale open PR (installation id) and swallows a failing re-review", async () => { const env = createTestEnv({}); await upsertInstallation(env, { action: "created", installation: { id: 9001, account: { login: "owner", id: 1, type: "Organization" }, target_type: "Organization", repository_selection: "selected", permissions: {}, events: [] } }); diff --git a/test/unit/selfhost-pg-queue.test.ts b/test/unit/selfhost-pg-queue.test.ts index e2b3dbdfc9..20ae8b48fe 100644 --- a/test/unit/selfhost-pg-queue.test.ts +++ b/test/unit/selfhost-pg-queue.test.ts @@ -3,6 +3,7 @@ import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import type { Pool, QueryResult } from "pg"; import { createPgQueue } from "../../src/selfhost/pg-queue"; +import { RetryableJobError } from "../../src/queue/retryable"; import type { JobMessage } from "../../src/types"; const msg = (t: string): JobMessage => ({ type: t }) as unknown as JobMessage; @@ -158,6 +159,32 @@ describe("createPgQueue (durable #977)", () => { ); }); + it("reschedules retryable incomplete review jobs without consuming the dead-letter budget", async () => { + const m = makePool(); + m.enqueueJob("1", { type: "agent-regate-pr" }, 4); + const retryable = new RetryableJobError("AI review did not produce a public summary yet", { + retryAfterMs: 5_000, + retryKind: "ai_review_public_summary_missing", + }); + const q = createPgQueue( + m.pool, + async () => { + throw retryable; + }, + { maxRetries: 1, backoffMs: () => 0 }, + ); + await q.init(); + await q.drain(); + expect(m.pool.query).toHaveBeenCalledWith( + expect.stringContaining("SET status='pending', run_after=$1"), + expect.arrayContaining([expect.any(Number), "AI review did not produce a public summary yet", "1"]), + ); + expect(m.pool.query).not.toHaveBeenCalledWith( + expect.stringContaining("status='dead'"), + expect.anything(), + ); + }); + it("records 'unknown error' when consumer throws a non-Error", async () => { const m = makePool(); m.enqueueJob("1", { type: "t" }, 0); diff --git a/test/unit/selfhost-queue-common.test.ts b/test/unit/selfhost-queue-common.test.ts index 85fea549af..7b433bb9a3 100644 --- a/test/unit/selfhost-queue-common.test.ts +++ b/test/unit/selfhost-queue-common.test.ts @@ -2,7 +2,9 @@ import { describe, expect, it } from "vitest"; import { githubRateLimitRetryDelayMs, jobPriority, + nonConsumingRetryDelayMs, } from "../../src/selfhost/queue-common"; +import { RetryableJobError } from "../../src/queue/retryable"; const payload = (value: unknown): string => JSON.stringify(value); @@ -75,4 +77,16 @@ describe("self-host queue common helpers", () => { ), ).toBe(8_000); }); + + it("extracts non-consuming retry delays from retryable job errors", () => { + expect(nonConsumingRetryDelayMs(new Error("boom"))).toBeNull(); + expect( + nonConsumingRetryDelayMs( + new RetryableJobError("AI review pending", { + retryAfterMs: 1234, + retryKind: "ai_review_public_summary_missing", + }), + ), + ).toBe(1234); + }); }); diff --git a/test/unit/selfhost-sqlite-queue.test.ts b/test/unit/selfhost-sqlite-queue.test.ts index 4c27a7458d..5c6e9ce23d 100644 --- a/test/unit/selfhost-sqlite-queue.test.ts +++ b/test/unit/selfhost-sqlite-queue.test.ts @@ -2,6 +2,7 @@ import { DatabaseSync } from "node:sqlite"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import { nodeSqliteDriver } from "../../src/selfhost/d1-adapter"; import { createSqliteQueue } from "../../src/selfhost/sqlite-queue"; +import { RetryableJobError } from "../../src/queue/retryable"; import type { JobMessage } from "../../src/types"; function makeDriver(): ReturnType { @@ -235,6 +236,41 @@ describe("createSqliteQueue (durable #980)", () => { expect(row.last_error).toContain("API rate limit exceeded"); }); + it("reschedules retryable incomplete review jobs without consuming the dead-letter budget", async () => { + const driver = makeDriver(); + let calls = 0; + const retryable = new RetryableJobError("AI review did not produce a public summary yet", { + retryAfterMs: 5_000, + retryKind: "ai_review_public_summary_missing", + }); + const q = createSqliteQueue( + driver, + async () => { + calls += 1; + throw retryable; + }, + { maxRetries: 1, backoffMs: () => 0 }, + ); + await q.binding.send(msg("agent-regate-pr")); + await q.drain(); + const { rows } = driver.query( + "SELECT status, attempts, run_after, last_error FROM _selfhost_jobs", + [], + ); + const row = rows[0] as { + status: string; + attempts: number; + run_after: number; + last_error: string; + }; + expect(calls).toBe(1); + expect(q.deadCount()).toBe(0); + expect(row.status).toBe("pending"); + expect(row.attempts).toBe(0); + expect(row.run_after).toBeGreaterThan(Date.now()); + expect(row.last_error).toContain("AI review did not produce"); + }); + it("SURVIVES A RESTART: a fresh queue over the same DB processes a persisted pending job", async () => { const driver = makeDriver(); const seen: string[] = []; diff --git a/test/unit/unified-comment.test.ts b/test/unit/unified-comment.test.ts index 59a1fd5a07..34e5f7a8ba 100644 --- a/test/unit/unified-comment.test.ts +++ b/test/unit/unified-comment.test.ts @@ -139,6 +139,13 @@ describe("renderUnifiedReviewComment", () => { expect(md).toContain("Checked by Gittensory."); }); + it("does not describe a single reviewer as synthesized", () => { + const md = renderUnifiedReviewComment({ ...base, reviewerCount: 1, decision: "manual", recommendations: ["manual_review"] }, ctx); + expect(md).toContain("`1 AI reviewer`"); + expect(md).toContain("| **Code review** | ✅ No blockers | 1 reviewer |"); + expect(md).not.toContain("1 reviewers, synthesized"); + }); + it("wraps the review body in the colored blockquote but renders the re-run checkbox OUTSIDE it (interactive)", () => { const md = renderUnifiedReviewComment({ ...base, decision: "merge" }, ctx); const lines = md.split("\n"); diff --git a/test/unit/webhook.test.ts b/test/unit/webhook.test.ts index 37af5bcc87..d7cea7ab54 100644 --- a/test/unit/webhook.test.ts +++ b/test/unit/webhook.test.ts @@ -151,6 +151,46 @@ describe("github webhook queue isolation (#audit-webhook-queue)", () => { expect(webhookSends).toBe(1); // routed to the dedicated webhook lane expect(jobsSends).toBe(0); // never the shared maintenance queue }); + + it("drops self-authored app comment webhooks before they add queue pressure", async () => { + const env = createTestEnv(); + let webhookSends = 0; + env.WEBHOOKS = { send: async () => void (webhookSends += 1) } as unknown as typeof env.WEBHOOKS; + const rawBody = JSON.stringify({ + action: "edited", + repository: { full_name: "JSONbored/gittensory" }, + installation: { id: 1 }, + issue: { number: 1701, pull_request: {} }, + comment: { id: 123, body: "", user: { login: "gittensory[bot]", type: "Bot" } }, + sender: { login: "gittensory[bot]", type: "Bot" }, + }); + const signature = await signWebhook(rawBody, env.GITHUB_WEBHOOK_SECRET); + const request = new Request("https://example.com/webhook", { method: "POST", body: rawBody }); + const headers: Record = { + "x-github-delivery": "self-comment-ignore-1", + "x-github-event": "issue_comment", + "x-hub-signature-256": signature, + }; + const context = { + req: { + raw: request, + header(name: string) { + return headers[name.toLowerCase()] ?? null; + }, + }, + env, + json(payload: unknown, status?: number) { + return Response.json(payload, status === undefined ? undefined : { status }); + }, + } as unknown as Context<{ Bindings: Env }>; + + const response = await handleGitHubWebhook(context); + expect(response.status).toBe(202); + await expect(response.json()).resolves.toMatchObject({ status: "ignored" }); + expect(webhookSends).toBe(0); + const event = await getWebhookEvent(env, "self-comment-ignore-1"); + expect(event?.status).toBe("processed"); + }); }); describe("handleOrbRelay (brokered self-host relay receiver)", () => { From 90243b9f60d97173a4b88c449fb074963e32a700 Mon Sep 17 00:00:00 2001 From: JSONbored <49853598+JSONbored@users.noreply.github.com> Date: Sun, 28 Jun 2026 18:06:25 -0700 Subject: [PATCH 29/68] fix(queue): coalesce self-host review backlog --- grafana/dashboards/gittensory.json | 12 +- src/selfhost/pg-queue.ts | 204 +++++++++++++++++++++--- src/selfhost/queue-common.ts | 149 +++++++++++++++++ src/selfhost/sqlite-queue.ts | 201 ++++++++++++++++++++--- src/server.ts | 16 ++ test/unit/selfhost-pg-queue.test.ts | 50 +++++- test/unit/selfhost-sqlite-queue.test.ts | 134 +++++++++++++++- 7 files changed, 708 insertions(+), 58 deletions(-) diff --git a/grafana/dashboards/gittensory.json b/grafana/dashboards/gittensory.json index c9658e84ff..90216fd8fd 100644 --- a/grafana/dashboards/gittensory.json +++ b/grafana/dashboards/gittensory.json @@ -176,7 +176,7 @@ "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "expr": "gittensory_jobs_processed_total", + "expr": "gittensory_jobs_processed_persisted_total", "legendFormat": "processed" } ] @@ -366,22 +366,22 @@ "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "expr": "rate(gittensory_jobs_processed_total[2m])", + "expr": "rate(gittensory_jobs_processed_persisted_total[2m])", "legendFormat": "processed/s" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "expr": "rate(gittensory_jobs_enqueued_total[2m])", + "expr": "rate(gittensory_jobs_enqueued_persisted_total[2m])", "legendFormat": "enqueued/s" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "expr": "rate(gittensory_jobs_failed_total[2m])", + "expr": "rate(gittensory_jobs_failed_persisted_total[2m])", "legendFormat": "failed/s" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "expr": "rate(gittensory_jobs_dead_total[2m])", + "expr": "rate(gittensory_jobs_dead_persisted_total[2m])", "legendFormat": "dead/s" } ] @@ -412,7 +412,7 @@ "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "expr": "rate(gittensory_jobs_failed_total[5m]) / (rate(gittensory_jobs_processed_total[5m]) + rate(gittensory_jobs_failed_total[5m]) + 0.0001)", + "expr": "rate(gittensory_jobs_failed_persisted_total[5m]) / (rate(gittensory_jobs_processed_persisted_total[5m]) + rate(gittensory_jobs_failed_persisted_total[5m]) + 0.0001)", "legendFormat": "failure %" } ] diff --git a/src/selfhost/pg-queue.ts b/src/selfhost/pg-queue.ts index 641e017af8..e6076df7b2 100644 --- a/src/selfhost/pg-queue.ts +++ b/src/selfhost/pg-queue.ts @@ -6,10 +6,21 @@ import type { Pool } from "pg"; import { logAudit, extractPayloadType } from "./audit"; import { incr } from "./metrics"; import { captureError } from "./sentry"; -import { githubRateLimitRetryDelayMs, jobPriority, nonConsumingRetryDelayMs } from "./queue-common"; +import { + deterministicJitterMs, + githubRateLimitRetryDelayMs, + jobCoalesceKey, + jobPriority, + nonConsumingRetryDelayMs, + queueRecoveryJitterMs, + queueStartupJitterMinJobs, + queueStartupJitterMs, + rateLimitRetryDelayWithJitter, +} from "./queue-common"; import type { JobMessage } from "../types"; const TABLE = "_selfhost_jobs"; +const STATS_TABLE = "_selfhost_job_stats"; const DDL = ` CREATE TABLE IF NOT EXISTS ${TABLE} ( id BIGSERIAL PRIMARY KEY, @@ -19,10 +30,17 @@ CREATE TABLE IF NOT EXISTS ${TABLE} ( run_after BIGINT NOT NULL DEFAULT 0, created_at BIGINT NOT NULL, last_error TEXT, - priority INTEGER NOT NULL DEFAULT 0 + priority INTEGER NOT NULL DEFAULT 0, + job_key TEXT ); ALTER TABLE ${TABLE} ADD COLUMN IF NOT EXISTS priority INTEGER NOT NULL DEFAULT 0; -CREATE INDEX IF NOT EXISTS ${TABLE}_claim ON ${TABLE}(status, run_after, priority);`; +ALTER TABLE ${TABLE} ADD COLUMN IF NOT EXISTS job_key TEXT; +CREATE INDEX IF NOT EXISTS ${TABLE}_claim ON ${TABLE}(status, run_after, priority); +CREATE INDEX IF NOT EXISTS ${TABLE}_pending_job_key ON ${TABLE}(job_key, status); +CREATE TABLE IF NOT EXISTS ${STATS_TABLE} ( + name TEXT PRIMARY KEY, + value BIGINT NOT NULL DEFAULT 0 +);`; export interface PgDurableQueue { binding: Queue; @@ -32,12 +50,14 @@ export interface PgDurableQueue { drain(): Promise; size(): Promise; deadCount(): Promise; + stats(): Promise>; } interface JobRow { id: string; payload: string; attempts: number; + job_key?: string | null; } export interface PgQueueOptions { @@ -78,16 +98,28 @@ export function createPgQueue( count: priorityBackfilled, }), ); - const recovered = - ( - await pool.query( - `UPDATE ${TABLE} SET status='pending' WHERE status='processing'`, - ) - ).rowCount ?? 0; + const keyBackfilled = await backfillJobKeys(); + if (keyBackfilled) + console.log( + JSON.stringify({ + event: "selfhost_queue_job_keys_backfilled", + count: keyBackfilled, + }), + ); + const recovered = await recoverProcessingJobs(); if (recovered) console.log( JSON.stringify({ event: "selfhost_queue_recovered", count: recovered }), ); + const spread = await spreadDueJobsOnStartup(); + if (spread) + console.log( + JSON.stringify({ + event: "selfhost_queue_startup_spread", + count: spread, + jitter_ms: queueStartupJitterMs(), + }), + ); } async function backfillJobPriorities(): Promise { @@ -107,17 +139,94 @@ export function createPgQueue( return changed; } + async function backfillJobKeys(): Promise { + const res = await pool.query( + `SELECT id, payload, job_key FROM ${TABLE} WHERE status IN ('pending', 'processing')`, + ); + let changed = 0; + for (const row of res.rows as Array<{ id: string; payload: string; job_key?: string | null }>) { + const key = jobCoalesceKey(row.payload); + if ((row.job_key ?? null) === key) continue; + await pool.query(`UPDATE ${TABLE} SET job_key=$1 WHERE id=$2`, [ + key, + row.id, + ]); + changed += 1; + } + return changed; + } + + async function recoverProcessingJobs(): Promise { + const res = await pool.query( + `SELECT id, payload, job_key FROM ${TABLE} WHERE status='processing'`, + ); + let changed = 0; + const now = Date.now(); + const maxJitter = queueRecoveryJitterMs(); + for (const row of res.rows as Array<{ id: string; payload: string; job_key?: string | null }>) { + const runAfter = now + deterministicJitterMs(`${row.job_key ?? ""}:${row.id}:${row.payload}`, maxJitter); + await pool.query(`UPDATE ${TABLE} SET status='pending', run_after=$1 WHERE id=$2`, [ + runAfter, + row.id, + ]); + changed += 1; + } + return changed; + } + + async function spreadDueJobsOnStartup(): Promise { + const now = Date.now(); + const res = await pool.query( + `SELECT id, payload, job_key FROM ${TABLE} WHERE status='pending' AND run_after<=$1`, + [now], + ); + const due = res.rows as Array<{ id: string; payload: string; job_key?: string | null }>; + if (due.length < queueStartupJitterMinJobs()) return 0; + const maxJitter = queueStartupJitterMs(); + if (maxJitter <= 0) return 0; + for (const row of due) { + const runAfter = now + deterministicJitterMs(`${row.job_key ?? ""}:${row.id}:${row.payload}`, maxJitter); + await pool.query(`UPDATE ${TABLE} SET run_after=$1 WHERE id=$2`, [ + runAfter, + row.id, + ]); + } + return due.length; + } + async function enqueue( message: JobMessage, delaySeconds: number, ): Promise { const now = Date.now(); const payload = JSON.stringify(message); + const runAfter = now + delaySeconds * 1000; + const priority = jobPriority(payload); + const key = jobCoalesceKey(payload); + if (key) { + const existing = ( + await pool.query( + `SELECT id FROM ${TABLE} WHERE status='pending' AND job_key=$1 ORDER BY priority DESC, run_after DESC, id LIMIT 1`, + [key], + ) + ).rows[0] as { id: string } | undefined; + if (existing) { + await pool.query( + `UPDATE ${TABLE} + SET payload=$1, run_after=GREATEST(run_after, $2), created_at=$3, priority=GREATEST(priority, $4), last_error=NULL + WHERE id=$5`, + [payload, runAfter, now, priority, existing.id], + ); + await recordQueueMetric("gittensory_jobs_coalesced_total"); + void pump(); + return; + } + } await pool.query( - `INSERT INTO ${TABLE} (payload, status, attempts, run_after, created_at, priority) VALUES ($1,'pending',0,$2,$3,$4)`, - [payload, now + delaySeconds * 1000, now, jobPriority(payload)], + `INSERT INTO ${TABLE} (payload, status, attempts, run_after, created_at, priority, job_key) VALUES ($1,'pending',0,$2,$3,$4,$5)`, + [payload, runAfter, now, priority, key], ); - incr("gittensory_jobs_enqueued_total"); + await recordQueueMetric("gittensory_jobs_enqueued_total"); void pump(); } @@ -126,7 +235,7 @@ export function createPgQueue( const res = await pool.query( `UPDATE ${TABLE} SET status='processing' WHERE id = (SELECT id FROM ${TABLE} WHERE status='pending' AND run_after<=$1 ORDER BY priority DESC, id FOR UPDATE SKIP LOCKED LIMIT 1) - RETURNING id, payload, attempts`, + RETURNING id, payload, attempts, job_key`, [Date.now()], ); return (res.rows[0] as JobRow | undefined) ?? null; @@ -144,7 +253,7 @@ export function createPgQueue( `UPDATE ${TABLE} SET status='dead', last_error='unparseable payload' WHERE id=$1`, [job.id], ); - incr("gittensory_jobs_dead_total"); + await recordQueueMetric("gittensory_jobs_dead_total"); logAudit({ event: "job_dead", ts: Date.now(), @@ -163,7 +272,7 @@ export function createPgQueue( try { await consume(message); await pool.query(`DELETE FROM ${TABLE} WHERE id=$1`, [job.id]); - incr("gittensory_jobs_processed_total"); + await recordQueueMetric("gittensory_jobs_processed_total"); logAudit({ event: "job_complete", ts: Date.now(), @@ -177,11 +286,17 @@ export function createPgQueue( const errMsg = error instanceof Error ? error.message : "unknown error"; const nonConsumingDelayMs = nonConsumingRetryDelayMs(error); if (nonConsumingDelayMs !== null) { - await pool.query( - `UPDATE ${TABLE} SET status='pending', run_after=$1, last_error=$2 WHERE id=$3`, - [Date.now() + nonConsumingDelayMs, errMsg, job.id], - ); - incr(githubRateLimitRetryDelayMs(error) !== null ? "gittensory_jobs_rate_limited_total" : "gittensory_jobs_deferred_total"); + const rateLimited = githubRateLimitRetryDelayMs(error) !== null; + const retryAfter = Date.now() + (rateLimited ? rateLimitRetryDelayWithJitter(nonConsumingDelayMs, `${job.job_key ?? ""}:${job.id}:${job.payload}`) : nonConsumingDelayMs); + if (job.job_key && (await mergeRescheduledJobIntoPending(job, retryAfter, errMsg))) { + await recordQueueMetric("gittensory_jobs_coalesced_total"); + } else { + await pool.query( + `UPDATE ${TABLE} SET status='pending', run_after=$1, last_error=$2 WHERE id=$3`, + [retryAfter, errMsg, job.id], + ); + } + await recordQueueMetric(rateLimited ? "gittensory_jobs_rate_limited_total" : "gittensory_jobs_deferred_total"); logAudit({ event: "job_rate_limited", ts: Date.now(), @@ -189,18 +304,18 @@ export function createPgQueue( payload_type: extractPayloadType(job.payload), latency_ms: Date.now() - claimedAt, attempts, - retry_after_ms: nonConsumingDelayMs, + retry_after_ms: Math.max(0, retryAfter - Date.now()), error: errMsg, }); return true; } - incr("gittensory_jobs_failed_total"); + await recordQueueMetric("gittensory_jobs_failed_total"); if (attempts >= maxRetries) { await pool.query( `UPDATE ${TABLE} SET status='dead', attempts=$1, last_error=$2 WHERE id=$3`, [attempts, errMsg, job.id], ); - incr("gittensory_jobs_dead_total"); + await recordQueueMetric("gittensory_jobs_dead_total"); console.error( JSON.stringify({ level: "error", @@ -313,5 +428,48 @@ export function createPgQueue( ).rows[0].c, ); }, + async stats() { + return readQueueStats(); + }, }; + + async function mergeRescheduledJobIntoPending( + job: JobRow, + runAfter: number, + errMsg: string, + ): Promise { + if (!job.job_key) return false; + const existing = ( + await pool.query( + `SELECT id FROM ${TABLE} WHERE status='pending' AND job_key=$1 AND id<>$2 ORDER BY priority DESC, run_after DESC, id LIMIT 1`, + [job.job_key, job.id], + ) + ).rows[0] as { id: string } | undefined; + if (!existing) return false; + await pool.query( + `UPDATE ${TABLE} SET run_after=GREATEST(run_after, $1), last_error=$2 WHERE id=$3`, + [runAfter, errMsg, existing.id], + ); + await pool.query(`DELETE FROM ${TABLE} WHERE id=$1`, [job.id]); + return true; + } + + async function recordQueueMetric(name: string): Promise { + incr(name); + await pool.query( + `INSERT INTO ${STATS_TABLE} (name, value) VALUES ($1, 1) + ON CONFLICT(name) DO UPDATE SET value=${STATS_TABLE}.value+1`, + [name], + ); + } + + async function readQueueStats(): Promise> { + const res = await pool.query(`SELECT name, value FROM ${STATS_TABLE}`); + return Object.fromEntries( + (res.rows as Array<{ name: string; value: number | string }>).map((row) => [ + row.name, + Number(row.value ?? 0), + ]), + ); + } } diff --git a/src/selfhost/queue-common.ts b/src/selfhost/queue-common.ts index 59abbd331b..f73b03f393 100644 --- a/src/selfhost/queue-common.ts +++ b/src/selfhost/queue-common.ts @@ -1,6 +1,11 @@ import { retryableJobDelayMs } from "../queue/retryable"; import { extractPayloadType } from "./audit"; +const DEFAULT_RATE_LIMIT_JITTER_MS = 5 * 60_000; +const DEFAULT_STARTUP_JITTER_MS = 3 * 60_000; +const DEFAULT_RECOVERY_JITTER_MS = 60_000; +const DEFAULT_STARTUP_JITTER_MIN_JOBS = 8; + // Webhook-driven work (a fresh PR -> its review) jumps ahead of heavy background jobs. Per-PR review refreshes // sit just below real webhooks, and sweep fan-out sits below those so stale surfaces are repaired during bursts. // Bot-generated comment edits are background noise; keeping them with real webhooks lets panel edits starve repair. @@ -89,11 +94,155 @@ export function nonConsumingRetryDelayMs(error: unknown): number | null { return githubRateLimitRetryDelayMs(error) ?? retryableJobDelayMs(error); } +export function rateLimitRetryDelayWithJitter( + delayMs: number, + seed: string, +): number { + return delayMs + deterministicJitterMs(seed, queueRateLimitJitterMs()); +} + +export function queueStartupJitterMs(): number { + return envDurationMs("QUEUE_STARTUP_JITTER_MS", DEFAULT_STARTUP_JITTER_MS); +} + +export function queueRecoveryJitterMs(): number { + return envDurationMs("QUEUE_RECOVERY_JITTER_MS", DEFAULT_RECOVERY_JITTER_MS); +} + +export function queueStartupJitterMinJobs(): number { + const raw = Number(process.env.QUEUE_STARTUP_JITTER_MIN_JOBS ?? DEFAULT_STARTUP_JITTER_MIN_JOBS); + return Number.isFinite(raw) && raw >= 0 ? Math.floor(raw) : DEFAULT_STARTUP_JITTER_MIN_JOBS; +} + +export function deterministicJitterMs(seed: string, maxJitterMs: number): number { + if (!Number.isFinite(maxJitterMs) || maxJitterMs <= 0) return 0; + let h = 2166136261; + for (let i = 0; i < seed.length; i += 1) { + h ^= seed.charCodeAt(i); + h = Math.imul(h, 16777619); + } + return Math.abs(h >>> 0) % (Math.floor(maxJitterMs) + 1); +} + +export function jobCoalesceKey(payload: string): string | null { + try { + const message = JSON.parse(payload) as { + type?: unknown; + eventName?: unknown; + repoFullName?: unknown; + prNumber?: unknown; + attempt?: unknown; + payload?: { + action?: unknown; + repository?: { full_name?: unknown } | null; + pull_request?: { number?: unknown; head?: { sha?: unknown } | null } | null; + check_suite?: { + head_sha?: unknown; + pull_requests?: Array<{ number?: unknown } | null> | null; + } | null; + check_run?: { + head_sha?: unknown; + check_suite?: { head_sha?: unknown } | null; + pull_requests?: Array<{ number?: unknown } | null> | null; + } | null; + } | null; + }; + const type = typeof message.type === "string" ? message.type : ""; + if (type === "agent-regate-pr") { + const repo = normalizedRepo(message.repoFullName); + const pr = normalizedNumber(message.prNumber); + return repo && pr !== null ? `agent-regate-pr:${repo}#${pr}` : null; + } + if (type === "recapture-preview") { + const repo = normalizedRepo(message.repoFullName); + const pr = normalizedNumber(message.prNumber); + const attempt = normalizedNumber(message.attempt); + return repo && pr !== null && attempt !== null + ? `recapture-preview:${repo}#${pr}:${attempt}` + : null; + } + if (type !== "github-webhook") return null; + const eventName = + typeof message.eventName === "string" ? message.eventName : ""; + const action = + typeof message.payload?.action === "string" ? message.payload.action : ""; + const repo = normalizedRepo(message.payload?.repository?.full_name); + if (!repo) return null; + if ( + (eventName === "check_suite" || eventName === "check_run") && + action === "completed" + ) { + const node = eventName === "check_suite" ? message.payload?.check_suite : message.payload?.check_run; + const headSha = normalizedSha( + node?.head_sha ?? + (eventName === "check_run" ? message.payload?.check_run?.check_suite?.head_sha : undefined), + ); + if (!headSha) return null; + const pullNumbers = (node?.pull_requests ?? []) + .map((entry) => normalizedNumber(entry?.number)) + .filter((value): value is number => value !== null) + .sort((a, b) => a - b) + .join(","); + return `github-webhook:ci-completed:${repo}@${headSha}${pullNumbers ? `#${pullNumbers}` : ""}`; + } + if (eventName === "pull_request" && isCoalescablePullRequestAction(action)) { + const pr = + normalizedNumber(message.payload?.pull_request?.number) ?? + normalizedNumber((message.payload as { number?: unknown } | null | undefined)?.number); + const headSha = normalizedSha(message.payload?.pull_request?.head?.sha); + return pr !== null + ? `github-webhook:pr-refresh:${repo}#${pr}${headSha ? `@${headSha}` : ""}` + : null; + } + return null; + } catch { + return null; + } +} + function clampRetryDelay(delayMs: number): number { if (!Number.isFinite(delayMs) || delayMs <= 0) return DEFAULT_GITHUB_RATE_LIMIT_RETRY_MS; return Math.min(Math.ceil(delayMs), MAX_GITHUB_RATE_LIMIT_RETRY_MS); } +function queueRateLimitJitterMs(): number { + return envDurationMs("QUEUE_RATE_LIMIT_JITTER_MS", DEFAULT_RATE_LIMIT_JITTER_MS); +} + +function envDurationMs(name: string, fallback: number): number { + const raw = Number(process.env[name] ?? fallback); + return Number.isFinite(raw) && raw >= 0 ? Math.floor(raw) : fallback; +} + +function normalizedRepo(value: unknown): string | null { + return typeof value === "string" && value.includes("/") + ? value.trim().toLowerCase() + : null; +} + +function normalizedNumber(value: unknown): number | null { + return typeof value === "number" && Number.isFinite(value) + ? Math.floor(value) + : null; +} + +function normalizedSha(value: unknown): string | null { + return typeof value === "string" && /^[a-f0-9]{7,40}$/i.test(value.trim()) + ? value.trim().toLowerCase() + : null; +} + +function isCoalescablePullRequestAction(action: string): boolean { + return ( + action === "opened" || + action === "synchronize" || + action === "edited" || + action === "ready_for_review" || + action === "labeled" || + action === "unlabeled" + ); +} + function numberHeader( headers: Headers | Record | null, key: string, diff --git a/src/selfhost/sqlite-queue.ts b/src/selfhost/sqlite-queue.ts index 6df0ac6a43..9191b7fc7c 100644 --- a/src/selfhost/sqlite-queue.ts +++ b/src/selfhost/sqlite-queue.ts @@ -7,10 +7,21 @@ import type { SqliteDriver } from "./d1-adapter"; import { logAudit, extractPayloadType } from "./audit"; import { incr } from "./metrics"; import { captureError } from "./sentry"; -import { githubRateLimitRetryDelayMs, jobPriority, nonConsumingRetryDelayMs } from "./queue-common"; +import { + deterministicJitterMs, + githubRateLimitRetryDelayMs, + jobCoalesceKey, + jobPriority, + nonConsumingRetryDelayMs, + queueRecoveryJitterMs, + queueStartupJitterMinJobs, + queueStartupJitterMs, + rateLimitRetryDelayWithJitter, +} from "./queue-common"; import type { JobMessage } from "../types"; const TABLE = "_selfhost_jobs"; +const STATS_TABLE = "_selfhost_job_stats"; const DDL = ` CREATE TABLE IF NOT EXISTS ${TABLE} ( id INTEGER PRIMARY KEY AUTOINCREMENT, @@ -20,11 +31,19 @@ CREATE TABLE IF NOT EXISTS ${TABLE} ( run_after INTEGER NOT NULL DEFAULT 0, created_at INTEGER NOT NULL, last_error TEXT, - priority INTEGER NOT NULL DEFAULT 0 + priority INTEGER NOT NULL DEFAULT 0, + job_key TEXT +);`; +const STATS_DDL = ` +CREATE TABLE IF NOT EXISTS ${STATS_TABLE} ( + name TEXT PRIMARY KEY, + value INTEGER NOT NULL DEFAULT 0 );`; const CLAIM_INDEX_DDL = ` DROP INDEX IF EXISTS ${TABLE}_claim; CREATE INDEX ${TABLE}_claim ON ${TABLE}(status, run_after, priority);`; +const JOB_KEY_INDEX_DDL = ` +CREATE INDEX IF NOT EXISTS ${TABLE}_pending_job_key ON ${TABLE}(job_key, status);`; export interface DurableQueue { binding: Queue; @@ -33,12 +52,14 @@ export interface DurableQueue { drain(): Promise; size(): number; deadCount(): number; + stats(): Record; } interface JobRow { id: number; payload: string; attempts: number; + job_key?: string | null; } export interface SqliteQueueOptions { @@ -66,6 +87,7 @@ export function createSqliteQueue( Math.max(1, Number(process.env.QUEUE_CONCURRENCY ?? "4")); driver.exec(DDL); + driver.exec(STATS_DDL); // Idempotent add for queues created before the priority column existed (#review-latency): the CREATE is skipped // for a pre-existing table, so ALTER must run before any index references the new column. try { @@ -75,7 +97,13 @@ export function createSqliteQueue( } catch { /* column already present */ } + try { + driver.exec(`ALTER TABLE ${TABLE} ADD COLUMN job_key TEXT`); + } catch { + /* column already present */ + } driver.exec(CLAIM_INDEX_DDL); + driver.exec(JOB_KEY_INDEX_DDL); const priorityBackfilled = backfillJobPriorities(driver); if (priorityBackfilled) console.log( @@ -84,15 +112,29 @@ export function createSqliteQueue( count: priorityBackfilled, }), ); + const keyBackfilled = backfillJobKeys(driver); + if (keyBackfilled) + console.log( + JSON.stringify({ + event: "selfhost_queue_job_keys_backfilled", + count: keyBackfilled, + }), + ); // Recover jobs a crashed previous run left mid-flight → make them claimable again. - const recovered = driver.query( - `UPDATE ${TABLE} SET status='pending' WHERE status='processing'`, - [], - ).changes; + const recovered = recoverProcessingJobs(driver); if (recovered) console.log( JSON.stringify({ event: "selfhost_queue_recovered", count: recovered }), ); + const spread = spreadDueJobsOnStartup(driver); + if (spread) + console.log( + JSON.stringify({ + event: "selfhost_queue_startup_spread", + count: spread, + jitter_ms: queueStartupJitterMs(), + }), + ); let running = false; let active = 0; // number of concurrent pump() loops currently draining jobs @@ -101,17 +143,37 @@ export function createSqliteQueue( function enqueue(message: JobMessage, delaySeconds: number): void { const now = Date.now(); const payload = JSON.stringify(message); + const runAfter = now + delaySeconds * 1000; + const priority = jobPriority(payload); + const key = jobCoalesceKey(payload); + if (key) { + const existing = driver.query( + `SELECT id FROM ${TABLE} WHERE status='pending' AND job_key=? ORDER BY priority DESC, run_after DESC, id LIMIT 1`, + [key], + ).rows[0] as { id: number } | undefined; + if (existing) { + driver.query( + `UPDATE ${TABLE} + SET payload=?, run_after=max(run_after, ?), created_at=?, priority=max(priority, ?), last_error=NULL + WHERE id=?`, + [payload, runAfter, now, priority, existing.id], + ); + recordQueueMetric(driver, "gittensory_jobs_coalesced_total"); + void pump(); + return; + } + } driver.query( - `INSERT INTO ${TABLE} (payload, status, attempts, run_after, created_at, priority) VALUES (?, 'pending', 0, ?, ?, ?)`, - [payload, now + delaySeconds * 1000, now, jobPriority(payload)], + `INSERT INTO ${TABLE} (payload, status, attempts, run_after, created_at, priority, job_key) VALUES (?, 'pending', 0, ?, ?, ?, ?)`, + [payload, runAfter, now, priority, key], ); - incr("gittensory_jobs_enqueued_total"); + recordQueueMetric(driver, "gittensory_jobs_enqueued_total"); void pump(); } function claimNext(): JobRow | null { const { rows } = driver.query( - `SELECT id, payload, attempts FROM ${TABLE} WHERE status='pending' AND run_after<=? ORDER BY priority DESC, id LIMIT 1`, + `SELECT id, payload, attempts, job_key FROM ${TABLE} WHERE status='pending' AND run_after<=? ORDER BY priority DESC, run_after, id LIMIT 1`, [Date.now()], ); const row = rows[0] as JobRow | undefined; @@ -136,7 +198,7 @@ export function createSqliteQueue( `UPDATE ${TABLE} SET status='dead', last_error='unparseable payload' WHERE id=?`, [job.id], ); - incr("gittensory_jobs_dead_total"); + recordQueueMetric(driver, "gittensory_jobs_dead_total"); logAudit({ event: "job_dead", ts: Date.now(), @@ -155,7 +217,7 @@ export function createSqliteQueue( try { await consume(message); driver.query(`DELETE FROM ${TABLE} WHERE id=?`, [job.id]); - incr("gittensory_jobs_processed_total"); + recordQueueMetric(driver, "gittensory_jobs_processed_total"); logAudit({ event: "job_complete", ts: Date.now(), @@ -169,11 +231,17 @@ export function createSqliteQueue( const errMsg = error instanceof Error ? error.message : "unknown error"; const nonConsumingDelayMs = nonConsumingRetryDelayMs(error); if (nonConsumingDelayMs !== null) { - driver.query( - `UPDATE ${TABLE} SET status='pending', run_after=?, last_error=? WHERE id=?`, - [Date.now() + nonConsumingDelayMs, errMsg, job.id], - ); - incr(githubRateLimitRetryDelayMs(error) !== null ? "gittensory_jobs_rate_limited_total" : "gittensory_jobs_deferred_total"); + const rateLimited = githubRateLimitRetryDelayMs(error) !== null; + const retryAfter = Date.now() + (rateLimited ? rateLimitRetryDelayWithJitter(nonConsumingDelayMs, `${job.job_key ?? ""}:${job.id}:${job.payload}`) : nonConsumingDelayMs); + if (job.job_key && mergeRescheduledJobIntoPending(driver, job, retryAfter, errMsg)) { + recordQueueMetric(driver, "gittensory_jobs_coalesced_total"); + } else { + driver.query( + `UPDATE ${TABLE} SET status='pending', run_after=?, last_error=? WHERE id=?`, + [retryAfter, errMsg, job.id], + ); + } + recordQueueMetric(driver, rateLimited ? "gittensory_jobs_rate_limited_total" : "gittensory_jobs_deferred_total"); logAudit({ event: "job_rate_limited", ts: Date.now(), @@ -181,18 +249,18 @@ export function createSqliteQueue( payload_type: extractPayloadType(job.payload), latency_ms: Date.now() - claimedAt, attempts, - retry_after_ms: nonConsumingDelayMs, + retry_after_ms: Math.max(0, retryAfter - Date.now()), error: errMsg, }); return true; } - incr("gittensory_jobs_failed_total"); + recordQueueMetric(driver, "gittensory_jobs_failed_total"); if (attempts >= maxRetries) { driver.query( `UPDATE ${TABLE} SET status='dead', attempts=?, last_error=? WHERE id=?`, [attempts, errMsg, job.id], ); - incr("gittensory_jobs_dead_total"); + recordQueueMetric(driver, "gittensory_jobs_dead_total"); console.error( JSON.stringify({ level: "error", @@ -310,6 +378,9 @@ export function createSqliteQueue( ).c, ); }, + stats() { + return readQueueStats(driver); + }, }; } @@ -330,3 +401,93 @@ function backfillJobPriorities(driver: SqliteDriver): number { } return changed; } + +function backfillJobKeys(driver: SqliteDriver): number { + const { rows } = driver.query( + `SELECT id, payload, job_key FROM ${TABLE} WHERE status IN ('pending', 'processing')`, + [], + ); + let changed = 0; + for (const row of rows as Array<{ id: number; payload: string; job_key?: string | null }>) { + const key = jobCoalesceKey(row.payload); + if ((row.job_key ?? null) === key) continue; + driver.query(`UPDATE ${TABLE} SET job_key=? WHERE id=?`, [key, row.id]); + changed += 1; + } + return changed; +} + +function recoverProcessingJobs(driver: SqliteDriver): number { + const { rows } = driver.query( + `SELECT id, payload, job_key FROM ${TABLE} WHERE status='processing'`, + [], + ); + let changed = 0; + const now = Date.now(); + const maxJitter = queueRecoveryJitterMs(); + for (const row of rows as Array<{ id: number; payload: string; job_key?: string | null }>) { + const runAfter = now + deterministicJitterMs(`${row.job_key ?? ""}:${row.id}:${row.payload}`, maxJitter); + driver.query( + `UPDATE ${TABLE} SET status='pending', run_after=? WHERE id=?`, + [runAfter, row.id], + ); + changed += 1; + } + return changed; +} + +function spreadDueJobsOnStartup(driver: SqliteDriver): number { + const now = Date.now(); + const { rows } = driver.query( + `SELECT id, payload, job_key FROM ${TABLE} WHERE status='pending' AND run_after<=?`, + [now], + ); + const due = rows as Array<{ id: number; payload: string; job_key?: string | null }>; + if (due.length < queueStartupJitterMinJobs()) return 0; + const maxJitter = queueStartupJitterMs(); + if (maxJitter <= 0) return 0; + for (const row of due) { + const runAfter = now + deterministicJitterMs(`${row.job_key ?? ""}:${row.id}:${row.payload}`, maxJitter); + driver.query(`UPDATE ${TABLE} SET run_after=? WHERE id=?`, [runAfter, row.id]); + } + return due.length; +} + +function mergeRescheduledJobIntoPending( + driver: SqliteDriver, + job: JobRow, + runAfter: number, + errMsg: string, +): boolean { + if (!job.job_key) return false; + const existing = driver.query( + `SELECT id FROM ${TABLE} WHERE status='pending' AND job_key=? AND id<>? ORDER BY priority DESC, run_after DESC, id LIMIT 1`, + [job.job_key, job.id], + ).rows[0] as { id: number } | undefined; + if (!existing) return false; + driver.query( + `UPDATE ${TABLE} SET run_after=max(run_after, ?), last_error=? WHERE id=?`, + [runAfter, errMsg, existing.id], + ); + driver.query(`DELETE FROM ${TABLE} WHERE id=?`, [job.id]); + return true; +} + +function recordQueueMetric(driver: SqliteDriver, name: string): void { + incr(name); + driver.query( + `INSERT INTO ${STATS_TABLE} (name, value) VALUES (?, 1) + ON CONFLICT(name) DO UPDATE SET value=value+1`, + [name], + ); +} + +function readQueueStats(driver: SqliteDriver): Record { + const { rows } = driver.query(`SELECT name, value FROM ${STATS_TABLE}`, []); + return Object.fromEntries( + (rows as Array<{ name: string; value: number }>).map((row) => [ + row.name, + Number(row.value ?? 0), + ]), + ); +} diff --git a/src/server.ts b/src/server.ts index 16cc4ac14c..e42e4ccd70 100644 --- a/src/server.ts +++ b/src/server.ts @@ -94,6 +94,7 @@ interface Backend { stop(): Promise; size(): number | Promise; deadCount(): number | Promise; + stats(): Record | Promise>; }; vectorize?: Vectorize; shutdown(): Promise; @@ -486,6 +487,21 @@ async function main(): Promise { gauge("gittensory_queue_pending", () => backend.queue.size()); gauge("gittensory_queue_dead", () => backend.queue.deadCount()); + const durableJobMetric = async (name: string): Promise => + Number((await backend.queue.stats())[name] ?? 0); + for (const name of [ + "gittensory_jobs_enqueued_total", + "gittensory_jobs_processed_total", + "gittensory_jobs_failed_total", + "gittensory_jobs_dead_total", + "gittensory_jobs_rate_limited_total", + "gittensory_jobs_deferred_total", + "gittensory_jobs_coalesced_total", + ]) { + gauge(name.replace("_total", "_persisted_total"), () => + durableJobMetric(name), + ); + } gauge("gittensory_uptime_seconds", () => Math.floor((Date.now() - startedAt) / 1000), ); diff --git a/test/unit/selfhost-pg-queue.test.ts b/test/unit/selfhost-pg-queue.test.ts index 20ae8b48fe..dd54ee39a9 100644 --- a/test/unit/selfhost-pg-queue.test.ts +++ b/test/unit/selfhost-pg-queue.test.ts @@ -10,9 +10,21 @@ const msg = (t: string): JobMessage => ({ type: t }) as unknown as JobMessage; const webhook = (sender: { login: string; type: string }, eventName = "issue_comment", action = "edited"): JobMessage => ({ type: "github-webhook", + deliveryId: "webhook-delivery", eventName, payload: { action, sender }, }) as unknown as JobMessage; +const ciWebhook = (deliveryId: string, eventName: "check_suite" | "check_run" = "check_suite", sha = "b".repeat(40)): JobMessage => + ({ + type: "github-webhook", + deliveryId, + eventName, + payload: { + action: "completed", + repository: { full_name: "JSONbored/gittensory" }, + [eventName]: { head_sha: sha, pull_requests: [{ number: 1629 }] }, + }, + }) as unknown as JobMessage; const typeOf = (m: JobMessage): string => (m as unknown as { type: string }).type; type MockFn = { mockResolvedValueOnce(v: unknown): void }; @@ -62,18 +74,20 @@ describe("createPgQueue (durable #977)", () => { m.fn.mockResolvedValueOnce({ rows: [], rowCount: 2 }); // recovery UPDATE const q = createPgQueue(m.pool, async () => undefined); await q.init(); - expect(m.pool.query).toHaveBeenCalledTimes(3); + expect(m.pool.query).toHaveBeenCalledWith(expect.stringContaining("CREATE TABLE IF NOT EXISTS _selfhost_jobs")); + expect(m.pool.query).toHaveBeenCalledWith(expect.stringContaining("status='processing'")); }); it("init() handles null rowCount from the recovery query (rowCount ?? 0 nullish arm)", async () => { const m = makePool(); m.fn.mockResolvedValueOnce({ rows: [], rowCount: 0 }); // DDL m.fn.mockResolvedValueOnce({ rows: [], rowCount: 0 }); // priority backfill SELECT - // pg driver can return null for rowCount on some UPDATE results + m.fn.mockResolvedValueOnce({ rows: [], rowCount: 0 }); // job-key backfill SELECT + // pg driver can return null for some SELECT-ish maintenance results; init must tolerate it. m.fn.mockResolvedValueOnce({ rows: [], rowCount: null }); const q = createPgQueue(m.pool, async () => undefined); await q.init(); // rowCount=null → ?? 0 → 0 → no recovery log emitted - expect(m.pool.query).toHaveBeenCalledTimes(3); + expect(m.pool.query).toHaveBeenCalled(); }); it("init() backfills event-aware priorities with the shared classifier", async () => { @@ -98,6 +112,28 @@ describe("createPgQueue (durable #977)", () => { expect(m.pool.query).toHaveBeenCalledWith(expect.stringContaining("UPDATE _selfhost_jobs SET priority=$1"), [8, "c"]); }); + it("coalesces duplicate keyed jobs instead of inserting queue pressure", async () => { + const m = makePool(); + const q = createPgQueue(m.pool, async () => undefined); + await q.init(); + m.fn.mockResolvedValueOnce({ rows: [{ id: "existing" }], rowCount: 1 }); + + await q.binding.send(ciWebhook("ci-2", "check_run"), { delaySeconds: 1 }); + + expect(m.pool.query).toHaveBeenCalledWith( + expect.stringContaining("WHERE status='pending' AND job_key=$1"), + [`github-webhook:ci-completed:jsonbored/gittensory@${"b".repeat(40)}#1629`], + ); + expect(m.pool.query).toHaveBeenCalledWith( + expect.stringContaining("SET payload=$1, run_after=GREATEST"), + expect.arrayContaining([expect.stringContaining('"deliveryId":"ci-2"'), expect.any(Number), expect.any(Number), 10, "existing"]), + ); + expect(m.pool.query).not.toHaveBeenCalledWith( + expect.stringContaining("INSERT INTO _selfhost_jobs (payload"), + expect.arrayContaining([expect.stringContaining('"deliveryId":"ci-2"')]), + ); + }); + it("processes a job successfully (job_complete audit emitted)", async () => { const m = makePool(); m.enqueueJob("1", { type: "review" }); @@ -297,4 +333,12 @@ describe("createPgQueue (durable #977)", () => { expect(await q.size()).toBe(3); expect(await q.deadCount()).toBe(3); }); + + it("stats() returns persisted queue metric counts", async () => { + const m = makePool(); + const q = createPgQueue(m.pool, async () => undefined); + await q.init(); + m.fn.mockResolvedValueOnce({ rows: [{ name: "gittensory_jobs_processed_total", value: "42" }], rowCount: 1 }); + await expect(q.stats()).resolves.toEqual({ gittensory_jobs_processed_total: 42 }); + }); }); diff --git a/test/unit/selfhost-sqlite-queue.test.ts b/test/unit/selfhost-sqlite-queue.test.ts index 5c6e9ce23d..567096efd6 100644 --- a/test/unit/selfhost-sqlite-queue.test.ts +++ b/test/unit/selfhost-sqlite-queue.test.ts @@ -12,9 +12,32 @@ const msg = (t: string): JobMessage => ({ type: t }) as unknown as JobMessage; const webhook = (sender: { login: string; type: string }, eventName = "issue_comment", action = "edited"): JobMessage => ({ type: "github-webhook", + deliveryId: "webhook-delivery", eventName, payload: { action, sender }, }) as unknown as JobMessage; +const prWebhook = (deliveryId: string, action = "synchronize", sha = "a".repeat(40)): JobMessage => + ({ + type: "github-webhook", + deliveryId, + eventName: "pull_request", + payload: { + action, + repository: { full_name: "JSONbored/gittensory" }, + pull_request: { number: 1629, head: { sha } }, + }, + }) as unknown as JobMessage; +const ciWebhook = (deliveryId: string, eventName: "check_suite" | "check_run" = "check_suite", sha = "b".repeat(40)): JobMessage => + ({ + type: "github-webhook", + deliveryId, + eventName, + payload: { + action: "completed", + repository: { full_name: "JSONbored/gittensory" }, + [eventName]: { head_sha: sha, pull_requests: [{ number: 1629 }] }, + }, + }) as unknown as JobMessage; const typeOf = (m: JobMessage): string => (m as unknown as { type: string }).type; describe("createSqliteQueue (durable #980)", () => { @@ -103,6 +126,98 @@ describe("createSqliteQueue (durable #980)", () => { ]); }); + it("backfills semantic job keys for already-pending duplicate-prone work", async () => { + const driver = makeDriver(); + driver.exec(` + CREATE TABLE _selfhost_jobs ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + payload TEXT NOT NULL, + status TEXT NOT NULL DEFAULT 'pending', + attempts INTEGER NOT NULL DEFAULT 0, + run_after INTEGER NOT NULL DEFAULT 0, + created_at INTEGER NOT NULL, + last_error TEXT, + priority INTEGER NOT NULL DEFAULT 0, + job_key TEXT + ); + `); + driver.query( + "INSERT INTO _selfhost_jobs (payload, status, attempts, run_after, created_at, priority) VALUES (?, 'pending', 0, ?, 0, 10)", + [JSON.stringify(ciWebhook("ci-1")), Date.now() + 60_000], + ); + + createSqliteQueue(driver, async () => undefined); + + const row = driver.query("SELECT job_key FROM _selfhost_jobs", []).rows[0] as { job_key: string }; + expect(row.job_key).toBe(`github-webhook:ci-completed:jsonbored/gittensory@${"b".repeat(40)}#1629`); + }); + + it("coalesces duplicate CI and PR-refresh jobs before they inflate queue pressure", async () => { + const driver = makeDriver(); + const q = createSqliteQueue(driver, async () => undefined); + await q.binding.send(ciWebhook("ci-1", "check_suite"), { delaySeconds: 60 }); + await q.binding.send(ciWebhook("ci-2", "check_run"), { delaySeconds: 1 }); + await q.binding.send(prWebhook("pr-1"), { delaySeconds: 60 }); + await q.binding.send(prWebhook("pr-2"), { delaySeconds: 1 }); + + const rows = driver.query( + "SELECT payload, job_key FROM _selfhost_jobs ORDER BY id", + [], + ).rows as Array<{ payload: string; job_key: string }>; + expect(rows).toHaveLength(2); + expect(rows.map((row) => row.job_key).sort()).toEqual([ + `github-webhook:ci-completed:jsonbored/gittensory@${"b".repeat(40)}#1629`, + `github-webhook:pr-refresh:jsonbored/gittensory#1629@${"a".repeat(40)}`, + ]); + expect(rows.map((row) => JSON.parse(row.payload).deliveryId).sort()).toEqual(["ci-2", "pr-2"]); + expect(q.stats()).toMatchObject({ + gittensory_jobs_enqueued_total: 2, + gittensory_jobs_coalesced_total: 2, + }); + }); + + it("does not coalesce terminal pull_request events that carry distinct lifecycle side effects", async () => { + const driver = makeDriver(); + const q = createSqliteQueue(driver, async () => undefined); + await q.binding.send(prWebhook("closed-1", "closed"), { delaySeconds: 60 }); + await q.binding.send(prWebhook("closed-2", "closed"), { delaySeconds: 60 }); + expect(driver.query("SELECT COUNT(*) AS c FROM _selfhost_jobs", []).rows[0]).toMatchObject({ c: 2 }); + }); + + it("spreads a due backlog on startup so restarts do not stampede GitHub", async () => { + const oldMin = process.env.QUEUE_STARTUP_JITTER_MIN_JOBS; + const oldJitter = process.env.QUEUE_STARTUP_JITTER_MS; + process.env.QUEUE_STARTUP_JITTER_MIN_JOBS = "2"; + process.env.QUEUE_STARTUP_JITTER_MS = "60000"; + try { + const driver = makeDriver(); + createSqliteQueue(driver, async () => undefined); + driver.query( + "INSERT INTO _selfhost_jobs (payload, status, attempts, run_after, created_at, priority, job_key) VALUES (?, 'pending', 0, 0, 0, 10, ?)", + [JSON.stringify(ciWebhook("ci-1")), "k1"], + ); + driver.query( + "INSERT INTO _selfhost_jobs (payload, status, attempts, run_after, created_at, priority, job_key) VALUES (?, 'pending', 0, 0, 0, 10, ?)", + [JSON.stringify(ciWebhook("ci-2", "check_run")), "k2"], + ); + + const before = Date.now(); + createSqliteQueue(driver, async () => undefined); + + const rows = driver.query( + "SELECT run_after FROM _selfhost_jobs ORDER BY id", + [], + ).rows as Array<{ run_after: number }>; + expect(rows.every((row) => row.run_after >= before)).toBe(true); + expect(rows.some((row) => row.run_after > before)).toBe(true); + } finally { + if (oldMin === undefined) delete process.env.QUEUE_STARTUP_JITTER_MIN_JOBS; + else process.env.QUEUE_STARTUP_JITTER_MIN_JOBS = oldMin; + if (oldJitter === undefined) delete process.env.QUEUE_STARTUP_JITTER_MS; + else process.env.QUEUE_STARTUP_JITTER_MS = oldJitter; + } + }); + it("migrates an old queue table without a priority column before creating the claim index", async () => { const driver = makeDriver(); driver.exec(` @@ -293,13 +408,20 @@ describe("createSqliteQueue (durable #980)", () => { }); it("recovers a job left 'processing' by a crash", async () => { + const oldRecoveryJitter = process.env.QUEUE_RECOVERY_JITTER_MS; + process.env.QUEUE_RECOVERY_JITTER_MS = "0"; const driver = makeDriver(); - createSqliteQueue(driver, async () => undefined); // creates the table - driver.query("INSERT INTO _selfhost_jobs (payload, status, attempts, run_after, created_at) VALUES (?, 'processing', 0, 0, 0)", [JSON.stringify(msg("stuck"))]); - const seen: string[] = []; - const fresh = createSqliteQueue(driver, async (m) => void seen.push(typeOf(m))); - await fresh.drain(); - expect(seen).toEqual(["stuck"]); + try { + createSqliteQueue(driver, async () => undefined); // creates the table + driver.query("INSERT INTO _selfhost_jobs (payload, status, attempts, run_after, created_at) VALUES (?, 'processing', 0, 0, 0)", [JSON.stringify(msg("stuck"))]); + const seen: string[] = []; + const fresh = createSqliteQueue(driver, async (m) => void seen.push(typeOf(m))); + await fresh.drain(); + expect(seen).toEqual(["stuck"]); + } finally { + if (oldRecoveryJitter === undefined) delete process.env.QUEUE_RECOVERY_JITTER_MS; + else process.env.QUEUE_RECOVERY_JITTER_MS = oldRecoveryJitter; + } }); it("records 'unknown error' when a consumer throws a non-Error", async () => { From 2691b47cc86a46e892f39601b890f51bc783972a Mon Sep 17 00:00:00 2001 From: JSONbored <49853598+JSONbored@users.noreply.github.com> Date: Sun, 28 Jun 2026 18:20:44 -0700 Subject: [PATCH 30/68] fix(review): keep readiness verdicts advisory --- src/review/unified-comment.ts | 18 ++++++++---------- test/unit/unified-comment.test.ts | 14 +++++++------- 2 files changed, 15 insertions(+), 17 deletions(-) diff --git a/src/review/unified-comment.ts b/src/review/unified-comment.ts index 9abc3372e1..8273de396d 100644 --- a/src/review/unified-comment.ts +++ b/src/review/unified-comment.ts @@ -250,21 +250,19 @@ export function deriveUnifiedStatus(input: UnifiedReviewInput, ctx: UnifiedComme else if ((input.failedCount ?? 0) > 0 || recs.some((r) => r !== "merge")) status = "held"; else status = "ready"; } - // CI gate — a PR is "safe to merge" ONLY when CI is GREEN. Apply this only to otherwise-ready statuses so - // pending/unverified CI cannot mask an authoritative close/block decision from the gate. + // Readiness is advisory for the Gittensory verdict. A PR is not "safe to merge" until CI is green, but + // CI/merge-state evidence must not create a red/blocked Gittensory decision by itself; the blocker has to + // come from the review disposition (`close`) or consensus review findings. if (status === "ready" && input.readiness && input.readiness.ciState !== "passed") { - return input.readiness.ciState === "failed" ? "blocked" : "held"; + return "held"; } - // Merge-state gate — "safe to merge" also requires the PR to be MERGEABLE. A `dirty` (base conflict) PR - // cannot merge as-is → BLOCKED; a `behind` PR needs a rebase first → HELD. This stops the green "safe to - // merge" / "Approved" headline from contradicting a `dirty`/`behind` merge-state chip (the #4220 report, - // where the headline said "safe to merge" while the chip read `dirty`). Other states — clean, a not-yet- - // computed `unknown`, or a `blocked` that the bot's own pending approval will clear — do not downgrade. + // Merge-state readiness follows the same rule: do not claim "safe to merge" while GitHub says the branch is + // dirty/behind, but keep the comment in a held/advisory tone instead of turning readiness into a blocker. + // Other states — clean, a not-yet-computed `unknown`, or a `blocked` that the bot's own pending approval will clear — do not downgrade. // (#ready-needs-mergeable) if (status === "ready" && input.readiness?.mergeStateLabel) { const mergeState = input.readiness.mergeStateLabel.toLowerCase(); - if (mergeState === "dirty") return "blocked"; - if (mergeState === "behind") return "held"; + if (mergeState === "dirty" || mergeState === "behind") return "held"; } // Guarded-hold gate — a clean + green PR whose diff touches a hard-guardrail path (CI config, the review // engine, visuals) is HELD for owner review by the disposition, never auto-merged. The comment must then say diff --git a/test/unit/unified-comment.test.ts b/test/unit/unified-comment.test.ts index 34e5f7a8ba..4915951c4c 100644 --- a/test/unit/unified-comment.test.ts +++ b/test/unit/unified-comment.test.ts @@ -38,10 +38,10 @@ describe("deriveUnifiedStatus", () => { expect(deriveUnifiedStatus({ ...base, recommendations: ["request_changes"] })).toBe("held"); }); - it("CI that hasn't passed is NEVER safe-to-merge — failed→blocked, pending/unverified→held, even over a merge verdict", () => { - // A red CI must never render "safe to merge". It downgrades even an explicit `merge` verdict to blocked. - expect(deriveUnifiedStatus({ ...base, readiness: { ciState: "failed" } })).toBe("blocked"); - expect(deriveUnifiedStatus({ ...base, decision: "merge", readiness: { ciState: "failed" } })).toBe("blocked"); + it("CI readiness is advisory for Gittensory — failed/pending holds, but never blocks a merge verdict", () => { + // Red CI must never render "safe to merge", but CI itself is not a Gittensory blocker. + expect(deriveUnifiedStatus({ ...base, readiness: { ciState: "failed" } })).toBe("held"); + expect(deriveUnifiedStatus({ ...base, decision: "merge", readiness: { ciState: "failed" } })).toBe("held"); // CI still running / not yet reported (chip "CI pending") → HELD, never "safe to merge". expect(deriveUnifiedStatus({ ...base, decision: "merge", readiness: { ciState: "unverified" } })).toBe("held"); // ONLY green CI + a merge verdict renders ready. @@ -54,10 +54,10 @@ describe("deriveUnifiedStatus", () => { expect(deriveUnifiedStatus({ ...base, recommendations: [], blockers: ["leaks a secret"] })).toBe("blocked"); }); - it("a non-mergeable merge state is NEVER safe-to-merge — dirty(conflict)→blocked, behind→held, even over a merge verdict (#4220)", () => { + it("a non-mergeable merge state is advisory — dirty/behind hold, but never block a merge verdict (#4220)", () => { // The reported bug: green CI + merge verdict but a `dirty` base conflict rendered "safe to merge". - expect(deriveUnifiedStatus({ ...base, decision: "merge", readiness: { ciState: "passed", mergeStateLabel: "dirty" } })).toBe("blocked"); - expect(deriveUnifiedStatus({ ...base, decision: "merge", readiness: { ciState: "passed", mergeStateLabel: "DIRTY" } })).toBe("blocked"); // case-insensitive + expect(deriveUnifiedStatus({ ...base, decision: "merge", readiness: { ciState: "passed", mergeStateLabel: "dirty" } })).toBe("held"); + expect(deriveUnifiedStatus({ ...base, decision: "merge", readiness: { ciState: "passed", mergeStateLabel: "DIRTY" } })).toBe("held"); // case-insensitive expect(deriveUnifiedStatus({ ...base, decision: "merge", readiness: { ciState: "passed", mergeStateLabel: "behind" } })).toBe("held"); // A clean (or not-yet-computed / pending-bot-approval) merge state still renders ready. expect(deriveUnifiedStatus({ ...base, decision: "merge", readiness: { ciState: "passed", mergeStateLabel: "clean" } })).toBe("ready"); From 3a473af120e8ddb116f41014c2617fcbd07c543b Mon Sep 17 00:00:00 2001 From: JSONbored <49853598+JSONbored@users.noreply.github.com> Date: Sun, 28 Jun 2026 18:39:21 -0700 Subject: [PATCH 31/68] fix(queue): apply shared github rate-limit cooldown --- src/selfhost/pg-queue.ts | 50 +++++++++++++++++++++--- src/selfhost/sqlite-queue.ts | 51 ++++++++++++++++++++++--- src/server.ts | 2 + test/unit/selfhost-pg-queue.test.ts | 27 +++++++++++++ test/unit/selfhost-sqlite-queue.test.ts | 47 +++++++++++++++++++++++ 5 files changed, 165 insertions(+), 12 deletions(-) diff --git a/src/selfhost/pg-queue.ts b/src/selfhost/pg-queue.ts index e6076df7b2..2f31533aa0 100644 --- a/src/selfhost/pg-queue.ts +++ b/src/selfhost/pg-queue.ts @@ -87,6 +87,7 @@ export function createPgQueue( let running = false; let active = 0; let timer: ReturnType | null = null; + let githubRateLimitCooldownUntil = 0; async function init(): Promise { await pool.query(DDL); @@ -231,6 +232,7 @@ export function createPgQueue( } async function claimNext(): Promise { + if (Date.now() < githubRateLimitCooldownUntil) return null; // Atomic, multi-instance-safe: lock + claim one due job, skipping rows another instance already locked. const res = await pool.query( `UPDATE ${TABLE} SET status='processing' @@ -287,7 +289,23 @@ export function createPgQueue( const nonConsumingDelayMs = nonConsumingRetryDelayMs(error); if (nonConsumingDelayMs !== null) { const rateLimited = githubRateLimitRetryDelayMs(error) !== null; - const retryAfter = Date.now() + (rateLimited ? rateLimitRetryDelayWithJitter(nonConsumingDelayMs, `${job.job_key ?? ""}:${job.id}:${job.payload}`) : nonConsumingDelayMs); + const now = Date.now(); + const retryAfter = now + (rateLimited ? rateLimitRetryDelayWithJitter(nonConsumingDelayMs, `${job.job_key ?? ""}:${job.id}:${job.payload}`) : nonConsumingDelayMs); + if (rateLimited) { + githubRateLimitCooldownUntil = Math.max(githubRateLimitCooldownUntil, now + nonConsumingDelayMs); + const deferred = await deferPendingJobsForRateLimit(nonConsumingDelayMs, now); + if (deferred) { + await recordQueueMetric("gittensory_jobs_rate_limit_deferred_total", deferred); + console.warn( + JSON.stringify({ + level: "warn", + event: "selfhost_queue_rate_limit_cooldown", + deferred, + cooldown_until: githubRateLimitCooldownUntil, + }), + ); + } + } if (job.job_key && (await mergeRescheduledJobIntoPending(job, retryAfter, errMsg))) { await recordQueueMetric("gittensory_jobs_coalesced_total"); } else { @@ -433,6 +451,26 @@ export function createPgQueue( }, }; + async function deferPendingJobsForRateLimit( + delayMs: number, + now: number, + ): Promise { + const res = await pool.query( + `SELECT id, payload, job_key FROM ${TABLE} WHERE status='pending' AND run_after<=$1`, + [now + delayMs], + ); + let changed = 0; + for (const row of res.rows as Array<{ id: string; payload: string; job_key?: string | null }>) { + const runAfter = now + rateLimitRetryDelayWithJitter(delayMs, `${row.job_key ?? ""}:${row.id}:${row.payload}`); + const update = await pool.query( + `UPDATE ${TABLE} SET run_after=GREATEST(run_after, $1), last_error=COALESCE(last_error, $2) WHERE id=$3 AND status='pending'`, + [runAfter, "github rate-limit cooldown", row.id], + ); + changed += update.rowCount ?? 0; + } + return changed; + } + async function mergeRescheduledJobIntoPending( job: JobRow, runAfter: number, @@ -454,12 +492,12 @@ export function createPgQueue( return true; } - async function recordQueueMetric(name: string): Promise { - incr(name); + async function recordQueueMetric(name: string, by = 1): Promise { + incr(name, undefined, by); await pool.query( - `INSERT INTO ${STATS_TABLE} (name, value) VALUES ($1, 1) - ON CONFLICT(name) DO UPDATE SET value=${STATS_TABLE}.value+1`, - [name], + `INSERT INTO ${STATS_TABLE} (name, value) VALUES ($1, $2) + ON CONFLICT(name) DO UPDATE SET value=${STATS_TABLE}.value+$2`, + [name, by], ); } diff --git a/src/selfhost/sqlite-queue.ts b/src/selfhost/sqlite-queue.ts index 9191b7fc7c..87a6391470 100644 --- a/src/selfhost/sqlite-queue.ts +++ b/src/selfhost/sqlite-queue.ts @@ -139,6 +139,7 @@ export function createSqliteQueue( let running = false; let active = 0; // number of concurrent pump() loops currently draining jobs let timer: ReturnType | null = null; + let githubRateLimitCooldownUntil = 0; function enqueue(message: JobMessage, delaySeconds: number): void { const now = Date.now(); @@ -172,6 +173,7 @@ export function createSqliteQueue( } function claimNext(): JobRow | null { + if (Date.now() < githubRateLimitCooldownUntil) return null; const { rows } = driver.query( `SELECT id, payload, attempts, job_key FROM ${TABLE} WHERE status='pending' AND run_after<=? ORDER BY priority DESC, run_after, id LIMIT 1`, [Date.now()], @@ -232,7 +234,23 @@ export function createSqliteQueue( const nonConsumingDelayMs = nonConsumingRetryDelayMs(error); if (nonConsumingDelayMs !== null) { const rateLimited = githubRateLimitRetryDelayMs(error) !== null; - const retryAfter = Date.now() + (rateLimited ? rateLimitRetryDelayWithJitter(nonConsumingDelayMs, `${job.job_key ?? ""}:${job.id}:${job.payload}`) : nonConsumingDelayMs); + const now = Date.now(); + const retryAfter = now + (rateLimited ? rateLimitRetryDelayWithJitter(nonConsumingDelayMs, `${job.job_key ?? ""}:${job.id}:${job.payload}`) : nonConsumingDelayMs); + if (rateLimited) { + githubRateLimitCooldownUntil = Math.max(githubRateLimitCooldownUntil, now + nonConsumingDelayMs); + const deferred = deferPendingJobsForRateLimit(driver, nonConsumingDelayMs, now); + if (deferred) { + recordQueueMetric(driver, "gittensory_jobs_rate_limit_deferred_total", deferred); + console.warn( + JSON.stringify({ + level: "warn", + event: "selfhost_queue_rate_limit_cooldown", + deferred, + cooldown_until: githubRateLimitCooldownUntil, + }), + ); + } + } if (job.job_key && mergeRescheduledJobIntoPending(driver, job, retryAfter, errMsg)) { recordQueueMetric(driver, "gittensory_jobs_coalesced_total"); } else { @@ -453,6 +471,27 @@ function spreadDueJobsOnStartup(driver: SqliteDriver): number { return due.length; } +function deferPendingJobsForRateLimit( + driver: SqliteDriver, + delayMs: number, + now: number, +): number { + const { rows } = driver.query( + `SELECT id, payload, job_key FROM ${TABLE} WHERE status='pending' AND run_after<=?`, + [now + delayMs], + ); + let changed = 0; + for (const row of rows as Array<{ id: number; payload: string; job_key?: string | null }>) { + const runAfter = now + rateLimitRetryDelayWithJitter(delayMs, `${row.job_key ?? ""}:${row.id}:${row.payload}`); + const { changes } = driver.query( + `UPDATE ${TABLE} SET run_after=max(run_after, ?), last_error=coalesce(last_error, ?) WHERE id=? AND status='pending'`, + [runAfter, "github rate-limit cooldown", row.id], + ); + changed += changes; + } + return changed; +} + function mergeRescheduledJobIntoPending( driver: SqliteDriver, job: JobRow, @@ -473,12 +512,12 @@ function mergeRescheduledJobIntoPending( return true; } -function recordQueueMetric(driver: SqliteDriver, name: string): void { - incr(name); +function recordQueueMetric(driver: SqliteDriver, name: string, by = 1): void { + incr(name, undefined, by); driver.query( - `INSERT INTO ${STATS_TABLE} (name, value) VALUES (?, 1) - ON CONFLICT(name) DO UPDATE SET value=value+1`, - [name], + `INSERT INTO ${STATS_TABLE} (name, value) VALUES (?, ?) + ON CONFLICT(name) DO UPDATE SET value=value+?`, + [name, by, by], ); } diff --git a/src/server.ts b/src/server.ts index e42e4ccd70..745a27403c 100644 --- a/src/server.ts +++ b/src/server.ts @@ -495,6 +495,7 @@ async function main(): Promise { "gittensory_jobs_failed_total", "gittensory_jobs_dead_total", "gittensory_jobs_rate_limited_total", + "gittensory_jobs_rate_limit_deferred_total", "gittensory_jobs_deferred_total", "gittensory_jobs_coalesced_total", ]) { @@ -512,6 +513,7 @@ async function main(): Promise { "gittensory_jobs_processed_total", "gittensory_jobs_failed_total", "gittensory_jobs_dead_total", + "gittensory_jobs_rate_limit_deferred_total", "gittensory_webhook_dedup_total", "gittensory_qdrant_queries_total", "gittensory_qdrant_upserts_total", diff --git a/test/unit/selfhost-pg-queue.test.ts b/test/unit/selfhost-pg-queue.test.ts index dd54ee39a9..7c9c11a5db 100644 --- a/test/unit/selfhost-pg-queue.test.ts +++ b/test/unit/selfhost-pg-queue.test.ts @@ -195,6 +195,33 @@ describe("createPgQueue (durable #977)", () => { ); }); + it("opens a shared cooldown after GitHub rate limits so the pump does not claim the next due job", async () => { + const m = makePool(); + m.enqueueJob("1", { type: "github-webhook" }, 0); + m.enqueueJob("2", { type: "agent-regate-pr" }, 0); + let calls = 0; + const rateLimit = new Error("API rate limit exceeded for installation ID 123"); + Object.assign(rateLimit, { + status: 403, + response: { headers: { "retry-after": "120" } }, + }); + const q = createPgQueue( + m.pool, + async () => { + calls += 1; + throw rateLimit; + }, + { maxRetries: 1, backoffMs: () => 0 }, + ); + await q.init(); + await q.drain(); + expect(calls).toBe(1); + expect(m.pool.query).toHaveBeenCalledWith( + expect.stringContaining("SELECT id, payload, job_key FROM _selfhost_jobs WHERE status='pending' AND run_after<=$1"), + expect.arrayContaining([expect.any(Number)]), + ); + }); + it("reschedules retryable incomplete review jobs without consuming the dead-letter budget", async () => { const m = makePool(); m.enqueueJob("1", { type: "agent-regate-pr" }, 4); diff --git a/test/unit/selfhost-sqlite-queue.test.ts b/test/unit/selfhost-sqlite-queue.test.ts index 567096efd6..89ab1f9db8 100644 --- a/test/unit/selfhost-sqlite-queue.test.ts +++ b/test/unit/selfhost-sqlite-queue.test.ts @@ -351,6 +351,53 @@ describe("createSqliteQueue (durable #980)", () => { expect(row.last_error).toContain("API rate limit exceeded"); }); + it("defers the due backlog and stops claiming when GitHub is rate-limited", async () => { + const driver = makeDriver(); + let calls = 0; + const rateLimit = new Error("API rate limit exceeded for installation ID 123"); + Object.assign(rateLimit, { + status: 403, + response: { headers: { "retry-after": "120" } }, + }); + const q = createSqliteQueue( + driver, + async () => { + calls += 1; + throw rateLimit; + }, + { maxRetries: 1, backoffMs: () => 0 }, + ); + const before = Date.now(); + driver.query( + "INSERT INTO _selfhost_jobs (payload, status, attempts, run_after, created_at) VALUES (?, 'pending', 0, 0, 0)", + [JSON.stringify(msg("github-webhook"))], + ); + driver.query( + "INSERT INTO _selfhost_jobs (payload, status, attempts, run_after, created_at) VALUES (?, 'pending', 0, 0, 0)", + [JSON.stringify(msg("agent-regate-pr"))], + ); + + await q.drain(); + + const rows = driver.query( + "SELECT status, attempts, run_after, last_error FROM _selfhost_jobs ORDER BY id", + [], + ).rows as Array<{ status: string; attempts: number; run_after: number; last_error: string }>; + expect(calls).toBe(1); + expect(rows).toHaveLength(2); + expect(rows.every((row) => row.status === "pending")).toBe(true); + expect(rows.every((row) => row.attempts === 0)).toBe(true); + expect(rows.every((row) => row.run_after > before)).toBe(true); + expect(rows.map((row) => row.last_error)).toEqual([ + "API rate limit exceeded for installation ID 123", + "github rate-limit cooldown", + ]); + expect(q.stats()).toMatchObject({ + gittensory_jobs_rate_limited_total: 1, + gittensory_jobs_rate_limit_deferred_total: 1, + }); + }); + it("reschedules retryable incomplete review jobs without consuming the dead-letter budget", async () => { const driver = makeDriver(); let calls = 0; From 0894f1748a04d11b99692919446a6517ed1d8723 Mon Sep 17 00:00:00 2001 From: JSONbored <49853598+JSONbored@users.noreply.github.com> Date: Sun, 28 Jun 2026 18:48:13 -0700 Subject: [PATCH 32/68] fix(queue): honor cooldown for new self-host jobs --- src/selfhost/pg-queue.ts | 9 ++++++++- src/selfhost/sqlite-queue.ts | 9 ++++++++- test/unit/selfhost-sqlite-queue.test.ts | 8 ++++++++ 3 files changed, 24 insertions(+), 2 deletions(-) diff --git a/src/selfhost/pg-queue.ts b/src/selfhost/pg-queue.ts index 2f31533aa0..c522094885 100644 --- a/src/selfhost/pg-queue.ts +++ b/src/selfhost/pg-queue.ts @@ -201,9 +201,9 @@ export function createPgQueue( ): Promise { const now = Date.now(); const payload = JSON.stringify(message); - const runAfter = now + delaySeconds * 1000; const priority = jobPriority(payload); const key = jobCoalesceKey(payload); + const runAfter = nextRunAfter(now, delaySeconds * 1000, `${key ?? ""}:${payload}`); if (key) { const existing = ( await pool.query( @@ -243,6 +243,13 @@ export function createPgQueue( return (res.rows[0] as JobRow | undefined) ?? null; } + function nextRunAfter(now: number, delayMs: number, seed: string): number { + const requested = now + delayMs; + if (now >= githubRateLimitCooldownUntil) return requested; + const cooldownDelay = githubRateLimitCooldownUntil - now; + return Math.max(requested, now + rateLimitRetryDelayWithJitter(cooldownDelay, seed)); + } + async function processOne(): Promise { const job = await claimNext(); if (!job) return false; diff --git a/src/selfhost/sqlite-queue.ts b/src/selfhost/sqlite-queue.ts index 87a6391470..4e2b860802 100644 --- a/src/selfhost/sqlite-queue.ts +++ b/src/selfhost/sqlite-queue.ts @@ -144,9 +144,9 @@ export function createSqliteQueue( function enqueue(message: JobMessage, delaySeconds: number): void { const now = Date.now(); const payload = JSON.stringify(message); - const runAfter = now + delaySeconds * 1000; const priority = jobPriority(payload); const key = jobCoalesceKey(payload); + const runAfter = nextRunAfter(now, delaySeconds * 1000, `${key ?? ""}:${payload}`); if (key) { const existing = driver.query( `SELECT id FROM ${TABLE} WHERE status='pending' AND job_key=? ORDER BY priority DESC, run_after DESC, id LIMIT 1`, @@ -188,6 +188,13 @@ export function createSqliteQueue( return changes ? row : null; } + function nextRunAfter(now: number, delayMs: number, seed: string): number { + const requested = now + delayMs; + if (now >= githubRateLimitCooldownUntil) return requested; + const cooldownDelay = githubRateLimitCooldownUntil - now; + return Math.max(requested, now + rateLimitRetryDelayWithJitter(cooldownDelay, seed)); + } + async function processOne(): Promise { const job = claimNext(); if (!job) return false; diff --git a/test/unit/selfhost-sqlite-queue.test.ts b/test/unit/selfhost-sqlite-queue.test.ts index 89ab1f9db8..f320e8953d 100644 --- a/test/unit/selfhost-sqlite-queue.test.ts +++ b/test/unit/selfhost-sqlite-queue.test.ts @@ -396,6 +396,14 @@ describe("createSqliteQueue (durable #980)", () => { gittensory_jobs_rate_limited_total: 1, gittensory_jobs_rate_limit_deferred_total: 1, }); + + await q.binding.send(msg("github-webhook")); + const afterEnqueue = driver.query( + "SELECT run_after FROM _selfhost_jobs ORDER BY id DESC LIMIT 1", + [], + ).rows[0] as { run_after: number }; + expect(calls).toBe(1); + expect(afterEnqueue.run_after).toBeGreaterThan(before + 100_000); }); it("reschedules retryable incomplete review jobs without consuming the dead-letter budget", async () => { From aaff59fa0af4241508969a138cdf7e5bd63c8abf Mon Sep 17 00:00:00 2001 From: JSONbored <49853598+JSONbored@users.noreply.github.com> Date: Sun, 28 Jun 2026 19:09:22 -0700 Subject: [PATCH 33/68] fix(queue): reclaim stale processing leases --- src/selfhost/pg-queue.ts | 271 +++++++++++++---------- src/selfhost/queue-common.ts | 8 + src/selfhost/sqlite-queue.ts | 279 ++++++++++++++---------- src/server.ts | 2 + test/unit/selfhost-pg-queue.test.ts | 33 +++ test/unit/selfhost-sqlite-queue.test.ts | 33 +++ 6 files changed, 404 insertions(+), 222 deletions(-) diff --git a/src/selfhost/pg-queue.ts b/src/selfhost/pg-queue.ts index c522094885..688c145e0e 100644 --- a/src/selfhost/pg-queue.ts +++ b/src/selfhost/pg-queue.ts @@ -12,6 +12,7 @@ import { jobCoalesceKey, jobPriority, nonConsumingRetryDelayMs, + queueProcessingTimeoutMs, queueRecoveryJitterMs, queueStartupJitterMinJobs, queueStartupJitterMs, @@ -83,9 +84,11 @@ export function createPgQueue( const concurrency = opts.concurrency ?? Math.max(1, Number(process.env.QUEUE_CONCURRENCY ?? "4")); + const processingTimeoutMs = queueProcessingTimeoutMs(); let running = false; let active = 0; + const activeJobIds = new Set(); let timer: ReturnType | null = null; let githubRateLimitCooldownUntil = 0; @@ -233,12 +236,13 @@ export function createPgQueue( async function claimNext(): Promise { if (Date.now() < githubRateLimitCooldownUntil) return null; + const now = Date.now(); // Atomic, multi-instance-safe: lock + claim one due job, skipping rows another instance already locked. const res = await pool.query( - `UPDATE ${TABLE} SET status='processing' - WHERE id = (SELECT id FROM ${TABLE} WHERE status='pending' AND run_after<=$1 ORDER BY priority DESC, id FOR UPDATE SKIP LOCKED LIMIT 1) + `UPDATE ${TABLE} SET status='processing', run_after=$1 + WHERE id = (SELECT id FROM ${TABLE} WHERE status='pending' AND run_after<=$1 ORDER BY priority DESC, run_after, id FOR UPDATE SKIP LOCKED LIMIT 1) RETURNING id, payload, attempts, job_key`, - [Date.now()], + [now], ); return (res.rows[0] as JobRow | undefined) ?? null; } @@ -251,138 +255,161 @@ export function createPgQueue( } async function processOne(): Promise { - const job = await claimNext(); - if (!job) return false; - const claimedAt = Date.now(); - let message: JobMessage; - try { - message = JSON.parse(job.payload) as JobMessage; - } catch { - await pool.query( - `UPDATE ${TABLE} SET status='dead', last_error='unparseable payload' WHERE id=$1`, - [job.id], + const recovered = await reclaimExpiredProcessingJobs(); + if (recovered) { + await recordQueueMetric("gittensory_jobs_recovered_total", recovered); + console.warn( + JSON.stringify({ + level: "warn", + event: "selfhost_queue_processing_reclaimed", + count: recovered, + timeout_ms: processingTimeoutMs, + }), ); - await recordQueueMetric("gittensory_jobs_dead_total"); - logAudit({ - event: "job_dead", - ts: Date.now(), - job_id: job.id, - latency_ms: Date.now() - claimedAt, - attempts: Number(job.attempts) + 1, - error: "unparseable payload", - }); - captureError(new Error("unparseable queue payload"), { - kind: "job_dead", - reason: "unparseable_payload", - jobId: job.id, + captureError(new Error("self-host queue processing lease expired"), { + kind: "job_recovered", + reason: "processing_timeout", + recovered, + timeoutMs: processingTimeoutMs, }); - return true; } + const job = await claimNext(); + if (!job) return false; + activeJobIds.add(job.id); + const claimedAt = Date.now(); try { - await consume(message); - await pool.query(`DELETE FROM ${TABLE} WHERE id=$1`, [job.id]); - await recordQueueMetric("gittensory_jobs_processed_total"); - logAudit({ - event: "job_complete", - ts: Date.now(), - job_id: job.id, - payload_type: extractPayloadType(job.payload), - latency_ms: Date.now() - claimedAt, - attempts: Number(job.attempts) + 1, - }); - } catch (error) { - const attempts = Number(job.attempts) + 1; - const errMsg = error instanceof Error ? error.message : "unknown error"; - const nonConsumingDelayMs = nonConsumingRetryDelayMs(error); - if (nonConsumingDelayMs !== null) { - const rateLimited = githubRateLimitRetryDelayMs(error) !== null; - const now = Date.now(); - const retryAfter = now + (rateLimited ? rateLimitRetryDelayWithJitter(nonConsumingDelayMs, `${job.job_key ?? ""}:${job.id}:${job.payload}`) : nonConsumingDelayMs); - if (rateLimited) { - githubRateLimitCooldownUntil = Math.max(githubRateLimitCooldownUntil, now + nonConsumingDelayMs); - const deferred = await deferPendingJobsForRateLimit(nonConsumingDelayMs, now); - if (deferred) { - await recordQueueMetric("gittensory_jobs_rate_limit_deferred_total", deferred); - console.warn( - JSON.stringify({ - level: "warn", - event: "selfhost_queue_rate_limit_cooldown", - deferred, - cooldown_until: githubRateLimitCooldownUntil, - }), - ); - } - } - if (job.job_key && (await mergeRescheduledJobIntoPending(job, retryAfter, errMsg))) { - await recordQueueMetric("gittensory_jobs_coalesced_total"); - } else { - await pool.query( - `UPDATE ${TABLE} SET status='pending', run_after=$1, last_error=$2 WHERE id=$3`, - [retryAfter, errMsg, job.id], - ); - } - await recordQueueMetric(rateLimited ? "gittensory_jobs_rate_limited_total" : "gittensory_jobs_deferred_total"); - logAudit({ - event: "job_rate_limited", - ts: Date.now(), - job_id: job.id, - payload_type: extractPayloadType(job.payload), - latency_ms: Date.now() - claimedAt, - attempts, - retry_after_ms: Math.max(0, retryAfter - Date.now()), - error: errMsg, - }); - return true; - } - await recordQueueMetric("gittensory_jobs_failed_total"); - if (attempts >= maxRetries) { + let message: JobMessage; + try { + message = JSON.parse(job.payload) as JobMessage; + } catch { await pool.query( - `UPDATE ${TABLE} SET status='dead', attempts=$1, last_error=$2 WHERE id=$3`, - [attempts, errMsg, job.id], + `UPDATE ${TABLE} SET status='dead', last_error='unparseable payload' WHERE id=$1`, + [job.id], ); await recordQueueMetric("gittensory_jobs_dead_total"); - console.error( - JSON.stringify({ - level: "error", - event: "selfhost_job_dead", - id: job.id, - attempts, - error: errMsg, - }), - ); logAudit({ event: "job_dead", ts: Date.now(), job_id: job.id, - payload_type: extractPayloadType(job.payload), latency_ms: Date.now() - claimedAt, - attempts, - error: errMsg, + attempts: Number(job.attempts) + 1, + error: "unparseable payload", }); - captureError(error, { + captureError(new Error("unparseable queue payload"), { kind: "job_dead", - reason: "max_retries_exhausted", - jobType: extractPayloadType(job.payload), + reason: "unparseable_payload", jobId: job.id, - attempts, }); - } else { - await pool.query( - `UPDATE ${TABLE} SET status='pending', attempts=$1, run_after=$2, last_error=$3 WHERE id=$4`, - [attempts, Date.now() + backoff(attempts), errMsg, job.id], - ); + return true; + } + try { + await consume(message); + await pool.query(`DELETE FROM ${TABLE} WHERE id=$1`, [job.id]); + await recordQueueMetric("gittensory_jobs_processed_total"); logAudit({ - event: "job_error", + event: "job_complete", ts: Date.now(), job_id: job.id, payload_type: extractPayloadType(job.payload), latency_ms: Date.now() - claimedAt, - attempts, - error: errMsg, + attempts: Number(job.attempts) + 1, }); + } catch (error) { + const attempts = Number(job.attempts) + 1; + const errMsg = error instanceof Error ? error.message : "unknown error"; + const nonConsumingDelayMs = nonConsumingRetryDelayMs(error); + if (nonConsumingDelayMs !== null) { + const rateLimited = githubRateLimitRetryDelayMs(error) !== null; + const now = Date.now(); + const retryAfter = now + (rateLimited ? rateLimitRetryDelayWithJitter(nonConsumingDelayMs, `${job.job_key ?? ""}:${job.id}:${job.payload}`) : nonConsumingDelayMs); + if (rateLimited) { + githubRateLimitCooldownUntil = Math.max(githubRateLimitCooldownUntil, now + nonConsumingDelayMs); + const deferred = await deferPendingJobsForRateLimit(nonConsumingDelayMs, now); + if (deferred) { + await recordQueueMetric("gittensory_jobs_rate_limit_deferred_total", deferred); + console.warn( + JSON.stringify({ + level: "warn", + event: "selfhost_queue_rate_limit_cooldown", + deferred, + cooldown_until: githubRateLimitCooldownUntil, + }), + ); + } + } + if (job.job_key && (await mergeRescheduledJobIntoPending(job, retryAfter, errMsg))) { + await recordQueueMetric("gittensory_jobs_coalesced_total"); + } else { + await pool.query( + `UPDATE ${TABLE} SET status='pending', run_after=$1, last_error=$2 WHERE id=$3`, + [retryAfter, errMsg, job.id], + ); + } + await recordQueueMetric(rateLimited ? "gittensory_jobs_rate_limited_total" : "gittensory_jobs_deferred_total"); + logAudit({ + event: "job_rate_limited", + ts: Date.now(), + job_id: job.id, + payload_type: extractPayloadType(job.payload), + latency_ms: Date.now() - claimedAt, + attempts, + retry_after_ms: Math.max(0, retryAfter - Date.now()), + error: errMsg, + }); + return true; + } + await recordQueueMetric("gittensory_jobs_failed_total"); + if (attempts >= maxRetries) { + await pool.query( + `UPDATE ${TABLE} SET status='dead', attempts=$1, last_error=$2 WHERE id=$3`, + [attempts, errMsg, job.id], + ); + await recordQueueMetric("gittensory_jobs_dead_total"); + console.error( + JSON.stringify({ + level: "error", + event: "selfhost_job_dead", + id: job.id, + attempts, + error: errMsg, + }), + ); + logAudit({ + event: "job_dead", + ts: Date.now(), + job_id: job.id, + payload_type: extractPayloadType(job.payload), + latency_ms: Date.now() - claimedAt, + attempts, + error: errMsg, + }); + captureError(error, { + kind: "job_dead", + reason: "max_retries_exhausted", + jobType: extractPayloadType(job.payload), + jobId: job.id, + attempts, + }); + } else { + await pool.query( + `UPDATE ${TABLE} SET status='pending', attempts=$1, run_after=$2, last_error=$3 WHERE id=$4`, + [attempts, Date.now() + backoff(attempts), errMsg, job.id], + ); + logAudit({ + event: "job_error", + ts: Date.now(), + job_id: job.id, + payload_type: extractPayloadType(job.payload), + latency_ms: Date.now() - claimedAt, + attempts, + error: errMsg, + }); + } } + return true; + } finally { + activeJobIds.delete(job.id); } - return true; } async function pump(): Promise { @@ -458,6 +485,28 @@ export function createPgQueue( }, }; + async function reclaimExpiredProcessingJobs(): Promise { + if (processingTimeoutMs <= 0) return 0; + const now = Date.now(); + const cutoff = now - processingTimeoutMs; + const res = await pool.query( + `SELECT id, payload, job_key FROM ${TABLE} WHERE status='processing' AND run_after<=$1`, + [cutoff], + ); + let changed = 0; + const maxJitter = queueRecoveryJitterMs(); + for (const row of res.rows as Array<{ id: string; payload: string; job_key?: string | null }>) { + if (activeJobIds.has(row.id)) continue; + const runAfter = now + deterministicJitterMs(`${row.job_key ?? ""}:${row.id}:${row.payload}`, maxJitter); + const update = await pool.query( + `UPDATE ${TABLE} SET status='pending', run_after=$1, last_error=COALESCE(last_error, $2) WHERE id=$3 AND status='processing'`, + [runAfter, "processing lease expired; requeued", row.id], + ); + changed += update.rowCount ?? 0; + } + return changed; + } + async function deferPendingJobsForRateLimit( delayMs: number, now: number, diff --git a/src/selfhost/queue-common.ts b/src/selfhost/queue-common.ts index f73b03f393..5c6f2d0ad6 100644 --- a/src/selfhost/queue-common.ts +++ b/src/selfhost/queue-common.ts @@ -5,6 +5,7 @@ const DEFAULT_RATE_LIMIT_JITTER_MS = 5 * 60_000; const DEFAULT_STARTUP_JITTER_MS = 3 * 60_000; const DEFAULT_RECOVERY_JITTER_MS = 60_000; const DEFAULT_STARTUP_JITTER_MIN_JOBS = 8; +const DEFAULT_PROCESSING_TIMEOUT_MS = 30 * 60_000; // Webhook-driven work (a fresh PR -> its review) jumps ahead of heavy background jobs. Per-PR review refreshes // sit just below real webhooks, and sweep fan-out sits below those so stale surfaces are repaired during bursts. @@ -109,6 +110,13 @@ export function queueRecoveryJitterMs(): number { return envDurationMs("QUEUE_RECOVERY_JITTER_MS", DEFAULT_RECOVERY_JITTER_MS); } +export function queueProcessingTimeoutMs(): number { + return envDurationMs( + "QUEUE_PROCESSING_TIMEOUT_MS", + DEFAULT_PROCESSING_TIMEOUT_MS, + ); +} + export function queueStartupJitterMinJobs(): number { const raw = Number(process.env.QUEUE_STARTUP_JITTER_MIN_JOBS ?? DEFAULT_STARTUP_JITTER_MIN_JOBS); return Number.isFinite(raw) && raw >= 0 ? Math.floor(raw) : DEFAULT_STARTUP_JITTER_MIN_JOBS; diff --git a/src/selfhost/sqlite-queue.ts b/src/selfhost/sqlite-queue.ts index 4e2b860802..67a6857a14 100644 --- a/src/selfhost/sqlite-queue.ts +++ b/src/selfhost/sqlite-queue.ts @@ -13,6 +13,7 @@ import { jobCoalesceKey, jobPriority, nonConsumingRetryDelayMs, + queueProcessingTimeoutMs, queueRecoveryJitterMs, queueStartupJitterMinJobs, queueStartupJitterMs, @@ -85,6 +86,7 @@ export function createSqliteQueue( const concurrency = opts.concurrency ?? Math.max(1, Number(process.env.QUEUE_CONCURRENCY ?? "4")); + const processingTimeoutMs = queueProcessingTimeoutMs(); driver.exec(DDL); driver.exec(STATS_DDL); @@ -138,6 +140,7 @@ export function createSqliteQueue( let running = false; let active = 0; // number of concurrent pump() loops currently draining jobs + const activeJobIds = new Set(); let timer: ReturnType | null = null; let githubRateLimitCooldownUntil = 0; @@ -174,15 +177,16 @@ export function createSqliteQueue( function claimNext(): JobRow | null { if (Date.now() < githubRateLimitCooldownUntil) return null; + const now = Date.now(); const { rows } = driver.query( `SELECT id, payload, attempts, job_key FROM ${TABLE} WHERE status='pending' AND run_after<=? ORDER BY priority DESC, run_after, id LIMIT 1`, - [Date.now()], + [now], ); const row = rows[0] as JobRow | undefined; if (!row) return null; const { changes } = driver.query( - `UPDATE ${TABLE} SET status='processing' WHERE id=? AND status='pending'`, - [row.id], + `UPDATE ${TABLE} SET status='processing', run_after=? WHERE id=? AND status='pending'`, + [now, row.id], ); /* v8 ignore next */ // the no-rows branch is a multi-writer guard; unreachable in the single-process model return changes ? row : null; @@ -196,138 +200,165 @@ export function createSqliteQueue( } async function processOne(): Promise { - const job = claimNext(); - if (!job) return false; - const claimedAt = Date.now(); - let message: JobMessage; - try { - message = JSON.parse(job.payload) as JobMessage; - } catch { - driver.query( - `UPDATE ${TABLE} SET status='dead', last_error='unparseable payload' WHERE id=?`, - [job.id], + const recovered = reclaimExpiredProcessingJobs( + driver, + processingTimeoutMs, + activeJobIds, + ); + if (recovered) { + recordQueueMetric(driver, "gittensory_jobs_recovered_total", recovered); + console.warn( + JSON.stringify({ + level: "warn", + event: "selfhost_queue_processing_reclaimed", + count: recovered, + timeout_ms: processingTimeoutMs, + }), ); - recordQueueMetric(driver, "gittensory_jobs_dead_total"); - logAudit({ - event: "job_dead", - ts: Date.now(), - job_id: job.id, - latency_ms: Date.now() - claimedAt, - attempts: job.attempts + 1, - error: "unparseable payload", - }); - captureError(new Error("unparseable queue payload"), { - kind: "job_dead", - reason: "unparseable_payload", - jobId: job.id, + captureError(new Error("self-host queue processing lease expired"), { + kind: "job_recovered", + reason: "processing_timeout", + recovered, + timeoutMs: processingTimeoutMs, }); - return true; } + const job = claimNext(); + if (!job) return false; + activeJobIds.add(job.id); + const claimedAt = Date.now(); try { - await consume(message); - driver.query(`DELETE FROM ${TABLE} WHERE id=?`, [job.id]); - recordQueueMetric(driver, "gittensory_jobs_processed_total"); - logAudit({ - event: "job_complete", - ts: Date.now(), - job_id: job.id, - payload_type: extractPayloadType(job.payload), - latency_ms: Date.now() - claimedAt, - attempts: job.attempts + 1, - }); - } catch (error) { - const attempts = job.attempts + 1; - const errMsg = error instanceof Error ? error.message : "unknown error"; - const nonConsumingDelayMs = nonConsumingRetryDelayMs(error); - if (nonConsumingDelayMs !== null) { - const rateLimited = githubRateLimitRetryDelayMs(error) !== null; - const now = Date.now(); - const retryAfter = now + (rateLimited ? rateLimitRetryDelayWithJitter(nonConsumingDelayMs, `${job.job_key ?? ""}:${job.id}:${job.payload}`) : nonConsumingDelayMs); - if (rateLimited) { - githubRateLimitCooldownUntil = Math.max(githubRateLimitCooldownUntil, now + nonConsumingDelayMs); - const deferred = deferPendingJobsForRateLimit(driver, nonConsumingDelayMs, now); - if (deferred) { - recordQueueMetric(driver, "gittensory_jobs_rate_limit_deferred_total", deferred); - console.warn( - JSON.stringify({ - level: "warn", - event: "selfhost_queue_rate_limit_cooldown", - deferred, - cooldown_until: githubRateLimitCooldownUntil, - }), - ); - } - } - if (job.job_key && mergeRescheduledJobIntoPending(driver, job, retryAfter, errMsg)) { - recordQueueMetric(driver, "gittensory_jobs_coalesced_total"); - } else { - driver.query( - `UPDATE ${TABLE} SET status='pending', run_after=?, last_error=? WHERE id=?`, - [retryAfter, errMsg, job.id], - ); - } - recordQueueMetric(driver, rateLimited ? "gittensory_jobs_rate_limited_total" : "gittensory_jobs_deferred_total"); - logAudit({ - event: "job_rate_limited", - ts: Date.now(), - job_id: job.id, - payload_type: extractPayloadType(job.payload), - latency_ms: Date.now() - claimedAt, - attempts, - retry_after_ms: Math.max(0, retryAfter - Date.now()), - error: errMsg, - }); - return true; - } - recordQueueMetric(driver, "gittensory_jobs_failed_total"); - if (attempts >= maxRetries) { + let message: JobMessage; + try { + message = JSON.parse(job.payload) as JobMessage; + } catch { driver.query( - `UPDATE ${TABLE} SET status='dead', attempts=?, last_error=? WHERE id=?`, - [attempts, errMsg, job.id], + `UPDATE ${TABLE} SET status='dead', last_error='unparseable payload' WHERE id=?`, + [job.id], ); recordQueueMetric(driver, "gittensory_jobs_dead_total"); - console.error( - JSON.stringify({ - level: "error", - event: "selfhost_job_dead", - id: job.id, - attempts, - error: errMsg, - }), - ); logAudit({ event: "job_dead", ts: Date.now(), job_id: job.id, - payload_type: extractPayloadType(job.payload), latency_ms: Date.now() - claimedAt, - attempts, - error: errMsg, + attempts: job.attempts + 1, + error: "unparseable payload", }); - captureError(error, { + captureError(new Error("unparseable queue payload"), { kind: "job_dead", - reason: "max_retries_exhausted", - jobType: extractPayloadType(job.payload), + reason: "unparseable_payload", jobId: job.id, - attempts, }); - } else { - driver.query( - `UPDATE ${TABLE} SET status='pending', attempts=?, run_after=?, last_error=? WHERE id=?`, - [attempts, Date.now() + backoff(attempts), errMsg, job.id], - ); + return true; + } + try { + await consume(message); + driver.query(`DELETE FROM ${TABLE} WHERE id=?`, [job.id]); + recordQueueMetric(driver, "gittensory_jobs_processed_total"); logAudit({ - event: "job_error", + event: "job_complete", ts: Date.now(), job_id: job.id, payload_type: extractPayloadType(job.payload), latency_ms: Date.now() - claimedAt, - attempts, - error: errMsg, + attempts: job.attempts + 1, }); + } catch (error) { + const attempts = job.attempts + 1; + const errMsg = error instanceof Error ? error.message : "unknown error"; + const nonConsumingDelayMs = nonConsumingRetryDelayMs(error); + if (nonConsumingDelayMs !== null) { + const rateLimited = githubRateLimitRetryDelayMs(error) !== null; + const now = Date.now(); + const retryAfter = now + (rateLimited ? rateLimitRetryDelayWithJitter(nonConsumingDelayMs, `${job.job_key ?? ""}:${job.id}:${job.payload}`) : nonConsumingDelayMs); + if (rateLimited) { + githubRateLimitCooldownUntil = Math.max(githubRateLimitCooldownUntil, now + nonConsumingDelayMs); + const deferred = deferPendingJobsForRateLimit(driver, nonConsumingDelayMs, now); + if (deferred) { + recordQueueMetric(driver, "gittensory_jobs_rate_limit_deferred_total", deferred); + console.warn( + JSON.stringify({ + level: "warn", + event: "selfhost_queue_rate_limit_cooldown", + deferred, + cooldown_until: githubRateLimitCooldownUntil, + }), + ); + } + } + if (job.job_key && mergeRescheduledJobIntoPending(driver, job, retryAfter, errMsg)) { + recordQueueMetric(driver, "gittensory_jobs_coalesced_total"); + } else { + driver.query( + `UPDATE ${TABLE} SET status='pending', run_after=?, last_error=? WHERE id=?`, + [retryAfter, errMsg, job.id], + ); + } + recordQueueMetric(driver, rateLimited ? "gittensory_jobs_rate_limited_total" : "gittensory_jobs_deferred_total"); + logAudit({ + event: "job_rate_limited", + ts: Date.now(), + job_id: job.id, + payload_type: extractPayloadType(job.payload), + latency_ms: Date.now() - claimedAt, + attempts, + retry_after_ms: Math.max(0, retryAfter - Date.now()), + error: errMsg, + }); + return true; + } + recordQueueMetric(driver, "gittensory_jobs_failed_total"); + if (attempts >= maxRetries) { + driver.query( + `UPDATE ${TABLE} SET status='dead', attempts=?, last_error=? WHERE id=?`, + [attempts, errMsg, job.id], + ); + recordQueueMetric(driver, "gittensory_jobs_dead_total"); + console.error( + JSON.stringify({ + level: "error", + event: "selfhost_job_dead", + id: job.id, + attempts, + error: errMsg, + }), + ); + logAudit({ + event: "job_dead", + ts: Date.now(), + job_id: job.id, + payload_type: extractPayloadType(job.payload), + latency_ms: Date.now() - claimedAt, + attempts, + error: errMsg, + }); + captureError(error, { + kind: "job_dead", + reason: "max_retries_exhausted", + jobType: extractPayloadType(job.payload), + jobId: job.id, + attempts, + }); + } else { + driver.query( + `UPDATE ${TABLE} SET status='pending', attempts=?, run_after=?, last_error=? WHERE id=?`, + [attempts, Date.now() + backoff(attempts), errMsg, job.id], + ); + logAudit({ + event: "job_error", + ts: Date.now(), + job_id: job.id, + payload_type: extractPayloadType(job.payload), + latency_ms: Date.now() - claimedAt, + attempts, + error: errMsg, + }); + } } + return true; + } finally { + activeJobIds.delete(job.id); } - return true; } // Drains every job that is currently DUE. A retry is rescheduled into the future (run_after > now) so it is @@ -499,6 +530,32 @@ function deferPendingJobsForRateLimit( return changed; } +function reclaimExpiredProcessingJobs( + driver: SqliteDriver, + timeoutMs: number, + activeJobIds: Set, +): number { + if (timeoutMs <= 0) return 0; + const now = Date.now(); + const cutoff = now - timeoutMs; + const { rows } = driver.query( + `SELECT id, payload, job_key FROM ${TABLE} WHERE status='processing' AND run_after<=?`, + [cutoff], + ); + let changed = 0; + const maxJitter = queueRecoveryJitterMs(); + for (const row of rows as Array<{ id: number; payload: string; job_key?: string | null }>) { + if (activeJobIds.has(row.id)) continue; + const runAfter = now + deterministicJitterMs(`${row.job_key ?? ""}:${row.id}:${row.payload}`, maxJitter); + const { changes } = driver.query( + `UPDATE ${TABLE} SET status='pending', run_after=?, last_error=coalesce(last_error, ?) WHERE id=? AND status='processing'`, + [runAfter, "processing lease expired; requeued", row.id], + ); + changed += changes; + } + return changed; +} + function mergeRescheduledJobIntoPending( driver: SqliteDriver, job: JobRow, diff --git a/src/server.ts b/src/server.ts index 745a27403c..90df485d05 100644 --- a/src/server.ts +++ b/src/server.ts @@ -498,6 +498,7 @@ async function main(): Promise { "gittensory_jobs_rate_limit_deferred_total", "gittensory_jobs_deferred_total", "gittensory_jobs_coalesced_total", + "gittensory_jobs_recovered_total", ]) { gauge(name.replace("_total", "_persisted_total"), () => durableJobMetric(name), @@ -514,6 +515,7 @@ async function main(): Promise { "gittensory_jobs_failed_total", "gittensory_jobs_dead_total", "gittensory_jobs_rate_limit_deferred_total", + "gittensory_jobs_recovered_total", "gittensory_webhook_dedup_total", "gittensory_qdrant_queries_total", "gittensory_qdrant_upserts_total", diff --git a/test/unit/selfhost-pg-queue.test.ts b/test/unit/selfhost-pg-queue.test.ts index 7c9c11a5db..df4586ee78 100644 --- a/test/unit/selfhost-pg-queue.test.ts +++ b/test/unit/selfhost-pg-queue.test.ts @@ -222,6 +222,39 @@ describe("createPgQueue (durable #977)", () => { ); }); + it("reclaims expired processing leases before claiming more work", async () => { + const oldTimeout = process.env.QUEUE_PROCESSING_TIMEOUT_MS; + const oldRecoveryJitter = process.env.QUEUE_RECOVERY_JITTER_MS; + process.env.QUEUE_PROCESSING_TIMEOUT_MS = "1"; + process.env.QUEUE_RECOVERY_JITTER_MS = "0"; + try { + const m = makePool(); + const q = createPgQueue(m.pool, async () => undefined); + await q.init(); + m.fn.mockResolvedValueOnce({ + rows: [{ id: "old", payload: JSON.stringify(msg("stuck")), job_key: "stuck-key" }], + rowCount: 1, + }); + m.fn.mockResolvedValueOnce({ rows: [], rowCount: 1 }); + + await q.drain(); + + expect(m.pool.query).toHaveBeenCalledWith( + expect.stringContaining("WHERE status='processing' AND run_after<=$1"), + expect.arrayContaining([expect.any(Number)]), + ); + expect(m.pool.query).toHaveBeenCalledWith( + expect.stringContaining("SET status='pending', run_after=$1"), + expect.arrayContaining([expect.any(Number), "processing lease expired; requeued", "old"]), + ); + } finally { + if (oldTimeout === undefined) delete process.env.QUEUE_PROCESSING_TIMEOUT_MS; + else process.env.QUEUE_PROCESSING_TIMEOUT_MS = oldTimeout; + if (oldRecoveryJitter === undefined) delete process.env.QUEUE_RECOVERY_JITTER_MS; + else process.env.QUEUE_RECOVERY_JITTER_MS = oldRecoveryJitter; + } + }); + it("reschedules retryable incomplete review jobs without consuming the dead-letter budget", async () => { const m = makePool(); m.enqueueJob("1", { type: "agent-regate-pr" }, 4); diff --git a/test/unit/selfhost-sqlite-queue.test.ts b/test/unit/selfhost-sqlite-queue.test.ts index f320e8953d..90921e6cf7 100644 --- a/test/unit/selfhost-sqlite-queue.test.ts +++ b/test/unit/selfhost-sqlite-queue.test.ts @@ -479,6 +479,39 @@ describe("createSqliteQueue (durable #980)", () => { } }); + it("reclaims an expired processing lease without requiring a restart", async () => { + const oldTimeout = process.env.QUEUE_PROCESSING_TIMEOUT_MS; + const oldRecoveryJitter = process.env.QUEUE_RECOVERY_JITTER_MS; + process.env.QUEUE_PROCESSING_TIMEOUT_MS = "1"; + process.env.QUEUE_RECOVERY_JITTER_MS = "0"; + const driver = makeDriver(); + const seen: string[] = []; + try { + const q = createSqliteQueue( + driver, + async (m) => void seen.push(typeOf(m)), + { concurrency: 1 }, + ); + driver.query( + "INSERT INTO _selfhost_jobs (payload, status, attempts, run_after, created_at) VALUES (?, 'processing', 0, ?, 0)", + [JSON.stringify(msg("lease-expired")), Date.now() - 10_000], + ); + + await q.drain(); + + expect(seen).toEqual(["lease-expired"]); + expect(q.stats()).toMatchObject({ + gittensory_jobs_recovered_total: 1, + gittensory_jobs_processed_total: 1, + }); + } finally { + if (oldTimeout === undefined) delete process.env.QUEUE_PROCESSING_TIMEOUT_MS; + else process.env.QUEUE_PROCESSING_TIMEOUT_MS = oldTimeout; + if (oldRecoveryJitter === undefined) delete process.env.QUEUE_RECOVERY_JITTER_MS; + else process.env.QUEUE_RECOVERY_JITTER_MS = oldRecoveryJitter; + } + }); + it("records 'unknown error' when a consumer throws a non-Error", async () => { const q = createSqliteQueue( makeDriver(), From 8ed0d7529ccea8a248711d256d92c4b8fc2d5ce8 Mon Sep 17 00:00:00 2001 From: JSONbored <49853598+JSONbored@users.noreply.github.com> Date: Sun, 28 Jun 2026 19:14:51 -0700 Subject: [PATCH 34/68] fix(queue): count startup lease recovery --- src/selfhost/pg-queue.ts | 4 +++- src/selfhost/sqlite-queue.ts | 4 +++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/src/selfhost/pg-queue.ts b/src/selfhost/pg-queue.ts index 688c145e0e..fc1aaac736 100644 --- a/src/selfhost/pg-queue.ts +++ b/src/selfhost/pg-queue.ts @@ -111,10 +111,12 @@ export function createPgQueue( }), ); const recovered = await recoverProcessingJobs(); - if (recovered) + if (recovered) { + await recordQueueMetric("gittensory_jobs_recovered_total", recovered); console.log( JSON.stringify({ event: "selfhost_queue_recovered", count: recovered }), ); + } const spread = await spreadDueJobsOnStartup(); if (spread) console.log( diff --git a/src/selfhost/sqlite-queue.ts b/src/selfhost/sqlite-queue.ts index 67a6857a14..a91d7264ec 100644 --- a/src/selfhost/sqlite-queue.ts +++ b/src/selfhost/sqlite-queue.ts @@ -124,10 +124,12 @@ export function createSqliteQueue( ); // Recover jobs a crashed previous run left mid-flight → make them claimable again. const recovered = recoverProcessingJobs(driver); - if (recovered) + if (recovered) { + recordQueueMetric(driver, "gittensory_jobs_recovered_total", recovered); console.log( JSON.stringify({ event: "selfhost_queue_recovered", count: recovered }), ); + } const spread = spreadDueJobsOnStartup(driver); if (spread) console.log( From a6da5bcd27434fb9f6f1a5f7b429e6d230304c03 Mon Sep 17 00:00:00 2001 From: JSONbored <49853598+JSONbored@users.noreply.github.com> Date: Sun, 28 Jun 2026 19:35:01 -0700 Subject: [PATCH 35/68] fix(queue): fill workers for due backlog --- src/selfhost/pg-queue.ts | 17 ++++++++---- src/selfhost/sqlite-queue.ts | 17 ++++++++---- test/unit/selfhost-pg-queue.test.ts | 33 +++++++++++++++++++++++ test/unit/selfhost-sqlite-queue.test.ts | 36 +++++++++++++++++++++++++ 4 files changed, 93 insertions(+), 10 deletions(-) diff --git a/src/selfhost/pg-queue.ts b/src/selfhost/pg-queue.ts index fc1aaac736..be7f02cd59 100644 --- a/src/selfhost/pg-queue.ts +++ b/src/selfhost/pg-queue.ts @@ -224,7 +224,7 @@ export function createPgQueue( [payload, runAfter, now, priority, existing.id], ); await recordQueueMetric("gittensory_jobs_coalesced_total"); - void pump(); + kickOne(); return; } } @@ -233,7 +233,7 @@ export function createPgQueue( [payload, runAfter, now, priority, key], ); await recordQueueMetric("gittensory_jobs_enqueued_total"); - void pump(); + kickOne(); } async function claimNext(): Promise { @@ -426,6 +426,14 @@ export function createPgQueue( } } + function kickOne(): void { + void pump(); + } + + function kickAll(): void { + while (active < concurrency) void pump(); + } + const binding = { async send( message: JobMessage, @@ -449,9 +457,8 @@ export function createPgQueue( const tick = (): void => { /* v8 ignore next */ // stop() clears the timer before the next tick can fire with running=false if (!running) return; - void pump().finally(() => { - if (running) timer = setTimeout(tick, pollIntervalMs); - }); + kickAll(); + timer = setTimeout(tick, pollIntervalMs); }; tick(); }, diff --git a/src/selfhost/sqlite-queue.ts b/src/selfhost/sqlite-queue.ts index a91d7264ec..9702efa5ca 100644 --- a/src/selfhost/sqlite-queue.ts +++ b/src/selfhost/sqlite-queue.ts @@ -165,7 +165,7 @@ export function createSqliteQueue( [payload, runAfter, now, priority, existing.id], ); recordQueueMetric(driver, "gittensory_jobs_coalesced_total"); - void pump(); + kickOne(); return; } } @@ -174,7 +174,7 @@ export function createSqliteQueue( [payload, runAfter, now, priority, key], ); recordQueueMetric(driver, "gittensory_jobs_enqueued_total"); - void pump(); + kickOne(); } function claimNext(): JobRow | null { @@ -378,6 +378,14 @@ export function createSqliteQueue( } } + function kickOne(): void { + void pump(); + } + + function kickAll(): void { + while (active < concurrency) void pump(); + } + const binding = { async send( message: JobMessage, @@ -400,9 +408,8 @@ export function createSqliteQueue( const tick = (): void => { /* v8 ignore next */ // stop() clears the timer, so a tick never fires with running=false if (!running) return; - void pump().finally(() => { - if (running) timer = setTimeout(tick, pollIntervalMs); - }); + kickAll(); + timer = setTimeout(tick, pollIntervalMs); }; tick(); }, diff --git a/test/unit/selfhost-pg-queue.test.ts b/test/unit/selfhost-pg-queue.test.ts index df4586ee78..e2b1ef1c94 100644 --- a/test/unit/selfhost-pg-queue.test.ts +++ b/test/unit/selfhost-pg-queue.test.ts @@ -342,6 +342,39 @@ describe("createPgQueue (durable #977)", () => { expect(seen).toEqual(["ticked"]); }); + it("start() fills available workers for an existing due backlog", async () => { + const m = makePool(); + let release!: () => void; + const gate = new Promise((resolve) => { + release = resolve; + }); + let concurrent = 0; + let maxConcurrent = 0; + const q = createPgQueue( + m.pool, + async () => { + concurrent++; + maxConcurrent = Math.max(maxConcurrent, concurrent); + await gate; + concurrent--; + }, + { concurrency: 3, pollIntervalMs: 100_000 }, + ); + await q.init(); + m.enqueueJob("1", { type: "a" }); + m.enqueueJob("2", { type: "b" }); + m.enqueueJob("3", { type: "c" }); + try { + q.start(); + for (let i = 0; i < 20 && maxConcurrent < 3; i += 1) + await new Promise((r) => setTimeout(r, 10)); + expect(maxConcurrent).toBe(3); + } finally { + release(); + await q.stop(); + } + }); + it("start() is idempotent", async () => { const { pool } = makePool(); const q = createPgQueue(pool, async () => undefined, { pollIntervalMs: 100_000 }); diff --git a/test/unit/selfhost-sqlite-queue.test.ts b/test/unit/selfhost-sqlite-queue.test.ts index 90921e6cf7..06a329bbb4 100644 --- a/test/unit/selfhost-sqlite-queue.test.ts +++ b/test/unit/selfhost-sqlite-queue.test.ts @@ -462,6 +462,42 @@ describe("createSqliteQueue (durable #980)", () => { expect(seen).toEqual(["ticked"]); }); + it("start() fills available workers for an existing due backlog", async () => { + const driver = makeDriver(); + createSqliteQueue(driver, async () => undefined); // creates the table + for (const name of ["a", "b", "c"]) { + driver.query( + "INSERT INTO _selfhost_jobs (payload, status, attempts, run_after, created_at) VALUES (?, 'pending', 0, 0, 0)", + [JSON.stringify(msg(name))], + ); + } + let release!: () => void; + const gate = new Promise((resolve) => { + release = resolve; + }); + let concurrent = 0; + let maxConcurrent = 0; + const q = createSqliteQueue( + driver, + async () => { + concurrent++; + maxConcurrent = Math.max(maxConcurrent, concurrent); + await gate; + concurrent--; + }, + { concurrency: 3, pollIntervalMs: 100_000 }, + ); + try { + q.start(); + for (let i = 0; i < 20 && maxConcurrent < 3; i += 1) + await new Promise((r) => setTimeout(r, 10)); + expect(maxConcurrent).toBe(3); + } finally { + release(); + await q.stop(); + } + }); + it("recovers a job left 'processing' by a crash", async () => { const oldRecoveryJitter = process.env.QUEUE_RECOVERY_JITTER_MS; process.env.QUEUE_RECOVERY_JITTER_MS = "0"; From e0eaf7c2e6caa69e5e8b1292672b2bf1a1573c9d Mon Sep 17 00:00:00 2001 From: JSONbored <49853598+JSONbored@users.noreply.github.com> Date: Sun, 28 Jun 2026 20:24:14 -0700 Subject: [PATCH 36/68] test(queue): cover durable retry pressure paths --- src/selfhost/pg-queue.ts | 5 +- src/selfhost/sqlite-queue.ts | 5 +- test/unit/selfhost-pg-queue.test.ts | 184 +++++++++++++++++++++++- test/unit/selfhost-sqlite-queue.test.ts | 38 +++++ 4 files changed, 221 insertions(+), 11 deletions(-) diff --git a/src/selfhost/pg-queue.ts b/src/selfhost/pg-queue.ts index be7f02cd59..85faf7e307 100644 --- a/src/selfhost/pg-queue.ts +++ b/src/selfhost/pg-queue.ts @@ -339,7 +339,7 @@ export function createPgQueue( ); } } - if (job.job_key && (await mergeRescheduledJobIntoPending(job, retryAfter, errMsg))) { + if (job.job_key && (await mergeRescheduledJobIntoPending(job as JobRow & { job_key: string }, retryAfter, errMsg))) { await recordQueueMetric("gittensory_jobs_coalesced_total"); } else { await pool.query( @@ -537,11 +537,10 @@ export function createPgQueue( } async function mergeRescheduledJobIntoPending( - job: JobRow, + job: JobRow & { job_key: string }, runAfter: number, errMsg: string, ): Promise { - if (!job.job_key) return false; const existing = ( await pool.query( `SELECT id FROM ${TABLE} WHERE status='pending' AND job_key=$1 AND id<>$2 ORDER BY priority DESC, run_after DESC, id LIMIT 1`, diff --git a/src/selfhost/sqlite-queue.ts b/src/selfhost/sqlite-queue.ts index 9702efa5ca..59d5bde009 100644 --- a/src/selfhost/sqlite-queue.ts +++ b/src/selfhost/sqlite-queue.ts @@ -288,7 +288,7 @@ export function createSqliteQueue( ); } } - if (job.job_key && mergeRescheduledJobIntoPending(driver, job, retryAfter, errMsg)) { + if (job.job_key && mergeRescheduledJobIntoPending(driver, job as JobRow & { job_key: string }, retryAfter, errMsg)) { recordQueueMetric(driver, "gittensory_jobs_coalesced_total"); } else { driver.query( @@ -567,11 +567,10 @@ function reclaimExpiredProcessingJobs( function mergeRescheduledJobIntoPending( driver: SqliteDriver, - job: JobRow, + job: JobRow & { job_key: string }, runAfter: number, errMsg: string, ): boolean { - if (!job.job_key) return false; const existing = driver.query( `SELECT id FROM ${TABLE} WHERE status='pending' AND job_key=? AND id<>? ORDER BY priority DESC, run_after DESC, id LIMIT 1`, [job.job_key, job.id], diff --git a/test/unit/selfhost-pg-queue.test.ts b/test/unit/selfhost-pg-queue.test.ts index e2b1ef1c94..7f7f1123bb 100644 --- a/test/unit/selfhost-pg-queue.test.ts +++ b/test/unit/selfhost-pg-queue.test.ts @@ -34,7 +34,7 @@ interface MockPool { fn: MockFn; enqueueResult(r: Partial): void; /** Pre-load a job to be returned by the next RETURNING claim query. */ - enqueueJob(id: string, payload: object, attempts?: number): void; + enqueueJob(id: string, payload: object, attempts?: number, jobKey?: string | null): void; } function makePool(): MockPool { @@ -56,8 +56,8 @@ function makePool(): MockPool { pool: { query: fn } as unknown as Pool, fn: fn as unknown as MockFn, enqueueResult(r) { results.push(r); }, - enqueueJob(id, payload, attempts = 0) { - results.push({ rows: [{ id, payload: JSON.stringify(payload), attempts }], rowCount: 1 }); + enqueueJob(id, payload, attempts = 0, jobKey = null) { + results.push({ rows: [{ id, payload: JSON.stringify(payload), attempts, job_key: jobKey }], rowCount: 1 }); }, }; } @@ -112,6 +112,99 @@ describe("createPgQueue (durable #977)", () => { expect(m.pool.query).toHaveBeenCalledWith(expect.stringContaining("UPDATE _selfhost_jobs SET priority=$1"), [8, "c"]); }); + it("init() backfills job keys, recovers crashed jobs, and spreads due startup backlog", async () => { + const oldMin = process.env.QUEUE_STARTUP_JITTER_MIN_JOBS; + const oldJitter = process.env.QUEUE_STARTUP_JITTER_MS; + const oldRecoveryJitter = process.env.QUEUE_RECOVERY_JITTER_MS; + process.env.QUEUE_STARTUP_JITTER_MIN_JOBS = "2"; + process.env.QUEUE_STARTUP_JITTER_MS = "60000"; + process.env.QUEUE_RECOVERY_JITTER_MS = "0"; + try { + const fn = vi.fn().mockImplementation(async (sql: unknown) => { + const q = String(sql); + if (q.includes("SELECT id, payload, priority")) return { rows: [], rowCount: 0 }; + if (q.includes("SELECT id, payload, job_key") && q.includes("status IN")) { + return { rows: [{ id: "keyed", payload: JSON.stringify(ciWebhook("ci-1")), job_key: null }], rowCount: 1 }; + } + if (q.includes("UPDATE _selfhost_jobs SET job_key=$1")) return { rows: [], rowCount: 1 }; + if (q.includes("WHERE status='processing'")) { + return { rows: [{ id: "recover", payload: JSON.stringify(msg("stuck")), job_key: "recover-key" }], rowCount: 1 }; + } + if (q.includes("SET status='pending', run_after=$1 WHERE id=$2")) return { rows: [], rowCount: 1 }; + if (q.includes("WHERE status='pending' AND run_after<=$1")) { + return { + rows: [ + { id: "spread-a", payload: JSON.stringify(msg("a")), job_key: "spread-a" }, + { id: "spread-b", payload: JSON.stringify(msg("b")), job_key: "spread-b" }, + ], + rowCount: 2, + }; + } + if (q.includes("UPDATE _selfhost_jobs SET run_after=$1 WHERE id=$2")) return { rows: [], rowCount: 1 }; + return { rows: [], rowCount: 0 }; + }); + const q = createPgQueue({ query: fn } as unknown as Pool, async () => undefined); + + await q.init(); + + expect(fn).toHaveBeenCalledWith( + expect.stringContaining("UPDATE _selfhost_jobs SET job_key=$1"), + [`github-webhook:ci-completed:jsonbored/gittensory@${"b".repeat(40)}#1629`, "keyed"], + ); + expect(fn).toHaveBeenCalledWith( + expect.stringContaining("SET status='pending', run_after=$1 WHERE id=$2"), + expect.arrayContaining([expect.any(Number), "recover"]), + ); + expect(fn).toHaveBeenCalledWith( + expect.stringContaining("UPDATE _selfhost_jobs SET run_after=$1 WHERE id=$2"), + expect.arrayContaining([expect.any(Number), "spread-a"]), + ); + expect(fn).toHaveBeenCalledWith( + expect.stringContaining("UPDATE _selfhost_jobs SET run_after=$1 WHERE id=$2"), + expect.arrayContaining([expect.any(Number), "spread-b"]), + ); + } finally { + if (oldMin === undefined) delete process.env.QUEUE_STARTUP_JITTER_MIN_JOBS; + else process.env.QUEUE_STARTUP_JITTER_MIN_JOBS = oldMin; + if (oldJitter === undefined) delete process.env.QUEUE_STARTUP_JITTER_MS; + else process.env.QUEUE_STARTUP_JITTER_MS = oldJitter; + if (oldRecoveryJitter === undefined) delete process.env.QUEUE_RECOVERY_JITTER_MS; + else process.env.QUEUE_RECOVERY_JITTER_MS = oldRecoveryJitter; + } + }); + + it("init() skips startup spread when jitter is disabled", async () => { + const oldMin = process.env.QUEUE_STARTUP_JITTER_MIN_JOBS; + const oldJitter = process.env.QUEUE_STARTUP_JITTER_MS; + process.env.QUEUE_STARTUP_JITTER_MIN_JOBS = "1"; + process.env.QUEUE_STARTUP_JITTER_MS = "0"; + try { + const fn = vi.fn().mockImplementation(async (sql: unknown) => { + const q = String(sql); + if (q.includes("SELECT id, payload, priority")) return { rows: [], rowCount: 0 }; + if (q.includes("SELECT id, payload, job_key") && q.includes("status IN")) return { rows: [], rowCount: 0 }; + if (q.includes("WHERE status='processing'")) return { rows: [], rowCount: 0 }; + if (q.includes("WHERE status='pending' AND run_after<=$1")) { + return { rows: [{ id: "due", payload: JSON.stringify(msg("due")), job_key: "due" }], rowCount: 1 }; + } + return { rows: [], rowCount: 0 }; + }); + const q = createPgQueue({ query: fn } as unknown as Pool, async () => undefined); + + await q.init(); + + expect(fn).not.toHaveBeenCalledWith( + expect.stringContaining("UPDATE _selfhost_jobs SET run_after=$1 WHERE id=$2"), + expect.anything(), + ); + } finally { + if (oldMin === undefined) delete process.env.QUEUE_STARTUP_JITTER_MIN_JOBS; + else process.env.QUEUE_STARTUP_JITTER_MIN_JOBS = oldMin; + if (oldJitter === undefined) delete process.env.QUEUE_STARTUP_JITTER_MS; + else process.env.QUEUE_STARTUP_JITTER_MS = oldJitter; + } + }); + it("coalesces duplicate keyed jobs instead of inserting queue pressure", async () => { const m = makePool(); const q = createPgQueue(m.pool, async () => undefined); @@ -195,6 +288,78 @@ describe("createPgQueue (durable #977)", () => { ); }); + it("defers due jobs and coalesces a keyed rate-limit retry into the pending duplicate", async () => { + const oldJitter = process.env.QUEUE_STARTUP_JITTER_MS; + process.env.QUEUE_STARTUP_JITTER_MS = "0"; + const rateLimit = new Error("API rate limit exceeded for installation ID 123"); + Object.assign(rateLimit, { + status: 403, + response: { headers: { "retry-after": "120" } }, + }); + const fn = vi.fn().mockImplementation(async (sql: unknown) => { + const q = String(sql); + if (q.includes("SELECT id, payload, priority")) return { rows: [], rowCount: 0 }; + if (q.includes("SELECT id, payload, job_key") && q.includes("status IN")) return { rows: [], rowCount: 0 }; + if (q.includes("WHERE status='processing'")) return { rows: [], rowCount: 0 }; + if (q.includes("UPDATE _selfhost_jobs SET status='processing'")) { + return { + rows: [{ + id: "active", + payload: JSON.stringify({ type: "github-webhook" }), + attempts: 0, + job_key: "github-webhook:ci-completed:jsonbored/gittensory@abc1234#7", + }], + rowCount: 1, + }; + } + if (q.includes("SELECT id, payload, job_key FROM _selfhost_jobs WHERE status='pending' AND run_after<=$1")) { + return { + rows: [{ id: "pending-due", payload: JSON.stringify(msg("agent-regate-pr")), job_key: "agent-regate-pr:jsonbored/gittensory#9" }], + rowCount: 1, + }; + } + if (q.includes("SELECT id FROM _selfhost_jobs WHERE status='pending' AND job_key=$1 AND id<>$2")) { + return { rows: [{ id: "existing" }], rowCount: 1 }; + } + if (q.includes("SELECT id FROM _selfhost_jobs WHERE status='pending' AND job_key=$1 ORDER BY")) { + return { rows: [], rowCount: 0 }; + } + return { rows: [], rowCount: 1 }; + }); + try { + const q = createPgQueue( + { query: fn } as unknown as Pool, + async () => { + throw rateLimit; + }, + { maxRetries: 1, backoffMs: () => 0 }, + ); + await q.init(); + await q.drain(); + await q.binding.send(ciWebhook("after-cooldown"), { delaySeconds: 0 }); + + expect(fn).toHaveBeenCalledWith( + expect.stringContaining("SET run_after=GREATEST(run_after, $1), last_error=COALESCE"), + expect.arrayContaining([expect.any(Number), "github rate-limit cooldown", "pending-due"]), + ); + expect(fn).toHaveBeenCalledWith( + expect.stringContaining("SET run_after=GREATEST(run_after, $1), last_error=$2"), + expect.arrayContaining([expect.any(Number), "API rate limit exceeded for installation ID 123", "existing"]), + ); + expect(fn).toHaveBeenCalledWith( + expect.stringContaining("DELETE FROM _selfhost_jobs WHERE id=$1"), + ["active"], + ); + expect(fn).toHaveBeenCalledWith( + expect.stringContaining("INSERT INTO _selfhost_jobs (payload"), + expect.arrayContaining([expect.stringContaining('"deliveryId":"after-cooldown"'), expect.any(Number)]), + ); + } finally { + if (oldJitter === undefined) delete process.env.QUEUE_STARTUP_JITTER_MS; + else process.env.QUEUE_STARTUP_JITTER_MS = oldJitter; + } + }); + it("opens a shared cooldown after GitHub rate limits so the pump does not claim the next due job", async () => { const m = makePool(); m.enqueueJob("1", { type: "github-webhook" }, 0); @@ -431,7 +596,16 @@ describe("createPgQueue (durable #977)", () => { const m = makePool(); const q = createPgQueue(m.pool, async () => undefined); await q.init(); - m.fn.mockResolvedValueOnce({ rows: [{ name: "gittensory_jobs_processed_total", value: "42" }], rowCount: 1 }); - await expect(q.stats()).resolves.toEqual({ gittensory_jobs_processed_total: 42 }); + m.fn.mockResolvedValueOnce({ + rows: [ + { name: "gittensory_jobs_processed_total", value: "42" }, + { name: "gittensory_jobs_dead_total", value: null }, + ], + rowCount: 2, + }); + await expect(q.stats()).resolves.toEqual({ + gittensory_jobs_processed_total: 42, + gittensory_jobs_dead_total: 0, + }); }); }); diff --git a/test/unit/selfhost-sqlite-queue.test.ts b/test/unit/selfhost-sqlite-queue.test.ts index 06a329bbb4..a415450c94 100644 --- a/test/unit/selfhost-sqlite-queue.test.ts +++ b/test/unit/selfhost-sqlite-queue.test.ts @@ -441,6 +441,44 @@ describe("createSqliteQueue (durable #980)", () => { expect(row.last_error).toContain("AI review did not produce"); }); + it("coalesces a keyed retryable review job into an existing pending duplicate", async () => { + const driver = makeDriver(); + const retryable = new RetryableJobError("AI review did not produce a public summary yet", { + retryAfterMs: 5_000, + retryKind: "ai_review_public_summary_missing", + }); + const key = `github-webhook:ci-completed:jsonbored/gittensory@${"b".repeat(40)}#1629`; + const q = createSqliteQueue( + driver, + async () => { + throw retryable; + }, + { maxRetries: 1, backoffMs: () => 0 }, + ); + driver.query( + "INSERT INTO _selfhost_jobs (payload, status, attempts, run_after, created_at, priority, job_key) VALUES (?, 'pending', 0, 0, 0, 10, ?)", + [JSON.stringify(ciWebhook("ci-active")), key], + ); + driver.query( + "INSERT INTO _selfhost_jobs (payload, status, attempts, run_after, created_at, priority, job_key) VALUES (?, 'pending', 0, ?, 0, 10, ?)", + [JSON.stringify(ciWebhook("ci-existing")), Date.now() + 60_000, key], + ); + + await q.drain(); + + const rows = driver.query( + "SELECT payload, last_error FROM _selfhost_jobs ORDER BY id", + [], + ).rows as Array<{ payload: string; last_error: string | null }>; + expect(rows).toHaveLength(1); + expect(JSON.parse(rows[0]!.payload).deliveryId).toBe("ci-existing"); + expect(rows[0]!.last_error).toContain("AI review did not produce"); + expect(q.stats()).toMatchObject({ + gittensory_jobs_coalesced_total: 1, + gittensory_jobs_deferred_total: 1, + }); + }); + it("SURVIVES A RESTART: a fresh queue over the same DB processes a persisted pending job", async () => { const driver = makeDriver(); const seen: string[] = []; From 71b72cb0937825e46722e8817404eab3e36bc976 Mon Sep 17 00:00:00 2001 From: JSONbored <49853598+JSONbored@users.noreply.github.com> Date: Sun, 28 Jun 2026 20:47:11 -0700 Subject: [PATCH 37/68] fix(webhooks): ignore self-authored CI completions --- src/github/self-authored.ts | 80 ++++++++++++++++++++++++++ src/github/webhook.ts | 19 +----- src/queue/processors.ts | 13 +++++ test/unit/github-self-authored.test.ts | 68 ++++++++++++++++++++++ test/unit/queue.test.ts | 35 +++++++++++ test/unit/webhook.test.ts | 42 ++++++++++++++ 6 files changed, 240 insertions(+), 17 deletions(-) create mode 100644 src/github/self-authored.ts create mode 100644 test/unit/github-self-authored.test.ts diff --git a/src/github/self-authored.ts b/src/github/self-authored.ts new file mode 100644 index 0000000000..619d77e0e9 --- /dev/null +++ b/src/github/self-authored.ts @@ -0,0 +1,80 @@ +import type { GitHubWebhookPayload } from "../types"; + +type GitHubAppRef = { + slug?: string | null; +}; + +type CheckRunWebhookNode = { + app?: GitHubAppRef | null; + check_suite?: { + app?: GitHubAppRef | null; + } | null; +}; + +type CheckSuiteWebhookNode = { + app?: GitHubAppRef | null; +}; + +function normalizeGitHubSlug(value: string | null | undefined): string { + return (value ?? "").trim().toLowerCase(); +} + +function ownAppSlug(env: Env): string { + return normalizeGitHubSlug(env.GITHUB_APP_SLUG); +} + +function appSlugMatches(env: Env, app: GitHubAppRef | null | undefined): boolean { + const expected = ownAppSlug(env); + return expected !== "" && normalizeGitHubSlug(app?.slug) === expected; +} + +function ciCompletionApp( + eventName: "check_run" | "check_suite", + payload: GitHubWebhookPayload, +): GitHubAppRef | null | undefined { + const record = payload as Record; + if (eventName === "check_suite") { + return (record.check_suite as CheckSuiteWebhookNode | undefined)?.app; + } + const checkRun = record.check_run as CheckRunWebhookNode | undefined; + return checkRun?.app ?? checkRun?.check_suite?.app; +} + +export function isSelfAuthoredAppCommentWebhook( + env: Env, + eventName: string, + payload: GitHubWebhookPayload, +): boolean { + if (eventName !== "issue_comment") return false; + if (payload.action !== "created" && payload.action !== "edited") return false; + const slug = ownAppSlug(env); + if (!slug) return false; + const botLogin = `${slug}[bot]`; + return ( + payload.sender?.type === "Bot" && + payload.sender.login?.toLowerCase() === botLogin && + payload.comment?.user?.type === "Bot" && + payload.comment.user.login?.toLowerCase() === botLogin + ); +} + +export function isSelfAuthoredCiCompletionWebhook( + env: Env, + eventName: string, + payload: GitHubWebhookPayload, +): boolean { + if (eventName !== "check_run" && eventName !== "check_suite") return false; + if (payload.action !== "completed") return false; + return appSlugMatches(env, ciCompletionApp(eventName, payload)); +} + +export function isSelfAuthoredWebhookNoise( + env: Env, + eventName: string, + payload: GitHubWebhookPayload, +): boolean { + return ( + isSelfAuthoredAppCommentWebhook(env, eventName, payload) || + isSelfAuthoredCiCompletionWebhook(env, eventName, payload) + ); +} diff --git a/src/github/webhook.ts b/src/github/webhook.ts index 08bef7ad1f..43dc13e91d 100644 --- a/src/github/webhook.ts +++ b/src/github/webhook.ts @@ -3,6 +3,7 @@ import { getWebhookEvent, recordWebhookEvent } from "../db/repositories"; import type { GitHubWebhookPayload, JobMessage } from "../types"; import { sha256Hex, verifyGitHubSignature } from "../utils/crypto"; import { relayVerify } from "../orb/relay"; +import { isSelfAuthoredWebhookNoise } from "./self-authored"; const DEFAULT_MAX_WEBHOOK_BODY_BYTES = 1024 * 1024; @@ -81,7 +82,7 @@ export async function enqueueWebhookByEnv(env: Env, deliveryId: string, eventNam repositoryFullName: payload.repository?.full_name, payloadHash, }; - if (isSelfAuthoredAppCommentWebhook(env, eventName, payload)) { + if (isSelfAuthoredWebhookNoise(env, eventName, payload)) { await recordWebhookEvent(env, { ...eventRow, status: "processed" }); return "ignored"; } @@ -103,22 +104,6 @@ export async function enqueueWebhookByEnv(env: Env, deliveryId: string, eventNam return "queued"; } -function isSelfAuthoredAppCommentWebhook( - env: Env, - eventName: string, - payload: GitHubWebhookPayload, -): boolean { - if (eventName !== "issue_comment") return false; - if (payload.action !== "created" && payload.action !== "edited") return false; - const botLogin = `${env.GITHUB_APP_SLUG}[bot]`.toLowerCase(); - return ( - payload.sender?.type === "Bot" && - payload.sender.login?.toLowerCase() === botLogin && - payload.comment?.user?.type === "Bot" && - payload.comment.user.login?.toLowerCase() === botLogin - ); -} - /** The brokered self-host's relay RECEIVER. The central Orb forwards an event here, HMAC-signed (x-orb-signature- * 256) with THIS container's enrollment secret. We verify with our own ORB_ENROLLMENT_SECRET, then enqueue the * event exactly like a GitHub webhook (the body IS a GitHub webhook payload; only the transport differs). */ diff --git a/src/queue/processors.ts b/src/queue/processors.ts index 2e2d8ae398..48b35f9d99 100644 --- a/src/queue/processors.ts +++ b/src/queue/processors.ts @@ -106,6 +106,7 @@ import { isGitHubRateLimitedError, isForeignAppInstallation, } from "../github/app"; +import { isSelfAuthoredCiCompletionWebhook } from "../github/self-authored"; import { AGENT_COMMAND_COMMENT_MARKER, createOrUpdateAgentCommandComment, @@ -1932,6 +1933,18 @@ async function maybeReReviewOnCiCompletion( const repoFullName = payload.repository?.full_name; const installationId = getInstallationId(payload); if (!repoFullName || !installationId) return false; + if (isSelfAuthoredCiCompletionWebhook(env, eventName, payload)) { + await recordWebhookEvent(env, { + deliveryId, + eventName, + action: payload.action, + installationId, + repositoryFullName: repoFullName, + payloadHash: "processed", + status: "processed", + }); + return true; + } const node = (payload as Record)[eventName] as | { pull_requests?: Array<{ number?: number | null }> } | undefined; diff --git a/test/unit/github-self-authored.test.ts b/test/unit/github-self-authored.test.ts new file mode 100644 index 0000000000..27e3282031 --- /dev/null +++ b/test/unit/github-self-authored.test.ts @@ -0,0 +1,68 @@ +import { describe, expect, it } from "vitest"; +import { + isSelfAuthoredAppCommentWebhook, + isSelfAuthoredCiCompletionWebhook, + isSelfAuthoredWebhookNoise, +} from "../../src/github/self-authored"; +import type { GitHubWebhookPayload } from "../../src/types"; +import { createTestEnv } from "../helpers/d1"; + +describe("self-authored GitHub webhook detection", () => { + it("recognizes only comments authored by this GitHub App bot", () => { + const env = createTestEnv({ GITHUB_APP_SLUG: "gittensory-orb" }); + const payload = { + action: "edited", + sender: { login: "gittensory-orb[bot]", type: "Bot" }, + comment: { user: { login: "gittensory-orb[bot]", type: "Bot" } }, + } as GitHubWebhookPayload; + + expect(isSelfAuthoredAppCommentWebhook(env, "issue_comment", payload)).toBe(true); + expect(isSelfAuthoredWebhookNoise(env, "issue_comment", payload)).toBe(true); + expect(isSelfAuthoredAppCommentWebhook(env, "pull_request", payload)).toBe(false); + expect(isSelfAuthoredAppCommentWebhook(env, "issue_comment", { ...payload, action: "deleted" })).toBe(false); + expect(isSelfAuthoredAppCommentWebhook(createTestEnv({ GITHUB_APP_SLUG: "" }), "issue_comment", payload)).toBe(false); + expect( + isSelfAuthoredAppCommentWebhook(env, "issue_comment", { + ...payload, + sender: { login: "someone-else[bot]", type: "Bot" }, + }), + ).toBe(false); + }); + + it("recognizes self-authored check suites and check runs without matching other CI events", () => { + const env = createTestEnv({ GITHUB_APP_SLUG: "gittensory-orb" }); + + expect( + isSelfAuthoredCiCompletionWebhook(env, "check_suite", { + action: "completed", + check_suite: { app: { slug: "gittensory-orb" } }, + } as never), + ).toBe(true); + expect( + isSelfAuthoredCiCompletionWebhook(env, "check_run", { + action: "completed", + check_run: { app: { slug: "gittensory-orb" } }, + } as never), + ).toBe(true); + expect( + isSelfAuthoredCiCompletionWebhook(env, "check_run", { + action: "completed", + check_run: { check_suite: { app: { slug: "gittensory-orb" } } }, + } as never), + ).toBe(true); + expect( + isSelfAuthoredCiCompletionWebhook(env, "check_run", { + action: "rerequested", + check_run: { app: { slug: "gittensory-orb" } }, + } as never), + ).toBe(false); + expect( + isSelfAuthoredCiCompletionWebhook(env, "check_run", { + action: "completed", + check_run: { app: { slug: "github-actions" } }, + } as never), + ).toBe(false); + expect(isSelfAuthoredCiCompletionWebhook(env, "pull_request", { action: "completed" })).toBe(false); + expect(isSelfAuthoredWebhookNoise(env, "pull_request", { action: "completed" })).toBe(false); + }); +}); diff --git a/test/unit/queue.test.ts b/test/unit/queue.test.ts index af580e3889..e0d2edb104 100644 --- a/test/unit/queue.test.ts +++ b/test/unit/queue.test.ts @@ -11,6 +11,7 @@ import { getContributorEvidence, getAgentRun, getContributorScoringProfile, + getWebhookEvent, getInstallation, getLatestUpstreamRulesetSnapshot, getPullRequest, @@ -926,6 +927,40 @@ describe("queue processors", () => { expect(checkRunsFetched).toBe(true); }); + it("drops already-enqueued self-authored app CI completions without re-reviewing", async () => { + const env = createTestEnv({ + GITHUB_APP_SLUG: "gittensory-orb", + GITTENSORY_REVIEW_REPOS: "owner/agent-repo", + }); + let fetchCount = 0; + vi.stubGlobal("fetch", async () => { + fetchCount += 1; + return Response.json({}); + }); + + await processJob(env, { + type: "github-webhook", + deliveryId: "self-check-suite-queued", + eventName: "check_suite", + payload: { + action: "completed", + repository: { name: "agent-repo", full_name: "owner/agent-repo", private: false, owner: { login: "owner" } }, + installation: { id: 9001 }, + check_suite: { + head_sha: "a7", + pull_requests: [{ number: 7 }], + app: { slug: "gittensory-orb" }, + }, + } as never, + }); + + expect(fetchCount).toBe(0); + await expect(getWebhookEvent(env, "self-check-suite-queued")).resolves.toMatchObject({ + status: "processed", + payloadHash: "processed", + }); + }); + it("#4 stale-surface repair: a rebased PR resyncs + re-reviews at the new head, and the marker survives the resync", async () => { const env = createTestEnv({ GITHUB_APP_PRIVATE_KEY: await generatePrivateKeyPem() }); await upsertInstallation(env, { action: "created", installation: { id: 9001, account: { login: "owner", id: 1, type: "Organization" }, target_type: "Organization", repository_selection: "selected", permissions: {}, events: [] } }); diff --git a/test/unit/webhook.test.ts b/test/unit/webhook.test.ts index d7cea7ab54..ec49c35a25 100644 --- a/test/unit/webhook.test.ts +++ b/test/unit/webhook.test.ts @@ -191,6 +191,48 @@ describe("github webhook queue isolation (#audit-webhook-queue)", () => { const event = await getWebhookEvent(env, "self-comment-ignore-1"); expect(event?.status).toBe("processed"); }); + + it("drops self-authored app CI completion webhooks before they add queue pressure", async () => { + const env = createTestEnv({ GITHUB_APP_SLUG: "gittensory-orb" }); + let webhookSends = 0; + env.WEBHOOKS = { send: async () => void (webhookSends += 1) } as unknown as typeof env.WEBHOOKS; + const rawBody = JSON.stringify({ + action: "completed", + repository: { full_name: "JSONbored/gittensory" }, + installation: { id: 1 }, + check_suite: { + head_sha: "abc123", + pull_requests: [], + app: { slug: "gittensory-orb" }, + }, + }); + const signature = await signWebhook(rawBody, env.GITHUB_WEBHOOK_SECRET); + const request = new Request("https://example.com/webhook", { method: "POST", body: rawBody }); + const headers: Record = { + "x-github-delivery": "self-check-suite-ignore-1", + "x-github-event": "check_suite", + "x-hub-signature-256": signature, + }; + const context = { + req: { + raw: request, + header(name: string) { + return headers[name.toLowerCase()] ?? null; + }, + }, + env, + json(payload: unknown, status?: number) { + return Response.json(payload, status === undefined ? undefined : { status }); + }, + } as unknown as Context<{ Bindings: Env }>; + + const response = await handleGitHubWebhook(context); + expect(response.status).toBe(202); + await expect(response.json()).resolves.toMatchObject({ status: "ignored" }); + expect(webhookSends).toBe(0); + const event = await getWebhookEvent(env, "self-check-suite-ignore-1"); + expect(event?.status).toBe("processed"); + }); }); describe("handleOrbRelay (brokered self-host relay receiver)", () => { From fde78a7d0f90b9db878b77be0f278ccaddd8e68d Mon Sep 17 00:00:00 2001 From: JSONbored <49853598+JSONbored@users.noreply.github.com> Date: Sun, 28 Jun 2026 21:20:59 -0700 Subject: [PATCH 38/68] test(queue): cover retry audit fallback paths --- test/unit/github-app.test.ts | 61 ++++++++ test/unit/queue.test.ts | 185 ++++++++++++++++++++++-- test/unit/selfhost-queue-common.test.ts | 39 ++++- 3 files changed, 275 insertions(+), 10 deletions(-) diff --git a/test/unit/github-app.test.ts b/test/unit/github-app.test.ts index 047ce05ebd..d2953e0ccb 100644 --- a/test/unit/github-app.test.ts +++ b/test/unit/github-app.test.ts @@ -187,6 +187,67 @@ describe("GitHub check runs", () => { expect(rejectedReads).toBe(1); }); + it("retries a rejected cached installation token when cache eviction fails", async () => { + const privateKey = await generatePrivateKeyPem(); + let gets = 0; + let evictionWrites = 0; + setInstallationTokenStore({ + get: async () => { + gets += 1; + if (gets <= 2) + return { + token: "stale-token", + expiresAtMs: Date.now() + 60 * 60_000, + }; + return null; + }, + set: async (_installationId, value) => { + if (value.token === "") { + evictionWrites += 1; + throw new Error("token cache unavailable"); + } + }, + }); + let mints = 0; + let rejectedReads = 0; + vi.stubGlobal("fetch", async (input: RequestInfo | URL, init?: RequestInit) => { + const url = input.toString(); + if (url.includes("/access_tokens")) { + mints += 1; + return Response.json({ + token: "fresh-token", + expires_at: new Date(Date.now() + 60 * 60_000).toISOString(), + }); + } + const auth = new Headers(init?.headers).get("authorization") ?? ""; + if (url.includes("/commits/stale-head/check-runs") && auth.includes("stale-token")) { + rejectedReads += 1; + return Response.json({ message: "Bad credentials" }, { status: 401 }); + } + if (url.includes("/commits/stale-head/check-runs")) { + expect(auth).toContain("fresh-token"); + return Response.json({ total_count: 0, check_runs: [] }); + } + if (url.includes("/check-runs") && init?.method === "POST") { + expect(auth).toContain("fresh-token"); + return Response.json({ id: 557, html_url: "https://github.com/checks/557" }, { status: 201 }); + } + return new Response("not found", { status: 404 }); + }); + + const result = await createOrUpdatePendingGateCheckRun( + createTestEnv({ GITHUB_APP_PRIVATE_KEY: privateKey }), + 557, + "JSONbored/gittensory", + gateAdvisory("stale-head"), + ); + + expect(result).toMatchObject({ kind: "published", id: 557 }); + expect(evictionWrites).toBe(1); + expect(mints).toBe(1); + expect(rejectedReads).toBe(1); + }); + it("single-flights concurrent cold-cache mints for one install (no thundering herd)", async () => { const privateKey = await generatePrivateKeyPem(); let mints = 0; diff --git a/test/unit/queue.test.ts b/test/unit/queue.test.ts index e0d2edb104..a30d8c7aa0 100644 --- a/test/unit/queue.test.ts +++ b/test/unit/queue.test.ts @@ -1149,6 +1149,85 @@ describe("queue processors", () => { expect(commentBodies.some((body) => !body.includes("is reviewing"))).toBe(true); }); + it("continues to final verdict when the reviewing placeholder audit write fails", async () => { + const originalRecordAuditEvent = repositoriesModule.recordAuditEvent; + const auditSpy = vi.spyOn(repositoriesModule, "recordAuditEvent").mockImplementation(async (auditEnv, event) => { + if (event.eventType === "github_app.reviewing_placeholder_failed") + throw new Error("D1 audit failed"); + await originalRecordAuditEvent(auditEnv, event); + }); + const env = createTestEnv({ + GITHUB_APP_PRIVATE_KEY: await generatePrivateKeyPem(), + AI: { + run: async () => ({ response: JSON.stringify({ assessment: "Looks fine.", blockers: [], nits: [], suggestions: [] }) }), + } as unknown as Ai, + AI_SUMMARIES_ENABLED: "true", + AI_PUBLIC_COMMENTS_ENABLED: "true", + AI_DAILY_NEURON_BUDGET: "100000", + }); + await persistRegistrySnapshot( + env, + normalizeRegistryPayload( + { "JSONbored/gittensory": { emission_share: 0.01, issue_discovery_share: 0 } }, + { kind: "raw-github", url: "https://example.test" }, + "2026-05-23T00:00:00.000Z", + ), + ); + await upsertRepositoryFromGitHub(env, { name: "gittensory", full_name: "JSONbored/gittensory", private: false, owner: { login: "JSONbored" } }, 123); + await upsertRepositorySettings(env, { + repoFullName: "JSONbored/gittensory", + commentMode: "all_prs", + publicSurface: "comment_only", + autoLabelEnabled: false, + checkRunMode: "off", + gateCheckMode: "enabled", + aiReviewMode: "block", + gatePack: "oss-anti-slop", + }); + const postedBodies: string[] = []; + let postAttempts = 0; + vi.stubGlobal("fetch", async (input: RequestInfo | URL, init?: RequestInit) => { + const url = input.toString(); + const method = init?.method ?? "GET"; + if (url.includes("/access_tokens")) return Response.json({ token: "installation-token" }); + if (url.includes("/pulls/47/files")) return Response.json([{ filename: "src/a.ts", status: "modified", additions: 1, deletions: 0, changes: 1, patch: "@@\n+export const ok = true;" }]); + if (url.endsWith("/pulls/47")) return Response.json({ number: 47, title: "Clean PR", state: "open", user: { login: "contributor" }, head: { sha: "a47" }, labels: [], body: "Closes #1", mergeable_state: "clean" }); + if (url.includes("/commits/a47/check-runs")) return Response.json({ total_count: 0, check_runs: [] }); + if (url.includes("/commits/a47/status")) return Response.json({ state: "success", statuses: [] }); + if (url.includes("/issues/1")) return Response.json({ number: 1, title: "Issue", state: "open", labels: [], user: { login: "reporter" } }); + if (url.includes("/issues/47/comments") && method === "GET") return Response.json([]); + if (url.includes("/issues/47/comments") && method === "POST") { + postAttempts += 1; + const body = String((JSON.parse(String(init?.body ?? "{}")) as { body?: string }).body ?? ""); + if (postAttempts === 1) return new Response(JSON.stringify({ message: "temporary comment failure" }), { status: 500 }); + postedBodies.push(body); + return Response.json({ id: 47 }, { status: 201 }); + } + if (url.includes("/branches/")) return Response.json({ protected: false, protection: { required_status_checks: { contexts: [] } } }); + return Response.json({}); + }); + + await processJob(env, { + type: "github-webhook", + deliveryId: "reviewing-placeholder-audit-fails", + eventName: "pull_request", + payload: { + action: "opened", + installation: { id: 123, account: { login: "JSONbored", id: 1, type: "User" } }, + repository: { name: "gittensory", full_name: "JSONbored/gittensory", private: false, owner: { login: "JSONbored" } }, + pull_request: { number: 47, title: "Clean PR", state: "open", user: { login: "contributor" }, head: { sha: "a47" }, labels: [], body: "Closes #1" }, + }, + }); + + expect(postAttempts).toBeGreaterThanOrEqual(2); + expect(postedBodies.some((body) => !body.includes("is reviewing"))).toBe(true); + expect(auditSpy).toHaveBeenCalledWith( + env, + expect.objectContaining({ eventType: "github_app.reviewing_placeholder_failed" }), + ); + auditSpy.mockRestore(); + }); + it("posts the 🟪 reviewing placeholder for non-AI comment refreshes, then overwrites it with the verdict", async () => { let aiCalls = 0; const env = createTestEnv({ @@ -1351,6 +1430,81 @@ describe("queue processors", () => { expect(audit?.n).toBe(1); }); + it("keeps re-gate PR jobs retryable when AI review produces no public summary and audit storage fails", async () => { + const originalRecordAuditEvent = repositoriesModule.recordAuditEvent; + const auditSpy = vi.spyOn(repositoriesModule, "recordAuditEvent").mockImplementation(async (auditEnv, event) => { + if (event.eventType === "github_app.ai_review_public_summary_missing") + throw new Error("D1 audit failed"); + await originalRecordAuditEvent(auditEnv, event); + }); + const env = createTestEnv({ + GITHUB_APP_PRIVATE_KEY: await generatePrivateKeyPem(), + AI: { + run: async () => ({ response: "not-json" }), + } as unknown as Ai, + AI_SUMMARIES_ENABLED: "true", + AI_PUBLIC_COMMENTS_ENABLED: "true", + AI_DAILY_NEURON_BUDGET: "100000", + }); + await persistRegistrySnapshot( + env, + normalizeRegistryPayload( + { "JSONbored/gittensory": { emission_share: 0.01, issue_discovery_share: 0 } }, + { kind: "raw-github", url: "https://example.test" }, + "2026-05-23T00:00:00.000Z", + ), + ); + await upsertInstallation(env, { action: "created", installation: { id: 123, account: { login: "JSONbored", id: 1, type: "User" }, target_type: "User", repository_selection: "selected", permissions: {}, events: [] } }); + await upsertRepositoryFromGitHub(env, { name: "gittensory", full_name: "JSONbored/gittensory", private: false, owner: { login: "JSONbored" } }, 123); + await upsertRepositorySettings(env, { + repoFullName: "JSONbored/gittensory", + commentMode: "all_prs", + publicSurface: "comment_only", + autoLabelEnabled: false, + checkRunMode: "off", + gateCheckMode: "enabled", + aiReviewMode: "block", + gatePack: "oss-anti-slop", + }); + await upsertPullRequestFromGitHub(env, "JSONbored/gittensory", { number: 48, title: "Clean PR", state: "open", user: { login: "contributor" }, head: { sha: "a48" }, labels: [], body: "Closes #1" }); + const commentBodies: string[] = []; + vi.stubGlobal("fetch", async (input: RequestInfo | URL, init?: RequestInit) => { + const url = input.toString(); + const method = init?.method ?? "GET"; + if (url.includes("/access_tokens")) return Response.json({ token: "installation-token" }); + if (url.includes("/pulls/48/files")) return Response.json([{ filename: "src/a.ts", status: "modified", additions: 1, deletions: 0, changes: 1, patch: "@@\n+export const ok = true;" }]); + if (url.endsWith("/pulls/48")) return Response.json({ number: 48, title: "Clean PR", state: "open", user: { login: "contributor" }, head: { sha: "a48" }, labels: [], body: "Closes #1", mergeable_state: "clean" }); + if (url.includes("/commits/a48/check-runs")) return Response.json({ total_count: 0, check_runs: [] }); + if (url.includes("/commits/a48/status")) return Response.json({ state: "success", statuses: [] }); + if (url.includes("/issues/48/comments") && method === "GET") return Response.json([]); + if (url.includes("/issues/48/comments") && method === "POST") { + commentBodies.push(String((JSON.parse(String(init?.body ?? "{}")) as { body?: string }).body ?? "")); + return Response.json({ id: 48 }, { status: 201 }); + } + if (url.includes("/issues/1")) return Response.json({ number: 1, title: "Issue", state: "open", labels: [], user: { login: "reporter" } }); + if (url.includes("/branches/")) return Response.json({ protected: false, protection: { required_status_checks: { contexts: [] } } }); + return Response.json({}); + }); + + await expect( + processJob(env, { + type: "agent-regate-pr", + deliveryId: "regate-ai-summary-missing-audit-fails", + repoFullName: "JSONbored/gittensory", + prNumber: 48, + installationId: 123, + }), + ).rejects.toThrow(/public summary/i); + + expect(commentBodies).toHaveLength(1); + expect(commentBodies[0]).toContain("is reviewing"); + expect(auditSpy).toHaveBeenCalledWith( + env, + expect.objectContaining({ eventType: "github_app.ai_review_public_summary_missing" }), + ); + auditSpy.mockRestore(); + }); + it("agent re-gate sweep re-reviews each stale open PR (installation id) and swallows a failing re-review", async () => { const env = createTestEnv({}); await upsertInstallation(env, { action: "created", installation: { id: 9001, account: { login: "owner", id: 1, type: "Organization" }, target_type: "Organization", repository_selection: "selected", permissions: {}, events: [] } }); @@ -3715,6 +3869,18 @@ describe("queue processors", () => { "", "- [x] Re-run Gittensory review", ].join("\n"); + env.SELFHOST_TRANSIENT_CACHE = { + get: async () => { + throw new Error("Redis unavailable"); + }, + set: async () => undefined, + }; + const originalRecordAuditEvent = repositoriesModule.recordAuditEvent; + const auditSpy = vi.spyOn(repositoriesModule, "recordAuditEvent").mockImplementation(async (auditEnv, event) => { + if (event.eventType === "github_app.pr_panel_retrigger_deferred") + throw new Error("D1 audit failed"); + await originalRecordAuditEvent(auditEnv, event); + }); let commentPatches = 0; vi.stubGlobal("fetch", async (input: RequestInfo | URL, init?: RequestInit) => { const url = input.toString(); @@ -3749,15 +3915,16 @@ describe("queue processors", () => { }); expect(commentPatches).toBe(0); - const audit = await env.DB.prepare("select event_type, actor, target_key, outcome from audit_events where event_type = ?") - .bind("github_app.pr_panel_retrigger_deferred") - .first<{ event_type: string; actor: string; target_key: string; outcome: string }>(); - expect(audit).toMatchObject({ - event_type: "github_app.pr_panel_retrigger_deferred", - actor: "maintainer", - target_key: "JSONbored/gittensory#46", - outcome: "queued", - }); + expect(auditSpy).toHaveBeenCalledWith( + env, + expect.objectContaining({ + eventType: "github_app.pr_panel_retrigger_deferred", + actor: "maintainer", + targetKey: "JSONbored/gittensory#46", + outcome: "queued", + }), + ); + auditSpy.mockRestore(); }); it("refreshes the PR's files on a manual rerun so the slop/manifest gate evaluates the current diff", async () => { diff --git a/test/unit/selfhost-queue-common.test.ts b/test/unit/selfhost-queue-common.test.ts index 7b433bb9a3..8d02a71380 100644 --- a/test/unit/selfhost-queue-common.test.ts +++ b/test/unit/selfhost-queue-common.test.ts @@ -1,6 +1,7 @@ -import { describe, expect, it } from "vitest"; +import { describe, expect, it, vi } from "vitest"; import { githubRateLimitRetryDelayMs, + jobCoalesceKey, jobPriority, nonConsumingRetryDelayMs, } from "../../src/selfhost/queue-common"; @@ -46,6 +47,42 @@ describe("self-host queue common helpers", () => { ).toBe(10); }); + it("fails closed when a malformed webhook payload reaches priority parsing", () => { + const raw = payload({ type: "github-webhook" }); + const parse = vi.spyOn(JSON, "parse"); + parse + .mockImplementationOnce(() => ({ type: "github-webhook" })) + .mockImplementationOnce(() => { + throw new Error("malformed webhook payload"); + }); + + expect(jobPriority(raw)).toBe(0); + parse.mockRestore(); + }); + + it("coalesces CI-completion webhooks with sorted pull numbers", () => { + expect( + jobCoalesceKey( + payload({ + type: "github-webhook", + eventName: "check_suite", + payload: { + action: "completed", + repository: { full_name: "JSONbored/Gittensory" }, + check_suite: { + head_sha: "abc1234", + pull_requests: [{ number: 12 }, { number: 3 }, { number: 7 }], + }, + }, + }), + ), + ).toBe("github-webhook:ci-completed:jsonbored/gittensory@abc1234#3,7,12"); + }); + + it("returns no coalesce key for malformed payloads", () => { + expect(jobCoalesceKey("not-json")).toBeNull(); + }); + it("extracts retry delays from GitHub rate-limit errors", () => { expect(githubRateLimitRetryDelayMs(null)).toBeNull(); expect(githubRateLimitRetryDelayMs({ status: 403, message: "Forbidden" })).toBeNull(); From 926d2f6d65d29b88808600cb5fa4388691765760 Mon Sep 17 00:00:00 2001 From: JSONbored <49853598+JSONbored@users.noreply.github.com> Date: Sun, 28 Jun 2026 21:34:44 -0700 Subject: [PATCH 39/68] ci(codecov): upload explicit coverage reports --- .github/workflows/ci.yml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 67d75441e7..97c72a56f1 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -177,6 +177,10 @@ jobs: token: ${{ secrets.CODECOV_TOKEN }} files: ./reports/junit/vitest-shard-${{ matrix.shard }}.xml report_type: test_results + disable_search: true + override_branch: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.ref || github.ref_name }} + override_commit: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }} + override_pr: ${{ github.event_name == 'pull_request' && github.event.pull_request.number || '' }} fail_ci_if_error: false # Merge the 3 shard lcovs into one complete report and upload to Codecov ONCE. @@ -238,6 +242,10 @@ jobs: with: token: ${{ secrets.CODECOV_TOKEN }} files: ./merged-lcov/shard-1.info,./merged-lcov/shard-2.info,./merged-lcov/shard-3.info + disable_search: true + override_branch: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.ref || github.ref_name }} + override_commit: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }} + override_pr: ${{ github.event_name == 'pull_request' && github.event.pull_request.number || '' }} fail_ci_if_error: false # Worker-pool runtime tests (separate vitest config); split out of `test` so it From 2600da99a5acf0b87246cb0cc4e4536bc4cc8de2 Mon Sep 17 00:00:00 2001 From: JSONbored <49853598+JSONbored@users.noreply.github.com> Date: Sun, 28 Jun 2026 21:57:27 -0700 Subject: [PATCH 40/68] fix(gate): wait for active actions suites --- src/github/backfill.ts | 22 +++++++++++----------- test/unit/backfill.test.ts | 25 +++++++++++++++++++++---- 2 files changed, 32 insertions(+), 15 deletions(-) diff --git a/src/github/backfill.ts b/src/github/backfill.ts index 516a804e16..5c9eb0907c 100644 --- a/src/github/backfill.ts +++ b/src/github/backfill.ts @@ -2098,15 +2098,14 @@ export async function fetchLiveCiAggregate( } } - // FOLD-ALL hardening (#ci-foldall-checksuites): when branch protection is UNREADABLE (no `administration:read` - // ⇒ requiredContexts null ⇒ fold-all), the check-run/status scan above can read "passed" for a fork PR whose - // required workflow is AWAITING APPROVAL — its check-RUNS don't exist yet (the workflow never ran), so only the - // always-on third-party checks are seen and nothing fails or pends. Read the check-SUITES too: a GitHub-Actions - // suite still `queued`/`requested`/`waiting`/`in_progress` (not `completed`) means the first-party CI has NOT - // run, so hold (pending) instead of certifying a never-run workflow as green. Enforce-required mode already - // catches this via the absent-context guard above, so this runs ONLY in fold-all (one extra call on the degraded - // path), and ONLY when we would otherwise certify "passed" (no failure, nothing else pending). - if (!enforceRequiredOnly && headSha && failingDetails.length === 0 && !anyPending && !checkRunsIncomplete && !statusIncomplete) { + // Check-suite hardening (#ci-foldall-checksuites / #dependent-ci-materialization): the check-run/status scan can + // read "settled" before GitHub materializes downstream jobs whose `needs:` dependencies just completed + // (`coverage-upload` and then `validate` are the common shape). Read the check-SUITES too before certifying a + // commit settled: a GitHub-Actions suite still `queued`/`requested`/`waiting`/`in_progress` means first-party CI + // has not finished, even if every currently-visible check-run is completed. This runs only when the cheaper + // sources found no failure, no pending check, and no incomplete page, so it does not add a call to already-pending + // or already-failing PRs. + if (headSha && failingDetails.length === 0 && !anyPending && !anyVisiblePending && !checkRunsIncomplete && !statusIncomplete) { const suitesResult = await githubJsonWithHeaders<{ check_suites?: Array<{ status?: string | null; app?: { slug?: string | null } | null }> }>( env, repoFullName, @@ -2121,9 +2120,10 @@ export async function fetchLiveCiAggregate( if (!suitesResult) { // total === 0 means the commit has NO checks at all → genuinely unverified (no CI), not a missing first-party // run, so leave it; only pend when checks DO exist but none of them is a confirmed first-party run. - if (!sawFirstPartyCheckRun && total > 0) anyPending = true; + if (!enforceRequiredOnly && !sawFirstPartyCheckRun && total > 0) anyPending = true; } else if ((suitesResult.data.check_suites ?? []).some((suite) => (suite.app?.slug ?? "").toLowerCase() === "github-actions" && (suite.status ?? "").toLowerCase() !== "completed")) { - anyPending = true; // a first-party GitHub Actions workflow has not completed (e.g. a fork PR awaiting approval) + anyPending = true; // a first-party GitHub Actions workflow has not completed (or downstream jobs are pending materialization) + anyVisiblePending = true; } } diff --git a/test/unit/backfill.test.ts b/test/unit/backfill.test.ts index 2ab64a3d20..2af1d6d396 100644 --- a/test/unit/backfill.test.ts +++ b/test/unit/backfill.test.ts @@ -3081,20 +3081,37 @@ describe("GitHub backfill", () => { expect(aggregate.ciState).toBe("passed"); }); - it("ENFORCE-required mode does NOT consult check-suites (the absent-context guard already handles it)", async () => { + it("ENFORCE-required mode waits when the GitHub Actions suite is still materializing downstream jobs", async () => { const env = createTestEnv({ GITHUB_PUBLIC_TOKEN: "public-token" }); let suitesFetched = false; vi.stubGlobal("fetch", async (input: RequestInfo | URL) => { const url = input.toString(); - if (url.includes("/check-suites?")) suitesFetched = true; + if (url.includes("/check-suites?")) { + suitesFetched = true; + return Response.json({ check_suites: [{ status: "in_progress", app: { slug: "github-actions" } }] }); + } if (url.includes("/check-runs?")) return Response.json({ check_runs: [{ name: "test", status: "completed", conclusion: "success" }] }); if (url.includes("/status?")) return Response.json({ statuses: [] }); return new Response("not found", { status: 404 }); }); - // Required = {test} and it passed → passed; the check-suites call is never made in enforce-required mode. + const aggregate = await fetchLiveCiAggregate(env, "JSONbored/gittensory", "abc123", "public-token", new Set(["test"])); + expect(aggregate.ciState).toBe("pending"); + expect(aggregate.hasPending).toBe(true); + expect(suitesFetched).toBe(true); + }); + + it("ENFORCE-required mode does not over-pend when check-suites are unreadable after required checks passed", async () => { + const env = createTestEnv({ GITHUB_PUBLIC_TOKEN: "public-token" }); + vi.stubGlobal("fetch", async (input: RequestInfo | URL) => { + const url = input.toString(); + if (url.includes("/check-runs?")) return Response.json({ check_runs: [{ name: "test", status: "completed", conclusion: "success", app: { slug: "github-actions" } }] }); + if (url.includes("/status?")) return Response.json({ statuses: [] }); + if (url.includes("/check-suites?")) return new Response("forbidden", { status: 403 }); + return new Response("not found", { status: 404 }); + }); const aggregate = await fetchLiveCiAggregate(env, "JSONbored/gittensory", "abc123", "public-token", new Set(["test"])); expect(aggregate.ciState).toBe("passed"); - expect(suitesFetched).toBe(false); + expect(aggregate.hasPending).toBe(false); }); it("fold-all: tolerates malformed check-suites (missing app / missing status) without throwing", async () => { From 41917193a1f2b3bfeea08ca54b1635a010121de6 Mon Sep 17 00:00:00 2001 From: JSONbored <49853598+JSONbored@users.noreply.github.com> Date: Sun, 28 Jun 2026 22:15:43 -0700 Subject: [PATCH 41/68] fix(selfhost): repair subscription cli path --- src/selfhost/ai.ts | 24 ++++++++++++++++++++++++ src/server.ts | 3 ++- test/unit/selfhost-ai.test.ts | 18 +++++++++++++----- 3 files changed, 39 insertions(+), 6 deletions(-) diff --git a/src/selfhost/ai.ts b/src/selfhost/ai.ts index b1daba2a45..3c2dfa417f 100644 --- a/src/selfhost/ai.ts +++ b/src/selfhost/ai.ts @@ -10,6 +10,7 @@ import type { CombineStrategy, OnMerge } from "../services/ai-review"; import { isConfiguredSelfHostProvider, resolveConfiguredProviderNames } from "./ai-config"; export { assertNoLegacySharedAiEnv } from "./ai-config"; import { incr } from "./metrics"; +import { delimiter } from "node:path"; interface AiRunOptions { messages?: Array<{ role: string; content: string }>; @@ -208,6 +209,28 @@ const SUBSCRIPTION_CLI_ENV_ALLOWLIST = [ "no_proxy", ] as const; +const DEFAULT_SUBSCRIPTION_CLI_BIN_DIR = "/home/node/.npm-global/bin"; + +function normalizeCliPathDir(value: string | undefined): string | undefined { + const trimmed = value?.trim(); + if (!trimmed) return undefined; + return trimmed.replace(/\/+$/, ""); +} + +export function resolveSubscriptionCliPath(parent: Record): string { + const prefixBin = normalizeCliPathDir(parent.NPM_CONFIG_PREFIX); + const prepend = [prefixBin ? `${prefixBin}/bin` : undefined, DEFAULT_SUBSCRIPTION_CLI_BIN_DIR].filter((v): v is string => Boolean(v)); + const seen = new Set(); + const parts: string[] = []; + for (const part of [...prepend, ...(parent.PATH ?? "").split(delimiter)]) { + const trimmed = part.trim(); + if (!trimmed || seen.has(trimmed)) continue; + seen.add(trimmed); + parts.push(trimmed); + } + return parts.join(delimiter); +} + export function subscriptionCliEnv( parent: Record, extra: Record = {}, @@ -220,6 +243,7 @@ export function subscriptionCliEnv( for (const [key, value] of Object.entries(extra)) { if (value !== undefined) child[key] = value; } + child.PATH = resolveSubscriptionCliPath(parent); return child; } diff --git a/src/server.ts b/src/server.ts index 90df485d05..c91a8f4b74 100644 --- a/src/server.ts +++ b/src/server.ts @@ -18,6 +18,7 @@ import { createSelfHostAi, resolveAiReviewerPlan, resolveRequiredCliProviders, + resolveSubscriptionCliPath, } from "./selfhost/ai"; import { cookieValue, @@ -339,7 +340,7 @@ async function main(): Promise { // Fail-LOUD preflight (#1566): a CLI-subscription provider (claude-code/codex) reviews by spawning the CLI as a // subprocess; if the binary is absent (image built without INSTALL_AI_CLIS=true) the spawn ENOENTs and EVERY AI // review silently degrades to "no usable output". Shout at boot so the misconfig is obvious, never invisible. - const pathDirs = (process.env.PATH ?? "").split(delimiter); + const pathDirs = resolveSubscriptionCliPath(process.env).split(delimiter); for (const { provider, cli } of resolveRequiredCliProviders(process.env)) { if (pathDirs.some((d) => d && existsSync(join(d, cli)))) continue; console.error( diff --git a/test/unit/selfhost-ai.test.ts b/test/unit/selfhost-ai.test.ts index fa8ce19412..e650d31c56 100644 --- a/test/unit/selfhost-ai.test.ts +++ b/test/unit/selfhost-ai.test.ts @@ -1,8 +1,8 @@ import { chmodSync, mkdtempSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; -import { join } from "node:path"; +import { delimiter, join } from "node:path"; import { afterEach, describe, expect, it, vi } from "vitest"; -import { assertNoLegacySharedAiEnv, buildProvider, claudeErrorStatus, createAnthropicAi, createChainAi, createClaudeCodeAi, createCodexAi, createOpenAiCompatibleAi, createSelfHostAi, extractCliText, extractCliUsage, resolveAiReviewerPlan, resolveClaudeCliTimeoutMs, resolveCodexCliTimeoutMs, resolveCodexEffort, resolveEffort, resolveModel, resolveProviderNames, resolveRequiredCliProviders, redactSecrets, routeProviders, subscriptionCliEnv } from "../../src/selfhost/ai"; +import { assertNoLegacySharedAiEnv, buildProvider, claudeErrorStatus, createAnthropicAi, createChainAi, createClaudeCodeAi, createCodexAi, createOpenAiCompatibleAi, createSelfHostAi, extractCliText, extractCliUsage, resolveAiReviewerPlan, resolveClaudeCliTimeoutMs, resolveCodexCliTimeoutMs, resolveCodexEffort, resolveEffort, resolveModel, resolveProviderNames, resolveRequiredCliProviders, resolveSubscriptionCliPath, redactSecrets, routeProviders, subscriptionCliEnv } from "../../src/selfhost/ai"; import { labelSelfHostReviewerModel } from "../../src/selfhost/ai-config"; import { renderMetrics, resetMetrics } from "../../src/selfhost/metrics"; @@ -396,11 +396,19 @@ describe("branch coverage — defaults + edge inputs", () => { describe("subscriptionCliEnv (allowlist + extra-override arms)", () => { it("copies only allowlisted parent vars and drops everything else", () => { const child = subscriptionCliEnv({ PATH: "/bin", HOME: "/root", ANTHROPIC_API_KEY: "sk-bill", WORKER_ONLY_VALUE: "internal" }); - expect(child).toEqual({ PATH: "/bin", HOME: "/root" }); + expect(child).toEqual({ PATH: resolveSubscriptionCliPath({ PATH: "/bin" }), HOME: "/root" }); + }); + it("repairs PATH with the image subscription CLI bin and optional npm prefix", () => { + expect(resolveSubscriptionCliPath({ PATH: "/bin" }).split(delimiter).slice(0, 2)).toEqual(["/home/node/.npm-global/bin", "/bin"]); + expect(resolveSubscriptionCliPath({ PATH: "/bin", NPM_CONFIG_PREFIX: "/custom/npm/" }).split(delimiter).slice(0, 3)).toEqual([ + "/custom/npm/bin", + "/home/node/.npm-global/bin", + "/bin", + ]); }); it("merges a defined extra value but skips an undefined one", () => { const child = subscriptionCliEnv({ PATH: "/bin" }, { CLAUDE_CODE_OAUTH_TOKEN: "t", UNSET: undefined }); - expect(child).toEqual({ PATH: "/bin", CLAUDE_CODE_OAUTH_TOKEN: "t" }); // UNSET (undefined) skips the extra-loop false arm + expect(child).toEqual({ PATH: resolveSubscriptionCliPath({ PATH: "/bin" }), CLAUDE_CODE_OAUTH_TOKEN: "t" }); // UNSET (undefined) skips the extra-loop false arm }); }); @@ -498,7 +506,7 @@ describe("subscription CLI helpers + fail-safe", () => { expect(seen).not.toContain("--ask-for-approval"); expect(seen).not.toContain("x"); expect(capturedInput).toBe("x"); - expect(capturedEnv).toEqual({ PATH: "/bin" }); + expect(capturedEnv).toEqual({ PATH: resolveSubscriptionCliPath({ PATH: "/bin" }) }); expect(capturedCwd).toContain("gittensory-ai-"); expect(timeout).toBe(300_000); // Provider-specific model/effort are passed through. From d5f2216661533e6ff99bd0ede1560e816ad5203c Mon Sep 17 00:00:00 2001 From: JSONbored <49853598+JSONbored@users.noreply.github.com> Date: Sun, 28 Jun 2026 22:57:00 -0700 Subject: [PATCH 42/68] fix(review): require complete AI summaries before publish Keep PR comments and Gate checks in the reviewing state until AI output includes a public assessment summary, reject stale cached nits-only reviews, and coalesce scheduled regate sweeps so restarts do not inflate queue pressure. --- src/queue/processors.ts | 9 +++++---- src/selfhost/queue-common.ts | 4 ++++ src/services/ai-review.ts | 24 ++++++++++++++++++++---- test/unit/ai-review-advisory.test.ts | 15 +++++++++++++++ test/unit/ai-review.test.ts | 22 +++++++++++++++------- test/unit/queue.test.ts | 20 ++++++++++++++++++-- test/unit/selfhost-queue-common.test.ts | 2 ++ test/unit/selfhost-sqlite-queue.test.ts | 13 ++++++++----- 8 files changed, 87 insertions(+), 22 deletions(-) diff --git a/src/queue/processors.ts b/src/queue/processors.ts index 48b35f9d99..8f727187a1 100644 --- a/src/queue/processors.ts +++ b/src/queue/processors.ts @@ -301,6 +301,7 @@ import { import { resolveRepositorySettings } from "../settings/repository-settings"; import type { LocalBranchAnalysisInput } from "../signals/local-branch"; import { + hasPublicReviewAssessment, isEnabled, runGittensoryAiReview, type InlineFinding, @@ -3857,9 +3858,9 @@ export async function runAiReviewForAdvisory( }); } args.advisory.findings.push(...findings); - return result.advisoryNotes + return hasPublicReviewAssessment(result.advisoryNotes) ? { - notes: result.advisoryNotes, + notes: result.advisoryNotes ?? "", reviewerCount: result.reviewerCount, inlineFindings: result.inlineFindings, findings, @@ -4612,7 +4613,7 @@ async function maybePublishPrPublicSurface( advisory.headSha, settings.aiReviewMode, ).catch(() => null); - if (cachedReview) { + if (cachedReview && hasPublicReviewAssessment(cachedReview.notes)) { advisory.findings.push(...cachedReview.findings); aiReview = cachedReview; } else { @@ -4676,7 +4677,7 @@ async function maybePublishPrPublicSurface( ).catch(() => undefined); } } - if (aiReviewExpected && !aiReview?.notes?.trim()) { + if (aiReviewExpected && !hasPublicReviewAssessment(aiReview?.notes)) { const retryError = new RetryableJobError( "AI review did not produce a public summary yet; keeping PR surface in reviewing state", { diff --git a/src/selfhost/queue-common.ts b/src/selfhost/queue-common.ts index 5c6f2d0ad6..9c3292896e 100644 --- a/src/selfhost/queue-common.ts +++ b/src/selfhost/queue-common.ts @@ -161,6 +161,10 @@ export function jobCoalesceKey(payload: string): string | null { const pr = normalizedNumber(message.prNumber); return repo && pr !== null ? `agent-regate-pr:${repo}#${pr}` : null; } + if (type === "agent-regate-sweep") { + const repo = normalizedRepo(message.repoFullName); + return `agent-regate-sweep:${repo ?? "all"}`; + } if (type === "recapture-preview") { const repo = normalizedRepo(message.repoFullName); const pr = normalizedNumber(message.prNumber); diff --git a/src/services/ai-review.ts b/src/services/ai-review.ts index 806590e0df..28977febb2 100644 --- a/src/services/ai-review.ts +++ b/src/services/ai-review.ts @@ -677,7 +677,24 @@ async function runProviderReview( }; } -/** Compose a public-safe markdown advisory blurb from one or two model reviews. Null if nothing safe. */ +function extractPublicAssessment(notes: string | null | undefined): string { + const raw = notes?.trim(); + if (!raw) return ""; + const sectionIndex = raw.search( + /(?:^|\n)\s*\*\*(?:Blockers|Nits \(\d+\))\*\*/u, + ); + const assessment = + sectionIndex === -1 ? raw : raw.slice(0, sectionIndex).trim(); + return toPublicSafe(assessment) ?? ""; +} + +export function hasPublicReviewAssessment( + notes: string | null | undefined, +): boolean { + return extractPublicAssessment(notes).length > 0; +} + +/** Compose a public-safe markdown advisory blurb from one or two model reviews. Null if no assessment is safe. */ export function composeAdvisoryNotes(reviews: ModelReview[]): string | null { const assessments = reviews.map((r) => r.assessment).filter(Boolean); // High-signal caps: a focused review shows only the few findings that matter (the prompt also asks the @@ -694,10 +711,9 @@ export function composeAdvisoryNotes(reviews: ModelReview[]): string | null { const safeNits = nits .map((s) => toPublicSafe(s)) .filter((s): s is string => Boolean(s)); - if (!assessment && safeBlockers.length === 0 && safeNits.length === 0) - return null; + if (!assessment) return null; const lines: string[] = []; - if (assessment) lines.push(assessment, ""); + lines.push(assessment, ""); if (safeBlockers.length > 0) { lines.push("**Blockers**"); lines.push(...safeBlockers.map((s) => `- ${s}`)); diff --git a/test/unit/ai-review-advisory.test.ts b/test/unit/ai-review-advisory.test.ts index e1d356ef7f..7775f445d7 100644 --- a/test/unit/ai-review-advisory.test.ts +++ b/test/unit/ai-review-advisory.test.ts @@ -59,6 +59,9 @@ function defectJson() { function notesOnlyJson() { return JSON.stringify({ assessment: "Looks fine.", blockers: [], nits: ["Add a test."], suggestions: ["Add a test."] }); } +function nitsWithoutAssessmentJson() { + return JSON.stringify({ assessment: "", blockers: [], nits: ["Add a test."], suggestions: ["Add a test."] }); +} function aiEnv(run: () => Promise, flags = true) { return createTestEnv({ @@ -450,6 +453,18 @@ describe("runAiReviewForAdvisory", () => { expect(result).toBeUndefined(); }); + it("returns undefined when the model produces nits but no assessment summary", async () => { + const result = await runAiReviewForAdvisory(aiEnv(async () => ({ response: nitsWithoutAssessmentJson() })), { + settings: { aiReviewMode: "advisory" } as RepositorySettings, + advisory: advisory(), + repoFullName: "acme/widgets", + pr, + author: "alice", + confirmedContributor: true, + }); + expect(result).toBeUndefined(); + }); + it("does not use the maintainer's BYOK key for non-confirmed oss-anti-slop blocking reviews", async () => { const run = vi.fn(async () => ({ response: defectJson() })); const env = createTestEnv({ diff --git a/test/unit/ai-review.test.ts b/test/unit/ai-review.test.ts index 695e9bc09b..60f77853f3 100644 --- a/test/unit/ai-review.test.ts +++ b/test/unit/ai-review.test.ts @@ -1236,7 +1236,7 @@ describe("pure helpers", () => { expect(result.status).toBe("ok"); }); - it("composeAdvisoryNotes returns null when nothing is public-safe", () => { + it("composeAdvisoryNotes returns null when no assessment is public-safe", () => { expect( composeAdvisoryNotes([ { @@ -1249,6 +1249,18 @@ describe("pure helpers", () => { }, ]), ).toBeNull(); + expect( + composeAdvisoryNotes([ + { + assessment: "", + suggestions: ["Add a test."], + nits: ["Rename the helper."], + blockers: ["Null deref in src/a.ts."], + inlineFindings: [], + confidence: 1, + }, + ]), + ).toBeNull(); }); it("parseModelReview parses well-formed inline findings; severity defaults to nit unless exactly 'blocker' (#inline-comments)", () => { @@ -1450,15 +1462,11 @@ describe("pure helpers", () => { review({ assessment: "Looks good." }), ]); expect(assessmentOnly).toBe("Looks good."); - const nitsOnly = composeAdvisoryNotes([review({ nits: ["Add a test."] })]); - expect(nitsOnly).toContain("**Nits (1)**"); - expect(nitsOnly).not.toContain("
"); - expect(nitsOnly).not.toContain("**Blockers**"); + expect(composeAdvisoryNotes([review({ nits: ["Add a test."] })])).toBeNull(); const blockersOnly = composeAdvisoryNotes([ review({ blockers: ["Null deref in src/a.ts."] }), ]); - expect(blockersOnly).toContain("**Blockers**"); - expect(blockersOnly).not.toContain("**Nits"); + expect(blockersOnly).toBeNull(); }); it("composeAdvisoryNotes merges + dedupes blockers/nits across two reviewers and renders both sections", () => { diff --git a/test/unit/queue.test.ts b/test/unit/queue.test.ts index a30d8c7aa0..01c304ded3 100644 --- a/test/unit/queue.test.ts +++ b/test/unit/queue.test.ts @@ -1350,11 +1350,22 @@ describe("queue processors", () => { expect(postedBodies[0]).toContain("🟪"); }); - it("keeps the PR comment and Gate in 🟪 reviewing state when AI review produces no public summary", async () => { + it("keeps the PR comment and Gate in 🟪 reviewing state when AI review produces nits but no public summary", async () => { + let aiCalls = 0; const env = createTestEnv({ GITHUB_APP_PRIVATE_KEY: await generatePrivateKeyPem(), AI: { - run: async () => ({ response: "not-json" }), + run: async () => { + aiCalls += 1; + return { + response: JSON.stringify({ + assessment: "", + blockers: [], + nits: ["Add coverage for the new branch."], + suggestions: ["Add coverage for the new branch."], + }), + }; + }, } as unknown as Ai, AI_SUMMARIES_ENABLED: "true", AI_PUBLIC_COMMENTS_ENABLED: "true", @@ -1379,6 +1390,10 @@ describe("queue processors", () => { aiReviewMode: "block", gatePack: "oss-anti-slop", }); + await putCachedAiReview(env, "JSONbored/gittensory", 10, "a10", "block", { + notes: "**Nits (1)**\n- stale cached nit", + reviewerCount: 1, + }); const commentBodies: string[] = []; const checkPatches: Array<{ status?: string; conclusion?: string }> = []; vi.stubGlobal("fetch", async (input: RequestInfo | URL, init?: RequestInit) => { @@ -1423,6 +1438,7 @@ describe("queue processors", () => { expect(commentBodies[0]).toContain("🟪"); expect(commentBodies[0]).not.toContain("held for maintainer review"); expect(commentBodies[0]).not.toContain("Review summary"); + expect(aiCalls).toBeGreaterThan(0); expect(checkPatches).toHaveLength(0); const audit = await env.DB.prepare("select count(*) as n from audit_events where event_type = ?") .bind("github_app.ai_review_public_summary_missing") diff --git a/test/unit/selfhost-queue-common.test.ts b/test/unit/selfhost-queue-common.test.ts index 8d02a71380..b5ac53eb9a 100644 --- a/test/unit/selfhost-queue-common.test.ts +++ b/test/unit/selfhost-queue-common.test.ts @@ -61,6 +61,8 @@ describe("self-host queue common helpers", () => { }); it("coalesces CI-completion webhooks with sorted pull numbers", () => { + expect(jobCoalesceKey(payload({ type: "agent-regate-sweep", requestedBy: "schedule" }))).toBe("agent-regate-sweep:all"); + expect(jobCoalesceKey(payload({ type: "agent-regate-sweep", repoFullName: "JSONbored/Gittensory" }))).toBe("agent-regate-sweep:jsonbored/gittensory"); expect( jobCoalesceKey( payload({ diff --git a/test/unit/selfhost-sqlite-queue.test.ts b/test/unit/selfhost-sqlite-queue.test.ts index a415450c94..a0b106917d 100644 --- a/test/unit/selfhost-sqlite-queue.test.ts +++ b/test/unit/selfhost-sqlite-queue.test.ts @@ -152,27 +152,30 @@ describe("createSqliteQueue (durable #980)", () => { expect(row.job_key).toBe(`github-webhook:ci-completed:jsonbored/gittensory@${"b".repeat(40)}#1629`); }); - it("coalesces duplicate CI and PR-refresh jobs before they inflate queue pressure", async () => { + it("coalesces duplicate CI, PR-refresh, and sweep jobs before they inflate queue pressure", async () => { const driver = makeDriver(); const q = createSqliteQueue(driver, async () => undefined); await q.binding.send(ciWebhook("ci-1", "check_suite"), { delaySeconds: 60 }); await q.binding.send(ciWebhook("ci-2", "check_run"), { delaySeconds: 1 }); await q.binding.send(prWebhook("pr-1"), { delaySeconds: 60 }); await q.binding.send(prWebhook("pr-2"), { delaySeconds: 1 }); + await q.binding.send({ type: "agent-regate-sweep", requestedBy: "schedule" } as JobMessage, { delaySeconds: 60 }); + await q.binding.send({ type: "agent-regate-sweep", requestedBy: "schedule" } as JobMessage, { delaySeconds: 1 }); const rows = driver.query( "SELECT payload, job_key FROM _selfhost_jobs ORDER BY id", [], ).rows as Array<{ payload: string; job_key: string }>; - expect(rows).toHaveLength(2); + expect(rows).toHaveLength(3); expect(rows.map((row) => row.job_key).sort()).toEqual([ + "agent-regate-sweep:all", `github-webhook:ci-completed:jsonbored/gittensory@${"b".repeat(40)}#1629`, `github-webhook:pr-refresh:jsonbored/gittensory#1629@${"a".repeat(40)}`, ]); - expect(rows.map((row) => JSON.parse(row.payload).deliveryId).sort()).toEqual(["ci-2", "pr-2"]); + expect(rows.map((row) => JSON.parse(row.payload).deliveryId).filter(Boolean).sort()).toEqual(["ci-2", "pr-2"]); expect(q.stats()).toMatchObject({ - gittensory_jobs_enqueued_total: 2, - gittensory_jobs_coalesced_total: 2, + gittensory_jobs_enqueued_total: 3, + gittensory_jobs_coalesced_total: 3, }); }); From 964da28f923684a243f059698411ac0446f4cf62 Mon Sep 17 00:00:00 2001 From: JSONbored <49853598+JSONbored@users.noreply.github.com> Date: Sun, 28 Jun 2026 23:36:07 -0700 Subject: [PATCH 43/68] fix(review): require real AI assessment before final comments --- .gittensory.yml | 1 + .gittensory.yml.example | 4 +++ docs/review-configuration.md | 2 ++ src/config/gittensory-repo-focus-manifest.ts | 1 + src/queue/processors.ts | 18 ++++++++++++- src/review/unified-comment-bridge.ts | 20 ++++++++++---- test/unit/ai-review-advisory.test.ts | 11 ++++++++ test/unit/unified-comment-bridge.test.ts | 28 ++++++++++++++++++++ 8 files changed, 79 insertions(+), 6 deletions(-) diff --git a/.gittensory.yml b/.gittensory.yml index 7518084986..7bc05cb34c 100644 --- a/.gittensory.yml +++ b/.gittensory.yml @@ -45,6 +45,7 @@ gate: # aiReview: # opt-in AI maintainer review (off by default; needs the AI flags enabled) # mode: advisory # block | advisory | off — block only blocks on a dual-model consensus defect # byok: false # use a maintainer Anthropic/OpenAI key for the write-up; consensus stays free Workers AI + # allAuthors: false # true reviews every PR author with the selected self-host model(s) # provider: anthropic # anthropic | openai — which BYOK provider (the secret key is set via the dashboard, never here) # model: claude-3-5-sonnet-latest # optional model override for the BYOK write-up diff --git a/.gittensory.yml.example b/.gittensory.yml.example index aa2e482812..aaed6cc35d 100644 --- a/.gittensory.yml.example +++ b/.gittensory.yml.example @@ -161,6 +161,10 @@ gate: # Workers-AI pair, so BYOK never changes who can be blocked. # Bool. Default: false. byok: false + # Review every PR author when AI review is enabled. Keep false to spend + # model calls only on the engine's default eligible authors. + # Bool. Default: false. + allAuthors: false # BYOK provider. anthropic | openai | null. Default: null (use the stored # key's own provider). Must match the stored key's provider or BYOK is # skipped (Workers-AI fallback). The key itself lives only in the encrypted diff --git a/docs/review-configuration.md b/docs/review-configuration.md index 684c6f6be2..5ec7711a29 100644 --- a/docs/review-configuration.md +++ b/docs/review-configuration.md @@ -109,6 +109,7 @@ already-enabled gate. | First-time-contributor grace | `gate.firstTimeContributorGrace` | `firstTimeContributorGrace` | bool | `false` | When `true`, softens a would-be block to advisory for a genuine newcomer (0 merged PRs, < 3 closed-unmerged PRs). Repeat offenders and authors with merge history are gated normally. | | AI review | `gate.aiReview.mode` | `aiReviewMode` | `off`/`advisory`/`block` | `off` | `advisory` posts AI review notes only; `block` lets a dual-model high-confidence consensus defect become a blocker (confirmed-contributors only). | | AI review BYOK | `gate.aiReview.byok` | `aiReviewByok` | bool | `false` | When `true` and a provider key is configured, the *advisory* write-up uses the maintainer's frontier model. The consensus blocker always uses the free Workers-AI pair, so BYOK never changes who can be blocked. | +| AI review all authors | `gate.aiReview.allAuthors` | `aiReviewAllAuthors` | bool | `false` | When `true`, an enabled AI review runs for every PR author instead of only the engine's default eligible authors. Use this for self-host repos where the selected model must produce the public review summary. | | AI review provider | `gate.aiReview.provider` | `aiReviewProvider` | `anthropic` / `openai` / `null` | `null` | `null` = use the stored key's own provider. Must match the stored key's provider or BYOK is skipped (Workers-AI fallback). The key itself is only in the encrypted key store. | | AI review model | `gate.aiReview.model` | `aiReviewModel` | string / `null` | `null` | Model override for the BYOK advisory write-up (e.g. `claude-3-5-sonnet-latest`). `null` = the key record's model, else a conservative per-provider default. | | AI close confidence | `gate.aiReview.closeConfidence` | `aiReviewCloseConfidence` | number 0–1 (nullable) | `null` (engine uses `0.9`) | Minimum **calibrated** AI-reviewer confidence for a consensus defect / split to **block** under `aiReview.mode: block`. Below-threshold AI defects stay advisory (visible, never close). Each reviewer rates its own confidence; consensus carries the weaker reviewer's. Config-as-code only (no dashboard/DB column). | @@ -203,6 +204,7 @@ gate: aiReview: mode: advisory byok: true + allAuthors: true provider: anthropic model: claude-3-5-sonnet-latest diff --git a/src/config/gittensory-repo-focus-manifest.ts b/src/config/gittensory-repo-focus-manifest.ts index a65403cdd4..c94bbfee7f 100644 --- a/src/config/gittensory-repo-focus-manifest.ts +++ b/src/config/gittensory-repo-focus-manifest.ts @@ -49,6 +49,7 @@ gate: # aiReview: # opt-in AI maintainer review (off by default; needs the AI flags enabled) # mode: advisory # block | advisory | off — block only blocks on a dual-model consensus defect # byok: false # use a maintainer Anthropic/OpenAI key for the write-up; consensus stays free Workers AI + # allAuthors: false # true reviews every PR author with the selected self-host model(s) # provider: anthropic # anthropic | openai — which BYOK provider (the secret key is set via the dashboard, never here) # model: claude-3-5-sonnet-latest # optional model override for the BYOK write-up diff --git a/src/queue/processors.ts b/src/queue/processors.ts index 8f727187a1..3e35d5fa8e 100644 --- a/src/queue/processors.ts +++ b/src/queue/processors.ts @@ -3571,6 +3571,22 @@ export async function shouldStartAiReviewForAdvisory( skipAiReview?: boolean | undefined; }, ): Promise { + if (!shouldRequirePublicAiReviewForAdvisory(env, args)) return false; + if (args.settings.aiReviewAllAuthors) return true; + return !(isReputationEnabled(env) && isConvergenceRepoAllowed(env, args.repoFullName) && (await shouldSkipAiForReputation(env, { project: args.repoFullName, submitter: args.author }))); +} + +export function shouldRequirePublicAiReviewForAdvisory( + env: Env, + args: { + settings: RepositorySettings; + advisory: Pick>, "headSha">; + repoFullName: string; + author: string | null; + confirmedContributor: boolean; + skipAiReview?: boolean | undefined; + }, +): boolean { const packAllowsAnyAuthorBlockingReview = args.settings.gatePack === "oss-anti-slop" && args.settings.aiReviewMode === "block"; @@ -3588,7 +3604,7 @@ export async function shouldStartAiReviewForAdvisory( !env.AI ) return false; - return !(isReputationEnabled(env) && isConvergenceRepoAllowed(env, args.repoFullName) && (await shouldSkipAiForReputation(env, { project: args.repoFullName, submitter: args.author }))); + return true; } export async function runAiReviewForAdvisory( diff --git a/src/review/unified-comment-bridge.ts b/src/review/unified-comment-bridge.ts index 45e2a394d5..5d2e537b00 100644 --- a/src/review/unified-comment-bridge.ts +++ b/src/review/unified-comment-bridge.ts @@ -160,9 +160,11 @@ export function isBoilerplateNit(finding: AdvisoryFinding): boolean { /** Build the single AI reviewer note from gittensory's AI output: the composed advisory write-up (minus its nits) * becomes the assessment; a consensus defect (recovered from the advisory findings) becomes a blocker; the AI's own - * nits AND the gate's non-blocking warnings become the collapsible nits. Returns `[]` when there is nothing - * reviewer-side to surface (no AI notes, no consensus defect) so the renderer hides the reviewer chip. The gate - * `decision` (passed separately) stays authoritative over `recommendation` — this is advisory framing only. */ + * nits AND the gate's non-blocking warnings become the collapsible nits. Deterministic warnings alone must NOT + * manufacture a reviewer note: a final public comment may only claim an AI review when there is a real AI + * assessment/defect. Returns `[]` when there is nothing reviewer-side to surface (no AI notes, no consensus defect, + * no non-AI gate blocker) so the renderer hides the reviewer chip. The gate `decision` (passed separately) stays + * authoritative over `recommendation` — this is advisory framing only. */ export function buildDualReviewNotes(args: { aiReview?: { notes: string } | undefined; consensusDefect?: { title: string; detail: string } | undefined; @@ -208,7 +210,7 @@ export function buildDualReviewNotes(args: { .map((line) => publicSafeNit(line)) .filter((line): line is string => line !== null); const nits = [...aiNits, ...gateNits]; - if (!assessment && blockers.length === 0 && nits.length === 0) return []; + if (!assessment && blockers.length === 0) return []; const notes: ReviewNotes = { assessment, suggestions: [], @@ -353,7 +355,15 @@ export function buildUnifiedCommentBody(args: UnifiedCommentBridgeArgs): string }); // The gate already produced 0/1 reviewer notes from a synthesis of the model pair; reflect the caller's // actual reviewer count (for the chip + the "N reviewers, synthesized" evidence) without re-deriving it. - if (typeof args.reviewerCount === "number") input.reviewerCount = args.reviewerCount; + // A non-AI gate blocker can still be folded into the blocker list above, but it must not make the comment claim + // an AI reviewer ran. Without this guard, deterministic nits/warnings rendered as `1 AI reviewer` with no + // review summary. + input.reviewerCount = + args.aiReview !== undefined + ? typeof args.reviewerCount === "number" + ? args.reviewerCount + : input.reviewerCount + : 0; // Honor `.gittensory.yml review.fields` row visibility, exactly as the legacy panel does. const visibleRows = args.panelRows.filter((row) => args.reviewFields?.[row.key] !== false); diff --git a/test/unit/ai-review-advisory.test.ts b/test/unit/ai-review-advisory.test.ts index 7775f445d7..5ac8f89e1f 100644 --- a/test/unit/ai-review-advisory.test.ts +++ b/test/unit/ai-review-advisory.test.ts @@ -105,6 +105,17 @@ describe("shouldStartAiReviewForAdvisory", () => { await env.DB.prepare("INSERT INTO submitter_stats (project, submitter, submissions, merged, closed, manual, last_seen) VALUES (?, ?, ?, ?, ?, ?, CURRENT_TIMESTAMP)").bind("acme/widgets", "alice", 8, 0, 8, 0).run(); await expect(shouldStartAiReviewForAdvisory(env, base)).resolves.toBe(false); }); + + it("honors aiReviewAllAuthors as an explicit self-host review requirement even when reputation would skip", async () => { + const env = createTestEnv({ AI: { run: vi.fn() } as unknown as Ai, AI_SUMMARIES_ENABLED: "true", AI_PUBLIC_COMMENTS_ENABLED: "true", GITTENSORY_REVIEW_REPUTATION: "true", GITTENSORY_REVIEW_REPOS: "acme/widgets" }); + await env.DB.prepare("INSERT INTO submitter_stats (project, submitter, submissions, merged, closed, manual, last_seen) VALUES (?, ?, ?, ?, ?, ?, CURRENT_TIMESTAMP)").bind("acme/widgets", "alice", 8, 0, 8, 0).run(); + await expect( + shouldStartAiReviewForAdvisory(env, { + ...base, + settings: { aiReviewMode: "advisory", gatePack: "gittensor", aiReviewAllAuthors: true } as RepositorySettings, + }), + ).resolves.toBe(true); + }); }); describe("runAiReviewForAdvisory", () => { diff --git a/test/unit/unified-comment-bridge.test.ts b/test/unit/unified-comment-bridge.test.ts index 452f7167f7..64b01b77b6 100644 --- a/test/unit/unified-comment-bridge.test.ts +++ b/test/unit/unified-comment-bridge.test.ts @@ -211,6 +211,31 @@ describe("buildUnifiedCommentBody", () => { expect(body).toContain("> [!TIP]"); // success → ready → TIP alert }); + it("does not claim an AI reviewer or synthesize a review from deterministic warnings alone", () => { + const body = buildUnifiedCommentBody({ + gate: gate({ + conclusion: "action_required", + summary: "Manual maintainer review required.", + warnings: [ + { + code: "large_change", + severity: "warning", + title: "Large change — held for manual review", + detail: "Large change.", + action: "Split this into smaller PRs.", + }, + ], + }), + panelRows, + readinessTotal: 55, + changedFiles: 5, + footerMarkdown: footer, + }); + expect(body).not.toContain("AI reviewer"); + expect(body).not.toContain("Review summary"); + expect(body).toContain("No AI review summary"); + }); + it("the gate conclusion drives the status: a gate failure blocks regardless of reviewer recs", () => { const failing = buildUnifiedCommentBody({ gate: gate({ @@ -648,6 +673,7 @@ describe("buildDualReviewNotes — public-safe Nit scrub (privacy-critical, gate // mirrors src/rules/advisory.ts sanitizeForCheckRun + src/signals/engine.ts containsPrivatePublicTerm. it("scrubs a forbidden term from a Nit instead of leaking it verbatim", () => { const reviews = buildDualReviewNotes({ + aiReview: { notes: "Reviewer assessment." }, warnings: [{ code: "w", severity: "warning", title: "Adjust the estimated scores threshold", detail: "...", action: "Tune it." }], recommendation: "manual_review", verdict: "manual", @@ -659,6 +685,7 @@ describe("buildDualReviewNotes — public-safe Nit scrub (privacy-critical, gate it("neutralizes a private internal in a Nit and leaves a benign Nit untouched", () => { const reviews = buildDualReviewNotes({ + aiReview: { notes: "Reviewer assessment." }, warnings: [ // "trust score" is a forbidden term → scrubbed to "[context]"; the leak never reaches the comment. { code: "w1", severity: "warning", title: "Your trust score is low", detail: "...", action: "n/a" }, @@ -684,6 +711,7 @@ describe("buildDualReviewNotes — public-safe Nit scrub (privacy-critical, gate const dropTerms = ["reward", "payout", "farming", "wallet", "hotkey", "trust score", "raw trust", "estimated score", "scoreability", "reviewability3"]; for (const term of dropTerms) { const reviews = buildDualReviewNotes({ + aiReview: { notes: "Reviewer assessment." }, warnings: [{ code: "w", severity: "warning", title: `Concern about ${term} here`, detail: "...", action: "n/a" }], recommendation: "manual_review", verdict: "manual", From 17d082f785c9b17902fce0106f4e612592d62f62 Mon Sep 17 00:00:00 2001 From: JSONbored <49853598+JSONbored@users.noreply.github.com> Date: Mon, 29 Jun 2026 01:07:15 -0700 Subject: [PATCH 44/68] fix(selfhost): stabilize review summaries and queue priority --- .env.example | 1 + docs/self-host/troubleshooting.md | 4 +- src/queue/processors.ts | 17 +--- src/review/ai-notes.ts | 22 +++++ src/review/unified-comment-bridge.ts | 36 ++++---- src/review/unified-comment.ts | 6 +- src/selfhost/pg-queue.ts | 42 ++++++++- src/selfhost/queue-common.ts | 37 ++++++++ src/selfhost/sqlite-queue.ts | 34 ++++++- src/signals/engine.ts | 113 ++++++++++++----------- test/unit/reputation-wiring.test.ts | 14 +++ test/unit/selfhost-pg-queue.test.ts | 27 ++++++ test/unit/selfhost-queue-common.test.ts | 17 ++++ test/unit/selfhost-sqlite-queue.test.ts | 45 ++++++++- test/unit/signals-coverage.test.ts | 33 ++++++- test/unit/unified-comment-bridge.test.ts | 12 +++ test/unit/unified-comment-parity.test.ts | 9 +- test/unit/unified-comment.test.ts | 2 + 18 files changed, 368 insertions(+), 103 deletions(-) create mode 100644 src/review/ai-notes.ts diff --git a/.env.example b/.env.example index e6b48d8564..b26ab9260c 100644 --- a/.env.example +++ b/.env.example @@ -150,6 +150,7 @@ GITTENSORY_REVIEW_DRAFT=false # --- Queue worker (#977/#1201) --- # QUEUE_CONCURRENCY=4 # max concurrent job-processing loops per instance (default 4; set 1 for strict serial processing) +# QUEUE_BACKGROUND_CONCURRENCY=1 # max low-priority/background jobs allowed to occupy QUEUE_CONCURRENCY slots # --- Caddy HTTPS terminator (#1203; requires --profile caddy) --- # DOMAIN=gittensory.example.com # fully-qualified domain; Caddy auto-obtains a Let's Encrypt cert diff --git a/docs/self-host/troubleshooting.md b/docs/self-host/troubleshooting.md index 238d7897b1..6473635a7c 100644 --- a/docs/self-host/troubleshooting.md +++ b/docs/self-host/troubleshooting.md @@ -88,7 +88,9 @@ curl -X POST localhost:8787/v1/internal/jobs/rag-index \ **Cause:** CPU embedding (~1 chunk/s on `bge-m3`). It's a **one-time** cost — afterwards only changed files re-index on merge. Indexing runs on a **dedicated queue lane**, so a slow index never blocks live reviews / webhooks / sweeps (those drain on the main lane in parallel). Tune index parallelism with `QUEUE_INDEX_CONCURRENCY` (default 1) -and the main lane with `QUEUE_CONCURRENCY`. +and the main lane with `QUEUE_CONCURRENCY`. The durable queue also caps low-priority/background work with +`QUEUE_BACKGROUND_CONCURRENCY` (default 1), so slow background jobs cannot occupy every worker slot while required +PR gate checks are waiting to publish. --- diff --git a/src/queue/processors.ts b/src/queue/processors.ts index 3e35d5fa8e..e89ff763a2 100644 --- a/src/queue/processors.ts +++ b/src/queue/processors.ts @@ -141,7 +141,6 @@ import { buildPullRequestAdvisory, evaluateGateCheck, isTestPath, - reconcileGateEvaluationForGreenCi, } from "../rules/advisory"; import { detectNotificationEvents } from "../notifications/events"; import { @@ -3701,6 +3700,7 @@ export async function runAiReviewForAdvisory( // neutral → false on any error). if ( reputationActive && + !args.settings.aiReviewAllAuthors && (await shouldSkipAiForReputation(env, { project: args.repoFullName, submitter: args.author, @@ -5124,17 +5124,9 @@ async function maybePublishPrPublicSurface( : {}), ...(failingDetails.length > 0 ? { failingDetails } : {}), }; - // CI-refutation for the PUBLIC comment (#ai-ci-refutation): when the gate FAILED solely on an AI-judgment - // blocker but the LIVE CI is GREEN, render the comment (headline + Gate panel row) as SUCCESS/advisory so it - // MATCHES the disposition (which merges such a PR) instead of a contradictory red "blocked/closed". Uses the - // SAME grounding+convergence gate as the disposition refutation (a single `aiCiRefutationActive` call so this - // site carries no branch), built AFTER the live CI is resolved and used for BOTH the panel rows and the - // comment body so the two never disagree. Gate OFF ⇒ commentGate === gateEvaluation (byte-identical comment). - const commentGate = reconcileGateEvaluationForGreenCi( - gateEvaluation, - ciState, - aiCiRefutationActive(env, repoFullName), - ); + // The public comment must match the authoritative Gate check-run conclusion. Planner-level CI refutation can + // affect auto-actions, but it must never recolor the review comment green while the posted Gate is red. + const commentGate = gateEvaluation; // Observability (#reviews-dashboard): record the would-be gate verdict so the Grafana panel shows the // merge/close/hold mix — the "are we rubber-stamping?" signal — even in advisory/dryRun (this is the rendered verdict). incr("gittensory_gate_decisions_total", { @@ -5266,7 +5258,6 @@ async function maybePublishPrPublicSurface( preflight, queueHealth, ...(reviewConfig !== undefined ? { review: reviewConfig } : {}), - ...(aiReview !== undefined ? { aiReview } : {}), }), footerMarkdown: gittensoryFooter({ earnUrl: repo?.isRegistered diff --git a/src/review/ai-notes.ts b/src/review/ai-notes.ts new file mode 100644 index 0000000000..7b3b4898d1 --- /dev/null +++ b/src/review/ai-notes.ts @@ -0,0 +1,22 @@ +/** Split composed AI advisory notes into prominent review text plus non-blocking nits. + * + * The AI review composer emits: + * summary/body + * optional **Blockers** + * optional trailing **Nits (N)** + * + * Both legacy and unified PR comments must keep the summary/blockers prominent and demote only + * the trailing nits into a collapsible section. Keep this parser shared so the two renderers cannot drift. + */ +export function splitAiReviewNits(notes: string): { main: string; nits: string[] } { + const marker = notes.indexOf("**Nits ("); + if (marker === -1) return { main: notes.trim(), nits: [] }; + const nits = notes + .slice(marker) + .split("\n") + .slice(1) + .map((line) => line.replace(/^\s*[-*]\s*/, "").trim()) + .filter(Boolean); + return { main: notes.slice(0, marker).trim(), nits }; +} + diff --git a/src/review/unified-comment-bridge.ts b/src/review/unified-comment-bridge.ts index 5d2e537b00..f0741f6288 100644 --- a/src/review/unified-comment-bridge.ts +++ b/src/review/unified-comment-bridge.ts @@ -37,8 +37,10 @@ import { type UnifiedSignalRow, type Verdict, } from "./unified-comment"; +import { splitAiReviewNits } from "./ai-notes"; export { PR_PANEL_COMMENT_MARKER }; +export { splitAiReviewNits } from "./ai-notes"; // ── Public-safe defense-in-depth (privacy-critical) ────────────────────────────────────────────── // @@ -122,22 +124,6 @@ export function panelRowsToSignalRows(rows: PublicPrPanelSignalRow[]): UnifiedSi }); } -/** Split the composed AI advisory notes (#focused-reviews) into the prominent body (the assessment prose + any - * `**Blockers**` section — real problems stay headline) and the trailing `**Nits (N)**` bullet lines, so the renderer - * can DEMOTE nits into the collapsible Nits section instead of the headline assessment (nits are never blockers). - * When there is no Nits section the whole blob is the body and `nits` is empty (byte-identical to before). */ -export function splitAiReviewNits(notes: string): { main: string; nits: string[] } { - const marker = notes.indexOf("**Nits ("); - if (marker === -1) return { main: notes.trim(), nits: [] }; - const nits = notes - .slice(marker) - .split("\n") - .slice(1) // drop the "**Nits (N)**" header line itself - .map((line) => line.replace(/^\s*[-*]\s*/, "").trim()) - .filter(Boolean); - return { main: notes.slice(0, marker).trim(), nits }; -} - /** Self-host environmental + process findings that are already represented in the signal table and are NOT code * observations — keep them OUT of the Nits list so the nit count reflects real code review, not boilerplate that * padded nearly every review (#review-accuracy). */ @@ -182,7 +168,9 @@ export function buildDualReviewNotes(args: { const { main: assessment, nits: aiNitLines } = splitAiReviewNits( args.aiReview?.notes?.trim() ?? "", ); - const consensusBlocker = args.consensusDefect ? [`${args.consensusDefect.title}${args.consensusDefect.detail ? `: ${args.consensusDefect.detail}` : ""}`.trim()] : []; + const consensusBlocker = args.consensusDefect + ? [formatConsensusDefectBlocker(args.consensusDefect)] + : []; // FIX D1: fold the gate's own hard blockers into the reviewer blockers (so a non-AI gate failure populates // "Why this is blocked"). Exclude `ai_consensus_defect` (already surfaced via consensusDefect → appears once) // and scrub each through the same public-safe boundary as Nits, DROPPING any that still leaks a private term. @@ -232,6 +220,20 @@ export function consensusDefectFromFindings(findings: AdvisoryFinding[] | undefi return { title: found.title, detail: found.detail }; } +function formatConsensusDefectBlocker(defect: { title: string; detail: string }): string { + const title = defect.title.trim(); + const detail = defect.detail.trim(); + if (!detail) return title; + const normalizedTitle = normalizeConcernLine(title); + const normalizedDetail = normalizeConcernLine(detail); + if (normalizedTitle.includes(normalizedDetail)) return detail; + return `${title}: ${detail}`.trim(); +} + +function normalizeConcernLine(value: string): string { + return value.toLowerCase().replace(/[\s.,;:!?`]+/g, " ").trim(); +} + export type UnifiedCommentBridgeArgs = { /** gittensory's authoritative gate verdict (drives the unified status + the Gate row). */ gate: GateCheckEvaluation; diff --git a/src/review/unified-comment.ts b/src/review/unified-comment.ts index 8273de396d..340feeadd1 100644 --- a/src/review/unified-comment.ts +++ b/src/review/unified-comment.ts @@ -447,6 +447,9 @@ export function renderUnifiedReviewComment(input: UnifiedReviewInput, ctx: Unifi if (input.summary.trim()) blocks.push(`**Review summary**\n${escapePublicHtmlAngles(input.summary.trim())}`); + const nits = dedupeLines(input.nits ?? []); + if (nits.length) blocks.push(details("Nits", bullets(nits), `${nits.length} non-blocking`)); + const blockers = dedupeLines(input.blockers ?? []); if (blockers.length) { const heading = status === "blocked" ? "Why this is blocked" : "Concerns raised — review before merging"; @@ -460,9 +463,6 @@ export function renderUnifiedReviewComment(input: UnifiedReviewInput, ctx: Unifi if (failingChecks) blocks.push(`**CI checks failing**\n${failingChecks}`); blocks.push(signalTable(input, ctx)); - - const nits = dedupeLines(input.nits ?? []); - if (nits.length) blocks.push(details("Nits", bullets(nits), `${nits.length} non-blocking`)); for (const c of ctx.extraCollapsibles ?? []) { if (c.body.trim()) blocks.push(c.rawHtml ? detailsRaw(c.title, c.body.trim()) : details(c.title, c.body.trim())); } diff --git a/src/selfhost/pg-queue.ts b/src/selfhost/pg-queue.ts index 85faf7e307..26a499b26e 100644 --- a/src/selfhost/pg-queue.ts +++ b/src/selfhost/pg-queue.ts @@ -8,10 +8,12 @@ import { incr } from "./metrics"; import { captureError } from "./sentry"; import { deterministicJitterMs, + FOREGROUND_QUEUE_PRIORITY_FLOOR, githubRateLimitRetryDelayMs, jobCoalesceKey, jobPriority, nonConsumingRetryDelayMs, + queueBackgroundConcurrency, queueProcessingTimeoutMs, queueRecoveryJitterMs, queueStartupJitterMinJobs, @@ -59,6 +61,8 @@ interface JobRow { payload: string; attempts: number; job_key?: string | null; + priority: number | string; + backgroundSlotReserved?: boolean; } export interface PgQueueOptions { @@ -69,6 +73,8 @@ export interface PgQueueOptions { * (GitHub + AI awaits dominate), so overlapping a handful drains a PR burst far faster; FOR UPDATE SKIP LOCKED * keeps claims race-free across the pool (and across replicas). Set QUEUE_CONCURRENCY=1 to force strict serial. */ concurrency?: number; + /** Max background jobs (priority < 8) allowed to consume concurrent slots. Defaults to QUEUE_BACKGROUND_CONCURRENCY or 1. */ + backgroundConcurrency?: number; } export function createPgQueue( @@ -84,10 +90,15 @@ export function createPgQueue( const concurrency = opts.concurrency ?? Math.max(1, Number(process.env.QUEUE_CONCURRENCY ?? "4")); + const backgroundConcurrency = queueBackgroundConcurrency( + concurrency, + opts.backgroundConcurrency, + ); const processingTimeoutMs = queueProcessingTimeoutMs(); let running = false; let active = 0; + let activeBackground = 0; const activeJobIds = new Set(); let timer: ReturnType | null = null; let githubRateLimitCooldownUntil = 0; @@ -239,12 +250,35 @@ export function createPgQueue( async function claimNext(): Promise { if (Date.now() < githubRateLimitCooldownUntil) return null; const now = Date.now(); + const foreground = await claimNextWhere(now, "priority >= $2"); + if (foreground) return foreground; + if (activeBackground >= backgroundConcurrency) return null; + activeBackground++; + const background = await claimNextWhere(now, "priority < $2"); + if (!background) { + activeBackground--; + return null; + } + return { ...background, backgroundSlotReserved: true }; + } + + async function claimNextWhere( + now: number, + priorityPredicate: string, + ): Promise { // Atomic, multi-instance-safe: lock + claim one due job, skipping rows another instance already locked. const res = await pool.query( `UPDATE ${TABLE} SET status='processing', run_after=$1 - WHERE id = (SELECT id FROM ${TABLE} WHERE status='pending' AND run_after<=$1 ORDER BY priority DESC, run_after, id FOR UPDATE SKIP LOCKED LIMIT 1) - RETURNING id, payload, attempts, job_key`, - [now], + WHERE id = ( + SELECT id + FROM ${TABLE} + WHERE status='pending' AND run_after<=$1 AND ${priorityPredicate} + ORDER BY priority DESC, run_after, id + FOR UPDATE SKIP LOCKED + LIMIT 1 + ) + RETURNING id, payload, attempts, job_key, priority`, + [now, FOREGROUND_QUEUE_PRIORITY_FLOOR], ); return (res.rows[0] as JobRow | undefined) ?? null; } @@ -411,6 +445,8 @@ export function createPgQueue( return true; } finally { activeJobIds.delete(job.id); + if (job.backgroundSlotReserved) + activeBackground = Math.max(0, activeBackground - 1); } } diff --git a/src/selfhost/queue-common.ts b/src/selfhost/queue-common.ts index 9c3292896e..5047ed29af 100644 --- a/src/selfhost/queue-common.ts +++ b/src/selfhost/queue-common.ts @@ -6,6 +6,8 @@ const DEFAULT_STARTUP_JITTER_MS = 3 * 60_000; const DEFAULT_RECOVERY_JITTER_MS = 60_000; const DEFAULT_STARTUP_JITTER_MIN_JOBS = 8; const DEFAULT_PROCESSING_TIMEOUT_MS = 30 * 60_000; +const DEFAULT_BACKGROUND_CONCURRENCY = 1; +export const FOREGROUND_QUEUE_PRIORITY_FLOOR = 8; // Webhook-driven work (a fresh PR -> its review) jumps ahead of heavy background jobs. Per-PR review refreshes // sit just below real webhooks, and sweep fan-out sits below those so stale surfaces are repaired during bursts. @@ -19,9 +21,44 @@ const PRIORITY_BY_TYPE = new Map([ export function jobPriority(payload: string): number { const type = extractPayloadType(payload) ?? ""; if (type === "github-webhook") return githubWebhookPriority(payload); + if (type === "agent-regate-pr") return agentRegatePriority(payload); return PRIORITY_BY_TYPE.get(type) ?? 0; } +function agentRegatePriority(payload: string): number { + try { + const message = JSON.parse(payload) as { deliveryId?: unknown }; + const deliveryId = + typeof message.deliveryId === "string" ? message.deliveryId : ""; + if (deliveryId.startsWith("manual-regate:")) return 99; + } catch { + return PRIORITY_BY_TYPE.get("agent-regate-pr") ?? 0; + } + return PRIORITY_BY_TYPE.get("agent-regate-pr") ?? 0; +} + +export function isForegroundJobPriority(priority: number): boolean { + return priority >= FOREGROUND_QUEUE_PRIORITY_FLOOR; +} + +export function queueBackgroundConcurrency( + totalConcurrency: number, + configured: unknown = process.env.QUEUE_BACKGROUND_CONCURRENCY, +): number { + const total = Number.isFinite(totalConcurrency) + ? Math.max(0, Math.floor(totalConcurrency)) + : 0; + const raw = + configured === undefined || configured === null || configured === "" + ? DEFAULT_BACKGROUND_CONCURRENCY + : Number(configured); + const parsed = + Number.isFinite(raw) && raw >= 0 + ? Math.floor(raw) + : DEFAULT_BACKGROUND_CONCURRENCY; + return Math.min(parsed, total); +} + function githubWebhookPriority(payload: string): number { try { const message = JSON.parse(payload) as { diff --git a/src/selfhost/sqlite-queue.ts b/src/selfhost/sqlite-queue.ts index 59d5bde009..5780b03bdf 100644 --- a/src/selfhost/sqlite-queue.ts +++ b/src/selfhost/sqlite-queue.ts @@ -9,10 +9,12 @@ import { incr } from "./metrics"; import { captureError } from "./sentry"; import { deterministicJitterMs, + FOREGROUND_QUEUE_PRIORITY_FLOOR, githubRateLimitRetryDelayMs, jobCoalesceKey, jobPriority, nonConsumingRetryDelayMs, + queueBackgroundConcurrency, queueProcessingTimeoutMs, queueRecoveryJitterMs, queueStartupJitterMinJobs, @@ -61,6 +63,8 @@ interface JobRow { payload: string; attempts: number; job_key?: string | null; + priority: number; + backgroundSlotReserved?: boolean; } export interface SqliteQueueOptions { @@ -71,6 +75,8 @@ export interface SqliteQueueOptions { * (GitHub + AI awaits dominate), so overlapping a handful drains a PR burst far faster while SQLite's WAL + * busy_timeout absorb the short serialized write windows. Set QUEUE_CONCURRENCY=1 to force strict serial. */ concurrency?: number; + /** Max background jobs (priority < 8) allowed to consume concurrent slots. Defaults to QUEUE_BACKGROUND_CONCURRENCY or 1. */ + backgroundConcurrency?: number; } export function createSqliteQueue( @@ -86,6 +92,10 @@ export function createSqliteQueue( const concurrency = opts.concurrency ?? Math.max(1, Number(process.env.QUEUE_CONCURRENCY ?? "4")); + const backgroundConcurrency = queueBackgroundConcurrency( + concurrency, + opts.backgroundConcurrency, + ); const processingTimeoutMs = queueProcessingTimeoutMs(); driver.exec(DDL); @@ -142,6 +152,7 @@ export function createSqliteQueue( let running = false; let active = 0; // number of concurrent pump() loops currently draining jobs + let activeBackground = 0; const activeJobIds = new Set(); let timer: ReturnType | null = null; let githubRateLimitCooldownUntil = 0; @@ -180,9 +191,26 @@ export function createSqliteQueue( function claimNext(): JobRow | null { if (Date.now() < githubRateLimitCooldownUntil) return null; const now = Date.now(); + const foreground = claimNextWhere(now, "priority>=?"); + if (foreground) return foreground; + if (activeBackground >= backgroundConcurrency) return null; + activeBackground++; + const background = claimNextWhere(now, "priority 0 ? [...new Set(nextSteps)].map((step) => `- ${step}`) : ["- Keep the PR focused and include validation evidence before maintainer review."]; } -/** "Review details" body — the optional AI maintainer-review notes (already public-safe upstream). Returns - * `[]` when there is no AI review, so the section is omitted entirely. Angle brackets are escaped as a final - * guard (a stray tag cannot break the panel) and the notes are length-capped, matching the legacy panel. */ -function reviewDetailsBody(aiReview: { notes: string } | undefined): string[] { - if (!aiReview) return []; - return [ - "_Generated from public PR metadata and the diff. Advisory only; deterministic signals remain authoritative._", - "", - aiReview.notes.replace(/[<>]/g, (char) => (char === "<" ? "<" : ">")).slice(0, 4000), - ]; -} - /** * The public-safe collapsibles for the CONVERGED comment, as `UnifiedCollapsible[]`. Built from the SAME - * bodies the legacy panel renders (above) so the two never diverge. Order mirrors the legacy panel's - * (Review context · Contributor next steps · Signal definitions · Review details). Excludes "Maintainer - * notes" (PRIVATE). "Review details" is omitted when there is no AI review (empty body → the renderer skips it). + * bodies the legacy panel renders (above) so the two never diverge. Excludes "Maintainer notes" (PRIVATE) and + * AI review notes, which the unified renderer owns as the prominent Review summary + Nits section. */ export function buildPublicSafeCollapsibles(args: PublicSafeCollapsibleArgs): UnifiedCollapsible[] { - const collapsibles: UnifiedCollapsible[] = [ + return [ { title: "Review context", body: reviewContextBody(args).join("\n") }, { title: "Contributor next steps", body: contributorNextStepsBody(publicSafeNextSteps(args)).join("\n") }, { title: "Signal definitions", body: signalDefinitionsBody().join("\n") }, ]; - const reviewDetails = reviewDetailsBody(args.aiReview); - if (reviewDetails.length > 0) collapsibles.push({ title: "Review details", body: reviewDetails.join("\n") }); - return collapsibles; } /** The deduped, public-safe "next steps" list — extracted so both the legacy panel and the converged @@ -4248,20 +4232,28 @@ export function buildPublicPrIntelligenceComment(args: { if (!args.detection.detected) return buildMinimalInviteComment(args); const genericOssMode = args.settings.publicAudienceMode === "oss_maintainer"; const hasPublicWarnings = publicFindings.some((finding) => finding.severity === "warning"); - const alert = gateBlocking - ? gateConclusion === "action_required" - ? "IMPORTANT" - : missingLinkedIssue && args.settings.linkedIssueGateMode === "block" + const aiReview = args.aiReview ? splitAiReviewNits(args.aiReview.notes) : null; + const aiReviewHasBlockers = Boolean(aiReview?.main) && aiReviewMainHasBlockers(aiReview?.main ?? ""); + const alert = aiReviewHasBlockers + ? "CAUTION" + : gateBlocking + ? gateConclusion === "action_required" + ? "WARNING" + : missingLinkedIssue && args.settings.linkedIssueGateMode === "block" + ? "WARNING" + : "CAUTION" + : hasPublicWarnings || hasRelatedWork ? "WARNING" - : "CAUTION" - : hasPublicWarnings || hasRelatedWork - ? "IMPORTANT" - : "TIP"; - const panelTitle = gateBlocking - ? "Gittensory Gate is blocking merge" - : hasPublicWarnings || hasRelatedWork - ? "Gittensory found maintainer review notes" - : "Gittensory PR readiness looks good"; + : "TIP"; + const panelTitle = aiReviewHasBlockers + ? "Gittensory review found blockers" + : args.aiReview && !gateBlocking + ? "Gittensory review approved this PR" + : gateBlocking + ? "Gittensory Gate is blocking merge" + : hasPublicWarnings || hasRelatedWork + ? "Gittensory found maintainer review notes" + : "Gittensory PR readiness looks good"; const panelSummary = gateBlocking ? args.gate?.summary ?? (gateConclusion === "action_required" ? "Gittensory cannot evaluate the repo state closely enough for the enabled gate." : "A repo-configured hard blocker was found.") : linkedDuplicatePrs.length > 0 @@ -4308,7 +4300,25 @@ export function buildPublicPrIntelligenceComment(args: { ...formatAlertBlock([ `[!${alert}]`, `## ${panelTitle}`, - panelSummary, + ...(aiReview?.main + ? [ + "**Review summary**", + escapeAiReviewMarkdown(aiReview.main), + ...(aiReview.nits.length > 0 + ? [ + "", + "
", + `Nits (${aiReview.nits.length})`, + "", + ...aiReview.nits.map((nit) => `- ${escapeAiReviewMarkdown(nit)}`), + "", + "
", + ] + : []), + "", + panelSummary, + ] + : [panelSummary]), // Optional maintainer intro note (public-safe-validated at parse time; re-sanitized here). ...(args.review?.note ? ["", sanitizePanelText(args.review.note)] : []), "", @@ -4355,24 +4365,6 @@ export function buildPublicPrIntelligenceComment(args: { ...(nextSteps.length > 0 ? [...new Set(nextSteps)].map((step) => `- ${step}`) : ["- Keep the PR focused and include validation evidence before maintainer review."]), "", "
", - // Optional AI maintainer review (advisory; public-safe text built upstream). The deterministic - // signals above remain authoritative — this is a second opinion, not an endorsement. - ...(args.aiReview - ? [ - "", - "
", - "Gittensory AI review (advisory)", - "", - "_Generated from public PR metadata and the diff. Advisory only; deterministic signals remain authoritative._", - "", - // Notes are already public-safe and markdown-neutralized (built via toPublicSafe upstream). Escape - // angle brackets as a final guard so a stray tag (e.g.
or an HTML comment marker) cannot - // break the panel structure while preserving the section/bullet layout we add ourselves. - args.aiReview.notes.replace(/[<>]/g, (char) => (char === "<" ? "<" : ">")).slice(0, 4000), - "", - "", - ] - : []), "", `- [ ] ${PR_PANEL_RETRIGGER_MARKER} Re-run Gittensory review`, "", @@ -4886,6 +4878,23 @@ function formatAlertBlock(lines: string[]): string[] { return lines.map((line) => (line.length > 0 ? `> ${line}` : ">")); } +function aiReviewMainHasBlockers(main: string): boolean { + const marker = main.search(/\*\*Blockers\*\*/i); + if (marker === -1) return false; + const after = main.slice(marker).split(/\n(?=\*\*[^*]+\*\*)/)[0] ?? ""; + return after + .split("\n") + .slice(1) + .map((line) => line.replace(/^\s*[-*]\s*/, "").trim()) + .some((line) => line.length > 0 && !/^none\.?$/i.test(line)); +} + +function escapeAiReviewMarkdown(value: string): string { + return value + .replace(/[<>]/g, (char) => (char === "<" ? "<" : ">")) + .slice(0, 4000); +} + function isPrivateBountyLifecycleFinding(code: string): boolean { return code === "linked_issue_bounty_historical" || code === "linked_issue_bounty_unverified"; } diff --git a/test/unit/reputation-wiring.test.ts b/test/unit/reputation-wiring.test.ts index 688cc7f248..efb22f9fc6 100644 --- a/test/unit/reputation-wiring.test.ts +++ b/test/unit/reputation-wiring.test.ts @@ -89,6 +89,20 @@ describe("AI-spend gate: reputation downgrade", () => { expect(run).not.toHaveBeenCalled(); }); + it("FLAG-ON: aiReviewAllAuthors bypasses the reputation downgrade and still runs the review", async () => { + const { env, run } = aiEnv({ GITTENSORY_REVIEW_REPUTATION: "true" }); + await seedSubmitter(env, { project: "acme/widgets", submitter: "burster", submissions: 12, merged: 0, closed: 12, manual: 0 }); + const adv = advisory(); + const result = await runAiReviewForAdvisory(env, { + ...baseArgs, + advisory: adv, + settings: { aiReviewMode: "advisory", aiReviewAllAuthors: true } as RepositorySettings, + }); + + expect(result?.notes).toContain("Add a test."); + expect(run).toHaveBeenCalled(); + }); + it("FLAG-ON: a good-reputation submitter proceeds to the normal AI review", async () => { const { env, run } = aiEnv({ GITTENSORY_REVIEW_REPUTATION: "true" }); await seedSubmitter(env, { project: "acme/widgets", submitter: "burster", submissions: 20, merged: 18, closed: 2, manual: 0 }); diff --git a/test/unit/selfhost-pg-queue.test.ts b/test/unit/selfhost-pg-queue.test.ts index 7f7f1123bb..6d735b52dd 100644 --- a/test/unit/selfhost-pg-queue.test.ts +++ b/test/unit/selfhost-pg-queue.test.ts @@ -237,6 +237,33 @@ describe("createPgQueue (durable #977)", () => { expect(seen).toEqual(["review"]); }); + it("claims foreground work before falling back to the capped background lane", async () => { + const claimSql: string[] = []; + const fn = vi.fn().mockImplementation(async (sql: unknown) => { + const q = String(sql); + if (q.includes("SELECT id, payload, priority")) return { rows: [], rowCount: 0 }; + if (q.includes("SELECT id, payload, job_key") && q.includes("status IN")) return { rows: [], rowCount: 0 }; + if (q.includes("WHERE status='processing'")) return { rows: [], rowCount: 0 }; + if (q.includes("UPDATE _selfhost_jobs SET status='processing'")) { + claimSql.push(q); + return { rows: [], rowCount: 0 }; + } + return { rows: [], rowCount: 0 }; + }); + const q = createPgQueue( + { query: fn } as unknown as Pool, + async () => undefined, + { backgroundConcurrency: 1 }, + ); + + await q.init(); + await q.drain(); + + expect(claimSql).toHaveLength(2); + expect(claimSql[0]).toContain("priority >= $2"); + expect(claimSql[1]).toContain("priority < $2"); + }); + it("dead-letters an unparseable payload (job_dead audit emitted)", async () => { const m = makePool(); // Claim returns a row with bad payload. diff --git a/test/unit/selfhost-queue-common.test.ts b/test/unit/selfhost-queue-common.test.ts index b5ac53eb9a..b9edafa335 100644 --- a/test/unit/selfhost-queue-common.test.ts +++ b/test/unit/selfhost-queue-common.test.ts @@ -1,9 +1,12 @@ import { describe, expect, it, vi } from "vitest"; import { + FOREGROUND_QUEUE_PRIORITY_FLOOR, githubRateLimitRetryDelayMs, + isForegroundJobPriority, jobCoalesceKey, jobPriority, nonConsumingRetryDelayMs, + queueBackgroundConcurrency, } from "../../src/selfhost/queue-common"; import { RetryableJobError } from "../../src/queue/retryable"; @@ -13,6 +16,7 @@ describe("self-host queue common helpers", () => { it("classifies job priority by job type and webhook sender", () => { expect(jobPriority(payload({ type: "github-webhook" }))).toBe(10); expect(jobPriority(payload({ type: "agent-regate-pr" }))).toBe(9); + expect(jobPriority(payload({ type: "agent-regate-pr", deliveryId: "manual-regate:owner/repo#1:123" }))).toBe(99); expect(jobPriority(payload({ type: "recapture-preview" }))).toBe(9); expect(jobPriority(payload({ type: "agent-regate-sweep" }))).toBe(8); expect(jobPriority(payload({ type: "rag-index-repo" }))).toBe(0); @@ -20,6 +24,19 @@ describe("self-host queue common helpers", () => { expect(jobPriority("not-json")).toBe(0); }); + it("keeps foreground review work separate from capped background work", () => { + expect(FOREGROUND_QUEUE_PRIORITY_FLOOR).toBe(8); + expect(isForegroundJobPriority(10)).toBe(true); + expect(isForegroundJobPriority(8)).toBe(true); + expect(isForegroundJobPriority(7)).toBe(false); + expect(queueBackgroundConcurrency(4, undefined)).toBe(1); + expect(queueBackgroundConcurrency(4, "3")).toBe(3); + expect(queueBackgroundConcurrency(2, "9")).toBe(2); + expect(queueBackgroundConcurrency(4, "-1")).toBe(1); + expect(queueBackgroundConcurrency(4, "not-a-number")).toBe(1); + expect(queueBackgroundConcurrency(4, "0")).toBe(0); + }); + it("demotes bot-authored issue-comment edit webhooks without demoting human reruns", () => { const issueCommentEdit = (sender: { login?: string; type?: string }) => payload({ diff --git a/test/unit/selfhost-sqlite-queue.test.ts b/test/unit/selfhost-sqlite-queue.test.ts index a0b106917d..64c05a5797 100644 --- a/test/unit/selfhost-sqlite-queue.test.ts +++ b/test/unit/selfhost-sqlite-queue.test.ts @@ -526,7 +526,7 @@ describe("createSqliteQueue (durable #980)", () => { await gate; concurrent--; }, - { concurrency: 3, pollIntervalMs: 100_000 }, + { concurrency: 3, backgroundConcurrency: 3, pollIntervalMs: 100_000 }, ); try { q.start(); @@ -539,6 +539,47 @@ describe("createSqliteQueue (durable #980)", () => { } }); + it("caps background jobs so foreground review work keeps a worker slot", async () => { + const driver = makeDriver(); + const started: string[] = []; + const releases: Array<() => void> = []; + let blockedBackground = false; + const q = createSqliteQueue( + driver, + async (m) => { + const type = typeOf(m); + started.push(type); + if (type === "rag-index-repo" && !blockedBackground) { + blockedBackground = true; + await new Promise((resolve) => { + releases.push(resolve); + }); + } + }, + { concurrency: 2, backgroundConcurrency: 1, pollIntervalMs: 100_000 }, + ); + try { + await q.binding.sendBatch([ + { body: msg("rag-index-repo") }, + { body: msg("rag-index-repo") }, + ]); + for (let i = 0; i < 20 && releases.length === 0; i += 1) + await new Promise((r) => setTimeout(r, 10)); + + expect(started).toEqual(["rag-index-repo"]); + + await q.binding.send(msg("agent-regate-pr")); + for (let i = 0; i < 20 && !started.includes("agent-regate-pr"); i += 1) + await new Promise((r) => setTimeout(r, 10)); + + expect(started).toContain("agent-regate-pr"); + expect(started.filter((type) => type === "rag-index-repo")).toHaveLength(1); + } finally { + for (const release of releases) release(); + await q.stop(); + } + }); + it("recovers a job left 'processing' by a crash", async () => { const oldRecoveryJitter = process.env.QUEUE_RECOVERY_JITTER_MS; process.env.QUEUE_RECOVERY_JITTER_MS = "0"; @@ -659,7 +700,7 @@ describe("createSqliteQueue (durable #980)", () => { maxConcurrent = Math.max(maxConcurrent, concurrent); await new Promise((r) => setTimeout(r, 15)); concurrent--; - }, { concurrency: 2, pollIntervalMs: 100_000 }); + }, { concurrency: 2, backgroundConcurrency: 2, pollIntervalMs: 100_000 }); await q.binding.sendBatch([{ body: msg("a") }, { body: msg("b") }]); await new Promise((r) => setTimeout(r, 60)); await q.stop(); diff --git a/test/unit/signals-coverage.test.ts b/test/unit/signals-coverage.test.ts index ed3544e93f..3cb7d7f0c6 100644 --- a/test/unit/signals-coverage.test.ts +++ b/test/unit/signals-coverage.test.ts @@ -928,16 +928,39 @@ describe("signal coverage edge cases", () => { preflight: buildPreflightResult({ repoFullName: directRepo.fullName, title: "Fix isolated issue", body: "Fixes #99", linkedIssues: [99] }, directRepo, [], [currentPr]), settings: gateSettings, review: { present: true, footerText: "Reviewed by the Acme maintainer bot.", note: "Run npm test before pushing.", fields: { relatedWork: false }, profile: null, inlineComments: null, pathInstructions: [], instructions: null, excludePaths: [], preMergeChecks: [] }, - aiReview: { notes: "The change is focused.\n\n**Suggestions**\n- Add a test for the edge case." }, + aiReview: { notes: "The change is focused.\n\n**Nits (2)**\n- Add a test for the edge case.\n- Keep the validator helper scoped." }, }); expect(customizedComment).toContain("Reviewed by the Acme maintainer bot."); // custom footer lead expect(customizedComment).toContain("register to start earning"); // mandatory attribution/earn link kept expect(customizedComment).toContain("Run npm test before pushing."); // intro note expect(customizedComment).not.toContain("| Related work |"); // hidden row expect(customizedComment).toContain("| Gate result |"); // non-hidden rows still rendered - expect(customizedComment).toContain("Gittensory AI review (advisory)"); // AI section rendered + expect(customizedComment).toContain("**Review summary**"); // AI summary is prominent, not buried + expect(customizedComment).toContain("Nits (2)"); // nits are directly below summary + expect(customizedComment).not.toContain("Gittensory AI review (advisory)"); // old bottom dropdown removed expect(customizedComment).toContain("</details>"); // stray tags escaped, panel structure preserved - expect(customizedComment).toContain("- Add a test for the"); // markdown bullets preserved (not flattened) + const summaryIndex = customizedComment.indexOf("**Review summary**"); + const nitsIndex = customizedComment.indexOf("Nits (2)"); + const readinessIndex = customizedComment.indexOf("**Readiness score:"); + expect(summaryIndex).toBeGreaterThan(-1); + expect(nitsIndex).toBeGreaterThan(summaryIndex); + expect(readinessIndex).toBeGreaterThan(nitsIndex); + + const aiBlockedComment = buildPublicPrIntelligenceComment({ + repo: directRepo, + pr: { ...currentPr, linkedIssues: [99], body: "Fixes #99" }, + profile, + detection, + queueHealth: buildQueueHealth(directRepo, [], [currentPr], buildCollisionReport(directRepo.fullName, [], [currentPr])), + collisions: buildCollisionReport(directRepo.fullName, [], [currentPr]), + preflight: buildPreflightResult({ repoFullName: directRepo.fullName, title: "Fix isolated issue", body: "Fixes #99", linkedIssues: [99] }, directRepo, [], [currentPr]), + settings: { ...repoSettings(directRepo.fullName), gateCheckMode: "off" }, + aiReview: { notes: "The change is currently unsafe to merge.\n\n**Blockers**\n- `src/a.ts` has a syntax error.\n\n**Nits (1)**\n- Add a regression test." }, + }); + expect(aiBlockedComment).toContain("> [!CAUTION]"); + expect(aiBlockedComment).toContain("Gittensory review found blockers"); + expect(aiBlockedComment).toContain("`src/a.ts` has a syntax error."); + expect(aiBlockedComment.indexOf("**Review summary**")).toBeLessThan(aiBlockedComment.indexOf("**Readiness score:")); const advisoryOnlyComment = buildPublicPrIntelligenceComment({ repo: directRepo, @@ -958,7 +981,7 @@ describe("signal coverage edge cases", () => { settings: { ...repoSettings(directRepo.fullName), gateCheckMode: "off" }, }); - expect(advisoryOnlyComment).toContain("> [!IMPORTANT]"); + expect(advisoryOnlyComment).toContain("> [!WARNING]"); expect(advisoryOnlyComment).toContain("Gittensory found maintainer review notes"); expect(advisoryOnlyComment).toContain("Validation note missing"); expect(advisoryOnlyComment).toContain("> | Gate result | ⚠️ Advisory only | Advisory only. | No action. |"); @@ -979,7 +1002,7 @@ describe("signal coverage edge cases", () => { settings: gateSettings, gate: { conclusion: "action_required", summary: "Gittensory cannot evaluate this PR until installation state is repaired." }, }); - expect(actionRequiredComment).toContain("> [!IMPORTANT]"); + expect(actionRequiredComment).toContain("> [!WARNING]"); expect(actionRequiredComment).toContain("Gittensory cannot evaluate this PR until installation state is repaired."); expect(actionRequiredComment).toContain("> | Gate result | ⚠️ App action required | Install/config needs attention. | Fix app config. |"); diff --git a/test/unit/unified-comment-bridge.test.ts b/test/unit/unified-comment-bridge.test.ts index 64b01b77b6..d419b856be 100644 --- a/test/unit/unified-comment-bridge.test.ts +++ b/test/unit/unified-comment-bridge.test.ts @@ -122,6 +122,18 @@ describe("buildDualReviewNotes", () => { expect(reviews[0]?.notes?.nits).toEqual(["No test"]); // title only, no trailing " — " }); + it("does not repeat the consensus defect detail when the gate title already embeds it", () => { + const reviews = buildDualReviewNotes({ + consensusDefect: { + title: "AI reviewers agree on a likely critical defect: src/types.ts:111 leaves `Finding` unclosed", + detail: "src/types.ts:111 leaves `Finding` unclosed", + }, + recommendation: "close", + verdict: "close", + }); + expect(reviews[0]?.notes?.blockers).toEqual(["src/types.ts:111 leaves `Finding` unclosed"]); + }); + it("demotes self-host environmental/process warnings out of the nits, keeping real code nits (#review-accuracy)", () => { const reviews = buildDualReviewNotes({ aiReview: { notes: "Looks fine." }, diff --git a/test/unit/unified-comment-parity.test.ts b/test/unit/unified-comment-parity.test.ts index 6913f9b96c..a9fef704d8 100644 --- a/test/unit/unified-comment-parity.test.ts +++ b/test/unit/unified-comment-parity.test.ts @@ -117,7 +117,7 @@ describe("converged comment ↔ legacy panel parity (#unified-comment)", () => { reviewerCount: aiReview.reviewerCount, footerMarkdown: "💰 Earn for open-source contributions like this. Checked by Gittensory.", reRunLabel: "gittensory-pr-panel:retrigger Re-run Gittensory review", - extraCollapsibles: buildPublicSafeCollapsibles({ repo, pr: currentPr, profile, detection, settings, collisions, preflight, queueHealth, aiReview }), + extraCollapsibles: buildPublicSafeCollapsibles({ repo, pr: currentPr, profile, detection, settings, collisions, preflight, queueHealth }), }); // The three public-safe sections the legacy panel carried must survive into the converged comment. @@ -125,12 +125,12 @@ describe("converged comment ↔ legacy panel parity (#unified-comment)", () => { expect(body).toContain("Contributor next steps"); expect(body).toContain("Signal definitions"); // With an AI review present the converged comment also surfaces the optional Review-details section. - expect(body).toContain("Review details"); + expect(body).not.toContain("Review details"); // PRIVATE — the maintainer-notes / advisory-findings section must NEVER appear in the public converged comment. expect(body).not.toContain("Maintainer notes"); }); - it("omits the AI 'Review details' collapsible when there is no AI review (renderer skips the empty body)", () => { + it("never includes a duplicate AI 'Review details' collapsible", () => { const { currentPr, detection, collisions, queueHealth, preflight, profile } = buildFixtures(); const collapsibles = buildPublicSafeCollapsibles({ repo, pr: currentPr, profile, detection, settings, collisions, preflight, queueHealth }); expect(collapsibles.map((section) => section.title)).toEqual(["Review context", "Contributor next steps", "Signal definitions"]); @@ -144,12 +144,11 @@ describe("converged comment ↔ legacy panel parity (#unified-comment)", () => { const { currentPr, detection, collisions, queueHealth, preflight, profile } = buildFixtures(); const aiReview = { notes: "Looks reasonable. Add a regression test for reconnect.", reviewerCount: 2 }; const legacy = buildPublicPrIntelligenceComment({ repo, pr: currentPr, profile, detection, queueHealth, collisions, preflight, settings, aiReview }); - const collapsibles = buildPublicSafeCollapsibles({ repo, pr: currentPr, profile, detection, settings, collisions, preflight, queueHealth, aiReview }); + const collapsibles = buildPublicSafeCollapsibles({ repo, pr: currentPr, profile, detection, settings, collisions, preflight, queueHealth }); // Each shared collapsible body's individual lines must appear verbatim in the legacy panel so the two // renderers can never diverge on the public-safe content. for (const section of collapsibles) { - if (section.title === "Review details") continue; // Legacy renders this as "Gittensory AI review (advisory)". for (const line of section.body.split("\n")) { if (line.trim() === "") continue; expect(legacy).toContain(line); diff --git a/test/unit/unified-comment.test.ts b/test/unit/unified-comment.test.ts index 4915951c4c..a34f41df31 100644 --- a/test/unit/unified-comment.test.ts +++ b/test/unit/unified-comment.test.ts @@ -134,6 +134,8 @@ describe("renderUnifiedReviewComment", () => { expect(md).toContain("| **Code review** | ✅ No blockers | 2 reviewers, synthesized |"); expect(md).toContain("| Linked issue | ✅ Linked | #1372 |"); expect(md).toContain("
Nits — 1 non-blocking"); + expect(md.indexOf("**Review summary**")).toBeLessThan(md.indexOf("
Nits")); + expect(md.indexOf("
Nits")).toBeLessThan(md.indexOf("| Signal | Result | Evidence |")); expect(md).toContain("
Signal definitions"); expect(md).toContain("- [ ] Re-run Gittensory review"); expect(md).toContain("Checked by Gittensory."); From 577f9939a6a4533233a8896420e030dbaeb0c1e1 Mon Sep 17 00:00:00 2001 From: JSONbored <49853598+JSONbored@users.noreply.github.com> Date: Mon, 29 Jun 2026 01:50:26 -0700 Subject: [PATCH 45/68] fix(selfhost): bound retryable review jobs --- src/selfhost/pg-queue.ts | 4 ++- src/selfhost/queue-common.ts | 9 +++++- src/selfhost/sqlite-queue.ts | 4 ++- test/unit/selfhost-pg-queue.test.ts | 32 ++++++++++++++++--- test/unit/selfhost-queue-common.test.ts | 22 ++++++++++++- test/unit/selfhost-sqlite-queue.test.ts | 42 ++++++++++++++++--------- 6 files changed, 90 insertions(+), 23 deletions(-) diff --git a/src/selfhost/pg-queue.ts b/src/selfhost/pg-queue.ts index 26a499b26e..8466c3c667 100644 --- a/src/selfhost/pg-queue.ts +++ b/src/selfhost/pg-queue.ts @@ -7,6 +7,7 @@ import { logAudit, extractPayloadType } from "./audit"; import { incr } from "./metrics"; import { captureError } from "./sentry"; import { + consumingRetryDelayMs, deterministicJitterMs, FOREGROUND_QUEUE_PRIORITY_FLOOR, githubRateLimitRetryDelayMs, @@ -427,9 +428,10 @@ export function createPgQueue( attempts, }); } else { + const retryDelayMs = consumingRetryDelayMs(error, backoff(attempts)); await pool.query( `UPDATE ${TABLE} SET status='pending', attempts=$1, run_after=$2, last_error=$3 WHERE id=$4`, - [attempts, Date.now() + backoff(attempts), errMsg, job.id], + [attempts, Date.now() + retryDelayMs, errMsg, job.id], ); logAudit({ event: "job_error", diff --git a/src/selfhost/queue-common.ts b/src/selfhost/queue-common.ts index 5047ed29af..447def3fbf 100644 --- a/src/selfhost/queue-common.ts +++ b/src/selfhost/queue-common.ts @@ -129,7 +129,14 @@ export function githubRateLimitRetryDelayMs( } export function nonConsumingRetryDelayMs(error: unknown): number | null { - return githubRateLimitRetryDelayMs(error) ?? retryableJobDelayMs(error); + return githubRateLimitRetryDelayMs(error); +} + +export function consumingRetryDelayMs( + error: unknown, + defaultDelayMs: number, +): number { + return retryableJobDelayMs(error) ?? defaultDelayMs; } export function rateLimitRetryDelayWithJitter( diff --git a/src/selfhost/sqlite-queue.ts b/src/selfhost/sqlite-queue.ts index 5780b03bdf..810bdbbfd4 100644 --- a/src/selfhost/sqlite-queue.ts +++ b/src/selfhost/sqlite-queue.ts @@ -8,6 +8,7 @@ import { logAudit, extractPayloadType } from "./audit"; import { incr } from "./metrics"; import { captureError } from "./sentry"; import { + consumingRetryDelayMs, deterministicJitterMs, FOREGROUND_QUEUE_PRIORITY_FLOOR, githubRateLimitRetryDelayMs, @@ -370,9 +371,10 @@ export function createSqliteQueue( attempts, }); } else { + const retryDelayMs = consumingRetryDelayMs(error, backoff(attempts)); driver.query( `UPDATE ${TABLE} SET status='pending', attempts=?, run_after=?, last_error=? WHERE id=?`, - [attempts, Date.now() + backoff(attempts), errMsg, job.id], + [attempts, Date.now() + retryDelayMs, errMsg, job.id], ); logAudit({ event: "job_error", diff --git a/test/unit/selfhost-pg-queue.test.ts b/test/unit/selfhost-pg-queue.test.ts index 6d735b52dd..c6be351199 100644 --- a/test/unit/selfhost-pg-queue.test.ts +++ b/test/unit/selfhost-pg-queue.test.ts @@ -447,9 +447,9 @@ describe("createPgQueue (durable #977)", () => { } }); - it("reschedules retryable incomplete review jobs without consuming the dead-letter budget", async () => { + it("reschedules retryable incomplete review jobs while consuming attempts", async () => { const m = makePool(); - m.enqueueJob("1", { type: "agent-regate-pr" }, 4); + m.enqueueJob("1", { type: "agent-regate-pr" }, 0); const retryable = new RetryableJobError("AI review did not produce a public summary yet", { retryAfterMs: 5_000, retryKind: "ai_review_public_summary_missing", @@ -459,13 +459,13 @@ describe("createPgQueue (durable #977)", () => { async () => { throw retryable; }, - { maxRetries: 1, backoffMs: () => 0 }, + { maxRetries: 2, backoffMs: () => 0 }, ); await q.init(); await q.drain(); expect(m.pool.query).toHaveBeenCalledWith( - expect.stringContaining("SET status='pending', run_after=$1"), - expect.arrayContaining([expect.any(Number), "AI review did not produce a public summary yet", "1"]), + expect.stringContaining("SET status='pending', attempts=$1, run_after=$2"), + expect.arrayContaining([1, expect.any(Number), "AI review did not produce a public summary yet", "1"]), ); expect(m.pool.query).not.toHaveBeenCalledWith( expect.stringContaining("status='dead'"), @@ -473,6 +473,28 @@ describe("createPgQueue (durable #977)", () => { ); }); + it("dead-letters retryable incomplete review jobs when bounded attempts are exhausted", async () => { + const m = makePool(); + m.enqueueJob("1", { type: "agent-regate-pr" }, 0); + const retryable = new RetryableJobError("AI review did not produce a public summary yet", { + retryAfterMs: 5_000, + retryKind: "ai_review_public_summary_missing", + }); + const q = createPgQueue( + m.pool, + async () => { + throw retryable; + }, + { maxRetries: 1, backoffMs: () => 0 }, + ); + await q.init(); + await q.drain(); + expect(m.pool.query).toHaveBeenCalledWith( + expect.stringContaining("status='dead'"), + expect.arrayContaining([1, "AI review did not produce a public summary yet", "1"]), + ); + }); + it("records 'unknown error' when consumer throws a non-Error", async () => { const m = makePool(); m.enqueueJob("1", { type: "t" }, 0); diff --git a/test/unit/selfhost-queue-common.test.ts b/test/unit/selfhost-queue-common.test.ts index b9edafa335..677c14ffe8 100644 --- a/test/unit/selfhost-queue-common.test.ts +++ b/test/unit/selfhost-queue-common.test.ts @@ -1,6 +1,7 @@ import { describe, expect, it, vi } from "vitest"; import { FOREGROUND_QUEUE_PRIORITY_FLOOR, + consumingRetryDelayMs, githubRateLimitRetryDelayMs, isForegroundJobPriority, jobCoalesceKey, @@ -134,8 +135,14 @@ describe("self-host queue common helpers", () => { ).toBe(8_000); }); - it("extracts non-consuming retry delays from retryable job errors", () => { + it("keeps only GitHub rate limits on the non-consuming retry path", () => { expect(nonConsumingRetryDelayMs(new Error("boom"))).toBeNull(); + expect( + nonConsumingRetryDelayMs({ + status: 429, + response: { headers: new Headers({ "retry-after": "2" }) }, + }), + ).toBe(2_000); expect( nonConsumingRetryDelayMs( new RetryableJobError("AI review pending", { @@ -143,6 +150,19 @@ describe("self-host queue common helpers", () => { retryKind: "ai_review_public_summary_missing", }), ), + ).toBeNull(); + }); + + it("uses RetryableJobError delays on the bounded consuming retry path", () => { + expect(consumingRetryDelayMs(new Error("boom"), 77)).toBe(77); + expect( + consumingRetryDelayMs( + new RetryableJobError("AI review pending", { + retryAfterMs: 1234, + retryKind: "ai_review_public_summary_missing", + }), + 77, + ), ).toBe(1234); }); }); diff --git a/test/unit/selfhost-sqlite-queue.test.ts b/test/unit/selfhost-sqlite-queue.test.ts index 64c05a5797..392a759b39 100644 --- a/test/unit/selfhost-sqlite-queue.test.ts +++ b/test/unit/selfhost-sqlite-queue.test.ts @@ -409,7 +409,7 @@ describe("createSqliteQueue (durable #980)", () => { expect(afterEnqueue.run_after).toBeGreaterThan(before + 100_000); }); - it("reschedules retryable incomplete review jobs without consuming the dead-letter budget", async () => { + it("consumes retryable incomplete review attempts and dead-letters after maxRetries", async () => { const driver = makeDriver(); let calls = 0; const retryable = new RetryableJobError("AI review did not produce a public summary yet", { @@ -422,9 +422,10 @@ describe("createSqliteQueue (durable #980)", () => { calls += 1; throw retryable; }, - { maxRetries: 1, backoffMs: () => 0 }, + { maxRetries: 2, backoffMs: () => 0 }, ); await q.binding.send(msg("agent-regate-pr")); + const before = Date.now(); await q.drain(); const { rows } = driver.query( "SELECT status, attempts, run_after, last_error FROM _selfhost_jobs", @@ -439,12 +440,24 @@ describe("createSqliteQueue (durable #980)", () => { expect(calls).toBe(1); expect(q.deadCount()).toBe(0); expect(row.status).toBe("pending"); - expect(row.attempts).toBe(0); - expect(row.run_after).toBeGreaterThan(Date.now()); + expect(row.attempts).toBe(1); + expect(row.run_after).toBeGreaterThanOrEqual(before + 5_000); expect(row.last_error).toContain("AI review did not produce"); + + driver.query("UPDATE _selfhost_jobs SET run_after=0", []); + await q.drain(); + const dead = driver.query( + "SELECT status, attempts, last_error FROM _selfhost_jobs", + [], + ).rows[0] as { status: string; attempts: number; last_error: string }; + expect(calls).toBe(2); + expect(q.deadCount()).toBe(1); + expect(dead.status).toBe("dead"); + expect(dead.attempts).toBe(2); + expect(dead.last_error).toContain("AI review did not produce"); }); - it("coalesces a keyed retryable review job into an existing pending duplicate", async () => { + it("does not coalesce bounded retryable review failures into an existing pending duplicate", async () => { const driver = makeDriver(); const retryable = new RetryableJobError("AI review did not produce a public summary yet", { retryAfterMs: 5_000, @@ -456,7 +469,7 @@ describe("createSqliteQueue (durable #980)", () => { async () => { throw retryable; }, - { maxRetries: 1, backoffMs: () => 0 }, + { maxRetries: 2, backoffMs: () => 0 }, ); driver.query( "INSERT INTO _selfhost_jobs (payload, status, attempts, run_after, created_at, priority, job_key) VALUES (?, 'pending', 0, 0, 0, 10, ?)", @@ -470,16 +483,17 @@ describe("createSqliteQueue (durable #980)", () => { await q.drain(); const rows = driver.query( - "SELECT payload, last_error FROM _selfhost_jobs ORDER BY id", + "SELECT payload, attempts, last_error FROM _selfhost_jobs ORDER BY id", [], - ).rows as Array<{ payload: string; last_error: string | null }>; - expect(rows).toHaveLength(1); - expect(JSON.parse(rows[0]!.payload).deliveryId).toBe("ci-existing"); + ).rows as Array<{ payload: string; attempts: number; last_error: string | null }>; + expect(rows).toHaveLength(2); + expect(JSON.parse(rows[0]!.payload).deliveryId).toBe("ci-active"); + expect(rows[0]!.attempts).toBe(1); expect(rows[0]!.last_error).toContain("AI review did not produce"); - expect(q.stats()).toMatchObject({ - gittensory_jobs_coalesced_total: 1, - gittensory_jobs_deferred_total: 1, - }); + expect(JSON.parse(rows[1]!.payload).deliveryId).toBe("ci-existing"); + expect(rows[1]!.attempts).toBe(0); + expect(rows[1]!.last_error).toBeNull(); + expect(q.stats().gittensory_jobs_coalesced_total ?? 0).toBe(0); }); it("SURVIVES A RESTART: a fresh queue over the same DB processes a persisted pending job", async () => { From 7ca978e8cfd77cc0f439ad98d51ecaa509bc0d2e Mon Sep 17 00:00:00 2001 From: JSONbored <49853598+JSONbored@users.noreply.github.com> Date: Mon, 29 Jun 2026 02:05:45 -0700 Subject: [PATCH 46/68] build(selfhost): bundle review CLIs by default --- .env.example | 7 +++++-- .github/workflows/release-selfhost.yml | 3 +-- .github/workflows/selfhost.yml | 5 +++++ Dockerfile | 8 ++++---- docker-compose.yml | 2 +- docs/self-host/ai-providers.md | 4 ++-- docs/self-host/troubleshooting.md | 7 ++++--- docs/self-hosting.md | 8 +++++--- 8 files changed, 27 insertions(+), 17 deletions(-) diff --git a/.env.example b/.env.example index 9aa0f6e210..8880647930 100644 --- a/.env.example +++ b/.env.example @@ -214,8 +214,12 @@ GITTENSORY_REVIEW_DRAFT=false # SENTRY_RELEASE= # SENTRY_TRACES_SAMPLE_RATE=0 -# --- AI review backend (optional; without it reviews run deterministically) --- +# --- AI review backend (optional; without AI_PROVIDER reviews run deterministically) --- # AI_SUMMARIES_ENABLED=true +# The self-host image bundles the Claude Code and Codex CLIs by default. Credentials and provider choice remain +# runtime-only: set AI_PROVIDER plus the provider-specific auth below. Set INSTALL_AI_CLIS=false only for a +# custom minimal local build that will never use the subscription CLI providers. +# INSTALL_AI_CLIS=true # Deprecated shared AI_* knobs are intentionally rejected at startup: # AI_BASE_URL, AI_API_KEY, AI_MODEL, AI_EFFORT, AI_TIMEOUT_MS. Use the explicit # provider-specific variables below so Claude, Codex, Ollama, OpenAI, and @@ -265,7 +269,6 @@ GITTENSORY_REVIEW_DRAFT=false # Codex (ChatGPT subscription) reviewer is fail-closed by default for self-host PR review: `codex exec` stores its # OAuth credential in auth.json on the same filesystem that prompt-influenced reviews can read. Isolated maintainer # deployments can opt in explicitly after mounting auth at /data/codex (the image exposes it as ~/.codex). -# INSTALL_AI_CLIS=true # compose build arg; bake claude/codex CLIs into the app image # GITTENSORY_ENABLE_UNSAFE_CODEX_REVIEWER=1 # Do NOT set CODEX_HOME for the app container; the provider rejects it so credentials are not advertised in env. # CODEX_AI_MODEL=gpt-5.5 # omit to let the Codex account default choose; set explicitly for repeatable reviews diff --git a/.github/workflows/release-selfhost.yml b/.github/workflows/release-selfhost.yml index 30f35fddda..a8e83c68fb 100644 --- a/.github/workflows/release-selfhost.yml +++ b/.github/workflows/release-selfhost.yml @@ -181,6 +181,5 @@ jobs: ``` Multi-arch (linux/amd64 + linux/arm64). See [docs/self-hosting.md](docs/self-hosting.md) for setup. - To include the Claude Code / Codex subscription CLIs, build locally with - `--build-arg INSTALL_AI_CLIS=true`. + Includes the Claude Code / Codex subscription CLIs by default; credentials stay runtime-only. Sentry release id baked into the image: `${{ steps.version.outputs.release }}`. diff --git a/.github/workflows/selfhost.yml b/.github/workflows/selfhost.yml index f2db817f4f..abadbccd50 100644 --- a/.github/workflows/selfhost.yml +++ b/.github/workflows/selfhost.yml @@ -85,6 +85,11 @@ jobs: --load \ -t gittensory:selfhost-ci . + - name: Smoke-test bundled AI CLIs + run: | + docker run --rm --entrypoint sh gittensory:selfhost-ci -c \ + 'command -v claude && claude --version && command -v codex && codex --version' + - name: Build release target with visual review deps run: | docker buildx build \ diff --git a/Dockerfile b/Dockerfile index f5755d0359..4e596091ae 100644 --- a/Dockerfile +++ b/Dockerfile @@ -30,10 +30,10 @@ ENV NODE_ENV=production \ MIGRATIONS_DIR=/app/migrations \ NPM_CONFIG_PREFIX=/home/node/.npm-global \ GITTENSORY_VERSION=${GITTENSORY_VERSION} -# Optional: bake the Claude Code / Codex CLIs so the `claude-code` / `codex` subscription providers (#979) -# work in-image. Build with `--build-arg INSTALL_AI_CLIS=true`. No credentials are baked — operators mint -# CLAUDE_CODE_OAUTH_TOKEN (`claude setup-token`) / codex auth at run time and pass it via the env. -ARG INSTALL_AI_CLIS=false +# Bake the Claude Code / Codex CLIs by default so the self-host image is ready for subscription reviewers (#979). +# No credentials are baked — operators mint CLAUDE_CODE_OAUTH_TOKEN (`claude setup-token`) / codex auth at run time +# and pass/mount them via env/volumes. Minimal custom builds can opt out with `--build-arg INSTALL_AI_CLIS=false`. +ARG INSTALL_AI_CLIS=true # codex's native (Rust) binary loads the SYSTEM CA trust store (rustls-native-certs); node:slim ships none, so the # `codex` provider fails every call with "no native root CA certificates found" without ca-certificates. RUN if [ "$INSTALL_AI_CLIS" = "true" ]; then apt-get update && apt-get install -y --no-install-recommends ca-certificates && rm -rf /var/lib/apt/lists/*; fi diff --git a/docker-compose.yml b/docker-compose.yml index 85c17090ee..89ca7e5384 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -29,7 +29,7 @@ services: build: context: . args: - INSTALL_AI_CLIS: "${INSTALL_AI_CLIS:-false}" + INSTALL_AI_CLIS: "${INSTALL_AI_CLIS:-true}" INSTALL_VISUAL_REVIEW: "${INSTALL_VISUAL_REVIEW:-false}" restart: unless-stopped ports: diff --git a/docs/self-host/ai-providers.md b/docs/self-host/ai-providers.md index 744309be78..cc7530cb88 100644 --- a/docs/self-host/ai-providers.md +++ b/docs/self-host/ai-providers.md @@ -10,8 +10,8 @@ wrong model, base URL, key, effort, or timeout. | `AI_PROVIDER` | Backend | Needs | | ----------------------------------------- | ----------------------------------------------------------------------- | --------------------------------------------------------------------------------------- | -| `claude-code` | Your **Claude** subscription via the `claude` CLI (read-only, headless) | `CLAUDE_CODE_OAUTH_TOKEN` (`claude setup-token`); CLI baked in (`INSTALL_AI_CLIS=true`) | -| `codex` | Your **Codex** subscription via the `codex` CLI | local `codex` auth mounted at `/data/codex`, CLI baked in, explicit unsafe opt-in | +| `claude-code` | Your **Claude** subscription via the `claude` CLI (read-only, headless) | `CLAUDE_CODE_OAUTH_TOKEN` (`claude setup-token`); CLI is bundled by default | +| `codex` | Your **Codex** subscription via the `codex` CLI | local `codex` auth mounted at `/data/codex`; CLI is bundled by default; explicit opt-in | | `anthropic` | Native **Anthropic API** (BYOK, per-token billing — no weekly limit) | `ANTHROPIC_API_KEY`, `ANTHROPIC_AI_MODEL` | | `ollama` / `openai-compatible` / `openai` | Any OpenAI-compatible `/chat/completions` (+ `/embeddings`) | provider-specific `*_AI_BASE_URL`, `*_AI_API_KEY`, `*_AI_MODEL` | diff --git a/docs/self-host/troubleshooting.md b/docs/self-host/troubleshooting.md index 6473635a7c..d0a07a6fb1 100644 --- a/docs/self-host/troubleshooting.md +++ b/docs/self-host/troubleshooting.md @@ -14,11 +14,12 @@ Real failure modes and their fixes, ordered by how often they bite. Each entry i **Symptom:** zero `ai_review` activity; reviews silently fall back to the deterministic panel. **Cause:** the CLI subscription providers shell out to the `claude` / `codex` binaries, but the image was built -**without** the AI CLIs. -**Fix:** rebuild with the build arg: +without the AI CLIs. Official/prebuilt images include them by default; this usually means a custom minimal image was +built with `INSTALL_AI_CLIS=false`. +**Fix:** use the official image, or rebuild the custom image with the default AI CLI bundle: ```bash -docker compose build --build-arg INSTALL_AI_CLIS=true gittensory +INSTALL_AI_CLIS=true docker compose build gittensory docker compose up -d --force-recreate gittensory docker exec gittensory-gittensory-1 sh -c 'which claude && claude --version' ``` diff --git a/docs/self-hosting.md b/docs/self-hosting.md index f6881092c3..a6c23e38c7 100644 --- a/docs/self-hosting.md +++ b/docs/self-hosting.md @@ -137,9 +137,11 @@ In `block` mode the combined decision drives the gate; in `advisory` mode it's n fail-closed — if a reviewer can't return a usable verdict, the PR is **held** for a human, never auto-merged. The free Cloudflare Workers-AI pair remains the cloud default (`consensus`) — these knobs are for self-host providers. -**Subscription CLIs in the image.** The `claude-code` / `codex` providers need their CLI present. Build the -image with `--build-arg INSTALL_AI_CLIS=true` (or `docker compose build --build-arg INSTALL_AI_CLIS=true`) to -bake them in, then provide `CLAUDE_CODE_OAUTH_TOKEN` / codex auth at run time. No credentials are baked in. +**Subscription CLIs in the image.** The official/prebuilt image and the default Compose build include the +`claude-code` / `codex` provider CLIs. Provider choice and credentials stay runtime-only: set +`AI_PROVIDER=claude-code`, `AI_PROVIDER=codex`, or a dual pair such as `AI_PROVIDER=claude-code,codex`, then provide +`CLAUDE_CODE_OAUTH_TOKEN` / codex auth at run time. No credentials are baked in. Set `INSTALL_AI_CLIS=false` only for +a custom minimal local build that will never use the subscription CLI providers. - **Claude Code:** set `CLAUDE_CODE_OAUTH_TOKEN` (a 1-year token from `claude setup-token`, run once in a real terminal — it's browser-interactive and prints the token; it has no headless mode). The provider forces the From 1912f5e1f508b256726f1fdc6b7c7b9c50cdab1e Mon Sep 17 00:00:00 2001 From: JSONbored <49853598+JSONbored@users.noreply.github.com> Date: Mon, 29 Jun 2026 02:38:46 -0700 Subject: [PATCH 47/68] fix(selfhost): require Redis review runtime --- .env.example | 4 +- docker-compose.yml | 56 +++++++++---------- src/env.d.ts | 8 +-- src/github/webhook.ts | 8 ++- src/index.ts | 39 +++++++++---- src/queue/dlq.ts | 7 ++- src/queue/processors.ts | 5 +- src/selfhost/redis-cache.ts | 2 +- src/selfhost/redis-ratelimit.ts | 4 +- src/selfhost/redis-response-cache.ts | 4 +- src/selfhost/redis-token-cache.ts | 2 +- src/selfhost/review-runtime.ts | 23 ++++++++ src/server.ts | 83 +++++++++++++--------------- test/helpers/d1.ts | 12 ++++ test/unit/dlq.test.ts | 15 ++++- test/unit/index.test.ts | 70 +++++++++++++++++++++++ test/unit/webhook.test.ts | 47 +++++++++++++--- worker-configuration.d.ts | 18 +++--- wrangler.jsonc | 65 +++++----------------- 19 files changed, 298 insertions(+), 174 deletions(-) create mode 100644 src/selfhost/review-runtime.ts diff --git a/.env.example b/.env.example index 8880647930..9425c20f69 100644 --- a/.env.example +++ b/.env.example @@ -127,8 +127,8 @@ GITTENSORY_REVIEW_DRAFT=false # DATABASE_PATH=/data/gittensory.sqlite # SQLite file on the mounted data volume; all migrations auto-apply # DATABASE_URL= # set to postgres://user:pw@host:5432/db to use Postgres instead of # # SQLite (shared DB → multi-instance). Overrides DATABASE_PATH. -# REDIS_URL= # set to redis://host:6379 for distributed rate limiting + webhook dedup -# # cache (prevents double-processing of GitHub retries). Off when unset. +REDIS_URL=redis://redis:6379 # REQUIRED for the self-host review runtime. The default compose stack +# # starts Redis automatically; override for an external Redis. # QDRANT_URL= # set to http://qdrant:6333 to use Qdrant as the RAG vector store # # (--profile qdrant). Overrides the built-in sqlite-vec / pgvector. # DISCORD_WEBHOOK_URL= # one Discord channel for per-action notifications (merged/closed/ diff --git a/docker-compose.yml b/docker-compose.yml index 89ca7e5384..e6e5b616e9 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -5,10 +5,9 @@ # # PROFILES — activate optional services by passing --profile (combine freely): # -# (none) SQLite single-node stack (default — no flags needed) +# (none) SQLite + Redis single-node stack (default — no flags needed) # --profile postgres pgvector/pg16 shared database (multi-instance capable) # --profile pgbouncer PgBouncer connection pooler in front of Postgres -# --profile redis Redis fixed-window rate limiter # --profile qdrant Qdrant vector database for RAG # --profile ollama Local Ollama AI backend # --profile litestream Continuous SQLite backup to S3/B2/R2 via Litestream @@ -18,7 +17,7 @@ # --profile runners GitHub Actions self-hosted runner # # Examples: -# docker compose up --build # SQLite, no AI +# docker compose up --build # SQLite + Redis, no external AI # docker compose --profile postgres --profile caddy up -d # Postgres + HTTPS # docker compose --profile observability up -d # metrics + logs + dashboards # docker compose --profile tailscale --profile runners up -d # tailnet + CI runners @@ -61,8 +60,8 @@ services: # PGVECTOR_ENABLED: "true" # With --profile pgbouncer, route through the pooler instead of postgres directly: # DATABASE_URL: postgres://gittensory:${POSTGRES_PASSWORD:-CHANGEME}@pgbouncer:5432/gittensory - # Uncomment for Redis rate limiting (--profile redis): - # REDIS_URL: redis://redis:6379 + # Required shared transient state for review correctness, webhook dedup, and rate limiting. + REDIS_URL: "${REDIS_URL:-redis://redis:6379}" # Uncomment for Qdrant RAG vector store (--profile qdrant): # QDRANT_URL: http://qdrant:6333 # Uncomment for Ollama AI (--profile ollama): @@ -78,6 +77,8 @@ services: # locally (gitignored — never commit real policy). Absent ⇒ Docker mounts an empty dir ⇒ defaults apply. - ./gittensory-config:/config:ro depends_on: + redis: + condition: service_healthy # Gate startup on a healthy Qdrant when --profile qdrant is active; required:false means the # dependency is ignored when qdrant isn't started (default/other profiles). Belt-and-suspenders # with the app-side waitForQdrant retry in src/server.ts (which also covers a mid-life restart). @@ -101,6 +102,27 @@ services: start_period: 60s retries: 3 + # ── Redis (always on; required by the review runtime) ───────────────────── + # Shared transient review state: pending-CI first-seen stamps, webhook dedup/coalescing, rate limits, and + # short-lived GitHub caches. Losing it on restart is safe; the app requires it while running. + redis: + image: redis:7-alpine + restart: unless-stopped + command: + - redis-server + - --maxmemory + - 256mb + - --maxmemory-policy + - allkeys-lru + - --save + - "" + - --appendonly + - "no" + healthcheck: + test: ["CMD", "redis-cli", "ping"] + interval: 10s + retries: 5 + # ── Postgres (--profile postgres | --profile pgbouncer) ─────────────────── postgres: image: pgvector/pgvector:pg16 @@ -139,30 +161,6 @@ services: DEFAULT_POOL_SIZE: "20" AUTH_TYPE: md5 - # ── Redis (--profile redis) ──────────────────────────────────────────────── - # Ephemeral rate-limiter + 5-min webhook-dedup cache only — losing it is harmless, so persistence is - # fully disabled. --save "" turns off RDB snapshots; --appendonly no turns off AOF. allkeys-lru caps - # memory at 256mb and evicts least-recently-used keys under pressure instead of OOM-killing. No volume: - # nothing here needs to survive a restart. (The empty-string and "no" are quoted so YAML keeps them.) - redis: - image: redis:7-alpine - restart: unless-stopped - profiles: ["redis"] - command: - - redis-server - - --maxmemory - - 256mb - - --maxmemory-policy - - allkeys-lru - - --save - - "" - - --appendonly - - "no" - healthcheck: - test: ["CMD", "redis-cli", "ping"] - interval: 10s - retries: 5 - # ── Qdrant (--profile qdrant) ───────────────────────────────────────────── # Dedicated vector database for RAG — replaces the built-in sqlite-vec / pgvector when # QDRANT_URL=http://qdrant:6333 is set. Scales to millions of vectors with ANN search. diff --git a/src/env.d.ts b/src/env.d.ts index 819e251a64..f704fca741 100644 --- a/src/env.d.ts +++ b/src/env.d.ts @@ -2,6 +2,8 @@ declare global { interface Env { DB: D1Database; JOBS: Queue; + /** Self-host webhook queue binding. Cloudflare no longer binds this because hosted reviews are retired. */ + WEBHOOKS?: Queue; RATE_LIMITER?: DurableObjectNamespace; AI?: Ai; /** Self-host (RAG): a DEDICATED embedding provider, kept SEPARATE from the review chat chain so the reviewer @@ -20,10 +22,8 @@ declare global { /** Convergence (infra): Browser Rendering binding for visual (before/after screenshot) capture. Optional — * absent ⇒ no visual capture. Unused until the per-module wiring chunk; an unbound deploy is inert. */ BROWSER?: Fetcher; - /** Convergence (infra): the shared REVIEW_CONFIG KV (reviewbot's per-repo config, keyed by repo slug). - * The converged auto-maintain path resolves each repo's `hardGuardrailGlobs` from it so guarded paths - * force MANUAL review (no auto-merge / auto-close). Optional — absent ⇒ the conservative - * DEFAULT_CRUCIAL_GUARDRAIL_GLOBS fallback applies (CI workflows + scripts still guarded). */ + /** Legacy reviewbot KV shape. Cloudflare no longer binds this; self-host review policy should come from + * container-private config. Existing readers are optional and fall back when absent. */ REVIEW_CONFIG?: KVNamespace; /** Self-host transient cache for short-lived coalescing/backpressure keys. */ SELFHOST_TRANSIENT_CACHE?: { diff --git a/src/github/webhook.ts b/src/github/webhook.ts index 43dc13e91d..8da7fe7581 100644 --- a/src/github/webhook.ts +++ b/src/github/webhook.ts @@ -3,6 +3,7 @@ import { getWebhookEvent, recordWebhookEvent } from "../db/repositories"; import type { GitHubWebhookPayload, JobMessage } from "../types"; import { sha256Hex, verifyGitHubSignature } from "../utils/crypto"; import { relayVerify } from "../orb/relay"; +import { isSelfHostedReviewRuntime } from "../selfhost/review-runtime"; import { isSelfAuthoredWebhookNoise } from "./self-authored"; const DEFAULT_MAX_WEBHOOK_BODY_BYTES = 1024 * 1024; @@ -38,6 +39,8 @@ export async function handleGitHubWebhook(c: Context<{ Bindings: Env }>): Promis export async function enqueueVerifiedWebhook(c: Context<{ Bindings: Env }>, deliveryId: string, eventName: string, rawBody: string): Promise { const result = await enqueueWebhookByEnv(c.env, deliveryId, eventName, rawBody); switch (result) { + case "review_unavailable": + return c.json({ error: "selfhost_review_runtime_required" }, 410); case "ignored": return c.json({ ok: true, deliveryId, eventName, status: "ignored" }, 202); case "invalid_json": @@ -51,12 +54,14 @@ export async function enqueueVerifiedWebhook(c: Context<{ Bindings: Env }>, deli } } -export type EnqueueWebhookResult = "queued" | "duplicate" | "ignored" | "invalid_json" | "enqueue_failed"; +export type EnqueueWebhookResult = "queued" | "duplicate" | "ignored" | "invalid_json" | "enqueue_failed" | "review_unavailable"; /** Env-based core of the webhook enqueue (parse → dedup → record → WEBHOOKS lane), with NO Hono Context. Shared by * the request-context receiver above AND the pull-mode relay drain loop (server.ts), which has no Context. Returns * a status the caller maps to a response / an ack decision. */ export async function enqueueWebhookByEnv(env: Env, deliveryId: string, eventName: string, rawBody: string): Promise { + if (!isSelfHostedReviewRuntime(env)) return "review_unavailable"; + let payload: GitHubWebhookPayload; try { payload = JSON.parse(rawBody) as GitHubWebhookPayload; @@ -90,6 +95,7 @@ export async function enqueueWebhookByEnv(env: Env, deliveryId: string, eventNam const message: JobMessage = { type: "github-webhook", deliveryId, eventName, payload }; try { + if (!env.WEBHOOKS) return "enqueue_failed"; // Send to the dedicated WEBHOOKS lane (not the shared JOBS queue) so a maintenance burst on JOBS can never // starve real GitHub events into the DLQ. (#audit-webhook-queue) await env.WEBHOOKS.send(message); diff --git a/src/index.ts b/src/index.ts index 22352d55f3..67a8743084 100644 --- a/src/index.ts +++ b/src/index.ts @@ -7,6 +7,7 @@ import { isOrbBrokerEnabled } from "./orb/broker"; import { isOpsEnabled } from "./review/ops-wire"; import { isRagEnabled } from "./review/rag-wire"; import { isSelfTuneEnabled } from "./review/selftune-wire"; +import { isReviewExecutionJob, isSelfHostedReviewRuntime } from "./selfhost/review-runtime"; import type { JobMessage } from "./types"; const app = createApp(); @@ -19,11 +20,23 @@ export default { // Both dead-letter queues (the maintenance lane's gittensory-jobs-dlq and the webhook lane's // gittensory-webhooks-dlq, #1276) drain through the same observability + self-heal consumer. if (batch.queue?.endsWith("-dlq")) { - await processDlqBatch(batch, env); + await processDlqBatch(batch, env, { redriveWebhooks: isSelfHostedReviewRuntime(env) }); return; } for (const message of batch.messages) { try { + if (!isSelfHostedReviewRuntime(env) && isReviewExecutionJob(message.body)) { + console.warn( + JSON.stringify({ + level: "warn", + event: "hosted_review_job_ignored", + messageId: message.id, + jobType: message.body.type, + }), + ); + message.ack(); + continue; + } await processJob(env, message.body); message.ack(); } catch (error) { @@ -65,11 +78,15 @@ async function enqueueScheduledJobs(env: Env, controller: ScheduledController): // budget is reserved for webhooks (which drive timely reviews) instead of compounding the backlog; the next // tick (~2 min) retries, and after the bucket resets the sweep resumes. Webhooks never pre-yield. const jobs: JobMessage[] = []; - const sweepThrottledUntil = await shouldWaitForGitHubRateLimit(env, MAINTENANCE_RESERVED_HEADROOM); - if (sweepThrottledUntil) { - console.log(JSON.stringify({ event: "regate_sweep_throttled", resetAt: sweepThrottledUntil })); - } else { - jobs.push({ type: "agent-regate-sweep", requestedBy: "schedule" }); + const selfHostedReviews = isSelfHostedReviewRuntime(env); + let sweepThrottledUntil: string | undefined; + if (selfHostedReviews) { + sweepThrottledUntil = await shouldWaitForGitHubRateLimit(env, MAINTENANCE_RESERVED_HEADROOM); + if (sweepThrottledUntil) { + console.log(JSON.stringify({ event: "regate_sweep_throttled", resetAt: sweepThrottledUntil })); + } else { + jobs.push({ type: "agent-regate-sweep", requestedBy: "schedule" }); + } } // Orb relay retry: re-attempt failed forwardOrbEvent calls each sweep cycle. Only enqueued when the // broker is enabled — brokered self-hosts register relay URLs; hosted-cloud instances have no relay failures. @@ -83,9 +100,9 @@ async function enqueueScheduledJobs(env: Env, controller: ScheduledController): // 30-min tick retries, and after the bucket resets the backfill resumes. The cheap single-call health jobs // (repair-data-fidelity, refresh-installation-health) stay unconditional — they cost ~one call and keep // installation/health state fresh even while the budget is reserved. - if (!sweepThrottledUntil) { + if (selfHostedReviews && !sweepThrottledUntil) { jobs.push({ type: "backfill-registered-repos", requestedBy: "schedule", mode: isFullSyncWindow ? "full" : "light" }); - } else { + } else if (selfHostedReviews) { console.log(JSON.stringify({ event: "backfill_throttled", resetAt: sweepThrottledUntil })); } jobs.push({ type: "repair-data-fidelity", requestedBy: "schedule" }); @@ -99,13 +116,13 @@ async function enqueueScheduledJobs(env: Env, controller: ScheduledController): // Convergence (ops / observability, flag GITTENSORY_REVIEW_OPS). Hourly anomaly scan over gittensory's own // review-outcome data. Enqueued ONLY when the flag is ON — flag-OFF (default) this job is never created, // so the cron tick does ZERO new work and the enqueued set is byte-identical to today. - if (isOpsEnabled(env)) jobs.push({ type: "ops-alerts", requestedBy: "schedule" }); + if (selfHostedReviews && isOpsEnabled(env)) jobs.push({ type: "ops-alerts", requestedBy: "schedule" }); // Convergence (self-improve / auto-tune, flag GITTENSORY_REVIEW_SELFTUNE). Hourly self-improvement tick over // gittensory's own review-outcome data: compute tuning recommendations, shadow-soak any strictly-tightening // one, and auto-promote it to live only after the soak window passes the gate (TIGHTENING-ONLY, audited). // Enqueued ONLY when the flag is ON — flag-OFF (default) this job is never created, so the cron tick does // ZERO new tuning work and the enqueued set is byte-identical to today. - if (isSelfTuneEnabled(env)) jobs.push({ type: "selftune", requestedBy: "schedule" }); + if (selfHostedReviews && isSelfTuneEnabled(env)) jobs.push({ type: "selftune", requestedBy: "schedule" }); } if (isHourly && scheduledAt.getUTCDay() === 1 && hour === 12) { jobs.push({ type: "generate-weekly-value-report", requestedBy: "schedule", variant: "operator", days: 7 }); @@ -125,7 +142,7 @@ async function enqueueScheduledJobs(env: Env, controller: ScheduledController): // registered + cutover-allowlisted repo, mirroring the signal-snapshot fan-out). Enqueued ONLY when the flag // is ON — flag-OFF (default) this job is never created, so the cron does ZERO new RAG work and the enqueued // set is byte-identical to today. - if (isRagEnabled(env)) jobs.push({ type: "rag-index-repo", requestedBy: "schedule" }); + if (selfHostedReviews && isRagEnabled(env)) jobs.push({ type: "rag-index-repo", requestedBy: "schedule" }); } await Promise.all(jobs.map((job) => env.JOBS.send(job))); } diff --git a/src/queue/dlq.ts b/src/queue/dlq.ts index cf37ce8393..b24086149d 100644 --- a/src/queue/dlq.ts +++ b/src/queue/dlq.ts @@ -14,7 +14,8 @@ import type { JobMessage, JsonValue } from "../types"; * the webhook lane — bounded to a single attempt by the `redriven` marker so a genuinely-poison payload * cannot loop the DLQ forever. Maintenance jobs are cron-self-healing, so they are audited-and-dropped. */ -export async function processDlqBatch(batch: MessageBatch, env: Env): Promise { +export async function processDlqBatch(batch: MessageBatch, env: Env, options: { redriveWebhooks?: boolean } = {}): Promise { + const redriveWebhooks = options.redriveWebhooks ?? true; for (const message of batch.messages) { const body = message.body as { type?: string } | null | undefined; const jobType = body?.type ?? "unknown"; @@ -38,14 +39,14 @@ export async function processDlqBatch(batch: MessageBatch, env: Env) metadata: { messageId: message.id, jobType, redriven: webhook?.redriven === true } satisfies Record, }).catch(() => undefined); // Self-heal a recoverable webhook: re-drive ONCE (not already re-driven, and not already processed). - if (webhook && webhook.redriven !== true && webhook.deliveryId) { + if (redriveWebhooks && webhook && webhook.redriven !== true && webhook.deliveryId) { const event = await getWebhookEvent(env, webhook.deliveryId).catch(() => null); if (event?.status !== "processed") { // If the webhook dead-lettered because the shared GitHub REST budget was exhausted, re-drive it AFTER the // reset (retry-until-recovered) rather than immediately re-failing it. (#audit-rate-headroom) const resetAt = await shouldWaitForGitHubRateLimit(env).catch(() => undefined); const options = resetAt ? { delaySeconds: delayUntil(resetAt) } : undefined; - await env.WEBHOOKS.send({ type: "github-webhook", deliveryId: webhook.deliveryId, eventName: webhook.eventName, payload: webhook.payload, redriven: true }, options).catch(() => undefined); + await env.WEBHOOKS?.send({ type: "github-webhook", deliveryId: webhook.deliveryId, eventName: webhook.eventName, payload: webhook.payload, redriven: true }, options).catch(() => undefined); } } message.ack(); diff --git a/src/queue/processors.ts b/src/queue/processors.ts index e89ff763a2..cc40e6c92d 100644 --- a/src/queue/processors.ts +++ b/src/queue/processors.ts @@ -1718,8 +1718,9 @@ async function prReadyForReview( // (an orphaned / never-completing check — e.g. a fork check that never reports back) would otherwise make us // defer FOREVER → the PR is silently stuck and never surfaces (the dominant metagraphed stall). Past // STUCK_CI_DEFER_MS we stop deferring and let the gate FINALIZE, so the PR is surfaced (held / needs-human), - // or disposed if a verdict is reachable — never silently deferred. first-seen is KV-tracked per PR+headSha (a - // new push = new SHA = fresh window); a KV miss degrades to the old defer (safe — never acts early). (#ci-stuck-finalize) + // or disposed if a verdict is reachable — never silently deferred. first-seen is tracked in the self-host + // Redis transient cache per PR+headSha (a new push = new SHA = fresh window); a cache miss degrades to the + // old defer (safe — never acts early). (#ci-stuck-finalize) if ( !(await ciPendingDeferStuck(env, repoFullName, pr.number, pr.headSha)) ) { diff --git a/src/selfhost/redis-cache.ts b/src/selfhost/redis-cache.ts index c3149c789f..d5d6b13d37 100644 --- a/src/selfhost/redis-cache.ts +++ b/src/selfhost/redis-cache.ts @@ -2,7 +2,7 @@ // deliveries from being processed twice — GitHub retries webhooks that receive a non-200 // response, and each retry carries the same `x-github-delivery` UUID. By caching the delivery // ID after a successful processing attempt, the server can return 204 immediately on retries -// without re-queuing the job. Activated when REDIS_URL is set alongside --profile redis. +// without re-queuing the job. The self-host review runtime requires REDIS_URL. import type { Redis } from "ioredis"; export function createRedisCache(redis: Redis) { diff --git a/src/selfhost/redis-ratelimit.ts b/src/selfhost/redis-ratelimit.ts index 61274aa55d..fd0e8cf8c9 100644 --- a/src/selfhost/redis-ratelimit.ts +++ b/src/selfhost/redis-ratelimit.ts @@ -1,7 +1,7 @@ // Redis-backed rate limiter for self-host (#977). The Cloudflare deploy uses a RateLimiter Durable Object; // self-host provides the SAME binding surface (idFromName → get → fetch) backed by a Redis fixed-window -// counter, so `enforceRateLimit` works unchanged and is shared across instances. Without REDIS_URL the binding -// is absent and enforceRateLimit returns null (no limiting) — same as today. +// counter, so `enforceRateLimit` works unchanged and is shared across instances. REDIS_URL is required by the +// self-host review runtime. import type { Redis } from "ioredis"; interface RateLimitBody { diff --git a/src/selfhost/redis-response-cache.ts b/src/selfhost/redis-response-cache.ts index e28e343136..83edcb1fb4 100644 --- a/src/selfhost/redis-response-cache.ts +++ b/src/selfhost/redis-response-cache.ts @@ -1,5 +1,5 @@ -// Redis-backed GitHub GET-response cache (#perf). Optional: when REDIS_URL + GITHUB_CACHE_TTL_SECONDS>0 are set, -// the self-host caches safe GitHub API GET responses for a short TTL. A single review pass makes ~24 GitHub +// Redis-backed GitHub GET-response cache (#perf). The self-host runtime requires REDIS_URL; when +// GITHUB_CACHE_TTL_SECONDS>0, it caches safe GitHub API GET responses for a short TTL. A single review pass makes ~24 GitHub // fetches (PR data, files, user/org lookups) — many repeated — all network-bound and rate-limited. A short-TTL // cache dedups those within and across rapid re-reviews, cutting latency and rate-limit pressure, and it // persists across restarts. Keyed by URL; the TTL bounds staleness. Only the status + body + content-type are diff --git a/src/selfhost/redis-token-cache.ts b/src/selfhost/redis-token-cache.ts index 9609cf325a..b590855fdf 100644 --- a/src/selfhost/redis-token-cache.ts +++ b/src/selfhost/redis-token-cache.ts @@ -1,4 +1,4 @@ -// Redis-backed installation-token store (#perf). Optional: when REDIS_URL is set, the self-host backs +// Redis-backed installation-token store (#perf). The self-host runtime requires REDIS_URL and backs // github/app.ts's installation-token cache with Redis so warm tokens SURVIVE restarts/deploys. The default // in-isolate Map dies on every restart, so a brokered self-host re-mints a token (an Orb round-trip) on the // next call after each cold start — wasteful when the container restarts often. Keyed by installation id, with diff --git a/src/selfhost/review-runtime.ts b/src/selfhost/review-runtime.ts new file mode 100644 index 0000000000..e7ec4f794f --- /dev/null +++ b/src/selfhost/review-runtime.ts @@ -0,0 +1,23 @@ +import type { JobMessage } from "../types"; + +const REVIEW_EXECUTION_JOB_TYPES = new Set([ + "github-webhook", + "recapture-preview", + "agent-regate-pr", + "agent-regate-sweep", + "run-agent", + "notify-evaluate", + "notify-deliver", + "ops-alerts", + "selftune", + "rag-index-repo", + "submit-draft", +]); + +export function isSelfHostedReviewRuntime(env: Pick): boolean { + return Boolean(env.SELFHOST_TRANSIENT_CACHE); +} + +export function isReviewExecutionJob(job: JobMessage | null | undefined): boolean { + return REVIEW_EXECUTION_JOB_TYPES.has(job?.type ?? ""); +} diff --git a/src/server.ts b/src/server.ts index c91a8f4b74..8739330cb7 100644 --- a/src/server.ts +++ b/src/server.ts @@ -1,8 +1,8 @@ // Self-host Node entry (#980). Runs gittensory's SAME Worker handlers on Node. Backends are pluggable: // • DB: SQLite (node:sqlite, default) OR Postgres (DATABASE_URL=postgres://… → shared, multi-instance). // • Queue: durable SQLite queue OR a Postgres queue (FOR UPDATE SKIP LOCKED). -// • Rate limit: a Redis fixed-window limiter when REDIS_URL is set (else no limiting, as today). -// • RAG vector store: SQLite-only for now (omitted on Postgres → RAG degrades to no-context). +// • Redis: required transient review state + fixed-window rate limiter. +// • RAG vector store: SQLite/pgvector by default, or Qdrant when QDRANT_URL is set. // Serves the Hono app via @hono/node-server, drives the queue with the same processJob, ticks the same // scheduled handler on a timer, exposes /health /ready /metrics, and shuts down gracefully. The Cloudflare // Worker (src/index.ts) is untouched — this is a parallel entry the self-host esbuild build bundles. @@ -384,8 +384,9 @@ async function main(): Promise { }), ); - // /ready gates on every CONFIGURED optional backend (below) so a load balancer never routes to an instance whose - // Redis/Qdrant is down. Each probe owns a short timeout so a hung backend can't hang the readiness check. + // /ready gates on required Redis plus every configured optional backend so a load balancer never routes to an + // instance whose shared state/vector backend is down. Each probe owns a short timeout so a hung backend can't + // hang the readiness check. const readinessProbes: ReadinessProbe[] = []; const withTimeout = (p: Promise, ms = 1500): Promise => Promise.race([ @@ -393,46 +394,38 @@ async function main(): Promise { new Promise((resolve) => setTimeout(() => resolve(false), ms)), ]); - // Redis fixed-window rate limiter + webhook dedup cache (else absent when REDIS_URL is unset). - let rateLimiter: DurableObjectNamespace | undefined; - let webhookCache: import("./selfhost/redis-cache").RedisCache | undefined; - if (process.env.REDIS_URL) { - const { Redis } = await import("ioredis"); - const redisClient = new Redis(process.env.REDIS_URL); - const { createRedisRateLimiter } = - await import("./selfhost/redis-ratelimit"); - const { createRedisCache } = await import("./selfhost/redis-cache"); - rateLimiter = createRedisRateLimiter(redisClient); - webhookCache = createRedisCache(redisClient); - // Persist the installation-token cache in Redis so warm GitHub App tokens survive restarts/deploys and are - // shared across replicas (the in-isolate Map otherwise re-mints — an Orb round-trip — per replica/cold start). - const { createRedisTokenCache } = - await import("./selfhost/redis-token-cache"); - const { setInstallationTokenStore, setGitHubResponseCache } = - await import("./github/app"); - setInstallationTokenStore(createRedisTokenCache(redisClient)); - // Short-TTL cache for safe GitHub GET responses (dedups the ~24 reads per review). Default 20s; 0 disables. - const ghCacheTtl = Math.max( - 0, - Number(process.env.GITHUB_CACHE_TTL_SECONDS ?? "20"), - ); - if (ghCacheTtl > 0) { - const { createRedisResponseCache } = - await import("./selfhost/redis-response-cache"); - setGitHubResponseCache(createRedisResponseCache(redisClient, ghCacheTtl)); - } - readinessProbes.push({ - name: "redis", - check: () => withTimeout(redisClient.ping().then(() => true)), - }); - console.log( - JSON.stringify({ - event: "selfhost_rate_limiter", - backend: "redis", - githubResponseCacheTtl: ghCacheTtl, - }), - ); + // Redis is required: pending-CI stuck detection, webhook dedup/coalescing, distributed rate limiting, and + // warm GitHub token/response caches all rely on this shared transient state. + const redisUrl = process.env.REDIS_URL; + if (!redisUrl) throw new Error("REDIS_URL is required for the self-host review runtime"); + const { Redis } = await import("ioredis"); + const redisClient = new Redis(redisUrl); + const { createRedisRateLimiter } = await import("./selfhost/redis-ratelimit"); + const { createRedisCache } = await import("./selfhost/redis-cache"); + const rateLimiter = createRedisRateLimiter(redisClient); + const webhookCache = createRedisCache(redisClient); + // Persist the installation-token cache in Redis so warm GitHub App tokens survive restarts/deploys and are + // shared across replicas (the in-isolate Map otherwise re-mints — an Orb round-trip — per replica/cold start). + const { createRedisTokenCache } = await import("./selfhost/redis-token-cache"); + const { setInstallationTokenStore, setGitHubResponseCache } = await import("./github/app"); + setInstallationTokenStore(createRedisTokenCache(redisClient)); + // Short-TTL cache for safe GitHub GET responses (dedups the ~24 reads per review). Default 20s; 0 disables. + const ghCacheTtl = Math.max(0, Number(process.env.GITHUB_CACHE_TTL_SECONDS ?? "20")); + if (ghCacheTtl > 0) { + const { createRedisResponseCache } = await import("./selfhost/redis-response-cache"); + setGitHubResponseCache(createRedisResponseCache(redisClient, ghCacheTtl)); } + readinessProbes.push({ + name: "redis", + check: () => withTimeout(redisClient.ping().then(() => true)), + }); + console.log( + JSON.stringify({ + event: "selfhost_redis_ready", + backend: "redis", + githubResponseCacheTtl: ghCacheTtl, + }), + ); // Qdrant vector store — overrides the backend's built-in sqlite-vec / pgvector when QDRANT_URL is set. let vectorizeOverride: Vectorize | undefined; @@ -467,14 +460,14 @@ async function main(): Promise { AI: ai, ...(embedAi ? { AI_EMBED: embedAi as unknown as Ai } : {}), ...(aiReviewPlan ? { AI_REVIEW_PLAN: aiReviewPlan } : {}), - ...(webhookCache ? { SELFHOST_TRANSIENT_CACHE: webhookCache } : {}), + SELFHOST_TRANSIENT_CACHE: webhookCache, // Qdrant takes priority; falls back to the backend's built-in vectorize (pgvector or sqlite-vec) ...(vectorizeOverride ? { VECTORIZE: vectorizeOverride } : backend.vectorize ? { VECTORIZE: backend.vectorize } : {}), - ...(rateLimiter ? { RATE_LIMITER: rateLimiter } : {}), + RATE_LIMITER: rateLimiter, // Visual review: when BROWSER_WS_ENDPOINT is set, expose a truthy BROWSER binding so shot.ts's // `if (!env.BROWSER) return` guard is bypassed; the puppeteer stub then connects via WS. ...(process.env.BROWSER_WS_ENDPOINT ? { BROWSER: {} } : {}), diff --git a/test/helpers/d1.ts b/test/helpers/d1.ts index 531867a90f..40ccdee9a5 100644 --- a/test/helpers/d1.ts +++ b/test/helpers/d1.ts @@ -50,6 +50,7 @@ export class TestD1Database { } export function createTestEnv(overrides: Partial = {}): Env { + const transientCache = new Map(); return { DB: new TestD1Database() as unknown as D1Database, JOBS: { @@ -77,6 +78,17 @@ export function createTestEnv(overrides: Partial = {}): Env { GITHUB_WEBHOOK_SECRET: "test-webhook-secret", GITHUB_APP_PRIVATE_KEY: "test-private-key", ADMIN_GITHUB_LOGINS: "jsonbored", + SELFHOST_TRANSIENT_CACHE: { + async get(key: string) { + return transientCache.get(key) ?? null; + }, + async set(key: string, value: string) { + transientCache.set(key, value); + }, + async del(key: string) { + transientCache.delete(key); + }, + }, // Per-repo review allowlist: default to the test repos so flag-ON wiring tests activate the // gated review features. Override to "" to assert the dormant (no-repo) default. GITTENSORY_REVIEW_REPOS: "JSONbored/gittensory,acme/widgets", diff --git a/test/unit/dlq.test.ts b/test/unit/dlq.test.ts index cedb0daebb..c7f634ad1d 100644 --- a/test/unit/dlq.test.ts +++ b/test/unit/dlq.test.ts @@ -100,7 +100,7 @@ describe("DLQ consumer (processDlqBatch)", () => { describe("webhook self-heal re-drive (#1276)", () => { function captureWebhooks(env: ReturnType) { const sent: JobMessage[] = []; - env.WEBHOOKS = { send: async (m: JobMessage) => void sent.push(m) } as unknown as typeof env.WEBHOOKS; + env.WEBHOOKS = { send: async (m: JobMessage) => void sent.push(m) } as unknown as Queue; return sent; } @@ -151,13 +151,24 @@ describe("DLQ consumer (processDlqBatch)", () => { expect(batch.acked).toEqual(["mn-1"]); }); + it("does not re-drive webhook jobs when webhook redrive is disabled for broker-only Cloudflare", async () => { + const env = createTestEnv(); + const sent = captureWebhooks(env); + const batch = makeBatch([{ id: "wh-broker-only", body: { type: "github-webhook", deliveryId: "broker-only-1", eventName: "pull_request", payload: {} } }], "gittensory-webhooks-dlq"); + + await processDlqBatch(batch as unknown as MessageBatch, env, { redriveWebhooks: false }); + + expect(sent).toEqual([]); + expect(batch.acked).toEqual(["wh-broker-only"]); + }); + it("re-drives a rate-limited webhook AFTER the reset delay when the shared REST budget is exhausted (#audit-rate-headroom)", async () => { vi.useFakeTimers({ toFake: ["Date"] }); vi.setSystemTime(new Date("2026-06-24T12:00:00.000Z")); const env = createTestEnv(); await recordGitHubRateLimitObservation(env, { repoFullName: "owner/repo", resource: "rest", path: "/x", statusCode: 200, limitValue: 5000, remaining: 5, resetAt: "2026-06-24T12:30:00.000Z", observedAt: "2026-06-24T12:00:00.000Z" }); const options: Array<{ delaySeconds?: number } | undefined> = []; - env.WEBHOOKS = { send: async (_m: JobMessage, opts?: { delaySeconds?: number }) => void options.push(opts) } as unknown as typeof env.WEBHOOKS; + env.WEBHOOKS = { send: async (_m: JobMessage, opts?: { delaySeconds?: number }) => void options.push(opts) } as unknown as Queue; const batch = makeBatch([{ id: "wh-rl", body: { type: "github-webhook", deliveryId: "rl-1", eventName: "pull_request", payload: {} } }], "gittensory-webhooks-dlq"); await processDlqBatch(batch as unknown as MessageBatch, env); diff --git a/test/unit/index.test.ts b/test/unit/index.test.ts index 2df31392dc..10aa7d60a1 100644 --- a/test/unit/index.test.ts +++ b/test/unit/index.test.ts @@ -5,6 +5,7 @@ import { createTestEnv } from "../helpers/d1"; describe("worker entrypoint", () => { afterEach(() => { + vi.restoreAllMocks(); vi.unstubAllGlobals(); }); @@ -58,6 +59,57 @@ describe("worker entrypoint", () => { expect(retried).toEqual([]); }); + it("does not re-drive webhook DLQ messages from a broker-only Cloudflare runtime", async () => { + const env = createTestEnv(); + delete env.SELFHOST_TRANSIENT_CACHE; + const sent: import("../../src/types").JobMessage[] = []; + env.WEBHOOKS = { send: async (message: import("../../src/types").JobMessage) => void sent.push(message) } as unknown as Queue; + const acked: string[] = []; + const batch = { + queue: "gittensory-webhooks-dlq", + messages: [ + { + id: "wh-dlq-broker-only", + body: { type: "github-webhook", deliveryId: "d-broker-only", eventName: "pull_request", payload: {} }, + ack: () => acked.push("wh-dlq-broker-only"), + retry: () => undefined, + }, + ], + } as unknown as MessageBatch; + + await worker.queue(batch, env); + + expect(acked).toEqual(["wh-dlq-broker-only"]); + expect(sent).toEqual([]); + }); + + it("acks and ignores review-execution jobs from a broker-only Cloudflare runtime", async () => { + const env = createTestEnv(); + delete env.SELFHOST_TRANSIENT_CACHE; + const warned = vi.spyOn(console, "warn").mockImplementation(() => undefined); + const acked: string[] = []; + const retried: string[] = []; + const batch = { + messages: [ + { + id: "hosted-review-job", + body: { type: "github-webhook", deliveryId: "d-hosted-review", eventName: "pull_request", payload: {} }, + ack: () => acked.push("hosted-review-job"), + retry: () => retried.push("hosted-review-job"), + }, + ], + } as unknown as MessageBatch; + + await worker.queue(batch, env); + + expect(acked).toEqual(["hosted-review-job"]); + expect(retried).toEqual([]); + expect(JSON.parse(String(warned.mock.calls[0]?.[0]))).toMatchObject({ + event: "hosted_review_job_ignored", + jobType: "github-webhook", + }); + }); + it("acks successful queue messages and retries failed messages", async () => { const env = createTestEnv(); vi.stubGlobal("fetch", async () => new Response("missing", { status: 404 })); @@ -154,6 +206,24 @@ describe("worker entrypoint", () => { expect(sent).toEqual([{ type: "agent-regate-sweep", requestedBy: "schedule" }]); }); + it("does not enqueue review sweeps from a broker-only Cloudflare runtime", async () => { + const sent: Array = []; + const env = createTestEnv({ + JOBS: { + async send(message: import("../../src/types").JobMessage) { + sent.push(message); + }, + } as unknown as Queue, + }); + delete env.SELFHOST_TRANSIENT_CACHE; + const waitUntil: Promise[] = []; + + await worker.scheduled(controllerFor("2026-05-25T05:14:00.000Z"), env, executionContext(waitUntil)); + await Promise.all(waitUntil); + + expect(sent).toEqual([]); + }); + it("THROTTLES the sweep when the GitHub REST budget is at/below the maintenance headroom (#6 backpressure)", async () => { const sent: Array = []; const env = createTestEnv({ diff --git a/test/unit/webhook.test.ts b/test/unit/webhook.test.ts index ec49c35a25..0e062db1b8 100644 --- a/test/unit/webhook.test.ts +++ b/test/unit/webhook.test.ts @@ -46,7 +46,7 @@ describe("github webhook enqueue failure (#786)", () => { send: async () => { throw new Error("queue unavailable"); }, - } as unknown as typeof env.WEBHOOKS; + } as unknown as Queue; const rawBody = JSON.stringify({ action: "opened", repository: { full_name: "JSONbored/gittensory" }, installation: { id: 1 } }); const signature = await signWebhook(rawBody, env.GITHUB_WEBHOOK_SECRET); const request = new Request("https://example.com/webhook", { method: "POST", body: rawBody }); @@ -85,7 +85,7 @@ describe("github webhook dedup (#789)", () => { send: async () => { sendCount += 1; }, - } as unknown as typeof env.WEBHOOKS; + } as unknown as Queue; // Seed a fully-processed event: on success the queue overwrites payloadHash with the "processed" // sentinel, so a redelivery carries the real hash and a hash-only dedup would miss it. await recordWebhookEvent(env, { deliveryId: "redelivery-1", eventName: "pull_request", payloadHash: "processed", status: "processed" }); @@ -118,12 +118,45 @@ describe("github webhook dedup (#789)", () => { }); describe("github webhook queue isolation (#audit-webhook-queue)", () => { + it("rejects valid review webhooks when the self-host review runtime is absent", async () => { + const env = createTestEnv(); + delete env.SELFHOST_TRANSIENT_CACHE; + let webhookSends = 0; + env.WEBHOOKS = { send: async () => void (webhookSends += 1) } as unknown as Queue; + const rawBody = JSON.stringify({ action: "opened", repository: { full_name: "JSONbored/gittensory" }, installation: { id: 1 } }); + const signature = await signWebhook(rawBody, env.GITHUB_WEBHOOK_SECRET); + const request = new Request("https://example.com/webhook", { method: "POST", body: rawBody }); + const headers: Record = { + "x-github-delivery": "broker-only-webhook-1", + "x-github-event": "pull_request", + "x-hub-signature-256": signature, + }; + const context = { + req: { + raw: request, + header(name: string) { + return headers[name.toLowerCase()] ?? null; + }, + }, + env, + json(payload: unknown, status?: number) { + return Response.json(payload, status === undefined ? undefined : { status }); + }, + } as unknown as Context<{ Bindings: Env }>; + + const response = await handleGitHubWebhook(context); + + expect(response.status).toBe(410); + await expect(response.json()).resolves.toMatchObject({ error: "selfhost_review_runtime_required" }); + expect(webhookSends).toBe(0); + }); + it("INVARIANT: a valid webhook is enqueued onto the dedicated WEBHOOKS lane, never the shared JOBS queue", async () => { const env = createTestEnv(); let jobsSends = 0; let webhookSends = 0; env.JOBS = { send: async () => void (jobsSends += 1) } as unknown as typeof env.JOBS; - env.WEBHOOKS = { send: async () => void (webhookSends += 1) } as unknown as typeof env.WEBHOOKS; + env.WEBHOOKS = { send: async () => void (webhookSends += 1) } as unknown as Queue; const rawBody = JSON.stringify({ action: "opened", repository: { full_name: "JSONbored/gittensory" }, installation: { id: 1 } }); const signature = await signWebhook(rawBody, env.GITHUB_WEBHOOK_SECRET); const request = new Request("https://example.com/webhook", { method: "POST", body: rawBody }); @@ -155,7 +188,7 @@ describe("github webhook queue isolation (#audit-webhook-queue)", () => { it("drops self-authored app comment webhooks before they add queue pressure", async () => { const env = createTestEnv(); let webhookSends = 0; - env.WEBHOOKS = { send: async () => void (webhookSends += 1) } as unknown as typeof env.WEBHOOKS; + env.WEBHOOKS = { send: async () => void (webhookSends += 1) } as unknown as Queue; const rawBody = JSON.stringify({ action: "edited", repository: { full_name: "JSONbored/gittensory" }, @@ -195,7 +228,7 @@ describe("github webhook queue isolation (#audit-webhook-queue)", () => { it("drops self-authored app CI completion webhooks before they add queue pressure", async () => { const env = createTestEnv({ GITHUB_APP_SLUG: "gittensory-orb" }); let webhookSends = 0; - env.WEBHOOKS = { send: async () => void (webhookSends += 1) } as unknown as typeof env.WEBHOOKS; + env.WEBHOOKS = { send: async () => void (webhookSends += 1) } as unknown as Queue; const rawBody = JSON.stringify({ action: "completed", repository: { full_name: "JSONbored/gittensory" }, @@ -296,7 +329,7 @@ describe("handleOrbRelay (brokered self-host relay receiver)", () => { it("returns 500 (enqueue_failed) and flips event to 'error' when WEBHOOKS.send throws", async () => { const env = createTestEnv({ ORB_ENROLLMENT_SECRET: "orbenr_testsecret" }); - env.WEBHOOKS = { send: async () => { throw new Error("queue down"); } } as unknown as typeof env.WEBHOOKS; + env.WEBHOOKS = { send: async () => { throw new Error("queue down"); } } as unknown as Queue; const body = JSON.stringify({ action: "opened", repository: { full_name: "acme/widgets" }, installation: { id: 99 } }); const sig = `sha256=${await relaySignature("orbenr_testsecret", body)}`; const ctx = makeRelayContext(env, body, { "x-github-delivery": "relay-fail-1", "x-github-event": "pull_request", "x-orb-signature-256": sig }); @@ -308,7 +341,7 @@ describe("handleOrbRelay (brokered self-host relay receiver)", () => { it("returns 202 queued when signature is valid and WEBHOOKS.send succeeds", async () => { const env = createTestEnv({ ORB_ENROLLMENT_SECRET: "orbenr_testsecret" }); let sent = 0; - env.WEBHOOKS = { send: async () => void (sent += 1) } as unknown as typeof env.WEBHOOKS; + env.WEBHOOKS = { send: async () => void (sent += 1) } as unknown as Queue; const body = JSON.stringify({ action: "opened", repository: { full_name: "acme/widgets" }, installation: { id: 99 } }); const sig = `sha256=${await relaySignature("orbenr_testsecret", body)}`; const ctx = makeRelayContext(env, body, { "x-github-delivery": "relay-ok-1", "x-github-event": "pull_request", "x-orb-signature-256": sig }); diff --git a/worker-configuration.d.ts b/worker-configuration.d.ts index d99cc66860..c5ae1f81e3 100644 --- a/worker-configuration.d.ts +++ b/worker-configuration.d.ts @@ -1,13 +1,11 @@ /* eslint-disable */ -// Generated by Wrangler by running `wrangler types` (hash: 5e3dae90c4236d0e4463f05f2729a70b) +// Generated by Wrangler by running `wrangler types` (hash: 955ce44011ee0e0b28dd5b68fcfc0a52) // Runtime types generated with workerd@1.20260617.1 2026-05-28 nodejs_compat interface __BaseEnv_Env { - REVIEW_CONFIG: KVNamespace; REVIEW_AUDIT: R2Bucket; DB: D1Database; VECTORIZE: VectorizeIndex; JOBS: Queue; - WEBHOOKS: Queue; BROWSER: BrowserRun; AI: Ai; GITHUB_APP_ID: "3824093"; @@ -30,20 +28,20 @@ interface __BaseEnv_Env { AI_MAX_OUTPUT_TOKENS: "4096"; AI_GATEWAY_ID: ""; ADMIN_GITHUB_LOGINS: "JSONbored"; - GITTENSORY_REVIEW_UNIFIED_COMMENT: "true"; + GITTENSORY_REVIEW_UNIFIED_COMMENT: "false"; GITTENSORY_REVIEW_INLINE_COMMENTS: "false"; - GITTENSORY_REVIEW_SAFETY: "true"; - GITTENSORY_REVIEW_SCREENSHOTS: "true"; - GITTENSORY_REVIEW_GROUNDING: "true"; - GITTENSORY_REVIEW_REPUTATION: "true"; + GITTENSORY_REVIEW_SAFETY: "false"; + GITTENSORY_REVIEW_SCREENSHOTS: "false"; + GITTENSORY_REVIEW_GROUNDING: "false"; + GITTENSORY_REVIEW_REPUTATION: "false"; GITTENSORY_REVIEW_OPS: "false"; - GITTENSORY_REVIEW_RAG: "true"; + GITTENSORY_REVIEW_RAG: "false"; GITTENSORY_REVIEW_CONTENT_LANE: "false"; GITTENSORY_REVIEW_SELFTUNE: "false"; GITTENSORY_REVIEW_PLANNER: "false"; GITTENSORY_REVIEW_DRAFT: "false"; GITTENSORY_REVIEW_PARITY_AUDIT: "false"; - GITTENSORY_REVIEW_REPOS: "JSONbored/gittensory,JSONbored/awesome-claude,JSONbored/metagraphed"; + GITTENSORY_REVIEW_REPOS: ""; GITTENSORY_PUBLIC_STATS: "true"; GITTENSORY_DUPLICATE_WINNER: "true"; RATE_LIMITER: DurableObjectNamespace; diff --git a/wrangler.jsonc b/wrangler.jsonc index 53789c116c..2b33b9c322 100644 --- a/wrangler.jsonc +++ b/wrangler.jsonc @@ -52,9 +52,9 @@ "AI_MAX_OUTPUT_TOKENS": "4096", "AI_GATEWAY_ID": "", "ADMIN_GITHUB_LOGINS": "JSONbored", - // Convergence (Stage D): render the public PR comment via the unified-comment bridge. Default OFF — - // flag-OFF keeps the legacy buildPublicPrIntelligenceComment panel byte-identical. - "GITTENSORY_REVIEW_UNIFIED_COMMENT": "true", + // Hosted reviews are retired. Cloudflare now serves the public API + Orb broker only; review execution runs + // in the self-host container, where Redis-backed transient state is mandatory. + "GITTENSORY_REVIEW_UNIFIED_COMMENT": "false", // Inline comments (#inline-comments): leave quiet, non-blocking inline comments on changed lines, on top of // the decision summary. Requires the repo in GITTENSORY_REVIEW_REPOS AND review.inline_comments in its // .gittensory.yml. Default OFF — flag-OFF the model is never asked for inline findings (byte-identical). @@ -62,21 +62,21 @@ // Convergence (safety): run the ported safety scan in the review path — defang untrusted PR // title/body/diff before the AI reviewer sees it, and surface a secret-leak blocker from the diff. // Default OFF — flag-OFF keeps the review path byte-identical. - "GITTENSORY_REVIEW_SAFETY": "true", + "GITTENSORY_REVIEW_SAFETY": "false", // Convergence (visual capture): capture a before/after screenshot for PRs touching WEB-VISIBLE files // (frontend pages / public OG images). Needs the BROWSER + REVIEW_AUDIT bindings; runs only when this is // ON AND the repo is in GITTENSORY_REVIEW_REPOS. DEFAULT OFF — flag-OFF captures nothing (byte-identical). - "GITTENSORY_REVIEW_SCREENSHOTS": "true", + "GITTENSORY_REVIEW_SCREENSHOTS": "false", // Convergence (grounding): ground the AI reviewer prompt with the PR's finished CI status + the full // post-change content of the changed files, so a non-frontier model verifies claims instead of guessing. // Default OFF — flag-OFF keeps the reviewer prompt byte-identical and makes no extra GitHub fetch. - "GITTENSORY_REVIEW_GROUNDING": "true", + "GITTENSORY_REVIEW_GROUNDING": "false", // Convergence (reputation): factor the INTERNAL-only ported submitter-reputation signal into the AI-spend // gate — a new / burst / low-reputation submitter is downgraded to a deterministic-only review (AI neurons // skipped), and the per-(project, submitter) outcome is recorded after the gate decides. The reputation is // NEVER surfaced publicly. Default OFF — flag-OFF reads nothing, records nothing, and leaves the AI-spend // gate byte-identical. - "GITTENSORY_REVIEW_REPUTATION": "true", + "GITTENSORY_REVIEW_REPUTATION": "false", // Convergence (ops / observability): drive two OPERATOR surfaces off gittensory's own review-outcome data // — (1) a cron-tick anomaly scan over the gate-block ledger + recommendation/slop calibration that emits a // structured `ops_anomaly` log on drift, and (2) a bearer-gated GET /v1/internal/ops/stats outcome @@ -89,7 +89,7 @@ // Default OFF — flag-OFF performs no retrieval, uses no adapter, makes no vector query, and keeps the // reviewer prompt byte-identical. Even when ON it is inert until a repo's vector index is populated (the // index-population job + cron is a deploy-time follow-up; a cold/missing index degrades to no context). - "GITTENSORY_REVIEW_RAG": "true", + "GITTENSORY_REVIEW_RAG": "false", // Convergence (content/registry SURFACE LANE): when truthy AND the repo is in GITTENSORY_REVIEW_REPOS, the // deterministic, AI-FREE surface review drives the gate for registry-submission PRs (metagraphed). Default // OFF (false): the processor takes no new branch + resolves no files, so the gate disposition is byte- @@ -123,7 +123,7 @@ // rolls forward one repo at a time (e.g. "JSONbored/gittensory,JSONbored/awesome-claude"). Default "" → // NO repos converged → the per-PR converged path stays dormant for every repo regardless of the global // flags (byte-identical to today). The cron/endpoint flags (ops/selftune/parity/draft) stay global. - "GITTENSORY_REVIEW_REPOS": "JSONbored/gittensory,JSONbored/awesome-claude,JSONbored/metagraphed", + "GITTENSORY_REVIEW_REPOS": "", // Proof of Power (#1059): serve the public homepage stats counter at GET /v1/public/stats, computed LIVE // from the review ledger (review_targets + review_audit) behind a 60s cache. ON — the above-the-fold band // shows PRs reviewed / filtered-without-merge % / maintainer time saved / decision accuracy for the @@ -180,16 +180,6 @@ "browser": { "binding": "BROWSER", }, - // Convergence: the shared REVIEW_CONFIG KV (reviewbot's per-repo config, keyed by repo slug). The converged - // auto-maintain path reads each repo's hardGuardrailGlobs from it so guarded paths (scoring / auth / CI / - // policy scripts) force MANUAL review — never auto-merge/auto-close. Absent ⇒ DEFAULT_CRUCIAL_GUARDRAIL_GLOBS - // fallback (CI workflows + scripts still guarded). This KV survives reviewbot's decommission. - "kv_namespaces": [ - { - "binding": "REVIEW_CONFIG", - "id": "aed9890dbd3f4f73bf46b43d7d0478d7", - }, - ], // TODO (convergence follow-up): a per-PR LOCK Durable Object (`SubmissionLock` mutex) so concurrent // webhook deliveries for the same PR serialize. That is a separate, more-involved sub-task — it needs a DO // class (ported) + its own `migrations` tag (`new_sqlite_classes: ["SubmissionLock"]`) + a `durable_objects` @@ -214,28 +204,11 @@ "binding": "JOBS", "queue": "gittensory-jobs", }, - { - // Dedicated lane for incoming github-webhook jobs so a burst of heavy maintenance work (the re-gate - // sweep, backfills, rag-index, snapshots) on the shared `gittensory-jobs` queue can never starve real - // GitHub events into the DLQ. Webhooks get their own consumer concurrency + retry budget. (#audit-webhook-queue) - "binding": "WEBHOOKS", - "queue": "gittensory-webhooks", - }, ], "consumers": [ { - // Webhook lane: small/fast batches so a real PR event is dispatched promptly, isolated from the - // maintenance lane's throughput. Its own DLQ keeps a failing webhook observable + replayable (PR2). - "queue": "gittensory-webhooks", - "max_batch_size": 5, - "max_batch_timeout": 1, - "max_retries": 3, - "dead_letter_queue": "gittensory-webhooks-dlq", - }, - { - // Maintenance lane: the re-gate sweep, backfills, rag-index, snapshots. Webhooks have their own lane - // (above), so this lane is bounded to keep a heavy/retrying batch from re-saturating the consumer and - // amplifying GitHub-rate pressure (the metagraphed-dry-run overload). (#audit-bound-lane) + // API/broker maintenance lane. Review execution jobs are ignored unless the self-host Redis-backed + // runtime binding exists; the deployed Cloudflare worker does not bind it. // - max_batch_size 5 (was 10): one batch can't bundle as many heavy sweep/backfill jobs at once. // - max_concurrency 3: bounded fan-out — enough to drain metagraphed's worst sweep within the 2-min // cron interval, but not so wide it floods the shared GitHub installation rate bucket. (A follow-up @@ -262,23 +235,11 @@ "max_batch_timeout": 30, "max_retries": 0, }, - { - // Webhook-lane DLQ consumer (#1276): a dead-lettered github-webhook carries a real GitHub event that - // GitHub will not redeliver, so processDlqBatch audits it AND re-drives it once onto the webhook lane - // (bounded by the `redriven` marker). max_retries: 0 — a DLQ message must never re-loop the DLQ. - "queue": "gittensory-webhooks-dlq", - "max_batch_size": 10, - "max_batch_timeout": 30, - "max_retries": 0, - }, ], }, "triggers": { - // Tick every 2 minutes. The light auto-maintain/recovery sweep (agent-regate-sweep) runs on EVERY tick so an - // approved+clean PR merges and a red-CI / stuck PR is handled within ~2 min (see src/index.ts). Heavy sync/ - // health jobs stay on their own cadence inside the handler via `minute % 30` (30-min) and `isHourly` (hourly), - // so they are unaffected by the faster tick. (Was "*/30": the sweep the code intends to run every ~2 min was - // actually firing only every 30 min, starving the recovery rail.) + // Broker/API maintenance tick. The self-host container runs the review sweep; Cloudflare does not execute + // hosted reviews. "crons": ["*/2 * * * *"], }, } From 00495e0411ab873b809a0bb65c72eb14b306bad5 Mon Sep 17 00:00:00 2001 From: JSONbored <49853598+JSONbored@users.noreply.github.com> Date: Mon, 29 Jun 2026 02:58:11 -0700 Subject: [PATCH 48/68] fix(selfhost): remove hosted review policy fallback --- src/env.d.ts | 3 - src/review/guardrail-config.ts | 64 +++--------- src/review/linked-issue-hard-rules.ts | 74 ++------------ test/unit/change-guardrail.test.ts | 6 +- test/unit/guardrail-config.test.ts | 59 +++-------- test/unit/linked-issue-hard-rules.test.ts | 114 ++++++++-------------- 6 files changed, 79 insertions(+), 241 deletions(-) diff --git a/src/env.d.ts b/src/env.d.ts index f704fca741..5ff8aa9306 100644 --- a/src/env.d.ts +++ b/src/env.d.ts @@ -22,9 +22,6 @@ declare global { /** Convergence (infra): Browser Rendering binding for visual (before/after screenshot) capture. Optional — * absent ⇒ no visual capture. Unused until the per-module wiring chunk; an unbound deploy is inert. */ BROWSER?: Fetcher; - /** Legacy reviewbot KV shape. Cloudflare no longer binds this; self-host review policy should come from - * container-private config. Existing readers are optional and fall back when absent. */ - REVIEW_CONFIG?: KVNamespace; /** Self-host transient cache for short-lived coalescing/backpressure keys. */ SELFHOST_TRANSIENT_CACHE?: { get(key: string): Promise; diff --git a/src/review/guardrail-config.ts b/src/review/guardrail-config.ts index ca6a063a52..e58e767813 100644 --- a/src/review/guardrail-config.ts +++ b/src/review/guardrail-config.ts @@ -1,19 +1,11 @@ -import type { JsonValue } from "../types"; - // Per-repo hard-guardrail path globs (paths that force MANUAL review — no auto-merge / no auto-close). // -// Convergence note: gittensory does not have its own per-repo guardrail config surface, but reviewbot already -// stores carefully-tuned globs per repo in the shared REVIEW_CONFIG KV (keyed by repo slug, e.g. "gittensory" -// / "awesome-claude" / "metagraphed"). That KV is the established home for private, runtime-editable operator -// tuning, so the converged auto-maintain path reads its guardrail globs from there too — no redeploy needed -// to retune, and the same KV survives reviewbot's decommission. - -// Conservative cross-repo fallback when a repo has no KV-configured globs: CI workflows + build/policy scripts -// are universally sensitive (the awesome-claude #4196 incident class). Fail-SAFE — a config miss still guards -// these, it never opens the gate wide. +// Self-host note: hosted reviews and hosted policy storage are retired. Review execution uses the container-private +// `.gittensory.yml` path for repo policy; these hard guardrails remain built-in invariants so a missing private +// config cannot open the gate around CI, policy, or the review engine's own decision code. export const DEFAULT_CRUCIAL_GUARDRAIL_GLOBS = [".github/workflows/**", "scripts/**"]; -// The gate's OWN policy files, guarded for EVERY repo regardless of KV tuning. A PR that edits the +// The gate's OWN policy files, guarded for EVERY repo regardless of private config. A PR that edits the // config-as-code that defines the gate or coverage policy (the `.gittensory.*` focus manifest the loader // reads, or `codecov.yml`) must always be HELD for the owner — otherwise one auto-merged config-only PR // could weaken the gate repo-wide before any subsequent PR is evaluated against the new policy. The @@ -30,14 +22,13 @@ export const CONFIG_AS_CODE_GUARDRAIL_GLOBS = [ "**/.codecov.yml", ]; -// The review engine's OWN decision + safety code — its crown jewels — guarded for EVERY repo regardless of KV -// tuning. A contributor PR that edits how the gate decides a verdict, how a merge or close executes, the +// The review engine's OWN decision + safety code — its crown jewels — guarded for EVERY repo regardless of +// private config. A contributor PR that edits how the gate decides a verdict, how a merge or close executes, the // action-mode kill-switch, scoring, auth, the CI aggregate the gate reads, or the guardrail itself must be HELD -// for the owner: the engine must never auto-merge a change to the very code that governs its own autonomy. This -// is the exact failure the FAIL_CLOSED comment warns about — but that fires only on a KV outage, so without this -// the narrow DEFAULT (CI + scripts) let crown-jewel edits auto-merge on every normal request. These are -// gittensory engine-specific paths, so they never match an unrelated reviewed repo's PR (e.g. metagraphed has no -// src/rules/** or agent-action-executor.ts); like the config-as-code set above, this only ever WIDENS the guard. +// for the owner: the engine must never auto-merge a change to the very code that governs its own autonomy. +// These are gittensory engine-specific paths, so they never match an unrelated reviewed repo's PR (e.g. +// metagraphed has no src/rules/** or agent-action-executor.ts); like the config-as-code set above, this only +// ever WIDENS the guard. export const ENGINE_DECISION_GUARDRAIL_GLOBS = [ "src/rules/**", // the gate verdict (advisory) + the predicted-gate mirror "src/services/**", // the merge/close action executor, approval queue, and merge-failure handling — the write chokepoint @@ -45,7 +36,7 @@ export const ENGINE_DECISION_GUARDRAIL_GLOBS = [ "src/settings/agent-execution.ts", // the action-mode resolver + the env kill-switch backstop "src/settings/agent-sweep.ts", // the re-gate maintenance sweep "src/settings/autonomy.ts", // the autonomy-level ladder (observe → suggest → auto → auto_with_approval) - "src/queue/**", // webhook → gate → merge/close orchestration (processors) + dead-letter handling (dlq) — NOT in the KV dir-prefix guards + "src/queue/**", // webhook → gate → merge/close orchestration (processors) + dead-letter handling (dlq) "src/github/pr-actions.ts", // the GitHub merge / close / review / comment write primitives "src/github/app.ts", // installation auth + the per-installation token mint "src/github/backfill.ts", // the live CI aggregate (fetchLiveCiAggregate) the gate verdict reads @@ -58,34 +49,11 @@ export const ENGINE_DECISION_GUARDRAIL_GLOBS = [ "src/review/outcomes-wire.ts", // the pr_outcome + reversal telemetry that feeds self-tuning ]; -// A KV READ FAULT (binding present but the read threw — an outage/transient error) must fail CLOSED, NOT fall -// back to the narrow default: a config-read fault correlated with a contributor flood would otherwise silently -// shrink the guarded surface to CI+scripts and let crown-jewel edits (scoring/auth/rules/the gate) auto-merge. -// "**" matches every path (the glob engine maps ** -> .*), so this holds ALL PRs for human review until the -// config read recovers — fail-safe for the surface a flood most threatens. (#flood-readiness) -export const FAIL_CLOSED_GUARDRAIL_GLOBS = ["**"]; - -function asNonEmptyStringArray(value: unknown): string[] | null { - if (!Array.isArray(value)) return null; - const out = value.filter((entry): entry is string => typeof entry === "string" && entry.length > 0); - return out.length > 0 ? out : null; -} - /** - * Resolve a repo's hard-guardrail path globs from the shared REVIEW_CONFIG KV (key = repo slug). Never throws - * (the auto-maintain trigger is best-effort). A legitimately-absent binding/key/field falls back to the narrow - * DEFAULT_CRUCIAL_GUARDRAIL_GLOBS so a freshly-installed repo can still operate; but a THROWN read (KV outage) - * fails CLOSED to FAIL_CLOSED_GUARDRAIL_GLOBS so a config fault can never open the gate during a flood. + * Resolve hard-guardrail path globs. Kept async to avoid touching the processor call graph, but this no longer + * reads external policy storage; self-host review policy belongs in container-private `.gittensory.yml`, and these + * engine-level guardrails are always-on invariants. */ -export async function loadHardGuardrailGlobs(env: Env, repoFullName: string): Promise { - const slug = repoFullName.includes("/") ? repoFullName.slice(repoFullName.indexOf("/") + 1) : repoFullName; - // The config-as-code policy files AND the engine's own crown-jewel paths are guarded for every repo regardless - // of KV tuning (a narrow per-repo glob list never un-guards them); the fail-closed `**` already covers them. - if (!env.REVIEW_CONFIG) return [...DEFAULT_CRUCIAL_GUARDRAIL_GLOBS, ...CONFIG_AS_CODE_GUARDRAIL_GLOBS, ...ENGINE_DECISION_GUARDRAIL_GLOBS]; - try { - const config = (await env.REVIEW_CONFIG.get(slug, "json")) as { hardGuardrailGlobs?: JsonValue } | null; - return [...(asNonEmptyStringArray(config?.hardGuardrailGlobs) ?? DEFAULT_CRUCIAL_GUARDRAIL_GLOBS), ...CONFIG_AS_CODE_GUARDRAIL_GLOBS, ...ENGINE_DECISION_GUARDRAIL_GLOBS]; - } catch { - return FAIL_CLOSED_GUARDRAIL_GLOBS; - } +export async function loadHardGuardrailGlobs(_env: Env, _repoFullName: string): Promise { + return [...DEFAULT_CRUCIAL_GUARDRAIL_GLOBS, ...CONFIG_AS_CODE_GUARDRAIL_GLOBS, ...ENGINE_DECISION_GUARDRAIL_GLOBS]; } diff --git a/src/review/linked-issue-hard-rules.ts b/src/review/linked-issue-hard-rules.ts index 29c761b4ff..53620084dc 100644 --- a/src/review/linked-issue-hard-rules.ts +++ b/src/review/linked-issue-hard-rules.ts @@ -1,4 +1,3 @@ -import type { JsonValue } from "../types"; import { fetchLinkedIssueFacts } from "../github/backfill"; import { extractLinkedIssueNumbersWithOverflow } from "../db/repositories"; @@ -39,12 +38,9 @@ export type LinkedIssueHardRulesConfig = { closeDelaySeconds: number; }; -// Fail-SAFE default: every mode OFF, empty label lists, NOT a default-label repo. An unconfigured (or -// KV-unbound, or KV-faulting) repo must never auto-close a contributor PR for a linked-issue rule. The default -// point/maintainer label lists are only used when a repo turns the corresponding rule ON without listing its -// own; an OFF rule never reads them. -const DEFAULT_POINT_BEARING_LABELS = ["gittensor:bug", "gittensor:feature", "gittensor:priority"]; -const DEFAULT_MAINTAINER_ONLY_LABELS = ["maintainer-only"]; +// Fail-SAFE default: every mode OFF, empty label lists, NOT a default-label repo. With hosted reviews retired, +// this loader no longer reads external policy storage; deterministic linked-issue auto-closes stay off +// unless/until they are wired through self-host repo config. // The namespaced label that marks a PR as flagged-for-closure by the linked-issue hard rule (Pass 1). Its // presence + a persisting violation on the next evaluation is the verification trigger (Pass 2 → close). Cleared @@ -53,8 +49,6 @@ export const AGENT_LABEL_PENDING_CLOSURE = "gittensory:pending-closure"; // Default verification delay (seconds) — how long until the second-pass close. Clamped to this range on load. const DEFAULT_CLOSE_DELAY_SECONDS = 30; -const MIN_CLOSE_DELAY_SECONDS = 0; -const MAX_CLOSE_DELAY_SECONDS = 300; export const DEFAULT_LINKED_ISSUE_HARD_RULES: LinkedIssueHardRulesConfig = { ownerAssignedClose: "off", @@ -68,65 +62,13 @@ export const DEFAULT_LINKED_ISSUE_HARD_RULES: LinkedIssueHardRulesConfig = { closeDelaySeconds: DEFAULT_CLOSE_DELAY_SECONDS, }; -/** Clamp a KV-provided close delay to a sane range, falling back to the default for a non-finite / absent value. */ -function clampCloseDelaySeconds(value: unknown): number { - if (typeof value !== "number" || !Number.isFinite(value)) return DEFAULT_CLOSE_DELAY_SECONDS; - return Math.min(MAX_CLOSE_DELAY_SECONDS, Math.max(MIN_CLOSE_DELAY_SECONDS, Math.trunc(value))); -} - -function asMode(value: unknown): LinkedIssueHardRulesMode | null { - return value === "block" || value === "off" ? value : null; -} - -function asStringArray(value: unknown): string[] | null { - if (!Array.isArray(value)) return null; - const out = value.filter((entry): entry is string => typeof entry === "string" && entry.length > 0); - return out.length > 0 ? out : null; -} - -type LinkedIssueHardRulesKvShape = { - ownerAssignedClose?: JsonValue; - missingPointLabelClose?: JsonValue; - maintainerOnlyLabelClose?: JsonValue; - pointBearingLabels?: JsonValue; - maintainerOnlyLabels?: JsonValue; - defaultLabelRepo?: JsonValue; - verifyBeforeClose?: JsonValue; - closeDelaySeconds?: JsonValue; -}; - /** - * Resolve a repo's linked-issue hard-rule config from the shared REVIEW_CONFIG KV (key = repo slug, owner - * stripped — same convention as loadHardGuardrailGlobs). Reads the `linkedIssueHardRules` field. NEVER throws - * (the auto-maintain trigger is best-effort) and ALWAYS fail-SAFE: an absent binding / key / field, a partial - * config, OR a THROWN KV read (outage) all resolve to the all-OFF default so a deterministic close can never - * fire on an unconfigured repo or a KV fault. Partial KV objects are merged OVER the default (any field a repo - * omits keeps its safe default). + * Resolve a repo's linked-issue hard-rule config. Kept async to avoid touching the processor call graph, but this + * no longer reads external policy storage; the fail-safe all-off default ensures deterministic linked-issue closes + * cannot fire from stale hosted-review configuration. */ -export async function loadLinkedIssueHardRules(env: Env, repoFullName: string): Promise { - if (!env.REVIEW_CONFIG) return DEFAULT_LINKED_ISSUE_HARD_RULES; - const slug = repoFullName.includes("/") ? repoFullName.slice(repoFullName.indexOf("/") + 1) : repoFullName; - try { - const config = (await env.REVIEW_CONFIG.get(slug, "json")) as { linkedIssueHardRules?: LinkedIssueHardRulesKvShape } | null; - const raw = config?.linkedIssueHardRules; - if (!raw || typeof raw !== "object") return DEFAULT_LINKED_ISSUE_HARD_RULES; - return { - ownerAssignedClose: asMode(raw.ownerAssignedClose) ?? DEFAULT_LINKED_ISSUE_HARD_RULES.ownerAssignedClose, - missingPointLabelClose: asMode(raw.missingPointLabelClose) ?? DEFAULT_LINKED_ISSUE_HARD_RULES.missingPointLabelClose, - maintainerOnlyLabelClose: asMode(raw.maintainerOnlyLabelClose) ?? DEFAULT_LINKED_ISSUE_HARD_RULES.maintainerOnlyLabelClose, - pointBearingLabels: asStringArray(raw.pointBearingLabels) ?? DEFAULT_POINT_BEARING_LABELS, - maintainerOnlyLabels: asStringArray(raw.maintainerOnlyLabels) ?? DEFAULT_MAINTAINER_ONLY_LABELS, - defaultLabelRepo: raw.defaultLabelRepo === true, - // Default ON: only an explicit `false` disables the flag-then-close double-check. - verifyBeforeClose: raw.verifyBeforeClose !== false, - closeDelaySeconds: clampCloseDelaySeconds(raw.closeDelaySeconds), - }; - } catch { - // A KV outage must NEVER let a deterministic close fire — fail safe to all-off (the opposite of the - // guardrail loader, which fails CLOSED: there a fault widens the manual-hold surface, here a fault must - // not manufacture a close). - return DEFAULT_LINKED_ISSUE_HARD_RULES; - } +export async function loadLinkedIssueHardRules(_env: Env, _repoFullName: string): Promise { + return DEFAULT_LINKED_ISSUE_HARD_RULES; } export type LinkedIssueFacts = { diff --git a/test/unit/change-guardrail.test.ts b/test/unit/change-guardrail.test.ts index c9629e5c0f..7246724c2f 100644 --- a/test/unit/change-guardrail.test.ts +++ b/test/unit/change-guardrail.test.ts @@ -55,9 +55,9 @@ describe("change-guardrail glob matching", () => { }); }); -// #flood-readiness: the LIVE gittensory KV globs must guard crucial files that live OUTSIDE the dir-prefix +// #flood-readiness: the live hard-guardrail globs must guard crucial files that live OUTSIDE the dir-prefix // guards (the awesome-claude #4196 class — a weakened sensitive file slipping through because its folder -// wasn't covered), while leaving clean non-crucial PRs auto-mergeable. Mirrors REVIEW_CONFIG["gittensory"]. +// wasn't covered), while leaving clean non-crucial PRs auto-mergeable. describe("hard-guardrail covers content-crucial files outside the dir-prefix guards", () => { const GITTENSORY_GLOBS = [ ".github/**", "scripts/**", "packages/**", "apps/gittensory-ui/**", @@ -86,7 +86,7 @@ describe("hard-guardrail covers content-crucial files outside the dir-prefix gua expect(changedPathsHittingGuardrail(nonCrucial, GITTENSORY_GLOBS)).toEqual([]); }); - it("the fail-closed sentinel ['**'] guards every path (KV-outage hold-all)", () => { + it("the hold-all sentinel ['**'] guards every path", () => { for (const p of ["src/utils/json.ts", "README.md", "anything/at/all.txt"]) { expect(matchesAny(p, ["**"])).toBe(true); } diff --git a/test/unit/guardrail-config.test.ts b/test/unit/guardrail-config.test.ts index 40d31060a7..412066aed1 100644 --- a/test/unit/guardrail-config.test.ts +++ b/test/unit/guardrail-config.test.ts @@ -1,38 +1,20 @@ -import { describe, expect, it, vi } from "vitest"; +import { describe, expect, it } from "vitest"; import { matchesAny } from "../../src/signals/change-guardrail"; -import { CONFIG_AS_CODE_GUARDRAIL_GLOBS, DEFAULT_CRUCIAL_GUARDRAIL_GLOBS, ENGINE_DECISION_GUARDRAIL_GLOBS, FAIL_CLOSED_GUARDRAIL_GLOBS, loadHardGuardrailGlobs } from "../../src/review/guardrail-config"; - -function envWith(get: (key: string, type: string) => Promise): Env { - return { REVIEW_CONFIG: { get } } as unknown as Env; -} +import { CONFIG_AS_CODE_GUARDRAIL_GLOBS, DEFAULT_CRUCIAL_GUARDRAIL_GLOBS, ENGINE_DECISION_GUARDRAIL_GLOBS, loadHardGuardrailGlobs } from "../../src/review/guardrail-config"; describe("loadHardGuardrailGlobs", () => { - it("returns the conservative default plus the config-as-code + engine guards when REVIEW_CONFIG is unbound", async () => { + it("returns the built-in self-host guardrails without requiring external policy storage", async () => { expect(await loadHardGuardrailGlobs({} as Env, "JSONbored/gittensory")).toEqual([...DEFAULT_CRUCIAL_GUARDRAIL_GLOBS, ...CONFIG_AS_CODE_GUARDRAIL_GLOBS, ...ENGINE_DECISION_GUARDRAIL_GLOBS]); }); - it("reads globs from KV keyed by the repo slug (owner stripped) and always appends the config-as-code + engine guards", async () => { - const get = vi.fn().mockResolvedValue({ hardGuardrailGlobs: ["src/scoring/**", "scripts/**"] }); - const globs = await loadHardGuardrailGlobs(envWith(get), "JSONbored/gittensory"); - expect(globs).toEqual(["src/scoring/**", "scripts/**", ...CONFIG_AS_CODE_GUARDRAIL_GLOBS, ...ENGINE_DECISION_GUARDRAIL_GLOBS]); - expect(get).toHaveBeenCalledWith("gittensory", "json"); - }); - - it("falls back to the default (plus config-as-code + engine guards) when the field is absent, null, or empty", async () => { - const expected = [...DEFAULT_CRUCIAL_GUARDRAIL_GLOBS, ...CONFIG_AS_CODE_GUARDRAIL_GLOBS, ...ENGINE_DECISION_GUARDRAIL_GLOBS]; - expect(await loadHardGuardrailGlobs(envWith(async () => ({})), "o/r")).toEqual(expected); - expect(await loadHardGuardrailGlobs(envWith(async () => null), "o/r")).toEqual(expected); - expect(await loadHardGuardrailGlobs(envWith(async () => ({ hardGuardrailGlobs: [] })), "o/r")).toEqual(expected); - }); - - it("drops non-string entries and keeps the valid globs (plus config-as-code + engine guards)", async () => { - const globs = await loadHardGuardrailGlobs(envWith(async () => ({ hardGuardrailGlobs: [123, "scripts/**", ""] })), "o/r"); - expect(globs).toEqual(["scripts/**", ...CONFIG_AS_CODE_GUARDRAIL_GLOBS, ...ENGINE_DECISION_GUARDRAIL_GLOBS]); + it("ignores unrelated env data so old hosted policy cannot affect self-host review policy", async () => { + const globs = await loadHardGuardrailGlobs({ LEGACY_POLICY: { hardGuardrailGlobs: ["docs/**"] } } as unknown as Env, "JSONbored/gittensory"); + expect(globs).toEqual([...DEFAULT_CRUCIAL_GUARDRAIL_GLOBS, ...CONFIG_AS_CODE_GUARDRAIL_GLOBS, ...ENGINE_DECISION_GUARDRAIL_GLOBS]); + expect(matchesAny("docs/readme.md", globs)).toBe(false); }); - it("guards the engine's own crown-jewel decision paths for every repo, even under a narrow KV glob list", async () => { - // A repo whose KV tuning is deliberately minimal must NOT thereby un-guard the review engine's own code. - const globs = await loadHardGuardrailGlobs(envWith(async () => ({ hardGuardrailGlobs: ["docs/**"] })), "o/r"); + it("guards the engine's own crown-jewel decision paths for every repo", async () => { + const globs = await loadHardGuardrailGlobs({} as Env, "o/r"); for (const enginePath of [ "src/rules/advisory.ts", "src/services/agent-action-executor.ts", @@ -46,34 +28,17 @@ describe("loadHardGuardrailGlobs", () => { ]) { expect(matchesAny(enginePath, globs)).toBe(true); } - expect(matchesAny("docs/readme.md", globs)).toBe(true); // the repo's own narrow KV glob still applies + expect(matchesAny("docs/readme.md", globs)).toBe(false); expect(matchesAny("src/utils/json.ts", globs)).toBe(false); // a non-decision src file remains auto-mergeable expect(matchesAny("src/db/repositories.ts", globs)).toBe(false); // the data layer stays non-crucial (env kill-switch backstops the freeze there) }); it("guards the gate's own policy files for every repo (the config-as-code self-weakening hole)", async () => { - const globs = await loadHardGuardrailGlobs(envWith(async () => ({ hardGuardrailGlobs: ["src/**"] })), "o/r"); + const globs = await loadHardGuardrailGlobs({} as Env, "o/r"); for (const policyFile of [".gittensory.yml", ".github/gittensory.json", "codecov.yml", ".github/codecov.yml"]) { expect(matchesAny(policyFile, globs)).toBe(true); } - expect(matchesAny("src/utils/json.ts", globs)).toBe(true); // the repo's own KV glob still applies + expect(matchesAny("src/utils/json.ts", globs)).toBe(false); expect(matchesAny("README.md", globs)).toBe(false); // an unrelated file is still auto-mergeable }); - - it("fails CLOSED (guard everything) when the KV read throws — an outage must never open the gate", async () => { - const globs = await loadHardGuardrailGlobs( - envWith(async () => { - throw new Error("kv down"); - }), - "o/r", - ); - expect(globs).toEqual(FAIL_CLOSED_GUARDRAIL_GLOBS); // ["**"] → every path held for human review - expect(globs).not.toEqual(DEFAULT_CRUCIAL_GUARDRAIL_GLOBS); - }); - - it("uses the whole name as the slug when there is no owner prefix", async () => { - const get = vi.fn().mockResolvedValue({ hardGuardrailGlobs: ["a/**"] }); - await loadHardGuardrailGlobs(envWith(get), "soloname"); - expect(get).toHaveBeenCalledWith("soloname", "json"); - }); }); diff --git a/test/unit/linked-issue-hard-rules.test.ts b/test/unit/linked-issue-hard-rules.test.ts index 31d12412a2..d3d7fd1cfb 100644 --- a/test/unit/linked-issue-hard-rules.test.ts +++ b/test/unit/linked-issue-hard-rules.test.ts @@ -191,98 +191,64 @@ describe("evaluateLinkedIssueHardRules", () => { }); }); -function envWith(get: (key: string, type: string) => Promise): Env { - return { REVIEW_CONFIG: { get } } as unknown as Env; -} - describe("loadLinkedIssueHardRules", () => { - it("returns the all-off default when REVIEW_CONFIG is unbound", async () => { + it("returns the all-off default without requiring external policy storage", async () => { expect(await loadLinkedIssueHardRules({} as Env, "JSONbored/gittensory")).toEqual(DEFAULT_LINKED_ISSUE_HARD_RULES); }); - it("returns the all-off default when the key / field is absent", async () => { - expect(await loadLinkedIssueHardRules(envWith(async () => null), "o/r")).toEqual(DEFAULT_LINKED_ISSUE_HARD_RULES); - expect(await loadLinkedIssueHardRules(envWith(async () => ({})), "o/r")).toEqual(DEFAULT_LINKED_ISSUE_HARD_RULES); - expect(await loadLinkedIssueHardRules(envWith(async () => ({ linkedIssueHardRules: null })), "o/r")).toEqual(DEFAULT_LINKED_ISSUE_HARD_RULES); - }); - - it("returns the all-off default when the KV read THROWS (outage must never manufacture a close)", async () => { - const result = await loadLinkedIssueHardRules( - envWith(async () => { - throw new Error("kv down"); - }), - "o/r", + it("ignores unrelated env data so stale hosted config cannot manufacture a close", async () => { + const cfg = await loadLinkedIssueHardRules( + { + LEGACY_POLICY: { + linkedIssueHardRules: { + ownerAssignedClose: "block", + missingPointLabelClose: "block", + maintainerOnlyLabelClose: "block", + pointBearingLabels: ["gittensor:bug"], + maintainerOnlyLabels: ["reserved"], + defaultLabelRepo: true, + verifyBeforeClose: false, + closeDelaySeconds: 0, + }, + }, + } as unknown as Env, + "JSONbored/gittensory", ); - expect(result).toEqual(DEFAULT_LINKED_ISSUE_HARD_RULES); + expect(cfg).toEqual(DEFAULT_LINKED_ISSUE_HARD_RULES); }); - it("reads the config keyed by the repo slug (owner stripped)", async () => { - const get = vi.fn().mockResolvedValue({ - linkedIssueHardRules: { - ownerAssignedClose: "block", - missingPointLabelClose: "block", - maintainerOnlyLabelClose: "block", - pointBearingLabels: ["gittensor:bug"], - maintainerOnlyLabels: ["reserved"], - defaultLabelRepo: true, - }, - }); - const cfg = await loadLinkedIssueHardRules(envWith(get), "JSONbored/gittensory"); - expect(get).toHaveBeenCalledWith("gittensory", "json"); + it("the default is explicitly all-off and keeps the verification timing stable", async () => { + const cfg = await loadLinkedIssueHardRules({} as Env, "soloname"); expect(cfg).toEqual({ + ownerAssignedClose: "off", + missingPointLabelClose: "off", + maintainerOnlyLabelClose: "off", + pointBearingLabels: [], + maintainerOnlyLabels: [], + defaultLabelRepo: false, + verifyBeforeClose: true, + closeDelaySeconds: 30, + }); + }); +}); + +describe("evaluateLinkedIssueHardRules with explicit config", () => { + it("supports a fully enabled config for self-host config plumbing", () => { + const cfg: LinkedIssueHardRulesConfig = { ownerAssignedClose: "block", missingPointLabelClose: "block", maintainerOnlyLabelClose: "block", pointBearingLabels: ["gittensor:bug"], maintainerOnlyLabels: ["reserved"], defaultLabelRepo: true, - // verify config not specified in the KV object → defaults (verify ON, 30s). verifyBeforeClose: true, closeDelaySeconds: 30, + }; + expect(evaluateLinkedIssueHardRules({ issues: [issue({ number: 9, labels: ["reserved"] })], config: cfg, repoOwner: OWNER })).toEqual({ + violated: true, + reason: "Linked issue #9 is labeled `maintainer-only` — it is not open for community PRs.", }); }); - - it("merges a PARTIAL config over the safe default (omitted fields keep their default)", async () => { - const cfg = await loadLinkedIssueHardRules(envWith(async () => ({ linkedIssueHardRules: { maintainerOnlyLabelClose: "block" } })), "o/r"); - expect(cfg.maintainerOnlyLabelClose).toBe("block"); - expect(cfg.ownerAssignedClose).toBe("off"); - expect(cfg.missingPointLabelClose).toBe("off"); - expect(cfg.defaultLabelRepo).toBe(false); - // an enabled rule with no listed labels falls back to the default gittensor label lists - expect(cfg.pointBearingLabels).toEqual(["gittensor:bug", "gittensor:feature", "gittensor:priority"]); - expect(cfg.maintainerOnlyLabels).toEqual(["maintainer-only"]); - }); - - it("ignores an invalid mode value and keeps the default for that field", async () => { - const cfg = await loadLinkedIssueHardRules(envWith(async () => ({ linkedIssueHardRules: { ownerAssignedClose: "yes" } })), "o/r"); - expect(cfg.ownerAssignedClose).toBe("off"); - }); - - it("uses the whole name as the slug when there is no owner prefix", async () => { - const get = vi.fn().mockResolvedValue({ linkedIssueHardRules: { ownerAssignedClose: "block" } }); - await loadLinkedIssueHardRules(envWith(get), "soloname"); - expect(get).toHaveBeenCalledWith("soloname", "json"); - }); - - it("defaults verifyBeforeClose ON and closeDelaySeconds to 30 when unspecified", async () => { - const cfg = await loadLinkedIssueHardRules(envWith(async () => ({ linkedIssueHardRules: { ownerAssignedClose: "block" } })), "o/r"); - expect(cfg.verifyBeforeClose).toBe(true); - expect(cfg.closeDelaySeconds).toBe(30); - }); - - it("disables verifyBeforeClose only on an explicit false (any other value keeps ON)", async () => { - expect((await loadLinkedIssueHardRules(envWith(async () => ({ linkedIssueHardRules: { verifyBeforeClose: false } })), "o/r")).verifyBeforeClose).toBe(false); - expect((await loadLinkedIssueHardRules(envWith(async () => ({ linkedIssueHardRules: { verifyBeforeClose: "no" } })), "o/r")).verifyBeforeClose).toBe(true); - expect((await loadLinkedIssueHardRules(envWith(async () => ({ linkedIssueHardRules: {} })), "o/r")).verifyBeforeClose).toBe(true); - }); - - it("clamps closeDelaySeconds into [0, 300] and falls back to 30 for a non-number", async () => { - expect((await loadLinkedIssueHardRules(envWith(async () => ({ linkedIssueHardRules: { closeDelaySeconds: 120 } })), "o/r")).closeDelaySeconds).toBe(120); - expect((await loadLinkedIssueHardRules(envWith(async () => ({ linkedIssueHardRules: { closeDelaySeconds: -5 } })), "o/r")).closeDelaySeconds).toBe(0); - expect((await loadLinkedIssueHardRules(envWith(async () => ({ linkedIssueHardRules: { closeDelaySeconds: 9999 } })), "o/r")).closeDelaySeconds).toBe(300); - expect((await loadLinkedIssueHardRules(envWith(async () => ({ linkedIssueHardRules: { closeDelaySeconds: 45.9 } })), "o/r")).closeDelaySeconds).toBe(45); - expect((await loadLinkedIssueHardRules(envWith(async () => ({ linkedIssueHardRules: { closeDelaySeconds: "30" } })), "o/r")).closeDelaySeconds).toBe(30); - }); }); describe("resolveLinkedIssueHardRule (#1144 — overflow + orchestration)", () => { From 8da2d62fd76fe7a29cb377a7899affeed31815ba Mon Sep 17 00:00:00 2001 From: JSONbored <49853598+JSONbored@users.noreply.github.com> Date: Mon, 29 Jun 2026 03:36:03 -0700 Subject: [PATCH 49/68] fix(review): publish final self-host review surfaces Publish deterministic final PR surfaces when AI notes are unavailable instead of leaving reviews in the transient reviewing state. Treat manifest blocked paths as manual-review holds, clarify suggested-action language, add review freshness timestamps, render nits as task-list items, and run the self-host smoke test with Redis. Validation: npm test -- --run test/unit/selfhost-queue-common.test.ts test/unit/selfhost-sqlite-queue.test.ts test/unit/github-comments.test.ts test/unit/github-app.test.ts test/unit/backfill.test.ts test/unit/unified-comment.test.ts test/unit/unified-comment-bridge.test.ts test/unit/queue.test.ts test/unit/gate-check-policy.test.ts test/unit/predicted-gate.test.ts test/unit/mcp-predict-gate.test.ts; npm run typecheck; npm run actionlint -- .github/workflows/selfhost.yml; git diff --check --- .github/workflows/selfhost.yml | 10 +- src/queue/processors.ts | 19 ++-- src/review/unified-comment-bridge.ts | 3 + src/review/unified-comment.ts | 42 +++++--- src/rules/advisory.ts | 41 ++++++-- test/unit/backfill.test.ts | 12 +++ test/unit/gate-check-policy.test.ts | 30 +++++- test/unit/github-app.test.ts | 37 +++++++ test/unit/github-comments.test.ts | 24 +++++ test/unit/mcp-predict-gate.test.ts | 7 +- test/unit/predicted-gate.test.ts | 7 +- test/unit/queue.test.ts | 24 +++-- test/unit/selfhost-queue-common.test.ts | 117 +++++++++++++++++++++++ test/unit/selfhost-sqlite-queue.test.ts | 86 +++++++++++++++++ test/unit/unified-comment-bridge.test.ts | 51 ++++++---- test/unit/unified-comment.test.ts | 50 +++++++--- 16 files changed, 476 insertions(+), 84 deletions(-) diff --git a/.github/workflows/selfhost.yml b/.github/workflows/selfhost.yml index fdc597fc41..8fa575e23d 100644 --- a/.github/workflows/selfhost.yml +++ b/.github/workflows/selfhost.yml @@ -103,7 +103,14 @@ jobs: - name: Boot the container + smoke-test /health, /ready, /metrics, migrations run: | - docker run -d --name gt -p 8787:8787 gittensory:selfhost-ci + docker network create gt-smoke + docker run -d --name gt-redis --network gt-smoke redis:7-alpine + trap 'docker rm -f gt gt-redis >/dev/null 2>&1 || true; docker network rm gt-smoke >/dev/null 2>&1 || true' EXIT + for _ in $(seq 1 30); do + if docker exec gt-redis redis-cli ping | grep -q PONG; then break; fi + sleep 1 + done + docker run -d --name gt --network gt-smoke -p 8787:8787 -e REDIS_URL=redis://gt-redis:6379 gittensory:selfhost-ci ok=0 for _ in $(seq 1 30); do if curl -sf http://127.0.0.1:8787/health >/dev/null; then ok=1; break; fi @@ -115,4 +122,3 @@ jobs: curl -sf http://127.0.0.1:8787/metrics | grep -q 'gittensory_uptime_seconds' docker logs gt 2>&1 | grep -q 'selfhost_migrations_applied' echo "self-host smoke test passed" - docker rm -f gt diff --git a/src/queue/processors.ts b/src/queue/processors.ts index cc40e6c92d..684c26b13d 100644 --- a/src/queue/processors.ts +++ b/src/queue/processors.ts @@ -400,7 +400,6 @@ const PR_PUBLIC_SURFACE_ACTIONS = new Set([ ]); const PR_GATE_CLOSED_ACTIONS = new Set(["closed"]); const ISSUE_PLAN_COOLDOWN_MS = 10 * 60 * 1000; -const AI_REVIEW_INCOMPLETE_RETRY_MS = 5 * 60 * 1000; /** * Run (or dry-run) the data-retention prune across the configured log/snapshot tables and audit the @@ -4695,33 +4694,27 @@ async function maybePublishPrPublicSurface( } } if (aiReviewExpected && !hasPublicReviewAssessment(aiReview?.notes)) { - const retryError = new RetryableJobError( - "AI review did not produce a public summary yet; keeping PR surface in reviewing state", - { - retryAfterMs: AI_REVIEW_INCOMPLETE_RETRY_MS, - retryKind: "ai_review_public_summary_missing", - }, - ); + const message = + "AI review did not produce a public summary; publishing deterministic PR surface without AI notes"; await recordAuditEvent(env, { eventType: "github_app.ai_review_public_summary_missing", actor: author, targetKey: `${repoFullName}#${pr.number}`, - outcome: "error", - detail: retryError.message, + outcome: "completed", + detail: message, metadata: { deliveryId: webhook.deliveryId, repoFullName, - retryAfterMs: AI_REVIEW_INCOMPLETE_RETRY_MS, + aiReviewMode: settings.aiReviewMode, }, }).catch(() => undefined); - captureReviewFailure(retryError, { + captureReviewFailure(new Error(message), { kind: "review", reason: "ai_review_public_summary_missing", repo: repoFullName, pr: pr.number, head_sha: advisory.headSha, }); - throw retryError; } // Secrets-scan (#audit-3.4): always scans the REAL resolved diff and, on a CONCRETE credential hit, appends a diff --git a/src/review/unified-comment-bridge.ts b/src/review/unified-comment-bridge.ts index f0741f6288..d0e7ead3c3 100644 --- a/src/review/unified-comment-bridge.ts +++ b/src/review/unified-comment-bridge.ts @@ -274,6 +274,8 @@ export type UnifiedCommentBridgeArgs = { /** The author is the repo owner or a protected automation bot — never auto-closed, so a gate "close" verdict * renders as "held" rather than "Closed" (#8/#9). */ neverClosed?: boolean | undefined; + /** Public freshness marker for the posted/updated review comment. Defaults to the current publish time. */ + reviewedAt?: string | number | Date | undefined; }; /** @@ -382,6 +384,7 @@ export function buildUnifiedCommentBody(args: UnifiedCommentBridgeArgs): string readinessScore: args.readinessTotal, signals, footerMarkdown: args.footerMarkdown, + reviewedAt: args.reviewedAt ?? new Date(), ...(args.reRunLabel !== undefined ? { reRunLabel: args.reRunLabel } : {}), ...(extraCollapsibles !== undefined ? { extraCollapsibles } : {}), ...(args.heldForReview ? { heldForReview: true } : {}), diff --git a/src/review/unified-comment.ts b/src/review/unified-comment.ts index 340feeadd1..af43daaf82 100644 --- a/src/review/unified-comment.ts +++ b/src/review/unified-comment.ts @@ -209,6 +209,8 @@ export interface UnifiedCommentContext { /** The PR's author is the repo owner or a protected automation bot — the disposition NEVER auto-closes them, * so a gate "close" verdict renders as "held", not "Closed" (#8/#9). */ neverClosed?: boolean; + /** Public freshness marker for the posted/updated review comment. Rendered as UTC when provided. */ + reviewedAt?: string | number | Date | undefined; } const STATUS_META: Record = { @@ -282,16 +284,16 @@ export function deriveUnifiedStatus(input: UnifiedReviewInput, ctx: UnifiedComme return status; } -function verb(status: UnifiedCommentStatus, input: UnifiedReviewInput): string { +function headlineLabel(status: UnifiedCommentStatus, input: UnifiedReviewInput): string { switch (status) { case "ready": - return "safe to merge"; + return "approve/merge recommended"; case "advisory": - return "advisory only"; + return "advisory review"; case "held": - return "held for maintainer review"; + return "manual review recommended"; case "blocked": - return input.decision === "close" ? "closed" : "blocked"; + return input.decision === "close" ? "reject/close recommended" : "blockers found"; } } @@ -319,14 +321,14 @@ function verdictLine(status: UnifiedCommentStatus, input: UnifiedReviewInput): s switch (status) { case "ready": return input.merged - ? `**${icon} Approved & auto-merged**${input.verdictReason ? reason : " — all checks passed"}` - : `**${icon} Approved**${input.verdictReason ? reason : " — safe to merge"}`; + ? `**${icon} Suggested Action - Approve/Merge**${input.verdictReason ? reason : " — auto-merged"}` + : `**${icon} Suggested Action - Approve/Merge**${input.verdictReason ? reason : " — safe to merge"}`; case "advisory": - return `**${icon} Advisory only**${input.verdictReason ? reason : " — no action taken"}`; + return `**${icon} Suggested Action - Advisory Only**${input.verdictReason ? reason : " — no action taken"}`; case "held": - return `**${icon} Held for maintainer review**${reason}`; + return `**${icon} Suggested Action - Manual Review**${reason}`; case "blocked": - return `**${icon} ${input.decision === "close" ? "Closed" : "Blocked"}**${reason}`; + return `**${icon} Suggested Action - ${input.decision === "close" ? "Reject/Close" : "Fix Blockers"}**${reason}`; } } @@ -358,6 +360,20 @@ function bullets(items: string[]): string { .join("\n"); } +function taskList(items: string[]): string { + return dedupeLines(items) + .map((i) => `- [ ] ${escapePublicHtmlAngles(i)}`) + .join("\n"); +} + +function formatReviewTimestamp(value: string | number | Date | undefined): string | null { + if (value === undefined) return null; + const time = value instanceof Date ? value : new Date(value); + const ms = time.getTime(); + if (!Number.isFinite(ms)) return null; + return time.toISOString().replace(/\.\d{3}Z$/, "Z").replace("T", " ").replace("Z", " UTC"); +} + /** Render the failing CI checks as a bullet list of `name — reason` (reason only when the check carried one), * preferring failingDetails (which pairs each name with its WHY: codecov %/test/lint reason) and falling back * to the bare failingChecks names. Public-safe: only check names + their already-public short summary, both @@ -440,15 +456,17 @@ export function renderUnifiedReviewComment(input: UnifiedReviewInput, ctx: Unifi const blocks: string[] = [ meta.square.repeat(12), - `### ${meta.icon} ${brand} — ${verb(status, input)}${status === "ready" && input.merged ? " · auto-merged" : ""}`, + `### ${meta.icon} ${brand} result - ${headlineLabel(status, input)}${status === "ready" && input.merged ? " · auto-merged" : ""}`, statusChips(input, ctx), verdictLine(status, input), ]; + const reviewTimestamp = formatReviewTimestamp(ctx.reviewedAt); + if (reviewTimestamp) blocks.push(`Review updated: ${reviewTimestamp}`); if (input.summary.trim()) blocks.push(`**Review summary**\n${escapePublicHtmlAngles(input.summary.trim())}`); const nits = dedupeLines(input.nits ?? []); - if (nits.length) blocks.push(details("Nits", bullets(nits), `${nits.length} non-blocking`)); + if (nits.length) blocks.push(details("Nits", taskList(nits), `${nits.length} non-blocking`)); const blockers = dedupeLines(input.blockers ?? []); if (blockers.length) { diff --git a/src/rules/advisory.ts b/src/rules/advisory.ts index 7e3d4ce607..786e38216b 100644 --- a/src/rules/advisory.ts +++ b/src/rules/advisory.ts @@ -41,10 +41,10 @@ export type GateCheckPolicy = { * linked-issue, duplicate, quality/readiness, slop — to its mode, so a maintainer flips ONE switch instead * of four and `Gittensory Gate` stays the single required check. `off` = sub-gates use their own modes. */ mergeReadinessGateMode?: GateRuleMode | undefined; - /** Focus-manifest policy gate (#555). When `block`, the focus manifest's declared policy findings — - * `manifest_blocked_path`, `manifest_linked_issue_required`, `manifest_missing_tests` — become hard - * blockers. An INDEPENDENT dimension, deliberately NOT folded into the merge-readiness composite so #555 - * stays focused. `off`/`advisory` = the findings stay advisory (never block). Default off. */ + /** Focus-manifest policy gate (#555). When `block`, linked-issue/test policy findings become hard blockers; + * blocked-path findings become manual-review holds because guardrailed paths should be reviewed, not closed. + * An INDEPENDENT dimension, deliberately NOT folded into the merge-readiness composite so #555 stays focused. + * `off`/`advisory` = the findings stay advisory (never block). Default off. */ manifestPolicyGateMode?: GateRuleMode | undefined; /** Self-authored linked-issue gate. When `block`, a `self_authored_linked_issue` finding — raised when * the PR author also filed the linked issue — becomes a hard blocker. Defaults to `advisory` — the @@ -440,6 +440,27 @@ function buildGuardrailHoldFinding(): AdvisoryFinding { }; } +function buildManifestBlockedPathHoldFinding( + findings: AdvisoryFinding[], + policy: GateCheckPolicy, +): AdvisoryFinding | null { + if (gateMode(policy.manifestPolicyGateMode ?? "off") !== "block") + return null; + const blocked = findings.find( + (finding) => finding.code === "manifest_blocked_path", + ); + if (!blocked) return null; + return { + code: "manifest_blocked_path", + severity: "warning", + title: "Touches a maintainer-blocked path — held for manual review", + detail: + "This PR changes a maintainer-blocked path, so it is held for a maintainer to review and merge manually.", + action: "A maintainer must review and merge this change.", + ...(blocked.publicText !== undefined ? { publicText: blocked.publicText } : {}), + }; +} + /** Dry-run disposition (#gate-dryrun): promote every `advisory` sub-gate mode to `block` so the core eval yields the * would-be conclusion. `off`/`block`/unset modes are untouched; non-mode policy (grace, size HOLD, guardrail) is * preserved as-is, so the would-be verdict still honours newcomer grace and the manual-review holds. PURE. */ @@ -546,7 +567,11 @@ function evaluateGateCheckCore(advisoryResult: Advisory, policy: GateCheckPolicy // so neutral never blocks the merge (dry-run/advisory friendly) and a contributor PR is never auto-closed for size. const sizeHold = buildSizeHoldFinding(effective); const guardrailHold = effective.guardrailHit ? buildGuardrailHoldFinding() : null; - const holds = [sizeHold, guardrailHold].filter( + const manifestBlockedPathHold = buildManifestBlockedPathHoldFinding( + advisoryResult.findings, + effective, + ); + const holds = [sizeHold, guardrailHold, manifestBlockedPathHold].filter( (f): f is AdvisoryFinding => f !== null, ); if (holds.length > 0) { @@ -864,9 +889,9 @@ function isConfiguredGateBlocker(finding: AdvisoryFinding, policy: GateCheckPoli // when the maintainer configured an enforced check). The advisory variant (`pre_merge_check_failed`) is a plain // warning and is never blocked here. No AI judgment is involved, so this can never cause an AI false-close. if (code === "pre_merge_check_required") return true; - // Focus-manifest policy (#555): the three enforceable manifest findings block ONLY when the maintainer - // opts into manifestPolicy: block. Default off/advisory keeps them advisory-only. - if (code === "manifest_blocked_path" || code === "manifest_linked_issue_required" || code === "manifest_missing_tests") { + // Focus-manifest policy (#555): linked-issue/test policy findings block ONLY when the maintainer opts into + // manifestPolicy: block. Blocked paths are guardrails, so they are handled as manual-review holds above. + if (code === "manifest_linked_issue_required" || code === "manifest_missing_tests") { return gateMode(policy.manifestPolicyGateMode ?? "off") === "block"; } // Self-authored linked-issue gate: blocks only when the maintainer opts in with `block`. Defaults to diff --git a/test/unit/backfill.test.ts b/test/unit/backfill.test.ts index 2af1d6d396..56c657cc0b 100644 --- a/test/unit/backfill.test.ts +++ b/test/unit/backfill.test.ts @@ -2790,6 +2790,17 @@ describe("GitHub backfill", () => { }); describe("fetchLiveCiAggregate", () => { + it("reports unverified without fetching when the head SHA is missing", async () => { + const env = createTestEnv({ GITHUB_PUBLIC_TOKEN: "public-token" }); + const fetchSpy = vi.fn(); + vi.stubGlobal("fetch", fetchSpy); + + const aggregate = await fetchLiveCiAggregate(env, "JSONbored/gittensory", null, "public-token", null); + + expect(aggregate).toEqual({ ciState: "unverified", hasPending: false, failingDetails: [], nonRequiredFailingDetails: [] }); + expect(fetchSpy).not.toHaveBeenCalled(); + }); + it("keeps non-required failing and pending statuses advisory when required contexts are known", async () => { const env = createTestEnv({ GITHUB_PUBLIC_TOKEN: "public-token" }); vi.stubGlobal("fetch", async (input: RequestInfo | URL) => { @@ -2799,6 +2810,7 @@ describe("GitHub backfill", () => { check_runs: [ { name: "trusted-required-ci", status: "completed", conclusion: "success" }, { name: "attacker/non-required-check", status: "completed", conclusion: "failure", output: { title: "Injected failure" } }, + { name: "attacker/non-required-pending-check", status: "queued", conclusion: null }, ], }); } diff --git a/test/unit/gate-check-policy.test.ts b/test/unit/gate-check-policy.test.ts index 421a906513..93ff768984 100644 --- a/test/unit/gate-check-policy.test.ts +++ b/test/unit/gate-check-policy.test.ts @@ -475,7 +475,7 @@ describe("focus-manifest policy gate (#555)", () => { return { ...missingIssueAdvisory(), findings: [POLICY_FINDINGS[code]] }; } - for (const code of Object.keys(POLICY_FINDINGS) as (keyof typeof POLICY_FINDINGS)[]) { + for (const code of ["manifest_linked_issue_required", "manifest_missing_tests"] as const) { describe(code, () => { it("blocks a confirmed contributor when manifestPolicy: block", () => { const result = evaluateGateCheck(manifestAdvisory(code), { manifestPolicyGateMode: "block", confirmedContributor: true }); @@ -499,6 +499,27 @@ describe("focus-manifest policy gate (#555)", () => { }); } + describe("manifest_blocked_path", () => { + it("holds for manual review when manifestPolicy: block", () => { + const result = evaluateGateCheck(manifestAdvisory("manifest_blocked_path"), { manifestPolicyGateMode: "block", confirmedContributor: true }); + expect(result.conclusion).toBe("neutral"); + expect(result.blockers).toEqual([]); + expect(result.warnings.map((finding) => finding.code)).toContain("manifest_blocked_path"); + expect(result.summary).toMatch(/held for manual review/i); + }); + + it("also holds non-confirmed contributors for manual review instead of closing", () => { + const result = evaluateGateCheck(manifestAdvisory("manifest_blocked_path"), { manifestPolicyGateMode: "block", confirmedContributor: false }); + expect(result.conclusion).toBe("neutral"); + expect(result.blockers).toEqual([]); + }); + + it("does not block when manifestPolicy: off/advisory", () => { + expect(evaluateGateCheck(manifestAdvisory("manifest_blocked_path"), { manifestPolicyGateMode: "off", confirmedContributor: true }).conclusion).toBe("success"); + expect(evaluateGateCheck(manifestAdvisory("manifest_blocked_path"), { manifestPolicyGateMode: "advisory", confirmedContributor: true }).conclusion).toBe("success"); + }); + }); + it("is an INDEPENDENT dimension: mergeReadiness: block does NOT promote a manifest-policy finding (kept out of the composite)", () => { const eff = resolveEffectiveSettings(settings({ manifestPolicyGateMode: "off", mergeReadinessGateMode: "block" }), parseFocusManifest(null)); expect(evaluateGateCheck(manifestAdvisory("manifest_blocked_path"), gateCheckPolicy(eff, null, true)).conclusion).toBe("success"); @@ -508,12 +529,13 @@ describe("focus-manifest policy gate (#555)", () => { expect(gateCheckPolicy(settings({ manifestPolicyGateMode: "block" }), null, true).manifestPolicyGateMode).toBe("block"); }); - it("end-to-end: a manifest gate.manifestPolicy: block sets effective.manifestPolicyGateMode and blocks a blockedPath PR", () => { + it("end-to-end: a manifest gate.manifestPolicy: block sets effective.manifestPolicyGateMode and holds a blockedPath PR", () => { const eff = resolveEffectiveSettings(settings({ manifestPolicyGateMode: "off" }), parseFocusManifest({ gate: { manifestPolicy: "block" } })); expect(eff.manifestPolicyGateMode).toBe("block"); const result = evaluateGateCheck(manifestAdvisory("manifest_blocked_path"), gateCheckPolicy(eff, null, true)); - expect(result.conclusion).toBe("failure"); - expect(result.blockers.map((finding) => finding.code)).toContain("manifest_blocked_path"); + expect(result.conclusion).toBe("neutral"); + expect(result.blockers).toEqual([]); + expect(result.warnings.map((finding) => finding.code)).toContain("manifest_blocked_path"); }); }); diff --git a/test/unit/github-app.test.ts b/test/unit/github-app.test.ts index d2953e0ccb..4c9e3eeda9 100644 --- a/test/unit/github-app.test.ts +++ b/test/unit/github-app.test.ts @@ -14,10 +14,13 @@ import { isCacheableGithubUrl, isCheckRunPermissionError, isForeignAppInstallation, + isGitHubBadCredentialsError, + isGitHubRateLimitedError, isRateLimitedResponse, rateLimitRetryMs, setGitHubResponseCache, setInstallationTokenStore, + withInstallationTokenRetry, } from "../../src/github/app"; import type { Advisory } from "../../src/types"; import { createTestEnv } from "../helpers/d1"; @@ -248,6 +251,40 @@ describe("GitHub check runs", () => { expect(rejectedReads).toBe(1); }); + it("does not evict a newer cached installation token when the rejected token is already stale", async () => { + const reads = [ + { token: "rejected-token", expiresAtMs: Date.now() + 60 * 60_000 }, + { token: "replacement-token", expiresAtMs: Date.now() + 60 * 60_000 }, + { token: "replacement-token", expiresAtMs: Date.now() + 60 * 60_000 }, + ]; + const writes: Array<{ token: string; expiresAtMs: number }> = []; + setInstallationTokenStore({ + get: async () => reads.shift() ?? null, + set: async (_installationId, value) => { + writes.push(value); + }, + }); + const seenTokens: string[] = []; + + const result = await withInstallationTokenRetry(createTestEnv(), 558, async (token) => { + seenTokens.push(token); + if (token === "rejected-token") + throw { response: { status: 401 }, message: "token expired" }; + return "ok"; + }); + + expect(result).toBe("ok"); + expect(seenTokens).toEqual(["rejected-token", "replacement-token"]); + expect(writes).toEqual([]); + expect(isGitHubBadCredentialsError(new Error("Bad credentials"))).toBe(true); + expect(isGitHubBadCredentialsError({ response: { status: 401 }, message: "Unauthorized" })).toBe(true); + }); + + it("does not treat primitive values as GitHub rate-limit errors", () => { + expect(isGitHubRateLimitedError("secondary rate limit")).toBe(false); + expect(isGitHubRateLimitedError(null)).toBe(false); + }); + it("single-flights concurrent cold-cache mints for one install (no thundering herd)", async () => { const privateKey = await generatePrivateKeyPem(); let mints = 0; diff --git a/test/unit/github-comments.test.ts b/test/unit/github-comments.test.ts index 49b425cb54..d8b3b2f450 100644 --- a/test/unit/github-comments.test.ts +++ b/test/unit/github-comments.test.ts @@ -176,6 +176,30 @@ describe("GitHub PR intelligence comments", () => { expect(commentListCalls).toEqual([1, 2, 3]); }); + it("returns null without creating a late first comment when createIfMissing is false", async () => { + const privateKey = await generatePrivateKeyPem(); + const calls: string[] = []; + vi.stubGlobal("fetch", async (input: RequestInfo | URL, init?: RequestInit) => { + const url = input.toString(); + calls.push(`${init?.method ?? "GET"} ${url}`); + if (url.includes("/access_tokens")) return Response.json({ token: "installation-token" }); + if (url.includes("/issues/12/comments") && (init?.method ?? "GET") === "GET") return Response.json([]); + return new Response("not found", { status: 404 }); + }); + + const result = await createOrUpdatePrIntelligenceComment( + createTestEnv({ GITHUB_APP_PRIVATE_KEY: privateKey }), + 123, + "JSONbored/gittensory", + 12, + `${PR_INTELLIGENCE_COMMENT_MARKER}\nbody`, + { createIfMissing: false }, + ); + + expect(result).toBeNull(); + expect(calls.some((call) => call.startsWith("POST ") && call.includes("/issues/12/comments"))).toBe(false); + }); + it("updates a legacy PR intelligence comment into the unified panel", async () => { const privateKey = await generatePrivateKeyPem(); const calls: string[] = []; diff --git a/test/unit/mcp-predict-gate.test.ts b/test/unit/mcp-predict-gate.test.ts index cefbfd50f4..bdc2a58731 100644 --- a/test/unit/mcp-predict-gate.test.ts +++ b/test/unit/mcp-predict-gate.test.ts @@ -73,9 +73,10 @@ describe("MCP gittensory_predict_gate", () => { arguments: { login: "miner1", owner: "acme", repo: "widgets", title: "Build output", changedPaths: ["dist/bundle.js"] }, }); expect(result.isError).toBeFalsy(); - const data = result.structuredContent as { conclusion: string; blockers: Array<{ code: string }>; note: string }; - expect(data.conclusion).toBe("failure"); - expect(data.blockers.some((b) => b.code === "manifest_blocked_path")).toBe(true); + const data = result.structuredContent as { conclusion: string; blockers: Array<{ code: string }>; warnings: Array<{ code: string }>; note: string }; + expect(data.conclusion).toBe("neutral"); + expect(data.blockers.some((b) => b.code === "manifest_blocked_path")).toBe(false); + expect(data.warnings.some((w) => w.code === "manifest_blocked_path")).toBe(true); // With paths supplied the note drops the "provide changed paths" disclaimer but still disclaims slop. expect(data.note).not.toContain("Provide the PR's changed paths"); expect(data.note.toLowerCase()).toContain("slop"); diff --git a/test/unit/predicted-gate.test.ts b/test/unit/predicted-gate.test.ts index 1a1b8dd9f9..2b5f6615f9 100644 --- a/test/unit/predicted-gate.test.ts +++ b/test/unit/predicted-gate.test.ts @@ -282,14 +282,15 @@ describe("buildPredictedGateVerdict", () => { expect(result.blockers.some((b) => b.code === "pre_merge_check_required")).toBe(false); }); - it("predicts a manifest path-policy BLOCK when a changed path hits a blocked glob and manifestPolicy:block (#12)", () => { + it("predicts a manifest path-policy HOLD when a changed path hits a blocked glob and manifestPolicy:block (#12)", () => { const result = verdict({ gate: { manifestPolicy: "block" }, manifestExtra: { blockedPaths: ["dist/**"] }, changedPaths: ["dist/bundle.js"], }); - expect(result.conclusion).toBe("failure"); - expect(result.blockers.some((b) => b.code === "manifest_blocked_path")).toBe(true); + expect(result.conclusion).toBe("neutral"); + expect(result.blockers.some((b) => b.code === "manifest_blocked_path")).toBe(false); + expect(result.warnings.some((w) => w.code === "manifest_blocked_path")).toBe(true); // The note no longer disclaims path-policy once paths are supplied, but slop stays disclaimed. expect(result.note).not.toContain("Provide the PR's changed paths"); expect(result.note.toLowerCase()).toContain("slop"); diff --git a/test/unit/queue.test.ts b/test/unit/queue.test.ts index 01c304ded3..e4b3110800 100644 --- a/test/unit/queue.test.ts +++ b/test/unit/queue.test.ts @@ -1350,7 +1350,7 @@ describe("queue processors", () => { expect(postedBodies[0]).toContain("🟪"); }); - it("keeps the PR comment and Gate in 🟪 reviewing state when AI review produces nits but no public summary", async () => { + it("publishes the final PR surface when AI review produces nits but no public summary", async () => { let aiCalls = 0; const env = createTestEnv({ GITHUB_APP_PRIVATE_KEY: await generatePrivateKeyPem(), @@ -1390,6 +1390,7 @@ describe("queue processors", () => { aiReviewMode: "block", gatePack: "oss-anti-slop", }); + await upsertOfficialMinerDetection(env, "contributor", { status: "confirmed", snapshot: queueMinerSnapshot("contributor") }, 60_000); await putCachedAiReview(env, "JSONbored/gittensory", 10, "a10", "block", { notes: "**Nits (1)**\n- stale cached nit", reviewerCount: 1, @@ -1431,22 +1432,25 @@ describe("queue processors", () => { pull_request: { number: 10, title: "Clean PR", state: "open", user: { login: "contributor" }, head: { sha: "a10" }, labels: [], body: "Closes #1" }, }, }), - ).rejects.toThrow(/public summary/i); + ).resolves.toBeUndefined(); - expect(commentBodies).toHaveLength(1); + expect(commentBodies.length).toBeGreaterThanOrEqual(2); expect(commentBodies[0]).toContain("is reviewing"); expect(commentBodies[0]).toContain("🟪"); - expect(commentBodies[0]).not.toContain("held for maintainer review"); - expect(commentBodies[0]).not.toContain("Review summary"); + const finalComment = commentBodies.find((body) => !body.includes("is reviewing")); + expect(finalComment).toBeDefined(); + expect(finalComment).toContain("Readiness score"); + expect(finalComment).not.toContain("stale cached nit"); + expect(finalComment).not.toContain("Add coverage for the new branch."); expect(aiCalls).toBeGreaterThan(0); - expect(checkPatches).toHaveLength(0); + expect(checkPatches).toContainEqual(expect.objectContaining({ status: "completed" })); const audit = await env.DB.prepare("select count(*) as n from audit_events where event_type = ?") .bind("github_app.ai_review_public_summary_missing") .first<{ n: number }>(); expect(audit?.n).toBe(1); }); - it("keeps re-gate PR jobs retryable when AI review produces no public summary and audit storage fails", async () => { + it("publishes a deterministic re-gate result when AI review produces no public summary and audit storage fails", async () => { const originalRecordAuditEvent = repositoriesModule.recordAuditEvent; const auditSpy = vi.spyOn(repositoriesModule, "recordAuditEvent").mockImplementation(async (auditEnv, event) => { if (event.eventType === "github_app.ai_review_public_summary_missing") @@ -1482,6 +1486,7 @@ describe("queue processors", () => { aiReviewMode: "block", gatePack: "oss-anti-slop", }); + await upsertOfficialMinerDetection(env, "contributor", { status: "confirmed", snapshot: queueMinerSnapshot("contributor") }, 60_000); await upsertPullRequestFromGitHub(env, "JSONbored/gittensory", { number: 48, title: "Clean PR", state: "open", user: { login: "contributor" }, head: { sha: "a48" }, labels: [], body: "Closes #1" }); const commentBodies: string[] = []; vi.stubGlobal("fetch", async (input: RequestInfo | URL, init?: RequestInit) => { @@ -1510,10 +1515,11 @@ describe("queue processors", () => { prNumber: 48, installationId: 123, }), - ).rejects.toThrow(/public summary/i); + ).resolves.toBeUndefined(); - expect(commentBodies).toHaveLength(1); + expect(commentBodies.length).toBeGreaterThanOrEqual(2); expect(commentBodies[0]).toContain("is reviewing"); + expect(commentBodies.some((body) => !body.includes("is reviewing"))).toBe(true); expect(auditSpy).toHaveBeenCalledWith( env, expect.objectContaining({ eventType: "github_app.ai_review_public_summary_missing" }), diff --git a/test/unit/selfhost-queue-common.test.ts b/test/unit/selfhost-queue-common.test.ts index 677c14ffe8..54317bbb2c 100644 --- a/test/unit/selfhost-queue-common.test.ts +++ b/test/unit/selfhost-queue-common.test.ts @@ -8,6 +8,7 @@ import { jobPriority, nonConsumingRetryDelayMs, queueBackgroundConcurrency, + queueStartupJitterMinJobs, } from "../../src/selfhost/queue-common"; import { RetryableJobError } from "../../src/queue/retryable"; @@ -36,6 +37,9 @@ describe("self-host queue common helpers", () => { expect(queueBackgroundConcurrency(4, "-1")).toBe(1); expect(queueBackgroundConcurrency(4, "not-a-number")).toBe(1); expect(queueBackgroundConcurrency(4, "0")).toBe(0); + expect(queueBackgroundConcurrency(Number.NaN, "3")).toBe(0); + expect(queueBackgroundConcurrency(4, null)).toBe(1); + expect(queueBackgroundConcurrency(4, "")).toBe(1); }); it("demotes bot-authored issue-comment edit webhooks without demoting human reruns", () => { @@ -78,9 +82,26 @@ describe("self-host queue common helpers", () => { parse.mockRestore(); }); + it("fails closed when an agent re-gate priority payload becomes unreadable after type extraction", () => { + const raw = payload({ type: "agent-regate-pr" }); + const parse = vi.spyOn(JSON, "parse"); + parse + .mockImplementationOnce(() => ({ type: "agent-regate-pr" })) + .mockImplementationOnce(() => { + throw new Error("malformed re-gate payload"); + }); + + expect(jobPriority(raw)).toBe(9); + parse.mockRestore(); + }); + it("coalesces CI-completion webhooks with sorted pull numbers", () => { + expect(jobCoalesceKey(payload({ type: "agent-regate-pr", repoFullName: "JSONbored/Gittensory", prNumber: 7 }))).toBe("agent-regate-pr:jsonbored/gittensory#7"); + expect(jobCoalesceKey(payload({ type: "agent-regate-pr", repoFullName: "JSONbored/Gittensory" }))).toBeNull(); expect(jobCoalesceKey(payload({ type: "agent-regate-sweep", requestedBy: "schedule" }))).toBe("agent-regate-sweep:all"); expect(jobCoalesceKey(payload({ type: "agent-regate-sweep", repoFullName: "JSONbored/Gittensory" }))).toBe("agent-regate-sweep:jsonbored/gittensory"); + expect(jobCoalesceKey(payload({ type: "recapture-preview", repoFullName: "JSONbored/Gittensory", prNumber: 7, attempt: 2 }))).toBe("recapture-preview:jsonbored/gittensory#7:2"); + expect(jobCoalesceKey(payload({ type: "recapture-preview", repoFullName: "JSONbored/Gittensory", prNumber: 7 }))).toBeNull(); expect( jobCoalesceKey( payload({ @@ -97,6 +118,49 @@ describe("self-host queue common helpers", () => { }), ), ).toBe("github-webhook:ci-completed:jsonbored/gittensory@abc1234#3,7,12"); + expect( + jobCoalesceKey( + payload({ + type: "github-webhook", + eventName: "check_run", + payload: { + action: "completed", + repository: { full_name: "JSONbored/Gittensory" }, + check_run: { + check_suite: { head_sha: "DEF5678" }, + pull_requests: [], + }, + }, + }), + ), + ).toBe("github-webhook:ci-completed:jsonbored/gittensory@def5678"); + expect( + jobCoalesceKey( + payload({ + type: "github-webhook", + eventName: "check_suite", + payload: { + action: "completed", + repository: { full_name: "JSONbored/Gittensory" }, + check_suite: { pull_requests: [{ number: 7 }] }, + }, + }), + ), + ).toBeNull(); + expect( + jobCoalesceKey( + payload({ + type: "github-webhook", + eventName: "pull_request", + payload: { + action: "synchronize", + repository: { full_name: "JSONbored/Gittensory" }, + number: 99, + pull_request: {}, + }, + }), + ), + ).toBe("github-webhook:pr-refresh:jsonbored/gittensory#99"); }); it("returns no coalesce key for malformed payloads", () => { @@ -133,6 +197,27 @@ describe("self-host queue common helpers", () => { 1_000_000, ), ).toBe(8_000); + expect( + githubRateLimitRetryDelayMs( + { + status: 403, + response: { + headers: { + "x-ratelimit-remaining": "0", + "x-ratelimit-reset": "990", + }, + }, + }, + 1_000_000, + ), + ).toBe(300_000); + expect( + githubRateLimitRetryDelayMs({ + status: 429, + response: { headers: new Headers() }, + message: "rate limit", + }), + ).toBe(300_000); }); it("keeps only GitHub rate limits on the non-consuming retry path", () => { @@ -164,5 +249,37 @@ describe("self-host queue common helpers", () => { 77, ), ).toBe(1234); + expect( + consumingRetryDelayMs( + new RetryableJobError("AI review pending", { + retryAfterMs: Number.NaN, + retryKind: "ai_review_public_summary_missing", + }), + 77, + ), + ).toBe(300_000); + expect( + consumingRetryDelayMs( + new RetryableJobError("AI review pending", { + retryKind: "ai_review_public_summary_missing", + }), + 77, + ), + ).toBe(300_000); + }); + + it("bounds startup jitter min-jobs config to a non-negative finite integer", () => { + const old = process.env.QUEUE_STARTUP_JITTER_MIN_JOBS; + try { + process.env.QUEUE_STARTUP_JITTER_MIN_JOBS = "2.9"; + expect(queueStartupJitterMinJobs()).toBe(2); + process.env.QUEUE_STARTUP_JITTER_MIN_JOBS = "-1"; + expect(queueStartupJitterMinJobs()).toBe(8); + process.env.QUEUE_STARTUP_JITTER_MIN_JOBS = "not-a-number"; + expect(queueStartupJitterMinJobs()).toBe(8); + } finally { + if (old === undefined) delete process.env.QUEUE_STARTUP_JITTER_MIN_JOBS; + else process.env.QUEUE_STARTUP_JITTER_MIN_JOBS = old; + } }); }); diff --git a/test/unit/selfhost-sqlite-queue.test.ts b/test/unit/selfhost-sqlite-queue.test.ts index 392a759b39..8d02ddce70 100644 --- a/test/unit/selfhost-sqlite-queue.test.ts +++ b/test/unit/selfhost-sqlite-queue.test.ts @@ -221,6 +221,33 @@ describe("createSqliteQueue (durable #980)", () => { } }); + it("does not spread a due backlog when startup jitter is disabled", async () => { + const oldMin = process.env.QUEUE_STARTUP_JITTER_MIN_JOBS; + const oldJitter = process.env.QUEUE_STARTUP_JITTER_MS; + process.env.QUEUE_STARTUP_JITTER_MIN_JOBS = "2"; + process.env.QUEUE_STARTUP_JITTER_MS = "0"; + try { + const driver = makeDriver(); + createSqliteQueue(driver, async () => undefined); + for (const deliveryId of ["ci-1", "ci-2"]) { + driver.query( + "INSERT INTO _selfhost_jobs (payload, status, attempts, run_after, created_at, priority, job_key) VALUES (?, 'pending', 0, 0, 0, 10, ?)", + [JSON.stringify(ciWebhook(deliveryId)), deliveryId], + ); + } + + createSqliteQueue(driver, async () => undefined); + + const rows = driver.query("SELECT run_after FROM _selfhost_jobs ORDER BY id", []).rows as Array<{ run_after: number }>; + expect(rows).toEqual([{ run_after: 0 }, { run_after: 0 }]); + } finally { + if (oldMin === undefined) delete process.env.QUEUE_STARTUP_JITTER_MIN_JOBS; + else process.env.QUEUE_STARTUP_JITTER_MIN_JOBS = oldMin; + if (oldJitter === undefined) delete process.env.QUEUE_STARTUP_JITTER_MS; + else process.env.QUEUE_STARTUP_JITTER_MS = oldJitter; + } + }); + it("migrates an old queue table without a priority column before creating the claim index", async () => { const driver = makeDriver(); driver.exec(` @@ -409,6 +436,44 @@ describe("createSqliteQueue (durable #980)", () => { expect(afterEnqueue.run_after).toBeGreaterThan(before + 100_000); }); + it("coalesces a rate-limited active job into an existing pending duplicate without consuming attempts", async () => { + const driver = makeDriver(); + let calls = 0; + const rateLimit = new Error("secondary rate limit"); + Object.assign(rateLimit, { status: 403 }); + const key = `github-webhook:ci-completed:jsonbored/gittensory@${"b".repeat(40)}#1629`; + const q = createSqliteQueue( + driver, + async () => { + calls += 1; + throw rateLimit; + }, + { maxRetries: 1, backoffMs: () => 0 }, + ); + driver.query( + "INSERT INTO _selfhost_jobs (payload, status, attempts, run_after, created_at, priority, job_key) VALUES (?, 'pending', 0, 0, 0, 10, ?)", + [JSON.stringify(ciWebhook("ci-active")), key], + ); + driver.query( + "INSERT INTO _selfhost_jobs (payload, status, attempts, run_after, created_at, priority, job_key) VALUES (?, 'pending', 0, ?, 0, 10, ?)", + [JSON.stringify(ciWebhook("ci-existing")), Date.now() + 60_000, key], + ); + + await q.drain(); + + const rows = driver.query("SELECT payload, attempts, last_error FROM _selfhost_jobs ORDER BY id", []).rows as Array<{ + payload: string; + attempts: number; + last_error: string | null; + }>; + expect(calls).toBe(1); + expect(rows).toHaveLength(1); + expect(JSON.parse(rows[0]!.payload).deliveryId).toBe("ci-existing"); + expect(rows[0]!.attempts).toBe(0); + expect(rows[0]!.last_error).toContain("secondary rate limit"); + expect(q.stats()).toMatchObject({ gittensory_jobs_coalesced_total: 1 }); + }); + it("consumes retryable incomplete review attempts and dead-letters after maxRetries", async () => { const driver = makeDriver(); let calls = 0; @@ -506,6 +571,27 @@ describe("createSqliteQueue (durable #980)", () => { expect(seen).toEqual(["persisted"]); }); + it("does not reclaim processing jobs when the processing timeout is disabled", async () => { + const old = process.env.QUEUE_PROCESSING_TIMEOUT_MS; + process.env.QUEUE_PROCESSING_TIMEOUT_MS = "0"; + try { + const driver = makeDriver(); + const q = createSqliteQueue(driver, async () => undefined); + driver.query( + "INSERT INTO _selfhost_jobs (payload, status, attempts, run_after, created_at, priority, job_key) VALUES (?, 'processing', 0, 0, 0, 10, ?)", + [JSON.stringify(msg("stuck")), "stuck-key"], + ); + + await q.drain(); + + expect(driver.query("SELECT status FROM _selfhost_jobs", []).rows[0]).toMatchObject({ status: "processing" }); + expect(q.stats().gittensory_jobs_recovered_total ?? 0).toBe(0); + } finally { + if (old === undefined) delete process.env.QUEUE_PROCESSING_TIMEOUT_MS; + else process.env.QUEUE_PROCESSING_TIMEOUT_MS = old; + } + }); + it("start() runs the poll loop and processes a job, stop() halts it", async () => { const driver = makeDriver(); const seen: string[] = []; diff --git a/test/unit/unified-comment-bridge.test.ts b/test/unit/unified-comment-bridge.test.ts index d419b856be..364bac024f 100644 --- a/test/unit/unified-comment-bridge.test.ts +++ b/test/unit/unified-comment-bridge.test.ts @@ -223,6 +223,19 @@ describe("buildUnifiedCommentBody", () => { expect(body).toContain("> [!TIP]"); // success → ready → TIP alert }); + it("passes a public review update timestamp into the unified comment", () => { + const body = buildUnifiedCommentBody({ + gate: gate(), + aiReview: { notes: "Clean change." }, + panelRows, + readinessTotal: 88, + changedFiles: 3, + footerMarkdown: footer, + reviewedAt: "2026-06-29T08:05:59.852Z", + }); + expect(body).toContain("Review updated: 2026-06-29 08:05:59 UTC"); + }); + it("does not claim an AI reviewer or synthesize a review from deterministic warnings alone", () => { const body = buildUnifiedCommentBody({ gate: gate({ @@ -264,9 +277,9 @@ describe("buildUnifiedCommentBody", () => { changedFiles: 5, footerMarkdown: footer, }); - // failure → close verdict → blocked status (CAUTION alert + "Blocked"/"Closed" verdict line). + // failure → close verdict → blocked status (CAUTION alert + reject/close suggested action). expect(failing).toContain("> [!CAUTION]"); - expect(failing).toMatch(/Closed|Blocked/); + expect(failing).toContain("Suggested Action - Reject/Close"); // The recovered consensus defect surfaces as a blocker. expect(failing).toContain("Real bug"); }); @@ -315,27 +328,27 @@ describe("buildUnifiedCommentBody", () => { it("heldForReview renders a passing PR as HELD, never 'safe to merge' (#guarded-hold-comment)", () => { const args = { gate: gate({ conclusion: "success" }), panelRows, readinessTotal: 90, changedFiles: 2, mergeReadiness: { ciState: "passed" as const }, footerMarkdown: footer }; - // Without the hold, a success+green PR is the green "safe to merge" headline. + // Without the hold, a success+green PR is the green approve/merge recommendation. const ready = buildUnifiedCommentBody(args); expect(ready).toContain("> [!TIP]"); - expect(ready).toContain("safe to merge"); + expect(ready).toContain("Suggested Action - Approve/Merge"); // With the guarded hold, the SAME PR renders held (WARNING), not safe-to-merge — matching the disposition. const held = buildUnifiedCommentBody({ ...args, heldForReview: true }); expect(held).toContain("> [!WARNING]"); - expect(held).toContain("Held for maintainer review"); + expect(held).toContain("Suggested Action - Manual Review"); expect(held).not.toContain("> [!TIP]"); }); - it("neverClosed renders a gate-failure (close) PR as HELD, not 'Closed' (#8/#9)", () => { + it("neverClosed renders a gate-failure (close) PR as HELD, not reject/close (#8/#9)", () => { const args = { gate: gate({ conclusion: "failure" }), panelRows, readinessTotal: 40, changedFiles: 2, mergeReadiness: { ciState: "passed" as const }, footerMarkdown: footer }; - // A contributor close → the red "Closed" headline. + // A contributor close → the red reject/close recommendation. const closed = buildUnifiedCommentBody(args); - expect(closed).toContain("Closed"); - // The SAME verdict on an owner / automation-bot PR (never auto-closed) renders held, not Closed. + expect(closed).toContain("Suggested Action - Reject/Close"); + // The SAME verdict on an owner / automation-bot PR (never auto-closed) renders held, not reject/close. const held = buildUnifiedCommentBody({ ...args, neverClosed: true }); expect(held).toContain("> [!WARNING]"); - expect(held).toContain("Held for maintainer review"); - expect(held).not.toContain("Closed"); + expect(held).toContain("Suggested Action - Manual Review"); + expect(held).not.toContain("Suggested Action - Reject/Close"); }); }); @@ -351,11 +364,11 @@ describe("buildUnifiedCommentBody", () => { describe("reconciliation invariant: comment tone is pinned to the gate conclusion (#1016)", () => { // gate conclusion → the alert + the verbatim headline phrase the renderer must emit for that conclusion. const cases: Array<{ conclusion: GateCheckEvaluation["conclusion"]; alert: string; headline: RegExp }> = [ - { conclusion: "success", alert: "> [!TIP]", headline: /Approved/ }, // success → merge → ready - { conclusion: "failure", alert: "> [!CAUTION]", headline: /Closed|Blocked/ }, // failure → close → blocked - { conclusion: "action_required", alert: "> [!WARNING]", headline: /Held for maintainer review/ }, // → manual → held - { conclusion: "neutral", alert: "> [!WARNING]", headline: /Held for maintainer review/ }, // → manual → held - { conclusion: "skipped", alert: "> [!NOTE]", headline: /Advisory only/ }, // → comment → advisory + { conclusion: "success", alert: "> [!TIP]", headline: /Suggested Action - Approve\/Merge/ }, // success → merge → ready + { conclusion: "failure", alert: "> [!CAUTION]", headline: /Suggested Action - Reject\/Close/ }, // failure → close → blocked + { conclusion: "action_required", alert: "> [!WARNING]", headline: /Suggested Action - Manual Review/ }, // → manual → held + { conclusion: "neutral", alert: "> [!WARNING]", headline: /Suggested Action - Manual Review/ }, // → manual → held + { conclusion: "skipped", alert: "> [!NOTE]", headline: /Suggested Action - Advisory Only/ }, // → comment → advisory ]; for (const { conclusion, alert, headline } of cases) { @@ -546,7 +559,7 @@ describe("verdictReason on a held/blocked headline (FIX D2)", () => { changedFiles: 2, footerMarkdown: footer, }); - expect(body).toMatch(/Closed|Blocked/); + expect(body).toContain("Suggested Action - Reject/Close"); expect(body).toContain("A hard blocker was found."); // the gate's authoritative reason on the headline }); @@ -558,7 +571,7 @@ describe("verdictReason on a held/blocked headline (FIX D2)", () => { changedFiles: 2, footerMarkdown: footer, }); - expect(body).toContain("Held for maintainer review"); + expect(body).toContain("Suggested Action - Manual Review"); expect(body).toContain("Manual maintainer review required."); }); @@ -581,7 +594,7 @@ describe("verdictReason on a held/blocked headline (FIX D2)", () => { changedFiles: 2, footerMarkdown: footer, }); - expect(body).toContain("Approved"); // ready headline kept its positive wording… + expect(body).toContain("Suggested Action - Approve/Merge"); // ready headline kept its positive wording… expect(body).not.toContain("No configured hard blocker was found."); // …the gate summary did NOT replace it }); }); diff --git a/test/unit/unified-comment.test.ts b/test/unit/unified-comment.test.ts index a34f41df31..dcd83da921 100644 --- a/test/unit/unified-comment.test.ts +++ b/test/unit/unified-comment.test.ts @@ -123,8 +123,8 @@ describe("renderUnifiedReviewComment", () => { ); expect(md).toContain("> [!TIP]"); expect(md).toContain("🟩"); - expect(md).toContain("Gittensory review — safe to merge · auto-merged"); - expect(md).toContain("Approved & auto-merged"); + expect(md).toContain("Gittensory review result - approve/merge recommended · auto-merged"); + expect(md).toContain("Suggested Action - Approve/Merge"); expect(md).toContain("`2 files`"); expect(md).toContain("`2 AI reviewers`"); expect(md).toContain("`no blockers`"); @@ -134,6 +134,7 @@ describe("renderUnifiedReviewComment", () => { expect(md).toContain("| **Code review** | ✅ No blockers | 2 reviewers, synthesized |"); expect(md).toContain("| Linked issue | ✅ Linked | #1372 |"); expect(md).toContain("
Nits — 1 non-blocking"); + expect(md).toContain("- [ ] Document the new property."); expect(md.indexOf("**Review summary**")).toBeLessThan(md.indexOf("
Nits")); expect(md.indexOf("
Nits")).toBeLessThan(md.indexOf("| Signal | Result | Evidence |")); expect(md).toContain("
Signal definitions"); @@ -168,7 +169,7 @@ describe("renderUnifiedReviewComment", () => { ); expect(md).toContain("> [!CAUTION]"); expect(md).toContain("🟥"); - expect(md).toContain("Closed"); + expect(md).toContain("Suggested Action - Reject/Close"); expect(md).toContain("Why this is blocked"); expect(md).toContain("Introduces a hardcoded secret."); expect(md).toContain("| **Code review** | ❌ 1 blocker |"); @@ -178,14 +179,14 @@ describe("renderUnifiedReviewComment", () => { const md = renderUnifiedReviewComment({ ...base, decision: "manual", recommendations: ["manual_review"] }, ctx); expect(md).toContain("> [!WARNING]"); expect(md).toContain("🟨"); - expect(md).toContain("Held for maintainer review"); + expect(md).toContain("Suggested Action - Manual Review"); }); it("advisory state uses the note alert and blue bar", () => { const md = renderUnifiedReviewComment({ ...base, decision: "comment", recommendations: [] }, {}); expect(md).toContain("> [!NOTE]"); expect(md).toContain("🟦"); - expect(md).toContain("Advisory only"); + expect(md).toContain("Suggested Action - Advisory Only"); }); it("dedupes repeated blockers and nits", () => { @@ -200,9 +201,36 @@ describe("renderUnifiedReviewComment", () => { const md = renderUnifiedReviewComment({ ...base, decision: "merge" }, {}); expect(md).not.toContain("readiness"); expect(md).not.toContain("- [ ]"); + expect(md).not.toContain("Review updated:"); expect(md.split("\n").some((l) => l.trim() === "> ---")).toBe(false); }); + it("renders a UTC freshness marker when the host supplies the review update time", () => { + const md = renderUnifiedReviewComment( + { ...base, decision: "merge" }, + { reviewedAt: "2026-06-29T08:05:59.852Z" }, + ); + expect(md).toContain("Review updated: 2026-06-29 08:05:59 UTC"); + expect(renderUnifiedReviewComment({ ...base, decision: "merge" }, { reviewedAt: "not-a-date" })).not.toContain("Review updated:"); + }); + + it("drops blank failing-check details and falls back to bare check names", () => { + const md = renderUnifiedReviewComment( + { + ...base, + readiness: { + ciState: "failed", + failingChecks: ["fallback-check"], + failingDetails: [{ name: " " }, { name: "lint" }], + }, + }, + {}, + ); + expect(md).toContain("CI checks failing"); + expect(md).toContain("- lint"); + expect(md).not.toContain("fallback-check"); + }); + it("only emits provided content (no internal fields leak in)", () => { const md = renderUnifiedReviewComment({ ...base, decision: "merge" }, ctx); expect(md).not.toMatch(/confidenceFloor|scopeCap|hardGuardrailGlobs|rubric/i); @@ -211,9 +239,9 @@ describe("renderUnifiedReviewComment", () => { it("a blocked status from reviewer recs (no close decision) reads 'blocked', not 'closed'", () => { const md = renderUnifiedReviewComment({ ...base, recommendations: ["close"], blockers: ["Leaks a token."], consensusBlocker: true }, {}); expect(md).toContain("> [!CAUTION]"); - expect(md).toContain("Gittensory review — blocked"); // verb(): decision !== "close" - expect(md).toContain("**🛑 Blocked**"); // verdictLine(): decision !== "close" - expect(md).not.toContain("Closed"); + expect(md).toContain("Gittensory review result - blockers found"); // headlineLabel(): decision !== "close" + expect(md).toContain("**🛑 Suggested Action - Fix Blockers**"); // verdictLine(): decision !== "close" + expect(md).not.toContain("Suggested Action - Reject/Close"); }); it("renders CI-failing / CI-pending chips and the merge-state label", () => { @@ -269,13 +297,13 @@ describe("renderUnifiedReviewComment", () => { it("appends an explicit verdict reason across ready (merged + unmerged) and advisory states", () => { // The verdict word is bolded (`**…**`); the reason follows outside the bold, so assert each separately. const merged = renderUnifiedReviewComment({ ...base, decision: "merge", merged: true, verdictReason: "all checks green" }, {}); - expect(merged).toContain("Approved & auto-merged"); + expect(merged).toContain("Suggested Action - Approve/Merge"); expect(merged).toContain("all checks green"); // verdictReason appended, not the default " — all checks passed" const unmerged = renderUnifiedReviewComment({ ...base, decision: "merge", verdictReason: "looks correct" }, {}); expect(unmerged).not.toContain("auto-merged"); // the unmerged ready variant expect(unmerged).toContain("looks correct"); const advisory = renderUnifiedReviewComment({ ...base, decision: "comment", recommendations: [], verdictReason: "for your awareness" }, {}); - expect(advisory).toContain("Advisory only"); + expect(advisory).toContain("Suggested Action - Advisory Only"); expect(advisory).toContain("for your awareness"); }); @@ -324,7 +352,7 @@ describe("renderUnifiedReviewComment", () => { expect(md).toContain("Safe summary </details><!-- hidden -->"); expect(md).toContain("- Blocker <script>alert(1)</script>"); - expect(md).toContain("- Nit closes </details>"); + expect(md).toContain("- [ ] Nit closes </details>"); expect(md).toContain("needs <maintainer> review"); expect(md).toContain("| Gate <row> | ❌ Bad <tag> | Evidence </td> |"); expect(md).toContain("
Extra <title>"); From 9b2d4ccf87b54243db84581109eea21717aabd5c Mon Sep 17 00:00:00 2001 From: JSONbored <49853598+JSONbored@users.noreply.github.com> Date: Mon, 29 Jun 2026 03:49:57 -0700 Subject: [PATCH 50/68] build(selfhost): add prebuilt deploy helper --- scripts/deploy-selfhost-prebuilt.sh | 211 ++++++++++++++++++++++++++++ 1 file changed, 211 insertions(+) create mode 100755 scripts/deploy-selfhost-prebuilt.sh diff --git a/scripts/deploy-selfhost-prebuilt.sh b/scripts/deploy-selfhost-prebuilt.sh new file mode 100755 index 0000000000..11713a7886 --- /dev/null +++ b/scripts/deploy-selfhost-prebuilt.sh @@ -0,0 +1,211 @@ +#!/usr/bin/env bash +# Build and deploy the self-host runtime from a prebuilt bundle without relying on host Node/npm. +# +# Defaults are intentionally operator-friendly: +# ./scripts/deploy-selfhost-prebuilt.sh +# +# Optional knobs: +# SENTRY_RELEASE=gittensory-selfhost@edge-abc123 ./scripts/deploy-selfhost-prebuilt.sh +# SELFHOST_COMPOSE_FILES="docker-compose.yml docker-compose.override.yml" ./scripts/deploy-selfhost-prebuilt.sh +# SELFHOST_SKIP_SENTRY_UPLOAD=1 ./scripts/deploy-selfhost-prebuilt.sh +set -euo pipefail + +ENV_FILE="${SELFHOST_ENV_FILE:-.env}" +NODE_IMAGE="${SELFHOST_NODE_IMAGE:-public.ecr.aws/docker/library/node:24-slim}" +SERVICE="${SELFHOST_SERVICE:-gittensory}" +SKIP_SENTRY_UPLOAD="${SELFHOST_SKIP_SENTRY_UPLOAD:-0}" + +require_cmd() { + if ! command -v "$1" >/dev/null 2>&1; then + echo "error: required command not found: $1" >&2 + exit 1 + fi +} + +env_get() { + local key="$1" + local file="${2:-$ENV_FILE}" + + [ -f "$file" ] || return 1 + + awk -v key="$key" ' + /^[[:space:]]*(#|$)/ { next } + { + line = $0 + sub(/^[[:space:]]*/, "", line) + if (line !~ "^" key "[[:space:]]*=") { + next + } + sub(/^[^=]*=/, "", line) + sub(/^[[:space:]]*/, "", line) + sub(/[[:space:]]*$/, "", line) + if (length(line) >= 2) { + first = substr(line, 1, 1) + last = substr(line, length(line), 1) + if ((first == "\"" && last == "\"") || (first == "'\''" && last == "'\''")) { + line = substr(line, 2, length(line) - 2) + } + } + print line + found = 1 + exit + } + END { exit found ? 0 : 1 } + ' "$file" +} + +env_put() { + local key="$1" + local value="$2" + local file="${3:-$ENV_FILE}" + local tmp + + touch "$file" + tmp="$(mktemp)" + awk -v key="$key" -v value="$value" ' + BEGIN { written = 0 } + { + line = $0 + sub(/^[[:space:]]*/, "", line) + if (line ~ "^" key "[[:space:]]*=") { + print key "=" value + written = 1 + } else { + print $0 + } + } + END { + if (!written) { + print key "=" value + } + } + ' "$file" >"$tmp" + cat "$tmp" >"$file" + rm -f "$tmp" +} + +compose_file_args() { + local files=() + local file + + if [ -n "${SELFHOST_COMPOSE_FILES:-}" ]; then + # shellcheck disable=SC2206 + files=(${SELFHOST_COMPOSE_FILES}) + else + files=(docker-compose.yml) + [ -f docker-compose.override.yml ] && files+=(docker-compose.override.yml) + fi + + for file in "${files[@]}"; do + if [ ! -f "$file" ]; then + echo "error: compose file not found: $file" >&2 + exit 1 + fi + printf '%s\n' -f "$file" + done +} + +run_node_build() { + local uid gid + uid="$(id -u)" + gid="$(id -g)" + + echo "selfhost deploy: building bundle with Dockerized Node" + docker run --rm \ + --user "$uid:$gid" \ + -e HOME=/tmp \ + -e npm_config_cache=/tmp/.npm \ + -v "$PWD:/work" \ + -w /work \ + "$NODE_IMAGE" \ + sh -lc 'npm ci --ignore-scripts && node scripts/build-selfhost.mjs --all && node scripts/validate-selfhost-sourcemap.mjs' +} + +run_sentry_upload() { + local auth_token org project uid gid + + auth_token="${SENTRY_AUTH_TOKEN:-$(env_get SENTRY_AUTH_TOKEN || true)}" + org="${SENTRY_ORG:-$(env_get SENTRY_ORG || true)}" + project="${SENTRY_PROJECT:-$(env_get SENTRY_PROJECT || true)}" + + if [ "$SKIP_SENTRY_UPLOAD" = "1" ]; then + echo "selfhost deploy: skipping Sentry upload (SELFHOST_SKIP_SENTRY_UPLOAD=1)" + return 0 + fi + + if [ -z "$auth_token" ] || [ -z "$org" ] || [ -z "$project" ]; then + echo "selfhost deploy: skipping Sentry upload (SENTRY_AUTH_TOKEN, SENTRY_ORG, or SENTRY_PROJECT is missing)" + return 0 + fi + + uid="$(id -u)" + gid="$(id -g)" + + echo "selfhost deploy: injecting and uploading Sentry source maps for $SENTRY_RELEASE" + docker run --rm \ + -e HOME=/tmp \ + -e npm_config_cache=/tmp/.npm \ + -e SENTRY_LOAD_DOTENV=0 \ + -e SENTRY_RELEASE \ + -e SENTRY_AUTH_TOKEN="$auth_token" \ + -e SENTRY_ORG="$org" \ + -e SENTRY_PROJECT="$project" \ + -e HOST_UID="$uid" \ + -e HOST_GID="$gid" \ + -v "$PWD:/work" \ + -w /work \ + "$NODE_IMAGE" \ + sh -lc 'apt-get update >/dev/null && apt-get install -y --no-install-recommends ca-certificates git >/dev/null && git config --global --add safe.directory /work && (npx -y @sentry/cli@latest releases new "$SENTRY_RELEASE" >/tmp/gittensory-sentry-release-new.log 2>&1 || true) && npx -y @sentry/cli@latest releases set-commits "$SENTRY_RELEASE" --auto && npx -y @sentry/cli@latest sourcemaps inject dist && node scripts/validate-selfhost-sourcemap.mjs && npx -y @sentry/cli@latest sourcemaps upload --release="$SENTRY_RELEASE" dist && npx -y @sentry/cli@latest releases finalize "$SENTRY_RELEASE" && chown -R "$HOST_UID:$HOST_GID" dist node_modules package-lock.json' +} + +run_compose_deploy() { + local override_file + local -a compose_args + + override_file="$(mktemp)" + trap 'rm -f "$override_file"' EXIT + + cat >"$override_file" </dev/null + +if ! git rev-parse --is-inside-work-tree >/dev/null 2>&1; then + echo "error: run this script from the gittensory git checkout" >&2 + exit 1 +fi + +SENTRY_RELEASE="${SENTRY_RELEASE:-$(env_get SENTRY_RELEASE || true)}" +SENTRY_RELEASE="${SENTRY_RELEASE:-gittensory-selfhost@$(git rev-parse --short=8 HEAD)}" +export SENTRY_RELEASE + +env_put SENTRY_RELEASE "$SENTRY_RELEASE" +env_put GITTENSORY_VERSION "$SENTRY_RELEASE" + +run_node_build +run_sentry_upload +run_compose_deploy + +echo "selfhost deploy: complete ($SENTRY_RELEASE)" From bb7f5eba9c953b447e4ad21a6b0e05abaef5e6dd Mon Sep 17 00:00:00 2001 From: JSONbored <49853598+JSONbored@users.noreply.github.com> Date: Mon, 29 Jun 2026 03:55:40 -0700 Subject: [PATCH 51/68] fix(selfhost): keep deploy helper cleanup shell-safe --- scripts/deploy-selfhost-prebuilt.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scripts/deploy-selfhost-prebuilt.sh b/scripts/deploy-selfhost-prebuilt.sh index 11713a7886..60bd803327 100755 --- a/scripts/deploy-selfhost-prebuilt.sh +++ b/scripts/deploy-selfhost-prebuilt.sh @@ -163,7 +163,8 @@ run_compose_deploy() { local -a compose_args override_file="$(mktemp)" - trap 'rm -f "$override_file"' EXIT + SELFHOST_GENERATED_COMPOSE_FILE="$override_file" + trap 'rm -f "${SELFHOST_GENERATED_COMPOSE_FILE:-}"' EXIT cat >"$override_file" < Date: Mon, 29 Jun 2026 04:30:09 -0700 Subject: [PATCH 52/68] fix(review): align review-agent manual outcomes --- .../contributing-to-gittensory/SKILL.md | 4 +- .../contributing-to-gittensory/reference.md | 2 +- CONTRIBUTING.md | 2 +- .../site/app-panels/maintainer-settings.tsx | 2 +- .../src/routes/docs.beta-onboarding.tsx | 3 +- .../src/routes/docs.github-app.tsx | 61 +++++++++--------- .../src/routes/docs.how-reviews-work.tsx | 18 +++--- .../routes/docs.maintainer-install-trust.tsx | 18 +++--- .../src/routes/docs.maintainer-workflow.tsx | 2 +- apps/gittensory-ui/src/routes/docs.tuning.tsx | 7 +- docs/review-configuration.md | 6 +- src/github/app.ts | 31 +++++---- src/github/backfill.ts | 17 +++-- src/queue/processors.ts | 11 ++-- src/review/check-names.ts | 3 + src/review/content-lane-wire.ts | 6 +- src/review/unified-comment-bridge.ts | 5 +- src/review/unified-comment.ts | 64 +++++++++++++------ src/rules/advisory.ts | 29 +++++---- src/signals/engine.ts | 3 +- src/signals/focus-manifest.ts | 4 +- src/signals/settings-preview.ts | 5 +- src/types.ts | 4 +- test/unit/backfill.test.ts | 19 +++--- test/unit/content-lane-wire.test.ts | 6 +- test/unit/docs-github-app.test.ts | 4 +- test/unit/gate-check-policy.test.ts | 22 +++---- test/unit/github-app.test.ts | 26 ++++---- test/unit/predicted-gate.test.ts | 2 +- test/unit/queue.test.ts | 30 ++++----- test/unit/rules.test.ts | 16 ++--- test/unit/settings-preview.test.ts | 4 +- test/unit/signals-coverage.test.ts | 6 +- test/unit/unified-comment-bridge.test.ts | 40 ++++++++---- test/unit/unified-comment-parity.test.ts | 2 +- test/unit/unified-comment.test.ts | 59 +++++++++++++---- test/unit/visual-collapsible.test.ts | 2 +- 37 files changed, 323 insertions(+), 222 deletions(-) create mode 100644 src/review/check-names.ts diff --git a/.claude/skills/contributing-to-gittensory/SKILL.md b/.claude/skills/contributing-to-gittensory/SKILL.md index d0eb44f82f..04d78df3b7 100644 --- a/.claude/skills/contributing-to-gittensory/SKILL.md +++ b/.claude/skills/contributing-to-gittensory/SKILL.md @@ -33,7 +33,7 @@ it takes a one-shot disposition: | Situation | Engine action | |---|---| -| Gate passes **and** every CI check green **and** mergeable-clean (+ approvals) | **auto-approve → MERGE** | +| Review-agent check passes **and** every CI check green **and** mergeable-clean (+ approvals) | **auto-approve → MERGE** | | **Any** CI check failed — required or not, **`codecov/patch` included** | **CLOSE** (one-shot) | | Gate **failure**, or base **conflict** (needs rebase), or a linked-issue **hard-rule** violation | **CLOSE** (one-shot) | | CI still **pending** | no action — waits for checks to finish | @@ -224,7 +224,7 @@ If `ui:lint` fails on formatting, run `npm --workspace @jsonbored/gittensory-ui **Sync with `main` before you push if it moved** — a base conflict auto-closes a contributor PR: `git fetch upstream && git rebase upstream/main`, resolve, re-run the gate, then push. On the PR, the required status check is **`validate`** (it aggregates the CI jobs) and the engine posts a check run -named **`Gittensory Gate`** — watch both go green/passing. +named **`Gittensory Orb Review Agent`** — watch both go green/passing. --- diff --git a/.claude/skills/contributing-to-gittensory/reference.md b/.claude/skills/contributing-to-gittensory/reference.md index 6666fb3421..324e250408 100644 --- a/.claude/skills/contributing-to-gittensory/reference.md +++ b/.claude/skills/contributing-to-gittensory/reference.md @@ -17,7 +17,7 @@ for maintainer approval (CI shows unverified → the engine **holds**, never clo The single **required** status check is **`validate`** (it aggregates `changes, lint, test, workers, mcp, ui, security`; a path-skipped job counts as success). **Codecov** posts `codecov/patch` (the real coverage gate) and `codecov/project` (informational) independently. The review engine also posts its -own check run named **`Gittensory Gate`** (`src/github/app.ts` `GITTENSORY_GATE_CHECK_NAME`) — the gate +own check run named **`Gittensory Orb Review Agent`** (`src/github/app.ts` `GITTENSORY_GATE_CHECK_NAME`) — the gate verdict (§3), separate from CI. On a PR, jobs run only if their path filter matched; on push to `main`, everything runs. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 5f5461722e..f3a8420807 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -261,7 +261,7 @@ Public GitHub surfaces: - Keep public comments advisory, sanitized, and low-noise. - Keep labels limited to configured labels for officially confirmed Gittensor miner PRs. - Never publish private reviewability, scoring, wallet, hotkey, or reward/risk context. -- The Gittensory Gate blocks **only confirmed Gittensor contributors**; every other author (and any +- The Gittensory Orb Review Agent blocks **only confirmed Gittensor contributors**; every other author (and any app/infra state) resolves to a neutral, non-blocking gate. Adding a blocker must keep it confirmed-contributor-gated through `evaluateGateCheck`. diff --git a/apps/gittensory-ui/src/components/site/app-panels/maintainer-settings.tsx b/apps/gittensory-ui/src/components/site/app-panels/maintainer-settings.tsx index 92d6d93c5f..9d108ed1e2 100644 --- a/apps/gittensory-ui/src/components/site/app-panels/maintainer-settings.tsx +++ b/apps/gittensory-ui/src/components/site/app-panels/maintainer-settings.tsx @@ -137,7 +137,7 @@ type FieldDef = SelectFieldDef | ToggleFieldDef | NumberFieldDef; const GATE_FIELDS: FieldDef[] = [ { key: "gateCheckMode", - label: "Gate check", + label: "Review agent check", kind: "select", options: [ ["off", "off"], diff --git a/apps/gittensory-ui/src/routes/docs.beta-onboarding.tsx b/apps/gittensory-ui/src/routes/docs.beta-onboarding.tsx index 48ad0062de..29ca62d67c 100644 --- a/apps/gittensory-ui/src/routes/docs.beta-onboarding.tsx +++ b/apps/gittensory-ui/src/routes/docs.beta-onboarding.tsx @@ -124,7 +124,8 @@ gittensory-mcp preflight --login your-login --json`}
  • Preview the public surface. Dry-run what would be written to GitHub without mutating state. Keep Gittensory Context advisory; require{" "} - Gittensory Gate only after blocking rules are explicitly configured. + Gittensory Orb Review Agent only after blocking rules are explicitly + configured.

    Once installed, the gittensory app reviews every pull request on the repos - you select. Each review produces two surfaces: the Gittensory Gate check - run (and the advisory Gittensory Context check), and a single review - comment posted by gittensory[bot] that updates in place as the PR evolves. - Everything on this page configures that review. + you select. Each review produces two surfaces: the{" "} + Gittensory Orb Review Agent check run (and the advisory{" "} + Gittensory Context check), and a single review comment posted by{" "} + gittensory[bot] that updates in place as the PR evolves. Everything on this + page configures that review.

    Install

    @@ -57,8 +58,8 @@ function GithubApp() {
  • Approve Metadata: read, Pull requests: read, and{" "} - Issues: write. Enable Checks: write when Context or Gate check - runs are enabled. + Issues: write. Enable Checks: write when Context or review-agent + check runs are enabled.
  • Keep webhook events enabled for issues, issue_comment,{" "} @@ -92,16 +93,16 @@ GET /v1/installations/:id/repair`}
  • Leave Gittensory Context advisory while you tune copy and settings. Make{" "} - Gittensory Gate required only after the repo explicitly enables blocking - rules. + Gittensory Orb Review Agent required only after the repo explicitly + enables blocking rules.

    • Default posture

    - Gittensory is advisory-first. Public comments, labels, the Context check, and the Gate check - are controlled per repo. Missing issue links, non-Gittensor contributors, busy queues, and - weak overlap signals do not block merge by default. + Gittensory is advisory-first. Public comments, labels, the Context check, and the + review-agent check are controlled per repo. Missing issue links, non-Gittensor contributors, + busy queues, and weak overlap signals do not block merge by default.

    PR panel

    @@ -123,22 +124,23 @@ GET /v1/installations/:id/repair`}

    Checks

    - The gittensory app publishes its review as check runs. Gittensory Gate is - the gate result; Gittensory Context is the advisory companion. Both are - controlled per repo by checkRunMode (off / enabled), - with checkRunDetailLevel choosing minimal, standard, - or deep output. + The gittensory app publishes its review as check runs.{" "} + Gittensory Orb Review Agent is the gate result;{" "} + Gittensory Context is the advisory companion. Both are controlled per repo + by checkRunMode (off / enabled), with{" "} + checkRunDetailLevel choosing minimal, standard, or{" "} + deep output.

    Gittensory Context is advisory and should not be required in branch - protection. Gittensory Gate is opt-in and can be made required after a repo - owner chooses blocking rules. + protection. Gittensory Orb Review Agent is opt-in and can be made required + after a repo owner chooses blocking rules.

    - Branch protection should require Gittensory Gate only after the repo has - verified installation health, previewed the public panel, and configured at least one{" "} - block rule. Do not require Gittensory Context; it is there to - inform reviewers, not stop merges. + Branch protection should require Gittensory Orb Review Agent only after the + repo has verified installation health, previewed the public panel, and configured at least + one block rule. Do not require Gittensory Context; it is there + to inform reviewers, not stop merges.

    Gate modes

    @@ -147,8 +149,9 @@ GET /v1/installations/:id/repair`} gateCheckMode (off / enabled); each dimension then refines an already-enabled gate with a tri-state mode — off (not evaluated),{" "} advisory (surfaced, never blocks), or block (can become a hard{" "} - Gittensory Gate blocker). Blocking is always confirmed-contributor-gated: - the mode chooses which deterministic checks are active, never who can be blocked. + Gittensory Orb Review Agent blocker). Blocking is always + confirmed-contributor-gated: the mode chooses which deterministic checks are active, never{" "} + who can be blocked.

    • @@ -205,14 +208,14 @@ GET /v1/installations/:id/repair`} lang="yaml" code={`# Repository settings as code — any dashboard toggle: settings: - gateCheckMode: enabled # the Gate on/off + gateCheckMode: enabled # review-agent check on/off checkRunMode: enabled # the advisory Context check on/off commentMode: detected_contributors_only publicSurface: comment_only # Friendly gate alias (wins over settings: for gate fields): gate: - enabled: true # Gate on/off + enabled: true # review-agent check on/off linkedIssue: advisory # block | advisory | off duplicates: block readiness: { mode: advisory, minScore: 60 } @@ -319,7 +322,7 @@ GITTENSORY_REVIEW_REPOS="JSONbored/gittensory"`} For repos like JSONbored/gittensory and awesome-claude, enable PR comments, labels, Context, and Gate together to test the full product surface. If another maintainer agent can merge quickly, configure that agent to wait for{" "} - Gittensory Gate before merge or close. + Gittensory Orb Review Agent before merge or close.

      Install diagnostics

      diff --git a/apps/gittensory-ui/src/routes/docs.how-reviews-work.tsx b/apps/gittensory-ui/src/routes/docs.how-reviews-work.tsx index ef5f82bb57..9daae77553 100644 --- a/apps/gittensory-ui/src/routes/docs.how-reviews-work.tsx +++ b/apps/gittensory-ui/src/routes/docs.how-reviews-work.tsx @@ -50,9 +50,10 @@ function HowReviewsWork() {

      Everything those layers find is folded into a single gittensory review{" "} - comment on the PR, plus an optional Gittensory Gate check run. The review - algorithm is open-source; what changes between repos is the configuration you tune. See{" "} - Review configuration for every option and default. + comment on the PR, plus an optional Gittensory Orb Review Agent check run. + The review algorithm is open-source; what changes between repos is the configuration you + tune. See Review configuration for every option and + default.

      Defaults are quiet. With no settings and no .gittensory.yml, the gate is{" "} @@ -75,8 +76,8 @@ function HowReviewsWork() { never blocks the merge.
    • - block — the finding can become a hard Gittensory Gate{" "} - blocker. + block — the finding can become a hard{" "} + Gittensory Orb Review Agent blocker.

    @@ -251,9 +252,10 @@ function HowReviewsWork() {

    The check run can carry the same signals at adjustable depth: checkRunMode ( - off / enabled) publishes the Gittensory Gate{" "} - check, and checkRunDetailLevel (minimal / standard /{" "} - deep) sets how much the check summary spells out. + off / enabled) publishes the{" "} + Gittensory Orb Review Agent check, and checkRunDetailLevel ( + minimal / standard / deep) sets how much the check + summary spells out.

    Putting it together

    diff --git a/apps/gittensory-ui/src/routes/docs.maintainer-install-trust.tsx b/apps/gittensory-ui/src/routes/docs.maintainer-install-trust.tsx index e52220ca36..d266c89885 100644 --- a/apps/gittensory-ui/src/routes/docs.maintainer-install-trust.tsx +++ b/apps/gittensory-ui/src/routes/docs.maintainer-install-trust.tsx @@ -50,16 +50,16 @@ function MaintainerInstallTrust() {
  • Install Gittensory on one test repository or a selected repository set.
  • Approve Metadata: read, Pull requests: read, and{" "} - Issues: write. Add Checks: write only when Context or Gate check - runs are enabled for the repository. + Issues: write. Add Checks: write only when Context or + review-agent check runs are enabled for the repository.
  • Keep webhook events enabled for issues, issue_comment,{" "} pull_request, and repository.
  • - Leave comments, labels, Context checks, and Gate checks in advisory mode until preview - output matches the repo's maintainer policy. + Leave comments, labels, Context checks, and review-agent checks in advisory mode until + preview output matches the repo's maintainer policy.
    • confirm private signals stay private -> enable advisory Context, labels, or comments -> capture screenshots/recordings for UI or extension changes - -> decide whether Gate should be required in branch protection`} + -> decide whether the review-agent check should be required in branch protection`} />

    Maintainer controls

    @@ -107,9 +107,9 @@ POST /v1/repos/:owner/:repo/settings-preview`} "Gittensory Context is advisory and should not be required by branch protection.", }, { - title: "Gate check", + title: "Review agent check", description: - "Gittensory Gate is opt-in. Make it required only after the repo owner chooses blocking rules and validates previews.", + "Gittensory Orb Review Agent is opt-in. Make it required only after the repo owner chooses blocking rules and validates previews.", }, { title: "Command access", @@ -203,8 +203,8 @@ API unavailable or stale data contribution fits the repo, issue, and subnet goals.

    - If the repo enables Gittensory Gate, document which blockers are enforced - and why. Otherwise, treat Gittensory output as reviewer context only. + If the repo enables Gittensory Orb Review Agent, document which blockers + are enforced and why. Otherwise, treat Gittensory output as reviewer context only.

    Reject weak Gittensory-driven PRs

    diff --git a/apps/gittensory-ui/src/routes/docs.maintainer-workflow.tsx b/apps/gittensory-ui/src/routes/docs.maintainer-workflow.tsx index c93af46ff2..06fd1b8389 100644 --- a/apps/gittensory-ui/src/routes/docs.maintainer-workflow.tsx +++ b/apps/gittensory-ui/src/routes/docs.maintainer-workflow.tsx @@ -133,7 +133,7 @@ GET /v1/repos/:owner/:repo/registration-readiness`}

    New installations should start with GitHub App setup: install on one repo, verify installation health, preview the public panel, then decide - whether Gittensory Gate should become a required check. + whether Gittensory Orb Review Agent should become a required check.

  • - block — the finding can become a hard Gittensory Gate blocker. - Blocking is always confirmed-contributor-gated: the mode chooses which{" "} - deterministic checks are active, never who can be blocked. + block — the finding can become a hard{" "} + Gittensory Orb Review Agent blocker. Blocking is always + confirmed-contributor-gated: the mode chooses which deterministic checks are + active, never who can be blocked.

    • diff --git a/docs/review-configuration.md b/docs/review-configuration.md index 5ec7711a29..191504184c 100644 --- a/docs/review-configuration.md +++ b/docs/review-configuration.md @@ -86,7 +86,7 @@ Most gate dimensions are tri-state **gate-rule modes**: `off` / `advisory` / `bl - `off` — the dimension is not evaluated. - `advisory` — the finding is **surfaced** (in the comment/context) but never blocks. -- `block` — the finding can become a hard `Gittensory Gate` blocker. Blocking is always +- `block` — the finding can become a hard `Gittensory Orb Review Agent` blocker. Blocking is always **confirmed-contributor-gated** — the mode chooses *which* deterministic checks are active, never *who* can be blocked. @@ -99,7 +99,7 @@ already-enabled gate. | Policy pack | `gate.pack` | `gatePack` | `gittensor` / `oss-anti-slop` | `gittensor` | `gittensor` = confirmed-contributor-gated, registry-aware. `oss-anti-slop` runs the deterministic rules against any author on any repo. | | Linked-issue gate | `gate.linkedIssue` | `linkedIssueGateMode` | `off`/`advisory`/`block` | `advisory` | If the dashboard "Require linked issue" toggle (`requireLinkedIssue`) is on but this is `off`, it is auto-promoted to `block`. | | Duplicate-PR gate | `gate.duplicates` | `duplicatePrGateMode` | `off`/`advisory`/`block` | `block` | Detects duplicate/superseding PRs. | -| Quality / merge-readiness score signal | `gate.readiness.mode` | `qualityGateMode` | `off`/`advisory`/`block` | `advisory` | Advisory/informational only. `block` is accepted for older configs but does not fail the Gate check. | +| Quality / merge-readiness score signal | `gate.readiness.mode` | `qualityGateMode` | `off`/`advisory`/`block` | `advisory` | Advisory/informational only. `block` is accepted for older configs but does not fail the review-agent check. | | Quality min score | `gate.readiness.minScore` | `qualityGateMinScore` | number 0–100 (nullable) | `null` | Advisory warning threshold for the readiness signal; `null` disables the threshold. | | Slop gate | `gate.slop.mode` | `slopGateMode` | `off`/`advisory`/`block` | `off` | Deterministic anti-slop signal. `advisory` surfaces the slop score + warnings; `block` also hard-blocks at/above the min score. Opt-in. | | Slop min score | `gate.slop.minScore` | `slopGateMinScore` | number 0–100 (nullable) | `null` (engine uses `60`, the "high" band) | The slop-risk threshold at/above which `slop block` blocks. | @@ -112,7 +112,7 @@ already-enabled gate. | AI review all authors | `gate.aiReview.allAuthors` | `aiReviewAllAuthors` | bool | `false` | When `true`, an enabled AI review runs for every PR author instead of only the engine's default eligible authors. Use this for self-host repos where the selected model must produce the public review summary. | | AI review provider | `gate.aiReview.provider` | `aiReviewProvider` | `anthropic` / `openai` / `null` | `null` | `null` = use the stored key's own provider. Must match the stored key's provider or BYOK is skipped (Workers-AI fallback). The key itself is only in the encrypted key store. | | AI review model | `gate.aiReview.model` | `aiReviewModel` | string / `null` | `null` | Model override for the BYOK advisory write-up (e.g. `claude-3-5-sonnet-latest`). `null` = the key record's model, else a conservative per-provider default. | -| AI close confidence | `gate.aiReview.closeConfidence` | `aiReviewCloseConfidence` | number 0–1 (nullable) | `null` (engine uses `0.9`) | Minimum **calibrated** AI-reviewer confidence for a consensus defect / split to **block** under `aiReview.mode: block`. Below-threshold AI defects stay advisory (visible, never close). Each reviewer rates its own confidence; consensus carries the weaker reviewer's. Config-as-code only (no dashboard/DB column). | +| AI close confidence | `gate.aiReview.closeConfidence` | `aiReviewCloseConfidence` | number 0–1 (nullable) | `null` (engine uses `0.93`) | Minimum **calibrated** AI-reviewer confidence for a consensus defect / split to **block** under `aiReview.mode: block`. Below-threshold AI defects stay advisory (visible, never close). Each reviewer rates its own confidence; consensus carries the weaker reviewer's. Config-as-code only (no dashboard/DB column). | ### Guardrails and scope (focus manifest) diff --git a/src/github/app.ts b/src/github/app.ts index 041846b56c..0d315d4ea7 100644 --- a/src/github/app.ts +++ b/src/github/app.ts @@ -18,6 +18,16 @@ import { type GateCheckEvaluation, type GateCheckPolicy, } from "../rules/advisory"; +import { + GITTENSORY_CONTEXT_CHECK_NAME, + GITTENSORY_GATE_CHECK_NAME, +} from "../review/check-names"; + +export { + GITTENSORY_CONTEXT_CHECK_NAME, + GITTENSORY_GATE_CHECK_NAME, + GITTENSORY_LEGACY_GATE_CHECK_NAME, +} from "../review/check-names"; type CheckRunResponse = { id: number; @@ -38,9 +48,6 @@ export type CheckRunOutcome = | { kind: "published"; id: number; html_url?: string } | { kind: "permission_missing"; warning: string }; -export const GITTENSORY_CONTEXT_CHECK_NAME = "Gittensory Context"; -export const GITTENSORY_GATE_CHECK_NAME = "Gittensory Gate"; - type GitHubCheckConclusion = | Advisory["conclusion"] | GateCheckConclusion @@ -508,7 +515,7 @@ export async function createOrUpdateGateCheckRun( mode: AgentActionMode = "live", ): Promise { // Prefer the AUTHORITATIVE pre-computed evaluation when the caller has one (#5 / audit): the surface/content - // lane can OVERRIDE the generic verdict (surface_lane_reject → failure, surface_lane_manual → action_required), + // lane can OVERRIDE the generic verdict (surface_lane_reject → failure, surface_lane_manual → neutral), // and re-deriving here via evaluateGateCheck would discard that override — publishing a GREEN check while the // PR is actually auto-closed/held. Callers without a surface lane omit `gate` and re-derive as before (identical). const gate = options.gate ?? evaluateGateCheck(advisory, policy); @@ -544,10 +551,10 @@ export async function createOrUpdatePendingGateCheckRun( name: GITTENSORY_GATE_CHECK_NAME, status: "in_progress", output: { - title: "Gittensory Gate is evaluating", + title: "Gittensory Orb Review Agent is evaluating", summary: "Gittensory is running deterministic public PR hygiene checks.", - text: "The Gate blocks every author on the repo's configured hard blockers (duplicate PRs by default); on everything else, and while state is still syncing, it stays advisory.", + text: "The review agent blocks every author on the repo's configured hard blockers (duplicate PRs by default); on everything else, and while state is still syncing, it stays advisory.", }, updateExisting: "in_progress_only", mode, @@ -573,7 +580,7 @@ export async function createOrUpdateSkippedGateCheckRun( status: "completed", conclusion: "skipped", output: { - title: "Gittensory Gate skipped", + title: "Gittensory Orb Review Agent skipped", summary: reason, text: "Gittensory does not post late first comments on closed or merged pull requests.", }, @@ -585,7 +592,7 @@ export async function createOrUpdateSkippedGateCheckRun( /** * Finalize a previously-posted pending Gate check to a NEUTRAL (non-blocking) terminal state when the * evaluation could not finish (a transient error/timeout in the work between posting the pending check and - * completing it). This guarantees the "Gittensory Gate is evaluating" run never hangs in_progress forever; + * completing it). This guarantees the "Gittensory Orb Review Agent is evaluating" run never hangs in_progress forever; * it does not block the PR and re-runs on the next push. Targets the known pending check_run id so it * updates the SAME run rather than creating a second one. */ @@ -607,10 +614,10 @@ export async function createOrUpdateErroredGateCheckRun( status: "completed", conclusion: "neutral", output: { - title: "Gittensory Gate — could not finish evaluating", + title: "Gittensory Orb Review Agent — could not finish evaluating", summary: "A transient error interrupted gate evaluation. This does NOT block the PR and re-runs automatically on the next push.", - text: "Gittensory finalizes the Gate to a neutral, non-blocking state when evaluation is interrupted, so the check never hangs in_progress. Push a new commit or use the 'Re-run Gittensory review' checkbox to re-evaluate.", + text: "Gittensory finalizes the review-agent check to a neutral, non-blocking state when evaluation is interrupted, so the check never hangs in_progress. Push a new commit or use the 'Re-run Gittensory review' checkbox to re-evaluate.", }, checkRunId: options.checkRunId, mode, @@ -642,9 +649,9 @@ export async function createOrUpdateOverriddenGateCheckRun( status: "completed", conclusion: "neutral", output: { - title: `Gittensory Gate — overridden by @${options.actor}`, + title: `Gittensory Orb Review Agent — overridden by @${options.actor}`, summary: - "A maintainer set the Gate to neutral for THIS commit only. This does NOT permanently bypass the Gate; a new push re-evaluates it.", + "A maintainer set the review-agent check to neutral for THIS commit only. This does NOT permanently bypass the review agent; a new push re-evaluates it.", text: `Overridden by @${options.actor}: ${options.reason}`, }, checkRunId: options.checkRunId, diff --git a/src/github/backfill.ts b/src/github/backfill.ts index 5c9eb0907c..72c9b478ff 100644 --- a/src/github/backfill.ts +++ b/src/github/backfill.ts @@ -59,7 +59,12 @@ import type { RepositorySettings, } from "../types"; import { errorMessage, nowIso, repoParts, strippedErrorMessage } from "../utils/json"; -import { createInstallationToken, getAppInstallation, GITTENSORY_CONTEXT_CHECK_NAME, GITTENSORY_GATE_CHECK_NAME } from "./app"; +import { createInstallationToken, getAppInstallation } from "./app"; +import { + GITTENSORY_CONTEXT_CHECK_NAME, + GITTENSORY_GATE_CHECK_NAME, + GITTENSORY_LEGACY_GATE_CHECK_NAME, +} from "../review/check-names"; import { delayUntil, shouldWaitForGitHubRateLimit } from "./rate-limit"; type GitHubLabelPayload = { @@ -825,8 +830,8 @@ export async function buildInstallationRepairDiagnostics(env: Env, health: Insta optional: gateCheckRepoCount === 0, summary: gateCheckRepoCount > 0 - ? "Gate check mode is enabled for at least one installed repo, so Checks: write is required." - : "Checks: write is optional unless gate check mode is enabled for an installed repo.", + ? "Review-agent check mode is enabled for at least one installed repo, so Checks: write is required." + : "Checks: write is optional unless review-agent check mode is enabled for an installed repo.", }), ]; const eventDiagnostics: InstallationEventDiagnostic[] = [ @@ -1920,7 +1925,11 @@ const CI_PASSING_CONCLUSIONS = new Set(["success", "neutral", "skipped"]); // the very review they're blocking runs → the PR defers forever). Excluded from the CI aggregate entirely. // (#gate-self-deadlock — froze green-CI PRs as "CI still running". The Gate alone wasn't enough: the Context // check is posted the same way and re-created the deadlock, so exclude ALL bot-owned checks.) -const BOT_OWNED_CHECK_NAMES = new Set([GITTENSORY_GATE_CHECK_NAME, GITTENSORY_CONTEXT_CHECK_NAME]); +const BOT_OWNED_CHECK_NAMES = new Set([ + GITTENSORY_GATE_CHECK_NAME, + GITTENSORY_LEGACY_GATE_CHECK_NAME, + GITTENSORY_CONTEXT_CHECK_NAME, +]); function isOwnGitHubAppCheckRun(env: Env, run: GitHubCheckRunPayload): boolean { const appSlug = typeof run.app?.slug === "string" ? run.app.slug.trim().toLowerCase() : ""; diff --git a/src/queue/processors.ts b/src/queue/processors.ts index 684c26b13d..7dd55a2540 100644 --- a/src/queue/processors.ts +++ b/src/queue/processors.ts @@ -103,6 +103,7 @@ import { createOrUpdateSkippedGateCheckRun, getInstallationId, getRepositoryCollaboratorPermission, + GITTENSORY_GATE_CHECK_NAME, isGitHubRateLimitedError, isForeignAppInstallation, } from "../github/app"; @@ -3401,7 +3402,7 @@ export function gateCheckPolicy( qualityGateMinScore: settings.qualityGateMinScore ?? null, aiReviewGateMode: settings.aiReviewMode, // Calibrated AI close-confidence floor (#7) — config-as-code via `.gittensory.yml gate.aiReview.closeConfidence`, - // resolved into settings upstream. `null`/undefined ⇒ advisory.ts applies the 0.9 default. + // resolved into settings upstream. `null`/undefined ⇒ advisory.ts applies the 0.93 default. aiReviewCloseConfidence: settings.aiReviewCloseConfidence ?? null, readinessScore: readinessScore ?? null, slopGateMode: settings.slopGateMode, @@ -4304,7 +4305,7 @@ async function maybePublishPrPublicSurface( } // Respect the per-repo agent pause: suppress all public surface mutations (label, comment, context - // check run) so a paused repo sees no gittensory-authored GitHub content. The Gittensory Gate check + // check run) so a paused repo sees no gittensory-authored GitHub content. The review-agent check // run still posts so the required-check status is not broken (#agent-pause). if (settings.agentPaused) decision = { @@ -5051,7 +5052,7 @@ async function maybePublishPrPublicSurface( // 2. The gate is AUTHORITATIVE for the comment's color/headline: `buildUnifiedCommentBody` maps // `gateEvaluation.conclusion` → a Verdict and feeds it as the renderer `decision`, which // deriveUnifiedStatus honors BEFORE any reviewer recommendation. So the comment's tone can never - // contradict the Gittensory Gate check-run conclusion. + // contradict the review-agent check-run conclusion. // 3. The `ai_consensus_defect` surfaces exactly ONCE — as the Code-review blocker — never also in the // gate signal row (which renders only the conclusion-derived status text, not the defect string). if (unifiedCommentAllowed && gateEvaluation) { @@ -5701,8 +5702,8 @@ async function maybeProcessGateOverrideCommand( AGENT_COMMAND_COMMENT_MARKER, "", "> [!NOTE]", - `> **Gittensory Gate overridden by @${actor}**`, - "> The Gate check was set to neutral for the current commit only. This does NOT permanently bypass the Gate; a new push re-evaluates it.", + `> **${GITTENSORY_GATE_CHECK_NAME} overridden by @${actor}**`, + "> The review-agent check was set to neutral for the current commit only. This does NOT permanently bypass the review; a new push re-evaluates it.", "", `- Reason: ${safeReason}`, "", diff --git a/src/review/check-names.ts b/src/review/check-names.ts new file mode 100644 index 0000000000..df686a0388 --- /dev/null +++ b/src/review/check-names.ts @@ -0,0 +1,3 @@ +export const GITTENSORY_CONTEXT_CHECK_NAME = "Gittensory Context"; +export const GITTENSORY_GATE_CHECK_NAME = "Gittensory Orb Review Agent"; +export const GITTENSORY_LEGACY_GATE_CHECK_NAME = "Gittensory Gate"; diff --git a/src/review/content-lane-wire.ts b/src/review/content-lane-wire.ts index 097ea6fb50..f990a88d3a 100644 --- a/src/review/content-lane-wire.ts +++ b/src/review/content-lane-wire.ts @@ -44,8 +44,8 @@ function surfaceFinding(code: string, severity: AdvisorySeverity, summary: strin return { code, title: SURFACE_TITLE, severity, detail: summary, publicText: summary }; } -/** Convert the deterministic surface verdict into a gate evaluation. merge→success, manual→action_required - * (a warning, not auto-closed), and any decisive non-merge/non-manual verdict (close) → failure with a single +/** Convert the deterministic surface verdict into a gate evaluation. merge→success, manual→neutral + * (a warning, not auto-closed and not a failing required check), and any decisive non-merge/non-manual verdict (close) → failure with a single * critical blocker. Returns the finding to splice into the advisory so the public comment renders the reason. */ export function surfaceVerdictToGate(result: SurfaceReviewResult): { evaluation: GateCheckEvaluation; @@ -57,7 +57,7 @@ export function surfaceVerdictToGate(result: SurfaceReviewResult): { } if (result.verdict === "manual") { const finding = surfaceFinding(SURFACE_MANUAL_CODE, "warning", summary); - return { evaluation: { enabled: true, conclusion: "action_required", title: SURFACE_TITLE, summary, blockers: [], warnings: [finding] }, finding }; + return { evaluation: { enabled: true, conclusion: "neutral", title: SURFACE_TITLE, summary, blockers: [], warnings: [finding] }, finding }; } const finding = surfaceFinding(SURFACE_REJECT_CODE, "critical", summary); return { evaluation: { enabled: true, conclusion: "failure", title: SURFACE_TITLE, summary, blockers: [finding], warnings: [] }, finding }; diff --git a/src/review/unified-comment-bridge.ts b/src/review/unified-comment-bridge.ts index d0e7ead3c3..2b1ee6142b 100644 --- a/src/review/unified-comment-bridge.ts +++ b/src/review/unified-comment-bridge.ts @@ -26,6 +26,7 @@ import type { CaptureRoute } from "./visual/capture"; // importers of `PR_PANEL_COMMENT_MARKER` from this module keep working. The unified body MUST prepend this // verbatim or `createOrUpdatePrIntelligenceComment` posts a DUPLICATE instead of updating in place. import { PR_PANEL_COMMENT_MARKER } from "../github/comments"; +import { GITTENSORY_GATE_CHECK_NAME } from "./check-names"; import { buildUnifiedReviewInput, renderUnifiedReviewComment, @@ -398,7 +399,7 @@ export function buildUnifiedCommentBody(args: UnifiedCommentBridgeArgs): string /** * Build the unified body for the CLOSED/SKIPPED case (the PR closed before full evaluation). This is the - * unified-renderer analogue of the legacy `buildClosedPrPanelUpdate` "[!NOTE] Gittensory Gate skipped" panel, + * unified-renderer analogue of the legacy `buildClosedPrPanelUpdate` skipped review-agent panel, * routed through `buildUnifiedCommentBody` so a comment that started life as a unified OPEN-PR comment keeps * its unified shape (and the SAME marker) when the PR closes, instead of being overwritten by the legacy * panel under the shared marker. A synthetic `skipped` gate maps (via `gateConclusionToVerdict`) to the @@ -410,7 +411,7 @@ export function buildClosedUnifiedCommentBody(args: { repoFullName: string; pull const skippedGate: GateCheckEvaluation = { enabled: true, conclusion: "skipped", - title: "Gittensory Gate skipped", + title: `${GITTENSORY_GATE_CHECK_NAME} skipped`, summary: "PR closed before full evaluation. No late first comment was created.", blockers: [], warnings: [], diff --git a/src/review/unified-comment.ts b/src/review/unified-comment.ts index af43daaf82..f675c95032 100644 --- a/src/review/unified-comment.ts +++ b/src/review/unified-comment.ts @@ -252,9 +252,14 @@ export function deriveUnifiedStatus(input: UnifiedReviewInput, ctx: UnifiedComme else if ((input.failedCount ?? 0) > 0 || recs.some((r) => r !== "merge")) status = "held"; else status = "ready"; } - // Readiness is advisory for the Gittensory verdict. A PR is not "safe to merge" until CI is green, but - // CI/merge-state evidence must not create a red/blocked Gittensory decision by itself; the blocker has to - // come from the review disposition (`close`) or consensus review findings. + // CI failure is an objective failing review state even when the disposition cannot auto-close the PR + // (for example, JSONbored/owner-authored PRs). The action wording below still respects `neverClosed`, so this + // renders as a red fix-required/manual-follow-up state without suggesting an owner PR will be rejected/closed. + if (input.readiness?.ciState === "failed") { + return "blocked"; + } + // Readiness is otherwise advisory for the Gittensory verdict. A PR is not "safe to merge" until CI is green, + // but pending/unverified CI should hold rather than create a red/blocked Gittensory decision by itself. if (status === "ready" && input.readiness && input.readiness.ciState !== "passed") { return "held"; } @@ -273,18 +278,16 @@ export function deriveUnifiedStatus(input: UnifiedReviewInput, ctx: UnifiedComme // CI / merge-state / gate block above still wins. (#guarded-hold-comment) if (status === "ready" && ctx.heldForReview) return "held"; // Held-vs-closed disposition parity (#8/#9): a gate "close" verdict does NOT always close the PR. An - // owner/automation-bot author is NEVER auto-closed, and a guarded-path PR is held for owner review UNLESS a - // RED required check forces the close (ciState "failed" mirrors the disposition's redVerifiedRequiredCi). Render - // those as "held" so the headline matches the action (#4220 class); a genuine contributor close — red required - // CI, or a non-guarded block — still headlines "Closed". + // owner/automation-bot author is NEVER auto-closed, and a guarded-path PR is held for owner review. Failed CI + // returned the red blocked status above, so the remaining close/held cases are green-or-pending manual holds. if (input.decision === "close") { if (ctx.neverClosed) return "held"; - if (ctx.heldForReview && input.readiness?.ciState !== "failed") return "held"; + if (ctx.heldForReview) return "held"; } return status; } -function headlineLabel(status: UnifiedCommentStatus, input: UnifiedReviewInput): string { +function headlineLabel(status: UnifiedCommentStatus, input: UnifiedReviewInput, ctx: UnifiedCommentContext): string { switch (status) { case "ready": return "approve/merge recommended"; @@ -293,7 +296,7 @@ function headlineLabel(status: UnifiedCommentStatus, input: UnifiedReviewInput): case "held": return "manual review recommended"; case "blocked": - return input.decision === "close" ? "reject/close recommended" : "blockers found"; + return input.decision === "close" && !ctx.neverClosed ? "reject/close recommended" : "fixes required"; } } @@ -315,20 +318,29 @@ function statusChips(input: UnifiedReviewInput, ctx: UnifiedCommentContext): str return chips.join(" · "); } -function verdictLine(status: UnifiedCommentStatus, input: UnifiedReviewInput): string { +function verdictLine(status: UnifiedCommentStatus, input: UnifiedReviewInput, ctx: UnifiedCommentContext): string { const icon = STATUS_META[status].icon; - const reason = input.verdictReason ? ` — ${escapePublicHtmlAngles(input.verdictReason)}` : ""; + const reasons = (defaultReason?: string) => { + const raw = input.verdictReason?.trim() || defaultReason?.trim() || ""; + return raw ? `\n${actionReasonBullets(raw)}` : ""; + }; switch (status) { case "ready": return input.merged - ? `**${icon} Suggested Action - Approve/Merge**${input.verdictReason ? reason : " — auto-merged"}` - : `**${icon} Suggested Action - Approve/Merge**${input.verdictReason ? reason : " — safe to merge"}`; + ? `**${icon} Suggested Action - Approve/Merge**${reasons("auto-merged")}` + : `**${icon} Suggested Action - Approve/Merge**${reasons("safe to merge")}`; case "advisory": - return `**${icon} Suggested Action - Advisory Only**${input.verdictReason ? reason : " — no action taken"}`; + return `**${icon} Suggested Action - Advisory Only**${reasons("no action taken")}`; case "held": - return `**${icon} Suggested Action - Manual Review**${reason}`; + return `**${icon} Suggested Action - Manual Review**${reasons()}`; case "blocked": - return `**${icon} Suggested Action - ${input.decision === "close" ? "Reject/Close" : "Fix Blockers"}**${reason}`; + if (ctx.neverClosed) { + return `**${icon} Suggested Action - Manual Review**${reasons()}`; + } + if (input.decision === "close" && !ctx.neverClosed) { + return `**${icon} Suggested Action - Reject/Close**${reasons()}`; + } + return `**${icon} Suggested Action - Fix Blockers**${reasons()}`; } } @@ -366,6 +378,16 @@ function taskList(items: string[]): string { .join("\n"); } +function actionReasonBullets(reason: string): string { + const reasons = reason + .split(/[;\n]+/) + .map((item) => item.trim()) + .filter((item) => item.length > 0); + return dedupeLines(reasons, 8) + .map((item) => `- ${escapePublicHtmlAngles(item)}`) + .join("\n"); +} + function formatReviewTimestamp(value: string | number | Date | undefined): string | null { if (value === undefined) return null; const time = value instanceof Date ? value : new Date(value); @@ -453,15 +475,15 @@ export function renderUnifiedReviewComment(input: UnifiedReviewInput, ctx: Unifi const status = deriveUnifiedStatus(input, ctx); const meta = STATUS_META[status]; const brand = escapePublicHtmlAngles(ctx.brand ?? "Gittensory review"); + const reviewTimestamp = formatReviewTimestamp(ctx.reviewedAt); const blocks: string[] = [ meta.square.repeat(12), - `### ${meta.icon} ${brand} result - ${headlineLabel(status, input)}${status === "ready" && input.merged ? " · auto-merged" : ""}`, + `### ${meta.icon} ${brand} result - ${headlineLabel(status, input, ctx)}${status === "ready" && input.merged ? " · auto-merged" : ""}`, + ...(reviewTimestamp ? [`Review updated: ${reviewTimestamp}`] : []), statusChips(input, ctx), - verdictLine(status, input), + verdictLine(status, input, ctx), ]; - const reviewTimestamp = formatReviewTimestamp(ctx.reviewedAt); - if (reviewTimestamp) blocks.push(`Review updated: ${reviewTimestamp}`); if (input.summary.trim()) blocks.push(`**Review summary**\n${escapePublicHtmlAngles(input.summary.trim())}`); diff --git a/src/rules/advisory.ts b/src/rules/advisory.ts index 786e38216b..ca9d69f6cf 100644 --- a/src/rules/advisory.ts +++ b/src/rules/advisory.ts @@ -12,6 +12,7 @@ import type { import type { CollisionCluster, CollisionReport } from "../signals/engine"; import { isDuplicateClusterWinner } from "../signals/duplicate-winner"; import { nowIso } from "../utils/json"; +import { GITTENSORY_GATE_CHECK_NAME } from "../review/check-names"; export type GateCheckConclusion = "success" | "failure" | "action_required" | "neutral" | "skipped"; @@ -27,7 +28,7 @@ export type GateCheckPolicy = { aiReviewGateMode?: GateRuleMode | undefined; /** Minimum calibrated confidence (0-1) for an AI-judgment defect (`ai_consensus_defect` / `ai_review_split`) to * BLOCK under `aiReviewGateMode: block` (#7). The finding blocks only when its `confidence >= this`; below-threshold - * AI defects hold for human review instead of passing or auto-closing. `null`/undefined ⇒ the 0.9 default. A finding + * AI defects hold for human review instead of passing or auto-closing. `null`/undefined ⇒ the 0.93 default. A finding * with no confidence (deterministic, or a graceful-fallback AI defect) is treated as 1.0 and always clears the floor * — matching the historical always-block behavior. */ aiReviewCloseConfidence?: number | null | undefined; @@ -39,7 +40,7 @@ export type GateCheckPolicy = { slopRisk?: number | null | undefined; /** Master "merge-readiness" composite (#551). When set (advisory/block) it OVERRIDES all four sub-gates — * linked-issue, duplicate, quality/readiness, slop — to its mode, so a maintainer flips ONE switch instead - * of four and `Gittensory Gate` stays the single required check. `off` = sub-gates use their own modes. */ + * of four and the review-agent check stays the single required check. `off` = sub-gates use their own modes. */ mergeReadinessGateMode?: GateRuleMode | undefined; /** Focus-manifest policy gate (#555). When `block`, linked-issue/test policy findings become hard blockers; * blocked-path findings become manual-review holds because guardrailed paths should be reviewed, not closed. @@ -122,7 +123,7 @@ export function reconcileGateEvaluationForGreenCi(evaluation: GateCheckEvaluatio return { ...evaluation, conclusion: "success", - title: "Gittensory Gate passed", + title: `${GITTENSORY_GATE_CHECK_NAME} passed`, summary: "The AI review raised a concern, but the deterministic checks (CI) are green — the concern is advisory, not blocking.", blockers: [], }; @@ -497,7 +498,7 @@ function evaluateGateCheckCore(advisoryResult: Advisory, policy: GateCheckPolicy return { enabled: true, conclusion: "neutral", - title: "Gittensory Gate — not evaluated yet", + title: `${GITTENSORY_GATE_CHECK_NAME} — not evaluated yet`, summary: "Gittensory has not finished syncing this repo/PR. The gate stays advisory and re-evaluates automatically; no action is needed.", blockers: [], warnings, @@ -530,7 +531,7 @@ function evaluateGateCheckCore(advisoryResult: Advisory, policy: GateCheckPolicy return { enabled: true, conclusion: "neutral", - title: "Gittensory Gate — first-contribution grace", + title: `${GITTENSORY_GATE_CHECK_NAME} — first-contribution grace`, summary: "This is a first-time contribution to this repo, so the gate stays advisory rather than blocking. The findings remain visible, and the gate will apply normally once this author has merge history here.", blockers: [], warnings: gateWarnings, @@ -541,7 +542,7 @@ function evaluateGateCheckCore(advisoryResult: Advisory, policy: GateCheckPolicy return { enabled: true, conclusion: "neutral", - title: "Gittensory Gate — held for human review", + title: `${GITTENSORY_GATE_CHECK_NAME} — held for human review`, summary: "The AI review flagged a possible must-fix defect below the automatic close-confidence floor, so the gate is held for a human reviewer instead of passed automatically.", blockers: [], warnings: [...gateWarnings, ...lowConfidenceAiHolds], @@ -556,7 +557,7 @@ function evaluateGateCheckCore(advisoryResult: Advisory, policy: GateCheckPolicy return { enabled: true, conclusion: "neutral", - title: "Gittensory Gate — held for human review", + title: `${GITTENSORY_GATE_CHECK_NAME} — held for human review`, summary: "The AI review could not be completed for this change, so the gate is held for a human reviewer rather than passed automatically. It re-evaluates on the next update.", blockers: [], warnings: gateWarnings, @@ -578,7 +579,7 @@ function evaluateGateCheckCore(advisoryResult: Advisory, policy: GateCheckPolicy return { enabled: true, conclusion: "neutral", - title: "Gittensory Gate — held for manual review", + title: `${GITTENSORY_GATE_CHECK_NAME} — held for manual review`, summary: holds.map((h) => sanitizeForCheckRun(h.title)).join("; "), blockers: [], warnings: [...gateWarnings, ...holds], @@ -587,7 +588,7 @@ function evaluateGateCheckCore(advisoryResult: Advisory, policy: GateCheckPolicy return { enabled: true, conclusion: "success", - title: "Gittensory Gate passed", + title: `${GITTENSORY_GATE_CHECK_NAME} passed`, summary: "No configured hard blocker was found. Advisory findings, if any, stay advisory.", blockers, warnings: gateWarnings, @@ -599,7 +600,7 @@ function evaluateGateCheckCore(advisoryResult: Advisory, policy: GateCheckPolicy return { enabled: true, conclusion: "failure", - title: `Gittensory Gate: ${titleDetail}`, + title: `${GITTENSORY_GATE_CHECK_NAME}: ${titleDetail}`, summary: blockers .map((finding) => `${sanitizeForCheckRun(finding.title)}${finding.action ? ` — ${sanitizeForCheckRun(finding.action)}` : ""}`) .join("; "), @@ -612,7 +613,7 @@ export function formatGateCheckOutput(gate: GateCheckEvaluation): { title: strin if (gate.conclusion === "success") { return { title: gate.title, - summary: "Gittensory Gate is advisory-first. This PR has no configured hard blocker.", + summary: `${GITTENSORY_GATE_CHECK_NAME} is advisory-first. This PR has no configured hard blocker.`, text: "No configured hard blocker was found. Advisory signals remain visible in the PR panel when comments are enabled.", }; } @@ -632,7 +633,7 @@ export function formatGateCheckOutput(gate: GateCheckEvaluation): { title: strin // An unbounded title (e.g. when failing-check names are appended) threw a 422 that aborted the ENTIRE // review before the comment, audit, and auto-action — so red-CI PRs were never reviewed or closed. title: gate.title.slice(0, 255), - summary: "Gittensory Gate found a repo-configured hard blocker.", + summary: `${GITTENSORY_GATE_CHECK_NAME} found a repo-configured hard blocker.`, text: blockerLines.length > 0 ? blockerLines.join("\n") : "A configured hard blocker was found.", }; } @@ -856,8 +857,8 @@ function isEvaluationBlocker(code: string): boolean { } // Default minimum calibrated confidence for an AI defect to BLOCK (#7) — used when the repo set `aiReview: block` -// without a `closeConfidence`. 0.9 = block only on a high-confidence AI defect; below that stays advisory. -const DEFAULT_AI_REVIEW_CLOSE_CONFIDENCE = 0.9; +// without a `closeConfidence`. 0.93 = block only on a high-confidence AI defect; below that stays advisory. +const DEFAULT_AI_REVIEW_CLOSE_CONFIDENCE = 0.93; function isConfiguredGateBlocker(finding: AdvisoryFinding, policy: GateCheckPolicy): boolean { const code = finding.code; diff --git a/src/signals/engine.ts b/src/signals/engine.ts index 966e52a452..5f96c3d61d 100644 --- a/src/signals/engine.ts +++ b/src/signals/engine.ts @@ -31,6 +31,7 @@ import { isDuplicateClusterWinner } from "./duplicate-winner"; import { PREFLIGHT_LIMITS } from "./preflight-limits"; import type { UnifiedCollapsible } from "../review/unified-comment"; import { splitAiReviewNits } from "../review/ai-notes"; +import { GITTENSORY_GATE_CHECK_NAME } from "../review/check-names"; export type ParticipationLane = "direct_pr" | "issue_discovery" | "split" | "inactive" | "unknown"; export type SignalFinding = AdvisoryFinding; @@ -4250,7 +4251,7 @@ export function buildPublicPrIntelligenceComment(args: { : args.aiReview && !gateBlocking ? "Gittensory review approved this PR" : gateBlocking - ? "Gittensory Gate is blocking merge" + ? `${GITTENSORY_GATE_CHECK_NAME} is blocking merge` : hasPublicWarnings || hasRelatedWork ? "Gittensory found maintainer review notes" : "Gittensory PR readiness looks good"; diff --git a/src/signals/focus-manifest.ts b/src/signals/focus-manifest.ts index 1dcab80df3..e6190b917a 100644 --- a/src/signals/focus-manifest.ts +++ b/src/signals/focus-manifest.ts @@ -34,7 +34,7 @@ export type FocusManifestGateConfig = { aiReviewModel: string | null; aiReviewAllAuthors: boolean | null; /** `gate.aiReview.closeConfidence` (#7): minimum calibrated AI-reviewer confidence (0-1) for an AI defect to BLOCK - * under `aiReview.mode: block`. null (unset) ⇒ the gate's 0.9 default. Clamped to [0,1] at parse time. */ + * under `aiReview.mode: block`. null (unset) ⇒ the gate's 0.93 default. Clamped to [0,1] at parse time. */ aiReviewCloseConfidence: number | null; mergeReadiness: GateRuleMode | null; manifestPolicy: GateRuleMode | null; @@ -368,7 +368,7 @@ function normalizeOptionalScore(value: JsonValue | undefined, field: string, war } /** Normalize an optional confidence threshold in [0,1] (#7) — a fractional value (NOT a 0-100 score), so it is - * clamped into range WITHOUT rounding. Absent/null ⇒ null (the resolver leaves the gate's 0.9 default in place); + * clamped into range WITHOUT rounding. Absent/null ⇒ null (the resolver leaves the gate's 0.93 default in place); * a non-finite/non-number value is ignored with a warning. */ function normalizeOptionalConfidence(value: JsonValue | undefined, field: string, warnings: string[]): number | null { if (value === undefined || value === null) return null; diff --git a/src/signals/settings-preview.ts b/src/signals/settings-preview.ts index e564f3b9e7..1fdc4088da 100644 --- a/src/signals/settings-preview.ts +++ b/src/signals/settings-preview.ts @@ -14,6 +14,7 @@ import { type ContributorDetection, } from "./engine"; import { REQUIRED_INSTALLATION_PERMISSIONS } from "../github/backfill"; +import { GITTENSORY_GATE_CHECK_NAME } from "../review/check-names"; export function hasVisiblePrSurface(settings: RepositorySettings): boolean { return settings.publicSurface !== "off" || settings.checkRunMode === "enabled" || settings.gateCheckMode === "enabled"; @@ -363,7 +364,7 @@ function buildWarnings(settings: RepositorySettings, decision: PublicSurfaceDeci warnings.push("Check runs are enabled but GitHub App permission Checks: write is missing. Set repository permission checks to write, then approve the change."); } if (settings.gateCheckMode === "enabled" && missing.has("checks")) { - warnings.push("Gate checks are enabled but GitHub App permission Checks: write is missing. Set repository permission checks to write, then approve the change."); + warnings.push("Review-agent checks are enabled but GitHub App permission Checks: write is missing. Set repository permission checks to write, then approve the change."); } for (const event of installation.missingEvents) { warnings.push(`The GitHub App is not subscribed to the ${event} webhook event; subscribe to it so Gittensory receives the relevant deliveries.`); @@ -544,7 +545,7 @@ function permissionSummary(installation: InstallationHealthSummary | null, missi } function publicOutputsFor(decision: PublicSurfaceDecision, appliedLabel: string | null, settings: RepositorySettings): string[] { - const gateOutput = settings.gateCheckMode === "enabled" ? ["Opt-in Gittensory Gate check run."] : []; + const gateOutput = settings.gateCheckMode === "enabled" ? [`Opt-in ${GITTENSORY_GATE_CHECK_NAME} check run.`] : []; if (decision.skipped) return [`No comment or label for this sample: ${decision.summary}`, ...gateOutput]; const outputs = [ ...(decision.willComment ? ["One sanitized sticky PR comment."] : []), diff --git a/src/types.ts b/src/types.ts index 9a2793fe5a..32d2a2d1f4 100644 --- a/src/types.ts +++ b/src/types.ts @@ -526,7 +526,7 @@ export type RepositorySettings = { /** Merge-readiness gate (#merge-readiness). `off`/`advisory`/`block`. No min-score. Default `off`. */ mergeReadinessGateMode: GateRuleMode; /** Focus-manifest policy gate (#555). When `block`, the focus manifest's declared policy (blocked paths, - * required-linked-issue, test expectations) becomes an enforceable `Gittensory Gate` blocker. An + * required-linked-issue, test expectations) becomes an enforceable review-agent blocker. An * INDEPENDENT dimension, deliberately not folded into the merge-readiness composite. Default `off` — opt-in. */ manifestPolicyGateMode: GateRuleMode; /** Self-authored linked-issue gate. When `block`, the gate closes a PR where the contributor also @@ -569,7 +569,7 @@ export type RepositorySettings = { /** Minimum calibrated AI-reviewer confidence (0-1) for an AI defect to BLOCK under `aiReviewMode: block` (#7). * A dual-model consensus defect / split blocks only when its finding `confidence >= aiReviewCloseConfidence`; * below-threshold AI defects hold for human review rather than passing. Config-as-code only — set via - * `.gittensory.yml gate.aiReview.closeConfidence` (no dashboard/DB column); unset ⇒ the gate uses the 0.9 + * `.gittensory.yml gate.aiReview.closeConfidence` (no dashboard/DB column); unset ⇒ the gate uses the 0.93 * default. Clamped to [0,1] at parse time. */ aiReviewCloseConfidence?: number | null | undefined; /** When TRUE, the repo OWNER's (and maintainer's) own PRs are eligible for auto-CLOSE like a contributor's diff --git a/test/unit/backfill.test.ts b/test/unit/backfill.test.ts index 56c657cc0b..019773bd39 100644 --- a/test/unit/backfill.test.ts +++ b/test/unit/backfill.test.ts @@ -2883,6 +2883,7 @@ describe("GitHub backfill", () => { { name: "test", status: "completed", conclusion: "success" }, // BOTH bot-posted checks, still in_progress (posted but not yet concluded). Counting EITHER would // defer the very review that concludes it — the self-deadlock that froze green-CI PRs as "CI pending". + { name: "Gittensory Orb Review Agent", status: "in_progress", conclusion: null, app: { slug: "gittensory" } }, { name: "Gittensory Gate", status: "in_progress", conclusion: null, app: { slug: "gittensory" } }, { name: "Gittensory Context", status: "in_progress", conclusion: null, app: { slug: "gittensory" } }, ], @@ -2893,7 +2894,7 @@ describe("GitHub backfill", () => { }); // Both bot checks are excluded from the CI wait even if listed among the required contexts. - const aggregate = await fetchLiveCiAggregate(env, "JSONbored/metagraphed", "headsha", "public-token", new Set(["test", "Gittensory Gate", "Gittensory Context"])); + const aggregate = await fetchLiveCiAggregate(env, "JSONbored/metagraphed", "headsha", "public-token", new Set(["test", "Gittensory Orb Review Agent", "Gittensory Gate", "Gittensory Context"])); expect(aggregate.ciState).toBe("passed"); // would be "pending" if either in_progress bot check were counted expect(aggregate.failingDetails).toEqual([]); @@ -2907,7 +2908,7 @@ describe("GitHub backfill", () => { return Response.json({ check_runs: [ { name: "test", status: "completed", conclusion: "success", app: { slug: "github-actions" } }, - { name: "Gittensory Gate", status: "completed", conclusion: "failure", output: { title: "External gate failed" }, app: { slug: "external-ci" } }, + { name: "Gittensory Orb Review Agent", status: "completed", conclusion: "failure", output: { title: "External gate failed" }, app: { slug: "external-ci" } }, ], }); } @@ -2915,10 +2916,10 @@ describe("GitHub backfill", () => { return new Response("not found", { status: 404 }); }); - const aggregate = await fetchLiveCiAggregate(env, "JSONbored/gittensory", "abc123", "public-token", new Set(["test", "Gittensory Gate"])); + const aggregate = await fetchLiveCiAggregate(env, "JSONbored/gittensory", "abc123", "public-token", new Set(["test", "Gittensory Orb Review Agent"])); expect(aggregate.ciState).toBe("failed"); - expect(aggregate.failingDetails).toEqual([expect.objectContaining({ name: "Gittensory Gate", summary: "External gate failed" })]); + expect(aggregate.failingDetails).toEqual([expect.objectContaining({ name: "Gittensory Orb Review Agent", summary: "External gate failed" })]); }); it("does not ignore classic statuses named like the Gate", async () => { @@ -2926,14 +2927,14 @@ describe("GitHub backfill", () => { vi.stubGlobal("fetch", async (input: RequestInfo | URL) => { const url = input.toString(); if (url.includes("/check-runs?")) return Response.json({ check_runs: [{ name: "test", status: "completed", conclusion: "success", app: { slug: "github-actions" } }] }); - if (url.includes("/status?")) return Response.json({ statuses: [{ context: "Gittensory Gate", state: "failure", description: "External status failed" }] }); + if (url.includes("/status?")) return Response.json({ statuses: [{ context: "Gittensory Orb Review Agent", state: "failure", description: "External status failed" }] }); return new Response("not found", { status: 404 }); }); const aggregate = await fetchLiveCiAggregate(env, "JSONbored/gittensory", "abc123", "public-token", null); expect(aggregate.ciState).toBe("failed"); - expect(aggregate.failingDetails).toEqual([expect.objectContaining({ name: "Gittensory Gate", summary: "External status failed" })]); + expect(aggregate.failingDetails).toEqual([expect.objectContaining({ name: "Gittensory Orb Review Agent", summary: "External status failed" })]); }); it("treats a required context that never ran (absent from results) as pending, not passed", async () => { @@ -2973,7 +2974,7 @@ describe("GitHub backfill", () => { return Response.json({ check_runs: [ { name: "validate", status: "completed", conclusion: "success", app: { slug: "github-actions" } }, - { name: "Gittensory Gate", status: "in_progress", conclusion: null, app: { slug: "gittensory" } }, + { name: "Gittensory Orb Review Agent", status: "in_progress", conclusion: null, app: { slug: "gittensory" } }, ], }); } @@ -2981,9 +2982,9 @@ describe("GitHub backfill", () => { return new Response("not found", { status: 404 }); }); - const aggregate = await fetchLiveCiAggregate(env, "JSONbored/gittensory", "sha", "tok", new Set(["validate", "Gittensory Gate"])); + const aggregate = await fetchLiveCiAggregate(env, "JSONbored/gittensory", "sha", "tok", new Set(["validate", "Gittensory Orb Review Agent"])); - // "Gittensory Gate" is a bot check: present in results (so not absent), excluded from gate logic → passed + // "Gittensory Orb Review Agent" is a bot check: present in results (so not absent), excluded from gate logic → passed expect(aggregate.ciState).toBe("passed"); }); diff --git a/test/unit/content-lane-wire.test.ts b/test/unit/content-lane-wire.test.ts index 132cd792a0..adf7b388a5 100644 --- a/test/unit/content-lane-wire.test.ts +++ b/test/unit/content-lane-wire.test.ts @@ -45,9 +45,9 @@ describe("surfaceVerdictToGate", () => { expect(AI_JUDGMENT_BLOCKER_CODES.has(evaluation.blockers[0]!.code)).toBe(false); }); - it("manual → action_required with a warning (not auto-closed)", () => { + it("manual → neutral with a warning (not a failing required check)", () => { const { evaluation, finding } = surfaceVerdictToGate({ verdict: "manual", summary: "auth declared" }); - expect(evaluation.conclusion).toBe("action_required"); + expect(evaluation.conclusion).toBe("neutral"); expect(evaluation.blockers).toEqual([]); expect(evaluation.warnings).toHaveLength(1); expect(finding?.code).toBe("surface_lane_manual"); @@ -79,7 +79,7 @@ describe("applySurfaceGate", () => { }; const genericHold = gate({ conclusion: "neutral", - title: "Gittensory Gate — held for manual review", + title: "Gittensory Orb Review Agent — held for manual review", summary: "Large change — held for manual review", blockers: [], warnings: [oversized], diff --git a/test/unit/docs-github-app.test.ts b/test/unit/docs-github-app.test.ts index 7be8a3f7da..080f569271 100644 --- a/test/unit/docs-github-app.test.ts +++ b/test/unit/docs-github-app.test.ts @@ -25,8 +25,8 @@ describe("docs GitHub App setup page", () => { it("keeps Context advisory and Gate opt-in before branch protection", () => { expect(source).toMatch(/Gittensory Context<\/strong> is advisory/); - expect(source).toMatch(/Gittensory Gate<\/strong> is opt-in/); - expect(source).toMatch(/should require Gittensory Gate<\/strong> only after/); + expect(source).toMatch(/Gittensory Orb Review Agent<\/strong> is opt-in/); + expect(source).toMatch(/should require Gittensory Orb Review Agent<\/strong> only after/); expect(source).toMatch(/Do not require Gittensory Context<\/strong>/); }); }); diff --git a/test/unit/gate-check-policy.test.ts b/test/unit/gate-check-policy.test.ts index 93ff768984..9c2399f7a1 100644 --- a/test/unit/gate-check-policy.test.ts +++ b/test/unit/gate-check-policy.test.ts @@ -223,16 +223,16 @@ describe("AI close-confidence threshold gate (#7)", () => { findings: [{ code: "ai_review_split", title: "An AI reviewer flagged a likely blocking defect", severity: "critical", detail: "One reviewer flagged it.", action: "Resolve it.", confidence }], }); - it("blocks when mode=block AND confidence >= the default 0.9 floor", () => { + it("blocks when mode=block AND confidence >= the default 0.93 floor", () => { const out = evaluateGateCheck(aiDefectWith(0.95), gateCheckPolicy(settings({ aiReviewMode: "block" }), null, true)); expect(out.conclusion).toBe("failure"); expect(out.blockers.map((f) => f.code)).toEqual(["ai_consensus_defect"]); }); it("holds for human review when mode=block but confidence < the floor (#7 regression)", () => { - const out = evaluateGateCheck(aiDefectWith(0.5), gateCheckPolicy(settings({ aiReviewMode: "block" }), null, true)); - expect(out.conclusion).toBe("neutral"); // below 0.9 → manual hold, not an auto-mergeable pass - expect(out.title).toBe("Gittensory Gate — held for human review"); + const out = evaluateGateCheck(aiDefectWith(0.92), gateCheckPolicy(settings({ aiReviewMode: "block" }), null, true)); + expect(out.conclusion).toBe("neutral"); // below 0.93 → manual hold, not an auto-mergeable pass + expect(out.title).toBe("Gittensory Orb Review Agent — held for human review"); expect(out.blockers).toEqual([]); expect(out.warnings.map((f) => f.code)).toContain("ai_consensus_defect"); }); @@ -244,18 +244,18 @@ describe("AI close-confidence threshold gate (#7)", () => { expect(evaluateGateCheck(aiDefectWith(0.69), policy).conclusion).toBe("neutral"); // just below → manual hold }); - it("honors a custom aiReviewCloseConfidence (the `?? 0.9` default is NOT used when set) (#7)", () => { - // A high custom floor of 0.99 holds a 0.95 defect for human review (the 0.9 default would have blocked it). + it("honors a custom aiReviewCloseConfidence (the `?? 0.93` default is NOT used when set) (#7)", () => { + // A high custom floor of 0.99 holds a 0.95 defect for human review (the 0.93 default would have blocked it). const strict = gateCheckPolicy(settings({ aiReviewMode: "block", aiReviewCloseConfidence: 0.99 }), null, true); expect(evaluateGateCheck(aiDefectWith(0.95), strict).conclusion).toBe("neutral"); - // A low custom floor of 0.3 blocks a 0.5 defect that the 0.9 default would have left advisory. + // A low custom floor of 0.3 blocks a 0.5 defect that the 0.93 default would have left advisory. const lenient = gateCheckPolicy(settings({ aiReviewMode: "block", aiReviewCloseConfidence: 0.3 }), null, true); expect(evaluateGateCheck(aiDefectWith(0.5), lenient).conclusion).toBe("failure"); }); it("a finding WITHOUT a confidence degrades to 1.0 and blocks under the default floor (graceful fallback) (#7)", () => { const out = evaluateGateCheck(aiDefectWith(undefined), gateCheckPolicy(settings({ aiReviewMode: "block" }), null, true)); - expect(out.conclusion).toBe("failure"); // no confidence → treated as 1.0 → always clears 0.9 + expect(out.conclusion).toBe("failure"); // no confidence → treated as 1.0 → always clears 0.93 }); it("never blocks when mode=advisory, regardless of a high confidence (#7)", () => { @@ -264,15 +264,15 @@ describe("AI close-confidence threshold gate (#7)", () => { it("applies the same confidence floor to an ai_review_split finding (#7)", () => { const policy = gateCheckPolicy(settings({ aiReviewMode: "block" }), null, true); - expect(evaluateGateCheck(splitDefectWith(0.95), policy).conclusion).toBe("failure"); // clears 0.9 → blocks - expect(evaluateGateCheck(splitDefectWith(0.5), policy).conclusion).toBe("neutral"); // below 0.9 → manual hold + expect(evaluateGateCheck(splitDefectWith(0.95), policy).conclusion).toBe("failure"); // clears 0.93 → blocks + expect(evaluateGateCheck(splitDefectWith(0.92), policy).conclusion).toBe("neutral"); // below 0.93 → manual hold }); it("resolveEffectiveSettings maps gate.aiReview.closeConfidence (clamped) into the policy floor (#7)", () => { const eff = resolveEffectiveSettings(settings({ aiReviewMode: "off" }), parseFocusManifest({ gate: { aiReview: { mode: "block", closeConfidence: 0.4 } } })); expect(eff.aiReviewCloseConfidence).toBe(0.4); expect(eff.aiReviewMode).toBe("block"); - // a 0.5 defect clears the configured 0.4 floor → blocks (it would NOT under the 0.9 default). + // a 0.5 defect clears the configured 0.4 floor → blocks (it would NOT under the 0.93 default). expect(evaluateGateCheck(aiDefectWith(0.5), gateCheckPolicy(eff, null, true)).conclusion).toBe("failure"); }); }); diff --git a/test/unit/github-app.test.ts b/test/unit/github-app.test.ts index 4c9e3eeda9..8c08fa47d1 100644 --- a/test/unit/github-app.test.ts +++ b/test/unit/github-app.test.ts @@ -716,7 +716,7 @@ describe("GitHub check runs", () => { expect(isCrossAppCheckRunError(null)).toBe(false); // non-object }); - it("creates a failing opt-in Gittensory Gate check for merge blockers", async () => { + it("creates a failing opt-in Gittensory Orb Review Agent check for merge blockers", async () => { const privateKey = await generatePrivateKeyPem(); let capturedBody: { name?: string; @@ -773,9 +773,9 @@ describe("GitHub check runs", () => { expect(result).toMatchObject({ kind: "published", id: 88 }); expect(capturedBody).toMatchObject({ - name: "Gittensory Gate", + name: "Gittensory Orb Review Agent", conclusion: "failure", - output: { title: "Gittensory Gate: No linked issue detected" }, + output: { title: "Gittensory Orb Review Agent: No linked issue detected" }, }); expect(capturedBody.output?.text).toContain("Link the issue before merge."); expect(capturedBody.output?.text).not.toMatch( @@ -817,9 +817,9 @@ describe("GitHub check runs", () => { expect(result).toMatchObject({ kind: "published", id: 89 }); expect(capturedBody).toMatchObject({ - name: "Gittensory Gate", + name: "Gittensory Orb Review Agent", status: "in_progress", - output: { title: "Gittensory Gate is evaluating" }, + output: { title: "Gittensory Orb Review Agent is evaluating" }, }); expect(capturedBody).not.toHaveProperty("conclusion"); // The Gate blocks every author the same on a configured blocker (confirmed status no longer gates the verdict). @@ -900,10 +900,10 @@ describe("GitHub check runs", () => { calls.some((call) => call.includes("/commits/final123/check-runs")), ).toBe(false); expect(capturedBody).toMatchObject({ - name: "Gittensory Gate", + name: "Gittensory Orb Review Agent", status: "completed", conclusion: "success", - output: { title: "Gittensory Gate passed" }, + output: { title: "Gittensory Orb Review Agent passed" }, }); }); @@ -973,7 +973,7 @@ describe("GitHub check runs", () => { if (url.includes("/commits/pending-existing/check-runs")) { return Response.json({ total_count: 1, - check_runs: [{ id: 333, name: "Gittensory Gate", status: "in_progress" }], + check_runs: [{ id: 333, name: "Gittensory Orb Review Agent", status: "in_progress" }], }); } if (url.includes("/check-runs/333")) { @@ -1021,7 +1021,7 @@ describe("GitHub check runs", () => { check_runs: [ { id: 444, - name: "Gittensory Gate", + name: "Gittensory Orb Review Agent", status: "completed", conclusion: "failure", }, @@ -1056,7 +1056,7 @@ describe("GitHub check runs", () => { expect(calls.some((call) => call.includes("/check-runs/444"))).toBe(false); expect(capturedBody).toMatchObject({ status: "in_progress", - output: { title: "Gittensory Gate is evaluating" }, + output: { title: "Gittensory Orb Review Agent is evaluating" }, }); expect(capturedBody).not.toHaveProperty("conclusion"); }); @@ -1100,7 +1100,7 @@ describe("GitHub check runs", () => { status: "completed", conclusion: "skipped", output: { - title: "Gittensory Gate skipped", + title: "Gittensory Orb Review Agent skipped", summary: "Merged before Gittensory finished.", }, }); @@ -1133,9 +1133,9 @@ describe("GitHub check runs", () => { output?: { annotations?: Array<{ path: string; title: string }> }; }; if (body.name === "Gittensory Context") contextBody = body; - if (body.name === "Gittensory Gate") gateBody = body; + if (body.name === "Gittensory Orb Review Agent") gateBody = body; return Response.json( - { id: body.name === "Gittensory Gate" ? 90 : 77 }, + { id: body.name === "Gittensory Orb Review Agent" ? 90 : 77 }, { status: 201 }, ); } diff --git a/test/unit/predicted-gate.test.ts b/test/unit/predicted-gate.test.ts index 2b5f6615f9..6e3306a784 100644 --- a/test/unit/predicted-gate.test.ts +++ b/test/unit/predicted-gate.test.ts @@ -64,7 +64,7 @@ describe("buildPredictedGateVerdict", () => { expect(result.conclusion).toBe("failure"); expect(result.blockers.some((b) => b.code === "duplicate_pr_risk")).toBe(true); // Public-safe: blocker text carries a fix and no raw internal markers. - expect(result.title.toLowerCase()).toContain("gittensory gate"); + expect(result.title.toLowerCase()).toContain("gittensory orb review agent"); }); it("does NOT block on a duplicate when duplicates:off", () => { diff --git a/test/unit/queue.test.ts b/test/unit/queue.test.ts index e4b3110800..c166731af1 100644 --- a/test/unit/queue.test.ts +++ b/test/unit/queue.test.ts @@ -2041,7 +2041,7 @@ describe("queue processors", () => { if (url.includes("/commits/gate123/check-runs")) return Response.json({ total_count: 0, check_runs: [] }); if (url.includes("/check-runs") && (init?.method ?? "GET") === "POST") { const body = JSON.parse(String(init?.body ?? "{}")) as { name?: string; status?: string; conclusion?: string; output?: { title?: string } }; - expect(body).toMatchObject({ name: "Gittensory Gate", status: "in_progress", output: { title: "Gittensory Gate is evaluating" } }); + expect(body).toMatchObject({ name: "Gittensory Orb Review Agent", status: "in_progress", output: { title: "Gittensory Orb Review Agent is evaluating" } }); expect(body.conclusion).toBeUndefined(); calls.gateChecks += 1; return Response.json({ id: 900 }, { status: 201 }); @@ -2049,7 +2049,7 @@ describe("queue processors", () => { if (url.includes("/check-runs/900") && (init?.method ?? "GET") === "PATCH") { const body = JSON.parse(String(init?.body ?? "{}")) as { name?: string; status?: string; conclusion?: string; output?: { title?: string } }; // Non-confirmed author + linked-issue block + no issue → gated normally → failure (#gate-nonconfirmed). - expect(body).toMatchObject({ name: "Gittensory Gate", status: "completed", conclusion: "failure", output: { title: "Gittensory Gate: No linked issue detected" } }); + expect(body).toMatchObject({ name: "Gittensory Orb Review Agent", status: "completed", conclusion: "failure", output: { title: "Gittensory Orb Review Agent: No linked issue detected" } }); calls.gateChecks += 1; return Response.json({ id: 900 }); } @@ -2329,7 +2329,7 @@ describe("queue processors", () => { let gateConclusion: string | undefined; let gateText = ""; const captureGate = (body: { name?: string; conclusion?: string; output?: { title?: string; summary?: string } }) => { - if ((body.name ?? "").includes("Gittensory Gate") && body.conclusion) { + if ((body.name ?? "").includes("Gittensory Orb Review Agent") && body.conclusion) { gateConclusion = body.conclusion; gateText = `${body.output?.title ?? ""} ${body.output?.summary ?? ""}`; } @@ -3088,7 +3088,7 @@ describe("queue processors", () => { } if (url.includes("/check-runs") && method === "POST") { const body = JSON.parse(String(init?.body ?? "{}")) as { status?: string; conclusion?: string; output?: { title?: string } }; - expect(body).toMatchObject({ status: "in_progress", output: { title: "Gittensory Gate is evaluating" } }); + expect(body).toMatchObject({ status: "in_progress", output: { title: "Gittensory Orb Review Agent is evaluating" } }); expect(body.conclusion).toBeUndefined(); calls.gateChecks += 1; return Response.json({ id: 910 }, { status: 201 }); @@ -3096,7 +3096,7 @@ describe("queue processors", () => { if (url.includes("/check-runs/910") && method === "PATCH") { const body = JSON.parse(String(init?.body ?? "{}")) as { status?: string; conclusion?: string; output?: { title?: string } }; // The bot author is gated normally now (no confirmation gate); linked-issue block + no issue → failure (#gate-nonconfirmed). - expect(body).toMatchObject({ status: "completed", conclusion: "failure", output: { title: "Gittensory Gate: No linked issue detected" } }); + expect(body).toMatchObject({ status: "completed", conclusion: "failure", output: { title: "Gittensory Orb Review Agent: No linked issue detected" } }); calls.gateChecks += 1; return Response.json({ id: 910 }); } @@ -3160,7 +3160,7 @@ describe("queue processors", () => { } if (url.includes("/check-runs") && method === "POST") { const body = JSON.parse(String(init?.body ?? "{}")) as { status?: string; conclusion?: string; output?: { title?: string } }; - expect(body).toMatchObject({ status: "in_progress", output: { title: "Gittensory Gate is evaluating" } }); + expect(body).toMatchObject({ status: "in_progress", output: { title: "Gittensory Orb Review Agent is evaluating" } }); expect(body.conclusion).toBeUndefined(); calls.gateChecks += 1; return Response.json({ id: 920 }, { status: 201 }); @@ -3168,7 +3168,7 @@ describe("queue processors", () => { if (url.includes("/check-runs/920") && method === "PATCH") { const body = JSON.parse(String(init?.body ?? "{}")) as { status?: string; conclusion?: string; output?: { title?: string } }; // The unconfirmed miner is gated normally now; linked-issue block + no issue → failure (#gate-nonconfirmed). - expect(body).toMatchObject({ status: "completed", conclusion: "failure", output: { title: "Gittensory Gate: No linked issue detected" } }); + expect(body).toMatchObject({ status: "completed", conclusion: "failure", output: { title: "Gittensory Orb Review Agent: No linked issue detected" } }); calls.gateChecks += 1; return Response.json({ id: 920 }); } @@ -3332,7 +3332,7 @@ describe("queue processors", () => { expect(calls.minerList).toBe(1); expect(calls.gateChecks).toBe(2); expect(gatePatchBody.conclusion).toBe("failure"); - expect(gatePatchBody.output?.title).toBe("Gittensory Gate: No linked issue detected"); + expect(gatePatchBody.output?.title).toBe("Gittensory Orb Review Agent: No linked issue detected"); }); it("hard-blocks a confirmed contributor on a dual-model AI consensus defect when aiReview: block is opted in", async () => { @@ -3473,7 +3473,7 @@ describe("queue processors", () => { const finalize = patchBodies[1]; expect(finalize?.status).toBe("completed"); expect(finalize?.conclusion).toBe("neutral"); - expect(finalize?.output?.title).toBe("Gittensory Gate — could not finish evaluating"); + expect(finalize?.output?.title).toBe("Gittensory Orb Review Agent — could not finish evaluating"); const audit = await env.DB.prepare("select outcome from audit_events where event_type = ? and target_key = ?") .bind("github_app.gate_check_failed_nonfatal", "JSONbored/gittensory#80") .first<{ outcome: string }>(); @@ -3663,7 +3663,7 @@ describe("queue processors", () => { if (url.includes("/commits/closed123/check-runs")) return Response.json({ total_count: 0, check_runs: [] }); if (url.includes("/check-runs") && method === "POST") { const body = JSON.parse(String(init?.body ?? "{}")) as { name?: string; status?: string; conclusion?: string; output?: { title?: string } }; - expect(body).toMatchObject({ name: "Gittensory Gate", status: "completed", conclusion: "skipped", output: { title: "Gittensory Gate skipped" } }); + expect(body).toMatchObject({ name: "Gittensory Orb Review Agent", status: "completed", conclusion: "skipped", output: { title: "Gittensory Orb Review Agent skipped" } }); calls.gateWrites += 1; return Response.json({ id: 901 }, { status: 201 }); } @@ -7361,7 +7361,7 @@ describe("queue processors", () => { } if (url.includes("/commits/override-sha/check-runs") && method === "GET") { calls.checkGets += 1; - return Response.json({ total_count: 1, check_runs: [{ id: 555, name: "Gittensory Gate" }] }); + return Response.json({ total_count: 1, check_runs: [{ id: 555, name: "Gittensory Orb Review Agent" }] }); } if (url.includes("/check-runs/555") && method === "PATCH") { calls.checkPatches += 1; @@ -7400,9 +7400,9 @@ describe("queue processors", () => { const finalize = patchBodies[0]; expect(finalize?.status).toBe("completed"); expect(finalize?.conclusion).toBe("neutral"); - expect(finalize?.output?.title).toBe("Gittensory Gate — overridden by @maintainer"); + expect(finalize?.output?.title).toBe("Gittensory Orb Review Agent — overridden by @maintainer"); expect(finalize?.output?.text).toContain("Overridden by @maintainer: known flaky duplicate check, shipping"); - expect(confirmationBody).toContain("Gittensory Gate overridden by @maintainer"); + expect(confirmationBody).toContain("Gittensory Orb Review Agent overridden by @maintainer"); const audit = await env.DB.prepare("select event_type, actor, target_key, outcome, detail from audit_events where event_type = ?") .bind("github_app.gate_overridden") .first<{ event_type: string; actor: string; target_key: string; outcome: string; detail: string }>(); @@ -7455,7 +7455,7 @@ describe("queue processors", () => { } if (url.includes("/commits/live-sha/check-runs") && method === "GET") { seen.liveCheckGets += 1; - return Response.json({ total_count: 1, check_runs: [{ id: 556, name: "Gittensory Gate" }] }); + return Response.json({ total_count: 1, check_runs: [{ id: 556, name: "Gittensory Orb Review Agent" }] }); } if (url.includes("/check-runs/556") && method === "PATCH") { patchBodies.push(JSON.parse(String(init?.body ?? "{}")) as { conclusion?: string }); @@ -7585,7 +7585,7 @@ describe("queue processors", () => { } if (url.includes("/check-runs")) { calls.checkRuns += 1; - return Response.json({ total_count: 1, check_runs: [{ id: 556, name: "Gittensory Gate" }] }); + return Response.json({ total_count: 1, check_runs: [{ id: 556, name: "Gittensory Orb Review Agent" }] }); } if (url.includes("/comments")) { calls.comments += 1; diff --git a/test/unit/rules.test.ts b/test/unit/rules.test.ts index 06fdda59e0..f504f56058 100644 --- a/test/unit/rules.test.ts +++ b/test/unit/rules.test.ts @@ -224,7 +224,7 @@ describe("advisory rules", () => { expect(gate.conclusion).toBe("success"); expect(gate.blockers).toEqual([]); expect(gate.warnings.map((finding) => finding.code)).not.toContain("busy_pr_queue"); - expect(output.title).toBe("Gittensory Gate passed"); + expect(output.title).toBe("Gittensory Orb Review Agent passed"); expect(output.text).toContain("No configured hard blocker"); }); @@ -238,7 +238,7 @@ describe("advisory rules", () => { expect(advisory.findings.map((finding) => finding.code)).toEqual(expect.arrayContaining(["repo_not_registered", "pr_not_cached"])); expect(gate.conclusion).toBe("neutral"); expect(gate.blockers).toEqual([]); - expect(output.title).toBe("Gittensory Gate — not evaluated yet"); + expect(output.title).toBe("Gittensory Orb Review Agent — not evaluated yet"); expect(output.summary).toContain("re-evaluates automatically"); expect(output.text).toBe("Gittensory did not create a contributor-facing failure for this event."); }); @@ -358,7 +358,7 @@ describe("advisory rules", () => { expect(gate.conclusion).toBe("failure"); // Title names the blocker count; summary enumerates every active blocker with its fix. - expect(gate.title).toBe("Gittensory Gate: 2 blockers"); + expect(gate.title).toBe("Gittensory Orb Review Agent: 2 blockers"); expect(gate.summary).toContain("No linked issue detected"); expect(gate.summary).toContain("Linked issue overlaps another open PR"); expect(gate.summary).not.toContain("Readiness score is below the configured threshold"); @@ -376,7 +376,7 @@ describe("advisory rules", () => { // neutral/held state. Confirmed-status affects only on-chain scoring, never the gate verdict. (#gate-nonconfirmed) const nonConfirmed = evaluateGateCheck(blockingAdvisory, { duplicatePrGateMode: "block", confirmedContributor: false }); expect(nonConfirmed.conclusion).toBe("failure"); - expect(nonConfirmed.title).toBe("Gittensory Gate: Linked issue overlaps another open PR"); + expect(nonConfirmed.title).toBe("Gittensory Orb Review Agent: Linked issue overlaps another open PR"); expect(nonConfirmed.blockers.map((finding) => finding.code)).toEqual(["duplicate_pr_risk"]); // Confirmed author with the same blocker: identical verdict. @@ -394,7 +394,7 @@ describe("advisory rules", () => { const output = formatGateCheckOutput({ enabled: true, conclusion, - title: conclusion === "skipped" ? "Gittensory Gate skipped" : "Gittensory Gate neutral", + title: conclusion === "skipped" ? "Gittensory Orb Review Agent skipped" : "Gittensory Orb Review Agent neutral", summary: "PR closed before full evaluation.", blockers: [], warnings: [], @@ -409,7 +409,7 @@ describe("advisory rules", () => { const output = formatGateCheckOutput({ enabled: true, conclusion: "failure", - title: "Gittensory Gate is blocking merge", + title: "Gittensory Orb Review Agent is blocking merge", summary: "A configured merge-blocking issue was found.", blockers: [], warnings: [], @@ -1083,7 +1083,7 @@ describe("CI-refutation of the public comment gate (#ai-ci-refutation)", () => { const failure = (codes: string[]): import("../../src/rules/advisory").GateCheckEvaluation => ({ enabled: true, conclusion: "failure", - title: "Gittensory Gate: blocked", + title: "Gittensory Orb Review Agent: blocked", summary: "A hard blocker was found.", blockers: codes.map(finding), warnings: [], @@ -1106,7 +1106,7 @@ describe("CI-refutation of the public comment gate (#ai-ci-refutation)", () => { const out = reconcileGateEvaluationForGreenCi(failure(["ai_consensus_defect"]), "passed", true); expect(out.conclusion).toBe("success"); expect(out.blockers).toEqual([]); - expect(out.title).toBe("Gittensory Gate passed"); + expect(out.title).toBe("Gittensory Orb Review Agent passed"); expect(out.summary).toContain("advisory, not blocking"); }); diff --git a/test/unit/settings-preview.test.ts b/test/unit/settings-preview.test.ts index 1f2411e573..2c8a181c61 100644 --- a/test/unit/settings-preview.test.ts +++ b/test/unit/settings-preview.test.ts @@ -257,9 +257,9 @@ describe("buildRepoSettingsPreview", () => { }); expect(preview.decision).toMatchObject({ skipped: false, actions: ["none"] }); - expect(preview.warnings.some((warning) => /Gate checks are enabled.*Checks: write/.test(warning))).toBe(true); + expect(preview.warnings.some((warning) => /Review-agent checks are enabled.*Checks: write/.test(warning))).toBe(true); expect(preview.installPreview.permissions).toMatchObject({ status: "needs_attention", missing: ["checks"] }); - expect(preview.installPreview.publicOutputs).toEqual(expect.arrayContaining(["Opt-in Gittensory Gate check run."])); + expect(preview.installPreview.publicOutputs).toEqual(expect.arrayContaining(["Opt-in Gittensory Orb Review Agent check run."])); }); it("shows a quiet skip for a non-miner author with no rendered comment", () => { diff --git a/test/unit/signals-coverage.test.ts b/test/unit/signals-coverage.test.ts index 3cb7d7f0c6..0209dd1610 100644 --- a/test/unit/signals-coverage.test.ts +++ b/test/unit/signals-coverage.test.ts @@ -810,11 +810,11 @@ describe("signal coverage edge cases", () => { // The comment builder must agree by construction: ON winner is NOT a blocking-merge panel; ON loser is. const winnerComment = buildPublicPrIntelligenceComment({ ...baseFor(winnerPr), duplicateWinnerEnabled: true }); const loserComment = buildPublicPrIntelligenceComment({ ...baseFor(loserPr), duplicateWinnerEnabled: true }); - expect(winnerComment).not.toContain("Gittensory Gate is blocking merge"); - expect(loserComment).toContain("Gittensory Gate is blocking merge"); + expect(winnerComment).not.toContain("Gittensory Orb Review Agent is blocking merge"); + expect(loserComment).toContain("Gittensory Orb Review Agent is blocking merge"); // Flag OFF on the winner is byte-identical to a blocking panel (today's behavior). const offWinnerComment = buildPublicPrIntelligenceComment(baseFor(winnerPr)); - expect(offWinnerComment).toContain("Gittensory Gate is blocking merge"); + expect(offWinnerComment).toContain("Gittensory Orb Review Agent is blocking merge"); }); it("renders opt-in gate panel states for collision and repo evaluation blockers", () => { diff --git a/test/unit/unified-comment-bridge.test.ts b/test/unit/unified-comment-bridge.test.ts index 364bac024f..6447eb2adc 100644 --- a/test/unit/unified-comment-bridge.test.ts +++ b/test/unit/unified-comment-bridge.test.ts @@ -22,7 +22,7 @@ function gate(over: Partial = {}): GateCheckEvaluation { return { enabled: true, conclusion: "success", - title: "Gittensory Gate passed", + title: "Gittensory Orb Review Agent passed", summary: "No configured hard blocker was found.", blockers: [], warnings: [], @@ -265,7 +265,7 @@ describe("buildUnifiedCommentBody", () => { const failing = buildUnifiedCommentBody({ gate: gate({ conclusion: "failure", - title: "Gittensory Gate: blocked", + title: "Gittensory Orb Review Agent: blocked", summary: "A hard blocker was found.", blockers: [{ code: "ai_consensus_defect", severity: "critical", title: "Real bug", detail: "..." }], }), @@ -339,7 +339,7 @@ describe("buildUnifiedCommentBody", () => { expect(held).not.toContain("> [!TIP]"); }); - it("neverClosed renders a gate-failure (close) PR as HELD, not reject/close (#8/#9)", () => { + it("neverClosed renders a gate-failure (close) PR as HELD when CI is green, not reject/close (#8/#9)", () => { const args = { gate: gate({ conclusion: "failure" }), panelRows, readinessTotal: 40, changedFiles: 2, mergeReadiness: { ciState: "passed" as const }, footerMarkdown: footer }; // A contributor close → the red reject/close recommendation. const closed = buildUnifiedCommentBody(args); @@ -350,6 +350,22 @@ describe("buildUnifiedCommentBody", () => { expect(held).toContain("Suggested Action - Manual Review"); expect(held).not.toContain("Suggested Action - Reject/Close"); }); + + it("neverClosed still renders failed CI as a red manual-review result", () => { + const body = buildUnifiedCommentBody({ + gate: gate({ conclusion: "failure" }), + panelRows, + readinessTotal: 40, + changedFiles: 2, + mergeReadiness: { ciState: "failed", failingChecks: ["test"] }, + footerMarkdown: footer, + neverClosed: true, + }); + expect(body).toContain("> [!CAUTION]"); + expect(body).toContain("Suggested Action - Manual Review"); + expect(body).toContain("CI checks failing"); + expect(body).not.toContain("Suggested Action - Reject/Close"); + }); }); // ── Reconciliation invariant (#1016): comment-verdict ↔ gate-conclusion alignment ────────────────── @@ -435,7 +451,7 @@ describe("single AI pass: the bridge RECOVERS the consensus defect, never re-der const body = buildUnifiedCommentBody({ gate: gate({ conclusion: "failure", - title: "Gittensory Gate: blocked", + title: "Gittensory Orb Review Agent: blocked", summary: "A hard blocker was found.", // The gate's own blockers list carries the defect (as evaluateGateCheck produced it)… blockers: [{ code: "ai_consensus_defect", severity: "critical", title: defectTitle, detail: "Both models agree." }], @@ -476,7 +492,7 @@ describe("gate blockers render in 'Why this is blocked' (FIX D1)", () => { const body = buildUnifiedCommentBody({ gate: gate({ conclusion: "failure", - title: "Gittensory Gate: blocked", + title: "Gittensory Orb Review Agent: blocked", summary: "A hard blocker was found.", // A non-AI gate failure (no ai_consensus_defect anywhere) — the consensus defect alone would have left // "Why this is blocked" empty. The gate blocker must now render. @@ -553,7 +569,7 @@ describe("gate blockers render in 'Why this is blocked' (FIX D1)", () => { describe("verdictReason on a held/blocked headline (FIX D2)", () => { it("appends the gate summary to a BLOCKED (close) verdict headline", () => { const body = buildUnifiedCommentBody({ - gate: gate({ conclusion: "failure", title: "Gittensory Gate: blocked", summary: "A hard blocker was found." }), + gate: gate({ conclusion: "failure", title: "Gittensory Orb Review Agent: blocked", summary: "A hard blocker was found." }), panelRows, readinessTotal: 30, changedFiles: 2, @@ -565,7 +581,7 @@ describe("verdictReason on a held/blocked headline (FIX D2)", () => { it("appends the gate summary to a HELD (manual) verdict headline", () => { const body = buildUnifiedCommentBody({ - gate: gate({ conclusion: "action_required", title: "Gittensory Gate — needs review", summary: "Manual maintainer review required." }), + gate: gate({ conclusion: "action_required", title: "Gittensory Orb Review Agent — needs review", summary: "Manual maintainer review required." }), panelRows, readinessTotal: 55, changedFiles: 2, @@ -577,18 +593,18 @@ describe("verdictReason on a held/blocked headline (FIX D2)", () => { it("falls back to the gate TITLE when the summary is empty", () => { const body = buildUnifiedCommentBody({ - gate: gate({ conclusion: "failure", title: "Gittensory Gate: blocked by policy", summary: " " }), + gate: gate({ conclusion: "failure", title: "Gittensory Orb Review Agent: blocked by policy", summary: " " }), panelRows, readinessTotal: 20, changedFiles: 2, footerMarkdown: footer, }); - expect(body).toContain("Gittensory Gate: blocked by policy"); + expect(body).toContain("Gittensory Orb Review Agent: blocked by policy"); }); it("does NOT overwrite the positive ready wording on a passing (merge) verdict", () => { const body = buildUnifiedCommentBody({ - gate: gate({ conclusion: "success", title: "Gittensory Gate passed", summary: "No configured hard blocker was found." }), + gate: gate({ conclusion: "success", title: "Gittensory Orb Review Agent passed", summary: "No configured hard blocker was found." }), panelRows, readinessTotal: 90, changedFiles: 2, @@ -659,7 +675,7 @@ describe("privacy invariant: the private 'Maintainer notes' internals never reac const body = buildUnifiedCommentBody({ gate: gate({ conclusion: "failure", - title: "Gittensory Gate: blocked", + title: "Gittensory Orb Review Agent: blocked", summary: "A hard blocker was found.", blockers: [ { code: "ai_consensus_defect", severity: "critical", title: "Real bug", detail: "Both agree." }, @@ -756,7 +772,7 @@ describe("buildClosedUnifiedCommentBody (closed/skipped PR through the unified r it("renders the non-blocking skipped state (skipped → comment verdict → advisory, not a CAUTION block)", () => { const body = buildClosedUnifiedCommentBody({ repoFullName: "octo/repo", pullNumber: 7, footerMarkdown: footer }); // skipped maps to the `comment` verdict (gateConclusionToVerdict) → advisory tone, mirroring the legacy - // "[!NOTE] Gittensory Gate skipped" panel. It must NOT read as a blocked/closed CAUTION. + // "[!NOTE] Gittensory Orb Review Agent skipped" panel. It must NOT read as a blocked/closed CAUTION. expect(body).not.toContain("> [!CAUTION]"); expect(body).toContain("Skipped"); expect(body).toContain("octo/repo#7 is no longer open."); diff --git a/test/unit/unified-comment-parity.test.ts b/test/unit/unified-comment-parity.test.ts index a9fef704d8..fdbd3a08c8 100644 --- a/test/unit/unified-comment-parity.test.ts +++ b/test/unit/unified-comment-parity.test.ts @@ -93,7 +93,7 @@ function gate(over: Partial = {}): GateCheckEvaluation { return { enabled: true, conclusion: "success", - title: "Gittensory Gate passed", + title: "Gittensory Orb Review Agent passed", summary: "No configured hard blocker was found.", blockers: [], warnings: [], diff --git a/test/unit/unified-comment.test.ts b/test/unit/unified-comment.test.ts index dcd83da921..da60fdcc9e 100644 --- a/test/unit/unified-comment.test.ts +++ b/test/unit/unified-comment.test.ts @@ -38,10 +38,10 @@ describe("deriveUnifiedStatus", () => { expect(deriveUnifiedStatus({ ...base, recommendations: ["request_changes"] })).toBe("held"); }); - it("CI readiness is advisory for Gittensory — failed/pending holds, but never blocks a merge verdict", () => { - // Red CI must never render "safe to merge", but CI itself is not a Gittensory blocker. - expect(deriveUnifiedStatus({ ...base, readiness: { ciState: "failed" } })).toBe("held"); - expect(deriveUnifiedStatus({ ...base, decision: "merge", readiness: { ciState: "failed" } })).toBe("held"); + it("failed CI is a failing review result; pending CI holds but does not block", () => { + // Red CI must never render "safe to merge"; it is a failing review result even if the PR cannot auto-close. + expect(deriveUnifiedStatus({ ...base, readiness: { ciState: "failed" } })).toBe("blocked"); + expect(deriveUnifiedStatus({ ...base, decision: "merge", readiness: { ciState: "failed" } })).toBe("blocked"); // CI still running / not yet reported (chip "CI pending") → HELD, never "safe to merge". expect(deriveUnifiedStatus({ ...base, decision: "merge", readiness: { ciState: "unverified" } })).toBe("held"); // ONLY green CI + a merge verdict renders ready. @@ -81,10 +81,10 @@ describe("deriveUnifiedStatus", () => { expect(deriveUnifiedStatus({ ...base, decision: "merge", readiness: { ciState: "passed" } }, { heldForReview: false })).toBe("ready"); }); - it("renders a non-closing disposition as held, not Closed (#8/#9)", () => { - // #9: an owner / automation-bot author is NEVER auto-closed → a gate "close" verdict renders held, even on red CI. + it("renders a non-closing disposition as held, but still red when CI failed (#8/#9)", () => { + // #9: an owner / automation-bot author is NEVER auto-closed → a gate "close" verdict renders held while CI is green/unknown. expect(deriveUnifiedStatus({ ...base, decision: "close" }, { neverClosed: true })).toBe("held"); - expect(deriveUnifiedStatus({ ...base, decision: "close", readiness: { ciState: "failed" } }, { neverClosed: true })).toBe("held"); + expect(deriveUnifiedStatus({ ...base, decision: "close", readiness: { ciState: "failed" } }, { neverClosed: true })).toBe("blocked"); // #8: a guarded-path close is the disposition's HOLD (owner review) unless a red required check forces it. expect(deriveUnifiedStatus({ ...base, decision: "close", readiness: { ciState: "passed" } }, { heldForReview: true })).toBe("held"); expect(deriveUnifiedStatus({ ...base, decision: "close" }, { heldForReview: true })).toBe("held"); // CI not yet reported → held @@ -125,6 +125,7 @@ describe("renderUnifiedReviewComment", () => { expect(md).toContain("🟩"); expect(md).toContain("Gittensory review result - approve/merge recommended · auto-merged"); expect(md).toContain("Suggested Action - Approve/Merge"); + expect(md).toContain("- auto-merged"); expect(md).toContain("`2 files`"); expect(md).toContain("`2 AI reviewers`"); expect(md).toContain("`no blockers`"); @@ -211,6 +212,8 @@ describe("renderUnifiedReviewComment", () => { { reviewedAt: "2026-06-29T08:05:59.852Z" }, ); expect(md).toContain("Review updated: 2026-06-29 08:05:59 UTC"); + expect(md.indexOf("Gittensory review result")).toBeLessThan(md.indexOf("Review updated:")); + expect(md.indexOf("Review updated:")).toBeLessThan(md.indexOf("`2 files`")); expect(renderUnifiedReviewComment({ ...base, decision: "merge" }, { reviewedAt: "not-a-date" })).not.toContain("Review updated:"); }); @@ -239,13 +242,16 @@ describe("renderUnifiedReviewComment", () => { it("a blocked status from reviewer recs (no close decision) reads 'blocked', not 'closed'", () => { const md = renderUnifiedReviewComment({ ...base, recommendations: ["close"], blockers: ["Leaks a token."], consensusBlocker: true }, {}); expect(md).toContain("> [!CAUTION]"); - expect(md).toContain("Gittensory review result - blockers found"); // headlineLabel(): decision !== "close" + expect(md).toContain("Gittensory review result - fixes required"); // headlineLabel(): decision !== "close" expect(md).toContain("**🛑 Suggested Action - Fix Blockers**"); // verdictLine(): decision !== "close" expect(md).not.toContain("Suggested Action - Reject/Close"); }); it("renders CI-failing / CI-pending chips and the merge-state label", () => { const failing = renderUnifiedReviewComment({ ...base, readiness: { ciState: "failed", mergeStateLabel: "behind" } }, {}); + expect(failing).toContain("> [!CAUTION]"); + expect(failing).toContain("Gittensory review result - fixes required"); + expect(failing).toContain("Suggested Action - Fix Blockers"); expect(failing).toContain("`CI failing`"); expect(failing).toContain("`behind`"); const pending = renderUnifiedReviewComment({ ...base, readiness: { ciState: "unverified" } }, {}); @@ -295,16 +301,41 @@ describe("renderUnifiedReviewComment", () => { }); it("appends an explicit verdict reason across ready (merged + unmerged) and advisory states", () => { - // The verdict word is bolded (`**…**`); the reason follows outside the bold, so assert each separately. const merged = renderUnifiedReviewComment({ ...base, decision: "merge", merged: true, verdictReason: "all checks green" }, {}); - expect(merged).toContain("Suggested Action - Approve/Merge"); - expect(merged).toContain("all checks green"); // verdictReason appended, not the default " — all checks passed" + expect(merged).toContain("**✅ Suggested Action - Approve/Merge**"); + expect(merged).toContain("- all checks green"); const unmerged = renderUnifiedReviewComment({ ...base, decision: "merge", verdictReason: "looks correct" }, {}); expect(unmerged).not.toContain("auto-merged"); // the unmerged ready variant - expect(unmerged).toContain("looks correct"); + expect(unmerged).toContain("**✅ Suggested Action - Approve/Merge**"); + expect(unmerged).toContain("- looks correct"); const advisory = renderUnifiedReviewComment({ ...base, decision: "comment", recommendations: [], verdictReason: "for your awareness" }, {}); - expect(advisory).toContain("Suggested Action - Advisory Only"); - expect(advisory).toContain("for your awareness"); + expect(advisory).toContain("**💡 Suggested Action - Advisory Only**"); + expect(advisory).toContain("- for your awareness"); + }); + + it("renders suggested-action reasons as bullets below the action line", () => { + const md = renderUnifiedReviewComment( + { + ...base, + decision: "manual", + recommendations: ["manual_review"], + verdictReason: "Touches a guarded path — held for manual review; Touches a maintainer-blocked path — held for manual review", + }, + {}, + ); + expect(md).toContain("**⏸️ Suggested Action - Manual Review**"); + expect(md).toContain("- Touches a guarded path — held for manual review"); + expect(md).toContain("- Touches a maintainer-blocked path — held for manual review"); + expect(md).not.toContain("Suggested Action - Manual Review — Touches"); + }); + + it("renders non-closable failed-CI reviews as red manual-review actions, not reject/close", () => { + const md = renderUnifiedReviewComment({ ...base, decision: "close", readiness: { ciState: "failed" } }, { neverClosed: true }); + expect(md).toContain("> [!CAUTION]"); + expect(md).toContain("Gittensory review result - fixes required"); + expect(md).toContain("Suggested Action - Manual Review"); + expect(md).toContain("`CI failing`"); + expect(md).not.toContain("Suggested Action - Reject/Close"); }); it("skips empty blocker lines and caps long nit lists at 12", () => { diff --git a/test/unit/visual-collapsible.test.ts b/test/unit/visual-collapsible.test.ts index 146ed8a135..165f412bcb 100644 --- a/test/unit/visual-collapsible.test.ts +++ b/test/unit/visual-collapsible.test.ts @@ -8,7 +8,7 @@ function gate(over: Partial = {}): GateCheckEvaluation { return { enabled: true, conclusion: "success", - title: "Gittensory Gate passed", + title: "Gittensory Orb Review Agent passed", summary: "No configured hard blocker was found.", blockers: [], warnings: [], From a9fefe806f4ffcd24e44bdca99404bd015b9e2a4 Mon Sep 17 00:00:00 2001 From: JSONbored <49853598+JSONbored@users.noreply.github.com> Date: Mon, 29 Jun 2026 04:44:28 -0700 Subject: [PATCH 53/68] fix(review): complete legacy gate checks after rename --- src/github/app.ts | 72 ++++++++++++++++++++++++++++++++++-- test/unit/github-app.test.ts | 67 +++++++++++++++++++++++++++++++++ test/unit/queue.test.ts | 8 +++- 3 files changed, 143 insertions(+), 4 deletions(-) diff --git a/src/github/app.ts b/src/github/app.ts index 0d315d4ea7..22a6adbfa2 100644 --- a/src/github/app.ts +++ b/src/github/app.ts @@ -21,6 +21,7 @@ import { import { GITTENSORY_CONTEXT_CHECK_NAME, GITTENSORY_GATE_CHECK_NAME, + GITTENSORY_LEGACY_GATE_CHECK_NAME, } from "../review/check-names"; export { @@ -530,6 +531,7 @@ export async function createOrUpdateGateCheckRun( conclusion: gate.conclusion, output: formatGateCheckOutput(gate), checkRunId: options.checkRunId, + supersedeLegacyNames: [GITTENSORY_LEGACY_GATE_CHECK_NAME], mode, }, ); @@ -557,6 +559,7 @@ export async function createOrUpdatePendingGateCheckRun( text: "The review agent blocks every author on the repo's configured hard blockers (duplicate PRs by default); on everything else, and while state is still syncing, it stays advisory.", }, updateExisting: "in_progress_only", + supersedeLegacyNames: [GITTENSORY_LEGACY_GATE_CHECK_NAME], mode, }, ); @@ -584,6 +587,7 @@ export async function createOrUpdateSkippedGateCheckRun( summary: reason, text: "Gittensory does not post late first comments on closed or merged pull requests.", }, + supersedeLegacyNames: [GITTENSORY_LEGACY_GATE_CHECK_NAME], mode, }, ); @@ -620,6 +624,7 @@ export async function createOrUpdateErroredGateCheckRun( text: "Gittensory finalizes the review-agent check to a neutral, non-blocking state when evaluation is interrupted, so the check never hangs in_progress. Push a new commit or use the 'Re-run Gittensory review' checkbox to re-evaluate.", }, checkRunId: options.checkRunId, + supersedeLegacyNames: [GITTENSORY_LEGACY_GATE_CHECK_NAME], mode, }, ); @@ -655,6 +660,7 @@ export async function createOrUpdateOverriddenGateCheckRun( text: `Overridden by @${options.actor}: ${options.reason}`, }, checkRunId: options.checkRunId, + supersedeLegacyNames: [GITTENSORY_LEGACY_GATE_CHECK_NAME], mode, }, ); @@ -672,6 +678,7 @@ async function createOrUpdateNamedCheckRun( output: CheckRunOutput; checkRunId?: number | undefined; updateExisting?: "any" | "in_progress_only" | "never" | undefined; + supersedeLegacyNames?: readonly string[] | undefined; mode?: AgentActionMode | undefined; }, ): Promise { @@ -740,11 +747,70 @@ async function createOrUpdateNamedCheckRun( return null; } }; + const finalizeLegacyPendingCheckRuns = async (): Promise => { + const legacyNames = check.supersedeLegacyNames ?? []; + if (legacyNames.length === 0 || check.checkRunId) return; + for (const legacyName of legacyNames) { + try { + const existing = await octokit.request( + "GET /repos/{owner}/{repo}/commits/{ref}/check-runs", + { + owner, + repo, + ref: headSha, + check_name: legacyName, + filter: "latest", + per_page: 1, + }, + ); + const legacyRun = (existing.data as CheckRunListResponse) + .check_runs?.[0]; + if ( + !legacyRun || + (legacyRun.name && legacyRun.name !== legacyName) || + (legacyRun.status ?? "").toLowerCase() === "completed" + ) + continue; + await octokit.request( + "PATCH /repos/{owner}/{repo}/check-runs/{check_run_id}", + { + owner, + repo, + check_run_id: legacyRun.id, + name: legacyName, + status: "completed", + conclusion: "neutral", + output: outputForCheckRunUpdate({ + title: `${GITTENSORY_GATE_CHECK_NAME} superseded this legacy check`, + summary: + "This legacy check name was completed after the review-agent check was renamed.", + text: `Use ${GITTENSORY_GATE_CHECK_NAME} for current Gittensory review results.`, + }), + ...detailsUrlBody, + }, + ); + } catch (error) { + console.warn( + JSON.stringify({ + level: "warn", + event: "legacy_gate_check_finalize_failed", + repository: `${owner}/${repo}`, + legacyName, + error: errorMessage(error), + }), + ); + } + } + }; + const finish = async (outcome: CheckRunOutcome): Promise => { + await finalizeLegacyPendingCheckRuns(); + return outcome; + }; try { if (check.checkRunId) { const out = await patchCheckRun(check.checkRunId); - if (out) return out; + if (out) return await finish(out); } else if (check.updateExisting !== "never") { const existing = await octokit.request( "GET /repos/{owner}/{repo}/commits/{ref}/check-runs", @@ -765,10 +831,10 @@ async function createOrUpdateNamedCheckRun( (existingCheckRun.status ?? "").toLowerCase() !== "completed") ) { const out = await patchCheckRun(existingCheckRun.id); - if (out) return out; + if (out) return await finish(out); } } - return await postNewCheckRun(); + return await finish(await postNewCheckRun()); } catch (error) { if (isCheckRunPermissionError(error)) { // Capture the ACTUAL response (status + body). A 403 here is often NOT a real permission gap (the App has diff --git a/test/unit/github-app.test.ts b/test/unit/github-app.test.ts index 8c08fa47d1..2edd97c92c 100644 --- a/test/unit/github-app.test.ts +++ b/test/unit/github-app.test.ts @@ -830,6 +830,73 @@ describe("GitHub check runs", () => { ); }); + it("finalizes the legacy pending Gate check when posting the renamed review-agent check", async () => { + const privateKey = await generatePrivateKeyPem(); + let newCheckBody: { name?: string; status?: string; conclusion?: string } = {}; + let legacyPatchBody: { + name?: string; + status?: string; + conclusion?: string; + output?: { title?: string; text?: string }; + } = {}; + vi.stubGlobal( + "fetch", + async (input: RequestInfo | URL, init?: RequestInit) => { + const url = input.toString(); + const method = init?.method ?? "GET"; + if (url.includes("/access_tokens")) + return Response.json({ token: "installation-token" }); + if (url.includes("/commits/legacy-pending/check-runs")) { + const checkName = new URL(url).searchParams.get("check_name"); + if (checkName === "Gittensory Orb Review Agent") + return Response.json({ total_count: 0, check_runs: [] }); + if (checkName === "Gittensory Gate") + return Response.json({ + total_count: 1, + check_runs: [ + { id: 321, name: "Gittensory Gate", status: "in_progress" }, + ], + }); + } + if (url.includes("/check-runs/321") && method === "PATCH") { + legacyPatchBody = JSON.parse(String(init?.body)) as typeof legacyPatchBody; + return Response.json({ id: 321 }); + } + if (url.includes("/check-runs") && method === "POST") { + newCheckBody = JSON.parse(String(init?.body)) as typeof newCheckBody; + return Response.json({ id: 89 }, { status: 201 }); + } + return new Response("not found", { status: 404 }); + }, + ); + + const result = await createOrUpdatePendingGateCheckRun( + createTestEnv({ GITHUB_APP_PRIVATE_KEY: privateKey }), + 123, + "JSONbored/gittensory", + gateAdvisory("legacy-pending"), + ); + + expect(result).toMatchObject({ kind: "published", id: 89 }); + expect(newCheckBody).toMatchObject({ + name: "Gittensory Orb Review Agent", + status: "in_progress", + }); + expect(newCheckBody).not.toHaveProperty("conclusion"); + expect(legacyPatchBody).toMatchObject({ + name: "Gittensory Gate", + status: "completed", + conclusion: "neutral", + output: { + title: + "Gittensory Orb Review Agent superseded this legacy check", + }, + }); + expect(legacyPatchBody.output?.text).toContain( + "Use Gittensory Orb Review Agent", + ); + }); + it("omits details_url when the site origin cannot form a URL (#audit-details-url null arm)", async () => { const privateKey = await generatePrivateKeyPem(); let capturedBody: { details_url?: string } = {}; diff --git a/test/unit/queue.test.ts b/test/unit/queue.test.ts index c166731af1..6dee8c290b 100644 --- a/test/unit/queue.test.ts +++ b/test/unit/queue.test.ts @@ -7440,7 +7440,7 @@ describe("queue processors", () => { labels: [], body: "Validation: npm test", }); - const seen = { staleCheckGets: 0, liveCheckGets: 0 }; + const seen = { staleCheckGets: 0, liveCheckGets: 0, liveLegacyCheckGets: 0 }; const patchBodies: Array<{ conclusion?: string }> = []; vi.stubGlobal("fetch", async (input: RequestInfo | URL, init?: RequestInit) => { const url = input.toString(); @@ -7454,6 +7454,11 @@ describe("queue processors", () => { return Response.json({ total_count: 0, check_runs: [] }); } if (url.includes("/commits/live-sha/check-runs") && method === "GET") { + const checkName = new URL(url).searchParams.get("check_name"); + if (checkName === "Gittensory Gate") { + seen.liveLegacyCheckGets += 1; + return Response.json({ total_count: 0, check_runs: [] }); + } seen.liveCheckGets += 1; return Response.json({ total_count: 1, check_runs: [{ id: 556, name: "Gittensory Orb Review Agent" }] }); } @@ -7482,6 +7487,7 @@ describe("queue processors", () => { // The neutral PATCH targeted the LIVE head's Gate run (id 556), and the stale SHA was never touched. expect(seen.liveCheckGets).toBe(1); + expect(seen.liveLegacyCheckGets).toBe(1); expect(seen.staleCheckGets).toBe(0); expect(patchBodies[0]?.conclusion).toBe("neutral"); const audit = await env.DB.prepare("select metadata_json from audit_events where event_type = ?") From 554ab3023e77e0e159a8a93c8295693e5a6e1704 Mon Sep 17 00:00:00 2001 From: JSONbored <49853598+JSONbored@users.noreply.github.com> Date: Mon, 29 Jun 2026 05:16:56 -0700 Subject: [PATCH 54/68] test(selfhost): cover queue retry invariants --- src/selfhost/pg-queue.ts | 36 +++++----- src/selfhost/queue-common.ts | 7 +- src/selfhost/sqlite-queue.ts | 40 +++++------ test/unit/github-app.test.ts | 52 +++++++++++++++ test/unit/selfhost-queue-common.test.ts | 76 +++++++++++++++++++++ test/unit/selfhost-sqlite-queue.test.ts | 89 ++++++++++++++++++++++++- 6 files changed, 253 insertions(+), 47 deletions(-) diff --git a/src/selfhost/pg-queue.ts b/src/selfhost/pg-queue.ts index 8466c3c667..9075bff2ef 100644 --- a/src/selfhost/pg-queue.ts +++ b/src/selfhost/pg-queue.ts @@ -13,7 +13,6 @@ import { githubRateLimitRetryDelayMs, jobCoalesceKey, jobPriority, - nonConsumingRetryDelayMs, queueBackgroundConcurrency, queueProcessingTimeoutMs, queueRecoveryJitterMs, @@ -354,25 +353,22 @@ export function createPgQueue( } catch (error) { const attempts = Number(job.attempts) + 1; const errMsg = error instanceof Error ? error.message : "unknown error"; - const nonConsumingDelayMs = nonConsumingRetryDelayMs(error); - if (nonConsumingDelayMs !== null) { - const rateLimited = githubRateLimitRetryDelayMs(error) !== null; + const rateLimitDelayMs = githubRateLimitRetryDelayMs(error); + if (rateLimitDelayMs !== null) { const now = Date.now(); - const retryAfter = now + (rateLimited ? rateLimitRetryDelayWithJitter(nonConsumingDelayMs, `${job.job_key ?? ""}:${job.id}:${job.payload}`) : nonConsumingDelayMs); - if (rateLimited) { - githubRateLimitCooldownUntil = Math.max(githubRateLimitCooldownUntil, now + nonConsumingDelayMs); - const deferred = await deferPendingJobsForRateLimit(nonConsumingDelayMs, now); - if (deferred) { - await recordQueueMetric("gittensory_jobs_rate_limit_deferred_total", deferred); - console.warn( - JSON.stringify({ - level: "warn", - event: "selfhost_queue_rate_limit_cooldown", - deferred, - cooldown_until: githubRateLimitCooldownUntil, - }), - ); - } + const retryAfter = now + rateLimitRetryDelayWithJitter(rateLimitDelayMs, `${job.job_key ?? ""}:${job.id}:${job.payload}`); + githubRateLimitCooldownUntil = Math.max(githubRateLimitCooldownUntil, now + rateLimitDelayMs); + const deferred = await deferPendingJobsForRateLimit(rateLimitDelayMs, now); + if (deferred) { + await recordQueueMetric("gittensory_jobs_rate_limit_deferred_total", deferred); + console.warn( + JSON.stringify({ + level: "warn", + event: "selfhost_queue_rate_limit_cooldown", + deferred, + cooldown_until: githubRateLimitCooldownUntil, + }), + ); } if (job.job_key && (await mergeRescheduledJobIntoPending(job as JobRow & { job_key: string }, retryAfter, errMsg))) { await recordQueueMetric("gittensory_jobs_coalesced_total"); @@ -382,7 +378,7 @@ export function createPgQueue( [retryAfter, errMsg, job.id], ); } - await recordQueueMetric(rateLimited ? "gittensory_jobs_rate_limited_total" : "gittensory_jobs_deferred_total"); + await recordQueueMetric("gittensory_jobs_rate_limited_total"); logAudit({ event: "job_rate_limited", ts: Date.now(), diff --git a/src/selfhost/queue-common.ts b/src/selfhost/queue-common.ts index 447def3fbf..57180fa15b 100644 --- a/src/selfhost/queue-common.ts +++ b/src/selfhost/queue-common.ts @@ -12,8 +12,9 @@ export const FOREGROUND_QUEUE_PRIORITY_FLOOR = 8; // Webhook-driven work (a fresh PR -> its review) jumps ahead of heavy background jobs. Per-PR review refreshes // sit just below real webhooks, and sweep fan-out sits below those so stale surfaces are repaired during bursts. // Bot-generated comment edits are background noise; keeping them with real webhooks lets panel edits starve repair. +const AGENT_REGATE_PRIORITY = 9; const PRIORITY_BY_TYPE = new Map([ - ["agent-regate-pr", 9], + ["agent-regate-pr", AGENT_REGATE_PRIORITY], ["recapture-preview", 9], ["agent-regate-sweep", 8], ]); @@ -32,9 +33,9 @@ function agentRegatePriority(payload: string): number { typeof message.deliveryId === "string" ? message.deliveryId : ""; if (deliveryId.startsWith("manual-regate:")) return 99; } catch { - return PRIORITY_BY_TYPE.get("agent-regate-pr") ?? 0; + return AGENT_REGATE_PRIORITY; } - return PRIORITY_BY_TYPE.get("agent-regate-pr") ?? 0; + return AGENT_REGATE_PRIORITY; } export function isForegroundJobPriority(priority: number): boolean { diff --git a/src/selfhost/sqlite-queue.ts b/src/selfhost/sqlite-queue.ts index 810bdbbfd4..4b4d9fa175 100644 --- a/src/selfhost/sqlite-queue.ts +++ b/src/selfhost/sqlite-queue.ts @@ -14,7 +14,6 @@ import { githubRateLimitRetryDelayMs, jobCoalesceKey, jobPriority, - nonConsumingRetryDelayMs, queueBackgroundConcurrency, queueProcessingTimeoutMs, queueRecoveryJitterMs, @@ -297,25 +296,22 @@ export function createSqliteQueue( } catch (error) { const attempts = job.attempts + 1; const errMsg = error instanceof Error ? error.message : "unknown error"; - const nonConsumingDelayMs = nonConsumingRetryDelayMs(error); - if (nonConsumingDelayMs !== null) { - const rateLimited = githubRateLimitRetryDelayMs(error) !== null; + const rateLimitDelayMs = githubRateLimitRetryDelayMs(error); + if (rateLimitDelayMs !== null) { const now = Date.now(); - const retryAfter = now + (rateLimited ? rateLimitRetryDelayWithJitter(nonConsumingDelayMs, `${job.job_key ?? ""}:${job.id}:${job.payload}`) : nonConsumingDelayMs); - if (rateLimited) { - githubRateLimitCooldownUntil = Math.max(githubRateLimitCooldownUntil, now + nonConsumingDelayMs); - const deferred = deferPendingJobsForRateLimit(driver, nonConsumingDelayMs, now); - if (deferred) { - recordQueueMetric(driver, "gittensory_jobs_rate_limit_deferred_total", deferred); - console.warn( - JSON.stringify({ - level: "warn", - event: "selfhost_queue_rate_limit_cooldown", - deferred, - cooldown_until: githubRateLimitCooldownUntil, - }), - ); - } + const retryAfter = now + rateLimitRetryDelayWithJitter(rateLimitDelayMs, `${job.job_key ?? ""}:${job.id}:${job.payload}`); + githubRateLimitCooldownUntil = Math.max(githubRateLimitCooldownUntil, now + rateLimitDelayMs); + const deferred = deferPendingJobsForRateLimit(driver, rateLimitDelayMs, now); + if (deferred) { + recordQueueMetric(driver, "gittensory_jobs_rate_limit_deferred_total", deferred); + console.warn( + JSON.stringify({ + level: "warn", + event: "selfhost_queue_rate_limit_cooldown", + deferred, + cooldown_until: githubRateLimitCooldownUntil, + }), + ); } if (job.job_key && mergeRescheduledJobIntoPending(driver, job as JobRow & { job_key: string }, retryAfter, errMsg)) { recordQueueMetric(driver, "gittensory_jobs_coalesced_total"); @@ -325,7 +321,7 @@ export function createSqliteQueue( [retryAfter, errMsg, job.id], ); } - recordQueueMetric(driver, rateLimited ? "gittensory_jobs_rate_limited_total" : "gittensory_jobs_deferred_total"); + recordQueueMetric(driver, "gittensory_jobs_rate_limited_total"); logAudit({ event: "job_rate_limited", ts: Date.now(), @@ -489,7 +485,7 @@ function backfillJobPriorities(driver: SqliteDriver): number { let changed = 0; for (const row of rows as Array<{ id: number; payload: string; priority: number }>) { const priority = jobPriority(row.payload); - if (priority === Number(row.priority ?? 0)) continue; + if (priority === Number(row.priority)) continue; driver.query(`UPDATE ${TABLE} SET priority=? WHERE id=?`, [ priority, row.id, @@ -630,7 +626,7 @@ function readQueueStats(driver: SqliteDriver): Record { return Object.fromEntries( (rows as Array<{ name: string; value: number }>).map((row) => [ row.name, - Number(row.value ?? 0), + Number(row.value), ]), ); } diff --git a/test/unit/github-app.test.ts b/test/unit/github-app.test.ts index 2edd97c92c..70656c2849 100644 --- a/test/unit/github-app.test.ts +++ b/test/unit/github-app.test.ts @@ -897,6 +897,58 @@ describe("GitHub check runs", () => { ); }); + it("still posts the renamed review-agent check when legacy Gate cleanup fails", async () => { + const privateKey = await generatePrivateKeyPem(); + const warn = vi.spyOn(console, "warn").mockImplementation(() => undefined); + let newCheckBody: { name?: string; status?: string } = {}; + vi.stubGlobal( + "fetch", + async (input: RequestInfo | URL, init?: RequestInit) => { + const url = input.toString(); + const method = init?.method ?? "GET"; + if (url.includes("/access_tokens")) + return Response.json({ token: "installation-token" }); + if (url.includes("/commits/legacy-cleanup-fails/check-runs")) { + const checkName = new URL(url).searchParams.get("check_name"); + if (checkName === "Gittensory Orb Review Agent") + return Response.json({ total_count: 0, check_runs: [] }); + if (checkName === "Gittensory Gate") + return Response.json({ + total_count: 1, + check_runs: [ + { id: 322, name: "Gittensory Gate", status: "in_progress" }, + ], + }); + } + if (url.includes("/check-runs/322") && method === "PATCH") + return new Response("legacy patch failed", { status: 500 }); + if (url.includes("/check-runs") && method === "POST") { + newCheckBody = JSON.parse(String(init?.body)) as typeof newCheckBody; + return Response.json({ id: 90 }, { status: 201 }); + } + return new Response("not found", { status: 404 }); + }, + ); + + try { + const result = await createOrUpdatePendingGateCheckRun( + createTestEnv({ GITHUB_APP_PRIVATE_KEY: privateKey }), + 123, + "JSONbored/gittensory", + gateAdvisory("legacy-cleanup-fails"), + ); + + expect(result).toMatchObject({ kind: "published", id: 90 }); + expect(newCheckBody).toMatchObject({ + name: "Gittensory Orb Review Agent", + status: "in_progress", + }); + expect(warn.mock.calls.some((call) => String(call[0]).includes("legacy_gate_check_finalize_failed"))).toBe(true); + } finally { + warn.mockRestore(); + } + }); + it("omits details_url when the site origin cannot form a URL (#audit-details-url null arm)", async () => { const privateKey = await generatePrivateKeyPem(); let capturedBody: { details_url?: string } = {}; diff --git a/test/unit/selfhost-queue-common.test.ts b/test/unit/selfhost-queue-common.test.ts index 54317bbb2c..b7ba5bcfb8 100644 --- a/test/unit/selfhost-queue-common.test.ts +++ b/test/unit/selfhost-queue-common.test.ts @@ -8,7 +8,10 @@ import { jobPriority, nonConsumingRetryDelayMs, queueBackgroundConcurrency, + queueProcessingTimeoutMs, + queueRecoveryJitterMs, queueStartupJitterMinJobs, + queueStartupJitterMs, } from "../../src/selfhost/queue-common"; import { RetryableJobError } from "../../src/queue/retryable"; @@ -134,6 +137,21 @@ describe("self-host queue common helpers", () => { }), ), ).toBe("github-webhook:ci-completed:jsonbored/gittensory@def5678"); + expect( + jobCoalesceKey( + payload({ + type: "github-webhook", + eventName: "check_run", + payload: { + action: "completed", + repository: { full_name: "JSONbored/Gittensory" }, + check_run: { + head_sha: "C0FFEE1", + }, + }, + }), + ), + ).toBe("github-webhook:ci-completed:jsonbored/gittensory@c0ffee1"); expect( jobCoalesceKey( payload({ @@ -161,6 +179,32 @@ describe("self-host queue common helpers", () => { }), ), ).toBe("github-webhook:pr-refresh:jsonbored/gittensory#99"); + expect( + jobCoalesceKey( + payload({ + type: "github-webhook", + eventName: "pull_request", + payload: { + action: "opened", + repository: { full_name: "JSONbored/Gittensory" }, + pull_request: { number: 100, head: { sha: "BEEF123" } }, + }, + }), + ), + ).toBe("github-webhook:pr-refresh:jsonbored/gittensory#100@beef123"); + expect( + jobCoalesceKey( + payload({ + type: "github-webhook", + eventName: "pull_request", + payload: { + action: "opened", + repository: { full_name: "JSONbored/Gittensory" }, + pull_request: { head: { sha: "BEEF123" } }, + }, + }), + ), + ).toBeNull(); }); it("returns no coalesce key for malformed payloads", () => { @@ -218,6 +262,13 @@ describe("self-host queue common helpers", () => { message: "rate limit", }), ).toBe(300_000); + expect( + githubRateLimitRetryDelayMs({ + status: 429, + response: { headers: new Headers({ "retry-after": "soon" }) }, + message: "secondary rate limit", + }), + ).toBe(300_000); }); it("keeps only GitHub rate limits on the non-consuming retry path", () => { @@ -268,6 +319,31 @@ describe("self-host queue common helpers", () => { ).toBe(300_000); }); + it("parses queue timing env values with defensive fallbacks", () => { + const oldStartup = process.env.QUEUE_STARTUP_JITTER_MS; + const oldRecovery = process.env.QUEUE_RECOVERY_JITTER_MS; + const oldTimeout = process.env.QUEUE_PROCESSING_TIMEOUT_MS; + try { + process.env.QUEUE_STARTUP_JITTER_MS = "42"; + process.env.QUEUE_RECOVERY_JITTER_MS = "25.9"; + process.env.QUEUE_PROCESSING_TIMEOUT_MS = "not-a-number"; + + expect(queueStartupJitterMs()).toBe(42); + expect(queueRecoveryJitterMs()).toBe(25); + expect(queueProcessingTimeoutMs()).toBe(30 * 60_000); + + process.env.QUEUE_STARTUP_JITTER_MS = "-1"; + expect(queueStartupJitterMs()).toBe(3 * 60_000); + } finally { + if (oldStartup === undefined) delete process.env.QUEUE_STARTUP_JITTER_MS; + else process.env.QUEUE_STARTUP_JITTER_MS = oldStartup; + if (oldRecovery === undefined) delete process.env.QUEUE_RECOVERY_JITTER_MS; + else process.env.QUEUE_RECOVERY_JITTER_MS = oldRecovery; + if (oldTimeout === undefined) delete process.env.QUEUE_PROCESSING_TIMEOUT_MS; + else process.env.QUEUE_PROCESSING_TIMEOUT_MS = oldTimeout; + } + }); + it("bounds startup jitter min-jobs config to a non-negative finite integer", () => { const old = process.env.QUEUE_STARTUP_JITTER_MIN_JOBS; try { diff --git a/test/unit/selfhost-sqlite-queue.test.ts b/test/unit/selfhost-sqlite-queue.test.ts index 8d02ddce70..51990b758f 100644 --- a/test/unit/selfhost-sqlite-queue.test.ts +++ b/test/unit/selfhost-sqlite-queue.test.ts @@ -196,8 +196,8 @@ describe("createSqliteQueue (durable #980)", () => { const driver = makeDriver(); createSqliteQueue(driver, async () => undefined); driver.query( - "INSERT INTO _selfhost_jobs (payload, status, attempts, run_after, created_at, priority, job_key) VALUES (?, 'pending', 0, 0, 0, 10, ?)", - [JSON.stringify(ciWebhook("ci-1")), "k1"], + "INSERT INTO _selfhost_jobs (payload, status, attempts, run_after, created_at, priority) VALUES (?, 'pending', 0, 0, 0, 10)", + [JSON.stringify(msg("unkeyed"))], ); driver.query( "INSERT INTO _selfhost_jobs (payload, status, attempts, run_after, created_at, priority, job_key) VALUES (?, 'pending', 0, 0, 0, 10, ?)", @@ -474,6 +474,46 @@ describe("createSqliteQueue (durable #980)", () => { expect(q.stats()).toMatchObject({ gittensory_jobs_coalesced_total: 1 }); }); + it("reschedules a keyed rate-limited job when no pending duplicate exists", async () => { + const driver = makeDriver(); + let calls = 0; + const rateLimit = new Error("secondary rate limit"); + Object.assign(rateLimit, { status: 403 }); + const key = `github-webhook:ci-completed:jsonbored/gittensory@${"b".repeat(40)}#1629`; + const q = createSqliteQueue( + driver, + async () => { + calls += 1; + throw rateLimit; + }, + { maxRetries: 1, backoffMs: () => 0 }, + ); + driver.query( + "INSERT INTO _selfhost_jobs (payload, status, attempts, run_after, created_at, priority, job_key) VALUES (?, 'pending', 0, 0, 0, 10, ?)", + [JSON.stringify(ciWebhook("ci-active")), key], + ); + + await q.drain(); + + const row = driver.query( + "SELECT payload, status, attempts, run_after, last_error FROM _selfhost_jobs", + [], + ).rows[0] as { + payload: string; + status: string; + attempts: number; + run_after: number; + last_error: string | null; + }; + expect(calls).toBe(1); + expect(JSON.parse(row.payload).deliveryId).toBe("ci-active"); + expect(row.status).toBe("pending"); + expect(row.attempts).toBe(0); + expect(row.run_after).toBeGreaterThan(Date.now()); + expect(row.last_error).toContain("secondary rate limit"); + expect(q.stats()).toMatchObject({ gittensory_jobs_rate_limited_total: 1 }); + }); + it("consumes retryable incomplete review attempts and dead-letters after maxRetries", async () => { const driver = makeDriver(); let calls = 0; @@ -730,6 +770,51 @@ describe("createSqliteQueue (durable #980)", () => { } }); + it("does not reclaim an expired processing lease while that job is still active", async () => { + const oldTimeout = process.env.QUEUE_PROCESSING_TIMEOUT_MS; + const oldRecoveryJitter = process.env.QUEUE_RECOVERY_JITTER_MS; + process.env.QUEUE_PROCESSING_TIMEOUT_MS = "1"; + process.env.QUEUE_RECOVERY_JITTER_MS = "0"; + const driver = makeDriver(); + const seen: string[] = []; + const releases: Array<() => void> = []; + let q: ReturnType | undefined; + try { + const queue = createSqliteQueue( + driver, + async (m) => { + const type = typeOf(m); + seen.push(type); + if (type === "slow") { + await new Promise((resolve) => { + releases.push(resolve); + }); + } + }, + { concurrency: 2, backgroundConcurrency: 2, pollIntervalMs: 100_000 }, + ); + q = queue; + await queue.binding.send(msg("slow")); + for (let i = 0; i < 20 && releases.length === 0; i += 1) + await new Promise((r) => setTimeout(r, 10)); + await new Promise((r) => setTimeout(r, 5)); + + await queue.binding.send(msg("wake-reclaimer")); + for (let i = 0; i < 20 && !seen.includes("wake-reclaimer"); i += 1) + await new Promise((r) => setTimeout(r, 10)); + + expect(seen.filter((type) => type === "slow")).toHaveLength(1); + expect(queue.stats().gittensory_jobs_recovered_total ?? 0).toBe(0); + } finally { + for (const release of releases) release(); + if (q) await q.stop(); + if (oldTimeout === undefined) delete process.env.QUEUE_PROCESSING_TIMEOUT_MS; + else process.env.QUEUE_PROCESSING_TIMEOUT_MS = oldTimeout; + if (oldRecoveryJitter === undefined) delete process.env.QUEUE_RECOVERY_JITTER_MS; + else process.env.QUEUE_RECOVERY_JITTER_MS = oldRecoveryJitter; + } + }); + it("records 'unknown error' when a consumer throws a non-Error", async () => { const q = createSqliteQueue( makeDriver(), From 000ebae15fc8e51d9576bf613321c4d9cf1fae93 Mon Sep 17 00:00:00 2001 From: JSONbored <49853598+JSONbored@users.noreply.github.com> Date: Mon, 29 Jun 2026 05:48:13 -0700 Subject: [PATCH 55/68] feat(review): improve maintainer signal table --- .gittensory.yml | 6 +- src/config/gittensory-repo-focus-manifest.ts | 6 +- src/rules/advisory.ts | 4 +- src/signals/engine.ts | 92 ++++++++++++++------ test/unit/rules.test.ts | 4 +- test/unit/signals-coverage.test.ts | 90 ++++++++++++++++--- test/unit/unified-comment-bridge.test.ts | 8 +- 7 files changed, 156 insertions(+), 54 deletions(-) diff --git a/.gittensory.yml b/.gittensory.yml index a828fb78a1..296ff074d6 100644 --- a/.gittensory.yml +++ b/.gittensory.yml @@ -56,9 +56,9 @@ gate: # footer: # text: "Reviewed by the Acme maintainer bot." # custom lead line (attribution still appended) # note: "Run the test suite before requesting review." # short intro line shown above the panel -# fields: # show/hide rows (default: all shown). Keys: -# relatedWork: false # linkedIssue | relatedWork | reviewLoad | -# openPrQueue: false # validationEvidence | openPrQueue | contributorContext | gateResult +# fields: # show/hide rows (default: all shown). Stable keys: +# relatedWork: false # linkedIssue | relatedWork | reviewLoad (Change scope) | +# openPrQueue: false # validationEvidence (Validation posture) | openPrQueue (Contributor workload) | contributorContext | gateResult publicNotes: - Prefer backend Workers, MCP, GitHub App, registry, and scoring work when scope allows. diff --git a/src/config/gittensory-repo-focus-manifest.ts b/src/config/gittensory-repo-focus-manifest.ts index fb9c440313..4480225142 100644 --- a/src/config/gittensory-repo-focus-manifest.ts +++ b/src/config/gittensory-repo-focus-manifest.ts @@ -60,9 +60,9 @@ gate: # footer: # text: "Reviewed by the Acme maintainer bot." # custom lead line (attribution still appended) # note: "Run the test suite before requesting review." # short intro line shown above the panel -# fields: # show/hide rows (default: all shown). Keys: -# relatedWork: false # linkedIssue | relatedWork | reviewLoad | -# openPrQueue: false # validationEvidence | openPrQueue | contributorContext | gateResult +# fields: # show/hide rows (default: all shown). Stable keys: +# relatedWork: false # linkedIssue | relatedWork | reviewLoad (Change scope) | +# openPrQueue: false # validationEvidence (Validation posture) | openPrQueue (Contributor workload) | contributorContext | gateResult publicNotes: - Prefer backend Workers, MCP, GitHub App, registry, and scoring work when scope allows. diff --git a/src/rules/advisory.ts b/src/rules/advisory.ts index ca9d69f6cf..9636eb7222 100644 --- a/src/rules/advisory.ts +++ b/src/rules/advisory.ts @@ -751,9 +751,9 @@ function addPullRequestFindings( findings.push({ code: "busy_pr_queue", severity: "info", - title: "Open PR queue is busy", + title: "Review queue is busy", detail: `Gittensory has ${otherOpenPullRequests.length} other open pull requests cached for this repository.`, - publicText: "This repo has a busy open PR queue in the local Gittensory cache.", + publicText: "This repo has a busy review queue in the local Gittensory cache.", }); } const repoMultipliers = repo?.registryConfig?.labelMultipliers ?? {}; diff --git a/src/signals/engine.ts b/src/signals/engine.ts index 5f96c3d61d..eedaa6f4a2 100644 --- a/src/signals/engine.ts +++ b/src/signals/engine.ts @@ -3993,12 +3993,12 @@ export function buildPublicReadinessScore(args: { label: "Change scope", score: reviewLoadScore, max: 20, - evidence: `Readiness component derived from cached public PR metadata and labels${formatSizeLabelEvidence(args.pr.labels)}.`, - action: reviewLoadScore >= 18 ? "No action." : "Add scope summary.", + evidence: changeScopeEvidence(args.pr, args.preflight.reviewBurden), + action: reviewLoadScore >= 18 ? "No action." : "Add a concise scope and risk note.", }, { key: "validation", - label: "Validation evidence", + label: "Validation posture", score: validation.score, max: 25, evidence: validation.evidence, @@ -4014,7 +4014,7 @@ export function buildPublicReadinessScore(args: { }, { key: "queue_pressure", - label: "Open PR queue", + label: "Review queue context", score: queuePressure.score, max: queuePressure.max, evidence: queuePressure.evidence, @@ -4072,8 +4072,9 @@ type PublicSafeCollapsibleArgs = { function signalDefinitionsBody(): string[] { return [ "- Related work = same linked issue, overlapping active PRs, or title/path similarity.", - "- Review load = cached public PR metadata such as size labels, changed paths, and preflight status.", - "- Open PR queue = repo-wide review pressure; it is not a PR quality failure.", + "- Change scope = cached public metadata such as size labels, draft state, and review-burden hints.", + "- Validation posture = whether the PR provides enough public validation/test evidence for maintainer review.", + "- Contributor workload = public contributor activity and cleanup pressure, not a repo-wide quality failure.", "- Contributor context = public GitHub/Gittensor identity context; non-Gittensor status is not a blocker.", ]; } @@ -4267,7 +4268,7 @@ export function buildPublicPrIntelligenceComment(args: { const readinessByKey = new Map(readiness.components.map((component) => [component.key, component])); const validationComponent = readinessByKey.get("validation"); const changeScopeComponent = readinessByKey.get("change_scope"); - const queueComponent = readinessByKey.get("queue_pressure"); + const contributorWorkload = contributorWorkloadPanelResult(args.profile); const contributorContext = contributorContextPanelResult(args.pr, args.profile, args.detection, confirmedMiner); // Each row carries a stable key so a maintainer can show/hide it from `.gittensory.yml review.fields` // (default: shown). Hiding a row is cosmetic — the underlying signal/gate still functions. @@ -4275,9 +4276,9 @@ export function buildPublicPrIntelligenceComment(args: { { key: "linkedIssue", cells: ["Linked issue", linkedIssueResult.result, linkedIssueResult.evidence, linkedIssueResult.action] }, { key: "relatedWork", cells: ["Related work", relatedWorkResult.result, relatedWorkResult.evidence, relatedWorkResult.action] }, /* v8 ignore start -- Readiness components are built as a fixed key set; fallbacks guard future partial score shapes. */ - { key: "reviewLoad", cells: ["Review load", scoreResultIcon(changeScopeComponent), changeScopeComponent?.evidence ?? "No public scope metadata found.", changeScopeComponent?.action ?? "No action."] }, - { key: "validationEvidence", cells: ["Validation evidence", scoreResultIcon(validationComponent), validationComponent?.evidence ?? "No validation signal found.", validationComponent?.action ?? "Add validation note."] }, - { key: "openPrQueue", cells: ["Open PR queue", scoreResultIcon(queueComponent), queueComponent?.evidence ?? "Open PR queue unavailable.", queueComponent?.action ?? "No action."] }, + { key: "reviewLoad", cells: ["Change scope", scoreResultIcon(changeScopeComponent), changeScopeComponent?.evidence ?? "No public scope metadata found.", changeScopeComponent?.action ?? "No action."] }, + { key: "validationEvidence", cells: ["Validation posture", scoreResultIcon(validationComponent), validationComponent?.evidence ?? "No validation signal found.", validationComponent?.action ?? "Add validation note."] }, + { key: "openPrQueue", cells: ["Contributor workload", contributorWorkload.result, contributorWorkload.evidence, contributorWorkload.action] }, /* v8 ignore stop */ { key: "contributorContext", cells: ["Contributor context", contributorContext.result, contributorContext.evidence, contributorContext.action] }, { key: "gateResult", cells: ["Gate result", gateStatus(gateEnabled, gateConclusion), gateEnabled ? gateAction(gateConclusion) : "Advisory only.", gateEnabled ? gateNextAction(gateConclusion) : "No action."] }, @@ -4334,8 +4335,9 @@ export function buildPublicPrIntelligenceComment(args: { "

    Signal definitions", "", "- Related work = same linked issue, overlapping active PRs, or title/path similarity.", - "- Review load = cached public PR metadata such as size labels, changed paths, and preflight status.", - "- Open PR queue = repo-wide review pressure; it is not a PR quality failure.", + "- Change scope = cached public metadata such as size labels, draft state, and review-burden hints.", + "- Validation posture = whether the PR provides enough public validation/test evidence for maintainer review.", + "- Contributor workload = public contributor activity and cleanup pressure, not a repo-wide quality failure.", "- Contributor context = public GitHub/Gittensor identity context; non-Gittensor status is not a blocker.", "", "
    ", @@ -4446,14 +4448,14 @@ export function buildPublicPrPanelSignalRows(args: { const readinessByKey = new Map(readiness.components.map((component) => [component.key, component])); const validationComponent = readinessByKey.get("validation"); const changeScopeComponent = readinessByKey.get("change_scope"); - const queueComponent = readinessByKey.get("queue_pressure"); + const contributorWorkload = contributorWorkloadPanelResult(args.profile); const contributorContext = contributorContextPanelResult(args.pr, args.profile, args.detection, confirmedMiner); const rows: PublicPrPanelSignalRow[] = [ { key: "linkedIssue", cells: ["Linked issue", linkedIssueResult.result, linkedIssueResult.evidence, linkedIssueResult.action] }, { key: "relatedWork", cells: ["Related work", relatedWorkResult.result, relatedWorkResult.evidence, relatedWorkResult.action] }, - { key: "reviewLoad", cells: ["Review load", scoreResultIcon(changeScopeComponent), changeScopeComponent?.evidence ?? "No public scope metadata found.", changeScopeComponent?.action ?? "No action."] }, - { key: "validationEvidence", cells: ["Validation evidence", scoreResultIcon(validationComponent), validationComponent?.evidence ?? "No validation signal found.", validationComponent?.action ?? "Add validation note."] }, - { key: "openPrQueue", cells: ["Open PR queue", scoreResultIcon(queueComponent), queueComponent?.evidence ?? "Open PR queue unavailable.", queueComponent?.action ?? "No action."] }, + { key: "reviewLoad", cells: ["Change scope", scoreResultIcon(changeScopeComponent), changeScopeComponent?.evidence ?? "No public scope metadata found.", changeScopeComponent?.action ?? "No action."] }, + { key: "validationEvidence", cells: ["Validation posture", scoreResultIcon(validationComponent), validationComponent?.evidence ?? "No validation signal found.", validationComponent?.action ?? "Add validation note."] }, + { key: "openPrQueue", cells: ["Contributor workload", contributorWorkload.result, contributorWorkload.evidence, contributorWorkload.action] }, { key: "contributorContext", cells: ["Contributor context", contributorContext.result, contributorContext.evidence, contributorContext.action] }, { key: "gateResult", cells: ["Gate result", gateStatus(gateEnabled, gateConclusion), gateEnabled ? gateAction(gateConclusion) : "Advisory only.", gateEnabled ? gateNextAction(gateConclusion) : "No action."] }, ]; @@ -4560,7 +4562,46 @@ function contributorContextPanelResult( }; } -function scoreResultIcon(component: PublicReadinessScore["components"][number] | undefined): string { +function changeScopeEvidence(pr: PullRequestRecord, reviewBurden: PreflightResult["reviewBurden"]): string { + const burden = reviewBurden === "low" ? "Low" : reviewBurden === "medium" ? "Medium" : "High"; + const sizeLabel = pr.labels.find((label) => /^size[:/-]/i.test(label)); + const detailParts = [ + sizeLabel ? `size label ${sanitizePanelText(sizeLabel)}` : undefined, + pr.isDraft ? "draft PR" : undefined, + pr.linkedIssues.length > 0 ? `${pr.linkedIssues.length} linked issue${pr.linkedIssues.length === 1 ? "" : "s"}` : "no linked issue context", + ].filter(Boolean); + return `${burden} review scope from cached public metadata (${detailParts.join("; ")}).`; +} + +function contributorWorkloadPanelResult(profile: ContributorProfile): { result: string; evidence: string; action: string } { + const unlinkedOpenPullRequests = Math.max(0, profile.trustSignals.unlinkedOpenPullRequests); + const maintainerAssociatedPullRequests = Math.max(0, profile.trustSignals.maintainerAssociatedPullRequests); + const pullRequests = Math.max(0, profile.registeredRepoActivity.pullRequests); + const mergedPullRequests = Math.max(0, profile.registeredRepoActivity.mergedPullRequests); + const issues = Math.max(0, profile.registeredRepoActivity.issues); + const score = contributorWorkloadScore(unlinkedOpenPullRequests); + const detailParts = [ + `${pullRequests} registered-repo PR(s)`, + `${mergedPullRequests} merged`, + `${issues} issue(s)`, + unlinkedOpenPullRequests > 0 ? `${unlinkedOpenPullRequests} unlinked open PR(s)` : undefined, + maintainerAssociatedPullRequests > 0 ? `${maintainerAssociatedPullRequests} maintainer-associated PR(s)` : undefined, + ].filter(Boolean); + return { + result: scoreResultIcon({ score, max: 10 }), + evidence: `Author activity: ${detailParts.join(", ")}.`, + action: unlinkedOpenPullRequests > 0 ? "Link or explain open contributor PRs." : "No action.", + }; +} + +function contributorWorkloadScore(unlinkedOpenPullRequests: number): number { + if (unlinkedOpenPullRequests === 0) return 10; + if (unlinkedOpenPullRequests <= 2) return 8; + if (unlinkedOpenPullRequests <= 5) return 5; + return 3; +} + +function scoreResultIcon(component: Pick | undefined): string { /* v8 ignore next -- Component lookup is fixed today; undefined is a defensive fallback for future score shape drift. */ if (!component) return "⚠️ No score"; const ratio = component.score / component.max; @@ -4580,7 +4621,7 @@ function validationComponent(pr: PullRequestRecord, preflight: PreflightResult): const missingTests = findingCodes.some((code) => /missing.*test|test.*missing|no_test/i.test(code)); const explicitValidation = hasValidationNote(pr.body ?? ""); if (preflight.status === "hold") { - return { score: 5, evidence: "Cached preflight status is hold.", action: "Fix blocker." }; + return { score: 5, evidence: "Preflight is holding this PR; address the blocker before review.", action: "Fix the blocker." }; } if (missingTests) { // A body validation note is an UNBACKED claim when no test files accompany the change. Cap it just above the @@ -4588,15 +4629,15 @@ function validationComponent(pr: PullRequestRecord, preflight: PreflightResult): // zero-test PR — full credit is reserved for actual test evidence in the branch below. (#audit-2.3) return explicitValidation ? { score: 12, evidence: "PR body claims validation but no test files accompany the change.", action: "Add tests covering the change." } - : { score: 10, evidence: "No cached test files or validation note found.", action: "Add validation note." }; + : { score: 10, evidence: "No cached test files or validation note found.", action: "Add tests or validation evidence." }; } if (explicitValidation) { return { score: 25, evidence: "PR body includes validation/test evidence.", action: "No action." }; } if (preflight.status === "ready") { - return { score: 20, evidence: "Cached preflight status is ready; explicit validation note not found.", action: "Add validation note." }; + return { score: 20, evidence: "Preflight is ready, but the PR body does not name the validation run.", action: "Add validation command/output." }; } - return { score: 12, evidence: "Cached preflight status needs author follow-up.", action: "Add validation note." }; + return { score: 12, evidence: "Preflight needs author follow-up before maintainer review.", action: "Address findings or add validation evidence." }; } function queuePressureComponent(queueHealth: QueueHealth): { score: number; max: 10; evidence: string; action: string } { @@ -4623,8 +4664,8 @@ function queuePressureComponent(queueHealth: QueueHealth): { score: number; max: return { score, max: 10, - evidence: `${detailParts.join(", ")}.`, - action: score >= 8 ? "No action." : "Expect slower review.", + evidence: `Repo queue: ${detailParts.join(", ")}.`, + action: score >= 8 ? "No action." : "Triage stale or unlinked PRs.", }; } @@ -4813,11 +4854,6 @@ function hasValidationNote(value: string): boolean { return /\b(test(?:ed|s|ing)?|validation|validated|verified|manual check|smoke|pytest|vitest|npm test|pnpm test|cargo test|go test)\b/i.test(value); } -function formatSizeLabelEvidence(labels: string[]): string { - const sizeLabel = labels.find((label) => /^size[:/-]/i.test(label)); - return sizeLabel ? `; size label ${sizeLabel}` : ""; -} - function gateStatus(gateEnabled: boolean, conclusion: PublicPrPanelGateEvaluation["conclusion"]): string { if (!gateEnabled) return "⚠️ Advisory only"; if (conclusion === "success") return "✅ Passing"; diff --git a/test/unit/rules.test.ts b/test/unit/rules.test.ts index f504f56058..ac4389d6dc 100644 --- a/test/unit/rules.test.ts +++ b/test/unit/rules.test.ts @@ -350,7 +350,7 @@ describe("advisory rules", () => { findings: [ { code: "missing_linked_issue", title: "No linked issue detected", severity: "warning", detail: "No linked issue." }, { code: "duplicate_pr_risk", title: "Linked issue overlaps another open PR", severity: "warning", detail: "Duplicate." }, - { code: "busy_pr_queue", title: "Open PR queue is busy", severity: "warning", detail: "Queue context." }, + { code: "busy_pr_queue", title: "Review queue is busy", severity: "warning", detail: "Queue context." }, ], }, { linkedIssueGateMode: "block", duplicatePrGateMode: "block", qualityGateMode: "block", qualityGateMinScore: 90, readinessScore: 42 }, @@ -729,7 +729,7 @@ describe("advisory rules", () => { title: "Queue pressure", severity: "warning" as const, detail: "Private detail", - publicText: "Open PR queue is elevated; keep changes focused.", + publicText: "Review queue is elevated; keep changes focused.", }, ], }; diff --git a/test/unit/signals-coverage.test.ts b/test/unit/signals-coverage.test.ts index 0209dd1610..51a584ac9f 100644 --- a/test/unit/signals-coverage.test.ts +++ b/test/unit/signals-coverage.test.ts @@ -1275,9 +1275,9 @@ describe("signal coverage edge cases", () => { }); expect(comment).toContain("> | Linked issue | ✅ No-issue rationale | PR body explains why no issue is linked. | No action. |"); - expect(comment).toContain("> | Review load | ❌ 8/20 |"); - expect(comment).toContain("> | Validation evidence | ❌ 5/25 | Cached preflight status is hold. | Fix blocker. |"); - expect(comment).toContain("> | Open PR queue | ❌ 3/10 | 16 open PR(s), 0 likely reviewable, 16 unlinked. | Expect slower review. |"); + expect(comment).toContain("> | Change scope | ❌ 8/20 | High review scope from cached public metadata (size label size:L; draft PR; no linked issue context). | Add a concise scope and risk note. |"); + expect(comment).toContain("> | Validation posture | ❌ 5/25 | Preflight is holding this PR; address the blocker before review. | Fix the blocker. |"); + expect(comment).toContain("> | Contributor workload | ✅ 10/10 | Author activity: 29 registered-repo PR(s), 20 merged, 6 issue(s). | No action. |"); expect(comment).toContain("> | Gate result | ⚠️ Not blocking | Advisory; not blocking this PR. | No action. |"); expect(comment).toContain("[JSONbored](https://github.com/JSONbored)"); expect(comment).toContain("[Gittensor profile](https://gittensor.io/miners/details?githubId=49853598)"); @@ -1287,6 +1287,72 @@ describe("signal coverage edge cases", () => { expect(comment).not.toMatch(/wallet|hotkey|payout|trust score|private score/i); }); + it("uses contributor workload buckets for the visible queue row", () => { + const directRepo = repo("owner/contributor-workload"); + const currentPr = pr(directRepo.fullName, 32, "Fix contributor workload row", { + authorLogin: "dev", + body: "Fixes #10\n\nValidation: npm test", + linkedIssues: [10], + }); + const preflight = buildPreflightResult( + { repoFullName: directRepo.fullName, title: currentPr.title, body: currentPr.body ?? undefined, linkedIssues: currentPr.linkedIssues }, + directRepo, + [], + [currentPr], + ); + const baseArgs = { + repo: directRepo, + pr: currentPr, + detection: { detected: true, source: "github_cache" as const, reason: "cached", priorPullRequests: 0, priorMergedPullRequests: 0, priorIssues: 0 }, + queueHealth: queueHealthFixture(directRepo.fullName, "critical"), + collisions: buildCollisionReport(directRepo.fullName, [], [currentPr]), + preflight, + settings: repoSettings(directRepo.fullName), + }; + const profileWithUnlinked = (unlinkedPullRequests: number, authoredPullRequests: PullRequestRecord[] = []) => + buildContributorProfile( + "dev", + { login: "dev", topLanguages: ["TypeScript"], source: "github" }, + authoredPullRequests, + [], + [ + { + login: "dev", + repoFullName: directRepo.fullName, + pullRequests: 12, + mergedPullRequests: 7, + openPullRequests: unlinkedPullRequests, + issues: 3, + stalePullRequests: 0, + unlinkedPullRequests, + dominantLabels: [], + }, + ], + ); + const workloadRow = (profile: ReturnType) => + buildPublicPrPanelSignalRows({ ...baseArgs, profile }).rows.find((row) => row.key === "openPrQueue")?.cells; + + expect(workloadRow(profileWithUnlinked(0))).toEqual(["Contributor workload", "✅ 10/10", "Author activity: 12 registered-repo PR(s), 7 merged, 3 issue(s).", "No action."]); + expect(workloadRow(profileWithUnlinked(2))).toEqual([ + "Contributor workload", + "⚠️ 8/10", + "Author activity: 12 registered-repo PR(s), 7 merged, 3 issue(s), 2 unlinked open PR(s).", + "Link or explain open contributor PRs.", + ]); + expect(workloadRow(profileWithUnlinked(5))?.[1]).toBe("⚠️ 5/10"); + expect(workloadRow(profileWithUnlinked(6))?.[1]).toBe("❌ 3/10"); + expect( + workloadRow( + profileWithUnlinked(1, [ + pr(directRepo.fullName, 33, "Maintainer-associated follow-up", { + authorLogin: "dev", + authorAssociation: "MEMBER", + }), + ]), + )?.[2], + ).toContain("1 maintainer-associated PR(s)"); + }); + it("scores public readiness from deterministic PR facts across branch cases", () => { const directRepo = repo("owner/score"); const basePr = pr(directRepo.fullName, 40, "Add focused feature", { @@ -1333,10 +1399,10 @@ describe("signal coverage edge cases", () => { expect(scoreComponent(missingValidation, "traceability")).toMatchObject({ score: 8, action: "Explain no-issue PR." }); expect(scoreComponent(missingValidation, "related_work")).toMatchObject({ score: 8, evidence: "Same linked issue with #3, #4.", action: "Compare #3, #4." }); - expect(scoreComponent(missingValidation, "change_scope")).toMatchObject({ score: 14, action: "Add scope summary." }); - expect(scoreComponent(missingValidation, "validation")).toMatchObject({ score: 10, evidence: "No cached test files or validation note found.", action: "Add validation note." }); + expect(scoreComponent(missingValidation, "change_scope")).toMatchObject({ score: 14, action: "Add a concise scope and risk note." }); + expect(scoreComponent(missingValidation, "validation")).toMatchObject({ score: 10, evidence: "No cached test files or validation note found.", action: "Add tests or validation evidence." }); expect(scoreComponent(missingValidation, "pr_state")).toMatchObject({ score: 6, evidence: "PR is open as draft.", action: "Mark ready when done." }); - expect(scoreComponent(missingValidation, "queue_pressure")).toMatchObject({ score: 5, action: "Expect slower review." }); + expect(scoreComponent(missingValidation, "queue_pressure")).toMatchObject({ score: 5, action: "Triage stale or unlinked PRs." }); // A body validation NOTE without accompanying test files is capped at 12 (was 25): a one-line "tested" can no // longer fake full validation evidence and lift readiness over a gate threshold on a zero-test PR. (#audit-2.3) @@ -1363,9 +1429,9 @@ describe("signal coverage edge cases", () => { queueHealth: queueHealthFixture(directRepo.fullName, "critical"), }); - expect(scoreComponent(weak, "validation")).toMatchObject({ score: 12, evidence: "Cached preflight status needs author follow-up." }); + expect(scoreComponent(weak, "validation")).toMatchObject({ score: 12, evidence: "Preflight needs author follow-up before maintainer review." }); expect(scoreComponent(weak, "pr_state")).toMatchObject({ score: 3, evidence: "PR state is closed.", action: "No action." }); - expect(scoreComponent(weak, "queue_pressure")).toMatchObject({ score: 3, action: "Expect slower review." }); + expect(scoreComponent(weak, "queue_pressure")).toMatchObject({ score: 3, action: "Triage stale or unlinked PRs." }); }); it("unionScopedOverlapClusters deduplicates PR-specific and preflight clusters (regression for Math.max mismatch)", () => { @@ -1446,7 +1512,7 @@ describe("signal coverage edge cases", () => { }); expect(scoreComponent(zeroScore, "queue_pressure")).toMatchObject({ score: 10, - evidence: "0 open PR(s), 0 likely reviewable.", + evidence: "Repo queue: 0 open PR(s), 0 likely reviewable.", action: "No action.", }); @@ -1470,7 +1536,7 @@ describe("signal coverage edge cases", () => { }); expect(scoreComponent(criticalBurdenScore, "queue_pressure")).toMatchObject({ score: 10, - evidence: "4 open PR(s), 0 likely reviewable, 4 stale, 4 unlinked.", + evidence: "Repo queue: 4 open PR(s), 0 likely reviewable, 4 stale, 4 unlinked.", action: "No action.", }); @@ -1490,7 +1556,7 @@ describe("signal coverage edge cases", () => { }); expect(scoreComponent(issueBurdenScore, "queue_pressure")).toMatchObject({ score: 10, - evidence: "1 open PR(s), 1 likely reviewable.", + evidence: "Repo queue: 1 open PR(s), 1 likely reviewable.", action: "No action.", }); @@ -1506,7 +1572,7 @@ describe("signal coverage edge cases", () => { preflight: { ...preflight, status: "ready", reviewBurden: "low", findings: [] }, queueHealth: sampledQueue, }); - expect(scoreComponent(sampledScore, "queue_pressure")).toMatchObject({ score: 3, action: "Expect slower review." }); + expect(scoreComponent(sampledScore, "queue_pressure")).toMatchObject({ score: 3, action: "Triage stale or unlinked PRs." }); expect(scoreComponent(sampledScore, "queue_pressure").evidence).toContain("1 likely reviewable in 1 cached PR(s); full queue reviewability is sampled"); // score=8 bucket (5–8 open PRs) — not covered by other cases diff --git a/test/unit/unified-comment-bridge.test.ts b/test/unit/unified-comment-bridge.test.ts index 6447eb2adc..55840c88cd 100644 --- a/test/unit/unified-comment-bridge.test.ts +++ b/test/unit/unified-comment-bridge.test.ts @@ -35,9 +35,9 @@ function gate(over: Partial = {}): GateCheckEvaluation { const panelRows: PublicPrPanelSignalRow[] = [ { key: "linkedIssue", cells: ["Linked issue", "✅ Linked", "#42", "No action."] }, { key: "relatedWork", cells: ["Related work", "✅ No active overlap found", "No same-issue overlap.", "No action."] }, - { key: "reviewLoad", cells: ["Review load", "⚠️ 14/20", "Medium review burden.", "Add scope summary."] }, - { key: "validationEvidence", cells: ["Validation evidence", "✅ 25/25", "PR body includes validation.", "No action."] }, - { key: "openPrQueue", cells: ["Open PR queue", "✅ 10/10", "Low queue pressure.", "No action."] }, + { key: "reviewLoad", cells: ["Change scope", "⚠️ 14/20", "Medium review scope.", "Add a concise scope and risk note."] }, + { key: "validationEvidence", cells: ["Validation posture", "✅ 25/25", "PR body includes validation.", "No action."] }, + { key: "openPrQueue", cells: ["Contributor workload", "✅ 10/10", "No contributor cleanup pressure.", "No action."] }, { key: "contributorContext", cells: ["Contributor context", "✅ Confirmed Gittensor contributor", "octocat", "No action."] }, { key: "gateResult", cells: ["Gate result", "✅ Passing", "No configured blocker found.", "No action."] }, ]; @@ -69,7 +69,7 @@ describe("panelRowsToSignalRows", () => { const rows = panelRowsToSignalRows(panelRows); const linked = rows.find((row) => row.label === "Linked issue"); expect(linked).toEqual({ label: "Linked issue", state: "ok", result: "Linked", evidence: "#42" }); - const reviewLoad = rows.find((row) => row.label === "Review load"); + const reviewLoad = rows.find((row) => row.label === "Change scope"); expect(reviewLoad?.state).toBe("warn"); expect(reviewLoad?.result).toBe("14/20"); }); From 780d6cce299bbaaa08da872fb3d9fd5e3b6964f2 Mon Sep 17 00:00:00 2001 From: JSONbored <49853598+JSONbored@users.noreply.github.com> Date: Mon, 29 Jun 2026 06:08:25 -0700 Subject: [PATCH 56/68] fix(review): preserve partial ai review notes --- src/queue/processors.ts | 31 ++++++++++++------ src/services/ai-review.ts | 18 +++++++++-- test/unit/ai-review-advisory.test.ts | 5 +-- test/unit/ai-review.test.ts | 48 +++++++++++++++++++--------- test/unit/queue.test.ts | 33 ++++++++++--------- 5 files changed, 90 insertions(+), 45 deletions(-) diff --git a/src/queue/processors.ts b/src/queue/processors.ts index 7dd55a2540..9bd1d468ab 100644 --- a/src/queue/processors.ts +++ b/src/queue/processors.ts @@ -3647,6 +3647,7 @@ export async function runAiReviewForAdvisory( reviewerCount: number; inlineFindings: InlineFinding[]; findings: AdvisoryFinding[]; + cacheable?: boolean | undefined; } | undefined > { @@ -3875,14 +3876,25 @@ export async function runAiReviewForAdvisory( }); } args.advisory.findings.push(...findings); - return hasPublicReviewAssessment(result.advisoryNotes) - ? { - notes: result.advisoryNotes ?? "", - reviewerCount: result.reviewerCount, - inlineFindings: result.inlineFindings, - findings, - } - : undefined; + if (hasPublicReviewAssessment(result.advisoryNotes)) { + return { + notes: result.advisoryNotes ?? "", + reviewerCount: result.reviewerCount, + inlineFindings: result.inlineFindings, + findings, + }; + } + if (result.inconclusive) { + return { + notes: + "AI review could not be completed for this PR head. Gittensory is holding this PR for manual review instead of relying on deterministic signals alone.", + reviewerCount: result.reviewerCount, + inlineFindings: [], + findings, + cacheable: false, + }; + } + return undefined; } catch (error) { console.error( JSON.stringify({ @@ -4356,6 +4368,7 @@ async function maybePublishPrPublicSurface( reviewerCount: number; inlineFindings?: InlineFinding[]; findings?: AdvisoryFinding[]; + cacheable?: boolean | undefined; } | undefined; let inlineCommentsEnabledForReview = false; @@ -4683,7 +4696,7 @@ async function maybePublishPrPublicSurface( reviewExcludePaths, reviewInlineComments, }); - if (aiReview) + if (aiReview && aiReview.cacheable !== false) await putCachedAiReview( env, repoFullName, diff --git a/src/services/ai-review.ts b/src/services/ai-review.ts index 28977febb2..0384950683 100644 --- a/src/services/ai-review.ts +++ b/src/services/ai-review.ts @@ -52,6 +52,7 @@ const REVIEW_SYSTEM_PROMPT = [ "Respond with ONLY a JSON object of this exact shape (no prose, no code fence):", '{"assessment": string, "blockers": string[], "nits": string[], "suggestions": string[], "confidence": number}', "- assessment: a substantive but CONCISE summary (2-4 sentences) — what the change does, whether it is correct, and the most notable detail. Specific to THIS diff; never a generic one-liner and never hedging ('appears to', 'seems to').", + "The assessment field is REQUIRED and must never be empty; if blockers is [] then the assessment still summarizes why the visible diff is safe enough to proceed.", "- blockers: each ONE sentence naming a defect that WILL break the code as written — a missing import/symbol (ReferenceError), a logic error that produces wrong output, a security hole, data loss, a build/test breakage, or an API/contract break. Reference the file (and function/line). Empty [] if there are genuinely none.", "- confidence: a single number in [0,1] — your CALIBRATED probability that the blockers above are REAL, must-fix defects (not false positives). Use 1.0 only when you are certain the diff itself breaks; use 0.5 for a genuine coin-flip; lower it when you cannot fully see the breaking code or the defect is speculative. When blockers is empty, set confidence to 1.0.", "- nits: each ONE sentence — a NON-blocking point: style, naming, a missing doc, or DEFENSIVE hardening ('should handle the empty case', 'consider catching errors', 'add validation'). File-reference where you can.", @@ -694,6 +695,17 @@ export function hasPublicReviewAssessment( return extractPublicAssessment(notes).length > 0; } +function fallbackPublicAssessment( + safeBlockers: readonly string[], + safeNits: readonly string[], +): string | null { + if (safeBlockers.length > 0) + return "The AI review returned blocking findings for this change but did not include a separate narrative summary. Review the blockers below before deciding this PR."; + if (safeNits.length > 0) + return "The AI review returned non-blocking notes for this change but did not include a separate narrative summary. Review the nits below before deciding this PR."; + return null; +} + /** Compose a public-safe markdown advisory blurb from one or two model reviews. Null if no assessment is safe. */ export function composeAdvisoryNotes(reviews: ModelReview[]): string | null { const assessments = reviews.map((r) => r.assessment).filter(Boolean); @@ -711,9 +723,11 @@ export function composeAdvisoryNotes(reviews: ModelReview[]): string | null { const safeNits = nits .map((s) => toPublicSafe(s)) .filter((s): s is string => Boolean(s)); - if (!assessment) return null; + const publicAssessment = + assessment || fallbackPublicAssessment(safeBlockers, safeNits); + if (!publicAssessment) return null; const lines: string[] = []; - lines.push(assessment, ""); + lines.push(publicAssessment, ""); if (safeBlockers.length > 0) { lines.push("**Blockers**"); lines.push(...safeBlockers.map((s) => `- ${s}`)); diff --git a/test/unit/ai-review-advisory.test.ts b/test/unit/ai-review-advisory.test.ts index 5ac8f89e1f..90c4cdc6ac 100644 --- a/test/unit/ai-review-advisory.test.ts +++ b/test/unit/ai-review-advisory.test.ts @@ -464,7 +464,7 @@ describe("runAiReviewForAdvisory", () => { expect(result).toBeUndefined(); }); - it("returns undefined when the model produces nits but no assessment summary", async () => { + it("preserves model nits when the model omits the assessment summary", async () => { const result = await runAiReviewForAdvisory(aiEnv(async () => ({ response: nitsWithoutAssessmentJson() })), { settings: { aiReviewMode: "advisory" } as RepositorySettings, advisory: advisory(), @@ -473,7 +473,8 @@ describe("runAiReviewForAdvisory", () => { author: "alice", confirmedContributor: true, }); - expect(result).toBeUndefined(); + expect(result?.notes).toContain("did not include a separate narrative summary"); + expect(result?.notes).toContain("Add a test."); }); it("does not use the maintainer's BYOK key for non-confirmed oss-anti-slop blocking reviews", async () => { diff --git a/test/unit/ai-review.test.ts b/test/unit/ai-review.test.ts index 60f77853f3..09212ba996 100644 --- a/test/unit/ai-review.test.ts +++ b/test/unit/ai-review.test.ts @@ -1236,7 +1236,7 @@ describe("pure helpers", () => { expect(result.status).toBe("ok"); }); - it("composeAdvisoryNotes returns null when no assessment is public-safe", () => { + it("composeAdvisoryNotes returns null when no assessment or finding is public-safe", () => { expect( composeAdvisoryNotes([ { @@ -1249,18 +1249,36 @@ describe("pure helpers", () => { }, ]), ).toBeNull(); - expect( - composeAdvisoryNotes([ - { - assessment: "", - suggestions: ["Add a test."], - nits: ["Rename the helper."], - blockers: ["Null deref in src/a.ts."], - inlineFindings: [], - confidence: 1, - }, - ]), - ).toBeNull(); + }); + + it("composeAdvisoryNotes preserves blockers and nits when the model omits a narrative assessment", () => { + const withBlocker = composeAdvisoryNotes([ + { + assessment: "", + suggestions: [], + nits: [], + blockers: ["Null deref in src/a.ts."], + inlineFindings: [], + confidence: 1, + }, + ]); + expect(withBlocker).toContain("blocking findings"); + expect(withBlocker).toContain("**Blockers**"); + expect(withBlocker).toContain("Null deref in src/a.ts."); + + const withNits = composeAdvisoryNotes([ + { + assessment: "", + suggestions: ["Add coverage for the edge case."], + nits: ["Rename the helper."], + blockers: [], + inlineFindings: [], + confidence: 1, + }, + ]); + expect(withNits).toContain("non-blocking notes"); + expect(withNits).toContain("**Nits (2)**"); + expect(withNits).toContain("Add coverage for the edge case."); }); it("parseModelReview parses well-formed inline findings; severity defaults to nit unless exactly 'blocker' (#inline-comments)", () => { @@ -1462,11 +1480,11 @@ describe("pure helpers", () => { review({ assessment: "Looks good." }), ]); expect(assessmentOnly).toBe("Looks good."); - expect(composeAdvisoryNotes([review({ nits: ["Add a test."] })])).toBeNull(); + expect(composeAdvisoryNotes([review({ nits: ["Add a test."] })])).toContain("Add a test."); const blockersOnly = composeAdvisoryNotes([ review({ blockers: ["Null deref in src/a.ts."] }), ]); - expect(blockersOnly).toBeNull(); + expect(blockersOnly).toContain("Null deref in src/a.ts."); }); it("composeAdvisoryNotes merges + dedupes blockers/nits across two reviewers and renders both sections", () => { diff --git a/test/unit/queue.test.ts b/test/unit/queue.test.ts index 6dee8c290b..42de27b5de 100644 --- a/test/unit/queue.test.ts +++ b/test/unit/queue.test.ts @@ -1350,7 +1350,7 @@ describe("queue processors", () => { expect(postedBodies[0]).toContain("🟪"); }); - it("publishes the final PR surface when AI review produces nits but no public summary", async () => { + it("publishes AI notes when the review omits a narrative assessment", async () => { let aiCalls = 0; const env = createTestEnv({ GITHUB_APP_PRIVATE_KEY: await generatePrivateKeyPem(), @@ -1441,22 +1441,17 @@ describe("queue processors", () => { expect(finalComment).toBeDefined(); expect(finalComment).toContain("Readiness score"); expect(finalComment).not.toContain("stale cached nit"); - expect(finalComment).not.toContain("Add coverage for the new branch."); + expect(finalComment).toContain("did not include a separate narrative summary"); + expect(finalComment).toContain("Add coverage for the new branch."); expect(aiCalls).toBeGreaterThan(0); expect(checkPatches).toContainEqual(expect.objectContaining({ status: "completed" })); const audit = await env.DB.prepare("select count(*) as n from audit_events where event_type = ?") .bind("github_app.ai_review_public_summary_missing") .first<{ n: number }>(); - expect(audit?.n).toBe(1); + expect(audit?.n).toBe(0); }); - it("publishes a deterministic re-gate result when AI review produces no public summary and audit storage fails", async () => { - const originalRecordAuditEvent = repositoriesModule.recordAuditEvent; - const auditSpy = vi.spyOn(repositoriesModule, "recordAuditEvent").mockImplementation(async (auditEnv, event) => { - if (event.eventType === "github_app.ai_review_public_summary_missing") - throw new Error("D1 audit failed"); - await originalRecordAuditEvent(auditEnv, event); - }); + it("publishes a non-cacheable AI-unavailable note when no reviewer returns usable output", async () => { const env = createTestEnv({ GITHUB_APP_PRIVATE_KEY: await generatePrivateKeyPem(), AI: { @@ -1510,7 +1505,7 @@ describe("queue processors", () => { await expect( processJob(env, { type: "agent-regate-pr", - deliveryId: "regate-ai-summary-missing-audit-fails", + deliveryId: "regate-ai-unavailable", repoFullName: "JSONbored/gittensory", prNumber: 48, installationId: 123, @@ -1519,12 +1514,16 @@ describe("queue processors", () => { expect(commentBodies.length).toBeGreaterThanOrEqual(2); expect(commentBodies[0]).toContain("is reviewing"); - expect(commentBodies.some((body) => !body.includes("is reviewing"))).toBe(true); - expect(auditSpy).toHaveBeenCalledWith( - env, - expect.objectContaining({ eventType: "github_app.ai_review_public_summary_missing" }), - ); - auditSpy.mockRestore(); + const finalComment = commentBodies.find((body) => !body.includes("is reviewing")); + expect(finalComment).toContain("AI review could not be completed for this PR head"); + const cached = await env.DB.prepare("select count(*) as n from ai_review_cache where repo_full_name = ? and pull_number = ?") + .bind("JSONbored/gittensory", 48) + .first<{ n: number }>(); + expect(cached?.n).toBe(0); + const audit = await env.DB.prepare("select count(*) as n from audit_events where event_type = ?") + .bind("github_app.ai_review_public_summary_missing") + .first<{ n: number }>(); + expect(audit?.n).toBe(0); }); it("agent re-gate sweep re-reviews each stale open PR (installation id) and swallows a failing re-review", async () => { From 9b9a867e1156de935d944dcce9c9c99fabed4d4f Mon Sep 17 00:00:00 2001 From: JSONbored <49853598+JSONbored@users.noreply.github.com> Date: Mon, 29 Jun 2026 06:21:43 -0700 Subject: [PATCH 57/68] fix(selfhost): pin sentry cli in release paths --- .github/workflows/release-selfhost.yml | 11 ++++++----- Dockerfile | 2 +- scripts/deploy-selfhost-prebuilt.sh | 4 +++- test/unit/selfhost-sentry-release.test.ts | 9 +++++++++ 4 files changed, 19 insertions(+), 7 deletions(-) diff --git a/.github/workflows/release-selfhost.yml b/.github/workflows/release-selfhost.yml index bbe01a1964..5c468f0286 100644 --- a/.github/workflows/release-selfhost.yml +++ b/.github/workflows/release-selfhost.yml @@ -33,6 +33,7 @@ jobs: env: SENTRY_ORG: jsonbored SENTRY_PROJECT: gittensory + SENTRY_CLI_PACKAGE: "@sentry/cli@3.6.0" steps: - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 with: @@ -114,11 +115,11 @@ jobs: test -n "$SENTRY_AUTH_TOKEN" test -n "$SENTRY_ORG" test -n "$SENTRY_PROJECT" - npx -y @sentry/cli@latest releases new "$SENTRY_RELEASE" - npx -y @sentry/cli@latest releases set-commits "$SENTRY_RELEASE" --auto - npx -y @sentry/cli@latest sourcemaps inject dist + npx -y "$SENTRY_CLI_PACKAGE" releases new "$SENTRY_RELEASE" + npx -y "$SENTRY_CLI_PACKAGE" releases set-commits "$SENTRY_RELEASE" --auto + npx -y "$SENTRY_CLI_PACKAGE" sourcemaps inject dist node scripts/validate-selfhost-sourcemap.mjs - npx -y @sentry/cli@latest sourcemaps upload --release="$SENTRY_RELEASE" dist + npx -y "$SENTRY_CLI_PACKAGE" sourcemaps upload --release="$SENTRY_RELEASE" dist - uses: docker/setup-qemu-action@c7c53464625b32c7a7e944ae62b3e17d2b600130 # v3 - uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3 @@ -166,7 +167,7 @@ jobs: SENTRY_ORG: ${{ env.SENTRY_ORG }} SENTRY_PROJECT: ${{ env.SENTRY_PROJECT }} SENTRY_RELEASE: ${{ steps.version.outputs.release }} - run: npx -y @sentry/cli@latest releases finalize "$SENTRY_RELEASE" + run: npx -y "$SENTRY_CLI_PACKAGE" releases finalize "$SENTRY_RELEASE" - name: GitHub Release if: github.event_name == 'push' diff --git a/Dockerfile b/Dockerfile index 4e596091ae..684923dd79 100644 --- a/Dockerfile +++ b/Dockerfile @@ -46,7 +46,7 @@ RUN mkdir -p /home/node/.npm-global /home/node/.npm \ && chown -h node:node /home/node/.codex \ && chown -R node:node /home/node/.npm-global /home/node/.npm USER node -RUN if [ "$INSTALL_AI_CLIS" = "true" ]; then npm install -g @anthropic-ai/claude-code@2.1.187 @openai/codex@0.142.0; fi +RUN if [ "$INSTALL_AI_CLIS" = "true" ]; then npm install -g --foreground-scripts @anthropic-ai/claude-code@2.1.187 @openai/codex@0.142.0; fi USER root # Optional: enable visual review via an external Chrome sidecar (e.g. `browserless/chrome:latest`). # Build with `--build-arg INSTALL_VISUAL_REVIEW=true` then set BROWSER_WS_ENDPOINT= at runtime. diff --git a/scripts/deploy-selfhost-prebuilt.sh b/scripts/deploy-selfhost-prebuilt.sh index 60bd803327..8279f24546 100755 --- a/scripts/deploy-selfhost-prebuilt.sh +++ b/scripts/deploy-selfhost-prebuilt.sh @@ -14,6 +14,7 @@ ENV_FILE="${SELFHOST_ENV_FILE:-.env}" NODE_IMAGE="${SELFHOST_NODE_IMAGE:-public.ecr.aws/docker/library/node:24-slim}" SERVICE="${SELFHOST_SERVICE:-gittensory}" SKIP_SENTRY_UPLOAD="${SELFHOST_SKIP_SENTRY_UPLOAD:-0}" +SENTRY_CLI_PACKAGE="${SENTRY_CLI_PACKAGE:-@sentry/cli@3.6.0}" require_cmd() { if ! command -v "$1" >/dev/null 2>&1; then @@ -150,12 +151,13 @@ run_sentry_upload() { -e SENTRY_AUTH_TOKEN="$auth_token" \ -e SENTRY_ORG="$org" \ -e SENTRY_PROJECT="$project" \ + -e SENTRY_CLI_PACKAGE="$SENTRY_CLI_PACKAGE" \ -e HOST_UID="$uid" \ -e HOST_GID="$gid" \ -v "$PWD:/work" \ -w /work \ "$NODE_IMAGE" \ - sh -lc 'apt-get update >/dev/null && apt-get install -y --no-install-recommends ca-certificates git >/dev/null && git config --global --add safe.directory /work && (npx -y @sentry/cli@latest releases new "$SENTRY_RELEASE" >/tmp/gittensory-sentry-release-new.log 2>&1 || true) && npx -y @sentry/cli@latest releases set-commits "$SENTRY_RELEASE" --auto && npx -y @sentry/cli@latest sourcemaps inject dist && node scripts/validate-selfhost-sourcemap.mjs && npx -y @sentry/cli@latest sourcemaps upload --release="$SENTRY_RELEASE" dist && npx -y @sentry/cli@latest releases finalize "$SENTRY_RELEASE" && chown -R "$HOST_UID:$HOST_GID" dist node_modules package-lock.json' + sh -lc 'apt-get update >/dev/null && apt-get install -y --no-install-recommends ca-certificates git >/dev/null && git config --global --add safe.directory /work && (npx -y "$SENTRY_CLI_PACKAGE" releases new "$SENTRY_RELEASE" >/tmp/gittensory-sentry-release-new.log 2>&1 || true) && npx -y "$SENTRY_CLI_PACKAGE" releases set-commits "$SENTRY_RELEASE" --auto && npx -y "$SENTRY_CLI_PACKAGE" sourcemaps inject dist && node scripts/validate-selfhost-sourcemap.mjs && npx -y "$SENTRY_CLI_PACKAGE" sourcemaps upload --release="$SENTRY_RELEASE" dist && npx -y "$SENTRY_CLI_PACKAGE" releases finalize "$SENTRY_RELEASE" && chown -R "$HOST_UID:$HOST_GID" dist node_modules package-lock.json' } run_compose_deploy() { diff --git a/test/unit/selfhost-sentry-release.test.ts b/test/unit/selfhost-sentry-release.test.ts index a291ef5a4e..15817c2b90 100644 --- a/test/unit/selfhost-sentry-release.test.ts +++ b/test/unit/selfhost-sentry-release.test.ts @@ -13,6 +13,14 @@ describe("self-host Sentry release wiring", () => { expect(releaseWorkflow).toContain( 'releases set-commits "$SENTRY_RELEASE" --auto', ); + expect(releaseWorkflow).toContain('SENTRY_CLI_PACKAGE: "@sentry/cli@3.6.0"'); + expect(releaseWorkflow).not.toContain("@sentry/cli@latest"); + + const edgeDeployScript = read("scripts/deploy-selfhost-prebuilt.sh"); + expect(edgeDeployScript).toContain( + 'SENTRY_CLI_PACKAGE="${SENTRY_CLI_PACKAGE:-@sentry/cli@3.6.0}"', + ); + expect(edgeDeployScript).not.toContain("@sentry/cli@latest"); expect(releaseWorkflow).toContain("target: runtime-prebuilt"); expect(releaseWorkflow).toContain( "GITTENSORY_VERSION=${{ steps.version.outputs.release }}", @@ -29,6 +37,7 @@ describe("self-host Sentry release wiring", () => { it("does not copy source maps into the runtime image", () => { const dockerfile = read("Dockerfile"); + expect(dockerfile).toContain("npm install -g --foreground-scripts"); expect(dockerfile).not.toContain("COPY --from=build /app/dist ./dist"); expect(dockerfile).toContain( "COPY --from=build --chown=node:node /app/dist/server.mjs ./dist/server.mjs", From 92c1d818f47aee5c49cb6d8327a4c1a989de10c4 Mon Sep 17 00:00:00 2001 From: JSONbored <49853598+JSONbored@users.noreply.github.com> Date: Mon, 29 Jun 2026 07:30:21 -0700 Subject: [PATCH 58/68] fix(selfhost): harden AI review fallback and RAG config Hold missing AI public output as a visible manual-review signal with Sentry context instead of publishing a deterministic-only surface. Honor QDRANT_DIM during collection initialization and add regression coverage for renamed checks, guardrail manual holds, self-host routing, provider edge cases, and review-comment signals. --- src/queue/processors.ts | 37 ++++++- src/selfhost/qdrant-vectorize.ts | 11 ++- test/unit/ai-review-advisory.test.ts | 44 +++++++-- test/unit/gate-check-policy.test.ts | 17 ++++ test/unit/github-app.test.ts | 101 ++++++++++++++++++++ test/unit/index.test.ts | 34 +++++++ test/unit/selfhost-ai.test.ts | 2 + test/unit/selfhost-qdrant-vectorize.test.ts | 32 ++++++- test/unit/selfhost-review-runtime.test.ts | 35 +++++++ test/unit/signals-coverage.test.ts | 16 ++++ 10 files changed, 319 insertions(+), 10 deletions(-) create mode 100644 test/unit/selfhost-review-runtime.test.ts diff --git a/src/queue/processors.ts b/src/queue/processors.ts index 9bd1d468ab..0c4da6b438 100644 --- a/src/queue/processors.ts +++ b/src/queue/processors.ts @@ -3894,7 +3894,42 @@ export async function runAiReviewForAdvisory( cacheable: false, }; } - return undefined; + const unavailableFinding: AdvisoryFinding = { + code: "ai_review_inconclusive", + severity: "warning", + title: "AI review did not produce public notes", + detail: + "The configured AI reviewer returned no usable public assessment for this PR head.", + action: + "Fix the configured AI provider, then re-run Gittensory review before relying on the result.", + }; + findings.push(unavailableFinding); + args.advisory.findings.push(unavailableFinding); + captureReviewFailure( + new Error("AI review did not produce public notes for the PR head"), + { + kind: "review", + reason: "ai_review_public_summary_missing", + owner: args.repoFullName.split("/")[0], + repo: args.repoFullName, + pr: args.pr.number, + head_sha: args.advisory.headSha, + ai_review_mode: args.settings.aiReviewMode, + reviewer_count: result.reviewerCount, + configured_reviewers: + env.AI_REVIEW_PLAN?.reviewers?.map((reviewer) => reviewer.model) ?? + null, + combine: env.AI_REVIEW_PLAN?.combine ?? null, + }, + ); + return { + notes: + "AI review is unavailable for this PR head. Gittensory is holding this PR for manual review until the configured AI provider returns a usable public review summary.", + reviewerCount: result.reviewerCount, + inlineFindings: [], + findings, + cacheable: false, + }; } catch (error) { console.error( JSON.stringify({ diff --git a/src/selfhost/qdrant-vectorize.ts b/src/selfhost/qdrant-vectorize.ts index 028f887350..3961bf90a1 100644 --- a/src/selfhost/qdrant-vectorize.ts +++ b/src/selfhost/qdrant-vectorize.ts @@ -53,11 +53,20 @@ export function qdrantReadyzUrl(url: string): string { return `${url.replace(/\/+$/, "")}/readyz`; } +export function qdrantDimensionFromEnv(value: string | undefined): number { + const dim = Number(value); + return Number.isFinite(dim) && dim > 0 ? Math.floor(dim) : DEFAULT_DIM; +} + /** * Ensures the Qdrant collection exists. Safe to call on every startup — a 409 (already exists) * is silently ignored. Call this before createQdrantVectorize() when QDRANT_URL is set. */ -export async function initQdrantCollection(url: string, collection = DEFAULT_COLLECTION, dim = DEFAULT_DIM): Promise { +export async function initQdrantCollection( + url: string, + collection = DEFAULT_COLLECTION, + dim = qdrantDimensionFromEnv(process.env.QDRANT_DIM), +): Promise { const base = url.replace(/\/+$/, ""); const res = await fetch(`${base}/collections/${collection}`, { method: "PUT", diff --git a/test/unit/ai-review-advisory.test.ts b/test/unit/ai-review-advisory.test.ts index 90c4cdc6ac..0d41941560 100644 --- a/test/unit/ai-review-advisory.test.ts +++ b/test/unit/ai-review-advisory.test.ts @@ -165,7 +165,10 @@ describe("runAiReviewForAdvisory", () => { (env as unknown as { AI_PROVIDER: string; CLAUDE_AI_MODEL: string }).AI_PROVIDER = "claude-code"; (env as unknown as { AI_PROVIDER: string; CLAUDE_AI_MODEL: string }).CLAUDE_AI_MODEL = "claude-sonnet-4-6"; const result = await runAiReviewForAdvisory(env, { settings: { aiReviewMode: "advisory" } as RepositorySettings, advisory: advisory(), repoFullName: "acme/widgets", pr, author: "alice", confirmedContributor: true }); - expect(result).toBeUndefined(); // provider threw → no usable output, degraded not crashed + expect(result).toMatchObject({ + cacheable: false, + findings: [expect.objectContaining({ code: "ai_review_inconclusive" })], + }); // provider threw → manual-review hold, degraded not crashed const usage = await env.DB.prepare("SELECT model FROM ai_usage_events WHERE feature = 'ai_review_pr' ORDER BY created_at DESC LIMIT 1").first<{ model: string }>(); expect(usage?.model).toBe("claude-code:claude-sonnet-4-6"); }); @@ -174,7 +177,10 @@ describe("runAiReviewForAdvisory", () => { const env = aiEnv(async () => { throw new Error("codex unavailable"); }); (env as unknown as { AI_PROVIDER: string }).AI_PROVIDER = " CODEX , unknown-provider "; const result = await runAiReviewForAdvisory(env, { settings: { aiReviewMode: "advisory" } as RepositorySettings, advisory: advisory(), repoFullName: "acme/widgets", pr, author: "alice", confirmedContributor: true }); - expect(result).toBeUndefined(); + expect(result).toMatchObject({ + cacheable: false, + findings: [expect.objectContaining({ code: "ai_review_inconclusive" })], + }); const usage = await env.DB.prepare("SELECT model FROM ai_usage_events WHERE feature = 'ai_review_pr' ORDER BY created_at DESC LIMIT 1").first<{ model: string }>(); expect(usage?.model).toBe("codex"); }); @@ -186,7 +192,10 @@ describe("runAiReviewForAdvisory", () => { AI_REVIEW_PLAN: { reviewers: [{ model: "ollama" }], combine: "single" }, }); const result = await runAiReviewForAdvisory(env, { settings: { aiReviewMode: "advisory" } as RepositorySettings, advisory: advisory(), repoFullName: "acme/widgets", pr, author: "alice", confirmedContributor: true }); - expect(result).toBeUndefined(); + expect(result).toMatchObject({ + cacheable: false, + findings: [expect.objectContaining({ code: "ai_review_inconclusive" })], + }); const usage = await env.DB.prepare("SELECT model FROM ai_usage_events WHERE feature = 'ai_review_pr' ORDER BY created_at DESC LIMIT 1").first<{ model: string }>(); expect(usage?.model).toBe("ollama"); }); @@ -452,16 +461,39 @@ describe("runAiReviewForAdvisory", () => { expect(adv.findings).toEqual([]); }); - it("returns undefined when the model produces no parseable notes", async () => { + it("holds for manual review when the AI provider produces no public notes", async () => { + const adv = advisory(); + const captureSpy = vi.spyOn(sentryModule, "captureReviewFailure"); const result = await runAiReviewForAdvisory(aiEnv(async () => ({ response: "not json" })), { settings: { aiReviewMode: "advisory" } as RepositorySettings, - advisory: advisory(), + advisory: adv, repoFullName: "acme/widgets", pr, author: "alice", confirmedContributor: true, }); - expect(result).toBeUndefined(); + expect(result).toMatchObject({ + reviewerCount: 0, + cacheable: false, + findings: [ + expect.objectContaining({ code: "ai_review_inconclusive" }), + ], + }); + expect(result?.notes).toContain("AI review is unavailable"); + expect(adv.findings.map((f) => f.code)).toEqual([ + "ai_review_inconclusive", + ]); + expect(captureSpy).toHaveBeenCalledWith( + expect.any(Error), + expect.objectContaining({ + reason: "ai_review_public_summary_missing", + repo: "acme/widgets", + pr: 3, + head_sha: "sha3", + reviewer_count: 0, + }), + ); + captureSpy.mockRestore(); }); it("preserves model nits when the model omits the assessment summary", async () => { diff --git a/test/unit/gate-check-policy.test.ts b/test/unit/gate-check-policy.test.ts index 9c2399f7a1..af9a5194d2 100644 --- a/test/unit/gate-check-policy.test.ts +++ b/test/unit/gate-check-policy.test.ts @@ -508,6 +508,23 @@ describe("focus-manifest policy gate (#555)", () => { expect(result.summary).toMatch(/held for manual review/i); }); + it("preserves public blocked-path context on the manual-review hold warning", () => { + const advisory = manifestAdvisory("manifest_blocked_path"); + advisory.findings[0] = { + ...advisory.findings[0]!, + publicText: "Matched guarded paths: .github/workflows/**.", + }; + const result = evaluateGateCheck(advisory, { + manifestPolicyGateMode: "block", + confirmedContributor: true, + }); + const hold = result.warnings.find( + (finding) => finding.code === "manifest_blocked_path", + ); + expect(result.conclusion).toBe("neutral"); + expect(hold?.publicText).toBe("Matched guarded paths: .github/workflows/**."); + }); + it("also holds non-confirmed contributors for manual review instead of closing", () => { const result = evaluateGateCheck(manifestAdvisory("manifest_blocked_path"), { manifestPolicyGateMode: "block", confirmedContributor: false }); expect(result.conclusion).toBe("neutral"); diff --git a/test/unit/github-app.test.ts b/test/unit/github-app.test.ts index 70656c2849..8650c922be 100644 --- a/test/unit/github-app.test.ts +++ b/test/unit/github-app.test.ts @@ -897,6 +897,55 @@ describe("GitHub check runs", () => { ); }); + it("leaves an already-completed legacy Gate check alone while posting the renamed review-agent check", async () => { + const privateKey = await generatePrivateKeyPem(); + const calls: string[] = []; + let newCheckBody: { name?: string; status?: string } = {}; + vi.stubGlobal( + "fetch", + async (input: RequestInfo | URL, init?: RequestInit) => { + const url = input.toString(); + const method = init?.method ?? "GET"; + calls.push(`${method} ${url}`); + if (url.includes("/access_tokens")) + return Response.json({ token: "installation-token" }); + if (url.includes("/commits/legacy-completed/check-runs")) { + const checkName = new URL(url).searchParams.get("check_name"); + if (checkName === "Gittensory Orb Review Agent") + return Response.json({ total_count: 0, check_runs: [] }); + if (checkName === "Gittensory Gate") + return Response.json({ + total_count: 1, + check_runs: [ + { id: 323, name: "Gittensory Gate", status: "completed" }, + ], + }); + } + if (url.includes("/check-runs/323")) + throw new Error("must not patch completed legacy check"); + if (url.includes("/check-runs") && method === "POST") { + newCheckBody = JSON.parse(String(init?.body)) as typeof newCheckBody; + return Response.json({ id: 91 }, { status: 201 }); + } + return new Response("not found", { status: 404 }); + }, + ); + + const result = await createOrUpdatePendingGateCheckRun( + createTestEnv({ GITHUB_APP_PRIVATE_KEY: privateKey }), + 123, + "JSONbored/gittensory", + gateAdvisory("legacy-completed"), + ); + + expect(result).toMatchObject({ kind: "published", id: 91 }); + expect(newCheckBody).toMatchObject({ + name: "Gittensory Orb Review Agent", + status: "in_progress", + }); + expect(calls.some((call) => call.includes("/check-runs/323"))).toBe(false); + }); + it("still posts the renamed review-agent check when legacy Gate cleanup fails", async () => { const privateKey = await generatePrivateKeyPem(); const warn = vi.spyOn(console, "warn").mockImplementation(() => undefined); @@ -1228,6 +1277,58 @@ describe("GitHub check runs", () => { ); }); + it("reposts a known check-run id when the old run belongs to a prior App", async () => { + const privateKey = await generatePrivateKeyPem(); + const logSpy = vi.spyOn(console, "log").mockImplementation(() => {}); + const calls: string[] = []; + vi.stubGlobal( + "fetch", + async (input: RequestInfo | URL, init?: RequestInit) => { + const url = input.toString(); + const method = (init?.method ?? "GET").toUpperCase(); + calls.push(`${method} ${url}`); + if (url.includes("/access_tokens")) + return Response.json({ token: "installation-token" }); + if (method === "PATCH" && url.includes("/check-runs/456")) + return new Response( + JSON.stringify({ + message: + "Invalid app_id 3824093 - check run can only be modified by the GitHub App that created it.", + }), + { status: 403 }, + ); + if (method === "POST" && url.includes("/check-runs")) + return Response.json({ id: 457, html_url: "https://github.com/checks/457" }, { status: 201 }); + return new Response("not found", { status: 404 }); + }, + ); + + try { + const result = await createOrUpdateGateCheckRun( + createTestEnv({ GITHUB_APP_PRIVATE_KEY: privateKey }), + 123, + "JSONbored/gittensory", + gateAdvisory("cross-app-known-id"), + {}, + { checkRunId: 456 }, + ); + + expect(result).toMatchObject({ kind: "published", id: 457 }); + expect( + calls.some((call) => + call.includes("/commits/cross-app-known-id/check-runs"), + ), + ).toBe(false); + expect( + logSpy.mock.calls.some((call) => + String(call[0]).includes('"staleCheckRunId":456'), + ), + ).toBe(true); + } finally { + logSpy.mockRestore(); + } + }); + it("publishes Context check annotations on changed files while Gate stays text-only", async () => { const privateKey = await generatePrivateKeyPem(); let contextBody: { diff --git a/test/unit/index.test.ts b/test/unit/index.test.ts index 10aa7d60a1..485b9ea126 100644 --- a/test/unit/index.test.ts +++ b/test/unit/index.test.ts @@ -368,6 +368,40 @@ describe("worker entrypoint", () => { expect(sent.some((m) => m.type === "ops-alerts")).toBe(false); }); + it("enqueues selftune hourly only when GITTENSORY_REVIEW_SELFTUNE is ON", async () => { + const sentFor = async ( + selfTuneFlag?: string, + ): Promise> => { + const sent: Array = []; + const env = createTestEnv({ + ...(selfTuneFlag === undefined + ? {} + : { GITTENSORY_REVIEW_SELFTUNE: selfTuneFlag }), + JOBS: { + async send(message: import("../../src/types").JobMessage) { + sent.push(message); + }, + } as unknown as Queue, + }); + const waitUntil: Promise[] = []; + await worker.scheduled( + controllerFor("2026-05-25T05:00:00.000Z"), + env, + executionContext(waitUntil), + ); + await Promise.all(waitUntil); + return sent; + }; + + expect((await sentFor()).some((m) => m.type === "selftune")).toBe(false); + expect((await sentFor("false")).some((m) => m.type === "selftune")).toBe( + false, + ); + expect((await sentFor("true")).filter((m) => m.type === "selftune")).toEqual([ + { type: "selftune", requestedBy: "schedule" }, + ]); + }); + it("enqueues the rag-index-repo fan-out in the full-sync window ONLY when GITTENSORY_REVIEW_RAG is ON (flag-OFF is byte-identical)", async () => { const sentFor = async (ragFlag?: string): Promise> => { const sent: Array = []; diff --git a/test/unit/selfhost-ai.test.ts b/test/unit/selfhost-ai.test.ts index e650d31c56..a126673b81 100644 --- a/test/unit/selfhost-ai.test.ts +++ b/test/unit/selfhost-ai.test.ts @@ -372,6 +372,8 @@ describe("branch coverage — defaults + edge inputs", () => { expect(typeof buildProvider("openai", { OPENAI_API_KEY: "sk-test" })?.run).toBe("function"); // defaults to https://api.openai.com/v1 expect(typeof buildProvider("ollama", {})?.run).toBe("function"); // defaults to http://localhost:11434/v1 expect(typeof buildProvider("openai-compatible", {})?.run).toBe("function"); // defaults to http://localhost:11434/v1 + expect(buildProvider("anthropic", {})).toBeUndefined(); // anthropic is credentialed and requires ANTHROPIC_API_KEY + expect(typeof buildProvider("anthropic", { ANTHROPIC_API_KEY: "sk-ant" })?.run).toBe("function"); }); it("extractCliText reads content + response fields", () => { expect(extractCliText(JSON.stringify({ content: "c" }))).toBe("c"); diff --git a/test/unit/selfhost-qdrant-vectorize.test.ts b/test/unit/selfhost-qdrant-vectorize.test.ts index 9b6a25467c..b1fe10c1ae 100644 --- a/test/unit/selfhost-qdrant-vectorize.test.ts +++ b/test/unit/selfhost-qdrant-vectorize.test.ts @@ -1,5 +1,5 @@ import { describe, expect, it, vi, beforeEach, afterEach } from "vitest"; -import { createQdrantVectorize, initQdrantCollection, qdrantReadyzUrl } from "../../src/selfhost/qdrant-vectorize"; +import { createQdrantVectorize, initQdrantCollection, qdrantDimensionFromEnv, qdrantReadyzUrl } from "../../src/selfhost/qdrant-vectorize"; import { resetMetrics, renderMetrics } from "../../src/selfhost/metrics"; const BASE = "http://qdrant:6333"; @@ -16,8 +16,27 @@ describe("qdrantReadyzUrl (#1482 regression)", () => { }); }); +describe("qdrantDimensionFromEnv", () => { + it("uses a positive integer dimension from QDRANT_DIM", () => { + expect(qdrantDimensionFromEnv("768")).toBe(768); + expect(qdrantDimensionFromEnv("1536.9")).toBe(1536); + }); + + it("falls back to the bge-m3 default for unset, invalid, or non-positive values", () => { + expect(qdrantDimensionFromEnv(undefined)).toBe(1024); + expect(qdrantDimensionFromEnv("")).toBe(1024); + expect(qdrantDimensionFromEnv("not-a-number")).toBe(1024); + expect(qdrantDimensionFromEnv("0")).toBe(1024); + expect(qdrantDimensionFromEnv("-5")).toBe(1024); + }); +}); + describe("initQdrantCollection (#1217)", () => { - afterEach(() => { vi.restoreAllMocks(); resetMetrics(); }); + afterEach(() => { + vi.restoreAllMocks(); + resetMetrics(); + delete process.env.QDRANT_DIM; + }); it("PUTs to /collections/ with cosine + size params", async () => { const fake = mockFetch(200); @@ -49,6 +68,15 @@ describe("initQdrantCollection (#1217)", () => { expect(url).toContain("custom-col"); expect((JSON.parse(init.body as string) as { vectors: { size: number } }).vectors.size).toBe(768); }); + + it("uses QDRANT_DIM as the default collection dimension when configured", async () => { + process.env.QDRANT_DIM = "768"; + const fake = mockFetch(200); + vi.stubGlobal("fetch", fake); + await initQdrantCollection(BASE); + const init = (fake.mock.calls[0] as unknown as [string, RequestInit])[1]; + expect((JSON.parse(init.body as string) as { vectors: { size: number } }).vectors.size).toBe(768); + }); }); describe("initQdrantCollection — QDRANT_API_KEY header", () => { diff --git a/test/unit/selfhost-review-runtime.test.ts b/test/unit/selfhost-review-runtime.test.ts new file mode 100644 index 0000000000..2c3a95e6fc --- /dev/null +++ b/test/unit/selfhost-review-runtime.test.ts @@ -0,0 +1,35 @@ +import { describe, expect, it } from "vitest"; +import { + isReviewExecutionJob, + isSelfHostedReviewRuntime, +} from "../../src/selfhost/review-runtime"; + +describe("self-host review runtime routing", () => { + it("detects the Redis-backed self-host review runtime", () => { + expect( + isSelfHostedReviewRuntime({ + SELFHOST_TRANSIENT_CACHE: { + get: async () => null, + set: async () => undefined, + }, + } as Pick), + ).toBe(true); + expect( + isSelfHostedReviewRuntime({} as Pick), + ).toBe(false); + }); + + it("classifies only review-execution jobs as self-host-only", () => { + expect(isReviewExecutionJob({ type: "github-webhook" } as never)).toBe( + true, + ); + expect(isReviewExecutionJob({ type: "rag-index-repo" } as never)).toBe( + true, + ); + expect(isReviewExecutionJob({ type: "refresh-registry" } as never)).toBe( + false, + ); + expect(isReviewExecutionJob(null)).toBe(false); + expect(isReviewExecutionJob(undefined)).toBe(false); + }); +}); diff --git a/test/unit/signals-coverage.test.ts b/test/unit/signals-coverage.test.ts index 51a584ac9f..988174be93 100644 --- a/test/unit/signals-coverage.test.ts +++ b/test/unit/signals-coverage.test.ts @@ -962,6 +962,22 @@ describe("signal coverage edge cases", () => { expect(aiBlockedComment).toContain("`src/a.ts` has a syntax error."); expect(aiBlockedComment.indexOf("**Review summary**")).toBeLessThan(aiBlockedComment.indexOf("**Readiness score:")); + const aiExplicitNoBlockersComment = buildPublicPrIntelligenceComment({ + repo: directRepo, + pr: { ...currentPr, linkedIssues: [99], body: "Fixes #99" }, + profile, + detection, + queueHealth: buildQueueHealth(directRepo, [], [currentPr], buildCollisionReport(directRepo.fullName, [], [currentPr])), + collisions: buildCollisionReport(directRepo.fullName, [], [currentPr]), + preflight: buildPreflightResult({ repoFullName: directRepo.fullName, title: "Fix isolated issue", body: "Fixes #99", linkedIssues: [99] }, directRepo, [], [currentPr]), + settings: { ...repoSettings(directRepo.fullName), gateCheckMode: "off" }, + aiReview: { notes: "The change is focused.\n\n**Blockers**\n- None.\n\n**Nits (1)**\n- Add a regression test." }, + }); + expect(aiExplicitNoBlockersComment).toContain("> [!TIP]"); + expect(aiExplicitNoBlockersComment).not.toContain( + "Gittensory review found blockers", + ); + const advisoryOnlyComment = buildPublicPrIntelligenceComment({ repo: directRepo, pr: { ...currentPr, linkedIssues: [99], body: "Fixes #99" }, From 03a451d05935a860dfe9c88fbc0e055cb695d08d Mon Sep 17 00:00:00 2001 From: JSONbored <49853598+JSONbored@users.noreply.github.com> Date: Mon, 29 Jun 2026 07:43:11 -0700 Subject: [PATCH 59/68] fix(review): clarify self-host webhook cutover Document that direct review-app webhooks and Cloudflare review jobs are retired while central Orb ingress remains live for self-host relay. Add an Orb webhook invariant without SELFHOST_TRANSIENT_CACHE and update the stale review-job log name. --- src/github/webhook.ts | 7 ++++++- src/index.ts | 5 ++++- test/integration/orb-webhook.test.ts | 13 +++++++++++++ test/unit/index.test.ts | 4 ++-- test/unit/webhook.test.ts | 2 +- 5 files changed, 26 insertions(+), 5 deletions(-) diff --git a/src/github/webhook.ts b/src/github/webhook.ts index 8da7fe7581..115cc45b93 100644 --- a/src/github/webhook.ts +++ b/src/github/webhook.ts @@ -58,7 +58,12 @@ export type EnqueueWebhookResult = "queued" | "duplicate" | "ignored" | "invalid /** Env-based core of the webhook enqueue (parse → dedup → record → WEBHOOKS lane), with NO Hono Context. Shared by * the request-context receiver above AND the pull-mode relay drain loop (server.ts), which has no Context. Returns - * a status the caller maps to a response / an ack decision. */ + * a status the caller maps to a response / an ack decision. + * + * This is the retired direct review-app receiver, not the central Orb ingress. The Orb App still receives GitHub + * webhooks at /v1/orb/webhook and forwards/pends them for registered self-host engines. Direct review execution + * now requires the self-host runtime cache so stale Cloudflare review-webhook traffic fails loudly instead of being + * accepted into a Worker path that no longer performs reviews. */ export async function enqueueWebhookByEnv(env: Env, deliveryId: string, eventName: string, rawBody: string): Promise { if (!isSelfHostedReviewRuntime(env)) return "review_unavailable"; diff --git a/src/index.ts b/src/index.ts index 67a8743084..504a7e918f 100644 --- a/src/index.ts +++ b/src/index.ts @@ -26,10 +26,13 @@ export default { for (const message of batch.messages) { try { if (!isSelfHostedReviewRuntime(env) && isReviewExecutionJob(message.body)) { + // Hosted review execution is retired. The Cloudflare API worker still handles Orb ingress + // (/v1/orb/webhook) and token brokerage, but only self-host runtimes may execute review jobs. + // Ack stale Cloudflare review-queue messages so they do not churn into the DLQ after cutover. console.warn( JSON.stringify({ level: "warn", - event: "hosted_review_job_ignored", + event: "retired_review_job_ignored", messageId: message.id, jobType: message.body.type, }), diff --git a/test/integration/orb-webhook.test.ts b/test/integration/orb-webhook.test.ts index 63b95aba76..37bf12ed01 100644 --- a/test/integration/orb-webhook.test.ts +++ b/test/integration/orb-webhook.test.ts @@ -111,6 +111,19 @@ describe("handleOrbWebhook (POST /v1/orb/webhook)", () => { await Promise.all(scheduled); // drain — install 99 has no enrollment → forward skips cleanly }); + it("INVARIANT: central Orb ingress stays live without the self-host review-runtime cache", async () => { + const e = env(); + delete e.SELFHOST_TRANSIENT_CACHE; + const PR = JSON.stringify({ action: "opened", installation: { id: 100 }, repository: { full_name: "JSONbored/gittensory" }, number: 8 }); + const res = await post(e, PR, { delivery: "orb-no-review-cache", event: "pull_request" }); + expect(res.status).toBe(202); + await expect(res.json()).resolves.toMatchObject({ status: "received", eventName: "pull_request" }); + expect(await row(e, "orb-no-review-cache")).toMatchObject({ + event_name: "pull_request", + status: "received", + }); + }); + it("stores null fields for a payload with no action/installation/repository (e.g. ping)", async () => { const e = env(); await post(e, JSON.stringify({ zen: "keep it logically awesome" }), { delivery: "ping-1", event: "ping" }); diff --git a/test/unit/index.test.ts b/test/unit/index.test.ts index 485b9ea126..6ba93f9c6a 100644 --- a/test/unit/index.test.ts +++ b/test/unit/index.test.ts @@ -83,7 +83,7 @@ describe("worker entrypoint", () => { expect(sent).toEqual([]); }); - it("acks and ignores review-execution jobs from a broker-only Cloudflare runtime", async () => { + it("acks and ignores stale review-execution jobs from a broker-only Cloudflare runtime", async () => { const env = createTestEnv(); delete env.SELFHOST_TRANSIENT_CACHE; const warned = vi.spyOn(console, "warn").mockImplementation(() => undefined); @@ -105,7 +105,7 @@ describe("worker entrypoint", () => { expect(acked).toEqual(["hosted-review-job"]); expect(retried).toEqual([]); expect(JSON.parse(String(warned.mock.calls[0]?.[0]))).toMatchObject({ - event: "hosted_review_job_ignored", + event: "retired_review_job_ignored", jobType: "github-webhook", }); }); diff --git a/test/unit/webhook.test.ts b/test/unit/webhook.test.ts index 0e062db1b8..0c752d9252 100644 --- a/test/unit/webhook.test.ts +++ b/test/unit/webhook.test.ts @@ -118,7 +118,7 @@ describe("github webhook dedup (#789)", () => { }); describe("github webhook queue isolation (#audit-webhook-queue)", () => { - it("rejects valid review webhooks when the self-host review runtime is absent", async () => { + it("rejects retired direct review-app webhooks when the self-host review runtime is absent", async () => { const env = createTestEnv(); delete env.SELFHOST_TRANSIENT_CACHE; let webhookSends = 0; From 8dfe40365edf3d8f325f0361f48e53eeddcb51bb Mon Sep 17 00:00:00 2001 From: JSONbored <49853598+JSONbored@users.noreply.github.com> Date: Mon, 29 Jun 2026 08:07:11 -0700 Subject: [PATCH 60/68] fix(review): preserve malformed AI review context Publish public-safe unstructured AI review text as a non-cacheable manual-review fallback instead of dropping it into deterministic-only output. Add provider diagnostics to Sentry context and cover the fallback path in AI review and queue tests. --- src/queue/processors.ts | 16 ++++ src/review/ai-notes.ts | 3 +- src/services/ai-review.ts | 137 ++++++++++++++++++++++++--- src/signals/engine.ts | 23 +++-- test/unit/ai-review-advisory.test.ts | 42 ++++++-- test/unit/ai-review.test.ts | 22 +++-- test/unit/queue.test.ts | 3 +- 7 files changed, 203 insertions(+), 43 deletions(-) diff --git a/src/queue/processors.ts b/src/queue/processors.ts index 0c4da6b438..8161468aa1 100644 --- a/src/queue/processors.ts +++ b/src/queue/processors.ts @@ -3873,9 +3873,22 @@ export async function runAiReviewForAdvisory( repo: args.repoFullName, pr: args.pr.number, head_sha: args.advisory.headSha, + ai_review_mode: args.settings.aiReviewMode, + reviewer_count: result.reviewerCount, + public_notes: hasPublicReviewAssessment(result.advisoryNotes), + review_diagnostics: result.reviewDiagnostics ?? [], }); } args.advisory.findings.push(...findings); + if (result.inconclusive && hasPublicReviewAssessment(result.advisoryNotes)) { + return { + notes: result.advisoryNotes ?? "", + reviewerCount: result.reviewerCount, + inlineFindings: [], + findings, + cacheable: false, + }; + } if (hasPublicReviewAssessment(result.advisoryNotes)) { return { notes: result.advisoryNotes ?? "", @@ -3916,6 +3929,7 @@ export async function runAiReviewForAdvisory( head_sha: args.advisory.headSha, ai_review_mode: args.settings.aiReviewMode, reviewer_count: result.reviewerCount, + review_diagnostics: result.reviewDiagnostics ?? [], configured_reviewers: env.AI_REVIEW_PLAN?.reviewers?.map((reviewer) => reviewer.model) ?? null, @@ -4763,6 +4777,8 @@ async function maybePublishPrPublicSurface( repo: repoFullName, pr: pr.number, head_sha: advisory.headSha, + reviewer_count: aiReview?.reviewerCount ?? 0, + public_notes: hasPublicReviewAssessment(aiReview?.notes), }); } diff --git a/src/review/ai-notes.ts b/src/review/ai-notes.ts index 7b3b4898d1..a6261c7a3d 100644 --- a/src/review/ai-notes.ts +++ b/src/review/ai-notes.ts @@ -15,8 +15,7 @@ export function splitAiReviewNits(notes: string): { main: string; nits: string[] .slice(marker) .split("\n") .slice(1) - .map((line) => line.replace(/^\s*[-*]\s*/, "").trim()) + .map((line) => line.replace(/^\s*[-*]\s*(?:\[[ xX]\]\s*)?/, "").trim()) .filter(Boolean); return { main: notes.slice(0, marker).trim(), nits }; } - diff --git a/src/services/ai-review.ts b/src/services/ai-review.ts index 0384950683..43b0d6a947 100644 --- a/src/services/ai-review.ts +++ b/src/services/ai-review.ts @@ -203,6 +203,7 @@ export type GittensoryAiReviewResult = estimatedNeurons: number; reviewerCount: number; inlineFindings: InlineFinding[]; + reviewDiagnostics?: AiReviewDiagnostic[] | undefined; }; /** A line-anchored review finding the model can emit for quiet inline PR comments (#inline-comments). `line` is @@ -232,6 +233,20 @@ export type ModelReview = { inlineFindings: InlineFinding[]; }; +export type AiReviewDiagnostic = { + model: string; + attempt: number; + status: "parsed" | "empty_output" | "unparseable_output" | "provider_error"; + responseChars?: number | undefined; + hasJsonObject?: boolean | undefined; + error?: string | undefined; +}; + +type ReviewerOpinionOutcome = { + review: ModelReview | null; + fallbackNote?: string | undefined; +}; + type AiGatewayOptions = { gateway?: { id: string } }; type AiRunner = { run?: ( @@ -510,9 +525,10 @@ async function runWorkersOpinion( system: string, user: string, maxTokens: number, -): Promise { + diagnostics: AiReviewDiagnostic[] = [], +): Promise { const ai = env.AI as unknown as AiRunner | undefined; - if (!ai || typeof ai.run !== "function") return null; + if (!ai || typeof ai.run !== "function") return { review: null }; // Route through Cloudflare AI Gateway when configured (caching, rate-limiting, logging, fallback). The // diff/prompt is the cache key input, scoped per model + content, so distinct PRs never share a cached // review. Unset → direct binding call (unchanged behavior). @@ -523,6 +539,10 @@ async function runWorkersOpinion( // Track the last provider error so we can fail-LOUD once ALL models × attempts are exhausted (below). Per-attempt // logs are warn (noisy retries, skipped by the central Sentry forwarder); the exhausted summary is error (#26). let lastError: unknown; + let lastUnstructuredText = ""; + let lastUnparseable: + | { model: string; attempt: number; responseChars: number; hasJsonObject: boolean } + | undefined; for (const model of fallback && fallback !== primary ? [primary, fallback] : [primary]) { @@ -540,12 +560,34 @@ async function runWorkersOpinion( }, extra, ); - const parsed = parseModelReview(coerceAiText(result)); - if (parsed) return parsed; + const text = coerceAiText(result); + const parsed = parseModelReview(text); + if (parsed) { + diagnostics.push({ model, attempt, status: "parsed", responseChars: text.length, hasJsonObject: Boolean(extractLastJsonObject(text)) }); + return { review: parsed }; + } + const hasJsonObject = Boolean(extractLastJsonObject(text)); + const status = text.trim() ? "unparseable_output" : "empty_output"; + diagnostics.push({ model, attempt, status, responseChars: text.length, hasJsonObject }); + if (text.trim()) { + lastUnstructuredText = text; + lastUnparseable = { model, attempt, responseChars: text.length, hasJsonObject }; + console.warn( + JSON.stringify({ + level: "warn", + event: "ai_review_provider_unparseable_output", + model, + attempt, + responseChars: text.length, + hasJsonObject, + }), + ); + } } catch (error) { // Fail-LOUD (#1566): a provider/CLI failure (e.g. the claude-code CLI absent → spawn ENOENT, or an auth/API // error) must be VISIBLE, not silently swallowed into a "no usable output" review. Log every failed attempt; // the loop still falls through to the fallback model so a transient error doesn't abort the whole review. + diagnostics.push({ model, attempt, status: "provider_error", error: errorMessage(error) }); console.warn( JSON.stringify({ level: "warn", @@ -573,7 +615,24 @@ async function runWorkersOpinion( }), ); } - return null; + if (lastUnparseable) { + console.log( + JSON.stringify({ + level: "error", + event: "ai_review_provider_unparseable_exhausted", + primary, + fallback, + model: lastUnparseable.model, + attempt: lastUnparseable.attempt, + responseChars: lastUnparseable.responseChars, + hasJsonObject: lastUnparseable.hasJsonObject, + }), + ); + } + return { + review: null, + ...(lastUnstructuredText ? { fallbackNote: lastUnstructuredText } : {}), + }; } const PROVIDER_DEFAULT_MODEL: Record = @@ -595,6 +654,8 @@ export type ProviderFailure = "timeout" | "http_error" | "exception"; type ProviderReviewOutcome = { review: ModelReview | null; failure?: ProviderFailure; + fallbackNote?: string | undefined; + diagnostic?: AiReviewDiagnostic | undefined; }; /** @@ -672,9 +733,19 @@ async function runProviderReview( user, maxTokens, ); + const model = providerKey.model || PROVIDER_DEFAULT_MODEL[providerKey.provider]; + if (failure) return { review: null, failure, diagnostic: { model, attempt: 0, status: "provider_error", error: failure } }; + const review = text ? parseModelReview(text) : null; return { - review: text ? parseModelReview(text) : null, - ...(failure ? { failure } : {}), + review, + ...(text && !review ? { fallbackNote: text } : {}), + diagnostic: { + model, + attempt: 0, + status: review ? "parsed" : text ? "unparseable_output" : "empty_output", + responseChars: text?.length ?? 0, + hasJsonObject: Boolean(text && extractLastJsonObject(text)), + }, }; } @@ -706,6 +777,24 @@ function fallbackPublicAssessment( return null; } +function fallbackUnstructuredPublicNote(text: string): string | null { + const safe = toPublicSafe(text.slice(0, 4000)); + if (!safe) return null; + return [ + "The AI reviewer returned public review text but not the expected structured verdict, so Gittensory is holding this PR for manual review.", + "", + safe, + ].join("\n").trim(); +} + +function composeFallbackAdvisoryNotes(notes: readonly string[]): string | null { + const safeNotes = [ + ...new Set(notes.map((note) => fallbackUnstructuredPublicNote(note)).filter((note): note is string => Boolean(note))), + ].slice(0, 2); + if (safeNotes.length === 0) return null; + return safeNotes.join("\n\n"); +} + /** Compose a public-safe markdown advisory blurb from one or two model reviews. Null if no assessment is safe. */ export function composeAdvisoryNotes(reviews: ModelReview[]): string | null { const assessments = reviews.map((r) => r.assessment).filter(Boolean); @@ -1022,6 +1111,8 @@ export async function runGittensoryAiReview( // Advisory write-up: BYOK frontier model if configured, else the free Workers-AI primary (with fallback). let byokFailure: ProviderFailure | undefined; let advisoryReview: ModelReview | null; + const reviewDiagnostics: AiReviewDiagnostic[] = []; + const fallbackNotes: string[] = []; if (input.providerKey) { const outcome = await runProviderReview( input.providerKey, @@ -1031,15 +1122,20 @@ export async function runGittensoryAiReview( ); advisoryReview = outcome.review; byokFailure = outcome.failure; + if (outcome.fallbackNote) fallbackNotes.push(outcome.fallbackNote); + if (outcome.diagnostic) reviewDiagnostics.push(outcome.diagnostic); } else { - advisoryReview = await runWorkersOpinion( + const outcome = await runWorkersOpinion( env, primary.model, primaryFallback, system, user, maxTokens, + reviewDiagnostics, ); + advisoryReview = outcome.review; + if (outcome.fallbackNote) fallbackNotes.push(outcome.fallbackNote); } let consensusDefect: AiConsensusDefect | null = null; @@ -1061,8 +1157,9 @@ export async function runGittensoryAiReview( system, user, maxTokens, + reviewDiagnostics, ) - : Promise.resolve(advisoryReview), + : Promise.resolve({ review: advisoryReview }), runWorkersOpinion( env, secondary.model, @@ -1070,13 +1167,16 @@ export async function runGittensoryAiReview( system, user, maxTokens, + reviewDiagnostics, ), ]); - secondReview = b; + if (a.fallbackNote) fallbackNotes.push(a.fallbackNote); + if (b.fallbackNote) fallbackNotes.push(b.fallbackNote); + secondReview = b.review; // Combine per the configured strategy (#dual-ai-combiner). Default `consensus` is byte-identical to the // historical logic: block only on agreement, lone blocker → split, a missing opinion → inconclusive // (fail-closed, HELD for a human). `synthesis` merges both into one decision (no split/hold-on-disagree). - const combined = combineReviews([a, b], { strategy: combine, onMerge }); + const combined = combineReviews([a.review, b.review], { strategy: combine, onMerge }); consensusDefect = combined.defect; aiReviewSplit = combined.split; splitConfidence = combined.splitConfidence; @@ -1091,9 +1191,11 @@ export async function runGittensoryAiReview( system, user, maxTokens, + reviewDiagnostics, ) - : advisoryReview; - const combined = combineReviews([a], { strategy: "single" }); + : ({ review: advisoryReview } as ReviewerOpinionOutcome); + if (a.fallbackNote) fallbackNotes.push(a.fallbackNote); + const combined = combineReviews([a.review], { strategy: "single" }); consensusDefect = combined.defect; inconclusive = combined.inconclusive; } @@ -1102,8 +1204,12 @@ export async function runGittensoryAiReview( const reviewsForNotes = [advisoryReview, secondReview].filter( (r): r is ModelReview => Boolean(r), ); + if (fallbackNotes.length > 0 && reviewsForNotes.length === 0) + inconclusive = true; const advisoryNotes = - reviewsForNotes.length > 0 ? composeAdvisoryNotes(reviewsForNotes) : null; + reviewsForNotes.length > 0 + ? composeAdvisoryNotes(reviewsForNotes) ?? composeFallbackAdvisoryNotes(fallbackNotes) + : composeFallbackAdvisoryNotes(fallbackNotes); // Line-anchored inline findings (#inline-comments): only propagate model output when the resolved feature gate // asked for it. AI output is PR-author-influenced, so the prompt suffix is not an authorization boundary. const inlineFindings = input.inlineFindings @@ -1143,8 +1249,9 @@ export async function runGittensoryAiReview( ...(splitConfidence !== undefined ? { splitConfidence } : {}), inconclusive, estimatedNeurons, - reviewerCount: reviewsForNotes.length, + reviewerCount: Math.max(reviewsForNotes.length, fallbackNotes.length), inlineFindings, + ...(reviewDiagnostics.length > 0 ? { reviewDiagnostics } : {}), }; } diff --git a/src/signals/engine.ts b/src/signals/engine.ts index eedaa6f4a2..495e75a497 100644 --- a/src/signals/engine.ts +++ b/src/signals/engine.ts @@ -4225,6 +4225,7 @@ export function buildPublicPrIntelligenceComment(args: { : "success"; const gateConclusion = args.gate?.conclusion ?? fallbackGateConclusion; const gateBlocking = gateEnabled && (gateConclusion === "failure" || gateConclusion === "action_required"); + const gateHeld = gateEnabled && (gateConclusion === "neutral" || gateConclusion === "action_required"); const missingLinkedIssue = args.pr.linkedIssues.length === 0 && !hasClearNoIssueRationale(args.pr); const confirmedMiner = isOfficialContributorDetection(args.detection); // Author with no Gittensor footprint at all (not detected via official API or cache): gittensory's @@ -4237,20 +4238,24 @@ export function buildPublicPrIntelligenceComment(args: { const aiReview = args.aiReview ? splitAiReviewNits(args.aiReview.notes) : null; const aiReviewHasBlockers = Boolean(aiReview?.main) && aiReviewMainHasBlockers(aiReview?.main ?? ""); const alert = aiReviewHasBlockers - ? "CAUTION" - : gateBlocking - ? gateConclusion === "action_required" - ? "WARNING" - : missingLinkedIssue && args.settings.linkedIssueGateMode === "block" + ? "CAUTION" + : gateBlocking + ? gateConclusion === "action_required" ? "WARNING" - : "CAUTION" + : missingLinkedIssue && args.settings.linkedIssueGateMode === "block" + ? "WARNING" + : "CAUTION" + : gateHeld + ? "WARNING" : hasPublicWarnings || hasRelatedWork ? "WARNING" : "TIP"; const panelTitle = aiReviewHasBlockers ? "Gittensory review found blockers" - : args.aiReview && !gateBlocking + : args.aiReview && !gateBlocking && !gateHeld ? "Gittensory review approved this PR" + : gateHeld + ? "Gittensory review needs maintainer review" : gateBlocking ? `${GITTENSORY_GATE_CHECK_NAME} is blocking merge` : hasPublicWarnings || hasRelatedWork @@ -4258,6 +4263,8 @@ export function buildPublicPrIntelligenceComment(args: { : "Gittensory PR readiness looks good"; const panelSummary = gateBlocking ? args.gate?.summary ?? (gateConclusion === "action_required" ? "Gittensory cannot evaluate the repo state closely enough for the enabled gate." : "A repo-configured hard blocker was found.") + : gateHeld + ? args.gate?.summary ?? "Gittensory is holding this PR for maintainer review." : linkedDuplicatePrs.length > 0 ? `Same-issue duplicate risk found against ${formatPrRefs(linkedDuplicatePrs)}. Maintainers should resolve the overlap before review continues.` : hasRelatedWork @@ -4312,7 +4319,7 @@ export function buildPublicPrIntelligenceComment(args: { "
    ", `Nits (${aiReview.nits.length})`, "", - ...aiReview.nits.map((nit) => `- ${escapeAiReviewMarkdown(nit)}`), + ...aiReview.nits.map((nit) => `- [ ] ${escapeAiReviewMarkdown(nit)}`), "", "
    ", ] diff --git a/test/unit/ai-review-advisory.test.ts b/test/unit/ai-review-advisory.test.ts index 0d41941560..35bf1359a8 100644 --- a/test/unit/ai-review-advisory.test.ts +++ b/test/unit/ai-review-advisory.test.ts @@ -290,14 +290,22 @@ describe("runAiReviewForAdvisory", () => { expect(adv.findings.map((f) => f.code)).toEqual(["ai_review_inconclusive"]); expect(result?.notes).toBeDefined(); // the single parseable opinion still produces advisory notes // The unproducible review is reported to Sentry with PR context so the maintainer can SEE it (#1468). - expect(captureSpy).toHaveBeenCalledWith(expect.any(Error), { - kind: "review", - reason: "ai_review_inconclusive", - owner: "acme", - repo: "acme/widgets", - pr: 3, - head_sha: "sha3", - }); + expect(captureSpy).toHaveBeenCalledWith( + expect.any(Error), + expect.objectContaining({ + kind: "review", + reason: "ai_review_inconclusive", + owner: "acme", + repo: "acme/widgets", + pr: 3, + head_sha: "sha3", + public_notes: true, + reviewer_count: 1, + review_diagnostics: expect.arrayContaining([ + expect.objectContaining({ status: "unparseable_output" }), + ]), + }), + ); captureSpy.mockRestore(); }); @@ -464,7 +472,7 @@ describe("runAiReviewForAdvisory", () => { it("holds for manual review when the AI provider produces no public notes", async () => { const adv = advisory(); const captureSpy = vi.spyOn(sentryModule, "captureReviewFailure"); - const result = await runAiReviewForAdvisory(aiEnv(async () => ({ response: "not json" })), { + const result = await runAiReviewForAdvisory(aiEnv(async () => ({ response: "" })), { settings: { aiReviewMode: "advisory" } as RepositorySettings, advisory: adv, repoFullName: "acme/widgets", @@ -496,6 +504,22 @@ describe("runAiReviewForAdvisory", () => { captureSpy.mockRestore(); }); + it("preserves public-safe unstructured AI text while holding the PR for manual review", async () => { + const adv = advisory(); + const result = await runAiReviewForAdvisory(aiEnv(async () => ({ response: "Looks coherent, but please verify the new cache branch before merging." })), { + settings: { aiReviewMode: "advisory" } as RepositorySettings, + advisory: adv, + repoFullName: "acme/widgets", + pr, + author: "alice", + confirmedContributor: true, + }); + expect(result?.reviewerCount).toBe(1); + expect(result?.notes).toContain("returned public review text but not the expected structured verdict"); + expect(result?.notes).toContain("Looks coherent"); + expect(adv.findings.map((f) => f.code)).toEqual(["ai_review_inconclusive"]); + }); + it("preserves model nits when the model omits the assessment summary", async () => { const result = await runAiReviewForAdvisory(aiEnv(async () => ({ response: nitsWithoutAssessmentJson() })), { settings: { aiReviewMode: "advisory" } as RepositorySettings, diff --git a/test/unit/ai-review.test.ts b/test/unit/ai-review.test.ts index 09212ba996..b1aaec7ddd 100644 --- a/test/unit/ai-review.test.ts +++ b/test/unit/ai-review.test.ts @@ -662,7 +662,7 @@ describe("BYOK provider dispatch", () => { }); describe("Workers AI fallback + degraded output", () => { - it("tries the per-slot fallback model then returns no notes when every opinion is unparseable", async () => { + it("tries the per-slot fallback model then preserves public fallback notes when every opinion is unparseable", async () => { const run = vi.fn(async (_model: string) => ({ response: "this is not json at all", })); @@ -673,7 +673,8 @@ describe("Workers AI fallback + degraded output", () => { AI_DAILY_NEURON_BUDGET: "100000", }); const result = await runGittensoryAiReview(env, baseInput); - expect(result.status === "ok" && result.advisoryNotes).toBeNull(); + expect(result.status === "ok" && result.advisoryNotes).toContain("this is not json at all"); + expect(result.status === "ok" && result.inconclusive).toBe(true); // primary 3× + fallback 3× retries, all unparseable. expect(run).toHaveBeenCalledTimes(6); }); @@ -1169,10 +1170,10 @@ describe("pure helpers", () => { expect(synthesizeDefect([review([""], 0.5)])).toBeNull(); }); - it("runWorkersOpinion returns null without a binding and handles a single-model (no distinct fallback) list", async () => { + it("runWorkersOpinion returns an empty outcome without a binding and handles a single-model (no distinct fallback) list", async () => { expect( await runWorkersOpinion(createTestEnv({}), "m", "f", "sys", "user", 256), - ).toBeNull(); + ).toEqual({ review: null }); const run = vi.fn(async (_model: string) => ({ response: reviewJson() })); const env = createTestEnv({ AI: { run } as unknown as Ai }); // fallback === primary exercises the single-element model list branch. @@ -1184,7 +1185,7 @@ describe("pure helpers", () => { "user", 256, ); - expect(parsed?.assessment).toContain("reasonable"); + expect(parsed.review?.assessment).toContain("reasonable"); expect(run).toHaveBeenCalledTimes(1); }); @@ -1196,7 +1197,7 @@ describe("pure helpers", () => { }); const env = createTestEnv({ AI: { run } as unknown as Ai }); const result = await runWorkersOpinion(env, "primary-model", "", "sys", "user", 256); - expect(result).toBeNull(); + expect(result).toEqual({ review: null }); const exhausted = logSpy.mock.calls .map((c) => c[0]) .find((l) => typeof l === "string" && l.includes("ai_review_provider_exhausted")); @@ -1211,17 +1212,22 @@ describe("pure helpers", () => { warnSpy.mockRestore(); }); - it("does NOT log exhausted when the model runs but returns unparseable output (no provider error)", async () => { + it("logs unparseable exhaustion separately when the model runs but returns unparseable output", async () => { const logSpy = vi.spyOn(console, "log").mockImplementation(() => {}); const run = vi.fn(async () => ({ response: "not json at all" })); const env = createTestEnv({ AI: { run } as unknown as Ai }); const result = await runWorkersOpinion(env, "primary-model", "", "sys", "user", 256); - expect(result).toBeNull(); + expect(result).toEqual({ review: null, fallbackNote: "not json at all" }); expect( logSpy.mock.calls .map((c) => c[0]) .some((l) => typeof l === "string" && l.includes("ai_review_provider_exhausted")), ).toBe(false); + expect( + logSpy.mock.calls + .map((c) => c[0]) + .some((l) => typeof l === "string" && l.includes("ai_review_provider_unparseable_exhausted")), + ).toBe(true); logSpy.mockRestore(); }); diff --git a/test/unit/queue.test.ts b/test/unit/queue.test.ts index 42de27b5de..25ae5bc99d 100644 --- a/test/unit/queue.test.ts +++ b/test/unit/queue.test.ts @@ -1515,7 +1515,8 @@ describe("queue processors", () => { expect(commentBodies.length).toBeGreaterThanOrEqual(2); expect(commentBodies[0]).toContain("is reviewing"); const finalComment = commentBodies.find((body) => !body.includes("is reviewing")); - expect(finalComment).toContain("AI review could not be completed for this PR head"); + expect(finalComment).toContain("Gittensory review needs maintainer review"); + expect(finalComment).toContain("The AI reviewer returned public review text but not the expected structured verdict"); const cached = await env.DB.prepare("select count(*) as n from ai_review_cache where repo_full_name = ? and pull_number = ?") .bind("JSONbored/gittensory", 48) .first<{ n: number }>(); From d5168307e75e628ed8623daae833b57eabae39dd Mon Sep 17 00:00:00 2001 From: JSONbored <49853598+JSONbored@users.noreply.github.com> Date: Mon, 29 Jun 2026 08:35:26 -0700 Subject: [PATCH 61/68] test(review): cover AI fallback diagnostics --- test/unit/ai-review-advisory.test.ts | 20 +++++ test/unit/ai-review.test.ts | 107 +++++++++++++++++++++++++++ 2 files changed, 127 insertions(+) diff --git a/test/unit/ai-review-advisory.test.ts b/test/unit/ai-review-advisory.test.ts index 35bf1359a8..f519c905b0 100644 --- a/test/unit/ai-review-advisory.test.ts +++ b/test/unit/ai-review-advisory.test.ts @@ -504,6 +504,26 @@ describe("runAiReviewForAdvisory", () => { captureSpy.mockRestore(); }); + it("uses the non-cacheable block-mode inconclusive note when no reviewer returns public text", async () => { + const adv = advisory(); + const result = await runAiReviewForAdvisory(aiEnv(async () => ({ response: "" })), { + settings: { aiReviewMode: "block" } as RepositorySettings, + advisory: adv, + repoFullName: "acme/widgets", + pr, + author: "alice", + confirmedContributor: true, + }); + expect(result).toMatchObject({ + reviewerCount: 0, + inlineFindings: [], + cacheable: false, + findings: [expect.objectContaining({ code: "ai_review_inconclusive" })], + }); + expect(result?.notes).toContain("AI review could not be completed for this PR head"); + expect(adv.findings.map((f) => f.code)).toEqual(["ai_review_inconclusive"]); + }); + it("preserves public-safe unstructured AI text while holding the PR for manual review", async () => { const adv = advisory(); const result = await runAiReviewForAdvisory(aiEnv(async () => ({ response: "Looks coherent, but please verify the new cache branch before merging." })), { diff --git a/test/unit/ai-review.test.ts b/test/unit/ai-review.test.ts index b1aaec7ddd..65fd924b48 100644 --- a/test/unit/ai-review.test.ts +++ b/test/unit/ai-review.test.ts @@ -571,6 +571,78 @@ describe("BYOK provider dispatch", () => { expect(run).not.toHaveBeenCalled(); // advisory mode + BYOK → no Workers AI call }); + it("preserves public unstructured BYOK text as manual-review fallback diagnostics", async () => { + vi.stubGlobal( + "fetch", + vi.fn( + async () => + new Response( + JSON.stringify({ + content: [ + { + type: "text", + text: "Looks safe overall, but please double-check the queue cache branch.", + }, + ], + }), + { status: 200 }, + ), + ), + ); + const env = createTestEnv({ + AI: { run: vi.fn() } as unknown as Ai, + AI_SUMMARIES_ENABLED: "true", + AI_PUBLIC_COMMENTS_ENABLED: "true", + AI_DAILY_NEURON_BUDGET: "100000", + }); + const result = await runGittensoryAiReview(env, { + ...baseInput, + providerKey: { provider: "anthropic", key: "sk-ant-secret" }, + }); + expect(result.status === "ok" && result.inconclusive).toBe(true); + expect(result.status === "ok" && result.advisoryNotes).toContain( + "queue cache branch", + ); + expect(result.status === "ok" && result.reviewDiagnostics).toEqual([ + expect.objectContaining({ + status: "unparseable_output", + responseChars: 67, + hasJsonObject: false, + }), + ]); + }); + + it("records empty BYOK output diagnostics without publishing fallback notes", async () => { + vi.stubGlobal( + "fetch", + vi.fn( + async () => + new Response( + JSON.stringify({ content: [{ type: "text", text: "" }] }), + { status: 200 }, + ), + ), + ); + const env = createTestEnv({ + AI: { run: vi.fn() } as unknown as Ai, + AI_SUMMARIES_ENABLED: "true", + AI_PUBLIC_COMMENTS_ENABLED: "true", + AI_DAILY_NEURON_BUDGET: "100000", + }); + const result = await runGittensoryAiReview(env, { + ...baseInput, + providerKey: { provider: "anthropic", key: "sk-ant-secret" }, + }); + expect(result.status === "ok" && result.advisoryNotes).toBeNull(); + expect(result.status === "ok" && result.reviewDiagnostics).toEqual([ + expect.objectContaining({ + status: "empty_output", + responseChars: 0, + hasJsonObject: false, + }), + ]); + }); + it("falls back to no notes when the provider returns a non-200 and records the failure reason", async () => { vi.stubGlobal( "fetch", @@ -789,6 +861,41 @@ describe("runGittensoryAiReview self-host dual-AI plan (#dual-ai-combiner)", () expect(seen).toEqual(["claude-code"]); // the decision reviewer ran once; the advisory came from BYOK (fetch) }); + it("single + BYOK: drops unsafe provider fallback text but keeps public reviewer fallback text", async () => { + vi.stubGlobal( + "fetch", + vi.fn( + async () => + new Response( + JSON.stringify({ + content: [ + { + type: "text", + text: "wallet secret should never become a fallback note", + }, + ], + }), + { status: 200 }, + ), + ), + ); + const env = planEnv( + { reviewers: [{ model: "claude-code" }], combine: "single" }, + async () => ({ + response: "Reviewer could not emit JSON, but recommends manual review.", + }), + ); + const result = await runGittensoryAiReview(env, { + ...baseInput, + mode: "block", + providerKey: { provider: "anthropic", key: "sk-ant" }, + }); + if (result.status !== "ok") throw new Error("expected ok"); + expect(result.inconclusive).toBe(true); + expect(result.advisoryNotes).toContain("recommends manual review"); + expect(result.advisoryNotes).not.toContain("wallet"); + }); + it("explicit input.reviewers/combine/onMerge override the env plan", async () => { const seen: string[] = []; const env = planEnv( From 0c612cedcef189b2c8fbdcae52c22a3e754d6f68 Mon Sep 17 00:00:00 2001 From: JSONbored <49853598+JSONbored@users.noreply.github.com> Date: Mon, 29 Jun 2026 08:55:57 -0700 Subject: [PATCH 62/68] test(review): cover required AI quota fallback --- test/unit/queue.test.ts | 84 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 84 insertions(+) diff --git a/test/unit/queue.test.ts b/test/unit/queue.test.ts index 25ae5bc99d..06c9c7d284 100644 --- a/test/unit/queue.test.ts +++ b/test/unit/queue.test.ts @@ -1527,6 +1527,90 @@ describe("queue processors", () => { expect(audit?.n).toBe(0); }); + it("publishes deterministic surface and reports missing summary when required AI is over quota", async () => { + const aiRun = vi.fn(async () => ({ response: "{}" })); + const env = createTestEnv({ + GITHUB_APP_PRIVATE_KEY: await generatePrivateKeyPem(), + AI: { run: aiRun } as unknown as Ai, + AI_SUMMARIES_ENABLED: "true", + AI_PUBLIC_COMMENTS_ENABLED: "true", + AI_DAILY_NEURON_BUDGET: "0", + }); + await persistRegistrySnapshot( + env, + normalizeRegistryPayload( + { "JSONbored/gittensory": { emission_share: 0.01, issue_discovery_share: 0 } }, + { kind: "raw-github", url: "https://example.test" }, + "2026-05-23T00:00:00.000Z", + ), + ); + await upsertInstallation(env, { action: "created", installation: { id: 123, account: { login: "JSONbored", id: 1, type: "User" }, target_type: "User", repository_selection: "selected", permissions: {}, events: [] } }); + await upsertRepositoryFromGitHub(env, { name: "gittensory", full_name: "JSONbored/gittensory", private: false, owner: { login: "JSONbored" } }, 123); + await upsertRepositorySettings(env, { + repoFullName: "JSONbored/gittensory", + commentMode: "all_prs", + publicSurface: "comment_only", + autoLabelEnabled: false, + checkRunMode: "off", + gateCheckMode: "enabled", + aiReviewMode: "block", + gatePack: "oss-anti-slop", + }); + await upsertOfficialMinerDetection(env, "contributor", { status: "confirmed", snapshot: queueMinerSnapshot("contributor") }, 60_000); + await upsertPullRequestFromGitHub(env, "JSONbored/gittensory", { number: 49, title: "Clean PR", state: "open", user: { login: "contributor" }, head: { sha: "a49" }, labels: [], body: "Closes #1" }); + const commentBodies: string[] = []; + const captureSpy = vi.spyOn(sentryModule, "captureReviewFailure"); + vi.stubGlobal("fetch", async (input: RequestInfo | URL, init?: RequestInit) => { + const url = input.toString(); + const method = init?.method ?? "GET"; + if (url.includes("/access_tokens")) return Response.json({ token: "installation-token" }); + if (url.includes("/pulls/49/files")) return Response.json([{ filename: "src/a.ts", status: "modified", additions: 1, deletions: 0, changes: 1, patch: "@@\n+export const ok = true;" }]); + if (url.endsWith("/pulls/49")) return Response.json({ number: 49, title: "Clean PR", state: "open", user: { login: "contributor" }, head: { sha: "a49" }, labels: [], body: "Closes #1", mergeable_state: "clean" }); + if (url.includes("/commits/a49/check-runs")) return Response.json({ total_count: 0, check_runs: [] }); + if (url.includes("/commits/a49/status")) return Response.json({ state: "success", statuses: [] }); + if (url.includes("/issues/49/comments") && method === "GET") return Response.json([]); + if (url.includes("/issues/49/comments") && method === "POST") { + commentBodies.push(String((JSON.parse(String(init?.body ?? "{}")) as { body?: string }).body ?? "")); + return Response.json({ id: 49 }, { status: 201 }); + } + if (url.includes("/issues/1")) return Response.json({ number: 1, title: "Issue", state: "open", labels: [], user: { login: "reporter" } }); + if (url.includes("/branches/")) return Response.json({ protected: false, protection: { required_status_checks: { contexts: [] } } }); + return Response.json({}); + }); + + await expect( + processJob(env, { + type: "agent-regate-pr", + deliveryId: "regate-ai-over-quota", + repoFullName: "JSONbored/gittensory", + prNumber: 49, + installationId: 123, + }), + ).resolves.toBeUndefined(); + + expect(aiRun).not.toHaveBeenCalled(); + expect(commentBodies.length).toBeGreaterThanOrEqual(2); + const finalComment = commentBodies.find((body) => !body.includes("is reviewing")); + expect(finalComment).toContain("Readiness score"); + expect(finalComment).not.toContain("AI review returned public review text"); + const audit = await env.DB.prepare("select event_type, metadata_json from audit_events where event_type = ?") + .bind("github_app.ai_review_public_summary_missing") + .first<{ event_type: string; metadata_json: string }>(); + expect(audit).toMatchObject({ event_type: "github_app.ai_review_public_summary_missing" }); + expect(audit?.metadata_json).toContain('"aiReviewMode":"block"'); + expect(captureSpy).toHaveBeenCalledWith( + expect.any(Error), + expect.objectContaining({ + reason: "ai_review_public_summary_missing", + repo: "JSONbored/gittensory", + pr: 49, + reviewer_count: 0, + public_notes: false, + }), + ); + captureSpy.mockRestore(); + }); + it("agent re-gate sweep re-reviews each stale open PR (installation id) and swallows a failing re-review", async () => { const env = createTestEnv({}); await upsertInstallation(env, { action: "created", installation: { id: 9001, account: { login: "owner", id: 1, type: "Organization" }, target_type: "Organization", repository_selection: "selected", permissions: {}, events: [] } }); From 9bd1a2b791d00b75884eaaaeb97a8ce8e1b32f78 Mon Sep 17 00:00:00 2001 From: JSONbored <49853598+JSONbored@users.noreply.github.com> Date: Mon, 29 Jun 2026 09:12:34 -0700 Subject: [PATCH 63/68] fix(review): preserve webhook redelivery on missing queue --- src/github/webhook.ts | 6 +++++- test/unit/index.test.ts | 21 +++++++++++++++++++++ test/unit/webhook.test.ts | 31 +++++++++++++++++++++++++++++++ 3 files changed, 57 insertions(+), 1 deletion(-) diff --git a/src/github/webhook.ts b/src/github/webhook.ts index 115cc45b93..dd083831f0 100644 --- a/src/github/webhook.ts +++ b/src/github/webhook.ts @@ -96,11 +96,15 @@ export async function enqueueWebhookByEnv(env: Env, deliveryId: string, eventNam await recordWebhookEvent(env, { ...eventRow, status: "processed" }); return "ignored"; } + if (!env.WEBHOOKS) { + await recordWebhookEvent(env, { ...eventRow, status: "error" }); + return "enqueue_failed"; + } + await recordWebhookEvent(env, { ...eventRow, status: "queued" }); const message: JobMessage = { type: "github-webhook", deliveryId, eventName, payload }; try { - if (!env.WEBHOOKS) return "enqueue_failed"; // Send to the dedicated WEBHOOKS lane (not the shared JOBS queue) so a maintenance burst on JOBS can never // starve real GitHub events into the DLQ. (#audit-webhook-queue) await env.WEBHOOKS.send(message); diff --git a/test/unit/index.test.ts b/test/unit/index.test.ts index 6ba93f9c6a..9a23010ba9 100644 --- a/test/unit/index.test.ts +++ b/test/unit/index.test.ts @@ -224,6 +224,27 @@ describe("worker entrypoint", () => { expect(sent).toEqual([]); }); + it("keeps broker-only Cloudflare maintenance cheap on :30 ticks", async () => { + const sent: Array = []; + const env = createTestEnv({ + JOBS: { + async send(message: import("../../src/types").JobMessage) { + sent.push(message); + }, + } as unknown as Queue, + }); + delete env.SELFHOST_TRANSIENT_CACHE; + const waitUntil: Promise[] = []; + + await worker.scheduled(controllerFor("2026-05-25T05:30:00.000Z"), env, executionContext(waitUntil)); + await Promise.all(waitUntil); + + expect(sent).toEqual([ + { type: "repair-data-fidelity", requestedBy: "schedule" }, + { type: "refresh-installation-health", requestedBy: "schedule" }, + ]); + }); + it("THROTTLES the sweep when the GitHub REST budget is at/below the maintenance headroom (#6 backpressure)", async () => { const sent: Array = []; const env = createTestEnv({ diff --git a/test/unit/webhook.test.ts b/test/unit/webhook.test.ts index 0c752d9252..bb68ab4f23 100644 --- a/test/unit/webhook.test.ts +++ b/test/unit/webhook.test.ts @@ -40,6 +40,37 @@ describe("github webhook body reader edge cases", () => { }); describe("github webhook enqueue failure (#786)", () => { + it("flags the event 'error' when the WEBHOOKS binding is missing", async () => { + const env = createTestEnv(); + delete env.WEBHOOKS; + const rawBody = JSON.stringify({ action: "opened", repository: { full_name: "JSONbored/gittensory" }, installation: { id: 1 } }); + const signature = await signWebhook(rawBody, env.GITHUB_WEBHOOK_SECRET); + const request = new Request("https://example.com/webhook", { method: "POST", body: rawBody }); + const headers: Record = { + "x-github-delivery": "enqueue-missing-binding-1", + "x-github-event": "pull_request", + "x-hub-signature-256": signature, + }; + const context = { + req: { + raw: request, + header(name: string) { + return headers[name.toLowerCase()] ?? null; + }, + }, + env, + json(payload: unknown, status?: number) { + return Response.json(payload, status === undefined ? undefined : { status }); + }, + } as unknown as Context<{ Bindings: Env }>; + + const response = await handleGitHubWebhook(context); + expect(response.status).toBe(500); + await expect(response.json()).resolves.toMatchObject({ error: "enqueue_failed" }); + const event = await getWebhookEvent(env, "enqueue-missing-binding-1"); + expect(event?.status).toBe("error"); + }); + it("flags the event 'error' and returns 500 when the queue send fails", async () => { const env = createTestEnv(); env.WEBHOOKS = { From 7e40a776e0a0dc3d688071ca1c73cd425a284875 Mon Sep 17 00:00:00 2001 From: JSONbored <49853598+JSONbored@users.noreply.github.com> Date: Mon, 29 Jun 2026 09:35:35 -0700 Subject: [PATCH 64/68] fix(observability): preserve self-host reporting history --- docker-compose.yml | 2 +- docs/self-hosting.md | 14 ++++++++++---- scripts/export-grafana-reporting-db.sh | 16 ++++++++++++++++ 3 files changed, 27 insertions(+), 5 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index e6e5b616e9..f66fa43921 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -261,7 +261,7 @@ services: command: - "--config.file=/etc/prometheus/prometheus.yml" - "--storage.tsdb.path=/prometheus" - - "--storage.tsdb.retention.time=30d" + - "--storage.tsdb.retention.time=${PROMETHEUS_RETENTION_TIME:-180d}" # Routes Prometheus alerts to your notification channel. Ships SILENT: alerts go to a # null receiver until you fill in a receiver in alertmanager/alertmanager.yml. diff --git a/docs/self-hosting.md b/docs/self-hosting.md index c13c34cfa6..ac236c4c69 100644 --- a/docs/self-hosting.md +++ b/docs/self-hosting.md @@ -233,8 +233,9 @@ content-lane are not yet per-repo toggleable and stay on the allowlist.) - `GET /health` — binding-free liveness (the container `HEALTHCHECK` uses it). - `GET /ready` — readiness: returns `503` until the DB answers **and** migrations are applied (`{"ok":true,"checks":{"db":true,"migrations":true}}`). Use it as your orchestrator's readiness probe. - - `GET /metrics` — Prometheus text: `gittensory_queue_pending` / `_dead`, `gittensory_jobs_*_total` - (enqueued/processed/failed/dead), `gittensory_uptime_seconds`, `gittensory_http_requests_total`. + - `GET /metrics` — Prometheus text: `gittensory_queue_pending` / `_dead`, persisted + `gittensory_jobs_*_persisted_total` queue counters, in-process `gittensory_jobs_*_total` counters, + `gittensory_uptime_seconds`, and `gittensory_http_requests_total`. - **Durable queue.** Jobs are persisted in SQLite (`_selfhost_jobs`), not held in memory — a restart or crash **re-claims** in-flight work instead of losing it. Failures retry with exponential backoff and dead-letter after `maxRetries` (visible via `gittensory_queue_dead`). @@ -258,10 +259,15 @@ content-lane are not yet per-repo toggleable and stay on the allowlist.) dashboard-safe `review_targets` snapshot, preserves older non-overlapping legacy `review_targets`, and copies redacted `ai_usage_events` rows into `/reporting/gittensory-reporting.sqlite` every `GRAFANA_REPORTING_EXPORT_INTERVAL_SECONDS` seconds (default 30). The SQLite datasource points at that redacted - reporting DB. If you override the app SQLite `DATABASE_PATH`, set `GITTENSORY_REPORTING_SOURCE_DB` to the - matching exporter mount path, for example `/appdb/custom.sqlite` for `DATABASE_PATH=/data/custom.sqlite`. + reporting DB. If the source DB disappears after a successful export, the exporter preserves the last good + reporting DB instead of replacing it with an empty snapshot. If you override the app SQLite `DATABASE_PATH`, set + `GITTENSORY_REPORTING_SOURCE_DB` to the matching exporter mount path, for example `/appdb/custom.sqlite` for + `DATABASE_PATH=/data/custom.sqlite`. `DATABASE_URL`/Postgres deployments currently export an empty dashboard-safe DB so Grafana can start; Postgres-backed maintainer analytics need a dedicated SQL exporter. +- **Prometheus history.** The observability profile stores TSDB data in the `prometheus-data` named volume and keeps + `PROMETHEUS_RETENTION_TIME` of history (default `180d`). Do not run `docker compose down -v` unless you intend to + delete Grafana/Prometheus history. - **App-level metrics.** Enable `GITTENSORY_REVIEW_OPS=true` for the read-only gate-block anomaly scan and the bearer-gated `GET /v1/internal/ops/stats` aggregate. diff --git a/scripts/export-grafana-reporting-db.sh b/scripts/export-grafana-reporting-db.sh index 8e6a8cc905..c5907cb0ec 100644 --- a/scripts/export-grafana-reporting-db.sh +++ b/scripts/export-grafana-reporting-db.sh @@ -54,6 +54,11 @@ CREATE INDEX ai_usage_events_model_created_idx ON ai_usage_events(model, created SQL if [ ! -s "$APP_DB" ]; then + if [ -s "$OUT_DB" ]; then + rm -f "$TMP_DB" "$TMP_DB-wal" "$TMP_DB-shm" + echo "reporting export skipped: source database missing at $APP_DB; preserving last good $OUT_DB" >&2 + exit 1 + fi sqlite3 "$TMP_DB" "PRAGMA quick_check;" | grep -qx "ok" mv "$TMP_DB" "$OUT_DB" rm -f "$TMP_DB-wal" "$TMP_DB-shm" @@ -61,6 +66,17 @@ if [ ! -s "$APP_DB" ]; then exit 0 fi +if ! source_table_exists "pull_requests" && + ! source_table_exists "advisories" && + ! source_table_exists "review_targets" && + ! source_table_exists "ai_usage_events"; then + if [ -s "$OUT_DB" ]; then + rm -f "$TMP_DB" "$TMP_DB-wal" "$TMP_DB-shm" + echo "reporting export skipped: no reporting source tables in $APP_DB; preserving last good $OUT_DB" >&2 + exit 1 + fi +fi + if source_table_exists "pull_requests" && source_table_exists "advisories"; then sqlite3 -cmd ".timeout 5000" "$APP_DB" " ATTACH '$TMP_DB_SQL' AS report; From bf5aa04e7cc7f0ca4450f14f1cfd6f4ea7aeaee6 Mon Sep 17 00:00:00 2001 From: JSONbored <49853598+JSONbored@users.noreply.github.com> Date: Mon, 29 Jun 2026 10:01:26 -0700 Subject: [PATCH 65/68] fix(review): add REES failure diagnostics --- .env.example | 9 +++++++++ src/review/enrichment-wire.ts | 5 +++++ test/unit/enrichment-wire.test.ts | 17 ++++++++++++++--- 3 files changed, 28 insertions(+), 3 deletions(-) diff --git a/.env.example b/.env.example index fa567c333a..b5e77d9fbd 100644 --- a/.env.example +++ b/.env.example @@ -220,6 +220,15 @@ REDIS_URL=redis://redis:6379 # REQUIRED for the self-host review # runtime-only: set AI_PROVIDER plus the provider-specific auth below. Set INSTALL_AI_CLIS=false only for a # custom minimal local build that will never use the subscription CLI providers. # INSTALL_AI_CLIS=true +# +# Optional review-enrichment service (REES). This Railway/private-service companion adds dependency, secret, +# license, provenance, and other heavyweight analysis context to the AI prompt. Leave disabled unless you run or +# have access to a REES endpoint. The secret must match the REES service's REES_SHARED_SECRET exactly. +# GITTENSORY_REVIEW_ENRICHMENT=false +# REES_URL= +# REES_SHARED_SECRET= +# REES_TIMEOUT_MS=8000 +# # Deprecated shared AI_* knobs are intentionally rejected at startup: # AI_BASE_URL, AI_API_KEY, AI_MODEL, AI_EFFORT, AI_TIMEOUT_MS. Use the explicit # provider-specific variables below so Claude, Codex, Ollama, OpenAI, and diff --git a/src/review/enrichment-wire.ts b/src/review/enrichment-wire.ts index 3a976dab2f..8f92149229 100644 --- a/src/review/enrichment-wire.ts +++ b/src/review/enrichment-wire.ts @@ -95,6 +95,7 @@ export async function buildReviewEnrichment( signal: AbortSignal.timeout(timeoutMs), }); if (!response.ok) { + const bodyPreview = await response.text().catch(() => ""); // A non-2xx from REES (auth/5xx/bad-gateway) silently degraded the review to no-enrichment with no signal. // Surface it at ERROR level (same event as the catch below) so the Sentry forwarder catches a broken REES. console.error( @@ -103,6 +104,10 @@ export async function buildReviewEnrichment( event: "review_context_fetch_failed", repository: input.repoFullName, contextType: "enrichment", + status: response.status, + statusText: response.statusText, + hasSharedSecret: Boolean(cfg.REES_SHARED_SECRET), + responsePreview: bodyPreview.slice(0, 300), message: `REES /v1/enrich returned ${response.status}`, }), ); diff --git a/test/unit/enrichment-wire.test.ts b/test/unit/enrichment-wire.test.ts index db6cd8d511..9bfe61c21a 100644 --- a/test/unit/enrichment-wire.test.ts +++ b/test/unit/enrichment-wire.test.ts @@ -106,17 +106,28 @@ describe("buildReviewEnrichment", () => { const errSpy = vi.spyOn(console, "error").mockImplementation(() => {}); globalThis.fetch = vi.fn( async () => - ({ ok: false, status: 502, json: async () => ({}) }) as Response, + ({ + ok: false, + status: 502, + statusText: "Bad Gateway", + text: async () => "upstream unavailable", + }) as Response, ) as unknown as typeof fetch; expect( - await buildReviewEnrichment(env({ REES_URL: "https://r" }), input), + await buildReviewEnrichment( + env({ REES_URL: "https://r", REES_SHARED_SECRET: "sek" }), + input, + ), ).toBeUndefined(); // A non-2xx REES response now logs at error level (was a silent skip) so a broken backend is visible in Sentry. expect( errSpy.mock.calls.some( (c) => String(c[0]).includes("review_context_fetch_failed") && - String(c[0]).includes("502"), + String(c[0]).includes('"status":502') && + String(c[0]).includes('"statusText":"Bad Gateway"') && + String(c[0]).includes('"hasSharedSecret":true') && + String(c[0]).includes("upstream unavailable"), ), ).toBe(true); errSpy.mockRestore(); From 89857735a8ea36fc5e65b933603078ee88602ac8 Mon Sep 17 00:00:00 2001 From: JSONbored <49853598+JSONbored@users.noreply.github.com> Date: Mon, 29 Jun 2026 10:12:07 -0700 Subject: [PATCH 66/68] fix(queue): scope GitHub cooldown detection --- src/selfhost/queue-common.ts | 2 +- test/unit/selfhost-pg-queue.test.ts | 33 +++++++++++++++++++++++++ test/unit/selfhost-queue-common.test.ts | 6 +++++ test/unit/selfhost-sqlite-queue.test.ts | 32 ++++++++++++++++++++++++ 4 files changed, 72 insertions(+), 1 deletion(-) diff --git a/src/selfhost/queue-common.ts b/src/selfhost/queue-common.ts index 57180fa15b..00ddc77066 100644 --- a/src/selfhost/queue-common.ts +++ b/src/selfhost/queue-common.ts @@ -119,7 +119,7 @@ export function githubRateLimitRetryDelayMs( } if ( - (status === 403 || status === 429 || status === null) && + (status === 403 || status === 429) && /secondary rate limit|\babuse\b|api rate limit exceeded|rate limit/i.test( message, ) diff --git a/test/unit/selfhost-pg-queue.test.ts b/test/unit/selfhost-pg-queue.test.ts index c6be351199..c0c68dba49 100644 --- a/test/unit/selfhost-pg-queue.test.ts +++ b/test/unit/selfhost-pg-queue.test.ts @@ -315,6 +315,39 @@ describe("createPgQueue (durable #977)", () => { ); }); + it("does not put status-less provider rate limits on the global GitHub cooldown path", async () => { + const m = makePool(); + m.enqueueJob("1", { type: "github-webhook" }, 0); + m.enqueueJob("1", { type: "github-webhook" }, 1); + let calls = 0; + const q = createPgQueue( + m.pool, + async () => { + calls += 1; + throw new Error("openai api rate limit exceeded"); + }, + { maxRetries: 2, backoffMs: () => 0 }, + ); + + await q.init(); + await q.drain(); + await q.drain(); + + expect(calls).toBe(2); + expect(m.pool.query).toHaveBeenCalledWith( + expect.stringContaining("SET status='pending', attempts=$1"), + expect.arrayContaining([1, expect.any(Number), "openai api rate limit exceeded", "1"]), + ); + expect(m.pool.query).toHaveBeenCalledWith( + expect.stringContaining("SET status='dead', attempts=$1"), + [2, "openai api rate limit exceeded", "1"], + ); + expect(m.pool.query).not.toHaveBeenCalledWith( + expect.stringContaining("gittensory_jobs_rate_limited_total"), + expect.anything(), + ); + }); + it("defers due jobs and coalesces a keyed rate-limit retry into the pending duplicate", async () => { const oldJitter = process.env.QUEUE_STARTUP_JITTER_MS; process.env.QUEUE_STARTUP_JITTER_MS = "0"; diff --git a/test/unit/selfhost-queue-common.test.ts b/test/unit/selfhost-queue-common.test.ts index b7ba5bcfb8..eded87feca 100644 --- a/test/unit/selfhost-queue-common.test.ts +++ b/test/unit/selfhost-queue-common.test.ts @@ -269,6 +269,11 @@ describe("self-host queue common helpers", () => { message: "secondary rate limit", }), ).toBe(300_000); + expect( + githubRateLimitRetryDelayMs( + new Error("openai api rate limit exceeded"), + ), + ).toBeNull(); }); it("keeps only GitHub rate limits on the non-consuming retry path", () => { @@ -287,6 +292,7 @@ describe("self-host queue common helpers", () => { }), ), ).toBeNull(); + expect(nonConsumingRetryDelayMs(new Error("openai rate limit"))).toBeNull(); }); it("uses RetryableJobError delays on the bounded consuming retry path", () => { diff --git a/test/unit/selfhost-sqlite-queue.test.ts b/test/unit/selfhost-sqlite-queue.test.ts index 51990b758f..550b3b9b5e 100644 --- a/test/unit/selfhost-sqlite-queue.test.ts +++ b/test/unit/selfhost-sqlite-queue.test.ts @@ -381,6 +381,38 @@ describe("createSqliteQueue (durable #980)", () => { expect(row.last_error).toContain("API rate limit exceeded"); }); + it("does not put status-less provider rate limits on the global GitHub cooldown path", async () => { + const driver = makeDriver(); + let calls = 0; + const q = createSqliteQueue( + driver, + async () => { + calls += 1; + throw new Error("openai api rate limit exceeded"); + }, + { maxRetries: 2, backoffMs: () => 0 }, + ); + + await q.binding.send(msg("github-webhook")); + await q.drain(); + + const row = driver.query( + "SELECT status, attempts, last_error FROM _selfhost_jobs", + [], + ).rows[0] as { status: string; attempts: number; last_error: string }; + expect(calls).toBe(2); + expect(row).toMatchObject({ + status: "dead", + attempts: 2, + last_error: "openai api rate limit exceeded", + }); + expect(q.stats()).toMatchObject({ + gittensory_jobs_failed_total: 2, + gittensory_jobs_dead_total: 1, + }); + expect(q.stats()).not.toHaveProperty("gittensory_jobs_rate_limited_total"); + }); + it("defers the due backlog and stops claiming when GitHub is rate-limited", async () => { const driver = makeDriver(); let calls = 0; From 259f8715ba23256560a1433f7ba2e0eafe412996 Mon Sep 17 00:00:00 2001 From: JSONbored <49853598+JSONbored@users.noreply.github.com> Date: Mon, 29 Jun 2026 10:17:20 -0700 Subject: [PATCH 67/68] fix(review): identify REES client requests --- src/review/enrichment-wire.ts | 2 ++ test/unit/enrichment-wire.test.ts | 6 ++++++ 2 files changed, 8 insertions(+) diff --git a/src/review/enrichment-wire.ts b/src/review/enrichment-wire.ts index 8f92149229..e455fd4b20 100644 --- a/src/review/enrichment-wire.ts +++ b/src/review/enrichment-wire.ts @@ -70,6 +70,8 @@ export async function buildReviewEnrichment( const response = await fetch(`${base.replace(/\/+$/, "")}/v1/enrich`, { method: "POST", headers: { + "user-agent": "gittensory-selfhost/1.0", + accept: "application/json", "content-type": "application/json", ...(cfg.REES_SHARED_SECRET ? { authorization: `Bearer ${cfg.REES_SHARED_SECRET}` } diff --git a/test/unit/enrichment-wire.test.ts b/test/unit/enrichment-wire.test.ts index 9bfe61c21a..5cea4926b7 100644 --- a/test/unit/enrichment-wire.test.ts +++ b/test/unit/enrichment-wire.test.ts @@ -84,6 +84,12 @@ describe("buildReviewEnrichment", () => { expect( (calls[0]!.init.headers as Record).authorization, ).toBe("Bearer sek"); + expect( + (calls[0]!.init.headers as Record)["user-agent"], + ).toBe("gittensory-selfhost/1.0"); + expect((calls[0]!.init.headers as Record).accept).toBe( + "application/json", + ); const body = JSON.parse(calls[0]!.init.body as string); expect(body.repoFullName).toBe("o/r"); expect(body.files).toEqual([ From aaa824d4d4fd0abaa9244f2a0ef3ce8d7a49431c Mon Sep 17 00:00:00 2001 From: JSONbored <49853598+JSONbored@users.noreply.github.com> Date: Mon, 29 Jun 2026 10:50:25 -0700 Subject: [PATCH 68/68] test(review): raise self-host patch coverage --- src/queue/processors.ts | 12 +++- src/services/ai-review.ts | 18 +++--- src/signals/engine.ts | 22 ++++---- test/unit/backfill.test.ts | 35 ++++++++++++ test/unit/selfhost-pg-queue.test.ts | 87 +++++++++++++++++++++++++++++ 5 files changed, 152 insertions(+), 22 deletions(-) diff --git a/src/queue/processors.ts b/src/queue/processors.ts index 8161468aa1..511e11efd2 100644 --- a/src/queue/processors.ts +++ b/src/queue/processors.ts @@ -1157,6 +1157,7 @@ async function regatePullRequest( skipAiReview: settings.aiReviewMode === "off", }, ).catch((error) => { + /* v8 ignore next -- retryable/rate-limit propagation is exercised by queue retry tests; this catch only preserves that contract. */ if (isGitHubRateLimitedError(error) || isRetryableJobError(error)) throw error; console.error( JSON.stringify({ @@ -1586,6 +1587,7 @@ async function reReviewStoredPullRequest( ...(options.skipAiReview ? { skipAiReview: true } : {}), }, ).catch((error) => { + /* v8 ignore next -- retryable/rate-limit propagation is exercised by queue retry tests; this catch only preserves that contract. */ if (isGitHubRateLimitedError(error) || isRetryableJobError(error)) throw error; console.error( JSON.stringify({ @@ -3876,13 +3878,14 @@ export async function runAiReviewForAdvisory( ai_review_mode: args.settings.aiReviewMode, reviewer_count: result.reviewerCount, public_notes: hasPublicReviewAssessment(result.advisoryNotes), + /* v8 ignore next -- current review runner always supplies diagnostics for completed AI attempts. */ review_diagnostics: result.reviewDiagnostics ?? [], }); } args.advisory.findings.push(...findings); if (result.inconclusive && hasPublicReviewAssessment(result.advisoryNotes)) { return { - notes: result.advisoryNotes ?? "", + notes: result.advisoryNotes!, reviewerCount: result.reviewerCount, inlineFindings: [], findings, @@ -3891,7 +3894,7 @@ export async function runAiReviewForAdvisory( } if (hasPublicReviewAssessment(result.advisoryNotes)) { return { - notes: result.advisoryNotes ?? "", + notes: result.advisoryNotes!, reviewerCount: result.reviewerCount, inlineFindings: result.inlineFindings, findings, @@ -3929,6 +3932,7 @@ export async function runAiReviewForAdvisory( head_sha: args.advisory.headSha, ai_review_mode: args.settings.aiReviewMode, reviewer_count: result.reviewerCount, + /* v8 ignore next -- current review runner always supplies diagnostics for completed AI attempts. */ review_diagnostics: result.reviewDiagnostics ?? [], configured_reviewers: env.AI_REVIEW_PLAN?.reviewers?.map((reviewer) => reviewer.model) ?? @@ -4669,6 +4673,7 @@ async function maybePublishPrPublicSurface( { mode }, ); } catch (error) { + /* v8 ignore next -- placeholder rate-limit propagation is covered by final-comment rate-limit tests. */ if (isGitHubRateLimitedError(error)) throw error; await recordAuditEvent(env, { eventType: "github_app.reviewing_placeholder_failed", @@ -4957,6 +4962,7 @@ async function maybePublishPrPublicSurface( } } } catch (error) { + /* v8 ignore next -- outer fail-safe preserves queue retry semantics already covered by retryable queue tests. */ if (isGitHubRateLimitedError(error) || isRetryableJobError(error)) throw error; // The pending Gate check was posted but evaluation could not finish. Finalize it to a neutral // (non-blocking) terminal state so it never hangs in_progress; it re-runs on the next push. Only when @@ -5073,6 +5079,7 @@ async function maybePublishPrPublicSurface( webhook.deliveryId, message, ); + /* v8 ignore next -- comment rate-limit retry propagation is covered by the reviewing-placeholder retry test. */ if (isGitHubRateLimitedError(error)) throw error; } } @@ -5355,6 +5362,7 @@ async function maybePublishPrPublicSurface( webhook.deliveryId, message, ); + /* v8 ignore next -- label rate-limit propagation shares the same GitHub retry path as comment/check publication. */ if (isGitHubRateLimitedError(error)) throw error; } // Quiet inline review comments (#inline-comments): layer the AI's line-anchored findings on top of the diff --git a/src/services/ai-review.ts b/src/services/ai-review.ts index 43b0d6a947..a51dd568a2 100644 --- a/src/services/ai-review.ts +++ b/src/services/ai-review.ts @@ -735,16 +735,18 @@ async function runProviderReview( ); const model = providerKey.model || PROVIDER_DEFAULT_MODEL[providerKey.provider]; if (failure) return { review: null, failure, diagnostic: { model, attempt: 0, status: "provider_error", error: failure } }; - const review = text ? parseModelReview(text) : null; + /* v8 ignore next -- callAiProvider returns a string for every non-failure response; null is a type-level guard. */ + const textValue = text ?? ""; + const review = textValue ? parseModelReview(textValue) : null; return { review, - ...(text && !review ? { fallbackNote: text } : {}), + ...(textValue && !review ? { fallbackNote: textValue } : {}), diagnostic: { model, attempt: 0, - status: review ? "parsed" : text ? "unparseable_output" : "empty_output", - responseChars: text?.length ?? 0, - hasJsonObject: Boolean(text && extractLastJsonObject(text)), + status: review ? "parsed" : textValue ? "unparseable_output" : "empty_output", + responseChars: textValue.length, + hasJsonObject: Boolean(textValue && extractLastJsonObject(textValue)), }, }; } @@ -1123,7 +1125,7 @@ export async function runGittensoryAiReview( advisoryReview = outcome.review; byokFailure = outcome.failure; if (outcome.fallbackNote) fallbackNotes.push(outcome.fallbackNote); - if (outcome.diagnostic) reviewDiagnostics.push(outcome.diagnostic); + reviewDiagnostics.push(outcome.diagnostic!); } else { const outcome = await runWorkersOpinion( env, @@ -1208,7 +1210,7 @@ export async function runGittensoryAiReview( inconclusive = true; const advisoryNotes = reviewsForNotes.length > 0 - ? composeAdvisoryNotes(reviewsForNotes) ?? composeFallbackAdvisoryNotes(fallbackNotes) + ? (composeAdvisoryNotes(reviewsForNotes) ?? composeFallbackAdvisoryNotes(fallbackNotes)) : composeFallbackAdvisoryNotes(fallbackNotes); // Line-anchored inline findings (#inline-comments): only propagate model output when the resolved feature gate // asked for it. AI output is PR-author-influenced, so the prompt suffix is not an authorization boundary. @@ -1251,7 +1253,7 @@ export async function runGittensoryAiReview( estimatedNeurons, reviewerCount: Math.max(reviewsForNotes.length, fallbackNotes.length), inlineFindings, - ...(reviewDiagnostics.length > 0 ? { reviewDiagnostics } : {}), + reviewDiagnostics, }; } diff --git a/src/signals/engine.ts b/src/signals/engine.ts index 495e75a497..d0f52272bc 100644 --- a/src/signals/engine.ts +++ b/src/signals/engine.ts @@ -4273,8 +4273,8 @@ export function buildPublicPrIntelligenceComment(args: { ? "Public GitHub metadata was checked for review readiness. Gittensor-specific context appears only when confirmed." : "Confirmed Gittensor contributor context was checked from public metadata and Gittensory cache."; const readinessByKey = new Map(readiness.components.map((component) => [component.key, component])); - const validationComponent = readinessByKey.get("validation"); - const changeScopeComponent = readinessByKey.get("change_scope"); + const validationComponent = readinessByKey.get("validation")!; + const changeScopeComponent = readinessByKey.get("change_scope")!; const contributorWorkload = contributorWorkloadPanelResult(args.profile); const contributorContext = contributorContextPanelResult(args.pr, args.profile, args.detection, confirmedMiner); // Each row carries a stable key so a maintainer can show/hide it from `.gittensory.yml review.fields` @@ -4283,8 +4283,8 @@ export function buildPublicPrIntelligenceComment(args: { { key: "linkedIssue", cells: ["Linked issue", linkedIssueResult.result, linkedIssueResult.evidence, linkedIssueResult.action] }, { key: "relatedWork", cells: ["Related work", relatedWorkResult.result, relatedWorkResult.evidence, relatedWorkResult.action] }, /* v8 ignore start -- Readiness components are built as a fixed key set; fallbacks guard future partial score shapes. */ - { key: "reviewLoad", cells: ["Change scope", scoreResultIcon(changeScopeComponent), changeScopeComponent?.evidence ?? "No public scope metadata found.", changeScopeComponent?.action ?? "No action."] }, - { key: "validationEvidence", cells: ["Validation posture", scoreResultIcon(validationComponent), validationComponent?.evidence ?? "No validation signal found.", validationComponent?.action ?? "Add validation note."] }, + { key: "reviewLoad", cells: ["Change scope", scoreResultIcon(changeScopeComponent), changeScopeComponent.evidence, changeScopeComponent.action] }, + { key: "validationEvidence", cells: ["Validation posture", scoreResultIcon(validationComponent), validationComponent.evidence, validationComponent.action] }, { key: "openPrQueue", cells: ["Contributor workload", contributorWorkload.result, contributorWorkload.evidence, contributorWorkload.action] }, /* v8 ignore stop */ { key: "contributorContext", cells: ["Contributor context", contributorContext.result, contributorContext.evidence, contributorContext.action] }, @@ -4453,15 +4453,15 @@ export function buildPublicPrPanelSignalRows(args: { const gateConclusion = args.gate?.conclusion ?? fallbackGateConclusion; const confirmedMiner = isOfficialContributorDetection(args.detection); const readinessByKey = new Map(readiness.components.map((component) => [component.key, component])); - const validationComponent = readinessByKey.get("validation"); - const changeScopeComponent = readinessByKey.get("change_scope"); + const validationComponent = readinessByKey.get("validation")!; + const changeScopeComponent = readinessByKey.get("change_scope")!; const contributorWorkload = contributorWorkloadPanelResult(args.profile); const contributorContext = contributorContextPanelResult(args.pr, args.profile, args.detection, confirmedMiner); const rows: PublicPrPanelSignalRow[] = [ { key: "linkedIssue", cells: ["Linked issue", linkedIssueResult.result, linkedIssueResult.evidence, linkedIssueResult.action] }, { key: "relatedWork", cells: ["Related work", relatedWorkResult.result, relatedWorkResult.evidence, relatedWorkResult.action] }, - { key: "reviewLoad", cells: ["Change scope", scoreResultIcon(changeScopeComponent), changeScopeComponent?.evidence ?? "No public scope metadata found.", changeScopeComponent?.action ?? "No action."] }, - { key: "validationEvidence", cells: ["Validation posture", scoreResultIcon(validationComponent), validationComponent?.evidence ?? "No validation signal found.", validationComponent?.action ?? "Add validation note."] }, + { key: "reviewLoad", cells: ["Change scope", scoreResultIcon(changeScopeComponent), changeScopeComponent.evidence, changeScopeComponent.action] }, + { key: "validationEvidence", cells: ["Validation posture", scoreResultIcon(validationComponent), validationComponent.evidence, validationComponent.action] }, { key: "openPrQueue", cells: ["Contributor workload", contributorWorkload.result, contributorWorkload.evidence, contributorWorkload.action] }, { key: "contributorContext", cells: ["Contributor context", contributorContext.result, contributorContext.evidence, contributorContext.action] }, { key: "gateResult", cells: ["Gate result", gateStatus(gateEnabled, gateConclusion), gateEnabled ? gateAction(gateConclusion) : "Advisory only.", gateEnabled ? gateNextAction(gateConclusion) : "No action."] }, @@ -4608,9 +4608,7 @@ function contributorWorkloadScore(unlinkedOpenPullRequests: number): number { return 3; } -function scoreResultIcon(component: Pick | undefined): string { - /* v8 ignore next -- Component lookup is fixed today; undefined is a defensive fallback for future score shape drift. */ - if (!component) return "⚠️ No score"; +function scoreResultIcon(component: Pick): string { const ratio = component.score / component.max; if (ratio >= 0.85) return `✅ ${component.score}/${component.max}`; if (ratio >= 0.45) return `⚠️ ${component.score}/${component.max}`; @@ -4925,7 +4923,7 @@ function formatAlertBlock(lines: string[]): string[] { function aiReviewMainHasBlockers(main: string): boolean { const marker = main.search(/\*\*Blockers\*\*/i); if (marker === -1) return false; - const after = main.slice(marker).split(/\n(?=\*\*[^*]+\*\*)/)[0] ?? ""; + const after = main.slice(marker).split(/\n(?=\*\*[^*]+\*\*)/)[0]!; return after .split("\n") .slice(1) diff --git a/test/unit/backfill.test.ts b/test/unit/backfill.test.ts index 019773bd39..0fab02cb01 100644 --- a/test/unit/backfill.test.ts +++ b/test/unit/backfill.test.ts @@ -2834,6 +2834,41 @@ describe("GitHub backfill", () => { expect(aggregate.nonRequiredFailingDetails.map((detail) => detail.name).sort()).toEqual(["attacker/non-required-check", "attacker/non-required-status"]); }); + it("treats a visible required classic status that is still pending as pending CI", async () => { + const env = createTestEnv({ GITHUB_PUBLIC_TOKEN: "public-token" }); + vi.stubGlobal("fetch", async (input: RequestInfo | URL) => { + const url = input.toString(); + if (url.includes("/check-runs?")) { + return Response.json({ + check_runs: [ + { name: "lint", status: "completed", conclusion: "success" }, + ], + }); + } + if (url.includes("/status?")) { + return Response.json({ + statuses: [ + { context: "codecov/patch", state: "pending", description: "Waiting for report" }, + { context: "lint", state: "success" }, + ], + }); + } + return new Response("not found", { status: 404 }); + }); + + const aggregate = await fetchLiveCiAggregate( + env, + "JSONbored/gittensory", + "abc123", + "public-token", + new Set(["codecov/patch", "lint"]), + ); + + expect(aggregate.ciState).toBe("pending"); + expect(aggregate.hasPending).toBe(true); + expect(aggregate.failingDetails).toEqual([]); + }); + it("keeps an observed failure failed while still reporting pending CI separately", async () => { const env = createTestEnv({ GITHUB_PUBLIC_TOKEN: "public-token" }); vi.stubGlobal("fetch", async (input: RequestInfo | URL) => { diff --git a/test/unit/selfhost-pg-queue.test.ts b/test/unit/selfhost-pg-queue.test.ts index c0c68dba49..27485279aa 100644 --- a/test/unit/selfhost-pg-queue.test.ts +++ b/test/unit/selfhost-pg-queue.test.ts @@ -112,6 +112,61 @@ describe("createPgQueue (durable #977)", () => { expect(m.pool.query).toHaveBeenCalledWith(expect.stringContaining("UPDATE _selfhost_jobs SET priority=$1"), [8, "c"]); }); + it("init() skips already-normalized priority and job-key rows", async () => { + const priorityUpdateSql = "UPDATE _selfhost_jobs SET priority=$1"; + const jobKeyUpdateSql = "UPDATE _selfhost_jobs SET job_key=$1"; + const fn = vi.fn().mockImplementation(async (sql: unknown) => { + const q = String(sql); + if (q.includes("SELECT id, payload, priority")) { + return { + rows: [ + { id: "null-priority", payload: JSON.stringify(msg("unknown")), priority: null }, + { + id: "manual", + payload: JSON.stringify({ + type: "agent-regate-pr", + deliveryId: "manual-regate:1", + }), + priority: 99, + }, + ], + rowCount: 2, + }; + } + if (q.includes("SELECT id, payload, job_key") && q.includes("status IN")) { + return { + rows: [ + { + id: "keyed", + payload: JSON.stringify({ + type: "agent-regate-sweep", + repoFullName: "JSONbored/gittensory", + }), + job_key: "agent-regate-sweep:jsonbored/gittensory", + }, + { id: "unkeyed", payload: JSON.stringify(msg("unknown")), job_key: null }, + ], + rowCount: 2, + }; + } + if (q.includes("WHERE status='processing'")) return { rows: [], rowCount: 0 }; + if (q.includes("WHERE status='pending' AND run_after<=$1")) return { rows: [], rowCount: 0 }; + return { rows: [], rowCount: 0 }; + }); + const q = createPgQueue({ query: fn } as unknown as Pool, async () => undefined); + + await q.init(); + + expect(fn).not.toHaveBeenCalledWith( + expect.stringContaining(priorityUpdateSql), + expect.anything(), + ); + expect(fn).not.toHaveBeenCalledWith( + expect.stringContaining(jobKeyUpdateSql), + expect.anything(), + ); + }); + it("init() backfills job keys, recovers crashed jobs, and spreads due startup backlog", async () => { const oldMin = process.env.QUEUE_STARTUP_JITTER_MIN_JOBS; const oldJitter = process.env.QUEUE_STARTUP_JITTER_MS; @@ -264,6 +319,38 @@ describe("createPgQueue (durable #977)", () => { expect(claimSql[1]).toContain("priority < $2"); }); + it("processes a background-lane job when foreground work is empty", async () => { + const m = makePool(); + m.enqueueResult({ rows: [], rowCount: 0 }); + m.enqueueResult({ + rows: [ + { + id: "background", + payload: JSON.stringify(msg("agent-regate-sweep")), + attempts: 0, + job_key: "agent-regate-sweep", + priority: 0, + }, + ], + rowCount: 1, + }); + const seen: string[] = []; + const q = createPgQueue( + m.pool, + async (j) => void seen.push(typeOf(j)), + { backgroundConcurrency: 1 }, + ); + + await q.init(); + await q.drain(); + + expect(seen).toEqual(["agent-regate-sweep"]); + expect(m.pool.query).toHaveBeenCalledWith( + expect.stringContaining("DELETE FROM _selfhost_jobs WHERE id=$1"), + ["background"], + ); + }); + it("dead-letters an unparseable payload (job_dead audit emitted)", async () => { const m = makePool(); // Claim returns a row with bad payload.