diff --git a/apps/server/src/sourceControl/GitHubApi.test.ts b/apps/server/src/sourceControl/GitHubApi.test.ts new file mode 100644 index 000000000000..85fe9a2dcfbe --- /dev/null +++ b/apps/server/src/sourceControl/GitHubApi.test.ts @@ -0,0 +1,497 @@ +import { describe, expect, it } from "@effect/vitest"; +import * as NodeServices from "@effect/platform-node/NodeServices"; +import * as Effect from "effect/Effect"; +import * as Layer from "effect/Layer"; +import * as Redacted from "effect/Redacted"; +import * as PlatformError from "effect/PlatformError"; +import * as TestClock from "effect/testing/TestClock"; +import * as Tracer from "effect/Tracer"; +import { ChildProcessSpawner } from "effect/process"; +import { VcsProcessSpawnError, VcsProcessTimeoutError } from "@t3tools/contracts"; +import { HttpClient, HttpClientResponse, type HttpClientRequest } from "effect/http"; + +import * as GitHubApi from "./GitHubApi.ts"; +import * as GitHubCredentials from "./GitHubCredentials.ts"; +import * as GitHubGraphQlBudget from "./githubGraphQlBudget.ts"; +import * as SourceControlRateLimit from "./SourceControlRateLimit.ts"; +import * as VcsProcess from "../vcs/VcsProcess.ts"; + +const NOW = Date.parse("2026-10-05T12:00:00.000Z"); + +function harness(respond: (request: HttpClientRequest.HttpClientRequest) => Response) { + const requests: Array = []; + let tokens = ["first", "second"]; + let invalidations = 0; + const credentials = Layer.succeed( + GitHubCredentials.GitHubCredentials, + GitHubCredentials.GitHubCredentials.of({ + get: (host) => + Effect.sync(() => ({ + host, + token: Redacted.make(tokens[0]!), + source: "gh" as const, + fingerprint: `${host}:${tokens[0]}`, + })), + invalidate: () => + Effect.sync(() => { + invalidations++; + tokens = tokens.slice(1); + }), + }), + ); + const http = Layer.succeed( + HttpClient.HttpClient, + HttpClient.make((request) => { + requests.push(request); + return Effect.succeed(HttpClientResponse.fromWeb(request, respond(request))); + }), + ); + const layer = GitHubApi.layer.pipe( + Layer.provide(Layer.mergeAll(credentials, http)), + Layer.provideMerge(GitHubGraphQlBudget.layer), + Layer.provideMerge(SourceControlRateLimit.layer), + ); + return { layer, requests, invalidations: () => invalidations }; +} + +const json = (body: unknown, init?: ResponseInit) => + new Response(JSON.stringify(body), { + ...init, + headers: { "content-type": "application/json", ...init?.headers }, + }); + +describe("gitHubApiUrls", () => { + it("maps github.com, GHE.com and GitHub Enterprise Server", () => { + expect(GitHubApi.gitHubApiUrls("GitHub.com")).toEqual({ + rest: "https://api.github.com", + graphql: "https://api.github.com/graphql", + }); + expect(GitHubApi.gitHubApiUrls("acme.ghe.com").graphql).toBe( + "https://api.acme.ghe.com/graphql", + ); + expect(GitHubApi.gitHubApiUrls("git.acme.internal")).toEqual({ + rest: "https://git.acme.internal/api/v3", + graphql: "https://git.acme.internal/api/graphql", + }); + }); +}); + +describe("environmentToken", () => { + it("follows gh's precedence per kind of host", () => { + const env = { GH_TOKEN: "gh", GITHUB_TOKEN: "github", GH_ENTERPRISE_TOKEN: "ghe" }; + expect(GitHubCredentials.environmentToken("github.com", env)).toBe("gh"); + expect(GitHubCredentials.environmentToken("acme.ghe.com", { GITHUB_TOKEN: "github" })).toBe( + "github", + ); + // An enterprise token only goes to the host GH_HOST names, never to whatever a remote says. + expect(GitHubCredentials.environmentToken("git.acme.internal", env)).toBeNull(); + expect( + GitHubCredentials.environmentToken("git.acme.internal", { + ...env, + GH_HOST: "git.acme.internal", + }), + ).toBe("ghe"); + expect(GitHubCredentials.environmentToken("git.acme.internal", { GH_TOKEN: "gh" })).toBeNull(); + }); +}); + +describe("GitHubApi", () => { + it.effect("sends GraphQL with the token and records the reported budget", () => { + const { layer, requests } = harness(() => + json({ + data: { + viewer: { login: "julius" }, + rateLimit: { cost: 1, limit: 5000, remaining: 4999, resetAt: "2026-10-05T13:00:00Z" }, + }, + }), + ); + return Effect.gen(function* () { + yield* TestClock.setTime(NOW); + const api = yield* GitHubApi.GitHubApi; + const body = yield* api.graphql({ + host: "github.com", + operation: "viewer", + query: "query { viewer { login } }", + }); + expect(body).toContain('"login":"julius"'); + expect(requests[0]!.url).toBe("https://api.github.com/graphql"); + expect(requests[0]!.headers.authorization).toBe("Bearer first"); + }).pipe(Effect.provide(layer)); + }); + + it.effect( + "traces the operation, path, query and cost, never the query string or variables", + () => { + const spans: Array = []; + const tracer = Tracer.make({ + span: (options) => { + const span = new Tracer.NativeSpan(options); + spans.push(span); + return span; + }, + }); + const { layer } = harness((request) => + request.url.endsWith("/graphql") + ? json( + { + data: { + viewer: { login: "julius" }, + rateLimit: { + cost: 3, + limit: 5000, + remaining: 4990, + resetAt: "2026-10-05T13:00:00Z", + }, + }, + }, + { headers: { "x-ratelimit-remaining": "4990", "x-ratelimit-resource": "graphql" } }, + ) + : json({ ok: true }, { headers: { "x-ratelimit-remaining": "4800" } }), + ); + return Effect.gen(function* () { + yield* TestClock.setTime(NOW); + const api = yield* GitHubApi.GitHubApi; + yield* api.graphql({ + host: "github.com", + operation: "getPullRequestDetail", + query: "query($body: String!) { viewer { login } }", + variables: { body: "secret user text" }, + }); + yield* api.rest({ + host: "github.com", + operation: "listWorkflowRuns", + path: "repos/acme/web/actions/runs?head_sha=abc123&branch=feat%2Fx", + }); + const byName = (name: string) => spans.filter((span) => span.name === name); + const graphql = Object.fromEntries(byName("GitHubApi.graphql")[0]!.attributes); + expect(graphql).toMatchObject({ + "github.operation": "getPullRequestDetail", + "github.graphql.query": "query($body: String!) { viewer { login } }", + "github.graphql.cost": 3, + "github.graphql.remaining": 4990, + }); + expect(String(graphql["github.graphql.query_hash"])).toMatch(/^[0-9a-f]{8}$/); + const rest = Object.fromEntries(byName("GitHubApi.send")[1]!.attributes); + expect(rest).toMatchObject({ + "github.operation": "listWorkflowRuns", + "github.kind": "rest", + "url.path": "/repos/acme/web/actions/runs", + "http.response.status_code": 200, + "github.ratelimit.remaining": 4800, + }); + // Nothing recorded carries the query string, the variables, or the token. + const everything = spans + .flatMap((span) => [...span.attributes].map(([key, value]) => `${key}=${String(value)}`)) + .join("\n"); + expect(everything).not.toContain("head_sha"); + expect(everything).not.toContain("secret user text"); + expect(everything).not.toContain("first"); + expect(spans.some((span) => span.attributes.has("url.full"))).toBe(false); + }).pipe(Effect.provide(layer), Effect.withTracer(tracer)); + }, + ); + + it.effect("fails a GraphQL answer that carries errors, naming GitHub's reason", () => { + const { layer } = harness(() => + json({ data: null, errors: [{ type: "FORBIDDEN", message: "Resource not accessible" }] }), + ); + return Effect.gen(function* () { + const api = yield* GitHubApi.GitHubApi; + const error = yield* Effect.flip( + api.graphql({ host: "github.com", operation: "detail", query: "query { viewer { id } }" }), + ); + expect(error._tag).toBe("GitHubApiResponseError"); + expect(error.message).toContain("Resource not accessible"); + }).pipe(Effect.provide(layer)); + }); + + it.effect( + "reads a NOT_FOUND answer with another error as a failure, not a missing resource", + () => { + const { layer } = harness(() => + json({ + data: null, + errors: [ + { type: "NOT_FOUND", message: "Could not resolve" }, + { message: "Something went wrong" }, + ], + }), + ); + return Effect.gen(function* () { + const api = yield* GitHubApi.GitHubApi; + const error = yield* Effect.flip( + api.graphql({ + host: "github.com", + operation: "detail", + query: "query { viewer { id } }", + }), + ); + expect(error._tag).toBe("GitHubApiResponseError"); + }).pipe(Effect.provide(layer)); + }, + ); + + it.effect("pauses the host after a GraphQL RATE_LIMITED answer until the reset", () => { + const reset = Math.floor(NOW / 1000) + 600; + const { layer, requests } = harness(() => + json( + { errors: [{ type: "RATE_LIMITED", message: "API rate limit exceeded" }] }, + { headers: { "x-ratelimit-remaining": "0", "x-ratelimit-reset": String(reset) } }, + ), + ); + return Effect.gen(function* () { + yield* TestClock.setTime(NOW); + const api = yield* GitHubApi.GitHubApi; + const first = yield* Effect.flip( + api.graphql({ host: "github.com", operation: "summary", query: "query { viewer { id } }" }), + ); + expect(first).toMatchObject({ _tag: "GitHubApiRateLimitError", retryAt: reset * 1000 }); + const second = yield* Effect.flip( + api.rest({ host: "github.com", operation: "stack", path: "repos/acme/web/stacks" }), + ); + expect(second).toMatchObject({ + _tag: "SourceControlRateLimitPausedError", + retryAt: reset * 1000, + }); + expect(requests).toHaveLength(1); + }).pipe(Effect.provide(layer)); + }); + + it.effect("carries GitHub's own reason for a refused REST request", () => { + const { layer } = harness(() => + json( + { + message: "Validation Failed", + errors: [ + { + resource: "PullRequest", + code: "custom", + message: "A pull request already exists for acme:feature.", + }, + ], + }, + { status: 422 }, + ), + ); + return Effect.gen(function* () { + const api = yield* GitHubApi.GitHubApi; + const error = yield* Effect.flip( + api.rest({ + host: "github.com", + operation: "createPullRequest", + method: "POST", + path: "repos/acme/web/pulls", + }), + ); + expect(error.message).toBe( + "GitHub returned an error: Validation Failed; A pull request already exists for acme:feature.", + ); + }).pipe(Effect.provide(layer)); + }); + + it.effect("lets an interactive request through a pause a background read recorded", () => { + const reset = Math.floor(NOW / 1000) + 600; + let call = 0; + const { layer, requests } = harness(() => + ++call === 1 + ? json( + { message: "API rate limit exceeded" }, + { + status: 403, + headers: { "x-ratelimit-remaining": "0", "x-ratelimit-reset": String(reset) }, + }, + ) + : json({ ok: true }), + ); + return Effect.gen(function* () { + yield* TestClock.setTime(NOW); + const api = yield* GitHubApi.GitHubApi; + const read = { host: "github.com", operation: "sweep", path: "repos/acme/web/pulls" }; + yield* Effect.flip(api.rest(read)); + expect((yield* Effect.flip(api.rest(read)))._tag).toBe("SourceControlRateLimitPausedError"); + const merged = yield* api + .rest({ + host: "github.com", + operation: "merge", + method: "PUT", + path: "repos/acme/web/pulls/7/merge", + }) + .pipe(Effect.provideService(GitHubApi.AllowGitHubReserve, true)); + expect(merged.status).toBe(200); + expect(requests).toHaveLength(2); + }).pipe(Effect.provide(layer)); + }); + + it.effect("reads an untyped GraphQL quota error as a rate limit", () => { + const reset = Math.floor(NOW / 1000) + 900; + const { layer } = harness(() => + json( + { errors: [{ message: "API rate limit already exceeded for user ID 1." }] }, + { headers: { "x-ratelimit-remaining": "0", "x-ratelimit-reset": String(reset) } }, + ), + ); + return Effect.gen(function* () { + yield* TestClock.setTime(NOW); + const api = yield* GitHubApi.GitHubApi; + const error = yield* Effect.flip( + api.graphql({ host: "github.com", operation: "lookup", query: "query { viewer { id } }" }), + ); + expect(error).toMatchObject({ _tag: "GitHubApiRateLimitError", retryAt: reset * 1000 }); + }).pipe(Effect.provide(layer)); + }); + + it.effect("maps REST 403 with an exhausted quota to a rate limit, and 304 to an answer", () => { + let call = 0; + const { layer } = harness(() => + ++call === 1 + ? new Response(null, { status: 304 }) + : json( + { message: "API rate limit exceeded" }, + { + status: 403, + headers: { "retry-after": "60" }, + }, + ), + ); + return Effect.gen(function* () { + yield* TestClock.setTime(NOW); + const api = yield* GitHubApi.GitHubApi; + const notModified = yield* api.rest({ + host: "github.com", + operation: "checks", + path: "repos/acme/web/pulls/7", + ifNoneMatch: '"abc"', + }); + expect(notModified.status).toBe(304); + const error = yield* Effect.flip( + api.rest({ host: "github.com", operation: "checks", path: "repos/acme/web/pulls/7" }), + ); + expect(error).toMatchObject({ _tag: "GitHubApiRateLimitError", retryAt: NOW + 60_000 }); + }).pipe(Effect.provide(layer)); + }); + + it.effect("drops a refused token so the next request asks the source again", () => { + const { layer, requests, invalidations } = harness((request) => + request.headers.authorization === "Bearer first" + ? new Response(null, { status: 401 }) + : json({ ok: true }), + ); + return Effect.gen(function* () { + const api = yield* GitHubApi.GitHubApi; + const refused = yield* Effect.flip( + api.rest({ host: "github.com", operation: "user", path: "user" }), + ); + expect(refused._tag).toBe("GitHubApiAuthenticationError"); + expect(invalidations()).toBe(1); + const answered = yield* api.rest({ host: "github.com", operation: "user", path: "user" }); + expect(answered.status).toBe(200); + expect(requests.map((request) => request.headers.authorization)).toEqual([ + "Bearer first", + "Bearer second", + ]); + }).pipe(Effect.provide(layer)); + }); + + it.effect("refuses to send a pinned credential to another host", () => { + const { layer, requests } = harness(() => json({})); + return Effect.gen(function* () { + const api = yield* GitHubApi.GitHubApi; + const error = yield* Effect.flip( + api.rest({ host: "git.acme.internal", operation: "user", path: "user" }), + ).pipe( + Effect.provideService(GitHubApi.PinnedGitHubCredential, { + host: "github.com", + token: Redacted.make("pinned"), + credentialFingerprint: "github.com:pinned", + }), + ); + expect(error._tag).toBe("GitHubApiAuthenticationError"); + expect(requests).toHaveLength(0); + }).pipe(Effect.provide(layer)); + }); +}); + +describe("GitHubCredentials", () => { + const credentialsWith = (run: VcsProcess.VcsProcess["Service"]["run"]) => + GitHubCredentials.layer.pipe( + Layer.provide(Layer.mock(VcsProcess.VcsProcess)({ run })), + Layer.provide(NodeServices.layer), + ); + + it.effect("fails with GitHubCliMissingError when gh is not on PATH", () => + Effect.gen(function* () { + const credentials = yield* GitHubCredentials.GitHubCredentials; + const error = yield* Effect.flip(credentials.get("git.acme.internal")); + expect(error._tag).toBe("GitHubCliMissingError"); + }).pipe( + Effect.provide( + credentialsWith(() => + Effect.fail( + new VcsProcessSpawnError({ + operation: "GitHubCredentials.get", + command: "gh", + cwd: "/", + cause: PlatformError.systemError({ + _tag: "NotFound", + module: "ChildProcess", + method: "spawn", + }), + }), + ), + ), + ), + ), + ); + + it.effect("does not cache a gh failure that is not about signing in", () => { + let calls = 0; + return Effect.gen(function* () { + const credentials = yield* GitHubCredentials.GitHubCredentials; + const first = yield* Effect.flip(credentials.get("github.com")); + expect(first._tag).toBe("GitHubCliFailedError"); + expect((yield* credentials.get("github.com")).source).toBe("gh"); + expect(calls).toBe(2); + }).pipe( + Effect.provide( + credentialsWith(() => + ++calls === 1 + ? Effect.fail( + new VcsProcessTimeoutError({ + operation: "GitHubCredentials.get", + command: "gh", + cwd: "/", + timeoutMs: 10_000, + }), + ) + : Effect.succeed({ + exitCode: ChildProcessSpawner.ExitCode(0), + stdout: "token\n", + stderr: "", + stdoutTruncated: false, + stderrTruncated: false, + }), + ), + ), + ); + }); + + it.effect("fails with GitHubNotSignedInError when gh prints no token", () => + Effect.gen(function* () { + const credentials = yield* GitHubCredentials.GitHubCredentials; + const error = yield* Effect.flip(credentials.get("git.acme.internal")); + expect(error._tag).toBe("GitHubNotSignedInError"); + }).pipe( + Effect.provide( + credentialsWith(() => + Effect.succeed({ + exitCode: ChildProcessSpawner.ExitCode(0), + stdout: "\n", + stderr: "", + stdoutTruncated: false, + stderrTruncated: false, + }), + ), + ), + ), + ); +}); diff --git a/apps/server/src/sourceControl/GitHubApi.ts b/apps/server/src/sourceControl/GitHubApi.ts new file mode 100644 index 000000000000..7b1a9f316a9c --- /dev/null +++ b/apps/server/src/sourceControl/GitHubApi.ts @@ -0,0 +1,593 @@ +import * as Clock from "effect/Clock"; +import * as Context from "effect/Context"; +import * as Data from "effect/Data"; +import * as Duration from "effect/Duration"; +import * as Effect from "effect/Effect"; +import * as Layer from "effect/Layer"; +import * as Option from "effect/Option"; +import * as Redacted from "effect/Redacted"; +import * as Schema from "effect/Schema"; +import * as Semaphore from "effect/Semaphore"; +import { HttpClient, HttpClientRequest, type HttpClientResponse } from "effect/http"; + +import { collectUint8StreamText } from "../stream/collectUint8StreamText.ts"; +import * as GitHubCredentials from "./GitHubCredentials.ts"; +import * as GitHubGraphQlBudget from "./githubGraphQlBudget.ts"; +import * as SourceControlRateLimit from "./SourceControlRateLimit.ts"; + +const DEFAULT_TIMEOUT = Duration.seconds(30); +const DEFAULT_MAX_RESPONSE_BYTES = 8 * 1024 * 1024; +/** Polite to GitHub's secondary limits, which punish bursts of concurrent requests. */ +const CONCURRENCY = 8; +const API_VERSION = "2022-11-28"; + +/** + * A credential already verified for one host. Every request made under it must target that + * host, so a page cannot read one account's data with another's token mid-flight. Server-local: + * never put its value in RPC payloads or cache keys. + */ +export const PinnedGitHubCredential = Context.Reference<{ + readonly host: string; + readonly token: Redacted.Redacted; + readonly credentialFingerprint: string; +} | null>("t3/sourceControl/PinnedGitHubCredential", { defaultValue: () => null }); + +/** + * Set by interactive callers (a user's read or write, not a background sweep). Requests made + * under it may spend the GraphQL reserve and go through a rate-limit pause: a user acting on a + * pull request should not be refused because a background read exhausted the quota. + */ +export const AllowGitHubReserve = Context.Reference( + "t3/sourceControl/AllowGitHubReserve", + { defaultValue: () => false }, +); + +export class GitHubApiRequestError extends Schema.TaggedError()( + "GitHubApiRequestError", + { host: Schema.String, operation: Schema.String, cause: Schema.Defect() }, +) { + override get message(): string { + return `Could not reach GitHub at ${this.host}.`; + } +} + +export class GitHubApiAuthenticationError extends Schema.TaggedError()( + "GitHubApiAuthenticationError", + { host: Schema.String, operation: Schema.String }, +) { + override get message(): string { + return `GitHub refused the credential for ${this.host}. Run \`gh auth login --hostname ${this.host}\` and retry.`; + } +} + +export class GitHubApiRateLimitError extends Schema.TaggedError()( + "GitHubApiRateLimitError", + { + host: Schema.String, + operation: Schema.String, + retryAt: Schema.optionalKey(Schema.Finite), + }, +) { + override get message(): string { + return "GitHub API rate limit exceeded."; + } +} + +export class GitHubApiNotFoundError extends Schema.TaggedError()( + "GitHubApiNotFoundError", + { host: Schema.String, operation: Schema.String }, +) { + override get message(): string { + return "GitHub could not find the requested resource, or the credential cannot see it."; + } +} + +/** + * GitHub answered with a failure that is none of the above. `githubErrors` carries GitHub's own + * error messages, from a GraphQL `errors` list or a REST `message`/`errors` body: they name the + * field and the reason ("A pull request already exists for acme:feature"), never a token. + */ +export class GitHubApiResponseError extends Schema.TaggedError()( + "GitHubApiResponseError", + { + host: Schema.String, + operation: Schema.String, + status: Schema.Int, + githubErrors: Schema.optionalKey(Schema.Array(Schema.String)), + }, +) { + override get message(): string { + return this.githubErrors !== undefined && this.githubErrors.length > 0 + ? `GitHub returned an error: ${this.githubErrors.join("; ")}` + : `GitHub returned HTTP ${this.status}.`; + } +} + +export type GitHubApiError = + | GitHubCredentials.GitHubCredentialUnavailableError + | GitHubApiRequestError + | GitHubApiAuthenticationError + | GitHubApiRateLimitError + | GitHubApiNotFoundError + | GitHubApiResponseError + | SourceControlRateLimit.SourceControlRateLimitPausedError; + +export interface GitHubRestResponse { + readonly status: number; + readonly headers: Readonly>; + readonly body: string; + readonly truncated: boolean; +} + +export interface GitHubRestInput { + readonly host: string; + readonly operation: string; + readonly method?: "GET" | "POST" | "PUT" | "PATCH" | "DELETE"; + /** Relative to the API root, e.g. `repos/acme/web/pulls/7`; query string included. */ + readonly path: string; + readonly body?: unknown; + /** Defaults to `application/vnd.github+json`. */ + readonly accept?: string; + /** Revalidates a cached answer. A 304 is returned rather than failed, and is free. */ + readonly ifNoneMatch?: string; + readonly maxResponseBytes?: number; + /** Overrides `AllowGitHubReserve` for this one request. */ + readonly allowReserve?: boolean; +} + +export interface GitHubGraphQlInput { + readonly host: string; + readonly operation: string; + readonly query: string; + readonly variables?: Readonly>; + readonly allowReserve?: boolean; +} + +export class GitHubApi extends Context.Service< + GitHubApi, + { + /** The raw JSON body of a successful GraphQL answer, with `rateLimit` recorded. */ + readonly graphql: (input: GitHubGraphQlInput) => Effect.Effect; + readonly rest: (input: GitHubRestInput) => Effect.Effect; + /** The credential a request to `host` would carry right now. */ + readonly credential: ( + host: string, + ) => Effect.Effect< + { readonly token: Redacted.Redacted; readonly fingerprint: string }, + GitHubApiError + >; + } +>()("t3/sourceControl/GitHubApi") {} + +function normalizeHost(host: string): string { + return host.trim().toLowerCase(); +} + +/** + * Where the API for a host lives. github.com and GHE.com data residency serve it from an `api.` + * subdomain; GitHub Enterprise Server serves it under `/api` on the instance itself. + */ +export function gitHubApiUrls(host: string): { readonly rest: string; readonly graphql: string } { + const normalized = normalizeHost(host); + if (normalized === "github.com") { + return { rest: "https://api.github.com", graphql: "https://api.github.com/graphql" }; + } + if (normalized.endsWith(".ghe.com")) { + return { + rest: `https://api.${normalized}`, + graphql: `https://api.${normalized}/graphql`, + }; + } + return { rest: `https://${normalized}/api/v3`, graphql: `https://${normalized}/api/graphql` }; +} + +/** GraphQL documents longer than this are cut in traces; the hash still identifies them. */ +const TRACED_QUERY_MAX_CHARS = 4_000; + +/** A stable short id for a GraphQL document, so traces group by query without its text. */ +function queryHash(query: string): string { + let hash = 0x811c9dc5; + for (let index = 0; index < query.length; index++) { + hash ^= query.charCodeAt(index); + hash = Math.imul(hash, 0x01000193); + } + return (hash >>> 0).toString(16).padStart(8, "0"); +} + +/** The numeric rate-limit headers GitHub sends, as span attributes. */ +function rateLimitAttributes( + headers: Readonly>, +): Record { + const attributes: Record = {}; + for (const [header, name] of [ + ["x-ratelimit-limit", "github.ratelimit.limit"], + ["x-ratelimit-remaining", "github.ratelimit.remaining"], + ["x-ratelimit-used", "github.ratelimit.used"], + ["x-ratelimit-reset", "github.ratelimit.reset"], + ] as const) { + const value = Number(headers[header]); + if (headers[header] !== undefined && Number.isFinite(value)) attributes[name] = value; + } + const resource = headers["x-ratelimit-resource"]; + if (resource !== undefined) attributes["github.ratelimit.resource"] = resource; + return attributes; +} + +const decodeGraphQlCost = Schema.decodeUnknownOption( + Schema.fromJsonString( + Schema.Struct({ + data: Schema.Struct({ + rateLimit: Schema.Struct({ cost: Schema.Number, remaining: Schema.Number }), + }), + }), + ), +); + +/** The pause GitHub asked for, from `retry-after` or the primary limit's reset. */ +function retryAtFrom( + headers: Readonly>, + now: number, +): number | undefined { + const fromRetryAfter = SourceControlRateLimit.retryAtFromHeader(headers["retry-after"], now); + if (fromRetryAfter !== undefined) return fromRetryAfter; + const reset = Number(headers["x-ratelimit-reset"]) * 1_000; + return Number.isFinite(reset) && reset > now ? reset : undefined; +} + +const decodeGraphQlErrors = Schema.decodeUnknownOption( + Schema.fromJsonString( + Schema.Struct({ + errors: Schema.NonEmptyArray( + Schema.Struct({ + type: Schema.optional(Schema.String), + message: Schema.optional(Schema.String), + }), + ), + }), + ), +); + +/** A REST failure body: `{ message, errors: [{ message } | { resource, field, code }] }`. */ +const decodeRestErrors = Schema.decodeUnknownOption( + Schema.fromJsonString( + Schema.Struct({ + message: Schema.optional(Schema.String), + errors: Schema.optional( + Schema.Array( + Schema.Union([ + Schema.String, + Schema.Struct({ + message: Schema.optional(Schema.String), + field: Schema.optional(Schema.String), + code: Schema.optional(Schema.String), + }), + ]), + ), + ), + }), + ), +); + +function restErrorMessages(body: string): ReadonlyArray | undefined { + return Option.match(decodeRestErrors(body), { + onNone: () => undefined, + onSome: (decoded) => { + const details = (decoded.errors ?? []).flatMap((error) => + typeof error === "string" + ? [error] + : error.message !== undefined + ? [error.message] + : error.field !== undefined && error.code !== undefined + ? [`${error.field} ${error.code}`] + : [], + ); + const messages = [...(decoded.message === undefined ? [] : [decoded.message]), ...details]; + return messages.length > 0 ? messages : undefined; + }, + }); +} + +/** What one GitHub answer means, decided once from its status, headers and body. */ +type Answer = Data.TaggedEnum<{ + Ok: {}; + RateLimited: {}; + Unauthorized: {}; + NotFound: {}; + Failed: { readonly messages: ReadonlyArray | undefined }; +}>; +const Answer = Data.taggedEnum(); + +/** + * GitHub reports a failed GraphQL document with HTTP 200 and an `errors` list, so a GraphQL body + * is read for its error types as well as its status. `gh api graphql` failed those too, and + * callers rely on that to fall back to narrower reads. + */ +function classify(input: { + readonly status: number; + readonly headers: Readonly>; + readonly body: string; + readonly graphql: boolean; + readonly acceptNotModified: boolean; +}): Answer { + const { status, headers, body } = input; + const errors = input.graphql + ? Option.getOrUndefined(decodeGraphQlErrors(body))?.errors + : undefined; + const types = errors?.flatMap((error) => (error.type === undefined ? [] : [error.type])) ?? []; + const messages = errors?.flatMap((error) => (error.message === undefined ? [] : [error.message])); + if ( + status === 429 || + types.includes("RATE_LIMITED") || + // An exhausted GraphQL quota can answer HTTP 200 with an untyped "API rate limit already + // exceeded" error; the headers say the same thing. + (input.graphql && errors !== undefined && headers["x-ratelimit-remaining"] === "0") || + (input.graphql && + messages !== undefined && + messages.some((message) => /rate limit (already )?exceeded/i.test(message))) || + (status === 403 && + (headers["x-ratelimit-remaining"] === "0" || + headers["retry-after"] !== undefined || + /rate limit/i.test(body))) + ) { + return Answer.RateLimited(); + } + if (status === 401) return Answer.Unauthorized(); + if ( + status === 404 || + (errors !== undefined && errors.every((error) => error.type === "NOT_FOUND")) + ) { + return Answer.NotFound(); + } + if (errors !== undefined) return Answer.Failed({ messages }); + if ((status >= 200 && status < 300) || (status === 304 && input.acceptNotModified)) { + return Answer.Ok(); + } + return Answer.Failed({ messages: input.graphql ? undefined : restErrorMessages(body) }); +} + +/** @public Service construction is part of the canonical Effect module API. */ +export const make = Effect.gen(function* () { + const httpClient = yield* HttpClient.HttpClient; + const credentials = yield* GitHubCredentials.GitHubCredentials; + const budget = yield* GitHubGraphQlBudget.GitHubGraphQlBudget; + const limits = yield* SourceControlRateLimit.SourceControlRateLimit; + const gate = yield* Semaphore.make(CONCURRENCY); + + const credential: GitHubApi["Service"]["credential"] = Effect.fn("GitHubApi.credential")( + function* (host) { + const normalized = normalizeHost(host); + const pinned = yield* PinnedGitHubCredential; + if (pinned !== null) { + // A pinned page only ever talks to the host it verified. + if (pinned.host !== normalized) { + return yield* new GitHubApiAuthenticationError({ + host: normalized, + operation: "credential", + }); + } + return { token: pinned.token, fingerprint: pinned.credentialFingerprint }; + } + const held = yield* credentials.get(normalized); + return { token: held.token, fingerprint: held.fingerprint }; + }, + ); + + /** + * Sends one request under the host's rate-limit pause and fails a refusal with the error that + * says why. `onSuccess` sees the answer only once it is known to be one. + */ + const send = Effect.fn("GitHubApi.send")(function* (input: { + readonly host: string; + readonly operation: string; + readonly request: HttpClientRequest.HttpClientRequest; + readonly maxResponseBytes: number; + readonly allowReserve: boolean; + readonly acceptNotModified: boolean; + /** Reads the body for GraphQL `errors`, which GitHub sends with HTTP 200. */ + readonly graphql?: boolean; + readonly timeout?: Duration.Duration | undefined; + }) { + const host = normalizeHost(input.host); + // Only the path: a query string can carry a SHA or a branch, and never needs to be in a trace. + yield* Effect.annotateCurrentSpan({ + "github.host": host, + "github.operation": input.operation, + "github.kind": input.graphql === true ? "graphql" : "rest", + "http.request.method": input.request.method, + "url.path": new URL(input.request.url).pathname, + }); + const { token, fingerprint } = yield* credential(host); + const scope = yield* SourceControlRateLimit.CredentialScope; + const key = { provider: "github" as const, host }; + const run = Effect.gen(function* () { + const lease = yield* limits + .check(key, input.allowReserve ? { allowPaused: true } : undefined) + .pipe( + Effect.tapError((paused) => + Effect.annotateCurrentSpan({ + "github.paused": true, + "github.retry_at": paused.retryAt, + }), + ), + ); + // One deadline covers the headers and the body: a host that answers headers and then stalls + // must not hold a slot of the shared gate for undici's own five-minute body timeout. + const { response, collected } = yield* Effect.gen(function* () { + const response: HttpClientResponse.HttpClientResponse = yield* httpClient + .execute( + input.request.pipe( + HttpClientRequest.bearerToken(Redacted.value(token)), + HttpClientRequest.setHeaders({ + "x-github-api-version": API_VERSION, + "user-agent": "t3code", + }), + ), + ) + .pipe( + // `GitHubApi.send` is the request's span. The client's own span would add `url.full` + // and `url.query`, which can carry SHAs and branch names, so it is off for GitHub. + Effect.provideService(HttpClient.TracerDisabledWhen, () => true), + ); + const collected = yield* collectUint8StreamText({ + stream: response.stream, + maxBytes: input.maxResponseBytes, + }).pipe( + // 204, 304 and many refusals carry no body at all, which is an empty answer. + Effect.catchIf( + (error) => error.reason._tag === "EmptyBodyError", + () => Effect.succeed({ text: "", truncated: false }), + ), + ); + return { response, collected }; + }).pipe( + Effect.timeout(input.timeout ?? DEFAULT_TIMEOUT), + Effect.mapError( + (cause) => new GitHubApiRequestError({ host, operation: input.operation, cause }), + ), + ); + const headers = response.headers; + const status = response.status; + yield* Effect.annotateCurrentSpan({ + "http.response.status_code": status, + ...rateLimitAttributes(headers), + }); + const context = { host, operation: input.operation }; + return yield* Answer.$match( + classify({ + status, + headers, + body: collected.text, + graphql: input.graphql === true, + acceptNotModified: input.acceptNotModified, + }), + { + Ok: () => + limits.recordSuccess({ ...key, lease }).pipe( + Effect.as({ + status, + headers, + body: collected.text, + truncated: collected.truncated, + }), + ), + RateLimited: () => + Effect.gen(function* () { + const retryAt = retryAtFrom(headers, yield* Clock.currentTimeMillis); + yield* limits.recordRateLimit({ ...key, lease, retryAt }); + yield* Effect.annotateCurrentSpan({ + "github.rate_limited": true, + ...(retryAt === undefined ? {} : { "github.retry_at": retryAt }), + }); + return yield* new GitHubApiRateLimitError({ + ...context, + ...(retryAt === undefined ? {} : { retryAt }), + }); + }), + // The source may hold a newer token than the one that was refused. + Unauthorized: () => + credentials + .invalidate(host) + .pipe(Effect.andThen(Effect.fail(new GitHubApiAuthenticationError(context)))), + NotFound: () => Effect.fail(new GitHubApiNotFoundError(context)), + Failed: ({ messages }) => + Effect.fail( + new GitHubApiResponseError({ + ...context, + status, + ...(messages === undefined ? {} : { githubErrors: messages }), + }), + ), + }, + ); + }); + // The rate-limit scope follows the credential, so a pause on one account never blocks another. + return yield* gate + .withPermit(run) + .pipe(Effect.provideService(SourceControlRateLimit.CredentialScope, scope || fingerprint)); + }); + + const rest: GitHubApi["Service"]["rest"] = (input) => { + const url = `${gitHubApiUrls(input.host).rest}/${input.path.replace(/^\/+/, "")}`; + const base = HttpClientRequest.make(input.method ?? "GET")(url).pipe( + HttpClientRequest.setHeader("accept", input.accept ?? "application/vnd.github+json"), + ); + const withEtag = + input.ifNoneMatch === undefined + ? base + : base.pipe(HttpClientRequest.setHeader("if-none-match", input.ifNoneMatch)); + const request = + input.body === undefined + ? withEtag + : withEtag.pipe(HttpClientRequest.bodyJsonUnsafe(input.body)); + return AllowGitHubReserve.pipe( + Effect.flatMap((interactive) => + send({ + host: input.host, + operation: input.operation, + request, + maxResponseBytes: input.maxResponseBytes ?? DEFAULT_MAX_RESPONSE_BYTES, + allowReserve: input.allowReserve ?? interactive, + acceptNotModified: input.ifNoneMatch !== undefined, + }), + ), + ); + }; + + const graphql: GitHubApi["Service"]["graphql"] = Effect.fn("GitHubApi.graphql")( + function* (input) { + const host = normalizeHost(input.host); + const { fingerprint } = yield* credential(host); + const scope = (yield* SourceControlRateLimit.CredentialScope) || fingerprint; + const allowReserve = input.allowReserve ?? (yield* AllowGitHubReserve); + // The document, never its variables: user text (bodies, search terms) travels as variables. + yield* Effect.annotateCurrentSpan({ + "github.operation": input.operation, + "github.graphql.query_hash": queryHash(input.query), + "github.graphql.query": + input.query.length > TRACED_QUERY_MAX_CHARS + ? `${input.query.slice(0, TRACED_QUERY_MAX_CHARS)}…` + : input.query, + }); + return yield* Effect.gen(function* () { + const query = yield* budget.query( + host, + input.query, + allowReserve ? { allowReserve: true } : undefined, + ); + const response = yield* send({ + host, + operation: input.operation, + request: HttpClientRequest.post(gitHubApiUrls(host).graphql).pipe( + HttpClientRequest.acceptJson, + HttpClientRequest.bodyJsonUnsafe({ query, variables: input.variables ?? {} }), + ), + maxResponseBytes: DEFAULT_MAX_RESPONSE_BYTES, + allowReserve, + acceptNotModified: false, + graphql: true, + }); + // A body cut at the byte cap hides any `errors` past the cut and cannot be decoded. + if (response.truncated) { + return yield* new GitHubApiResponseError({ + host, + operation: input.operation, + status: response.status, + }); + } + yield* budget.observe(host, response.body); + const cost = Option.getOrUndefined(decodeGraphQlCost(response.body)); + if (cost !== undefined) { + yield* Effect.annotateCurrentSpan({ + "github.graphql.cost": cost.data.rateLimit.cost, + "github.graphql.remaining": cost.data.rateLimit.remaining, + }); + } + return response.body; + }).pipe(Effect.provideService(SourceControlRateLimit.CredentialScope, scope)); + }, + ); + + return GitHubApi.of({ graphql, rest, credential }); +}); + +export const layer = Layer.effect(GitHubApi, make); diff --git a/apps/server/src/sourceControl/GitHubCli.ts b/apps/server/src/sourceControl/GitHubCli.ts index 3a13ac69e9ef..0097794c7b9d 100644 --- a/apps/server/src/sourceControl/GitHubCli.ts +++ b/apps/server/src/sourceControl/GitHubCli.ts @@ -23,6 +23,7 @@ import { normalizeGitRemoteUrl } from "@t3tools/shared/git"; import { decodeJsonResult } from "@t3tools/shared/schemaJson"; import * as VcsProcess from "../vcs/VcsProcess.ts"; +import * as GitHubApi from "./GitHubApi.ts"; import * as GitHubGraphQlBudget from "./githubGraphQlBudget.ts"; import * as SourceControlRateLimit from "./SourceControlRateLimit.ts"; import { @@ -35,11 +36,7 @@ import { const DEFAULT_TIMEOUT_MS = 30_000; /** Server-local credential scope; never put its value in RPC payloads or cache keys. */ -export const PinnedGitHubCredential = Context.Reference<{ - readonly host: string; - readonly token: Redacted.Redacted; - readonly credentialFingerprint: string; -} | null>("t3/sourceControl/PinnedGitHubCredential", { defaultValue: () => null }); +export const PinnedGitHubCredential = GitHubApi.PinnedGitHubCredential; export const AllowGitHubReserve = Context.Reference( "t3/sourceControl/AllowGitHubReserve", diff --git a/apps/server/src/sourceControl/GitHubCredentials.ts b/apps/server/src/sourceControl/GitHubCredentials.ts new file mode 100644 index 000000000000..7207373bcf31 --- /dev/null +++ b/apps/server/src/sourceControl/GitHubCredentials.ts @@ -0,0 +1,189 @@ +import * as Cache from "effect/Cache"; +import * as Context from "effect/Context"; +import * as Crypto from "effect/Crypto"; +import * as Duration from "effect/Duration"; +import * as Effect from "effect/Effect"; +import * as Hex from "effect/encoding/Hex"; +import * as Exit from "effect/Exit"; +import * as Layer from "effect/Layer"; +import * as Option from "effect/Option"; +import * as PlatformError from "effect/PlatformError"; +import * as Redacted from "effect/Redacted"; +import * as Schema from "effect/Schema"; + +import { HostProcessEnvironment, HostProcessWorkingDirectory } from "@t3tools/shared/hostProcess"; + +import * as VcsProcess from "../vcs/VcsProcess.ts"; + +/** How long a token is reused before `gh` is asked again, so a `gh auth switch` applies soon. */ +const TOKEN_TTL = Duration.minutes(5); +/** No credential is retried sooner, so a fresh `gh auth login` takes effect on the next read. */ +const MISSING_TTL = Duration.seconds(10); + +export const GitHubCredentialSource = Schema.Literals(["env", "gh"]); +export type GitHubCredentialSource = typeof GitHubCredentialSource.Type; + +export interface GitHubCredential { + readonly host: string; + readonly token: Redacted.Redacted; + readonly source: GitHubCredentialSource; + /** A digest of host and token: safe for cache keys and rate-limit scopes, never the token. */ + readonly fingerprint: string; +} + +/** Nothing in the environment, and no `gh` on PATH to ask. */ +export class GitHubCliMissingError extends Schema.TaggedError()( + "GitHubCliMissingError", + { host: Schema.String }, +) { + override get message(): string { + return `No GitHub credential for ${this.host}: set GH_TOKEN, or install the GitHub CLI and run \`gh auth login\`.`; + } +} + +/** `gh` is installed but holds no login for the host. */ +export class GitHubNotSignedInError extends Schema.TaggedError()( + "GitHubNotSignedInError", + { host: Schema.String }, +) { + override get message(): string { + return `No GitHub credential for ${this.host}: run \`gh auth login --hostname ${this.host}\`.`; + } +} + +/** `gh auth token` timed out or failed for a reason other than having no login. */ +export class GitHubCliFailedError extends Schema.TaggedError()( + "GitHubCliFailedError", + { host: Schema.String, cause: Schema.Defect() }, +) { + override get message(): string { + return `The GitHub CLI could not hand over a credential for ${this.host}. Check \`gh auth status\` on the server.`; + } +} + +/** There is no token for the host. */ +export type GitHubCredentialUnavailableError = + | GitHubCliMissingError + | GitHubNotSignedInError + | GitHubCliFailedError; + +/** + * Where GitHub tokens come from. Callers ask per host and never see how the token was found, + * so another source (an in-app OAuth login) slots in here without touching any of them. + */ +export class GitHubCredentials extends Context.Service< + GitHubCredentials, + { + readonly get: ( + host: string, + ) => Effect.Effect; + /** Drops the held token after GitHub refused it, so the next read asks its source again. */ + readonly invalidate: (host: string) => Effect.Effect; + } +>()("t3/sourceControl/GitHubCredentials") {} + +function normalizeHost(host: string): string { + return host.trim().toLowerCase(); +} + +/** Hosts gh treats as GitHub.com-like for `GH_TOKEN`: github.com and GHE.com data residency. */ +function isGitHubDotCom(host: string): boolean { + return host === "github.com" || host.endsWith(".ghe.com"); +} + +/** + * The environment token for a host, in gh's precedence order. gh hands `GH_ENTERPRISE_TOKEN` to + * any non-github.com host; here it only goes to the host `GH_HOST` names, because a remote URL + * picks the host and a hostile one must not receive an enterprise token. + */ +export function environmentToken( + host: string, + env: Readonly>, +): string | null { + const names = isGitHubDotCom(host) + ? ["GH_TOKEN", "GITHUB_TOKEN"] + : env.GH_HOST?.trim().toLowerCase() === host + ? ["GH_ENTERPRISE_TOKEN", "GITHUB_ENTERPRISE_TOKEN"] + : []; + for (const name of names) { + const value = env[name]?.trim(); + if (value) return value; + } + return null; +} + +/** @public Service construction is part of the canonical Effect module API. */ +export const make = Effect.gen(function* () { + const process = yield* VcsProcess.VcsProcess; + const crypto = yield* Crypto.Crypto; + const environment = yield* HostProcessEnvironment; + const workingDirectory = yield* HostProcessWorkingDirectory; + + /** `host:sha256(token)`, safe for cache keys and rate-limit scopes. */ + const fingerprintOf = (host: string, token: string) => + crypto.digest("SHA-256", new TextEncoder().encode(token)).pipe( + Effect.map((digest) => `${host}:${Hex.encode(digest)}`), + // Hashing a string in memory has no platform failure worth a typed error. + Effect.orDie, + ); + + const fromGh = (host: string) => + process + .run({ + operation: "GitHubCredentials.get", + command: "gh", + args: ["auth", "token", "--hostname", host], + cwd: workingDirectory, + // Never let gh print the token into a debug log. + env: { GH_DEBUG: "", GH_PROMPT_DISABLED: "1" }, + timeoutMs: 10_000, + }) + .pipe( + Effect.mapError((error) => + error._tag === "VcsProcessSpawnError" && + error.cause instanceof PlatformError.PlatformError && + error.cause.reason._tag === "NotFound" + ? new GitHubCliMissingError({ host }) + : // gh exits non-zero with "no oauth token" when it has no login for the host. + error._tag === "VcsProcessExitError" + ? new GitHubNotSignedInError({ host }) + : new GitHubCliFailedError({ host, cause: error }), + ), + Effect.map((output) => output.stdout.trim()), + Effect.filterOrFail( + (token) => token !== "", + () => new GitHubNotSignedInError({ host }), + ), + ); + + const lookup = Effect.fn("GitHubCredentials.lookup")(function* (host: string) { + const fromEnv = environmentToken(host, environment); + const token = fromEnv ?? (yield* fromGh(host)); + return { + host, + token: Redacted.make(token), + source: fromEnv !== null ? "env" : "gh", + fingerprint: yield* fingerprintOf(host, token), + } satisfies GitHubCredential; + }); + + const cache = yield* Cache.makeWith(lookup, { + capacity: 32, + // A transient gh failure (a timeout, a locked keyring) is asked again on the next read. + timeToLive: (exit) => + Exit.isSuccess(exit) + ? TOKEN_TTL + : Exit.findErrorOption(exit).pipe( + Option.exists((error) => error._tag === "GitHubCliFailedError"), + ) + ? Duration.zero + : MISSING_TTL, + }); + + return GitHubCredentials.of({ + get: (host) => Cache.get(cache, normalizeHost(host)), + invalidate: (host) => Cache.invalidate(cache, normalizeHost(host)), + }); +}); + +export const layer = Layer.effect(GitHubCredentials, make);