fix(evalboard): variant-run coverage, averaged repeat stats, and per-task model name

evalboard/README.md

@@ -36,14 +36,18 @@ show up in the index — empty shells and the `latest` symlink are filtered out.
 
 `<task-id>` is the same string the eval framework writes to
 `task_results[].task_id` (e.g., `skill-flow-calculator`) and equals the
-subdir name under `<run-id>/default/`.
+subdir name under `<run-id>/<variant-id>/`, where `<variant-id>` is the
+experiment arm — `default` for a single-config run, or the arm name (e.g.
+`opus`, `with-skill`) in an A/B run. The task page selects the arm via `?v=`
+(mirroring `?r=` for replicates); a bare URL resolves the run's actual arm.
 
 ## Conventions
 
 - `/api/file?run=<id>&path=<relpath>` serves `.flow`, `.uipx`, etc. with
   path-traversal guard (`resolveSafePath`).
-- `/api/download?run=<id>[&task=<id>]` streams a zip of a task folder (with
-  `task`) or the whole run (without). Files are gathered by `collectTaskFiles`
+- `/api/download?run=<id>[&task=<id>][&v=<variant>]` streams a zip of a task
+  folder (with `task`; `v` selects the arm, default `default`) or the whole run
+  (without `task`). Files are gathered by `collectTaskFiles`
   / `collectRunFiles`, which reuse the `walkArtifacts` noise filter, and zipped
   by `lib/zip.ts` (a dependency-free DEFLATE writer).
 - Pass rows render green (`bg-green-50 text-green-700`), failures render red

evalboard/app/api/download/__tests__/route.test.ts

@@ -0,0 +1,73 @@
+import { promises as fs } from "node:fs";
+import os from "node:os";
+import path from "node:path";
+import { afterEach, beforeEach, describe, expect, test, vi } from "vitest";
+
+// collectTaskFiles reads RUNS_DIR, resolved from EVALBOARD_LOCAL_RUNS_DIR at
+// import time — so stub the env to a throwaway runs dir and import a fresh
+// module copy (like collect.test.ts / the refresh route test).
+let tmp: string;
+
+async function write(rel: string, body: string): Promise<void> {
+    const abs = path.join(tmp, rel);
+    await fs.mkdir(path.dirname(abs), { recursive: true });
+    await fs.writeFile(abs, body);
+}
+
+async function loadGet() {
+    vi.resetModules();
+    vi.stubEnv("EVALBOARD_LOCAL_RUNS_DIR", tmp);
+    return (await import("../route")).GET;
+}
+
+function get(qs: string): Request {
+    return new Request(`http://test/api/download?${qs}`, { method: "GET" });
+}
+
+const RUN = "2026-01-01_00-00-00";
+
+beforeEach(async () => {
+    tmp = await fs.mkdtemp(path.join(os.tmpdir(), "evalboard-download-"));
+    // A/B task: only under the glm-5-2 arm (no default/ subtree).
+    await write(`${RUN}/glm-5-2/ab-task/00/task.json`, "{}");
+    await write(`${RUN}/glm-5-2/ab-task/00/artifacts/out.txt`, "glm out");
+    // Single-config task: under default/.
+    await write(`${RUN}/default/solo-task/00/task.json`, "{}");
+});
+
+afterEach(async () => {
+    vi.unstubAllEnvs();
+    await fs.rm(tmp, { recursive: true, force: true });
+});
+
+describe("GET /api/download — variant (?v=) wiring", () => {
+    test("zips the requested arm's subtree", async () => {
+        const GET = await loadGet();
+        const res = await GET(get(`run=${RUN}&task=ab-task&v=glm-5-2`));
+        expect(res.status).toBe(200);
+        expect(res.headers.get("Content-Type")).toBe("application/zip");
+        // Non-empty archive → the arm's files were found. (Dropping the variant
+        // arg in the route would look in default/ab-task, which doesn't exist,
+        // and 404 — this assertion kills that mutation.)
+        expect(Number(res.headers.get("Content-Length"))).toBeGreaterThan(0);
+    });
+
+    test("a single-config task downloads with no ?v (default arm)", async () => {
+        const GET = await loadGet();
+        const res = await GET(get(`run=${RUN}&task=solo-task`));
+        expect(res.status).toBe(200);
+        expect(res.headers.get("Content-Type")).toBe("application/zip");
+    });
+
+    test("an unknown arm 404s rather than zipping the wrong subtree", async () => {
+        const GET = await loadGet();
+        const res = await GET(get(`run=${RUN}&task=ab-task&v=nope`));
+        expect(res.status).toBe(404);
+    });
+
+    test("missing run -> 400", async () => {
+        const GET = await loadGet();
+        const res = await GET(get(`task=ab-task`));
+        expect(res.status).toBe(400);
+    });
+});

evalboard/app/api/download/route.ts

@@ -6,20 +6,24 @@ import { createZip, type ZipEntry } from "@/lib/zip";
 export const dynamic = "force-dynamic";
 
 // Bundle a task folder, or a whole run, into a zip download.
-//   ?run=<id>&task=<id>  → just that task's folder (default/<taskId>/)
-//   ?run=<id>            → the entire run folder (run.json + every task dir)
+//   ?run=<id>&task=<id>[&v=<variant>]  → just that task's folder
+//                                        (<variant>/<taskId>/, variant default "default")
+//   ?run=<id>                          → the entire run folder (run.json + every task dir)
 // minus the usual scaffolding noise. In blob mode the collect* helpers fetch
 // the needed blobs first, so this mirrors what the page would load.
 export async function GET(req: Request) {
     const url = new URL(req.url);
     const runId = url.searchParams.get("run");
     const taskId = url.searchParams.get("task");
+    // Which A/B variant's copy of the task to zip. Absent → "default" (the
+    // single-config subdir), so single-model download links are unchanged.
+    const variant = url.searchParams.get("v") ?? undefined;
     if (!runId) {
         return new NextResponse("missing run", { status: 400 });
     }
 
     const files = taskId
-        ? await collectTaskFiles(runId, taskId)
+        ? await collectTaskFiles(runId, taskId, variant)
         : await collectRunFiles(runId);
     if (!files) {
         return new NextResponse("not found", { status: 404 });

evalboard/app/api/refresh/__tests__/route.test.ts

@@ -87,9 +87,10 @@ describe("POST /api/refresh", () => {
         });
 
         test('traversal id ".." -> 400, cache root untouched', async () => {
-            // ".." passes isValidId (dots are word-ish) but clearRunCacheDir's
-            // strict-child check rejects it, so the route returns 400 and the
-            // cache root is never the rm target.
+            // isValidId now rejects "." / ".." outright, so clearRunCacheDir
+            // returns false and the route 400s — and even if that guard were
+            // loosened, its strict-child check still refuses to rm the cache
+            // root. Belt and suspenders; the cache root is never the rm target.
             const marker = path.join(cache, "keep.txt");
             await fs.writeFile(marker, "x");
 

evalboard/app/runs/[id]/[...task]/page.tsx

@@ -9,6 +9,7 @@ import {
     replicateDirName,
 } from "@/lib/runs";
 import { readTaskReview } from "@/lib/reviews";
+import { DEFAULT_VARIANT, firstParam, variantLinkParam } from "@/lib/variant";
 import { fmtCompact, fmtRunTime, humanizeTaskId } from "@/lib/format";
 import { StatusPill } from "@/lib/pills";
 import { ChipButton } from "../chips";
@@ -33,40 +34,54 @@ export default async function TaskPage({
     searchParams,
 }: {
     params: Promise<{ id: string; task: string[] }>;
-    searchParams: Promise<{ r?: string }>;
+    // Next yields `string | string[]` for a repeated query key; declare it
+    // honestly and normalize with firstParam so `?v=a&v=b` can't reach a reader
+    // as an array (which would throw a 500 at path.join).
+    searchParams: Promise<{ r?: string | string[]; v?: string | string[] }>;
 }) {
     const { id, task: taskSegments } = await params;
-    const { r } = await searchParams;
+    const { r, v } = await searchParams;
     const taskId = taskSegments.join("/");
     // Replicate index from ?r=NN — repeated runs of one task share this task
     // path, so the query param is what selects which replicate's <NN>/ dir to
     // open. Absent / non-numeric / negative → replicate 0 (the single result a
     // non-repeated or legacy run has).
-    const parsedR = Number(r);
+    const parsedR = Number(firstParam(r));
     const replicate =
-        r != null && Number.isInteger(parsedR) && parsedR >= 0 ? parsedR : 0;
-    const task = await readTaskDetail(id, taskId, replicate);
+        Number.isInteger(parsedR) && parsedR >= 0 ? parsedR : 0;
+    // Variant (arm) from ?v=NAME — in an A/B run several variants share this task
+    // path, so ?v selects which arm's <variant>/ subdir to open. When ABSENT,
+    // readTaskDetail resolves the run's actual arm (single-arm tasks just work;
+    // this is what keeps pre-existing ?v-less deep links from 404-ing). The
+    // resolved arm comes back on task.variant and drives every other reader.
+    const task = await readTaskDetail(id, taskId, replicate, firstParam(v));
     if (!task) notFound();
+    const variant = task.variant ?? DEFAULT_VARIANT;
 
-    // Replicate indices available for this task — drives the run selector below.
-    // [0] (or fewer) for a non-repeated task, so the selector self-hides.
-    const replicates = await readTaskReplicates(id, taskId);
+    // Replicate indices available for this task/variant — drives the run
+    // selector below. [0] (or fewer) for a non-repeated task, so it self-hides.
+    const replicates = await readTaskReplicates(id, taskId, variant);
 
-    // variant is always "default" here; the replicate selects the <NN>/ dir.
+    // The replicate selects the <NN>/ dir within the variant's subtree.
     // readTaskReview returns null for older runs that predate the review feature.
     const review = await readTaskReview(
         id,
-        "default",
+        variant,
         taskId,
         replicateDirName(replicate),
     );
 
-    const log = await readLogTail(id, taskId, replicate);
+    const log = await readLogTail(id, taskId, replicate, variant);
     const conversation = parseConversation(
-        await readConversationLog(id, taskId, replicate),
+        await readConversationLog(id, taskId, replicate, variant),
     );
     const { flowDebug } = task;
 
+    // Preserve ?v= on in-page links (replicate selector, download) so switching
+    // replicates or downloading stays on the SAME arm. "default" is the implicit
+    // fallback, so it's omitted to keep single-config URLs clean.
+    const variantParam = variantLinkParam(variant);
+
     return (
         <div className="space-y-6">
             <nav className="text-sm text-gray-500 flex items-center gap-2 flex-wrap">
@@ -100,7 +115,7 @@ export default async function TaskPage({
                             return (
                                 <Link
                                     key={ri}
-                                    href={`/runs/${id}/${taskId}?r=${ri}`}
+                                    href={`/runs/${id}/${taskId}?r=${ri}${variantParam}`}
                                     // Keep the scroll position when switching runs
                                     // — Next scrolls to top on nav by default.
                                     scroll={false}
@@ -125,14 +140,31 @@ export default async function TaskPage({
                         {humanizeTaskId(taskId)}
                     </h1>
                     <StatusPill status={task.status} relabel />
+                    {/* Which LLM ran this task. Always shown when known so a
+                        single-model run still names its model, and an A/B run's
+                        per-model page is unambiguous. The variant alias (e.g.
+                        "kimi-k3") rides in the tooltip when it differs. */}
+                    {task.model && (
+                        <span
+                            className="inline-flex items-center gap-1.5 rounded-md border border-gray-200 bg-gray-50 px-2 py-0.5 font-mono text-xs text-gray-700"
+                            title={
+                                variant !== "default"
+                                    ? `model ${task.model} · variant ${variant}`
+                                    : `model ${task.model}`
+                            }
+                        >
+                            <span className="text-gray-400">model</span>
+                            {task.model}
+                        </span>
+                    )}
                     {/* Download zips the task folder from blob storage — an
                         internal-hosting surface (no blob backend in the public
                         OSS edition). See lib/edition.ts. */}
                     {isInternal && (
                         <a
                             href={`/api/download?run=${encodeURIComponent(
                                 id,
-                            )}&task=${encodeURIComponent(taskId)}`}
+                            )}&task=${encodeURIComponent(taskId)}${variantParam}`}
                             className="ml-auto inline-flex items-center gap-1.5 rounded-md border border-gray-300 bg-white px-3 py-1.5 text-sm font-medium text-gray-700 hover:bg-gray-50 hover:text-studio-blue"
                             download
                         >

evalboard/app/runs/[id]/__tests__/run-view.render.test.tsx

@@ -32,6 +32,7 @@ function row(
         cacheCreationTokens: null,
         cacheReadTokens: null,
         model: null,
+        variant: null,
         tags: [],
         skill: null,
         matureSkipped: false,

evalboard/app/runs/[id]/__tests__/run-view.test.ts

@@ -22,6 +22,7 @@ function row(
         cacheCreationTokens: null,
         cacheReadTokens: null,
         model: null,
+        variant: null,
         tags: [],
         skill: null,
         matureSkipped: false,

evalboard/app/runs/[id]/__tests__/task-grid.test.tsx

@@ -25,6 +25,7 @@ function row(
         cacheCreationTokens: null,
         cacheReadTokens: null,
         model: null,
+        variant: null,
         tags: [],
         skill: null,
         matureSkipped: false,
@@ -397,3 +398,105 @@ describe("TaskGrid — replicates", () => {
         ).toBe("1/2✓");
     });
 });
+
+describe("TaskGrid — multi-model (A/B) runs", () => {
+    test("shows a Model column and one row per model when models differ", () => {
+        render(
+            <TaskGrid
+                runId="r1"
+                tasks={[
+                    row("t", 3, 5, {
+                        variant: "kimi-k3",
+                        model: "moonshotai/kimi-k3",
+                    }),
+                    row("t", 3, 5, {
+                        variant: "glm-5-2",
+                        model: "z-ai/glm-5.2",
+                    }),
+                ]}
+            />,
+        );
+        const table = screen.getByRole("table");
+        // The Model column header appears only for multi-model runs.
+        expect(
+            within(table).getByRole("columnheader", { name: /Model/i }),
+        ).toBeInTheDocument();
+        // Both models are rendered — the two variants stay distinct rows, not
+        // collapsed into one (the multi-model "no info displayed" bug).
+        expect(within(table).getByText("moonshotai/kimi-k3")).toBeInTheDocument();
+        expect(within(table).getByText("z-ai/glm-5.2")).toBeInTheDocument();
+    });
+
+    test("each model's row links to its OWN variant via ?v=", () => {
+        render(
+            <TaskGrid
+                runId="r1"
+                tasks={[
+                    row("t", 3, 5, {
+                        variant: "kimi-k3",
+                        model: "moonshotai/kimi-k3",
+                    }),
+                    row("t", 3, 5, {
+                        variant: "glm-5-2",
+                        model: "z-ai/glm-5.2",
+                    }),
+                ]}
+            />,
+        );
+        const table = screen.getByRole("table");
+        const links = within(table).getAllByRole("link", { name: /^t/i });
+        const hrefs = links.map((l) => l.getAttribute("href")).sort();
+        expect(hrefs).toEqual(["/runs/r1/t?v=glm-5-2", "/runs/r1/t?v=kimi-k3"]);
+    });
+
+    test("hides the arm column for a single-arm run", () => {
+        render(
+            <TaskGrid
+                runId="r1"
+                tasks={[
+                    row("a", 3, 5, { variant: "default", model: "claude-sonnet-4-6" }),
+                    row("b", 3, 5, { variant: "default", model: "claude-sonnet-4-6" }),
+                ]}
+            />,
+        );
+        const table = screen.getByRole("table");
+        expect(
+            within(table).queryByRole("columnheader", { name: /Model/i }),
+        ).toBeNull();
+        expect(
+            within(table).queryByRole("columnheader", { name: /Variant/i }),
+        ).toBeNull();
+    });
+
+    test("same-model A/B: shows a Variant column labeling each arm", () => {
+        // Skill on/off (or terse/detailed): same model, different variant. Rows
+        // split on variant, so they must be distinguishable — gating on distinct
+        // MODEL would render two identical unlabeled rows (the reported gap).
+        render(
+            <TaskGrid
+                runId="r1"
+                tasks={[
+                    row("t", 3, 5, { variant: "bare", model: "claude-sonnet-4-6" }),
+                    row("t", 3, 5, {
+                        variant: "with-skill",
+                        model: "claude-sonnet-4-6",
+                    }),
+                ]}
+            />,
+        );
+        const table = screen.getByRole("table");
+        // Header reads "Variant" (models don't differ), not "Model".
+        expect(
+            within(table).getByRole("columnheader", { name: /Variant/i }),
+        ).toBeInTheDocument();
+        // Both arms are labeled and distinct.
+        expect(within(table).getByText("bare")).toBeInTheDocument();
+        expect(within(table).getByText("with-skill")).toBeInTheDocument();
+        // Each arm links to its own ?v=.
+        const hrefs = within(table)
+            .getAllByRole("link", { name: /^t/i })
+            .map((l) => l.getAttribute("href"))
+            .sort();
+        expect(hrefs).toEqual(["/runs/r1/t?v=bare", "/runs/r1/t?v=with-skill"]);
+    });
+});