diff --git a/electron/ai-edition/agent-tools.test.ts b/electron/ai-edition/agent-tools.test.ts index 4fe706b8..5f46c96e 100644 --- a/electron/ai-edition/agent-tools.test.ts +++ b/electron/ai-edition/agent-tools.test.ts @@ -1640,3 +1640,382 @@ describe("getCursorTrack", () => { expect(isMutatingTool("getCursorTrack")).toBe(false); }); }); + +// ─── D-ANCHOR: a zoom's focus is something a result can be held to ────────── +// +// `focus` was the one argument of a zoom write that no result ever mentioned. A +// span covering no clip is refused, a depth off the table is refused, and the +// span that actually landed is reported — but a focus ON the pointer and a focus +// half a frame away from it produced byte-identical results, so nothing +// downstream could tell the two apart. These tests hold both halves: the +// measurement is present and correct where the runtime can make it, and ABSENT +// everywhere it would be a claim the runtime has no standing to make. + +/** A pointer parked at one place over a source window, sampled at 10 Hz — the + * shape a zoom's focus is usually aimed at, and the one whose spread is 0 so a + * test can assert the reported position exactly. */ +function parkedSamples(fromSec: number, toSec: number, cx: number, cy: number) { + const out: Array<{ timeMs: number; cx: number; cy: number }> = []; + for (let ms = Math.round(fromSec * 1000); ms <= Math.round(toSec * 1000); ms += 100) { + out.push({ timeMs: ms, cx, cy }); + } + return out; +} + +function withTrack( + samples: Array<{ timeMs: number; cx: number; cy: number }>, + assetId = "asset_1", +) { + return { cursorTelemetry: { load: { status: "ok" as const, assetId, samples } } }; +} + +/** Two recordings, laid back to back. Their clips cover DIFFERENT ruler spans + * from IDENTICAL source windows, which is what makes reading one asset's track + * as if it described the other silently plausible. */ +function twoAssetDocument(): AxcutDocument { + const base = createEmptyDocument({ title: "Two", projectId: "proj_two" }); + return documentSchema.parse({ + ...base, + project: { ...base.project, primaryAssetId: "asset_1" }, + assets: [ + { id: "asset_1", kind: "video", label: "Screen", originalPath: "C:/a.mp4", durationSec: 10 }, + { id: "asset_2", kind: "video", label: "B-roll", originalPath: "C:/b.mp4", durationSec: 10 }, + ], + timeline: { + ...base.timeline, + clips: [ + { + id: "clip_a", + assetId: "asset_1", + sourceStartSec: 0, + sourceEndSec: 10, + timelineStartSec: 0, + timelineEndSec: 10, + wordRefs: [], + origin: "user", + reason: "", + }, + { + id: "clip_b", + assetId: "asset_2", + sourceStartSec: 0, + sourceEndSec: 10, + timelineStartSec: 10, + timelineEndSec: 20, + wordRefs: [], + origin: "user", + reason: "", + }, + ], + }, + }); +} + +/** Zoom ids are fresh uuids, so two runs of the same write differ in the one + * place that carries no meaning. Everything else must match byte for byte. */ +function withoutIds(document: AxcutDocument | undefined): string { + return JSON.stringify(document).replace(/zoom_[0-9a-f-]+/g, "zoom_x"); +} + +describe("addZoom answers for the focus it was given", () => { + it("reports where the pointer really was, beside the focus it received", () => { + // The samples outside the span are the other half of the assertion: a + // report that averaged the whole track would land nowhere near (0.8, 0.7). + const samples = [...parkedSamples(2, 6, 0.8, 0.7), ...parkedSamples(20, 25, 0.1, 0.2)]; + const result = executeAgentTool( + fixtureDocument(), + "addZoom", + JSON.stringify({ startSec: 2, endSec: 6, focus: { cx: 0.2, cy: 0.1 } }), + withTrack(samples), + ); + const anchor = JSON.parse(result.resultJson).cursorAnchor; + + expect(result.ok).toBe(true); + expect(anchor.available).toBe(true); + expect(anchor.focus).toEqual({ cx: 0.2, cy: 0.1 }); + expect(anchor.cursor).toEqual({ cx: 0.8, cy: 0.7 }); + expect(anchor.offset).toBeCloseTo(Math.hypot(0.6, 0.6), 3); + // A parked pointer has nowhere to stray: `spread` is what tells a reader + // whether the single position above stands for anything. + expect(anchor.spread).toBe(0); + expect(anchor.samples).toBe(41); + }); + + it("echoes the default focus a call never set", () => { + // The case a caller cannot reconstruct from its own arguments: it sent no + // focus, so it has no record that it asked for the centre of the frame. + const result = executeAgentTool( + fixtureDocument(), + "addZoom", + JSON.stringify({ startSec: 2, endSec: 6 }), + withTrack(parkedSamples(2, 6, 0.8, 0.7)), + ); + const anchor = JSON.parse(result.resultJson).cursorAnchor; + + expect(anchor.focus).toEqual({ cx: 0.5, cy: 0.5 }); + expect(anchor.cursor).not.toEqual(anchor.focus); + }); + + it("reports; it does not place — the document is identical either way", () => { + // THE invariant. This is a reporting change: a zoom written by a runtime + // that can read telemetry and one written by a runtime that cannot must be + // the same zoom, in the same place, described to the user the same way. + const args = JSON.stringify({ startSec: 2, endSec: 6, focus: { cx: 0.2, cy: 0.1 } }); + const blind = executeAgentTool(fixtureDocument(), "addZoom", args); + const seeing = executeAgentTool( + fixtureDocument(), + "addZoom", + args, + withTrack(parkedSamples(2, 6, 0.8, 0.7)), + ); + + expect(withoutIds(seeing.document)).toEqual(withoutIds(blind.document)); + expect(seeing.summary).toEqual(blind.summary); + // …and the ONLY difference is in the report. + expect(seeing.resultJson).toContain("cursorAnchor"); + expect(blind.resultJson).not.toContain("cursorAnchor"); + }); + + it("measures the span that LANDED, not the span that was asked for", () => { + // CLAMP. `addZoom {20,40}` on a 24.70 s clip stores 20–24.704. Measuring the + // requested window would let 152 samples the zoom never covers outvote the + // 48 it does, and answer (0.1, 0.1) for a pointer that sat at (0.9, 0.9) + // throughout the zoom. + const samples = [...parkedSamples(20, 24.7, 0.9, 0.9), ...parkedSamples(24.8, 40, 0.1, 0.1)]; + const result = executeAgentTool( + shortSingleClip(), + "addZoom", + JSON.stringify({ startSec: 20, endSec: 40 }), + withTrack(samples), + ); + const payload = JSON.parse(result.resultJson); + + expect(payload.clamped).toBe(true); + expect(payload.cursorAnchor.cursor).toEqual({ cx: 0.9, cy: 0.9 }); + expect(payload.cursorAnchor.samples).toBe(48); + }); + + it("spans every fragment when a zoom is split across two clips", () => { + // FRAGMENTATION. Two fragments, two source windows, one answer: reporting + // only the first would describe half the zoom and say nothing about it. + const samples = [ + ...parkedSamples(25, 29.9, 0.7, 0.3), + ...parkedSamples(30, 35, 0.7, 0.3), + ...parkedSamples(36, 40, 0.1, 0.9), + ]; + const result = executeAgentTool( + fixtureDocument(), + "addZoom", + JSON.stringify({ startSec: 25, endSec: 35 }), + withTrack(samples), + ); + const payload = JSON.parse(result.resultJson); + + expect(payload.fragments).toBe(2); + // 50 from the first fragment and 51 from the second: the first alone would + // be 50, and the decoy after the zoom ends would drag the position off + // (0.7, 0.3) if the window were the ruler span rather than the anchors. + expect(payload.cursorAnchor.samples).toBe(101); + expect(payload.cursorAnchor.cursor).toEqual({ cx: 0.7, cy: 0.3 }); + }); + + it("has no window to report on when nothing was placed", () => { + // NOTHING PLACED. The span covers no clip, so the write is refused and + // there is no landed window to measure — the refusal must not grow a + // measurement of a zoom that does not exist. + const telemetry = withTrack(parkedSamples(0, 30, 0.8, 0.7)); + const nowhere = executeAgentTool( + fixtureDocument(), + "addZoom", + JSON.stringify({ startSec: 90, endSec: 95 }), + telemetry, + ); + expect(nowhere.ok).toBe(false); + expect(nowhere.resultJson).toContain("covers no clip"); + expect(nowhere.resultJson).not.toContain("cursorAnchor"); + + // Same telemetry, a span that does cover a clip: the field is there. + const somewhere = executeAgentTool( + fixtureDocument(), + "addZoom", + JSON.stringify({ startSec: 2, endSec: 6 }), + telemetry, + ); + expect(somewhere.resultJson).toContain("cursorAnchor"); + }); + + it("stays silent when the runtime could not look, and never calls that an absence", () => { + // Blindness is not evidence. A runtime with no reader wired, and an asset + // whose sidecar was checked and is genuinely missing, both leave the field + // OFF — an absent field claims nothing, while a present one saying "no + // cursor here" would put our limit into the answer as their fact. + const args = JSON.stringify({ startSec: 2, endSec: 6 }); + const blind = [ + undefined, + { cursorTelemetry: {} }, + { cursorTelemetry: { load: { status: "unavailable" as const, assetId: "asset_1" } } }, + { cursorTelemetry: { load: { status: "no-sidecar" as const, assetId: "asset_1" } } }, + ]; + for (const options of blind) { + const result = executeAgentTool(fixtureDocument(), "addZoom", args, options); + expect(result.ok).toBe(true); + expect(result.resultJson).not.toContain("cursorAnchor"); + expect(result.resultJson).not.toMatch(/cursor|pointer|telemetry/i); + } + // The control: the same call, on a runtime that could read the track. + expect( + executeAgentTool(fixtureDocument(), "addZoom", args, withTrack(parkedSamples(2, 6, 0.8, 0.7))) + .resultJson, + ).toContain("cursorAnchor"); + }); + + it("says the SPAN carries no sample, not that the recording carries none", () => { + // The track was read and it simply does not reach here — a finding about + // this window, and the note has to keep it that size. + const result = executeAgentTool( + fixtureDocument(), + "addZoom", + JSON.stringify({ startSec: 20, endSec: 25 }), + withTrack(parkedSamples(0, 5, 0.8, 0.7)), + ); + const anchor = JSON.parse(result.resultJson).cursorAnchor; + + expect(anchor.available).toBe(false); + expect(anchor.reason).toBe("no-samples"); + expect(anchor.note).toMatch(/fact about this span/i); + expect(anchor.note).not.toMatch(/no cursor data|has no cursor|no pointer data/i); + + // One sample inside the span is enough to turn it into a measurement: the + // reason is about coverage, never about the recording. + const covered = executeAgentTool( + fixtureDocument(), + "addZoom", + JSON.stringify({ startSec: 20, endSec: 25 }), + withTrack([...parkedSamples(0, 5, 0.8, 0.7), { timeMs: 22_000, cx: 0.4, cy: 0.6 }]), + ); + expect(JSON.parse(covered.resultJson).cursorAnchor).toMatchObject({ + available: true, + cursor: { cx: 0.4, cy: 0.6 }, + samples: 1, + }); + }); + + it("will not read a position off frames a trim cuts out of playback", () => { + // The fixture trims source 10–12. Those instants are recorded and never + // seen, so a focus argued from them would describe a frame no viewer + // reaches — the same untruth as reporting the span that was requested + // instead of the one that was stored. + const samples = [...parkedSamples(10, 12, 0.9, 0.1), ...parkedSamples(12.1, 14, 0.3, 0.6)]; + const result = executeAgentTool( + fixtureDocument(), + "addZoom", + JSON.stringify({ startSec: 10, endSec: 14 }), + withTrack(samples), + ); + const anchor = JSON.parse(result.resultJson).cursorAnchor; + + expect(anchor.cursor).toEqual({ cx: 0.3, cy: 0.6 }); + expect(anchor.samples).toBe(20); + + // And when the trim takes ALL of them, that is said rather than averaged in. + const allCut = executeAgentTool( + fixtureDocument(), + "addZoom", + JSON.stringify({ startSec: 10, endSec: 12 }), + withTrack(parkedSamples(10, 12, 0.9, 0.1)), + ); + expect(JSON.parse(allCut.resultJson).cursorAnchor).toMatchObject({ + available: false, + reason: "trimmed-out", + }); + }); + + it("never measures a zoom against another recording's track", () => { + // Both clips draw source 0–10 from their own asset, so asset_1's samples + // fit asset_2's window arithmetically and would be read as a plausible + // answer about footage they do not describe. Silence is the only honest + // reply, and it is not a claim that asset_2 has no telemetry. + const telemetry = withTrack(parkedSamples(0, 10, 0.8, 0.7), "asset_1"); + const otherAsset = executeAgentTool( + twoAssetDocument(), + "addZoom", + JSON.stringify({ startSec: 12, endSec: 18 }), + telemetry, + ); + expect(otherAsset.ok).toBe(true); + expect(otherAsset.resultJson).not.toContain("cursorAnchor"); + + const ownAsset = executeAgentTool( + twoAssetDocument(), + "addZoom", + JSON.stringify({ startSec: 2, endSec: 8 }), + telemetry, + ); + expect(JSON.parse(ownAsset.resultJson).cursorAnchor).toMatchObject({ + available: true, + cursor: { cx: 0.8, cy: 0.7 }, + }); + }); + + it("gives every region of a batch its own answer", () => { + const result = executeAgentTool( + fixtureDocument(), + "addZooms", + JSON.stringify({ + regions: [ + { startSec: 2, endSec: 6 }, + { startSec: 20, endSec: 24 }, + ], + }), + withTrack(parkedSamples(2, 6, 0.8, 0.7)), + ); + const applied = JSON.parse(result.resultJson).applied; + + expect(applied[0].cursorAnchor).toMatchObject({ + available: true, + cursor: { cx: 0.8, cy: 0.7 }, + }); + // One measured, one not: a batch answer that leaned on the first region + // would describe the second with a position taken from somewhere else. + expect(applied[1].cursorAnchor).toMatchObject({ available: false, reason: "no-samples" }); + }); +}); + +describe("setZoom answers for the focus it kept", () => { + function seededZoom(): { document: AxcutDocument; zoomId: string } { + const document = executeAgentTool( + fixtureDocument(), + "addZoom", + JSON.stringify({ startSec: 2, endSec: 6, focus: { cx: 0.2, cy: 0.1 } }), + ).document as AxcutDocument; + return { document, zoomId: document.zoomRanges[0].id }; + } + + it("measures the moved span against the focus the call did not touch", () => { + // Reshaping a zoom is mostly a question about where it now points, and the + // focus is read back off the document rather than off the arguments — + // exactly as `renderedScale` already is. + const { document, zoomId } = seededZoom(); + const result = executeAgentTool( + document, + "setZoom", + JSON.stringify({ zoomId, startSec: 20, endSec: 24 }), + withTrack(parkedSamples(20, 24, 0.55, 0.45)), + ); + const anchor = JSON.parse(result.resultJson).cursorAnchor; + + expect(anchor.focus).toEqual({ cx: 0.2, cy: 0.1 }); + expect(anchor.cursor).toEqual({ cx: 0.55, cy: 0.45 }); + }); + + it("says nothing about the pointer when no track was read", () => { + const { document, zoomId } = seededZoom(); + const result = executeAgentTool( + document, + "setZoom", + JSON.stringify({ zoomId, startSec: 20, endSec: 24 }), + ); + expect(result.ok).toBe(true); + expect(result.resultJson).not.toContain("cursorAnchor"); + }); +}); diff --git a/electron/ai-edition/agent-tools.ts b/electron/ai-edition/agent-tools.ts index b7f17388..5a096639 100644 --- a/electron/ai-edition/agent-tools.ts +++ b/electron/ai-edition/agent-tools.ts @@ -38,6 +38,7 @@ import { replacePillSpan, resolvePillIds, } from "../../src/lib/ai-edition/timeline/timelineMap"; +import { trimAppliesToClip } from "../../src/lib/ai-edition/timeline/trim-mapping"; // ponytail: relative, and it has to stay that way — `electron/` never resolves // the `@/` alias (the main-process build does not declare it), which is why the // scale table was moved out of `components/video-editor/types.ts` to be @@ -932,6 +933,146 @@ export function resolveCursorAssetId( return assetId ?? document.project.primaryAssetId ?? document.assets[0]?.id ?? null; } +// ─── What the pointer was doing where the zoom landed ────────────────────── +// +// ponytail: `focus` is the one thing a zoom write says that nothing ever +// checked. A span covering no clip is refused, a depth outside the table is +// refused, and the result reports the span that really landed — but a focus on +// the pointer and a focus half a frame off it produced byte-identical results, +// so a caller had no way to find out which of the two it had just written. The +// result now carries where the pointer ACTUALLY was over the window the zoom +// landed on, beside the focus the call used. +// +// It informs; it decides nothing. Framing a slide, a face, or a corner the +// pointer never visits is a legitimate zoom: nothing is moved, nothing is +// refused, and this is a measurement the caller is free to disagree with. The +// only thing that changes is that the difference is on the page instead of +// nowhere. + +/** Per-axis median, not the mean. A pointer that crosses the frame and comes + * back averages to the middle of a path it spent no time on, while the median + * lands where it actually sat. `spread` is what says whether either number + * describes anything. */ +function medianOf(values: number[]): number { + const sorted = [...values].sort((a, b) => a - b); + const mid = sorted.length >> 1; + return sorted.length % 2 === 1 ? sorted[mid] : (sorted[mid - 1] + sorted[mid]) / 2; +} + +function round3(value: number): number { + return Math.round(value * 1000) / 1000; +} + +/** + * Where the recorded pointer was over the span a zoom write LANDED on. + * + * A zoom is authored in VIRTUAL seconds and the pointer is recorded on the + * asset's SOURCE clock, so something has to map between them — and that map + * already exists, computed once by `anchorRawRegionsToClips`, which writes + * `sourceStartSec`/`sourceEndSec` onto every fragment as it ventilates a span + * across clips. So the source windows are read back OFF THE FRAGMENTS the write + * just stored, never re-derived from `timelineStartSec`. A second derivation + * would be free to drift, and it would drift on exactly the cases that make this + * report worth having: a CLAMPED span and a span SPLIT across two clips both + * land on source windows that are not the ones asked for, and both are already + * right in the anchors. A fragment whose clip draws on another asset contributes + * nothing — this telemetry does not describe that media. + * + * Absence is never a claim. The field is left OFF when the runtime holds no + * telemetry for the footage under the span, which covers "no reader wired", + * "this asset has no sidecar" and "the zoom landed on another asset's clip" + * alike; none of those is evidence about the recording, and the tool description + * says so. `available: false` is emitted only for the two things that ARE + * findings about this span: nothing was recorded over it, or everything recorded + * over it is cut out of playback. + */ +function cursorAnchorReport( + // `id` is not read; it is what makes this a fragment of a stored region rather + // than an all-optional bag TypeScript would let any object satisfy. + regions: Array<{ id: string; clipId?: string; sourceStartSec?: number; sourceEndSec?: number }>, + document: AxcutDocument, + focus: { cx: number; cy: number }, + telemetry: CursorTelemetryContext | undefined, +): Record | undefined { + const load = telemetry?.load; + if (load?.status !== "ok") return undefined; + const byId = new Map(document.timeline.clips.map((c) => [c.id, c])); + const windows = regions.flatMap((region) => { + const clip = region.clipId ? byId.get(region.clipId) : undefined; + if (!clip || clip.assetId !== load.assetId) return []; + if (region.sourceStartSec === undefined || region.sourceEndSec === undefined) return []; + return [{ clip, startSec: region.sourceStartSec, endSec: region.sourceEndSec }]; + }); + if (windows.length === 0) return undefined; + + const xs: number[] = []; + const ys: number[] = []; + let cutOut = 0; + for (const sample of load.samples) { + if ( + !Number.isFinite(sample.timeMs) || + !Number.isFinite(sample.cx) || + !Number.isFinite(sample.cy) + ) { + continue; + } + const atSec = sample.timeMs / 1000; + const covering = windows.find((w) => atSec >= w.startSec && atSec <= w.endSec); + if (!covering) continue; + // `trimAppliesToClip` is THE rule for "is this cut on this clip", and the + // fragment names its clip, so the question is answered exactly once here. + // A trimmed instant is one the viewer never reaches: a position argued from + // frames that do not play would be the same kind of untruth as a span that + // reports the edges it was asked for rather than the ones it got. + if ( + document.timeline.trimRanges.some( + (t) => trimAppliesToClip(t, covering.clip) && atSec >= t.startSec && atSec <= t.endSec, + ) + ) { + cutOut += 1; + continue; + } + xs.push(sample.cx); + ys.push(sample.cy); + } + + if (xs.length === 0) { + return cutOut > 0 + ? { + available: false, + reason: "trimmed-out", + note: + "The pointer WAS recorded over this span, but a trim cuts every one of those " + + "instants out of playback, so none of them describes what a viewer sees here.", + } + : { + available: false, + reason: "no-samples", + note: + "This recording's pointer telemetry covers no instant of this span. That is a " + + "fact about this span, not about the recording.", + }; + } + + const cx = medianOf(xs); + const cy = medianOf(ys); + let spread = 0; + for (let i = 0; i < xs.length; i += 1) { + spread = Math.max(spread, Math.hypot(xs[i] - cx, ys[i] - cy)); + } + return { + available: true, + // Echoed, including the default a call that omitted `focus` silently got: + // "you asked for the centre" is the half of the comparison the caller + // cannot reconstruct from its own arguments. + focus: { cx: focus.cx, cy: focus.cy }, + cursor: { cx: round3(cx), cy: round3(cy) }, + offset: round3(Math.hypot(cx - focus.cx, cy - focus.cy)), + spread: round3(spread), + samples: xs.length, + }; +} + export function executeAgentTool( document: AxcutDocument, name: string, @@ -1348,6 +1489,10 @@ export function executeAgentTool( ...document, zoomRanges: [...document.zoomRanges, ...placed] as AxcutDocument["zoomRanges"], }; + // Measured over what was STORED, never over what was asked for: `placed` + // is the clamped, ventilated truth, so the report cannot end up + // describing a window the zoom does not occupy. + const anchor = cursorAnchorReport(placed, document, zoom.focus, options?.cursorTelemetry); return { ok: true, document: next, @@ -1358,6 +1503,7 @@ export function executeAgentTool( // model turns into "3×" for a frame that renders 1.80×. renderedScale: effectiveZoomScale(zoom), ...landingReport(landing, startMs / 1000, endMs / 1000), + ...(anchor ? { cursorAnchor: anchor } : {}), }), summary: `added zoom ${formatSec(landing.startSec)} – ${formatSec(landing.endSec)} ` + @@ -1419,6 +1565,18 @@ export function executeAgentTool( // re-ventilated, and `renderedScale` is the only number the viewer sees. const landed = new Set(landing.ids); const strength = rebuiltZooms.find((z) => landed.has(z.id)); + // The EFFECTIVE focus, read off the document exactly like `renderedScale` + // is: a setZoom that moved only the span still gets told what its + // untouched focus now looks at, which is most of the reason to reshape a + // zoom at all. + const anchor = strength + ? cursorAnchorReport( + rebuiltZooms.filter((z) => landed.has(z.id)), + document, + strength.focus, + options?.cursorTelemetry, + ) + : undefined; return { ok: true, document: next, @@ -1429,6 +1587,7 @@ export function executeAgentTool( : {}), ...(clearsCustomScale ? { clearedCustomScale: true } : {}), ...landingReport(landing, startMs / 1000, endMs / 1000), + ...(anchor ? { cursorAnchor: anchor } : {}), }), summary: `updated zoom ${formatSec(landing.startSec)} – ${formatSec(landing.endSec)}` + diff --git a/electron/ai-edition/deep-agent/service.test.ts b/electron/ai-edition/deep-agent/service.test.ts index f567da0e..f9691208 100644 --- a/electron/ai-edition/deep-agent/service.test.ts +++ b/electron/ai-edition/deep-agent/service.test.ts @@ -576,3 +576,67 @@ describe("cursor telemetry reaches the model", () => { expect(TOOL_DESCRIPTIONS.getCursorTrack).toMatch(/unavailable/); }); }); + +// ─── The zoom writes answer for their focus too ───────────────────────────── +// +// `executeAgentTool` is synchronous, so a sidecar read has to happen out here, +// in the wrapper, before the executor runs. It used to happen for exactly one +// tool — which is why a zoom could name a focus and no layer, from the schema +// down to the stored region, was ever in a position to say what was actually at +// that point of the frame. +describe("a zoom write is measured against the recorded track", () => { + it("reads the track for a zoom, and not for a tool with no focus to answer for", async () => { + const asked: string[] = []; + const runtime = { + cursor: { + read: async ({ assetId }: { assetId: string }) => { + asked.push(assetId); + return { status: "ok" as const, assetId, samples: SAMPLES }; + }, + }, + }; + const { sink } = recordingSink(); + const tools: BuiltTool[] = buildTools({ current: fixtureDocument() }, sink, true, runtime); + const zoom = tools.find((t) => t.name === "addZoom"); + const trim = tools.find((t) => t.name === "addTrim"); + if (!zoom || !trim) throw new Error("addZoom / addTrim are not built"); + + // The pointer sits at (0.8, 0.25) across this span while the call aims at + // the opposite corner: the write still lands, and the difference is on the + // page instead of nowhere. + const payload = JSON.parse( + String(await zoom.invoke({ startSec: 4, endSec: 5.6, focus: { cx: 0.1, cy: 0.9 } })), + ); + expect(payload.cursorAnchor).toMatchObject({ + available: true, + focus: { cx: 0.1, cy: 0.9 }, + cursor: { cx: 0.8, cy: 0.25 }, + }); + // No assetId is passed by a zoom write, so the wrapper resolves the primary + // asset — the same resolution the executor then reports against. + expect(asked).toEqual(["asset_1"]); + + // A trim has no focus and nothing to check: it must not pay for the read. + await trim.invoke({ startSec: 1, endSec: 2 }); + expect(asked).toEqual(["asset_1"]); + }); +}); + +describe("what the descriptions say about a zoom's focus", () => { + it("offers the measurement without turning it into an instruction", () => { + expect(TOOL_DESCRIPTIONS.addZoom).toMatch(/cursorAnchor/); + expect(TOOL_DESCRIPTIONS.addZoom).toMatch(/measurement, not a correction/); + // The absence rule, spelled out in the tool that will most often omit the + // field: a runtime that could not read a track has said nothing about the + // recording, and the prose is the only thing standing between that silence + // and a model reporting it as a finding. + expect(TOOL_DESCRIPTIONS.addZoom).toMatch(/never that the recording has none/); + + // …and NOT a rule about where to zoom. A description telling the model to + // put its focus on the pointer would swap its reading of the recording for + // a heuristic and cap it there — the same trade the tool layer refuses when + // it hands over a track instead of a list of moments. + expect(TOOL_DESCRIPTIONS.addZoom).not.toMatch(/focus (?:should|must|has to|needs to)/i); + expect(SYSTEM_PROMPT).not.toMatch(/cursorAnchor/); + }); +}); diff --git a/electron/ai-edition/deep-agent/service.ts b/electron/ai-edition/deep-agent/service.ts index e025827e..d804a3b1 100644 --- a/electron/ai-edition/deep-agent/service.ts +++ b/electron/ai-edition/deep-agent/service.ts @@ -155,9 +155,9 @@ export const TOOL_DESCRIPTIONS: Record = { "Reorder a placed clip: move `clipId` so it plays just before `beforeClipId` (pass null, or omit it, to move it last). Ids come from getCurrentDocument, where each clip carries its `index` and its label in `reason`. This preserves every clip id, every source range, every trim, and the zooms / speed regions / annotations anchored to each clip. This is the tool for 'swap these clips', 'put X first' and 'change the clip order' — replaceTimeline cannot reorder anything.", replaceTimeline: "Replace the whole timeline with the given kept intervals of the primary asset's source time. Everything outside the intervals becomes a trim. The intervals are SORTED, so this can never reorder clips — use moveClip for that. DO NOT use this for 'cut silences' or 'remove pauses' — the user has likely placed clips on the timeline that you'd be discarding. Use this ONLY when the user explicitly asks you to rebuild the timeline from scratch (e.g. 'start over with the kept intervals from the transcript'). It is refused when it would merge away, shorten or drop an existing clip; the refusal names them and the tool to use instead.", - addZoom: `Add a zoom-in over a span of the edited timeline (virtual seconds). depth is an ORDINAL 1–6, not a factor: it selects a magnification from a fixed table (${ZOOM_DEPTH_LEGEND}), so the default depth 3 renders at 1.80×. The result reports renderedScale — quote that, never the depth, when telling the user how strong the zoom is. focus is the zoom centre in 0–1 fractions of the frame (default centre). Use for 'zoom in on …' and the smart-zoom pass.`, - addZooms: `Add MANY zooms in one call: \`regions\` is a list, each entry taking exactly the fields addZoom takes (same depth table, ${ZOOM_DEPTH_LEGEND}). Use this for the smart-zoom pass, where you have decided every zoom before emitting the first one — sending them one at a time costs one round trip each. Each region stands or falls ALONE: one that covers no clip is refused by itself and listed in \`refused\` with its index and the reason, while the others are still applied. The result leads with requested / appliedCount / refusedCount, and each applied entry carries its renderedScale — quote that, never the depth.`, - setZoom: `Move, resize, or restyle an existing zoom by id (virtual-timeline seconds). Only the fields you pass are changed. depth selects from the same table (${ZOOM_DEPTH_LEGEND}); if the zoom carries a customScale (getCurrentDocument shows it as depthIsOverridden), that custom value is what renders, and passing depth clears it so the depth takes effect — the result says so. The result reports the resulting renderedScale.`, + addZoom: `Add a zoom-in over a span of the edited timeline (virtual seconds). depth is an ORDINAL 1–6, not a factor: it selects a magnification from a fixed table (${ZOOM_DEPTH_LEGEND}), so the default depth 3 renders at 1.80×. The result reports renderedScale — quote that, never the depth, when telling the user how strong the zoom is. focus is the zoom centre in 0–1 fractions of the frame (default centre). When the recording's pointer telemetry can be read for the footage under the span, the result also carries \`cursorAnchor\`: \`focus\` echoes the value this call used (including the default, if you left it out), \`cursor\` is where the pointer ACTUALLY was over the span the zoom landed on — the median of the recorded samples, \`spread\` being how far the farthest one strays from it — and \`offset\` is the distance between the two, in frame fractions. It is a measurement, not a correction: nothing is moved and no call is refused over it, and a zoom framing a slide, a face, or a region the pointer never enters is a legitimate choice. \`available:false\` names what it found instead (\`no-samples\`, \`trimmed-out\`). Its ABSENCE means no telemetry was read for that footage — never that the recording has none; assets[].hasCursorTelemetry and getCursorTrack are what answer that. Use for 'zoom in on …' and the smart-zoom pass.`, + addZooms: `Add MANY zooms in one call: \`regions\` is a list, each entry taking exactly the fields addZoom takes (same depth table, ${ZOOM_DEPTH_LEGEND}). Use this for the smart-zoom pass, where you have decided every zoom before emitting the first one — sending them one at a time costs one round trip each. Each region stands or falls ALONE: one that covers no clip is refused by itself and listed in \`refused\` with its index and the reason, while the others are still applied. The result leads with requested / appliedCount / refusedCount, and each applied entry carries its renderedScale — quote that, never the depth — plus the same \`cursorAnchor\` addZoom reports, whenever the footage under that region has readable pointer telemetry.`, + setZoom: `Move, resize, or restyle an existing zoom by id (virtual-timeline seconds). Only the fields you pass are changed. depth selects from the same table (${ZOOM_DEPTH_LEGEND}); if the zoom carries a customScale (getCurrentDocument shows it as depthIsOverridden), that custom value is what renders, and passing depth clears it so the depth takes effect — the result says so. The result reports the resulting renderedScale, and — when the footage under the span has readable pointer telemetry — the same \`cursorAnchor\` addZoom reports, measured against the zoom's EFFECTIVE focus, so a call that moved only the span still learns what its unchanged focus is now looking at.`, addSpeed: "Add a speed-change region over a span of the edited timeline (virtual seconds). speed > 1 fast-forwards, < 1 slows down (default 1.5×). Use to speed through slow stretches without cutting them.", setSpeed: @@ -210,6 +210,29 @@ interface ToolRuntime { availableByAssetId?: Record; } +/** + * The tools whose RESULT depends on the recorded pointer track, so the async + * wrapper knows to do the read before entering the synchronous executor. + * + * ponytail: the zoom writes are on this list, not only the reader. A `focus` + * that nothing reports back on is a `focus` nobody can check — the write + * answered `ok` whether it framed the pointer or the opposite corner. They pass + * no assetId, so the read resolves to the primary asset and the executor reports + * the anchor ONLY for fragments whose clip draws on that same asset: measured + * against the right media, or left off, never inferred from the wrong one. + * + * No cache. The read is a local JSON parse, `addZooms` is what keeps a whole + * zoom pass to one call rather than N, and nothing on this path memoises today — + * a cache here would be one more thing to invalidate for a saving nobody has + * measured. + */ +const TOOLS_READING_CURSOR: ReadonlySet = new Set([ + "getCursorTrack", + "addZoom", + "addZooms", + "setZoom", +]); + // One document tool: run it through the shared executor, advance the holder so // the next tool in the turn sees the edit, and emit exactly ONE start/end pair // carrying the executor's REAL verdict. @@ -237,10 +260,9 @@ function documentTool( // gate every mutation passes through, and it has to stay testable // without a filesystem). So the load happens here and its verdict — // including "I could not look" — goes in as data. - const load = - name === "getCursorTrack" - ? await loadCursorTelemetry(holder.current, args, runtime) - : undefined; + const load = TOOLS_READING_CURSOR.has(name) + ? await loadCursorTelemetry(holder.current, args, runtime) + : undefined; const execution = executeAgentTool(holder.current, name, JSON.stringify(args), { editsAllowed, cursorTelemetry: { availableByAssetId: runtime.availableByAssetId, load }, diff --git a/technical-documentation/architecture/ai-agent.md b/technical-documentation/architecture/ai-agent.md index 8e59015a..6278474e 100644 --- a/technical-documentation/architecture/ai-agent.md +++ b/technical-documentation/architecture/ai-agent.md @@ -47,16 +47,16 @@ The model never free-writes the project document. It can only call the fixed set |---|---|---| | `getCurrentDocument` | Reads a compact project, asset, clip, trim, and modifier snapshot with explicit time bases. Each asset reports `hasCameraTrack` / `cameraVisible` / `hasCursorTelemetry` beside `hasTranscript` (`hasCursorTelemetry` is three-valued: `true`, `false` when the asset was checked and has none, `null` when it was not checked — never `false` for something we failed to look at), the document reports `hasAnyCamera` and `autoFocusAll`, and each zoom reports the `renderedScale` the viewer will see plus `customScale` / `depthIsOverridden` when a custom scale makes its `depth` inert. | Nothing. | | `getTranscript` | Reads the transcript segments for an asset, or the primary asset, in full. On the production path a segment is one word, so a half-hour recording is a few thousand of them — there is no cap, and no per-model context budget to derive one from. | Nothing. | -| `getCursorTrack` | Reads the recorded pointer telemetry for an asset as a DIGEST: the moments the cursor sat still or clicked, each with its hold, its average position, its click count, its source time and the `virtualSec` that `addZoom` takes — never the raw samples. Answers `available:false` with `reason:"no-sidecar"` (checked, this asset has none) or `reason:"unavailable"` (could not be read from here), and the two are never conflated. | Nothing. | +| `getCursorTrack` | Reads the recorded pointer telemetry for an asset as REAL SAMPLES, reduced per axis against time (Douglas–Peucker) rather than summarised: every point returned is one that was recorded, carrying its source `atSec`, `cx`/`cy`, the `virtualSec` that `addZoom` takes (left off the points, with `virtualEqualsSource` on the envelope, when the two coincide everywhere), a `shape` index per distinct pointer bitmap, and `kind` / `trimmed` where they apply. Answers `available:false` with `reason:"no-sidecar"` (checked, this asset has none) or `reason:"unavailable"` (could not be read from here), and the two are never conflated. | Nothing. | | `addTrim` | Adds one source-time cut inside a clip. | `timeline.trimRanges`. | | `addTrims` | Adds many cuts in one call, replaying `addTrim` per entry so the rules cannot drift apart. Each range stands alone: one that cannot be placed is refused by itself and named with its index and reason while the rest are applied, and the result leads with `requested` / `appliedCount` / `refusedCount`. Only a batch where nothing landed is an error. | `timeline.trimRanges`. | | `setTrim` | Moves or resizes an existing source-time trim. | The matching `timeline.trimRanges` entry. | | `setClipRange` | Changes a clip's source in/out points and relays clips back-to-back. | The clip range and any anchored regions clamped or removed by the shared timeline mutator. | | `moveClip` | Reorders a placed clip by naming the clip it should play before (`null` = last). Preserves every clip id, source range, trim and anchored modifier. | Timeline clip order; anchored modifiers' derived ms follow their clip. | | `replaceTimeline` | Rebuilds the primary-asset timeline from kept source-time intervals. Preserves the id, origin and label of every clip an interval reproduces exactly, carries existing trims through, and never touches another asset's trims. Refused when it would merge away, shorten or drop a clip, or when the intervals are not ascending (a reorder it cannot perform — the refusal points at `moveClip`). | Timeline clips and trim ranges. | -| `addZoom` | Adds a clip-anchored zoom over virtual timeline time. `depth` is an ordinal selecting from `ZOOM_DEPTH_SCALES` (1.25×–5.0×, non-linear); the result reports the resulting `renderedScale`. | `zoomRanges`. | -| `addZooms` | Adds many zooms in one call, replaying `addZoom` per entry, with the same per-entry refusal and reporting contract as `addTrims`. | `zoomRanges`. | -| `setZoom` | Moves, resizes, or restyles a zoom pill. Changing `depth` clears any `customScale` on that pill — otherwise the write is a no-op at render — and says so in the result. | The clip-anchored `zoomRanges` fragments represented by that pill. | +| `addZoom` | Adds a clip-anchored zoom over virtual timeline time. `depth` is an ordinal selecting from `ZOOM_DEPTH_SCALES` (1.25×–5.0×, non-linear); the result reports the resulting `renderedScale`. When the pointer track can be read for the footage under the span it also reports `cursorAnchor`: the `focus` the call used (default included) beside the median recorded pointer position over the span the zoom LANDED on, with `spread`, `offset` and the sample count. The source windows come from the stored fragments' own `sourceStartSec`/`sourceEndSec`, so a clamped or split zoom is described by the windows it really occupies; trimmed instants are left out; and the field is OMITTED — never guessed, never turned into an absence of data — when no track covers that footage. | `zoomRanges`. | +| `addZooms` | Adds many zooms in one call, replaying `addZoom` per entry, with the same per-entry refusal and reporting contract as `addTrims`. Each applied entry carries its own `cursorAnchor`. | `zoomRanges`. | +| `setZoom` | Moves, resizes, or restyles a zoom pill. Changing `depth` clears any `customScale` on that pill — otherwise the write is a no-op at render — and says so in the result. Reports `cursorAnchor` against the zoom's EFFECTIVE focus, read back off the document, so a span-only edit still learns what its unchanged focus now looks at. | The clip-anchored `zoomRanges` fragments represented by that pill. | | `addSpeed` | Adds a clip-anchored speed region over virtual timeline time. | `legacyEditor.speedRegions`. | | `setSpeed` | Moves, resizes, or changes an existing speed pill. | The corresponding `legacyEditor.speedRegions` fragments. | | `addAnnotation` | Adds a positioned text annotation over virtual timeline time. | `annotations`. | @@ -89,6 +89,6 @@ Chat sessions and checkpoints live only in nested process-memory `Map` objects i - Chat sessions and message checkpoints have no durable persistence. - `allowAgentEdits` has no per-turn approval channel. When it is off the agent reads freely, is told by its system prompt to state the edit and ask, and has every write refused by `executeAgentTool` with a `consent_required` payload; the returned document is withheld as well. But there is no way for the user to answer "yes, go ahead" for one turn — they have to re-enable the setting in Settings → AI, which is what the refusal tells the model to say. -- Cursor telemetry is read through an injected `CursorTelemetryReader` (`electron/ipc/handlers.ts` builds the only production one, behind `resolveApprovedVideoPath`). A runtime with no reader wired answers `reason: "unavailable"` on every call and reports `hasCursorTelemetry: null` — honest, and useless. There is no cache: a turn probes each asset's sidecar once and reads it in full only if the model asks. -- The digest reports dwells and click counts, not the raw samples, and `readCursorSidecar`'s normalizer flattens `double-click` / `right-click` / `middle-click` to `move` before the digest sees them, so those clicks are currently invisible. The digest already counts the wide union; the narrowing is upstream, in `CursorRecordingSample`. +- Cursor telemetry is read through an injected `CursorTelemetryReader` (`electron/ipc/handlers.ts` builds the only production one, behind `resolveApprovedVideoPath`). A runtime with no reader wired answers `reason: "unavailable"` on every call and reports `hasCursorTelemetry: null` — honest, and useless. There is no cache: a turn probes each asset's sidecar once, then reads it in full for every call in `TOOLS_READING_CURSOR` — `getCursorTrack` and the three zoom writes, which need it to report `cursorAnchor`. N separate `addZoom` calls therefore re-read the same file N times; `addZooms` is what keeps a whole zoom pass to one read. +- `readCursorSidecar`'s normalizer flattens `double-click` / `right-click` / `middle-click` to `move` before the track is built, so those clicks reach the model as plain moves rather than as the events they were. The narrowing is upstream, in `CursorRecordingSample`. - The deep-agent instance is rebuilt for every turn without a LangGraph checkpointer, so stateful agent threads do not persist independently of the explicit chat history.