diff --git a/electron/ai-edition/agent-tools.test.ts b/electron/ai-edition/agent-tools.test.ts index fc8ba4085..4bad12b91 100644 --- a/electron/ai-edition/agent-tools.test.ts +++ b/electron/ai-edition/agent-tools.test.ts @@ -2505,3 +2505,112 @@ describe("addTrim shapes the cut like the transcript pane", () => { expect(lastTrim(result.document)?.endSec).toBe(15); }); }); + +describe("zoom transitions are reported, not hidden (#1028)", () => { + // 1.8× (depth 3): each move lasts 0.6 + 0.55·ln 1.8 ≈ 0.923 s of screen time. + const MOVE_SEC = 0.923; + + function run(document: AxcutDocument, name: string, args: unknown) { + const result = executeAgentTool(document, name, JSON.stringify(args)); + expect(result.ok).toBe(true); + return { document: result.document ?? document, payload: JSON.parse(result.resultJson) }; + } + + function zoomedAt10to12() { + return run(shortSingleClip(), "addZoom", { startSec: 10, endSec: 12, depth: 3 }); + } + + function snapshotZoom(document: AxcutDocument) { + return run(document, "getCurrentDocument", {}).payload.zoomRanges[0]; + } + + it("gives each zoom its move windows on the timeline", () => { + const added = zoomedAt10to12(); + expect(added.payload.cutTransitions).toBeUndefined(); + const zoom = snapshotZoom(added.document); + expect(zoom.transitionSec).toBe(MOVE_SEC); + expect(zoom.zoomInFromSec).toBe(10 - MOVE_SEC); + expect(zoom.zoomOutUntilSec).toBe(12 + MOVE_SEC); + expect(zoom.cutByTrim).toBeUndefined(); + }); + + it("warns when a trim at the zoom's end cuts its whole zoom-out (the issue's repro)", () => { + const trimmed = run(zoomedAt10to12().document, "addTrim", { startSec: 12, endSec: 14 }); + const zoomId = snapshotZoom(trimmed.document).id; + expect(trimmed.payload.cutTransitions).toEqual([{ zoomId, side: "out", cutSec: MOVE_SEC }]); + expect(snapshotZoom(trimmed.document).cutByTrim).toBe("out"); + }); + + it("flags a zoom on the second clip of an asset that a trim without clipId cuts", () => { + // Two clips over one recording, and a pre-v7 trim (no clipId): playback cuts it from + // both clips, while the ruler maps it through the first one only. + const base = shortSingleClip(); + const clip = base.timeline.clips[0]; + const length = (clip.sourceEndSec ?? 0) - clip.sourceStartSec; + const twoClips: AxcutDocument = { + ...base, + timeline: { + ...base.timeline, + clips: [ + clip, + { + ...clip, + id: "clip_2", + timelineStartSec: length, + timelineEndSec: 2 * length, + }, + ], + trimRanges: [ + { + id: "trim_legacy", + assetId: clip.assetId, + startSec: 12, + endSec: 14, + reason: "", + origin: "user", + }, + ], + }, + }; + const added = run(twoClips, "addZoom", { + startSec: length + 10, + endSec: length + 12, + depth: 3, + }); + expect(added.payload.cutTransitions).toEqual([ + { zoomId: added.payload.zoomId, side: "out", cutSec: MOVE_SEC }, + ]); + }); + + it("stays quiet about a trim clear of both windows", () => { + const trimmed = run(zoomedAt10to12().document, "addTrim", { startSec: 14, endSec: 15 }); + expect(trimmed.payload.cutTransitions).toBeUndefined(); + }); + + it("warns the zoom write too, when it lands next to an existing cut", () => { + const trimmed = run(shortSingleClip(), "addTrim", { startSec: 8, endSec: 9.5 }); + const added = run(trimmed.document, "addZoom", { startSec: 10, endSec: 12, depth: 3 }); + expect(added.payload.cutTransitions).toEqual([ + { zoomId: added.payload.zoomId, side: "in", cutSec: 0.423 }, + ]); + expect(snapshotZoom(added.document).cutByTrim).toBe("in"); + }); + + it("repeats the warning when setZoom moves a zoom that is still cut", () => { + const trimmed = run(zoomedAt10to12().document, "addTrim", { startSec: 12, endSec: 14 }); + const zoomId = snapshotZoom(trimmed.document).id; + const moved = run(trimmed.document, "setZoom", { zoomId, startSec: 9 }); + expect(moved.payload.cutTransitions).toEqual([ + { zoomId: moved.payload.zoomId, side: "out", cutSec: MOVE_SEC }, + ]); + }); + + it("stretches the zoom-out across a speed region, and catches a trim placed there", () => { + const sped = run(zoomedAt10to12().document, "addSpeed", { startSec: 12, endSec: 20, speed: 3 }); + expect(snapshotZoom(sped.document).zoomOutUntilSec).toBe(14.77); + const trimmed = run(sped.document, "addTrims", { ranges: [{ startSec: 14, endSec: 15 }] }); + expect(trimmed.payload.applied[0].cutTransitions).toEqual([ + { zoomId: snapshotZoom(trimmed.document).id, side: "out", cutSec: 0.77 }, + ]); + }); +}); diff --git a/electron/ai-edition/agent-tools.ts b/electron/ai-edition/agent-tools.ts index 3ffc9b215..b8ffb68a4 100644 --- a/electron/ai-edition/agent-tools.ts +++ b/electron/ai-edition/agent-tools.ts @@ -48,7 +48,10 @@ import { replacePillSpan, resolvePillIds, } from "../../src/lib/ai-edition/timeline/timelineMap"; -import { trimAppliesToClip } from "../../src/lib/ai-edition/timeline/trim-mapping"; +import { + trimAppliesToClip, + trimToTimelineSpan, +} from "../../src/lib/ai-edition/timeline/trim-mapping"; // ponytail: relative, and it has to stay that way — `electron/` never resolves // the `@/` alias (the main-process build does not declare it), which is why the // scale table was moved out of `components/video-editor/types.ts` to be @@ -57,6 +60,10 @@ import { effectiveZoomScale, ZOOM_DEPTH_LEGEND, } from "../../src/lib/ai-edition/timeline/zoom-scale"; +import { + transitionCutsMs, + zoomTransitions, +} from "../../src/lib/ai-edition/timeline/zoom-transitions"; import { SETTING_BOUNDS } from "../../src/lib/projectDefaults"; export interface AgentToolExecution { @@ -713,6 +720,56 @@ function roundSec(ms: number): number { return Math.round(ms) / 1000; } +// Each zoom pill's camera moves in virtual ms, and how much of each a trim cuts off (#1028). +// The pill shows the hold only: the move in runs before it, the move out after it, and a +// trim over either makes the export jump at the cut. +function zoomTransitionsForAgent(document: AxcutDocument) { + const legacy = document.legacyEditor as Record | null; + const speedRegions = + (legacy?.speedRegions as + | Array<{ id: string; startMs: number; endMs: number; speed: number }> + | undefined) ?? []; + // A trim with no clipId cuts every clip of its asset in playback, while the ruler maps it + // through one: measured per clip, so a zoom on any of them learns of the cut. + const trims = document.timeline.trimRanges.flatMap((trim) => { + const clips = + trim.clipId !== undefined + ? [trim] + : document.timeline.clips + .filter((clip) => clip.assetId === trim.assetId) + .map((clip) => ({ ...trim, clipId: clip.id })); + return clips.flatMap((anchored) => { + const span = trimToTimelineSpan(anchored, document.timeline.clips); + return span ? [{ startMs: span.start * 1000, endMs: span.end * 1000 }] : []; + }); + }); + return coalesceForAgent(document.zoomRanges).map((zoom) => { + const transitions = zoomTransitions({ ...zoom, scale: effectiveZoomScale(zoom) }, speedRegions); + return { zoom, transitions, cut: transitionCutsMs(zoom, transitions, trims) }; + }); +} + +/** The transitions this edit cut that were not cut, or not this way, before it: the warning a + * zoom or trim write carries, like `cursorAnchor`: reported, never refused. */ +function cutTransitionsReport(before: AxcutDocument, after: AxcutDocument) { + const cutsOf = (document: AxcutDocument) => + zoomTransitionsForAgent(document).flatMap(({ zoom, cut }) => + (["in", "out"] as const) + .map((side) => ({ + zoomId: zoom.id, + side, + cutSec: roundSec(side === "in" ? cut.inMs : cut.outMs), + key: `${zoom.id}:${side}:${zoom.startMs}:${zoom.endMs}`, + })) + .filter((c) => c.cutSec > 0), + ); + const known = new Set(cutsOf(before).map((c) => `${c.key}:${c.cutSec}`)); + const cuts = cutsOf(after) + .filter((c) => !known.has(`${c.key}:${c.cutSec}`)) + .map(({ key: _key, ...c }) => c); + return cuts.length ? { cutTransitions: cuts } : {}; +} + // Compact projection of the document for the model: everything it needs to // reference ids and times, nothing it doesn't (no waveform paths, no history). // @@ -764,7 +821,10 @@ export function documentSnapshotForModel( zoomNote: `renderedScale is what the viewer sees (depth is an ordinal, not a factor: ${ZOOM_DEPTH_LEGEND}). ` + "When a zoom carries customScale it wins over depth and depthIsOverridden is true — " + - "a setZoom that only changes depth on such a zoom clears customScale so the depth takes effect.", + "a setZoom that only changes depth on such a zoom clears customScale so the depth takes effect. " + + "startSec–endSec is where the zoom HOLDS: it animates in over zoomInFromSec–startSec and out over " + + "endSec–zoomOutUntilSec (transitionSec of screen time each; a speed region scales that on the " + + "timeline, longer above 1x and shorter below, so read the bounds rather than adding transitionSec). cutByTrim names the move a trim cuts into (in, out or both): the export jumps at that cut.", project: { id: document.project.id, title: document.project.title }, primaryAssetId: document.project.primaryAssetId ?? document.assets[0]?.id ?? null, autoFocusAll, @@ -821,10 +881,16 @@ export function documentSnapshotForModel( endSec: s.endSec, reason: s.reason, })), - zoomRanges: coalesceForAgent(document.zoomRanges).map((z) => ({ + zoomRanges: zoomTransitionsForAgent(document).map(({ zoom: z, transitions, cut }) => ({ id: z.id, startSec: roundSec(z.startMs), endSec: roundSec(z.endMs), + zoomInFromSec: roundSec(transitions.inFromMs), + zoomOutUntilSec: roundSec(transitions.outUntilMs), + transitionSec: roundSec(transitions.durationMs), + ...(cut.inMs > 0 || cut.outMs > 0 + ? { cutByTrim: cut.inMs > 0 ? (cut.outMs > 0 ? "both" : "in") : "out" } + : {}), depth: z.depth, renderedScale: effectiveZoomScale(z), // Emitted only when set: an unconditional `customScale: null` on every @@ -1501,7 +1567,12 @@ export function executeAgentTool( return { ok: true, document: next, - resultJson: JSON.stringify({ trimRangeId: trim.id, startSec, endSec }), + resultJson: JSON.stringify({ + trimRangeId: trim.id, + startSec, + endSec, + ...cutTransitionsReport(document, next), + }), summary: `added trim ${formatSec(startSec)} – ${formatSec(endSec)}`, }; } @@ -1548,7 +1619,12 @@ export function executeAgentTool( return { ok: true, document: next, - resultJson: JSON.stringify({ trimRangeId, startSec, endSec }), + resultJson: JSON.stringify({ + trimRangeId, + startSec, + endSec, + ...cutTransitionsReport(document, next), + }), summary: `moved trim to ${formatSec(startSec)} – ${formatSec(endSec)}`, }; } @@ -1744,6 +1820,7 @@ export function executeAgentTool( renderedScale: effectiveZoomScale(zoom), ...landingReport(landing, startMs / 1000, endMs / 1000), ...(anchor ? { cursorAnchor: anchor } : {}), + ...cutTransitionsReport(document, next), }), summary: `added zoom ${formatSec(landing.startSec)} – ${formatSec(landing.endSec)} ` + @@ -1828,6 +1905,7 @@ export function executeAgentTool( ...(clearsCustomScale ? { clearedCustomScale: true } : {}), ...landingReport(landing, startMs / 1000, endMs / 1000), ...(anchor ? { cursorAnchor: anchor } : {}), + ...cutTransitionsReport(document, next), }), summary: `updated zoom ${formatSec(landing.startSec)} – ${formatSec(landing.endSec)}` + diff --git a/electron/ai-edition/deep-agent/service.test.ts b/electron/ai-edition/deep-agent/service.test.ts index 5aebb9a33..78326fdfc 100644 --- a/electron/ai-edition/deep-agent/service.test.ts +++ b/electron/ai-edition/deep-agent/service.test.ts @@ -390,6 +390,14 @@ describe("what the descriptions say about zoom strength", () => { expect(TOOL_DESCRIPTIONS.addCameraFullscreen).toMatch(/hasCameraTrack/); expect(SYSTEM_PROMPT).toMatch(/hasCameraTrack/); }); + + it("tells every zoom and trim write where a zoom's moves are (#1028)", () => { + for (const name of ["addZoom", "addZooms", "setZoom", "addTrim", "addTrims", "setTrim"]) { + expect(TOOL_DESCRIPTIONS[name]).toContain("zoomOutUntilSec"); + expect(TOOL_DESCRIPTIONS[name]).toContain("cutTransitions"); + } + expect(SYSTEM_PROMPT).toMatch(/zoomInFromSec.*untrimmed/); + }); }); // ── D-CONSENT ─────────────────────────────────────────────────────────────── diff --git a/electron/ai-edition/deep-agent/service.ts b/electron/ai-edition/deep-agent/service.ts index 21f5b79a5..b7924e870 100644 --- a/electron/ai-edition/deep-agent/service.ts +++ b/electron/ai-edition/deep-agent/service.ts @@ -25,6 +25,7 @@ import type { AxcutDocument } from "../../../src/lib/ai-edition/schema"; // (`depth/2 + 0.5`) that was purged from two rendering sites and survived here, // wrong at both ends of a table that actually runs 1.25× to 5.0×. import { ZOOM_DEPTH_LEGEND } from "../../../src/lib/ai-edition/timeline/zoom-scale"; +import { ZOOM_TRANSITION_LEGEND } from "../../../src/lib/ai-edition/timeline/zoom-transitions"; import { type AgentToolExecution, addAnnotationArgs, @@ -120,6 +121,7 @@ const BASE_SYSTEM_PROMPT = [ "- Silences, pauses and dead stretches are removed as trims INSIDE the placed clip. Send them together with addTrims once you know the ranges; addTrim is for a single cut or a correction. The placed clip stays the canonical cut; it is not rebuilt to drop them.", "- Changing where a clip starts or ends within its source is setClipRange — the clip's in/out, distinct from a trim.", `- addZoom takes a virtual-timeline span (depth is an ordinal 1–6 selecting from a fixed table — ${ZOOM_DEPTH_LEGEND} — never a multiplier; focus in 0–1 frame fractions). addSpeed changes pacing over a span. addAnnotation puts text on screen. addCameraFullscreen enlarges the webcam, and only does something where assets[].hasCameraTrack is true.`, + "- A zoom animates in just before its span and out just after it (zoomInFromSec, zoomOutUntilSec in getCurrentDocument). Leave those windows untrimmed, or the export jumps at the cut.", "- addAudio lays an imported voiceover or music file over a span. It plays an asset the project already has (kind 'audio'); importing or recording one is the editor's job, not a tool you have — so when the project has none, say so rather than naming an id that does not exist.", "- moveClip changes the order of placed clips, one call per clip that moves, preserving ids, source ranges, trims and anchored effects. replaceTimeline rebuilds the timeline from kept intervals and sorts them, so it cannot reorder anything.", "- Deleting is a first-class action, not a workaround: removeTrim, removeModifier, removeClip. Never fake a deletion by re-adding an element or zeroing it out (span 0, speed 1×) — that leaves it in the document and misreports what you did.", @@ -142,6 +144,10 @@ export function buildSystemPrompt(options: { editsAllowed: boolean }): string { * case, and the string the surface tests measure the wire against. */ export const SYSTEM_PROMPT = buildSystemPrompt({ editsAllowed: true }); +// #1028: a zoom's camera moves sit OUTSIDE its span, where nothing on the ruler shows them. +const ZOOM_TRANSITIONS_NOTE = `A zoom HOLDS over its span: it animates in during the window just before startSec and out during the window just after endSec. Each move lasts 0.6 s + 0.55 s × ln(renderedScale) of SCREEN time (${ZOOM_TRANSITION_LEGEND}), so a speed region scales the timeline it covers by its speed: longer above 1×, shorter below. getCurrentDocument gives each zoom's zoomInFromSec and zoomOutUntilSec, in virtual seconds like the zoom. A trim inside either window cuts that move and the export jumps at the cut: leave the windows untrimmed.`; +const CUT_TRANSITIONS_RESULT = `When the result cuts a zoom's move in or out, it carries \`cutTransitions\`: which zoom, which side, and cutSec, the timeline seconds of the move lost. It is a warning, nothing is refused: move the cut or the zoom unless a hard cut is what you want.`; + export const TOOL_DESCRIPTIONS: Record = { getCurrentDocument: "Read a compact snapshot of the current project: assets (with durations), timeline clips and trim ranges (source-time), and the zoom / speed / annotation effects (virtual, edited-timeline time). Call this before editing if the snapshot in the system prompt may be stale. The AxcutDocument is the single source of truth — your edits should preserve the user's placed clips and any timeline state they have already set up.", @@ -153,21 +159,18 @@ export const TOOL_DESCRIPTIONS: Record = { 'Read the transcript one WORD at a time for an asset: each word\'s id, text, start/end seconds, and — only when it is not plain transcription — `source` ("user" for a word the user corrected, "synth" for one they typed in) and `originalText` (what the transcriber had heard before the correction). This is the ONLY read that gives you the ids setWordText takes; getTranscript answers in segments, whose ids belong to a different namespace and are not accepted there. A whole transcript is large, so pass startSec/endSec to read just the passage you mean to fix. Omit assetId for the primary asset.', setWordText: "Correct ONE word's text, by the id getTranscriptWords returns. This changes the TRANSCRIPT and nothing else: the captions follow it, the film is untouched and no audio is cut. Use it when the transcriber misheard something — a name, a technical term — and the user asks for it to read correctly. Passing an empty string BLANKS the word: it keeps its place in the media but leaves the captions, which is how a junk token like \"(inaudible)\" is removed without cutting the speech around it. Writing the transcriber's own text back clears the correction. This is NOT how you make a spoken word go away — that removes only the label and leaves the film saying it; use addTrim, which cuts the audio with it.", - addTrim: - "Add ONE trim range: a cut of a span inside a clip (this source-time span will not be played or exported) that does NOT split the clip. Times are in seconds of the asset's source time. This is the preferred (and for 'remove silences' requests, the only) way to handle silences; it preserves the user's placed clips and only adds a cut. When you have several cuts to make, use addTrims and send them together — this one is for a single cut or a later correction. A cut belongs to ONE clip: `clipId` is inferred when a single clip covers the range, but when several clips draw on the same asset over it the call FAILS and lists them — pass the `clipId` you mean (ids come from getCurrentDocument).", - addTrims: - "Add MANY trim ranges in one call: `ranges` is a list, each entry taking exactly the fields addTrim takes. Use this whenever you have more than one cut to make — 'remove the silences' on a half-hour recording is hundreds of cuts, and sending them one at a time costs one round trip each. Each range stands or falls ALONE: one that cannot be placed is refused by itself and listed in `refused` with its index and the reason, while every other range is still applied. Nothing is rolled back, so a single bad bound never costs you the rest. The result leads with requested / appliedCount / refusedCount so you can see a partial outcome without re-reading the document — report what was refused rather than claiming the whole list landed.", - setTrim: - "Move or resize an existing trim range by id. Times are source-time seconds. The cut follows to whichever clip the new range lands in, when that clip is unambiguous.", + addTrim: `Add ONE trim range: a cut of a span inside a clip (this source-time span will not be played or exported) that does NOT split the clip. Times are in seconds of the asset's source time. This is the preferred (and for 'remove silences' requests, the only) way to handle silences; it preserves the user's placed clips and only adds a cut. When you have several cuts to make, use addTrims and send them together — this one is for a single cut or a later correction. A cut belongs to ONE clip: \`clipId\` is inferred when a single clip covers the range, but when several clips draw on the same asset over it the call FAILS and lists them — pass the \`clipId\` you mean (ids come from getCurrentDocument). ${ZOOM_TRANSITIONS_NOTE} ${CUT_TRANSITIONS_RESULT}`, + addTrims: `Add MANY trim ranges in one call: \`ranges\` is a list, each entry taking exactly the fields addTrim takes. Use this whenever you have more than one cut to make — 'remove the silences' on a half-hour recording is hundreds of cuts, and sending them one at a time costs one round trip each. Each range stands or falls ALONE: one that cannot be placed is refused by itself and listed in \`refused\` with its index and the reason, while every other range is still applied. Nothing is rolled back, so a single bad bound never costs you the rest. The result leads with requested / appliedCount / refusedCount so you can see a partial outcome without re-reading the document — report what was refused rather than claiming the whole list landed. ${ZOOM_TRANSITIONS_NOTE} Each applied entry carries the same \`cutTransitions\` warning addTrim reports.`, + setTrim: `Move or resize an existing trim range by id. Times are source-time seconds. The cut follows to whichever clip the new range lands in, when that clip is unambiguous. ${ZOOM_TRANSITIONS_NOTE} ${CUT_TRANSITIONS_RESULT}`, setClipRange: "Set a clip's in/out points (source-time seconds) to shorten its head or tail — distinct from a trim (which cuts a span inside the clip). All clips are re-laid back-to-back afterwards, so downstream clips shift automatically. Use this ONLY when the user explicitly asks to shorten or extend a user-placed clip. Do NOT use this for 'remove silences' or 'cut pauses' — for those, use addTrim.", moveClip: "Reorder a placed clip: move `clipId` so it plays just before `beforeClipId` (pass null, or omit it, to move it last). Ids come from getCurrentDocument, where each clip carries its `index` and its label in `reason`. This preserves every clip id, every source range, every trim, and the zooms / speed regions / annotations anchored to each clip. This is the tool for 'swap these clips', 'put X first' and 'change the clip order' — replaceTimeline cannot reorder anything.", replaceTimeline: "Replace the whole timeline with the given kept intervals of the primary asset's source time. Everything outside the intervals becomes a trim. The intervals are SORTED, so this can never reorder clips — use moveClip for that. DO NOT use this for 'cut silences' or 'remove pauses' — the user has likely placed clips on the timeline that you'd be discarding. Use this ONLY when the user explicitly asks you to rebuild the timeline from scratch (e.g. 'start over with the kept intervals from the transcript'). It is refused when it would merge away, shorten or drop an existing clip; the refusal names them and the tool to use instead.", - addZoom: `Add a zoom-in over a span of the edited timeline (virtual seconds). depth is an ORDINAL 1–6, not a factor: it selects a magnification from a fixed table (${ZOOM_DEPTH_LEGEND}), so the default depth 3 renders at 1.80×. The result reports renderedScale — quote that, never the depth, when telling the user how strong the zoom is. focus is the zoom centre in 0–1 fractions of the frame (default centre). When the recording's pointer telemetry can be read for the footage under the span, the result also carries \`cursorAnchor\`: \`focus\` echoes the value this call used (including the default, if you left it out), \`cursor\` is where the pointer ACTUALLY was over the span the zoom landed on — the median of the recorded samples, \`spread\` being how far the farthest one strays from it — and \`offset\` is the distance between the two, in frame fractions. It is a measurement, not a correction: nothing is moved and no call is refused over it, and a zoom framing a slide, a face, or a region the pointer never enters is a legitimate choice. \`available:false\` names what it found instead (\`no-samples\`, \`trimmed-out\`). Its ABSENCE means no telemetry was read for that footage — never that the recording has none; assets[].hasCursorTelemetry and getCursorTrack are what answer that. Use for 'zoom in on …' and the smart-zoom pass.`, - addZooms: `Add MANY zooms in one call: \`regions\` is a list, each entry taking exactly the fields addZoom takes (same depth table, ${ZOOM_DEPTH_LEGEND}). Use this for the smart-zoom pass, where you have decided every zoom before emitting the first one — sending them one at a time costs one round trip each. Each region stands or falls ALONE: one that covers no clip is refused by itself and listed in \`refused\` with its index and the reason, while the others are still applied. The result leads with requested / appliedCount / refusedCount, and each applied entry carries its renderedScale — quote that, never the depth — plus the same \`cursorAnchor\` addZoom reports, whenever the footage under that region has readable pointer telemetry.`, - setZoom: `Move, resize, or restyle an existing zoom by id (virtual-timeline seconds). Only the fields you pass are changed. depth selects from the same table (${ZOOM_DEPTH_LEGEND}); if the zoom carries a customScale (getCurrentDocument shows it as depthIsOverridden), that custom value is what renders, and passing depth clears it so the depth takes effect — the result says so. The result reports the resulting renderedScale, and — when the footage under the span has readable pointer telemetry — the same \`cursorAnchor\` addZoom reports, measured against the zoom's EFFECTIVE focus, so a call that moved only the span still learns what its unchanged focus is now looking at.`, + addZoom: `Add a zoom-in over a span of the edited timeline (virtual seconds). depth is an ORDINAL 1–6, not a factor: it selects a magnification from a fixed table (${ZOOM_DEPTH_LEGEND}), so the default depth 3 renders at 1.80×. The result reports renderedScale — quote that, never the depth, when telling the user how strong the zoom is. focus is the zoom centre in 0–1 fractions of the frame (default centre). When the recording's pointer telemetry can be read for the footage under the span, the result also carries \`cursorAnchor\`: \`focus\` echoes the value this call used (including the default, if you left it out), \`cursor\` is where the pointer ACTUALLY was over the span the zoom landed on — the median of the recorded samples, \`spread\` being how far the farthest one strays from it — and \`offset\` is the distance between the two, in frame fractions. It is a measurement, not a correction: nothing is moved and no call is refused over it, and a zoom framing a slide, a face, or a region the pointer never enters is a legitimate choice. \`available:false\` names what it found instead (\`no-samples\`, \`trimmed-out\`). Its ABSENCE means no telemetry was read for that footage — never that the recording has none; assets[].hasCursorTelemetry and getCursorTrack are what answer that. Use for 'zoom in on …' and the smart-zoom pass. ${ZOOM_TRANSITIONS_NOTE} ${CUT_TRANSITIONS_RESULT}`, + addZooms: `Add MANY zooms in one call: \`regions\` is a list, each entry taking exactly the fields addZoom takes (same depth table, ${ZOOM_DEPTH_LEGEND}). Use this for the smart-zoom pass, where you have decided every zoom before emitting the first one — sending them one at a time costs one round trip each. Each region stands or falls ALONE: one that covers no clip is refused by itself and listed in \`refused\` with its index and the reason, while the others are still applied. The result leads with requested / appliedCount / refusedCount, and each applied entry carries its renderedScale — quote that, never the depth — plus the same \`cursorAnchor\` addZoom reports, whenever the footage under that region has readable pointer telemetry, and the same \`cutTransitions\` warning. ${ZOOM_TRANSITIONS_NOTE}`, + setZoom: `Move, resize, or restyle an existing zoom by id (virtual-timeline seconds). Only the fields you pass are changed. depth selects from the same table (${ZOOM_DEPTH_LEGEND}); if the zoom carries a customScale (getCurrentDocument shows it as depthIsOverridden), that custom value is what renders, and passing depth clears it so the depth takes effect — the result says so. The result reports the resulting renderedScale, and — when the footage under the span has readable pointer telemetry — the same \`cursorAnchor\` addZoom reports, measured against the zoom's EFFECTIVE focus, so a call that moved only the span still learns what its unchanged focus is now looking at. ${ZOOM_TRANSITIONS_NOTE} ${CUT_TRANSITIONS_RESULT}`, addSpeed: "Add a speed-change region over a span of the edited timeline (virtual seconds). speed > 1 fast-forwards, < 1 slows down, from 0.25× to 16× (default 1.5×). Use to speed through slow stretches without cutting them.", setSpeed: diff --git a/src/lib/ai-edition/timeline/speed.ts b/src/lib/ai-edition/timeline/speed.ts index e90eb922f..446ad2df5 100644 --- a/src/lib/ai-edition/timeline/speed.ts +++ b/src/lib/ai-edition/timeline/speed.ts @@ -36,3 +36,24 @@ export function screenTimeMs(regions: readonly SpeedRegion[], timeMs: number): n } return screen; } + +/** + * Inverse of `screenTimeMs`: the timeline time shown at screen time `screenMs`. Mirror of + * `ScreenClock::source_at` in `crates/compositor/src/regions.rs`. + */ +export function timelineTimeMs(regions: readonly SpeedRegion[], screenMs: number): number { + let lead = 0; + let coveredTo = Number.NEGATIVE_INFINITY; + for (const region of [...regions].sort((a, b) => a.startMs - b.startMs)) { + const start = Math.max(region.startMs, coveredTo); + if (region.endMs <= start) continue; + const screenStart = start - lead; + if (screenMs <= screenStart) break; + coveredTo = region.endMs; + const speed = Number.isFinite(region.speed) && region.speed > 0 ? region.speed : 1; + const screenEnd = screenStart + (region.endMs - start) / speed; + if (screenMs < screenEnd) return start + (screenMs - screenStart) * speed; + lead += (region.endMs - start) * (1 - 1 / speed); + } + return screenMs + lead; +} diff --git a/src/lib/ai-edition/timeline/zoom-transitions.test.ts b/src/lib/ai-edition/timeline/zoom-transitions.test.ts new file mode 100644 index 000000000..2d7bab314 --- /dev/null +++ b/src/lib/ai-edition/timeline/zoom-transitions.test.ts @@ -0,0 +1,67 @@ +import { describe, expect, it } from "vitest"; +import { zoomTransitionMs } from "../../zoomMath/constants"; +import { type SpeedRegion, screenTimeMs, timelineTimeMs } from "./speed"; +import { transitionCutsMs, zoomTransitions } from "./zoom-transitions"; + +const W = zoomTransitionMs(1.8); // ≈ 923 ms +const zoom = { startMs: 10_000, endMs: 12_000, scale: 1.8 }; + +describe("timelineTimeMs", () => { + it("inverts screenTimeMs, overlapping regions included", () => { + const regions: SpeedRegion[] = [ + { id: "a", startMs: 2000, endMs: 6000, speed: 2 }, + { id: "b", startMs: 4000, endMs: 9000, speed: 4 }, + { id: "c", startMs: 12_000, endMs: 13_000, speed: 0.5 }, + ]; + for (const t of [0, 1999, 2000, 3000, 6000, 7500, 9000, 12_500, 20_000]) { + expect(timelineTimeMs(regions, screenTimeMs(regions, t))).toBeCloseTo(t, 6); + } + }); +}); + +describe("zoomTransitions", () => { + it("puts the moves just outside the pill, deeper zooms longer", () => { + const t = zoomTransitions(zoom, []); + expect(t.inFromMs).toBeCloseTo(10_000 - W, 6); + expect(t.outUntilMs).toBeCloseTo(12_000 + W, 6); + expect(zoomTransitions({ ...zoom, scale: 5 }, []).durationMs).toBeGreaterThan(t.durationMs); + }); + + it("stretches a move across the timeline inside a speed region (issue #1028 variant)", () => { + const t = zoomTransitions(zoom, [{ id: "s", startMs: 12_000, endMs: 20_000, speed: 3 }]); + expect(t.outUntilMs).toBeCloseTo(12_000 + 3 * W, 6); + expect(t.inFromMs).toBeCloseTo(10_000 - W, 6); + }); + + it("never starts a zoom-in before the timeline", () => { + expect(zoomTransitions({ startMs: 300, endMs: 2000, scale: 1.8 }, []).inFromMs).toBe(0); + }); +}); + +describe("transitionCutsMs", () => { + const t = zoomTransitions(zoom, []); + + it("reports the whole zoom-out lost to a trim starting at the pill's end (issue #1028)", () => { + const cut = transitionCutsMs(zoom, t, [{ startMs: 12_000, endMs: 14_000 }]); + expect(cut.inMs).toBe(0); + expect(cut.outMs).toBeCloseTo(W, 6); + }); + + it("measures a partial cut from the first trimmed instant", () => { + expect(transitionCutsMs(zoom, t, [{ startMs: 12_500, endMs: 13_000 }]).outMs).toBeCloseTo( + 12_000 + W - 12_500, + 6, + ); + expect(transitionCutsMs(zoom, t, [{ startMs: 9000, endMs: 9500 }]).inMs).toBeCloseTo( + 9500 - (10_000 - W), + 6, + ); + }); + + it("ignores trims clear of the windows, inside the pill, or swallowing the whole zoom", () => { + const none = { inMs: 0, outMs: 0 }; + expect(transitionCutsMs(zoom, t, [{ startMs: 13_000, endMs: 14_000 }])).toEqual(none); + expect(transitionCutsMs(zoom, t, [{ startMs: 10_500, endMs: 11_000 }])).toEqual(none); + expect(transitionCutsMs(zoom, t, [{ startMs: 9000, endMs: 14_000 }])).toEqual(none); + }); +}); diff --git a/src/lib/ai-edition/timeline/zoom-transitions.ts b/src/lib/ai-edition/timeline/zoom-transitions.ts new file mode 100644 index 000000000..6d0440ea4 --- /dev/null +++ b/src/lib/ai-edition/timeline/zoom-transitions.ts @@ -0,0 +1,77 @@ +// Where a zoom's camera moves sit on the timeline (#1028). A zoom pill shows the HOLD: the +// zoom-in runs entirely before its start and the zoom-out entirely after its end, each +// lasting `zoomTransitionMs(scale)` of SCREEN time (`computeRegionStrength`, mirror of +// `zoom_region_strength` in crates/compositor/src/regions.rs). These helpers turn those +// windows into timeline ms, through the same speed-aware clock, and measure what a trim cuts +// off them: the render drops the trimmed frames, so the camera jumps at the cut instead of +// easing. +// +// Chained zooms (closer than 1.5 s, panned between) are measured as if apart. That is the +// right answer for trims: a trim between two zooms splits them into separate segments, which +// breaks the chain, and each then needs its own windows untrimmed. + +import { zoomTransitionMs } from "../../zoomMath/constants"; +import { type SpeedRegion, screenTimeMs, timelineTimeMs } from "./speed"; +import { ZOOM_DEPTH_SCALES } from "./zoom-scale"; + +export interface Span { + startMs: number; + endMs: number; +} + +export interface ZoomTransitions { + /** Timeline ms where the zoom-in starts. It lands on the zoom's start. */ + inFromMs: number; + /** Timeline ms where the zoom-out, which leaves from the zoom's end, is over. */ + outUntilMs: number; + /** Screen ms each move lasts, whatever the speed regions do to the footage. */ + durationMs: number; +} + +/** "0.72 s at 1.25×, …": every depth's move duration, for the agent's tool descriptions. */ +export const ZOOM_TRANSITION_LEGEND = Object.values(ZOOM_DEPTH_SCALES) + .sort((a, b) => a - b) + .map((scale) => `${(zoomTransitionMs(scale) / 1000).toFixed(2)} s at ${scale.toFixed(2)}×`) + .join(", "); + +export function zoomTransitions( + zoom: Span & { scale: number }, + speedRegions: readonly SpeedRegion[], +): ZoomTransitions { + const durationMs = zoomTransitionMs(zoom.scale); + return { + inFromMs: Math.max( + 0, + timelineTimeMs(speedRegions, screenTimeMs(speedRegions, zoom.startMs) - durationMs), + ), + outUntilMs: timelineTimeMs(speedRegions, screenTimeMs(speedRegions, zoom.endMs) + durationMs), + durationMs, + }; +} + +/** + * Timeline ms of each move the trims cut off (0 when untouched). The zoom-in only plays after + * the last trimmed instant before the zoom, the zoom-out only until the first one after it. + * A zoom entirely under a trim never plays and the compositor renders it without envelopes + * (`under_trim`), so it has nothing to cut. + */ +export function transitionCutsMs( + zoom: Span, + transitions: ZoomTransitions, + trims: readonly Span[], +): { inMs: number; outMs: number } { + if (trims.some((t) => t.startMs <= zoom.startMs && t.endMs >= zoom.endMs)) { + return { inMs: 0, outMs: 0 }; + } + let resumesAt = transitions.inFromMs; + let jumpsAt = transitions.outUntilMs; + for (const t of trims) { + if (t.startMs < zoom.startMs && t.endMs > transitions.inFromMs) { + resumesAt = Math.max(resumesAt, Math.min(t.endMs, zoom.startMs)); + } + if (t.endMs > zoom.endMs && t.startMs < transitions.outUntilMs) { + jumpsAt = Math.min(jumpsAt, Math.max(t.startMs, zoom.endMs)); + } + } + return { inMs: resumesAt - transitions.inFromMs, outMs: transitions.outUntilMs - jumpsAt }; +} diff --git a/technical-documentation/architecture/ai-agent.md b/technical-documentation/architecture/ai-agent.md index 3536f67cc..e43e446b1 100644 --- a/technical-documentation/architecture/ai-agent.md +++ b/technical-documentation/architecture/ai-agent.md @@ -45,10 +45,10 @@ The model never free-writes the project document. It can only call the fixed set | Tool | What it does | What it mutates | |---|---|---| -| `getCurrentDocument` | Reads a compact project, asset, clip, trim, and modifier snapshot with explicit time bases. Each asset reports `hasCameraTrack` / `cameraVisible` / `hasCursorTelemetry` beside `hasTranscript` (`hasCursorTelemetry` is three-valued: `true`, `false` when the asset was checked and has none, `null` when it was not checked — never `false` for something we failed to look at), the document reports `hasAnyCamera` and `autoFocusAll`, and each zoom reports the `renderedScale` the viewer will see plus `customScale` / `depthIsOverridden` when a custom scale makes its `depth` inert. | Nothing. | +| `getCurrentDocument` | Reads a compact project, asset, clip, trim, and modifier snapshot with explicit time bases. Each asset reports `hasCameraTrack` / `cameraVisible` / `hasCursorTelemetry` beside `hasTranscript` (`hasCursorTelemetry` is three-valued: `true`, `false` when the asset was checked and has none, `null` when it was not checked — never `false` for something we failed to look at), the document reports `hasAnyCamera` and `autoFocusAll`, and each zoom reports the `renderedScale` the viewer will see plus `customScale` / `depthIsOverridden` when a custom scale makes its `depth` inert. Each zoom also reports where its camera moves sit on the ruler, outside the pill: `zoomInFromSec`, `zoomOutUntilSec` (speed-aware, from `zoomTransitions` in `src/lib/ai-edition/timeline/zoom-transitions.ts`), `transitionSec`, and `cutByTrim` (`in` / `out` / `both`) when a trim cuts a move, so the export jumps there (#1028). | Nothing. | | `getTranscript` | Reads the transcript segments for an asset, or the primary asset, in full. On the production path a segment is one word, so a half-hour recording is a few thousand of them — there is no cap, and no per-model context budget to derive one from. | Nothing. | | `getCursorTrack` | Reads the recorded pointer telemetry for an asset as REAL SAMPLES, reduced per axis against time (Douglas–Peucker) rather than summarised: every point returned is one that was recorded, carrying its source `atSec`, `cx`/`cy`, the `virtualSec` that `addZoom` takes (left off the points, with `virtualEqualsSource` on the envelope, when the two coincide everywhere), a `shape` index per distinct pointer bitmap, and `kind` / `trimmed` where they apply. Answers `available:false` with `reason:"no-sidecar"` (checked, this asset has none) or `reason:"unavailable"` (could not be read from here), and the two are never conflated. | Nothing. | -| `addTrim` | Adds one source-time cut inside a clip. | `timeline.trimRanges`. | +| `addTrim` | Adds one source-time cut inside a clip. Like the zoom writes and `setTrim`, it reports `cutTransitions` (zoom, side, seconds lost) when the edit newly cuts a zoom's move in or out; a warning, never a refusal. | `timeline.trimRanges`. | | `addTrims` | Adds many cuts in one call, replaying `addTrim` per entry so the rules cannot drift apart. Each range stands alone: one that cannot be placed is refused by itself and named with its index and reason while the rest are applied, and the result leads with `requested` / `appliedCount` / `refusedCount`. Only a batch where nothing landed is an error. | `timeline.trimRanges`. | | `setTrim` | Moves or resizes an existing source-time trim. | The matching `timeline.trimRanges` entry. | | `setClipRange` | Changes a clip's source in/out points and relays clips back-to-back. | The clip range and any anchored regions clamped or removed by the shared timeline mutator. |