diff --git a/src/components/ai-edition/NewEditorShell.tsx b/src/components/ai-edition/NewEditorShell.tsx index 08b52f21e..f522c8cb9 100644 --- a/src/components/ai-edition/NewEditorShell.tsx +++ b/src/components/ai-edition/NewEditorShell.tsx @@ -12,6 +12,8 @@ import { createId } from "@/lib/ai-edition/document/ids"; import { documentAfterProbedDuration, PLACEHOLDER_DURATION_SEC, + projectRawTimelineSecToPlayback, + readSpeedRegions, } from "@/lib/ai-edition/document/timeline"; import { type InsertSide, @@ -37,6 +39,7 @@ import { useTimeline } from "@/lib/ai-edition/store/useTimeline"; import { isGeneratedAssetId } from "@/lib/ai-edition/timeline/clip-parts"; import { mergeCloseCuts } from "@/lib/ai-edition/timeline/cut-breath"; import { newRegionDurationSec } from "@/lib/ai-edition/timeline/newRegionDuration"; +import { removedRawSpans } from "@/lib/ai-edition/timeline/programme-time"; import { dropTrimPillsByIds, trimAppliesToClip, @@ -980,16 +983,30 @@ export function NewEditorShell() { // placed every voiceover one full take-length to the right of where it was // spoken. Captured on the way in, used on the way out. const voiceoverStartSecRef = useRef(0); + const voiceoverEndSecRef = useRef(0); + const voiceoverMaxRawEndSecRef = useRef(0); + const voiceoverRecordingPlaybackMapRef = + useRef(undefined); const openVoiceoverFlow = useCallback(() => { const doc = useProjectStore.getState().document; if (!doc) return; const total = doc.timeline.clips.reduce((max, c) => Math.max(max, c.timelineEndSec), 0); const playhead = useProjectStore.getState().currentTimeSec; + const speedRegions = readSpeedRegions(doc); + const outputAt = (rawSec: number) => + projectRawTimelineSecToPlayback( + doc.timeline.clips, + doc.timeline.trimRanges, + rawSec, + speedRegions, + ); voiceoverStartSecRef.current = playhead; + voiceoverEndSecRef.current = playhead; + voiceoverMaxRawEndSecRef.current = total; // Recording stops itself at the end of the timeline: a take can never // outlive the video it was recorded over. - setVoiceoverFlow({ maxDurationSec: Math.max(0.5, total - playhead) }); + setVoiceoverFlow({ maxDurationSec: Math.max(0.5, outputAt(total) - outputAt(playhead)) }); }, []); // Silences the timeline's own audio tracks for the duration of a take — see @@ -999,12 +1016,44 @@ export function NewEditorShell() { const handleVoiceoverRecordingStart = useCallback(() => { // Re-capture: the user may have scrubbed between opening the dialog and // hitting Record, and playback starts from wherever the playhead is now. - voiceoverStartSecRef.current = useProjectStore.getState().currentTimeSec; + const doc = useProjectStore.getState().document; + const playhead = useProjectStore.getState().currentTimeSec; + voiceoverStartSecRef.current = playhead; + voiceoverEndSecRef.current = playhead; + voiceoverRecordingPlaybackMapRef.current = undefined; + if (doc) { + const total = doc.timeline.clips.reduce((max, c) => Math.max(max, c.timelineEndSec), 0); + const speedRegions = readSpeedRegions(doc); + const outputAt = (rawSec: number) => + projectRawTimelineSecToPlayback( + doc.timeline.clips, + doc.timeline.trimRanges, + rawSec, + speedRegions, + ); + voiceoverMaxRawEndSecRef.current = total; + voiceoverRecordingPlaybackMapRef.current = { + removedRawSpans: removedRawSpans(doc.timeline.clips, doc.timeline.trimRanges) + .filter((span) => span.endSec > playhead && span.startSec < total) + .map(({ startSec, endSec }) => ({ startSec, endSec })), + speedRegions: speedRegions.filter( + (region) => region.endMs / 1000 > playhead && region.startMs / 1000 < total, + ), + }; + setVoiceoverFlow({ + maxDurationSec: Math.max(0.5, outputAt(total) - outputAt(playhead)), + }); + } setVoiceoverRecording(true); if (videoElement?.paused) void videoElement.play().catch(() => undefined); }, [videoElement]); const handleVoiceoverRecordingStop = useCallback(() => { + const playhead = useProjectStore.getState().currentTimeSec; + voiceoverEndSecRef.current = Math.max( + voiceoverStartSecRef.current, + Math.min(voiceoverMaxRawEndSecRef.current, playhead), + ); setVoiceoverRecording(false); videoElement?.pause(); }, [videoElement]); @@ -1012,10 +1061,13 @@ export function NewEditorShell() { const handleVoiceoverReady = useCallback( async (assetId: string, durationSec: number) => { setVoiceoverFlow(null); + const startSec = voiceoverStartSecRef.current; + const rawSpanSec = Math.max(0, voiceoverEndSecRef.current - startSec); await tl.addAudioTrack(assetId, voiceoverStartSecRef.current, { kind: "voiceover", durationSec, - spanSec: durationSec, + spanSec: rawSpanSec > 0 ? rawSpanSec : durationSec, + recordingPlaybackMap: voiceoverRecordingPlaybackMapRef.current, }); }, [tl], @@ -1409,6 +1461,7 @@ export function NewEditorShell() { isMac, togglePlay, handleSeek, + openVoiceoverFlow, ]); const showTimeline = mode !== "rec"; diff --git a/src/lib/ai-edition/document/audioTracks.test.ts b/src/lib/ai-edition/document/audioTracks.test.ts index 25702630a..58018c471 100644 --- a/src/lib/ai-edition/document/audioTracks.test.ts +++ b/src/lib/ai-edition/document/audioTracks.test.ts @@ -5,8 +5,10 @@ import { audioGhostExtent, collapseTracksToPills, computeAudioRowCount, + cutTakeHead, packAudioTrackRows, patchAudioTrack, + reanchorAudioTracks, removeAudioTrack, resolveFadeSecs, slipAudioOffsetMs, @@ -76,6 +78,57 @@ describe("anchorAudioTrackFragments", () => { expect(frags[1].offsetMs).toBe(7000); }); + it("uses the recorded playback clock when a take crosses a clip boundary", () => { + const frags = anchorAudioTrackFragments( + track({ + kind: "voiceover", + startMs: 0, + endMs: 13_000, + durationSec: 8, + recordingPlaybackMap: { + removedRawSpans: [{ startSec: 5, endSec: 10 }], + speedRegions: [{ startMs: 0, endMs: 5000, speed: 2 }], + }, + }), + twoClips, + makeId, + ); + + expect(frags.map((fragment) => fragment.offsetMs)).toEqual([0, 2500]); + }); + + it("keeps recorded fragments together when it re-anchors a take", () => { + const trackBeforeReanchor = track({ + kind: "voiceover", + startMs: 0, + endMs: 13_000, + durationSec: 8, + recordingPlaybackMap: { + removedRawSpans: [{ startSec: 5, endSec: 10 }], + speedRegions: [{ startMs: 0, endMs: 5000, speed: 2 }], + }, + }); + const fragments = anchorAudioTrackFragments(trackBeforeReanchor, twoClips, makeId); + + const reanchored = reanchorAudioTracks(fragments, twoClips, makeId); + expect(collapseTracksToPills(reanchored)).toHaveLength(1); + expect(reanchored.map((fragment) => fragment.offsetMs)).toEqual([0, 2500]); + }); + + it("cuts a recorded take head by its captured playback time", () => { + const take = track({ + kind: "voiceover", + startMs: 0, + offsetMs: 0, + recordingPlaybackMap: { + removedRawSpans: [{ startSec: 5, endSec: 10 }], + speedRegions: [{ startMs: 0, endMs: 5000, speed: 2 }], + }, + }); + + expect(cutTakeHead(take, 7).offsetMs).toBe(2500); + }); + it("keeps the fades on the outer edges only", () => { const frags = anchorAudioTrackFragments( track({ startMs: 5000, endMs: 15_000, fadeInMs: 500, fadeOutMs: 800 }), diff --git a/src/lib/ai-edition/document/audioTracks.ts b/src/lib/ai-edition/document/audioTracks.ts index 83dd74a1e..334c489d2 100644 --- a/src/lib/ai-edition/document/audioTracks.ts +++ b/src/lib/ai-edition/document/audioTracks.ts @@ -19,6 +19,7 @@ import type { AxcutAudioTrack, AxcutClip, AxcutDocument } from "../schema"; import { isGeneratedAssetId } from "../timeline/clip-parts"; +import { outputDurationOfRawSpan } from "../timeline/playback-time"; import { anchorRegionsWithDerivedMs, clampSpanAgainstNeighbours, @@ -63,7 +64,11 @@ export function anchorAudioTrackFragments( // Looping restarts the window on its own, so an advanced offset would // double-count the fold; the mixer and the preview both wrap within // `durationSec - offsetMs`, which every fragment shares. - offsetMs: track.loop ? track.offsetMs : track.offsetMs + elapsedMs, + offsetMs: track.loop + ? track.offsetMs + : track.recordingPlaybackMap + ? Math.round(recordedSourceAtRaw(track, fragment.startMs / 1000) * 1000) + : track.offsetMs + elapsedMs, fadeInMs: index === 0 ? track.fadeInMs : 0, fadeOutMs: index === last ? track.fadeOutMs : 0, }; @@ -134,10 +139,11 @@ function hangsTogether( right: AxcutAudioTrack, assetOf: Map, ): boolean { - const runsOn = - Math.abs(left.offsetMs - right.offsetMs) <= 1 || - (!right.loop && Math.abs(left.offsetMs + (left.endMs - left.startMs) - right.offsetMs) <= 1); - return runsOn && (Math.abs(left.endMs - right.startMs) <= 1 || mediaRunsOn(left, right, assetOf)); + return ( + recordingPlaybackMapsMatch(left.recordingPlaybackMap, right.recordingPlaybackMap) && + (Math.abs(left.offsetMs - right.offsetMs) <= 1 || sourceClockContinues(left, right)) && + (Math.abs(left.endMs - right.startMs) <= 1 || mediaRunsOn(left, right, assetOf)) + ); } /** One stretch of one media, cut in two: what an inserted word leaves either side of it. */ @@ -182,7 +188,9 @@ function joinContiguousTakes(pills: AxcutAudioTrack[]): AxcutAudioTrack[] { * advances, so its file says nothing about the join: two loops at one offset are one take when * neither fades across the seam, which is the seam splitting one leaves (#1011). */ function takesJoin(left: AxcutAudioTrack, right: AxcutAudioTrack): boolean { - const spanMs = left.endMs - left.startMs; + const sourceContinues = left.loop + ? sourceClockContinues(left, right) && left.fadeOutMs === 0 && right.fadeInMs === 0 + : sourceClockContinues(left, right); return ( left.assetId === right.assetId && left.kind === right.kind && @@ -190,20 +198,100 @@ function takesJoin(left: AxcutAudioTrack, right: AxcutAudioTrack): boolean { left.gainDb === right.gainDb && left.muted === right.muted && Math.abs(left.endMs - right.startMs) <= 1 && - (left.loop - ? left.offsetMs === right.offsetMs && left.fadeOutMs === 0 && right.fadeInMs === 0 - : Math.abs(left.offsetMs + spanMs - right.offsetMs) <= 1) + sourceContinues + ); +} + +function sourceClockContinues(left: AxcutAudioTrack, right: AxcutAudioTrack): boolean { + if (!recordingPlaybackMapsMatch(left.recordingPlaybackMap, right.recordingPlaybackMap)) { + return false; + } + if (left.loop) return left.offsetMs === right.offsetMs; + if (left.recordingPlaybackMap) { + return Math.abs(recordedSourceEndMs(left) - right.offsetMs) <= 1; + } + return Math.abs(left.offsetMs + (left.endMs - left.startMs) - right.offsetMs) <= 1; +} + +function recordingPlaybackMapsMatch( + left: AxcutAudioTrack["recordingPlaybackMap"], + right: AxcutAudioTrack["recordingPlaybackMap"], +): boolean { + if (!left || !right) return left === right; + return ( + left.removedRawSpans.length === right.removedRawSpans.length && + left.removedRawSpans.every( + (span, index) => + span.startSec === right.removedRawSpans[index].startSec && + span.endSec === right.removedRawSpans[index].endSec, + ) && + left.speedRegions.length === right.speedRegions.length && + left.speedRegions.every( + (region, index) => + region.startMs === right.speedRegions[index].startMs && + region.endMs === right.speedRegions[index].endMs && + region.speed === right.speedRegions[index].speed, + ) ); } +function recordedSourceEndMs(track: AxcutAudioTrack): number { + return Math.round(recordedSourceAtRaw(track, track.endMs / 1000) * 1000); +} + +function recordedSourceAtRaw( + track: Pick, + rawSec: number, +): number { + const map = track.recordingPlaybackMap; + const start = track.startMs / 1000; + const end = Math.max(start, rawSec); + if (!map) return track.offsetMs / 1000 + end - start; + const cuts = [...map.removedRawSpans] + .filter((span) => span.endSec > start && span.startSec < end) + .sort((a, b) => a.startSec - b.startSec); + let raw = start; + let sourceSec = track.offsetMs / 1000; + for (const cut of cuts) { + const cutStart = Math.max(raw, cut.startSec); + if (cutStart > raw) { + sourceSec += outputDurationOfRawSpan(raw, cutStart, map.speedRegions); + } + raw = Math.max(raw, Math.min(end, cut.endSec)); + if (raw >= end) break; + } + if (raw < end) sourceSec += outputDurationOfRawSpan(raw, end, map.speedRegions); + return sourceSec; +} + /** * A clip edit that cuts a fragment's head off cuts the same stretch off its take's file, so what * is left goes on playing the audio over its own footage (#1011). A loop restarts on its own and * keeps its offset; a region with no file passes through. */ export function cutTakeHead(region: T, cutSec: number): T { - const take = region as T & { offsetMs?: unknown; loop?: unknown }; + const take = region as T & { + offsetMs?: unknown; + loop?: unknown; + startMs?: unknown; + recordingPlaybackMap?: AxcutAudioTrack["recordingPlaybackMap"]; + }; if (!(cutSec > 0) || typeof take.offsetMs !== "number" || take.loop === true) return region; + if (take.recordingPlaybackMap && typeof take.startMs === "number") { + return { + ...region, + offsetMs: Math.round( + recordedSourceAtRaw( + { + startMs: take.startMs, + offsetMs: take.offsetMs, + recordingPlaybackMap: take.recordingPlaybackMap, + }, + take.startMs / 1000 + cutSec, + ) * 1000, + ), + }; + } return { ...region, offsetMs: take.offsetMs + Math.round(cutSec * 1000) }; } diff --git a/src/lib/ai-edition/document/insertionTrack.test.ts b/src/lib/ai-edition/document/insertionTrack.test.ts index e7fcd8fc4..25533ab8b 100644 --- a/src/lib/ai-edition/document/insertionTrack.test.ts +++ b/src/lib/ai-edition/document/insertionTrack.test.ts @@ -164,6 +164,57 @@ describe("retextGeneratedTrack", () => { }); }); +describe("insertions in a live-recorded take", () => { + it("uses the take's playback source offset and restores it after removal", () => { + const sourceAligned = doc(); + sourceAligned.timeline.trimRanges = [ + { + id: "trim_1", + assetId: "a1", + clipId: "c1", + startSec: 5, + endSec: 10, + origin: "user", + reason: "", + }, + ]; + sourceAligned.audioTracks = sourceAligned.audioTracks.map((take) => ({ + ...take, + endMs: 15_000, + durationSec: 8, + recordingPlaybackMap: { + removedRawSpans: [{ startSec: 5, endSec: 10 }], + speedRegions: [], + }, + })); + + const edited = insertGeneratedTrack(sourceAligned, "vo", "v2", "before", "hi"); + const insertedTake = collapseTracksToPills(edited.audioTracks).find( + (take) => take.assetId === "vo" && take.startMs > 10_000, + ); + expect(insertedTake).toMatchObject({ + offsetMs: 6000, + recordingPlaybackMap: { removedRawSpans: [{ startSec: 5, endSec: 10 }] }, + }); + expect( + collapseTracksToPills(edited.audioTracks).find((take) => take.assetId === "ext:synth_1") + ?.recordingPlaybackMap, + ).toBeUndefined(); + + const restored = removeGeneratedTracks(edited, ["synth_1"]); + expect( + collapseTracksToPills(restored.audioTracks).filter((take) => take.assetId === "vo"), + ).toMatchObject([ + { + startMs: 2000, + endMs: 15_000, + offsetMs: 0, + recordingPlaybackMap: { removedRawSpans: [{ startSec: 5, endSec: 10 }] }, + }, + ]); + }); +}); + describe("the two lanes stay out of each other's way", () => { it("a word added to the FILM leaves the take's own span alone", () => { // Settled deliberately: the take has its own audio and keeps talking against a picture diff --git a/src/lib/ai-edition/document/insertionTrack.ts b/src/lib/ai-edition/document/insertionTrack.ts index c9a3af977..b0fb7a163 100644 --- a/src/lib/ai-edition/document/insertionTrack.ts +++ b/src/lib/ai-edition/document/insertionTrack.ts @@ -47,13 +47,20 @@ function takeFor( document: AxcutDocument, assetId: string, fileSec: number, -): { pill: AxcutAudioTrack; rawSec: number } | null { +): { pill: AxcutAudioTrack; rawSec: number; sourceSec: number } | null { const removed = removedRawSpans(document.timeline.clips, document.timeline.trimRanges); for (const pill of collapseTracksToPills(document.audioTracks)) { if (pill.assetId !== assetId || pill.loop) continue; for (const piece of takeProgramme(pill, removed)) { + if (piece.kind !== "play") continue; if (fileSec < piece.sourceStartSec - EPS || fileSec > piece.sourceEndSec + EPS) continue; - return { pill, rawSec: piece.rawStartSec + (fileSec - piece.sourceStartSec) }; + const sourceSpan = piece.sourceEndSec - piece.sourceStartSec; + const rawSpan = piece.rawEndSec - piece.rawStartSec; + return { + pill, + rawSec: piece.rawStartSec + ((fileSec - piece.sourceStartSec) / sourceSpan) * rawSpan, + sourceSec: fileSec, + }; } } return null; @@ -92,7 +99,7 @@ export function insertGeneratedTrack( if (!found) { throw new Error(`Cannot insert beside word "${anchorWordId}": no take speaks that moment`); } - const { pill, rawSec } = found; + const { pill, rawSec, sourceSec } = found; const wordId = nextGeneratedWordId(document); const durationSec = extensionDurationSec(trimmed); @@ -120,6 +127,7 @@ export function insertGeneratedTrack( durationSec, fadeInMs: 0, fadeOutMs: 0, + recordingPlaybackMap: undefined, }; const right: AxcutAudioTrack = { ...pill, @@ -128,7 +136,7 @@ export function insertGeneratedTrack( startMs: atMs + spanMs, // The take is as long as the audio it holds, so its tail moves by the whole insertion. endMs: pill.endMs + spanMs, - offsetMs: pill.offsetMs + (atMs - pill.startMs), + offsetMs: Math.round(sourceSec * 1000), fadeInMs: 0, }; diff --git a/src/lib/ai-edition/document/timeline.ts b/src/lib/ai-edition/document/timeline.ts index 4e1a75141..4c421fc1e 100644 --- a/src/lib/ai-edition/document/timeline.ts +++ b/src/lib/ai-edition/document/timeline.ts @@ -22,6 +22,10 @@ import type { export type PlaybackSegment = AxcutClip; import { type Interval, subtractInterval } from "../timeline/intervals"; +import { outputDurationOfRawSpan, type PlaybackSpeedRegion } from "../timeline/playback-time"; + +export type { PlaybackSpeedRegion } from "../timeline/playback-time"; + import { keptRawSpans } from "../timeline/programme-time"; import { anchoredToRawSpanSec, @@ -234,12 +238,6 @@ export function resolvePlaybackSegments( * delayed every track by the total trim duration ahead of it. The preview already lands them * correctly because its playhead jumps across trims; this makes the render agree. */ -export interface PlaybackSpeedRegion { - startMs: number; - endMs: number; - speed: number; -} - /** * The project's speed regions as every reader should see them. They live on the legacy editor * envelope, which no schema checks, so this is where their bound applies: a region without a @@ -257,43 +255,6 @@ export function readSpeedRegions ({ ...r, speed: clampToBound(r.speed, "playbackSpeed") })); } -/** - * Output seconds a raw interval `[fromSec, toSec)` occupies once the speed - * regions covering it are applied: a 2x stretch of raw time takes half as long - * to play, so it contributes half its raw length to the programme. - * - * Subdivides at every speed boundary the interval crosses and integrates - * `1 / speed` piecewise. Regions are matched on the raw ruler, the same - * coordinate their pills are drawn in. - */ -function outputDurationOfRawSpan( - fromSec: number, - toSec: number, - speedRegions: PlaybackSpeedRegion[], -): number { - if (toSec <= fromSec) return 0; - if (speedRegions.length === 0) return toSec - fromSec; - // Every boundary inside the span, so each piece has one constant speed. - const cuts = new Set([fromSec, toSec]); - for (const region of speedRegions) { - for (const edge of [region.startMs / 1000, region.endMs / 1000]) { - if (edge > fromSec && edge < toSec) cuts.add(edge); - } - } - const edges = [...cuts].sort((a, b) => a - b); - let out = 0; - for (let i = 0; i < edges.length - 1; i++) { - const start = edges[i]; - const end = edges[i + 1]; - const mid = (start + end) / 2; - const region = speedRegions.find( - (r) => mid >= r.startMs / 1000 && mid < r.endMs / 1000 && r.speed > 0, - ); - out += (end - start) / (region?.speed ?? 1); - } - return out; -} - export function projectRawTimelineSecToPlayback( clips: AxcutClip[], trimRanges: AxcutTrimRange[], diff --git a/src/lib/ai-edition/schema/index.test.ts b/src/lib/ai-edition/schema/index.test.ts index 38aabd870..706eb5034 100644 --- a/src/lib/ai-edition/schema/index.test.ts +++ b/src/lib/ai-edition/schema/index.test.ts @@ -1089,6 +1089,23 @@ describe("audio tracks (issue #350)", () => { expect(track.endMs).toBe(6000); }); + it("persists the playback clock for a newly recorded voiceover", () => { + const track = createAudioTrack({ + assetId: "asset_1", + durationSec: 8, + kind: "voiceover", + spanSec: 13, + recordingPlaybackMap: { + removedRawSpans: [{ startSec: 5, endSec: 10 }], + speedRegions: [], + }, + }); + + expect(audioTrackSchema.parse(track).recordingPlaybackMap?.removedRawSpans).toEqual([ + { startSec: 5, endSec: 10 }, + ]); + }); + it("createAudioTrack still gives a grabbable span to a zero-duration source", () => { const track = createAudioTrack({ assetId: "asset_1", durationSec: 0 }); expect(track.endMs).toBeGreaterThan(track.startMs); diff --git a/src/lib/ai-edition/schema/index.ts b/src/lib/ai-edition/schema/index.ts index 05edecab0..1ddab0cf1 100644 --- a/src/lib/ai-edition/schema/index.ts +++ b/src/lib/ai-edition/schema/index.ts @@ -582,6 +582,25 @@ export const audioTrackSchema = endGteStart( // Full source duration of the underlying file, cached here so the timeline // can lay out the pill before the asset is re-probed on load. durationSec: z.number().nonnegative().default(0), + // Playback cuts and rates in effect when a live take was recorded. Current cuts still + // mute the track, while this snapshot maps its source clock to the recorded playback. + recordingPlaybackMap: z + .object({ + removedRawSpans: z.array( + z.object({ + startSec: z.number().nonnegative(), + endSec: z.number().nonnegative(), + }), + ), + speedRegions: z.array( + z.object({ + startMs: z.number().nonnegative(), + endMs: z.number().nonnegative(), + speed: z.number().positive(), + }), + ), + }) + .optional(), offsetMs: z.number().int().nonnegative().default(0), gainDb: z.number().min(-60).max(12).default(0), loop: z.boolean().default(false), @@ -1265,6 +1284,7 @@ export type AxcutAnnotationRegion = z.infer; export type AxcutZoomRegion = z.infer; export type AxcutCameraTrack = z.infer; export type AxcutAudioTrack = z.infer; +export type AxcutRecordingPlaybackMap = NonNullable; export type AxcutLegacyEditor = z.infer; export type AxcutDocument = z.infer; export type AxcutDocumentInput = z.input; @@ -1345,6 +1365,8 @@ export function createAudioTrack(input: { * `spanSec` when the caller wants a shorter placement than the file. */ timelineStartSec?: number; spanSec?: number; + /** Playback cuts and rates in effect while a live take was recorded. */ + recordingPlaybackMap?: AxcutRecordingPlaybackMap; label?: string; }): AxcutAudioTrack { const startMs = Math.round(Math.max(0, input.timelineStartSec ?? 0) * 1000); @@ -1356,6 +1378,7 @@ export function createAudioTrack(input: { id: createId("audio"), assetId: input.assetId, kind, + ...(input.recordingPlaybackMap ? { recordingPlaybackMap: input.recordingPlaybackMap } : {}), ...audioTrackDefaults(kind), durationSec: input.durationSec, startMs, diff --git a/src/lib/ai-edition/store/projectStore.test.ts b/src/lib/ai-edition/store/projectStore.test.ts index b49bae656..14411dded 100644 --- a/src/lib/ai-edition/store/projectStore.test.ts +++ b/src/lib/ai-edition/store/projectStore.test.ts @@ -407,6 +407,39 @@ describe("useProjectStore", () => { expect(useProjectStore.getState().selectedAudioTrackId).toBe(id); }); + it("keeps the recorded voiceover playback clock in the saved document", async () => { + useProjectStore.setState({ + projectId: "proj_test", + document: { ...sampleDoc, assets: [audioAsset] }, + revision: 1, + status: "ready", + error: null, + }); + bridgeMocks.save.mockImplementation((document: unknown) => + Promise.resolve({ success: true, document }), + ); + + await useProjectStore.getState().addAudioTrack("audio_1", 0, { + kind: "voiceover", + durationSec: 8, + spanSec: 13, + recordingPlaybackMap: { + removedRawSpans: [{ startSec: 5, endSec: 10 }], + speedRegions: [], + }, + }); + + expect(useProjectStore.getState().document?.audioTracks[0]).toMatchObject({ + startMs: 0, + endMs: 13_000, + durationSec: 8, + recordingPlaybackMap: { + removedRawSpans: [{ startSec: 5, endSec: 10 }], + speedRegions: [], + }, + }); + }); + it("addAudioTrack refuses a non-audio (or unknown) asset and selects nothing", async () => { useProjectStore.setState({ projectId: "proj_test", diff --git a/src/lib/ai-edition/store/projectStore.ts b/src/lib/ai-edition/store/projectStore.ts index ec60b230b..05b2d8838 100644 --- a/src/lib/ai-edition/store/projectStore.ts +++ b/src/lib/ai-edition/store/projectStore.ts @@ -6,7 +6,13 @@ import { nativeBridgeClient } from "@/native/client"; import { placeAudioTrackInDocument } from "../document/audioTracks"; import { createId } from "../document/ids"; import { type Interval, replaceTimeline as replaceTimelineOp } from "../document/timeline"; -import { type AxcutAsset, type AxcutDocument, createAudioTrack, documentSchema } from "../schema"; +import { + type AxcutAsset, + type AxcutDocument, + type AxcutRecordingPlaybackMap, + createAudioTrack, + documentSchema, +} from "../schema"; import { probeAudioDuration, probeVideoDimensions } from "../timeline/duration"; import { clearHistory, currentWriteEpoch, pushHistory } from "./undoStack"; @@ -183,6 +189,7 @@ export interface ProjectState { durationSec?: number; /** Timeline span, when it should differ from the source duration. */ spanSec?: number; + recordingPlaybackMap?: AxcutRecordingPlaybackMap; }, ) => Promise; setSelectedAudioTrackId: (id: string | null) => void; @@ -500,6 +507,7 @@ export const useProjectStore = create((set, get) => ({ durationSec: options?.durationSec ?? asset.durationSec ?? 0, kind: options?.kind, spanSec: options?.spanSec, + recordingPlaybackMap: options?.recordingPlaybackMap, // Default to the playhead (RAW/document timeline seconds — the clock the // ruler and playhead use, NOT the trim-compressed output programme), // matching the timeline hook's placement. A voiceover passes the playhead diff --git a/src/lib/ai-edition/store/useTimeline.ts b/src/lib/ai-edition/store/useTimeline.ts index 7c173f870..56efc967e 100644 --- a/src/lib/ai-edition/store/useTimeline.ts +++ b/src/lib/ai-edition/store/useTimeline.ts @@ -1446,7 +1446,12 @@ export function useTimeline() { async ( assetId: string, timelineStartSec?: number, - options?: { kind?: "voiceover" | "music"; durationSec?: number; spanSec?: number }, + options?: { + kind?: "voiceover" | "music"; + durationSec?: number; + spanSec?: number; + recordingPlaybackMap?: AxcutAudioTrack["recordingPlaybackMap"]; + }, ): Promise => { const id = await storeAddAudioTrack(assetId, timelineStartSec ?? playheadSec(), options); if (id) { diff --git a/src/lib/ai-edition/timeline/aggregated-transcript.lanes.test.ts b/src/lib/ai-edition/timeline/aggregated-transcript.lanes.test.ts index 9cfebde27..3ce2f2241 100644 --- a/src/lib/ai-edition/timeline/aggregated-transcript.lanes.test.ts +++ b/src/lib/ai-edition/timeline/aggregated-transcript.lanes.test.ts @@ -88,6 +88,57 @@ describe("voiceoverPlacements", () => { [4, 4, 6], ]); }); + + it("places a live-recorded take's later words after the removed raw span", () => { + const placements = voiceoverPlacements( + [ + track({ + id: "recorded", + startMs: 0, + endMs: 13_000, + recordingPlaybackMap: { + removedRawSpans: [{ startSec: 5, endSec: 10 }], + speedRegions: [], + }, + }), + ], + [{ startSec: 5, endSec: 10, trimIds: ["trim"] }], + ); + + expect( + placements.map((placement) => [ + placement.timelineStartSec, + placement.sourceStartSec, + placement.sourceEndSec, + ]), + ).toEqual([ + [0, 0, 5], + [10, 5, 8], + ]); + }); + + it("maps transcript words through the playback rate captured for a live take", () => { + const placements = voiceoverPlacements([ + track({ + id: "recorded", + startMs: 0, + endMs: 8000, + recordingPlaybackMap: { + removedRawSpans: [], + speedRegions: [{ startMs: 2000, endMs: 6000, speed: 2 }], + }, + }), + ]); + + expect( + placements.map((placement) => [placement.timelineStartSec, placement.placementRawEndSec]), + ).toEqual([ + [0, 2], + [2, 6], + [6, 8], + ]); + expect(placementRawSec(placements[1], 3)).toBe(4); + }); }); describe("lanePlacements", () => { diff --git a/src/lib/ai-edition/timeline/aggregated-transcript.test.ts b/src/lib/ai-edition/timeline/aggregated-transcript.test.ts index 1a62d7e4c..edf6981c4 100644 --- a/src/lib/ai-edition/timeline/aggregated-transcript.test.ts +++ b/src/lib/ai-edition/timeline/aggregated-transcript.test.ts @@ -432,6 +432,31 @@ describe("findCueWordId", () => { expect(findCueWordId([section], 2)).toBeNull(); }); + it("maps a speed-adjusted raw cue back to the placement's source clock", () => { + const section = makeSection( + "vo_1", + "asset_vo", + [ + ["before", 2, 2.9], + ["atCue", 2.9, 3.1], + ["after", 3.1, 4], + ], + { + sourceStartSec: 2, + sourceEndSec: 4, + timelineStartSec: 2, + timelineEndSec: 6, + }, + ); + const speedAdjusted = { + ...section, + clip: { ...section.clip, placementRawEndSec: 6 }, + }; + + // Raw 4 maps to source 3; treating the raw offset as a source offset picks "after". + expect(findCueWordId([speedAdjusted], 4)).toBe("vo_1:atCue"); + }); + // Two clips over the same media project the SAME transcript words twice, so the cue // has to be resolved against the one that is actually playing. describe("two clips over the same media", () => { diff --git a/src/lib/ai-edition/timeline/aggregated-transcript.ts b/src/lib/ai-edition/timeline/aggregated-transcript.ts index 1527a7f37..1acdacc68 100644 --- a/src/lib/ai-edition/timeline/aggregated-transcript.ts +++ b/src/lib/ai-edition/timeline/aggregated-transcript.ts @@ -43,6 +43,8 @@ export interface TranscriptPlacement { /** Where the window lands on the RAW ruler. Source time is per asset, so this is * the only thing that turns a word back into a moment the playhead can seek to. */ timelineStartSec: number; + /** Raw end for placements whose source clock does not run 1:1 with the ruler. */ + placementRawEndSec?: number; } /** Which lane's speech the transcript tab is reading. */ @@ -56,6 +58,17 @@ export type TranscriptLane = "recording" | "voiceover"; * and not in source time (issue #560). */ export function placementRawSec(placement: TranscriptPlacement, sourceSec: number): number { + if ( + typeof placement.sourceEndSec === "number" && + typeof placement.placementRawEndSec === "number" + ) { + const sourceSpan = placement.sourceEndSec - placement.sourceStartSec; + if (sourceSpan <= 0) return placement.timelineStartSec; + const rawSpan = placement.placementRawEndSec - placement.timelineStartSec; + return ( + placement.timelineStartSec + ((sourceSec - placement.sourceStartSec) / sourceSpan) * rawSpan + ); + } return placement.timelineStartSec + (sourceSec - placement.sourceStartSec); } @@ -64,7 +77,7 @@ export function placementRawExtent(placement: TranscriptPlacement): RawSpan | nu if (placement.sourceEndSec === undefined) return null; return { startSec: placement.timelineStartSec, - endSec: placementRawSec(placement, placement.sourceEndSec), + endSec: placement.placementRawEndSec ?? placementRawSec(placement, placement.sourceEndSec), }; } @@ -338,6 +351,7 @@ export function voiceoverPlacements( sourceStartSec: piece.sourceStartSec, sourceEndSec: piece.sourceEndSec, timelineStartSec: piece.rawStartSec, + placementRawEndSec: piece.rawEndSec, })), ); } @@ -396,7 +410,14 @@ export function findCueWordId(sections: ClipSection[], rawSec: number | null): s if (!match) return null; // Back to the placement's own source clock, which is what the words are stamped in. - const t = match.clip.sourceStartSec + (rawSec - match.clip.timelineStartSec); + const sourceSpan = + (match.clip.sourceEndSec ?? match.clip.sourceStartSec) - match.clip.sourceStartSec; + const rawSpan = + (match.clip.placementRawEndSec ?? match.clip.timelineStartSec) - match.clip.timelineStartSec; + const t = + typeof match.clip.placementRawEndSec === "number" && rawSpan > 0 + ? match.clip.sourceStartSec + ((rawSec - match.clip.timelineStartSec) / rawSpan) * sourceSpan + : match.clip.sourceStartSec + (rawSec - match.clip.timelineStartSec); let previous: string | null = null; for (const cw of match.words) { if (isSilenceWord(cw.word)) continue; diff --git a/src/lib/ai-edition/timeline/playback-time.ts b/src/lib/ai-edition/timeline/playback-time.ts new file mode 100644 index 000000000..c8bb8821c --- /dev/null +++ b/src/lib/ai-edition/timeline/playback-time.ts @@ -0,0 +1,40 @@ +export interface PlaybackSpeedRegion { + startMs: number; + endMs: number; + speed: number; +} + +/** + * Output seconds a raw interval occupies after speed regions apply: a 2x stretch of raw time + * takes half as long to play. Integrates 1 / speed piecewise at every region boundary. + */ +export function outputDurationOfRawSpan( + fromSec: number, + toSec: number, + speedRegions: readonly PlaybackSpeedRegion[], +): number { + if (toSec <= fromSec) return 0; + if (speedRegions.length === 0) return toSec - fromSec; + const boundaries = new Set([fromSec, toSec]); + for (const region of speedRegions) { + for (const edge of [region.startMs / 1000, region.endMs / 1000]) { + if (edge > fromSec && edge < toSec) boundaries.add(edge); + } + } + const edges = [...boundaries].sort((a, b) => a - b); + let duration = 0; + for (let i = 0; i < edges.length - 1; i++) { + const start = edges[i]; + const end = edges[i + 1]; + const mid = (start + end) / 2; + const region = speedRegions.find( + (candidate) => + mid >= candidate.startMs / 1000 && + mid < candidate.endMs / 1000 && + Number.isFinite(candidate.speed) && + candidate.speed > 0, + ); + duration += (end - start) / (region?.speed ?? 1); + } + return duration; +} diff --git a/src/lib/ai-edition/timeline/take-programme.test.ts b/src/lib/ai-edition/timeline/take-programme.test.ts index def48cc48..93918e096 100644 --- a/src/lib/ai-edition/timeline/take-programme.test.ts +++ b/src/lib/ai-edition/timeline/take-programme.test.ts @@ -73,6 +73,54 @@ describe("takePlaybackAt", () => { it("has nothing to say outside the take", () => { expect(takePlaybackAt(pieces, 12)).toBeNull(); }); + + it("keeps a voiceover recorded against playback aligned after an existing trim", () => { + const recorded = { + ...TAKE, + endMs: 13_000, + recordingPlaybackMap: { + removedRawSpans: [{ startSec: 5, endSec: 10 }], + speedRegions: [], + }, + }; + const playback = takeProgramme(recorded, removedRawSpans(CLIPS, [trim(5, 10)])); + + expect(takePlaybackAt(playback, 7)).toMatchObject({ targetTimeSec: 5, shouldPlay: false }); + expect(takePlaybackAt(playback, 10)).toMatchObject({ targetTimeSec: 5, shouldPlay: true }); + expect(takePlaybackAt(playback, 12)).toMatchObject({ targetTimeSec: 7, shouldPlay: true }); + }); + + it("does not compress the source for trims added after recording", () => { + const recorded = { + ...TAKE, + endMs: 13_000, + recordingPlaybackMap: { removedRawSpans: [], speedRegions: [] }, + }; + const playback = takeProgramme(recorded, removedRawSpans(CLIPS, [trim(5, 10)])); + + expect(takePlaybackAt(playback, 7)).toMatchObject({ targetTimeSec: 7, shouldPlay: false }); + expect(takePlaybackAt(playback, 10)).toMatchObject({ targetTimeSec: 10, shouldPlay: true }); + expect(takePlaybackAt(playback, 12)).toMatchObject({ targetTimeSec: 12, shouldPlay: true }); + }); + + it("advances recorded source time at the playback rate", () => { + const recorded = { + ...TAKE, + endMs: 12_000, + recordingPlaybackMap: { + removedRawSpans: [{ startSec: 5, endSec: 10 }], + speedRegions: [ + { startMs: 0, endMs: 5000, speed: 2 }, + { startMs: 10_000, endMs: 12_000, speed: 2 }, + ], + }, + }; + const playback = takeProgramme(recorded, removedRawSpans(CLIPS, [trim(5, 10)])); + + expect(takePlaybackAt(playback, 2)).toMatchObject({ targetTimeSec: 1, shouldPlay: true }); + expect(takePlaybackAt(playback, 7)).toMatchObject({ targetTimeSec: 2.5, shouldPlay: false }); + expect(takePlaybackAt(playback, 11)).toMatchObject({ targetTimeSec: 3, shouldPlay: true }); + }); }); // ─── The preview and the export, held to each other ───────────────────────── diff --git a/src/lib/ai-edition/timeline/take-programme.ts b/src/lib/ai-edition/timeline/take-programme.ts index 1d0017054..f713bc162 100644 --- a/src/lib/ai-edition/timeline/take-programme.ts +++ b/src/lib/ai-edition/timeline/take-programme.ts @@ -1,13 +1,14 @@ -// One walk over a voice-over take (issue #560). +// One walk over a voice-over take (issues #560 and #1195). // -// A CUT under a take takes time away: the voice is silent through it with its own clock -// still running, which is what keeps the words after a cut landing on the picture they -// belong to. That is the whole of what this does. +// Imported voiceovers keep their source clock running under a cut: they were recorded against +// the raw timeline, so words after the cut keep their original source positions. Live-recorded +// takes use the playback map captured at recording time, because the recorder never captured the +// skipped intervals. Current cuts still mute either kind of take. // -// It used to also add time, for an insertion inside the take. It does not any more: an -// insertion IS a track of its own (`document/insertionTrack.ts`), so the take arrives here -// already split into pills that each play an uninterrupted stretch of their file. Both -// cursors therefore advance together everywhere, and the walk is a subtraction again. +// An insertion inside the take used to add time here. It does not any more: an +// insertion IS a track of its own (`document/insertionTrack.ts`), so each resulting pill plays +// an uninterrupted stretch of its file. Imported takes consume raw time; recorded takes consume +// output time on kept stretches. Neither walk needs a second pass for insertions. // // It runs on the PILL, never on a stored fragment. The document stores one fragment per clip // a take covers, and the mixer sums with `+=` at an absolute offset — so a fragment-wise walk @@ -15,13 +16,14 @@ // correct for the music and loop paths that still read it. import type { AxcutAudioTrack } from "../schema"; +import { outputDurationOfRawSpan } from "./playback-time"; import type { RemovedRawSpan } from "./programme-time"; /** One stretch of a take, in playback order. */ export interface TakePiece { /** * `play` — the file is heard. `removed` — the film lost this stretch, so the take is - * silent through it, its own clock still running underneath. + * silent through it. Recorded takes hold their source clock; imported takes keep theirs moving. */ kind: "play" | "removed"; /** Stored RAW ruler seconds. */ @@ -36,7 +38,7 @@ const EPSILON_SEC = 1e-9; /** The take, stretch by stretch, in playback order: what is heard, and what a cut mutes. */ export function takeProgramme( - pill: Pick, + pill: Pick, removed: readonly RemovedRawSpan[], ): TakePiece[] { const rawStart = pill.startMs / 1000; @@ -47,6 +49,45 @@ export function takeProgramme( .filter((span) => span.endSec > rawStart && span.startSec < rawEnd) .sort((a, b) => a.startSec - b.startSec); + if (pill.recordingPlaybackMap) { + const recordingCuts = pill.recordingPlaybackMap.removedRawSpans.filter( + (span) => span.endSec > rawStart && span.startSec < rawEnd, + ); + const boundaries = new Set([rawStart, rawEnd]); + for (const span of [...cuts, ...recordingCuts]) { + if (span.startSec > rawStart && span.startSec < rawEnd) boundaries.add(span.startSec); + if (span.endSec > rawStart && span.endSec < rawEnd) boundaries.add(span.endSec); + } + for (const region of pill.recordingPlaybackMap.speedRegions) { + for (const edge of [region.startMs / 1000, region.endMs / 1000]) { + if (edge > rawStart && edge < rawEnd) boundaries.add(edge); + } + } + const edges = [...boundaries].sort((a, b) => a - b); + const pieces: TakePiece[] = []; + let source = sourceStart; + for (let i = 0; i < edges.length - 1; i++) { + const raw = edges[i]; + const to = edges[i + 1]; + if (to - raw <= EPSILON_SEC) continue; + const mid = (raw + to) / 2; + const sourceRemoved = recordingCuts.some((span) => mid >= span.startSec && mid < span.endSec); + const currentRemoved = cuts.some((span) => mid >= span.startSec && mid < span.endSec); + const sourceEnd = sourceRemoved + ? source + : source + outputDurationOfRawSpan(raw, to, pill.recordingPlaybackMap.speedRegions); + pieces.push({ + kind: sourceRemoved || currentRemoved ? "removed" : "play", + rawStartSec: raw, + rawEndSec: to, + sourceStartSec: source, + sourceEndSec: sourceEnd, + }); + source = sourceEnd; + } + return pieces; + } + const pieces: TakePiece[] = []; let raw = rawStart; // The source clock never parks, so it is a plain shift of the raw one. @@ -101,7 +142,9 @@ export function takePlaybackAt( if (rawSec < piece.rawStartSec) break; if (rawSec >= piece.rawEndSec) continue; - const target = piece.sourceStartSec + (rawSec - piece.rawStartSec); + const rawSpan = piece.rawEndSec - piece.rawStartSec; + const sourceSpan = piece.sourceEndSec - piece.sourceStartSec; + const target = piece.sourceStartSec + ((rawSec - piece.rawStartSec) / rawSpan) * sourceSpan; return { targetTimeSec: target, shouldPlay: piece.kind === "play" }; } return null; diff --git a/src/native/sceneDescription.test.ts b/src/native/sceneDescription.test.ts index 0d84461b5..a9c25ac43 100644 --- a/src/native/sceneDescription.test.ts +++ b/src/native/sceneDescription.test.ts @@ -15,6 +15,7 @@ import { import { prepareClickSound } from "@/lib/ai-edition/clickSound"; import type { AxcutAsset, + AxcutAudioTrack, AxcutClip, AxcutDocument, AxcutZoomRegion, @@ -2601,7 +2602,7 @@ describe("buildSceneDescription.audioTracks", () => { durationSec: 30, }); // A 10s span starting at raw 5s, playing the source from 2s in. - const track = { + const track: AxcutAudioTrack = { id: "trk1", assetId: "aud", kind: "music" as const, @@ -2668,7 +2669,7 @@ describe("buildSceneDescription.audioTracks", () => { // through and ends early, deliberately, and has no words whose redness must be true. /** One 10s clip, cut over raw 4..6. */ - function cutDoc(tracks: Array & { kind: "music" | "voiceover" }>) { + function cutDoc(tracks: AxcutAudioTrack[]) { return makeDoc({ assets: [ audioAsset, @@ -2702,7 +2703,7 @@ describe("buildSceneDescription.audioTracks", () => { } /** A voiceover over the whole 10s, reading its file from the head. */ - const voice = { + const voice: AxcutAudioTrack = { ...track, kind: "voiceover" as const, startMs: 0, @@ -2721,6 +2722,27 @@ describe("buildSceneDescription.audioTracks", () => { expect(entries[1]).toMatchObject({ startSec: 4, trimStartSec: 6, trimEndSec: 10 }); }); + it("keeps a live recorded take aligned after the cut in the rendered mix", () => { + const doc = cutDoc([ + { + ...voice, + endMs: 10_000, + recordingPlaybackMap: { + removedRawSpans: [{ startSec: 4, endSec: 6 }], + speedRegions: [], + }, + }, + ]); + doc.assets = doc.assets.map((asset) => + asset.id === "aud" ? { ...asset, durationSec: 8 } : asset, + ); + const entries = buildSceneDescription(doc).audioTracks; + + expect(entries).toHaveLength(2); + expect(entries[0]).toMatchObject({ startSec: 0, trimStartSec: 0, trimEndSec: 4 }); + expect(entries[1]).toMatchObject({ startSec: 4, trimStartSec: 4, trimEndSec: 8 }); + }); + it("keeps the fades on the take's outer edges across a split", () => { const entries = buildSceneDescription( cutDoc([{ ...voice, fadeInMs: 500, fadeOutMs: 500 }]),