fix(editor): sync voiceovers with trimmed sections

src/components/ai-edition/NewEditorShell.tsx

@@ -12,6 +12,8 @@ import { createId } from "@/lib/ai-edition/document/ids";
 import {
 	documentAfterProbedDuration,
 	PLACEHOLDER_DURATION_SEC,
+	projectRawTimelineSecToPlayback,
+	readSpeedRegions,
 } from "@/lib/ai-edition/document/timeline";
 import {
 	type InsertSide,
@@ -37,6 +39,7 @@ import { useTimeline } from "@/lib/ai-edition/store/useTimeline";
 import { isGeneratedAssetId } from "@/lib/ai-edition/timeline/clip-parts";
 import { mergeCloseCuts } from "@/lib/ai-edition/timeline/cut-breath";
 import { newRegionDurationSec } from "@/lib/ai-edition/timeline/newRegionDuration";
+import { removedRawSpans } from "@/lib/ai-edition/timeline/programme-time";
 import {
 	dropTrimPillsByIds,
 	trimAppliesToClip,
@@ -980,16 +983,30 @@ export function NewEditorShell() {
 	// placed every voiceover one full take-length to the right of where it was
 	// spoken. Captured on the way in, used on the way out.
 	const voiceoverStartSecRef = useRef(0);
+	const voiceoverEndSecRef = useRef(0);
+	const voiceoverMaxRawEndSecRef = useRef(0);
+	const voiceoverRecordingPlaybackMapRef =
+		useRef<AxcutAudioTrack["recordingPlaybackMap"]>(undefined);
 
 	const openVoiceoverFlow = useCallback(() => {
 		const doc = useProjectStore.getState().document;
 		if (!doc) return;
 		const total = doc.timeline.clips.reduce((max, c) => Math.max(max, c.timelineEndSec), 0);
 		const playhead = useProjectStore.getState().currentTimeSec;
+		const speedRegions = readSpeedRegions(doc);
+		const outputAt = (rawSec: number) =>
+			projectRawTimelineSecToPlayback(
+				doc.timeline.clips,
+				doc.timeline.trimRanges,
+				rawSec,
+				speedRegions,
+			);
 		voiceoverStartSecRef.current = playhead;
+		voiceoverEndSecRef.current = playhead;
+		voiceoverMaxRawEndSecRef.current = total;
 		// Recording stops itself at the end of the timeline: a take can never
 		// outlive the video it was recorded over.
-		setVoiceoverFlow({ maxDurationSec: Math.max(0.5, total - playhead) });
+		setVoiceoverFlow({ maxDurationSec: Math.max(0.5, outputAt(total) - outputAt(playhead)) });
 	}, []);
 
 	// Silences the timeline's own audio tracks for the duration of a take — see
@@ -999,23 +1016,58 @@ export function NewEditorShell() {
 	const handleVoiceoverRecordingStart = useCallback(() => {
 		// Re-capture: the user may have scrubbed between opening the dialog and
 		// hitting Record, and playback starts from wherever the playhead is now.
-		voiceoverStartSecRef.current = useProjectStore.getState().currentTimeSec;
+		const doc = useProjectStore.getState().document;
+		const playhead = useProjectStore.getState().currentTimeSec;
+		voiceoverStartSecRef.current = playhead;
+		voiceoverEndSecRef.current = playhead;
+		voiceoverRecordingPlaybackMapRef.current = undefined;
+		if (doc) {
+			const total = doc.timeline.clips.reduce((max, c) => Math.max(max, c.timelineEndSec), 0);
+			const speedRegions = readSpeedRegions(doc);
+			const outputAt = (rawSec: number) =>
+				projectRawTimelineSecToPlayback(
+					doc.timeline.clips,
+					doc.timeline.trimRanges,
+					rawSec,
+					speedRegions,
+				);
+			voiceoverMaxRawEndSecRef.current = total;
+			voiceoverRecordingPlaybackMapRef.current = {
+				removedRawSpans: removedRawSpans(doc.timeline.clips, doc.timeline.trimRanges)
+					.filter((span) => span.endSec > playhead && span.startSec < total)
+					.map(({ startSec, endSec }) => ({ startSec, endSec })),
+				speedRegions: speedRegions.filter(
+					(region) => region.endMs / 1000 > playhead && region.startMs / 1000 < total,
+				),
+			};
+			setVoiceoverFlow({
+				maxDurationSec: Math.max(0.5, outputAt(total) - outputAt(playhead)),
+			});
+		}
 		setVoiceoverRecording(true);
 		if (videoElement?.paused) void videoElement.play().catch(() => undefined);
 	}, [videoElement]);
 
 	const handleVoiceoverRecordingStop = useCallback(() => {
+		const playhead = useProjectStore.getState().currentTimeSec;
+		voiceoverEndSecRef.current = Math.max(
+			voiceoverStartSecRef.current,
+			Math.min(voiceoverMaxRawEndSecRef.current, playhead),
+		);
 		setVoiceoverRecording(false);
 		videoElement?.pause();
 	}, [videoElement]);
 
 	const handleVoiceoverReady = useCallback(
 		async (assetId: string, durationSec: number) => {
 			setVoiceoverFlow(null);
+			const startSec = voiceoverStartSecRef.current;
+			const rawSpanSec = Math.max(0, voiceoverEndSecRef.current - startSec);
 			await tl.addAudioTrack(assetId, voiceoverStartSecRef.current, {
 				kind: "voiceover",
 				durationSec,
-				spanSec: durationSec,
+				spanSec: rawSpanSec > 0 ? rawSpanSec : durationSec,
+				recordingPlaybackMap: voiceoverRecordingPlaybackMapRef.current,
 			});
 		},
 		[tl],
@@ -1409,6 +1461,7 @@ export function NewEditorShell() {
 		isMac,
 		togglePlay,
 		handleSeek,
+		openVoiceoverFlow,
 	]);
 
 	const showTimeline = mode !== "rec";

src/lib/ai-edition/document/audioTracks.test.ts

@@ -5,8 +5,10 @@ import {
 	audioGhostExtent,
 	collapseTracksToPills,
 	computeAudioRowCount,
+	cutTakeHead,
 	packAudioTrackRows,
 	patchAudioTrack,
+	reanchorAudioTracks,
 	removeAudioTrack,
 	resolveFadeSecs,
 	slipAudioOffsetMs,
@@ -76,6 +78,57 @@ describe("anchorAudioTrackFragments", () => {
 		expect(frags[1].offsetMs).toBe(7000);
 	});
 
+	it("uses the recorded playback clock when a take crosses a clip boundary", () => {
+		const frags = anchorAudioTrackFragments(
+			track({
+				kind: "voiceover",
+				startMs: 0,
+				endMs: 13_000,
+				durationSec: 8,
+				recordingPlaybackMap: {
+					removedRawSpans: [{ startSec: 5, endSec: 10 }],
+					speedRegions: [{ startMs: 0, endMs: 5000, speed: 2 }],
+				},
+			}),
+			twoClips,
+			makeId,
+		);
+
+		expect(frags.map((fragment) => fragment.offsetMs)).toEqual([0, 2500]);
+	});
+
+	it("keeps recorded fragments together when it re-anchors a take", () => {
+		const trackBeforeReanchor = track({
+			kind: "voiceover",
+			startMs: 0,
+			endMs: 13_000,
+			durationSec: 8,
+			recordingPlaybackMap: {
+				removedRawSpans: [{ startSec: 5, endSec: 10 }],
+				speedRegions: [{ startMs: 0, endMs: 5000, speed: 2 }],
+			},
+		});
+		const fragments = anchorAudioTrackFragments(trackBeforeReanchor, twoClips, makeId);
+
+		const reanchored = reanchorAudioTracks(fragments, twoClips, makeId);
+		expect(collapseTracksToPills(reanchored)).toHaveLength(1);
+		expect(reanchored.map((fragment) => fragment.offsetMs)).toEqual([0, 2500]);
+	});
+
+	it("cuts a recorded take head by its captured playback time", () => {
+		const take = track({
+			kind: "voiceover",
+			startMs: 0,
+			offsetMs: 0,
+			recordingPlaybackMap: {
+				removedRawSpans: [{ startSec: 5, endSec: 10 }],
+				speedRegions: [{ startMs: 0, endMs: 5000, speed: 2 }],
+			},
+		});
+
+		expect(cutTakeHead(take, 7).offsetMs).toBe(2500);
+	});
+
 	it("keeps the fades on the outer edges only", () => {
 		const frags = anchorAudioTrackFragments(
 			track({ startMs: 5000, endMs: 15_000, fadeInMs: 500, fadeOutMs: 800 }),

src/lib/ai-edition/document/audioTracks.ts

@@ -19,6 +19,7 @@
 
 import type { AxcutAudioTrack, AxcutClip, AxcutDocument } from "../schema";
 import { isGeneratedAssetId } from "../timeline/clip-parts";
+import { outputDurationOfRawSpan } from "../timeline/playback-time";
 import {
 	anchorRegionsWithDerivedMs,
 	clampSpanAgainstNeighbours,
@@ -63,7 +64,11 @@ export function anchorAudioTrackFragments(
 			// Looping restarts the window on its own, so an advanced offset would
 			// double-count the fold; the mixer and the preview both wrap within
 			// `durationSec - offsetMs`, which every fragment shares.
-			offsetMs: track.loop ? track.offsetMs : track.offsetMs + elapsedMs,
+			offsetMs: track.loop
+				? track.offsetMs
+				: track.recordingPlaybackMap
+					? Math.round(recordedSourceAtRaw(track, fragment.startMs / 1000) * 1000)
+					: track.offsetMs + elapsedMs,
 			fadeInMs: index === 0 ? track.fadeInMs : 0,
 			fadeOutMs: index === last ? track.fadeOutMs : 0,
 		};
@@ -134,10 +139,11 @@ function hangsTogether(
 	right: AxcutAudioTrack,
 	assetOf: Map<string, string>,
 ): boolean {
-	const runsOn =
-		Math.abs(left.offsetMs - right.offsetMs) <= 1 ||
-		(!right.loop && Math.abs(left.offsetMs + (left.endMs - left.startMs) - right.offsetMs) <= 1);
-	return runsOn && (Math.abs(left.endMs - right.startMs) <= 1 || mediaRunsOn(left, right, assetOf));
+	return (
+		recordingPlaybackMapsMatch(left.recordingPlaybackMap, right.recordingPlaybackMap) &&
+		(Math.abs(left.offsetMs - right.offsetMs) <= 1 || sourceClockContinues(left, right)) &&
+		(Math.abs(left.endMs - right.startMs) <= 1 || mediaRunsOn(left, right, assetOf))
+	);
 }
 
 /** One stretch of one media, cut in two: what an inserted word leaves either side of it. */
@@ -182,28 +188,110 @@ function joinContiguousTakes(pills: AxcutAudioTrack[]): AxcutAudioTrack[] {
  *  advances, so its file says nothing about the join: two loops at one offset are one take when
  *  neither fades across the seam, which is the seam splitting one leaves (#1011). */
 function takesJoin(left: AxcutAudioTrack, right: AxcutAudioTrack): boolean {
-	const spanMs = left.endMs - left.startMs;
+	const sourceContinues = left.loop
+		? sourceClockContinues(left, right) && left.fadeOutMs === 0 && right.fadeInMs === 0
+		: sourceClockContinues(left, right);
 	return (
 		left.assetId === right.assetId &&
 		left.kind === right.kind &&
 		left.loop === right.loop &&
 		left.gainDb === right.gainDb &&
 		left.muted === right.muted &&
 		Math.abs(left.endMs - right.startMs) <= 1 &&
-		(left.loop
-			? left.offsetMs === right.offsetMs && left.fadeOutMs === 0 && right.fadeInMs === 0
-			: Math.abs(left.offsetMs + spanMs - right.offsetMs) <= 1)
+		sourceContinues
+	);
+}
+
+function sourceClockContinues(left: AxcutAudioTrack, right: AxcutAudioTrack): boolean {
+	if (!recordingPlaybackMapsMatch(left.recordingPlaybackMap, right.recordingPlaybackMap)) {
+		return false;
+	}
+	if (left.loop) return left.offsetMs === right.offsetMs;
+	if (left.recordingPlaybackMap) {
+		return Math.abs(recordedSourceEndMs(left) - right.offsetMs) <= 1;
+	}
+	return Math.abs(left.offsetMs + (left.endMs - left.startMs) - right.offsetMs) <= 1;
+}
+
+function recordingPlaybackMapsMatch(
+	left: AxcutAudioTrack["recordingPlaybackMap"],
+	right: AxcutAudioTrack["recordingPlaybackMap"],
+): boolean {
+	if (!left || !right) return left === right;
+	return (
+		left.removedRawSpans.length === right.removedRawSpans.length &&
+		left.removedRawSpans.every(
+			(span, index) =>
+				span.startSec === right.removedRawSpans[index].startSec &&
+				span.endSec === right.removedRawSpans[index].endSec,
+		) &&
+		left.speedRegions.length === right.speedRegions.length &&
+		left.speedRegions.every(
+			(region, index) =>
+				region.startMs === right.speedRegions[index].startMs &&
+				region.endMs === right.speedRegions[index].endMs &&
+				region.speed === right.speedRegions[index].speed,
+		)
 	);
 }
 
+function recordedSourceEndMs(track: AxcutAudioTrack): number {
+	return Math.round(recordedSourceAtRaw(track, track.endMs / 1000) * 1000);
+}
+
+function recordedSourceAtRaw(
+	track: Pick<AxcutAudioTrack, "startMs" | "offsetMs" | "recordingPlaybackMap">,
+	rawSec: number,
+): number {
+	const map = track.recordingPlaybackMap;
+	const start = track.startMs / 1000;
+	const end = Math.max(start, rawSec);
+	if (!map) return track.offsetMs / 1000 + end - start;
+	const cuts = [...map.removedRawSpans]
+		.filter((span) => span.endSec > start && span.startSec < end)
+		.sort((a, b) => a.startSec - b.startSec);
+	let raw = start;
+	let sourceSec = track.offsetMs / 1000;
+	for (const cut of cuts) {
+		const cutStart = Math.max(raw, cut.startSec);
+		if (cutStart > raw) {
+			sourceSec += outputDurationOfRawSpan(raw, cutStart, map.speedRegions);
+		}
+		raw = Math.max(raw, Math.min(end, cut.endSec));
+		if (raw >= end) break;
+	}
+	if (raw < end) sourceSec += outputDurationOfRawSpan(raw, end, map.speedRegions);
+	return sourceSec;
+}
+
 /**
  * A clip edit that cuts a fragment's head off cuts the same stretch off its take's file, so what
  * is left goes on playing the audio over its own footage (#1011). A loop restarts on its own and
  * keeps its offset; a region with no file passes through.
  */
 export function cutTakeHead<T extends object>(region: T, cutSec: number): T {
-	const take = region as T & { offsetMs?: unknown; loop?: unknown };
+	const take = region as T & {
+		offsetMs?: unknown;
+		loop?: unknown;
+		startMs?: unknown;
+		recordingPlaybackMap?: AxcutAudioTrack["recordingPlaybackMap"];
+	};
 	if (!(cutSec > 0) || typeof take.offsetMs !== "number" || take.loop === true) return region;
+	if (take.recordingPlaybackMap && typeof take.startMs === "number") {
+		return {
+			...region,
+			offsetMs: Math.round(
+				recordedSourceAtRaw(
+					{
+						startMs: take.startMs,
+						offsetMs: take.offsetMs,
+						recordingPlaybackMap: take.recordingPlaybackMap,
+					},
+					take.startMs / 1000 + cutSec,
+				) * 1000,
+			),
+		};
+	}
 	return { ...region, offsetMs: take.offsetMs + Math.round(cutSec * 1000) };
 }
 

src/lib/ai-edition/document/insertionTrack.test.ts

@@ -164,6 +164,57 @@ describe("retextGeneratedTrack", () => {
 	});
 });
 
+describe("insertions in a live-recorded take", () => {
+	it("uses the take's playback source offset and restores it after removal", () => {
+		const sourceAligned = doc();
+		sourceAligned.timeline.trimRanges = [
+			{
+				id: "trim_1",
+				assetId: "a1",
+				clipId: "c1",
+				startSec: 5,
+				endSec: 10,
+				origin: "user",
+				reason: "",
+			},
+		];
+		sourceAligned.audioTracks = sourceAligned.audioTracks.map((take) => ({
+			...take,
+			endMs: 15_000,
+			durationSec: 8,
+			recordingPlaybackMap: {
+				removedRawSpans: [{ startSec: 5, endSec: 10 }],
+				speedRegions: [],
+			},
+		}));
+
+		const edited = insertGeneratedTrack(sourceAligned, "vo", "v2", "before", "hi");
+		const insertedTake = collapseTracksToPills(edited.audioTracks).find(
+			(take) => take.assetId === "vo" && take.startMs > 10_000,
+		);
+		expect(insertedTake).toMatchObject({
+			offsetMs: 6000,
+			recordingPlaybackMap: { removedRawSpans: [{ startSec: 5, endSec: 10 }] },
+		});
+		expect(
+			collapseTracksToPills(edited.audioTracks).find((take) => take.assetId === "ext:synth_1")
+				?.recordingPlaybackMap,
+		).toBeUndefined();
+
+		const restored = removeGeneratedTracks(edited, ["synth_1"]);
+		expect(
+			collapseTracksToPills(restored.audioTracks).filter((take) => take.assetId === "vo"),
+		).toMatchObject([
+			{
+				startMs: 2000,
+				endMs: 15_000,
+				offsetMs: 0,
+				recordingPlaybackMap: { removedRawSpans: [{ startSec: 5, endSec: 10 }] },
+			},
+		]);
+	});
+});
+
 describe("the two lanes stay out of each other's way", () => {
 	it("a word added to the FILM leaves the take's own span alone", () => {
 		// Settled deliberately: the take has its own audio and keeps talking against a picture