From 9326d9f1bafa615b200435a74792d2d485e1c68b Mon Sep 17 00:00:00 2001
From: EtienneLescot <etiennelescot@gmail.com>
Date: Mon, 19 Oct 2026 22:18:08 +0200
Subject: [PATCH] fix(audio): play a track once across its pill, so speed
 regions keep it on the clock

The document stores a track as one fragment per clip it covers, each fragment's
offset advanced by the ruler time before it. The export mixed, and the preview
played, every fragment on its own: at its projected head, from its
ruler-advanced offset. Under a slowed clip a bed played its ruler seconds of file
over twice as many output seconds, went silent for the rest, then came back late;
through a sped-up clip the slices overlapped. Measured on a 50 s film: 0.6 s of
silence at the end of a 0.5x shot, and the music 2.7 s behind by the end.

Both now take each track once, from its head fragment, as the whole pill: the
voiceover path already did this to stop a take playing on top of itself.
---
 src/components/ai-edition/VirtualPreview.tsx | 43 +++++++++---------
 src/lib/ai-edition/document/audioTracks.ts   | 16 +++++++
 src/native/sceneDescription.test.ts          | 46 ++++++++++++++++++++
 src/native/sceneDescription.ts               | 31 +++++--------
 4 files changed, 96 insertions(+), 40 deletions(-)

diff --git a/src/components/ai-edition/VirtualPreview.tsx b/src/components/ai-edition/VirtualPreview.tsx
index 4e4ec9e550..19c0fb1175 100644
--- a/src/components/ai-edition/VirtualPreview.tsx
+++ b/src/components/ai-edition/VirtualPreview.tsx
@@ -11,8 +11,8 @@ import {
 	takeCrossedClickHits,
 } from "@/lib/ai-edition/clickSound";
 import {
-	collapseTracksToPills,
 	resolveFadeSecs,
+	takesByHeadFragment,
 	trackGroupId,
 } from "@/lib/ai-edition/document/audioTracks";
 import {
@@ -875,23 +875,22 @@ export function VirtualPreview({
 	// One walk per take, recomputed only when the cuts or the insertions move. The rAF asks
 	// it every frame per track, and walking on each would be wasteful.
 	const takePiecesRef = useRef<Map<string, TakePiece[]>>(new Map());
-	const takeHeadsRef = useRef<Map<string, string>>(new Map());
+	// Each take keyed by the fragment whose element plays it, as the export mixes it.
+	const takesRef = useRef<Map<string, AxcutAudioTrack>>(new Map());
 	const removedRef = useRef(removedRawSpans(clips, trimRanges));
 	removedRef.current = useMemo(() => removedRawSpans(clips, trimRanges), [clips, trimRanges]);
 	const takeWalks = useMemo(() => {
 		const pieces = new Map<string, TakePiece[]>();
-		const heads = new Map<string, string>();
 		const removed = removedRawSpans(clips, trimRanges);
-		for (const pill of collapseTracksToPills(audioTracks)) {
+		const takes = takesByHeadFragment(audioTracks);
+		for (const pill of takes.values()) {
 			if (pill.kind !== "voiceover" || pill.loop) continue;
-			const groupId = trackGroupId(pill);
-			heads.set(groupId, pill.id);
-			pieces.set(groupId, takeProgramme(pill, removed));
+			pieces.set(trackGroupId(pill), takeProgramme(pill, removed));
 		}
-		return { pieces, heads };
+		return { pieces, takes };
 	}, [audioTracks, clips, trimRanges]);
 	takePiecesRef.current = takeWalks.pieces;
-	takeHeadsRef.current = takeWalks.heads;
+	takesRef.current = takeWalks.takes;
 	// Trim-narrowed (`resolvePlaybackSegments`) — used ONLY to detect "has the <video>'s own
 	// currentTime drifted into a trim" and where to jump it back out to. Everything ELSE in
 	// this component (`clips`/`clipsRef` above, virtualTimeSec, zoom/speed region lookups,
@@ -1033,9 +1032,18 @@ export function VirtualPreview({
 				}
 			}
 			duck.db = nextMusicDuckDb(duck.db, (now - duck.voiceAt) / 1000, frameSec);
-			for (const track of audioTracksRef.current) {
-				const el = audioTrackElsRef.current.get(track.id);
+			for (const fragment of audioTracksRef.current) {
+				const el = audioTrackElsRef.current.get(fragment.id);
 				if (!el) continue;
+				// A take plays once, from its head fragment's element, as its whole pill: the
+				// document keeps one fragment per clip the take covers, and letting each play its
+				// own slice put a bed back on the ruler under a speed region (see
+				// `takesByHeadFragment`), and a voiceover on top of itself.
+				const track = takesRef.current.get(fragment.id);
+				if (!track) {
+					if (!el.paused) el.pause();
+					continue;
+				}
 				const outputStartSec = projectRawTimelineSecToPlayback(
 					clipsRef.current,
 					trimRangesRef.current,
@@ -1062,22 +1070,15 @@ export function VirtualPreview({
 						),
 				);
 				// A voiceover follows the cuts AND its own insertions, through one walk over
-				// its PILL. Every fragment but the head is silenced: the document keeps one
-				// per clip the take covers, and letting each play its own slice would put the
-				// take on top of itself, exactly as it would in the export.
+				// its pill.
 				//
 				// A bed plays through a cut and ends early, on purpose. A looping voiceover
 				// keeps the bed's treatment — step 6 of #560 refuses that combination, and
 				// inventing semantics for it would be the worse answer.
-				const takeGroupId = trackGroupId(track);
 				const takePieces =
 					track.kind === "voiceover" && !track.loop
-						? takePiecesRef.current.get(takeGroupId)
+						? takePiecesRef.current.get(trackGroupId(track))
 						: undefined;
-				if (takePieces && takeHeadsRef.current.get(takeGroupId) !== track.id) {
-					if (!el.paused) el.pause();
-					continue;
-				}
 				const trackTarget = takePieces
 					? (takePlaybackAt(takePieces, virtualTimeSecRef.current) ?? {
 							targetTimeSec: track.offsetMs / 1000,
@@ -1109,7 +1110,7 @@ export function VirtualPreview({
 				} else {
 					trackGainDb += duck.db;
 				}
-				const trackGainNode = audioTrackGainNodesRef.current.get(track.id);
+				const trackGainNode = audioTrackGainNodesRef.current.get(fragment.id);
 				if (trackGainNode) {
 					trackGainNode.gain.value = audioGainScalar(trackGainDb) * fade;
 					if (el.volume !== 1) el.volume = 1;
diff --git a/src/lib/ai-edition/document/audioTracks.ts b/src/lib/ai-edition/document/audioTracks.ts
index 83dd74a1ef..a65d59d41e 100644
--- a/src/lib/ai-edition/document/audioTracks.ts
+++ b/src/lib/ai-edition/document/audioTracks.ts
@@ -239,6 +239,22 @@ export function collapseTracksToPills(tracks: AxcutAudioTrack[]): AxcutAudioTrac
 	});
 }
 
+/**
+ * Every take once, keyed by the id of the fragment that plays it: its first. The export and the
+ * preview both play a take from there, at 1x, across the whole pill. Played fragment by fragment,
+ * each one its own slice, a bed was back on the ruler under a speed region: silent at the end of a
+ * slowed clip, then behind; on top of itself through a sped-up one.
+ */
+export function takesByHeadFragment(tracks: AxcutAudioTrack[]): Map<string, AxcutAudioTrack> {
+	const heads = new Map<string, string>();
+	for (const track of [...tracks].sort((a, b) => a.startMs - b.startMs)) {
+		if (!heads.has(trackGroupId(track))) heads.set(trackGroupId(track), track.id);
+	}
+	return new Map(
+		collapseTracksToPills(tracks).map((pill) => [heads.get(trackGroupId(pill)) ?? pill.id, pill]),
+	);
+}
+
 /** Drop every fragment of a track, and its asset when nothing else needs it.
  *
  *  An imported audio asset is only ever reachable through its track — audio is
diff --git a/src/native/sceneDescription.test.ts b/src/native/sceneDescription.test.ts
index 567fe925eb..9c259673b4 100644
--- a/src/native/sceneDescription.test.ts
+++ b/src/native/sceneDescription.test.ts
@@ -2941,6 +2941,52 @@ describe("buildSceneDescription.audioTracks", () => {
 		expect(entry.startSec).toBeCloseTo(3, 6);
 	});
 
+	it("plays a bed spanning a slowed clip as one take, on the clock, not on the ruler", () => {
+		// The document stores a track as one fragment per clip it covers, each fragment's offset
+		// advanced by the ruler time before it. Mixed fragment by fragment, the bed under a 0.5x
+		// clip played its 4 ruler seconds of file over 8 output seconds, went silent for the other
+		// 4, then came back 4 seconds behind the music it should have been playing.
+		const screen = makeAsset({ id: "scr", originalPath: "/screen.mp4", durationSec: 20 });
+		const fragment = { ...track, trackId: "trk1", startMs: 0, offsetMs: 0, endMs: 4000 };
+		const doc = makeDoc({
+			assets: [screen, audioAsset],
+			clips: [
+				makeClip({
+					id: "c1",
+					assetId: "scr",
+					sourceStartSec: 0,
+					sourceEndSec: 4,
+					timelineStartSec: 0,
+					timelineEndSec: 4,
+				}),
+				makeClip({
+					id: "c2",
+					assetId: "scr",
+					sourceStartSec: 4,
+					sourceEndSec: 8,
+					timelineStartSec: 4,
+					timelineEndSec: 8,
+				}),
+			],
+			legacyEditor: { speedRegions: [{ id: "s1", startMs: 0, endMs: 4000, speed: 0.5 }] },
+			audioTracks: [
+				{ ...fragment, clipId: "c1", sourceStartSec: 0, sourceEndSec: 4 },
+				{
+					...fragment,
+					id: "trk1-b",
+					clipId: "c2",
+					sourceStartSec: 4,
+					sourceEndSec: 8,
+					startMs: 4000,
+					endMs: 8000,
+					offsetMs: 4000,
+				},
+			],
+		});
+		const entries = buildSceneDescription(doc).audioTracks;
+		expect(entries.map((e) => [e.startSec, e.trimStartSec, e.trimEndSec])).toEqual([[0, 0, 8]]);
+	});
+
 	it("drops a track whose asset has no resolvable path", () => {
 		const doc = makeDoc({ assets: [], audioTracks: [track] });
 		expect(buildSceneDescription(doc).audioTracks).toEqual([]);
diff --git a/src/native/sceneDescription.ts b/src/native/sceneDescription.ts
index 71e920b38c..44117795b7 100644
--- a/src/native/sceneDescription.ts
+++ b/src/native/sceneDescription.ts
@@ -31,7 +31,7 @@ import {
 	getCaptionTranslations,
 } from "@/lib/ai-edition/captions";
 import { clickSoundForDocument } from "@/lib/ai-edition/clickSound";
-import { collapseTracksToPills, trackGroupId } from "@/lib/ai-edition/document/audioTracks";
+import { takesByHeadFragment } from "@/lib/ai-edition/document/audioTracks";
 import { createId } from "@/lib/ai-edition/document/ids";
 import { isFormatFillActive, pickOutputDims } from "@/lib/ai-edition/document/outputFormat";
 import {
@@ -884,15 +884,12 @@ export function buildSceneDescription(
 	// question, and it does not depend on the track.
 	// Placed once: the projection below counts them, so a track after a pause lands where
 	const removed = removedRawSpans(projectedClips, document.timeline.trimRanges);
-	// The take's pills, keyed by group. A voiceover is walked ONCE per pill and never per
-	// stored fragment: the document keeps one fragment per clip a take covers, so walking
-	// them separately would emit overlapping entries and `overlay_track_pcm` sums with `+=`
-	// at an absolute offset — the export would contain the take playing on top of itself.
-	const voiceoverPills = new Map(
-		collapseTracksToPills(document.audioTracks)
-			.filter((pill) => pill.kind === "voiceover" && !pill.loop)
-			.map((pill) => [trackGroupId(pill), pill]),
-	);
+	// Every take is mixed ONCE, from its head fragment, as its whole pill — never per stored
+	// fragment. The document keeps one fragment per clip a take covers: a voiceover walked per
+	// fragment would play on top of itself (`overlay_track_pcm` sums with `+=`), and a bed laid
+	// per fragment, each at its projected head with its ruler-advanced offset, went silent at the
+	// end of a slowed clip and came back late (see `takesByHeadFragment`).
+	const takes = takesByHeadFragment(document.audioTracks);
 	// Click hits for the "mouse clicks" toggle, placed on the programme this export assembles.
 	// Synchronous: the cues were read from the take's sidecar when the sound was switched on.
 	// Its row lives under "Show cursor", like the click impact: a hidden cursor makes no sound.
@@ -900,8 +897,9 @@ export function buildSceneDescription(
 		clickSound: settings.cursorShow && settings.cursor.clickSound,
 		clickSoundGainDb: settings.cursor.clickSoundGainDb,
 	});
-	const audioTracks = document.audioTracks.flatMap((track) => {
-		if (track.muted) return [];
+	const audioTracks = document.audioTracks.flatMap((fragment) => {
+		const track = takes.get(fragment.id);
+		if (!track || track.muted) return [];
 		const asset = assetById.get(track.assetId);
 		if (!asset?.originalPath) return [];
 		const sourceDurationSec = asset.durationSec ?? track.durationSec;
@@ -957,17 +955,12 @@ export function buildSceneDescription(
 		// #560 refuses that combination outright, and inventing semantics for something
 		// about to be banned would be the worse answer.
 		if (track.kind === "voiceover" && !track.loop) {
-			const groupId = trackGroupId(track);
-			const pill = voiceoverPills.get(groupId);
-			// Emitted from the group's HEAD fragment only — every other fragment of the same
-			// take is already covered by the pill's own walk.
-			if (!pill || pill.id !== track.id) return [];
-			const rawSpanSec = Math.max(0, pill.endMs / 1000 - pill.startMs / 1000);
+			const rawSpanSec = Math.max(0, track.endMs / 1000 - track.startMs / 1000);
 			// Unprobed assets have no real duration to cap with; the RAW span is how much
 			// file the take covers, which is the honest fallback once the cuts are taken out.
 			const voWindowSec =
 				sourceDurationSec > 0 ? Math.max(0, sourceDurationSec - offsetSec) : rawSpanSec;
-			const kept = takeProgramme(pill, removed)
+			const kept = takeProgramme(track, removed)
 				.filter((piece) => piece.kind === "play")
 				.map((piece) => ({
 					...base,