Skip to content

fix(audio): play a track once across its pill, so speed regions keep it on the clock #1355

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
EtienneLescot merged 1 commit into from
Oct 23, 2026
Merged

fix(audio): play a track once across its pill, so speed regions keep it on the clock #1355

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
43 changes: 22 additions & 21 deletions src/components/ai-edition/VirtualPreview.tsx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,8 +11,8 @@ import {
takeCrossedClickHits,
} from "@/lib/ai-edition/clickSound";
import {
collapseTracksToPills,
resolveFadeSecs,
takesByHeadFragment,
trackGroupId,
} from "@/lib/ai-edition/document/audioTracks";
import {
Expand Down Expand Up @@ -875,23 +875,22 @@ export function VirtualPreview({
// One walk per take, recomputed only when the cuts or the insertions move. The rAF asks
// it every frame per track, and walking on each would be wasteful.
const takePiecesRef = useRef<Map<string, TakePiece[]>>(new Map());
const takeHeadsRef = useRef<Map<string, string>>(new Map());
// Each take keyed by the fragment whose element plays it, as the export mixes it.
const takesRef = useRef<Map<string, AxcutAudioTrack>>(new Map());
const removedRef = useRef(removedRawSpans(clips, trimRanges));
removedRef.current = useMemo(() => removedRawSpans(clips, trimRanges), [clips, trimRanges]);
const takeWalks = useMemo(() => {
const pieces = new Map<string, TakePiece[]>();
const heads = new Map<string, string>();
const removed = removedRawSpans(clips, trimRanges);
for (const pill of collapseTracksToPills(audioTracks)) {
const takes = takesByHeadFragment(audioTracks);
for (const pill of takes.values()) {
if (pill.kind !== "voiceover" || pill.loop) continue;
const groupId = trackGroupId(pill);
heads.set(groupId, pill.id);
pieces.set(groupId, takeProgramme(pill, removed));
pieces.set(trackGroupId(pill), takeProgramme(pill, removed));
}
return { pieces, heads };
return { pieces, takes };
}, [audioTracks, clips, trimRanges]);
takePiecesRef.current = takeWalks.pieces;
takeHeadsRef.current = takeWalks.heads;
takesRef.current = takeWalks.takes;
// Trim-narrowed (`resolvePlaybackSegments`) — used ONLY to detect "has the <video>'s own
// currentTime drifted into a trim" and where to jump it back out to. Everything ELSE in
// this component (`clips`/`clipsRef` above, virtualTimeSec, zoom/speed region lookups,
Expand Down Expand Up @@ -1033,9 +1032,18 @@ export function VirtualPreview({
}
}
duck.db = nextMusicDuckDb(duck.db, (now - duck.voiceAt) / 1000, frameSec);
for (const track of audioTracksRef.current) {
const el = audioTrackElsRef.current.get(track.id);
for (const fragment of audioTracksRef.current) {
const el = audioTrackElsRef.current.get(fragment.id);
if (!el) continue;
// A take plays once, from its head fragment's element, as its whole pill: the
// document keeps one fragment per clip the take covers, and letting each play its
// own slice put a bed back on the ruler under a speed region (see
// `takesByHeadFragment`), and a voiceover on top of itself.
const track = takesRef.current.get(fragment.id);
if (!track) {
if (!el.paused) el.pause();
continue;
}
const outputStartSec = projectRawTimelineSecToPlayback(
clipsRef.current,
trimRangesRef.current,
Expand All @@ -1062,22 +1070,15 @@ export function VirtualPreview({
),
);
// A voiceover follows the cuts AND its own insertions, through one walk over
// its PILL. Every fragment but the head is silenced: the document keeps one
// per clip the take covers, and letting each play its own slice would put the
// take on top of itself, exactly as it would in the export.
// its pill.
//
// A bed plays through a cut and ends early, on purpose. A looping voiceover
// keeps the bed's treatment — step 6 of #560 refuses that combination, and
// inventing semantics for it would be the worse answer.
const takeGroupId = trackGroupId(track);
const takePieces =
track.kind === "voiceover" && !track.loop
? takePiecesRef.current.get(takeGroupId)
? takePiecesRef.current.get(trackGroupId(track))
: undefined;
if (takePieces && takeHeadsRef.current.get(takeGroupId) !== track.id) {
if (!el.paused) el.pause();
continue;
}
const trackTarget = takePieces
? (takePlaybackAt(takePieces, virtualTimeSecRef.current) ?? {
targetTimeSec: track.offsetMs / 1000,
Expand Down Expand Up @@ -1109,7 +1110,7 @@ export function VirtualPreview({
} else {
trackGainDb += duck.db;
}
const trackGainNode = audioTrackGainNodesRef.current.get(track.id);
const trackGainNode = audioTrackGainNodesRef.current.get(fragment.id);
if (trackGainNode) {
trackGainNode.gain.value = audioGainScalar(trackGainDb) * fade;
if (el.volume !== 1) el.volume = 1;
Expand Down
16 changes: 16 additions & 0 deletions src/lib/ai-edition/document/audioTracks.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -239,6 +239,22 @@ export function collapseTracksToPills(tracks: AxcutAudioTrack[]): AxcutAudioTrac
});
}

/**
* Every take once, keyed by the id of the fragment that plays it: its first. The export and the
* preview both play a take from there, at 1x, across the whole pill. Played fragment by fragment,
* each one its own slice, a bed was back on the ruler under a speed region: silent at the end of a
* slowed clip, then behind; on top of itself through a sped-up one.
*/
export function takesByHeadFragment(tracks: AxcutAudioTrack[]): Map<string, AxcutAudioTrack> {
const heads = new Map<string, string>();
for (const track of [...tracks].sort((a, b) => a.startMs - b.startMs)) {
if (!heads.has(trackGroupId(track))) heads.set(trackGroupId(track), track.id);
}
return new Map(
collapseTracksToPills(tracks).map((pill) => [heads.get(trackGroupId(pill)) ?? pill.id, pill]),
);
}

/** Drop every fragment of a track, and its asset when nothing else needs it.
*
* An imported audio asset is only ever reachable through its track — audio is
Expand Down
46 changes: 46 additions & 0 deletions src/native/sceneDescription.test.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2941,6 +2941,52 @@ describe("buildSceneDescription.audioTracks", () => {
expect(entry.startSec).toBeCloseTo(3, 6);
});

it("plays a bed spanning a slowed clip as one take, on the clock, not on the ruler", () => {
// The document stores a track as one fragment per clip it covers, each fragment's offset
// advanced by the ruler time before it. Mixed fragment by fragment, the bed under a 0.5x
// clip played its 4 ruler seconds of file over 8 output seconds, went silent for the other
// 4, then came back 4 seconds behind the music it should have been playing.
const screen = makeAsset({ id: "scr", originalPath: "/screen.mp4", durationSec: 20 });
const fragment = { ...track, trackId: "trk1", startMs: 0, offsetMs: 0, endMs: 4000 };
const doc = makeDoc({
assets: [screen, audioAsset],
clips: [
makeClip({
id: "c1",
assetId: "scr",
sourceStartSec: 0,
sourceEndSec: 4,
timelineStartSec: 0,
timelineEndSec: 4,
}),
makeClip({
id: "c2",
assetId: "scr",
sourceStartSec: 4,
sourceEndSec: 8,
timelineStartSec: 4,
timelineEndSec: 8,
}),
],
legacyEditor: { speedRegions: [{ id: "s1", startMs: 0, endMs: 4000, speed: 0.5 }] },
audioTracks: [
{ ...fragment, clipId: "c1", sourceStartSec: 0, sourceEndSec: 4 },
{
...fragment,
id: "trk1-b",
clipId: "c2",
sourceStartSec: 4,
sourceEndSec: 8,
startMs: 4000,
endMs: 8000,
offsetMs: 4000,
},
],
});
const entries = buildSceneDescription(doc).audioTracks;
expect(entries.map((e) => [e.startSec, e.trimStartSec, e.trimEndSec])).toEqual([[0, 0, 8]]);
});

it("drops a track whose asset has no resolvable path", () => {
const doc = makeDoc({ assets: [], audioTracks: [track] });
expect(buildSceneDescription(doc).audioTracks).toEqual([]);
Expand Down
31 changes: 12 additions & 19 deletions src/native/sceneDescription.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,7 @@ import {
getCaptionTranslations,
} from "@/lib/ai-edition/captions";
import { clickSoundForDocument } from "@/lib/ai-edition/clickSound";
import { collapseTracksToPills, trackGroupId } from "@/lib/ai-edition/document/audioTracks";
import { takesByHeadFragment } from "@/lib/ai-edition/document/audioTracks";
import { createId } from "@/lib/ai-edition/document/ids";
import { isFormatFillActive, pickOutputDims } from "@/lib/ai-edition/document/outputFormat";
import {
Expand Down Expand Up @@ -884,24 +884,22 @@ export function buildSceneDescription(
// question, and it does not depend on the track.
// Placed once: the projection below counts them, so a track after a pause lands where
const removed = removedRawSpans(projectedClips, document.timeline.trimRanges);
// The take's pills, keyed by group. A voiceover is walked ONCE per pill and never per
// stored fragment: the document keeps one fragment per clip a take covers, so walking
// them separately would emit overlapping entries and `overlay_track_pcm` sums with `+=`
// at an absolute offset — the export would contain the take playing on top of itself.
const voiceoverPills = new Map(
collapseTracksToPills(document.audioTracks)
.filter((pill) => pill.kind === "voiceover" && !pill.loop)
.map((pill) => [trackGroupId(pill), pill]),
);
// Every take is mixed ONCE, from its head fragment, as its whole pill — never per stored
// fragment. The document keeps one fragment per clip a take covers: a voiceover walked per
// fragment would play on top of itself (`overlay_track_pcm` sums with `+=`), and a bed laid
// per fragment, each at its projected head with its ruler-advanced offset, went silent at the
// end of a slowed clip and came back late (see `takesByHeadFragment`).
const takes = takesByHeadFragment(document.audioTracks);
// Click hits for the "mouse clicks" toggle, placed on the programme this export assembles.
// Synchronous: the cues were read from the take's sidecar when the sound was switched on.
// Its row lives under "Show cursor", like the click impact: a hidden cursor makes no sound.
const clickSound = clickSoundForDocument(document, {
clickSound: settings.cursorShow && settings.cursor.clickSound,
clickSoundGainDb: settings.cursor.clickSoundGainDb,
});
const audioTracks = document.audioTracks.flatMap((track) => {
if (track.muted) return [];
const audioTracks = document.audioTracks.flatMap((fragment) => {
const track = takes.get(fragment.id);
if (!track || track.muted) return [];
const asset = assetById.get(track.assetId);
if (!asset?.originalPath) return [];
const sourceDurationSec = asset.durationSec ?? track.durationSec;
Expand Down Expand Up @@ -957,17 +955,12 @@ export function buildSceneDescription(
// #560 refuses that combination outright, and inventing semantics for something
// about to be banned would be the worse answer.
if (track.kind === "voiceover" && !track.loop) {
const groupId = trackGroupId(track);
const pill = voiceoverPills.get(groupId);
// Emitted from the group's HEAD fragment only — every other fragment of the same
// take is already covered by the pill's own walk.
if (!pill || pill.id !== track.id) return [];
const rawSpanSec = Math.max(0, pill.endMs / 1000 - pill.startMs / 1000);
const rawSpanSec = Math.max(0, track.endMs / 1000 - track.startMs / 1000);
// Unprobed assets have no real duration to cap with; the RAW span is how much
// file the take covers, which is the honest fallback once the cuts are taken out.
const voWindowSec =
sourceDurationSec > 0 ? Math.max(0, sourceDurationSec - offsetSec) : rawSpanSec;
const kept = takeProgramme(pill, removed)
const kept = takeProgramme(track, removed)
.filter((piece) => piece.kind === "play")
.map((piece) => ({
...base,
Expand Down
Loading