From 2107dc16673f724a08f9ec6ae305ecb6f5cd043c Mon Sep 17 00:00:00 2001 From: Calvin Grunewald Date: Sun, 25 Oct 2026 14:34:02 -0700 Subject: [PATCH] Fixes from using tiers 1 and 2 by hand Screen recording - Clicks are counted, not polled, so a trackpad tap is never missed. - Click times line up with the video: screencapture starts 0.25-1.1 s late, measured from the take's birth time instead of a fixed guess. - Missing Screen Recording permission is said at Start, and asked for. Playback - The clock waits for each stream's first frame, not the paused still: heavy effects no longer start a third of a second behind. - ctl status reports how playback kept up. Captions - Whisper's tokens joined into words; unspaced writing keeps each token's own time. Captions never run past the clip heard. - Japanese and Chinese grouped into word-like pieces (kana to kanji, punctuation), joined with U+200B so karaoke lights each one. - Korean is spaced writing. - Titles in writing their font lacks are set in one font that has it, not a fallback per run (kanji came out in a Chinese serif). - Word and letter pieces with no space between meet halfway. - Drawn titles are versioned, so cached drawings are made again. Timeline - Keyframe marks: click to go to one, drag to move it (one undo step). - Ruler labels show half seconds; clip names clear the transition tab. Co-Authored-By: Claude Opus 5.5 --- README.md | 11 +- crates/splice-app/src/editor.rs | 40 ++++++ crates/splice-app/src/editor/agent.rs | 1 + crates/splice-app/src/editor/effects.rs | 12 +- crates/splice-app/src/player.rs | 63 ++++++++- crates/splice-app/src/recorder.rs | 100 +++++++++----- crates/splice-app/src/speech.rs | 94 +++++++++++-- crates/splice-app/src/timeline.rs | 38 ++++- crates/splice-app/src/titles.rs | 112 ++++++++++++++- crates/splice-core/src/captions.rs | 176 ++++++++++++++++++++---- crates/splice-core/src/look.rs | 32 +++++ crates/splice-core/src/time.rs | 15 +- skill/splice/SKILL.md | 7 +- 13 files changed, 613 insertions(+), 88 deletions(-) diff --git a/README.md b/README.md index d2372d5..4690c7e 100644 --- a/README.md +++ b/README.md @@ -111,7 +111,9 @@ make install-app # Splice.app in /Applications - **Captions** (Edit → Add Captions from Speech, or the clip's Sound section): Splice hears what is said, on your Mac (whisper.cpp, `brew install whisper-cpp`; a 148 MB speech model downloads once), and lays - captions over it, each word lighting up as it is said. Each caption is a + captions over it, each word lighting up as it is said. Any language whisper + knows works; Japanese and Chinese, written without spaces, are grouped into + words to light up and set in one font that has them. Each caption is a title you can edit. `.srt` and `.vtt` files load too. - **Colour** shelf: full-frame colours for backgrounds and title cards, a half-black tint, and light leaks that wash warm or cool light over a shot. @@ -125,6 +127,8 @@ make install-app # Splice.app in /Applications the playhead; set another somewhere else and it glides between them (linear, smooth, ease in, ease out, or hold). Dragging a keyframed picture on the stage keys it where the playhead is. ⌥K keys place and size at once. + A selected clip shows its keyframes as marks along its foot on the + timeline: click one to go to it, drag it to change when it happens. - **Transition:** the way into a clip from the one before it: dissolve, dip to black or white, wipe or push four ways, or zoom, centred on the cut, with a length. A tab on the timeline marks each one. ⇧⌘T adds a dissolve @@ -290,7 +294,10 @@ recorder, so there is no OBS and no extra install. - The take comes into the media panel. In Story view it goes into the chosen beat instead. - The first recording asks for Screen Recording permission. It is granted to - whatever launched Splice: the app, or your terminal. + whatever launched Splice: the app, or your terminal. Without it, Start says + so at once. A rebuilt app is new to macOS (it is signed only locally), so + after `make install-app` turn the permission off and on again in System + Settings → Privacy & Security → Screen Recording. ## Projects diff --git a/crates/splice-app/src/editor.rs b/crates/splice-app/src/editor.rs index af5f43e..9368e69 100644 --- a/crates/splice-app/src/editor.rs +++ b/crates/splice-app/src/editor.rs @@ -102,6 +102,13 @@ enum Drag { left: bool, before: Project, }, + /// A chosen clip's keyframes at one moment, moving along it. + Keyframe { + id: ClipId, + at: Time, + before: Project, + moved: bool, + }, /// The end of a story beat, making it longer or shorter. BeatEnd { id: splice_core::story::BeatId, @@ -974,6 +981,20 @@ impl Editor { window.focus(&self.focus); let p = self.local(event.position); let shift = event.modifiers.shift || event.modifiers.platform; + // A keyframe's mark: the playhead goes to it, and a drag moves it. + if !shift + && let Some((id, at)) = timeline::keyframe_at(&self.doc.project, &self.view, &self.selection, p) + && let Some(start) = self.doc.project.clip(id).map(|c| c.start) + { + self.seek(start + at, cx); + self.drag = Drag::Keyframe { + id, + at, + before: self.doc.project.clone(), + moved: false, + }; + return cx.notify(); + } match timeline::hit(&self.doc.project, &self.view, p) { Hit::Ruler => { self.drag = Drag::Scrub; @@ -1142,6 +1163,22 @@ impl Editor { project.set_beat_duration(*id, t - start); self.doc.project = project; } + Drag::Keyframe { id, at, before, moved } => { + let Some(clip) = before.clip(*id) else { return }; + let frame = before.frame(); + let to = ((view.t(p.x) - clip.start).clamp(0, clip.duration) + frame / 2) / frame * frame; + if !*moved && (to - *at).abs() < frame { + return; + } + *moved = true; + let mut project = before.clone(); + if let Some(c) = project.clip_mut(*id) { + c.look.keys.move_moment(*at, to); + } + let start = clip.start; + self.doc.project = project; + self.seek(start + to, cx); + } Drag::Trim { id, left, before } => { let mut t = view.t(p.x); let mut points = before.snap_points(&[*id]); @@ -1170,6 +1207,9 @@ impl Editor { Drag::Move { before, moved: true, .. } + | Drag::Keyframe { + before, moved: true, .. + } | Drag::Trim { before, .. } | Drag::BeatEnd { before, .. } => { self.doc.commit(before); diff --git a/crates/splice-app/src/editor/agent.rs b/crates/splice-app/src/editor/agent.rs index 2fdb711..6efc14d 100644 --- a/crates/splice-app/src/editor/agent.rs +++ b/crates/splice-app/src/editor/agent.rs @@ -977,6 +977,7 @@ impl Editor { "duration": s(project.duration()), "playhead": s(self.player.position), "playing": self.player.is_playing(), + "playback": self.player.keeping, "selection": self.selection.iter().map(|c| c.0).collect::>(), "media": project.media.len(), "tracks": project.tracks.len(), diff --git a/crates/splice-app/src/editor/effects.rs b/crates/splice-app/src/editor/effects.rs index 78a2715..9b11ed1 100644 --- a/crates/splice-app/src/editor/effects.rs +++ b/crates/splice-app/src/editor/effects.rs @@ -470,20 +470,20 @@ impl Editor { editor.captioning.remove(&id); // Where the clip is now: it may have moved while it was heard. let now = editor.doc.project.clip(id).cloned(); - let heard: anyhow::Result<(Vec, Time, f64)> = match (heard, now) { + let heard: anyhow::Result<(Vec, Time, f64, Time)> = match (heard, now) { (Err(e), _) => Err(e), (Ok(_), None) => Err(anyhow::anyhow!("the clip was removed while it was heard")), (Ok(_), Some(c)) if c.source_in != from || c.speed != clip.speed || c.reverse => Err( anyhow::anyhow!("the clip was trimmed or retimed while it was heard; caption it again"), ), - (Ok(h), Some(c)) => Ok((h, c.start, c.speed)), + (Ok(h), Some(c)) => Ok((h, c.start, c.speed, c.end())), }; let answer = match heard { - Ok((heard, _, _)) if heard.is_empty() => { + Ok((heard, ..)) if heard.is_empty() => { editor.say_error("No speech heard in that clip"); serde_json::json!({ "ok": false, "error": "no speech heard" }) } - Ok((heard, start, speed)) => { + Ok((heard, start, speed, end)) => { // From source time to timeline time, at the clip's speed. let at = |t: Time| start + (t as f64 / speed).round() as Time; let words: Vec = heard @@ -494,6 +494,10 @@ impl Editor { end: at(w.end), }) .collect(); + let layout = splice_core::captions::Layout { + until: Some(end), + ..layout + }; let ids = editor.edit(cx, |p| p.add_captions(&words, &layout)); editor.selection = ids.clone(); editor.say(format!( diff --git a/crates/splice-app/src/player.rs b/crates/splice-app/src/player.rs index 1920d23..b6d08c1 100644 --- a/crates/splice-app/src/player.rs +++ b/crates/splice-app/src/player.rs @@ -8,7 +8,7 @@ use crate::library::Library; use gpui::RenderImage; use splice_core::compose; use splice_core::model::{Clip, ClipId, Media, MediaId, MediaKind, Project, Source}; -use splice_core::time::{SECOND, Time, floor_to_frame}; +use splice_core::time::{self, SECOND, Time, floor_to_frame}; use std::collections::{HashMap, HashSet}; use std::sync::Arc; use std::sync::mpsc::{Receiver, Sender, channel}; @@ -90,6 +90,23 @@ pub fn backdrop_size(full: (u32, u32)) -> (u32, u32) { ((full.0 / 4).max(16) & !1, (full.1 / 4).max(16) & !1) } +/// How playback has kept up since it last started: for `ctl status`, and for +/// telling a heavy stack of effects from a smooth one. +#[derive(Clone, Copy, Debug, Default, serde::Serialize)] +pub struct Keeping { + /// Seconds played. + pub seconds: f64, + /// New frames drawn, all clips together. + pub frames: u64, + /// Decoded frames passed over because a later one was already due. + pub skipped: u64, + /// Looks at the clock that found a clip's picture over two frames behind. + pub behind: u64, + pub looks: u64, + /// The furthest any clip's picture fell behind, in seconds. + pub worst: f64, +} + /// What the editor should do after a tick. #[derive(Default)] pub struct Tick { @@ -112,6 +129,10 @@ pub struct Player { still_tx: Sender, still_rx: Receiver, retired: Vec>, + /// Clips whose stream has given a frame since playback was asked for: + /// the clock waits for them, not for the stills shown while paused. + flowing: HashSet, + pub keeping: Keeping, } impl Player { @@ -128,6 +149,8 @@ impl Player { still_tx, still_rx, retired: Vec::new(), + flowing: HashSet::new(), + keeping: Keeping::default(), } } @@ -145,6 +168,8 @@ impl Player { self.position = 0; } self.state = State::Priming(Instant::now()); + self.flowing.clear(); + self.keeping = Keeping::default(); } pub fn pause(&mut self) { @@ -157,6 +182,7 @@ impl Player { self.position = t.max(0); if self.is_playing() { self.state = State::Priming(Instant::now()); + self.flowing.clear(); } } @@ -325,6 +351,24 @@ impl Player { let due = if clip.reverse { source - half } else { source + half }; if let Some((t, size, image)) = self.pull(clip.id, due) { self.show(clip.id, media.id, filters.clone(), t, size, image); + self.flowing.insert(clip.id); + self.keeping.frames += 1; + } + if let State::Playing { from, .. } = self.state { + self.keeping.seconds = time::secs(position - from); + self.keeping.looks += 1; + if let Some(shown) = self.shown.get(&clip.id) { + let gap = if clip.reverse { + shown.source - due + } else { + due - shown.source + }; + let late = time::secs(gap) / clip.speed.max(0.01); + if late > 2.0 * time::secs(frame) { + self.keeping.behind += 1; + } + self.keeping.worst = self.keeping.worst.max(late); + } } } else { let current = self @@ -387,7 +431,14 @@ impl Player { } if let State::Priming(since) = self.state { - let ready = stack.iter().all(|w| self.shown.contains_key(&w.clip.id)); + let ready = stack.iter().all(|w| { + let streamed = w.media.kind == MediaKind::Video && !w.clip.frozen; + if streamed { + self.flowing.contains(&w.clip.id) + } else { + self.shown.contains_key(&w.clip.id) + } + }); if ready || now - since > PRIME_TIMEOUT { self.state = State::Playing { anchor: now, @@ -480,13 +531,18 @@ impl Player { let size = stream.size; let reverse = stream.reverse; let mut latest = None; + let mut skipped = 0; loop { if stream.pending.is_none() { let t = stream.decoder.next_time(); stream.pending = stream.decoder.try_next().map(|image| (t, image)); } match stream.pending.take() { - Some((t, image)) if (if reverse { t >= due } else { t <= due }) => latest = Some((t, image)), + Some((t, image)) if (if reverse { t >= due } else { t <= due }) => { + if latest.replace((t, image)).is_some() { + skipped += 1; + } + } Some(early) => { stream.pending = Some(early); break; @@ -494,6 +550,7 @@ impl Player { None => break, } } + self.keeping.skipped += skipped; latest.map(|(t, image)| (t, size, image)) } diff --git a/crates/splice-app/src/recorder.rs b/crates/splice-app/src/recorder.rs index 9bd1ef2..b2650cd 100644 --- a/crates/splice-app/src/recorder.rs +++ b/crates/splice-app/src/recorder.rs @@ -155,6 +155,8 @@ pub struct Recording { child: Child, pub path: PathBuf, pub started: Instant, + /// The same moment by the clock files are stamped with. + begun: std::time::SystemTime, pub source: Source, /// Watching the mouse for clicks, until told to stop. clicks: Option<(Arc, std::thread::JoinHandle>)>, @@ -199,9 +201,11 @@ struct Point { #[link(name = "CoreGraphics", kind = "framework")] unsafe extern "C" { - fn CGEventSourceButtonState(state: u32, button: u32) -> bool; + fn CGEventSourceCounterForEventType(state: u32, event: u32) -> u32; fn CGEventCreate(source: *const std::ffi::c_void) -> *mut std::ffi::c_void; fn CGEventGetLocation(event: *mut std::ffi::c_void) -> Point; + fn CGPreflightScreenCaptureAccess() -> bool; + fn CGRequestScreenCaptureAccess() -> bool; } #[link(name = "CoreFoundation", kind = "framework")] @@ -222,11 +226,12 @@ fn mouse() -> Option<(f64, f64)> { } } -/// Whether the left button is down (no permission needed to ask). -fn pressed() -> bool { +/// How many times the left button has gone down since the login session +/// began (no permission needed to ask). +fn presses() -> u32 { const COMBINED: u32 = 0; - const LEFT: u32 = 0; - unsafe { CGEventSourceButtonState(COMBINED, LEFT) } + const LEFT_MOUSE_DOWN: u32 = 1; + unsafe { CGEventSourceCounterForEventType(COMBINED, LEFT_MOUSE_DOWN) } } /// The part of the screen `source` records, in screen points from the main @@ -295,19 +300,31 @@ fn window_bounds(id: u32) -> Option<(f64, f64, f64, f64)> { None } -/// screencapture starts recording about this long after it is started. -const LATE: f64 = 0.2; +/// When the file says nothing, how long after it is started screencapture +/// begins. +const LATE: f64 = 0.5; + +/// How long after `started` the take's first frame was: screencapture +/// takes a while to begin (a quarter second to more than one, measured), and +/// its file is born then. +fn lateness(path: &std::path::Path, started: std::time::SystemTime) -> f64 { + std::fs::metadata(path) + .and_then(|m| m.created()) + .ok() + .and_then(|born| born.duration_since(started).ok()) + .map(|d| d.as_secs_f64()) + .filter(|&d| d < 5.0) + .unwrap_or(LATE) +} /// Clicks in the recording's own time, without the click that stopped it /// (the Stop button, or Splice in the Dock) in its last half second. -fn settle(clicks: Vec, ended: f64) -> Vec { +/// `late` is how long after the clicks' clock the take begins. +fn settle(clicks: Vec, ended: f64, late: f64) -> Vec { clicks .into_iter() - .filter(|c| c.t < ended - 0.5) - .map(|c| Click { - t: (c.t - LATE).max(0.0), - ..c - }) + .filter(|c| c.t < ended - 0.5 && c.t >= late) + .map(|c| Click { t: c.t - late, ..c }) .collect() } @@ -315,13 +332,14 @@ fn settle(clicks: Vec, ended: f64) -> Vec { fn watch(area: (f64, f64, f64, f64), started: Instant, stop: Arc) -> Vec { let (ax, ay, aw, ah) = area; let mut clicks = Vec::new(); - let mut was = pressed(); + // Counting presses rather than looking at the button: a tap on a + // trackpad can go down and up between two looks. + let mut seen = presses(); while !stop.load(Ordering::Relaxed) { - let now = pressed(); - if now - && !was - && let Some((x, y)) = mouse() - { + let count = presses(); + let new = count != seen; + seen = count; + if new && let Some((x, y)) = mouse() { let (u, v) = ((x - ax) / aw.max(1.0), (y - ay) / ah.max(1.0)); if (0.0..=1.0).contains(&u) && (0.0..=1.0).contains(&v) { clicks.push(Click { @@ -331,7 +349,6 @@ fn watch(area: (f64, f64, f64, f64), started: Instant, stop: Arc) -> }); } } - was = now; std::thread::sleep(std::time::Duration::from_millis(8)); } clicks @@ -355,19 +372,11 @@ impl Recording { libc_kill(self.child.id() as i32, 2); } } - if let Some((stop, watcher)) = self.clicks.take() { - let ended = self.started.elapsed().as_secs_f64(); + let ended = self.started.elapsed().as_secs_f64(); + let clicks = self.clicks.take().and_then(|(stop, watcher)| { stop.store(true, Ordering::Relaxed); - if let Ok(clicks) = watcher.join() - && let clicks = settle(clicks, ended) - && !clicks.is_empty() - { - let _ = std::fs::write( - clicks_file(&self.path), - serde_json::to_string(&clicks).unwrap_or_default(), - ); - } - } + watcher.join().ok() + }); let output = self.child.wait_with_output()?; let ok = std::fs::metadata(&self.path).is_ok_and(|m| m.len() > 0); if !ok { @@ -380,6 +389,15 @@ impl Recording { why }); } + if let Some(clicks) = clicks + && let clicks = settle(clicks, ended, lateness(&self.path, self.begun)) + && !clicks.is_empty() + { + let _ = std::fs::write( + clicks_file(&self.path), + serde_json::to_string(&clicks).unwrap_or_default(), + ); + } Ok(self.path) } } @@ -396,6 +414,16 @@ pub fn takes() -> PathBuf { /// Starts recording `source`. pub fn start(source: Source, options: &Options) -> Result { + // Without permission, screencapture writes nothing, and that would only + // show when the take ends. Asking puts Splice in the list (and, the + // first time, brings up the system's prompt). + if !unsafe { CGPreflightScreenCaptureAccess() } { + unsafe { CGRequestScreenCaptureAccess() }; + anyhow::bail!( + "Splice needs Screen Recording permission: turn it on in System Settings → Privacy & Security → \ + Screen Recording, then reopen Splice" + ); + } let dir = takes(); std::fs::create_dir_all(&dir)?; let stamp = { @@ -442,6 +470,7 @@ pub fn start(source: Source, options: &Options) -> Result { .spawn() .context("starting screencapture")?; let started = Instant::now(); + let begun = std::time::SystemTime::now(); // Clicks are noted for zooming in on later. let clicks = area(&source).and_then(|area| { let stop = Arc::new(AtomicBool::new(false)); @@ -456,6 +485,7 @@ pub fn start(source: Source, options: &Options) -> Result { child, path, started, + begun, source, clicks, }) @@ -475,9 +505,11 @@ mod tests { y: 0.99, }, ]; - let kept = settle(clicks, 10.0); + let kept = settle(clicks, 10.0, 0.4); assert_eq!(kept.len(), 1); - assert!((kept[0].t - 0.8).abs() < 1e-9, "in the recording's own time"); + assert!((kept[0].t - 0.6).abs() < 1e-9, "in the recording's own time"); + let early = settle(vec![Click { t: 0.1, x: 0.5, y: 0.5 }], 10.0, 0.4); + assert!(early.is_empty(), "a click before the first frame is not in the take"); } #[test] diff --git a/crates/splice-app/src/speech.rs b/crates/splice-app/src/speech.rs index 373e975..4cb5740 100644 --- a/crates/splice-app/src/speech.rs +++ b/crates/splice-app/src/speech.rs @@ -6,6 +6,7 @@ use crate::ffmpeg; use crate::session::dirs; use anyhow::{Context, Result, bail}; use sha2::{Digest, Sha256}; +use splice_core::captions::unspaced; use splice_core::time::{Time, secs}; use std::path::{Path, PathBuf}; use std::process::{Command, Stdio}; @@ -106,7 +107,7 @@ pub fn transcribe(path: &Path, from: Time, length: Time) -> Result> { .arg(&model) .arg("-f") .arg(&wav) - .args(["-l", "auto", "-ml", "1", "-sow", "-oj", "-np", "-of"]) + .args(["-l", "auto", "-ml", "1", "-oj", "-np", "-of"]) .arg(work.join("words")) .stdout(Stdio::null()) .stderr(Stdio::piped()) @@ -118,25 +119,62 @@ pub fn transcribe(path: &Path, from: Time, length: Time) -> Result> { let text = String::from_utf8_lossy(&output.stderr); bail!("whisper could not hear it: {}", text.lines().last().unwrap_or("")); } - parse(&json?) + // Whisper's last times can run past the end of what it heard. + Ok(parse(&json?)? + .into_iter() + .filter(|w| w.start < length) + .map(|w| Heard { + end: w.end.min(length), + ..w + }) + .collect()) } -/// whisper-cli's JSON, one segment a word (`-ml 1 -sow`), as words. +/// whisper-cli's JSON, one segment a token (`-ml 1`), as words: a token +/// without a space before it is the rest of the word before ("every" + +/// "body", "work" + "."), except in writing without spaces (Japanese, +/// Chinese), where each token keeps its own time. fn parse(json: &str) -> Result> { let value: serde_json::Value = serde_json::from_str(json)?; let segments = value["transcription"].as_array().cloned().unwrap_or_default(); - Ok(segments - .iter() - .filter_map(|s| { - let text = s["text"].as_str()?.trim().trim_matches('"').to_string(); + let mut words: Vec = Vec::new(); + for s in &segments { + let (Some(text), Some(a), Some(z)) = ( + s["text"].as_str(), + s["offsets"]["from"].as_i64(), + s["offsets"]["to"].as_i64(), + ) else { + continue; + }; + if text.trim().is_empty() { + continue; + } + let (start, end) = (a * 1000, z.max(a) * 1000); + let first = text.chars().next().unwrap_or(' '); + let rest = !first.is_whitespace() + && words.last().is_some_and(|w| { + let last = w.text.chars().last().unwrap_or(' '); + !(unspaced(last) && unspaced(first)) + }); + match words.last_mut() { + Some(word) if rest => { + word.text.push_str(text); + word.end = word.end.max(end); + } + _ => words.push(Heard { + text: text.trim().to_string(), + start, + end, + }), + } + } + Ok(words + .into_iter() + .filter_map(|w| { + let text = w.text.trim().trim_matches('"').to_string(); // Whisper marks what it cannot make out in brackets: [MUSIC], (laughs). let noise = text.starts_with('[') || text.starts_with('('); - let (a, z) = (s["offsets"]["from"].as_i64()?, s["offsets"]["to"].as_i64()?); - (!text.is_empty() && !noise).then_some(Heard { - text, - start: a * 1000, - end: z.max(a) * 1000, - }) + (!text.is_empty() && !noise).then_some(Heard { text, ..w }) }) .collect()) } @@ -145,6 +183,36 @@ fn parse(json: &str) -> Result> { mod tests { use super::*; + #[test] + fn tokens_join_into_words_except_where_words_are_unspaced() { + let json = r#"{"transcription":[ + {"offsets":{"from":0,"to":100},"text":" Today"}, + {"offsets":{"from":100,"to":200},"text":","}, + {"offsets":{"from":200,"to":300},"text":" I"}, + {"offsets":{"from":300,"to":400},"text":"'m"}, + {"offsets":{"from":400,"to":500},"text":" [MUS"}, + {"offsets":{"from":500,"to":600},"text":"IC]"}, + {"offsets":{"from":600,"to":700},"text":" 今日は"}, + {"offsets":{"from":700,"to":800},"text":"動画"}, + {"offsets":{"from":800,"to":900},"text":"。"}]}"#; + let words: Vec<(String, Time, Time)> = parse(json) + .unwrap() + .into_iter() + .map(|w| (w.text, w.start / 1000, w.end / 1000)) + .collect(); + assert_eq!( + words, + vec![ + ("Today,".into(), 0, 200), + ("I'm".into(), 200, 400), + ("今日は".into(), 600, 700), + // Unspaced tokens stay apart; captions group them. + ("動画".into(), 700, 800), + ("。".into(), 800, 900), + ] + ); + } + #[test] fn reads_whisper_words() { let json = r#"{"transcription":[ diff --git a/crates/splice-app/src/timeline.rs b/crates/splice-app/src/timeline.rs index b69c150..16feddc 100644 --- a/crates/splice-app/src/timeline.rs +++ b/crates/splice-app/src/timeline.rs @@ -171,6 +171,34 @@ pub fn hit(project: &Project, view: &View, p: Point) -> Hit { Hit::Empty } +/// The keyframe mark under `p`, among the chosen clips' (they show their +/// marks along their foot): the clip, and the keyframe's clip time. +pub fn keyframe_at(project: &Project, view: &View, chosen: &[ClipId], p: Point) -> Option<(ClipId, Time)> { + let n = project.tracks.len(); + for &id in chosen { + let (Some(clip), Some(slot)) = (project.clip(id), project.slot(id)) else { + continue; + }; + let y = view.track_y(slot.track, n) + ROW - 9.0; + if p.y < y - 3.0 || p.y > y + 10.0 { + continue; + } + let near = clip + .look + .keys + .moments() + .into_iter() + .filter(|k| (0..=clip.duration).contains(k)) + .map(|k| (k, (view.x(clip.start + k) - p.x).abs())) + .filter(|(_, d)| *d <= 6.0) + .min_by(|a, b| a.1.total_cmp(&b.1)); + if let Some((k, _)) = near { + return Some((id, k)); + } + } + None +} + /// Where a transition's tab is drawn on a row at `y` (timeline coordinates): /// across the stretch it takes, at least wide enough to hold. pub fn transition_tab(view: &View, cut: &splice_core::transition::Cut, y: f32) -> Bounds { @@ -397,7 +425,15 @@ pub fn paint(b: Bounds, frame: Frame, window: &mut Window, cx: &mut App) if art.outdated && x1 - x0 > 16.0 { rect(window, area(x1 - 11.0, y + 5.0, 6.0, 6.0), Theme::accent(), 3.0); } - let label_x = x0.max(GUTTER) + 6.0; + let mut label_x = x0.max(GUTTER) + 6.0; + // Clear of the tab of the transition into it. + if let Some(cut) = project.cut_into(clip.id) { + let tab = transition_tab(&view, &cut, 0.0); + let right = f32::from(tab.origin.x + tab.size.width); + if right + 4.0 > label_x { + label_x = right + 4.0; + } + } // Under the ruler, the name stays on the part still in view. let y = y.max(top).min(y + ROW - 21.0); if x1 - label_x > 24.0 { diff --git a/crates/splice-app/src/titles.rs b/crates/splice-app/src/titles.rs index 0eafd1d..86c939a 100644 --- a/crates/splice-app/src/titles.rs +++ b/crates/splice-app/src/titles.rs @@ -3,6 +3,7 @@ //! the picture the viewer shows is the very picture export overlays. use cosmic_text::{Attrs, Buffer, Color, Family, FontSystem, Metrics, Shaping, Style, SwashCache, Weight}; +use splice_core::captions::WORD_BREAK; use splice_core::export::{Rendered, Resolve}; use splice_core::look::Mask; use splice_core::model::{Align, Clip, Generator, Media, Title}; @@ -171,6 +172,44 @@ fn draw_title(title: &Title, canvas: (u32, u32)) -> Picture { } } +/// The family to set `text` in: `font`, unless it has no letters for the +/// text's writing (most Latin fonts have no kanji or hangul). Then one font +/// for all of it, chosen by what it holds; left to fallback, each run of a +/// script finds its own, so kanji beside kana could come out in a Chinese +/// serif. +fn family_for(system: &mut FontSystem, font: &str, text: &str, bold: bool) -> String { + let script = |range: std::ops::RangeInclusive| text.chars().any(|c| range.contains(&(c as u32))); + let wanted = if script(0x3040..=0x30ff) { + "Hiragino Sans" + } else if script(0xac00..=0xd7af) || script(0x1100..=0x11ff) { + "Apple SD Gothic Neo" + } else if script(0x3400..=0x9fff) { + "PingFang SC" + } else if script(0x0e00..=0x0e7f) { + "Thonburi" + } else { + return font.to_string(); + }; + let Some(sample) = text.chars().find(|c| !c.is_ascii() && !c.is_whitespace()) else { + return font.to_string(); + }; + let weight = if bold { Weight::BOLD } else { Weight::NORMAL }; + let has = |system: &mut FontSystem, family: &str| { + let id = system.db().query(&cosmic_text::fontdb::Query { + families: &[cosmic_text::fontdb::Family::Name(family)], + weight, + ..Default::default() + }); + id.and_then(|id| system.get_font(id, weight)) + .is_some_and(|f| f.as_swash().charmap().map(sample) != 0) + }; + if has(system, font) || !has(system, wanted) { + font.to_string() + } else { + wanted.to_string() + } +} + /// Draws a title as its parts (see `Parts`). pub fn draw_title_parts(title: &Title, canvas: (u32, u32)) -> Parts { let size = (title.size * canvas.1 as f32).clamp(4.0, 2000.0); @@ -183,8 +222,9 @@ pub fn draw_title_parts(title: &Title, canvas: (u32, u32)) -> Parts { let mut guard = fonts().lock().unwrap(); let Fonts { system, cache } = &mut *guard; + let family = family_for(system, &title.font, text, title.bold); let attrs = Attrs::new() - .family(Family::Name(&title.font)) + .family(Family::Name(&family)) .weight(if title.bold { Weight::BOLD } else { Weight::NORMAL }) .style(if title.italic { Style::Italic } else { Style::Normal }); let align = match title.align { @@ -253,8 +293,20 @@ pub fn draw_title_parts(title: &Title, canvas: (u32, u32)) -> Parts { *word += 1; } }; + let mut after = 0; for glyph in run.glyphs.iter() { - let blank = run.text.get(glyph.start..glyph.end).is_none_or(|t| t.trim().is_empty()); + let text = run.text.get(glyph.start..glyph.end); + let blank = text.is_none_or(|t| t.trim().trim_matches(WORD_BREAK).is_empty()); + // A break between words of unspaced writing draws nothing, + // and may leave no glyph of its own. + let broken = run + .text + .get(after.min(glyph.start)..glyph.start) + .is_some_and(|t| t.contains(WORD_BREAK)); + after = glyph.end; + if broken { + close(&mut current, &mut units, &mut word); + } if blank { close(&mut current, &mut units, &mut word); continue; @@ -275,6 +327,21 @@ pub fn draw_title_parts(title: &Title, canvas: (u32, u32)) -> Parts { }); } } + // Neighbours whose margins reach into each other (letters, or words + // with no space between) meet halfway, so one lit does not light the + // edge of the next. + for list in [&mut units.words, &mut units.letters] { + for i in 1..list.len() { + let (a, b) = (list[i - 1].rect, list[i].rect); + let (a_right, b_right) = (a[0] + a[2], b[0] + b[2]); + if a[1] == b[1] && b[0] > a[0] && b[0] < a_right { + let middle = (b[0] + a_right) / 2; + list[i - 1].rect[2] = middle - a[0]; + list[i].rect[0] = middle; + list[i].rect[2] = b_right - middle; + } + } + } units }; @@ -370,8 +437,9 @@ pub fn text_picture( let text = if text.is_empty() { " " } else { text }; let mut guard = fonts().lock().unwrap(); let Fonts { system, cache } = &mut *guard; + let family = family_for(system, font, text, bold); let attrs = Attrs::new() - .family(Family::Name(font)) + .family(Family::Name(&family)) .weight(if bold { Weight::BOLD } else { Weight::NORMAL }) .style(if italic { Style::Italic } else { Style::Normal }); let align = match align { @@ -440,9 +508,14 @@ fn blur(values: &mut [f32], w: usize, h: usize, radius: usize) { } } +/// Goes up when drawing changes how the same title looks, so drawings kept +/// from before are made again. +const DRAWING: u32 = 3; + /// A stable key for a drawing: what is drawn, its mask and the canvas. pub fn key(generator: &Generator, mask: Option<&Mask>, canvas: (u32, u32)) -> u64 { let mut hasher = std::collections::hash_map::DefaultHasher::new(); + DRAWING.hash(&mut hasher); serde_json::to_string(generator).unwrap_or_default().hash(&mut hasher); serde_json::to_string(&mask).unwrap_or_default().hash(&mut hasher); canvas.hash(&mut hasher); @@ -658,6 +731,39 @@ pub fn drawn_dir() -> PathBuf { mod tests { use super::*; + #[test] + fn writing_a_font_lacks_is_set_in_one_font_that_has_it() { + let mut guard = fonts().lock().unwrap(); + let system = &mut guard.system; + if system + .db() + .faces() + .all(|f| f.families.iter().all(|(n, _)| n != "Hiragino Sans")) + { + return; // Not a Mac with its Japanese fonts. + } + assert_eq!( + family_for(system, "Avenir Next", "画面を録画して", true), + "Hiragino Sans" + ); + assert_eq!(family_for(system, "Avenir Next", "Hello", true), "Avenir Next"); + assert_eq!( + family_for(system, "Hiragino Mincho ProN", "画面を", false), + "Hiragino Mincho ProN", + "a font that has it is kept" + ); + drop(guard); + // Words of it, marked apart without spaces, are each a word to animate. + let title = Title { + text: "今日は\u{200B}動画の\u{200B}編集方法を".into(), + ..Title::default() + }; + let parts = draw_title_parts(&title, (1920, 1080)); + assert_eq!(parts.units.words.len(), 3); + let [a, b] = [parts.units.words[0].rect, parts.units.words[1].rect]; + assert_eq!(a[0] + a[2], b[0], "touching, not overlapping"); + } + #[test] fn draws_text_with_a_system_font() { let title = Title { diff --git a/crates/splice-core/src/captions.rs b/crates/splice-core/src/captions.rs index 0b21a58..0f41207 100644 --- a/crates/splice-core/src/captions.rs +++ b/crates/splice-core/src/captions.rs @@ -24,6 +24,8 @@ pub struct Layout { pub chars: usize, /// Light words up as they are said (else each line appears whole). pub karaoke: bool, + /// No caption goes past this (the end of the clip heard). + pub until: Option