Fixes from using tiers 1 and 2 by hand

README.md

@@ -111,7 +111,9 @@ make install-app   # Splice.app in /Applications
 - **Captions** (Edit → Add Captions from Speech, or the clip's Sound
   section): Splice hears what is said, on your Mac (whisper.cpp,
   `brew install whisper-cpp`; a 148 MB speech model downloads once), and lays
-  captions over it, each word lighting up as it is said. Each caption is a
+  captions over it, each word lighting up as it is said. Any language whisper
+  knows works; Japanese and Chinese, written without spaces, are grouped into
+  words to light up and set in one font that has them. Each caption is a
   title you can edit. `.srt` and `.vtt` files load too.
 - **Colour** shelf: full-frame colours for backgrounds and title cards, a
   half-black tint, and light leaks that wash warm or cool light over a shot.
@@ -125,6 +127,8 @@ make install-app   # Splice.app in /Applications
     the playhead; set another somewhere else and it glides between them
     (linear, smooth, ease in, ease out, or hold). Dragging a keyframed picture
     on the stage keys it where the playhead is. ⌥K keys place and size at once.
+    A selected clip shows its keyframes as marks along its foot on the
+    timeline: click one to go to it, drag it to change when it happens.
   - **Transition:** the way into a clip from the one before it: dissolve, dip
     to black or white, wipe or push four ways, or zoom, centred on the cut,
     with a length. A tab on the timeline marks each one. ⇧⌘T adds a dissolve
@@ -290,7 +294,10 @@ recorder, so there is no OBS and no extra install.
 - The take comes into the media panel. In Story view it goes into the chosen
   beat instead.
 - The first recording asks for Screen Recording permission. It is granted to
-  whatever launched Splice: the app, or your terminal.
+  whatever launched Splice: the app, or your terminal. Without it, Start says
+  so at once. A rebuilt app is new to macOS (it is signed only locally), so
+  after `make install-app` turn the permission off and on again in System
+  Settings → Privacy & Security → Screen Recording.
 
 ## Projects
 

crates/splice-app/src/editor.rs

@@ -102,6 +102,13 @@ enum Drag {
         left: bool,
         before: Project,
     },
+    /// A chosen clip's keyframes at one moment, moving along it.
+    Keyframe {
+        id: ClipId,
+        at: Time,
+        before: Project,
+        moved: bool,
+    },
     /// The end of a story beat, making it longer or shorter.
     BeatEnd {
         id: splice_core::story::BeatId,
@@ -974,6 +981,20 @@ impl Editor {
         window.focus(&self.focus);
         let p = self.local(event.position);
         let shift = event.modifiers.shift || event.modifiers.platform;
+        // A keyframe's mark: the playhead goes to it, and a drag moves it.
+        if !shift
+            && let Some((id, at)) = timeline::keyframe_at(&self.doc.project, &self.view, &self.selection, p)
+            && let Some(start) = self.doc.project.clip(id).map(|c| c.start)
+        {
+            self.seek(start + at, cx);
+            self.drag = Drag::Keyframe {
+                id,
+                at,
+                before: self.doc.project.clone(),
+                moved: false,
+            };
+            return cx.notify();
+        }
         match timeline::hit(&self.doc.project, &self.view, p) {
             Hit::Ruler => {
                 self.drag = Drag::Scrub;
@@ -1142,6 +1163,22 @@ impl Editor {
                 project.set_beat_duration(*id, t - start);
                 self.doc.project = project;
             }
+            Drag::Keyframe { id, at, before, moved } => {
+                let Some(clip) = before.clip(*id) else { return };
+                let frame = before.frame();
+                let to = ((view.t(p.x) - clip.start).clamp(0, clip.duration) + frame / 2) / frame * frame;
+                if !*moved && (to - *at).abs() < frame {
+                    return;
+                }
+                *moved = true;
+                let mut project = before.clone();
+                if let Some(c) = project.clip_mut(*id) {
+                    c.look.keys.move_moment(*at, to);
+                }
+                let start = clip.start;
+                self.doc.project = project;
+                self.seek(start + to, cx);
+            }
             Drag::Trim { id, left, before } => {
                 let mut t = view.t(p.x);
                 let mut points = before.snap_points(&[*id]);
@@ -1170,6 +1207,9 @@ impl Editor {
             Drag::Move {
                 before, moved: true, ..
             }
+            | Drag::Keyframe {
+                before, moved: true, ..
+            }
             | Drag::Trim { before, .. }
             | Drag::BeatEnd { before, .. } => {
                 self.doc.commit(before);

crates/splice-app/src/editor/agent.rs

@@ -977,6 +977,7 @@ impl Editor {
             "duration": s(project.duration()),
             "playhead": s(self.player.position),
             "playing": self.player.is_playing(),
+            "playback": self.player.keeping,
             "selection": self.selection.iter().map(|c| c.0).collect::<Vec<_>>(),
             "media": project.media.len(),
             "tracks": project.tracks.len(),

crates/splice-app/src/editor/effects.rs

@@ -470,20 +470,20 @@ impl Editor {
                 editor.captioning.remove(&id);
                 // Where the clip is now: it may have moved while it was heard.
                 let now = editor.doc.project.clip(id).cloned();
-                let heard: anyhow::Result<(Vec<crate::speech::Heard>, Time, f64)> = match (heard, now) {
+                let heard: anyhow::Result<(Vec<crate::speech::Heard>, Time, f64, Time)> = match (heard, now) {
                     (Err(e), _) => Err(e),
                     (Ok(_), None) => Err(anyhow::anyhow!("the clip was removed while it was heard")),
                     (Ok(_), Some(c)) if c.source_in != from || c.speed != clip.speed || c.reverse => Err(
                         anyhow::anyhow!("the clip was trimmed or retimed while it was heard; caption it again"),
                     ),
-                    (Ok(h), Some(c)) => Ok((h, c.start, c.speed)),
+                    (Ok(h), Some(c)) => Ok((h, c.start, c.speed, c.end())),
                 };
                 let answer = match heard {
-                    Ok((heard, _, _)) if heard.is_empty() => {
+                    Ok((heard, ..)) if heard.is_empty() => {
                         editor.say_error("No speech heard in that clip");
                         serde_json::json!({ "ok": false, "error": "no speech heard" })
                     }
-                    Ok((heard, start, speed)) => {
+                    Ok((heard, start, speed, end)) => {
                         // From source time to timeline time, at the clip's speed.
                         let at = |t: Time| start + (t as f64 / speed).round() as Time;
                         let words: Vec<splice_core::captions::Word> = heard
@@ -494,6 +494,10 @@ impl Editor {
                                 end: at(w.end),
                             })
                             .collect();
+                        let layout = splice_core::captions::Layout {
+                            until: Some(end),
+                            ..layout
+                        };
                         let ids = editor.edit(cx, |p| p.add_captions(&words, &layout));
                         editor.selection = ids.clone();
                         editor.say(format!(

crates/splice-app/src/player.rs

@@ -8,7 +8,7 @@ use crate::library::Library;
 use gpui::RenderImage;
 use splice_core::compose;
 use splice_core::model::{Clip, ClipId, Media, MediaId, MediaKind, Project, Source};
-use splice_core::time::{SECOND, Time, floor_to_frame};
+use splice_core::time::{self, SECOND, Time, floor_to_frame};
 use std::collections::{HashMap, HashSet};
 use std::sync::Arc;
 use std::sync::mpsc::{Receiver, Sender, channel};
@@ -90,6 +90,23 @@ pub fn backdrop_size(full: (u32, u32)) -> (u32, u32) {
     ((full.0 / 4).max(16) & !1, (full.1 / 4).max(16) & !1)
 }
 
+/// How playback has kept up since it last started: for `ctl status`, and for
+/// telling a heavy stack of effects from a smooth one.
+#[derive(Clone, Copy, Debug, Default, serde::Serialize)]
+pub struct Keeping {
+    /// Seconds played.
+    pub seconds: f64,
+    /// New frames drawn, all clips together.
+    pub frames: u64,
+    /// Decoded frames passed over because a later one was already due.
+    pub skipped: u64,
+    /// Looks at the clock that found a clip's picture over two frames behind.
+    pub behind: u64,
+    pub looks: u64,
+    /// The furthest any clip's picture fell behind, in seconds.
+    pub worst: f64,
+}
+
 /// What the editor should do after a tick.
 #[derive(Default)]
 pub struct Tick {
@@ -112,6 +129,10 @@ pub struct Player {
     still_tx: Sender<StillResult>,
     still_rx: Receiver<StillResult>,
     retired: Vec<Arc<RenderImage>>,
+    /// Clips whose stream has given a frame since playback was asked for:
+    /// the clock waits for them, not for the stills shown while paused.
+    flowing: HashSet<ClipId>,
+    pub keeping: Keeping,
 }
 
 impl Player {
@@ -128,6 +149,8 @@ impl Player {
             still_tx,
             still_rx,
             retired: Vec::new(),
+            flowing: HashSet::new(),
+            keeping: Keeping::default(),
         }
     }
 
@@ -145,6 +168,8 @@ impl Player {
             self.position = 0;
         }
         self.state = State::Priming(Instant::now());
+        self.flowing.clear();
+        self.keeping = Keeping::default();
     }
 
     pub fn pause(&mut self) {
@@ -157,6 +182,7 @@ impl Player {
         self.position = t.max(0);
         if self.is_playing() {
             self.state = State::Priming(Instant::now());
+            self.flowing.clear();
         }
     }
 
@@ -325,6 +351,24 @@ impl Player {
                 let due = if clip.reverse { source - half } else { source + half };
                 if let Some((t, size, image)) = self.pull(clip.id, due) {
                     self.show(clip.id, media.id, filters.clone(), t, size, image);
+                    self.flowing.insert(clip.id);
+                    self.keeping.frames += 1;
+                }
+                if let State::Playing { from, .. } = self.state {
+                    self.keeping.seconds = time::secs(position - from);
+                    self.keeping.looks += 1;
+                    if let Some(shown) = self.shown.get(&clip.id) {
+                        let gap = if clip.reverse {
+                            shown.source - due
+                        } else {
+                            due - shown.source
+                        };
+                        let late = time::secs(gap) / clip.speed.max(0.01);
+                        if late > 2.0 * time::secs(frame) {
+                            self.keeping.behind += 1;
+                        }
+                        self.keeping.worst = self.keeping.worst.max(late);
+                    }
                 }
             } else {
                 let current = self
@@ -387,7 +431,14 @@ impl Player {
         }
 
         if let State::Priming(since) = self.state {
-            let ready = stack.iter().all(|w| self.shown.contains_key(&w.clip.id));
+            let ready = stack.iter().all(|w| {
+                let streamed = w.media.kind == MediaKind::Video && !w.clip.frozen;
+                if streamed {
+                    self.flowing.contains(&w.clip.id)
+                } else {
+                    self.shown.contains_key(&w.clip.id)
+                }
+            });
             if ready || now - since > PRIME_TIMEOUT {
                 self.state = State::Playing {
                     anchor: now,
@@ -480,20 +531,26 @@ impl Player {
         let size = stream.size;
         let reverse = stream.reverse;
         let mut latest = None;
+        let mut skipped = 0;
         loop {
             if stream.pending.is_none() {
                 let t = stream.decoder.next_time();
                 stream.pending = stream.decoder.try_next().map(|image| (t, image));
             }
             match stream.pending.take() {
-                Some((t, image)) if (if reverse { t >= due } else { t <= due }) => latest = Some((t, image)),
+                Some((t, image)) if (if reverse { t >= due } else { t <= due }) => {
+                    if latest.replace((t, image)).is_some() {
+                        skipped += 1;
+                    }
+                }
                 Some(early) => {
                     stream.pending = Some(early);
                     break;
                 }
                 None => break,
             }
         }
+        self.keeping.skipped += skipped;
         latest.map(|(t, image)| (t, size, image))
     }