diff --git a/CHANGELOG.md b/CHANGELOG.md index aa5f941..f2c6ecf 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,28 @@ follow semantic versioning as described in [CONTRIBUTING.md](CONTRIBUTING.md). ## [Unreleased] +## [0.5.0] - 2026-11-18 + +### Added + +- Precise motion for large frames: `SessionOptions::precise_motion`, `preciseMotion` in the browser module, + `FH_PRECISE=1` in the environment for the C API. Off by default. On frames above 1280 px on the long side the pair + stage finds the motion on a 400-line picture and refines it on an 800-line one (`internal` auto picks 1408 x 800 for + 1080p) instead of one pass at 480 lines, with the same weights. Captured 1080p 120 fps video (UVG, 14 clips): 33.83 + against 33.42 dB, LPIPS 0.159 against 0.165; synthetic 1080p pans and moving objects: 35.41 against 33.35 dB. It + costs GPU time: 1.94 against 1.06 ms per generated frame at 1080p on an RTX 5090. No effect on smaller frames or + with `fast_render`. + +### Changed + +- With static protection on (the default), the still test of the render trusts a pixel only after three unchanged + source pairs, like the protection itself: one pair in which a moving detail happened to cover the same place twice + no longer holds it still. On the realtime defaults: UVG 1080p 33.42 dB and LPIPS 0.165 (0.4.1: 33.23 and 0.173), + synthetic 1080p 33.35 dB (32.58); ground-truth triplets ATD 31.78, CS2 29.39, natural 38.16 dB (31.66, 29.32, + 38.18). Local contrast of the in-between on UVG is 0.75 of the ground truth's (0.77): the held pixels were sharp + and often wrong. The age pass of static protection costs 0.08 to 0.10 ms more per pair on an RTX 4060 Ti; a + session without static protection is unchanged. + ## [0.4.1] - 2026-11-17 ### Changed diff --git a/Cargo.lock b/Cargo.lock index 29337fa..a539f27 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -548,7 +548,7 @@ dependencies = [ [[package]] name = "flyhigh-capi" -version = "0.4.1" +version = "0.5.0" dependencies = [ "anyhow", "flyhigh-engine", @@ -558,7 +558,7 @@ dependencies = [ [[package]] name = "flyhigh-cli" -version = "0.4.1" +version = "0.5.0" dependencies = [ "anyhow", "bytemuck", @@ -573,7 +573,7 @@ dependencies = [ [[package]] name = "flyhigh-engine" -version = "0.4.1" +version = "0.5.0" dependencies = [ "anyhow", "ash", @@ -590,7 +590,7 @@ dependencies = [ [[package]] name = "flyhigh-options" -version = "0.4.1" +version = "0.5.0" dependencies = [ "proc-macro2", "quote", @@ -599,7 +599,7 @@ dependencies = [ [[package]] name = "flyhigh-player" -version = "0.4.1" +version = "0.5.0" dependencies = [ "anyhow", "flyhigh-engine", @@ -610,7 +610,7 @@ dependencies = [ [[package]] name = "flyhigh-web" -version = "0.4.1" +version = "0.5.0" dependencies = [ "console_error_panic_hook", "flyhigh-engine", diff --git a/Cargo.toml b/Cargo.toml index b1fe96f..7110e5d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -5,7 +5,7 @@ members = ["crates/flyhigh-options", "crates/flyhigh-engine", "crates/flyhigh-ca default-members = ["crates/flyhigh-options", "crates/flyhigh-engine", "crates/flyhigh-capi", "crates/flyhigh-cli", "examples/player"] [workspace.package] -version = "0.4.1" +version = "0.5.0" edition = "2021" license = "Apache-2.0" repository = "https://github.com/getflyhigh/flyhigh-engine" diff --git a/crates/flyhigh-engine/src/api.rs b/crates/flyhigh-engine/src/api.rs index e313668..a4cbcca 100644 --- a/crates/flyhigh-engine/src/api.rs +++ b/crates/flyhigh-engine/src/api.rs @@ -111,7 +111,15 @@ pub struct Options { } impl Options { pub fn new(width: u32, height: u32) -> Options { - let internal = if width.max(height) > 960 { Some(default_internal(width, height)) } else { None }; + let internal = if width.max(height) > 960 { + Some(if crate::options::precise_default() && crate::options::precise_applies(width, height) { + crate::options::precise_internal(width, height) + } else { + default_internal(width, height) + }) + } else { + None + }; Options { width, height, @@ -148,6 +156,7 @@ impl Options { linear_t: self.linear_t, protection_mask: false, near: crate::options::near_default(), + precise: crate::options::precise_default() && crate::options::precise_applies(self.width, self.height), } } } diff --git a/crates/flyhigh-engine/src/gen_tex.rs b/crates/flyhigh-engine/src/gen_tex.rs index 2015dfd..595e1fb 100644 --- a/crates/flyhigh-engine/src/gen_tex.rs +++ b/crates/flyhigh-engine/src/gen_tex.rs @@ -64,11 +64,14 @@ pub struct TexOptions { pub protection_mask: bool, /// nearest-source regions (`NetOpts::near`, realtime mode) pub near: bool, + /// precise motion (`SessionOptions::precise_motion`): the two-level pair stage; `iw` x `ih` is its fine level + pub precise: bool, } impl TexOptions { /// defaults: k 1, internal size `default_internal`, realtime, repair `DEFAULT_REPAIR`, no cadence, model sampler, no static protection pub fn new(w: u32, h: u32) -> TexOptions { - let (iw, ih) = crate::default_internal(w, h); + let precise = crate::options::precise_default() && crate::options::precise_applies(w, h); + let (iw, ih) = if precise { crate::options::precise_internal(w, h) } else { crate::default_internal(w, h) }; TexOptions { w, h, @@ -86,6 +89,7 @@ impl TexOptions { linear_t: false, protection_mask: false, near: crate::options::near_default(), + precise, } } } @@ -193,6 +197,15 @@ impl GenTex { opts: TexOptions, tune: Option<&crate::TuneTable>, ) -> Result { + // fast render has no cascade: precise motion set by the environment default falls back to the plain rung + let mut opts = opts; + opts.precise &= crate::options::precise_applies(opts.w, opts.h); + if opts.precise && opts.fast_render { + opts.precise = false; + if (opts.iw, opts.ih) == crate::options::precise_internal(opts.w, opts.h) { + (opts.iw, opts.ih) = crate::default_internal(opts.w, opts.h); + } + } anyhow::ensure!((1..=3).contains(&opts.k), "k must be 1..3"); anyhow::ensure!(opts.repair <= 2, "repair must be 0..2"); anyhow::ensure!(opts.w >= 64 && opts.h >= 64, "frame size too small (min 64 x 64)"); @@ -215,6 +228,8 @@ impl GenTex { decisive: crate::options::decisive_default() && opts.mode == TexMode::Realtime, still: crate::options::still_default() && opts.mode == TexMode::Realtime, near: opts.near && opts.mode == TexMode::Realtime, + cascade: crate::options::cascade_for(opts.precise).0, + cascade_iters: crate::options::cascade_for(opts.precise).1, }; let mut sess = Vec::new(); for _ in 0..n_sess { diff --git a/crates/flyhigh-engine/src/gpu/pipeline.rs b/crates/flyhigh-engine/src/gpu/pipeline.rs index 13df30e..0a78d83 100644 --- a/crates/flyhigh-engine/src/gpu/pipeline.rs +++ b/crates/flyhigh-engine/src/gpu/pipeline.rs @@ -20,6 +20,7 @@ fn on_static(gx: f32, gy: f32) -> bool {\n\ let hook = " // static hook\n"; let rule = " let sa = on_static(gxa, gya); let sb = on_static(gxb, gyb);\n\ var raw = 0.0;\n\ + stillw = stillw * select(0.0, 1.0, abs(f32(age[y * pp.W + x])) >= STILL_RUN);\n\ if (sa && sb) { rk = 0.0; raw = 1.0; } else if (sa) { rk = 0.0; bwa = 0.0; bwb = 1.0; } else if (sb) { rk = 0.0; bwa = 1.0; bwb = 0.0; }\n"; let out_old = " outp[(c * pp.H + y) * pp.W + x] = f16(o[c]);"; let out_new = " let i = (c * pp.H + y) * pp.W + x;\n\ diff --git a/crates/flyhigh-engine/src/model/graph.rs b/crates/flyhigh-engine/src/model/graph.rs index cee8101..5b25501 100644 --- a/crates/flyhigh-engine/src/model/graph.rs +++ b/crates/flyhigh-engine/src/model/graph.rs @@ -66,13 +66,118 @@ pub(crate) fn count_uses_unrolled(ops: &[Value], k: u32, uses: &mut HashMap u32 { + op["times"].as_u64().map_or(k, |n| n as u32) +} + +fn renamed(op: &Value, suffix: &str) -> Value { + let mut op = op.clone(); + if suffix.is_empty() { + return op; + } + for key in ["in", "out", "carry"] { + if let Some(list) = op.get_mut(key).and_then(Value::as_array_mut) { + for v in list { + if let Some(n) = v.as_str() { + *v = Value::String(format!("{n}{suffix}")); + } + } + } + } + if let Some(body) = op.get_mut("body").and_then(Value::as_array_mut) { + for b in body { + *b = renamed(b, suffix); + } + } + op +} + +/// The cascade form of a graph: the pair stage runs whole at the coarsest of `levels` pyramid levels (each half the +/// size of the one before), then at every finer level the encoders run again and the fine block (the graph's repeat +/// body) continues from the flow, visibility and hidden state of the level below, upsampled: `fine_iters[i]` times +/// at the i-th level above the coarsest (the last entry repeats). +/// The coarse level keeps the motion range of a low internal size; the fine levels give the flow the precision of the +/// frame's own resolution, which a low internal size alone lacks (0.4 px of error at 1080p from a 480-line grid). +/// The mid stage is unchanged and reads the finest level under the graph's own names. `None` when the graph has no +/// repeat with carried tensors before its mid stage. +pub fn cascade(ops: &[Value], levels: u32, fine_iters: &[u32]) -> Option> { + use std::collections::HashSet; + let ms = mid_start(ops); + let rep = ops[..ms].iter().position(|o| o["op"] == "repeat")?; + let carry = names(&ops[rep], "carry"); + if levels < 2 || carry.is_empty() || fine_iters.is_empty() { + return None; + } + let produced_before: HashSet = ops[..rep].iter().flat_map(|o| names(o, "out")).collect(); + // what a finer level needs from the ops before the repeat: the closure of the given reads, stopping at the + // carried tensors (they come from the level below) and at the graph inputs + let closure = |seed: &[String]| -> Vec { + let mut need: HashSet = + seed.iter().filter(|n| produced_before.contains(*n) && !carry.contains(*n)).cloned().collect(); + let mut keep = vec![false; rep]; + for i in (0..rep).rev() { + if names(&ops[i], "out").iter().any(|o| need.contains(o)) { + keep[i] = true; + need.extend(names(&ops[i], "in").into_iter().filter(|n| !carry.contains(n))); + } + } + keep + }; + let mut reads = HashMap::new(); + count_uses(ops[rep]["body"].as_array()?, &mut reads); + let body_reads: Vec = reads.into_keys().collect(); + // the finest level also feeds every op after the repeat whose result is read at all + let mut used = HashMap::new(); + count_uses(ops, &mut used); + let mut tail_reads = body_reads.clone(); + for o in &ops[rep + 1..] { + if o["op"] == "render_pix" || names(o, "out").iter().any(|n| used.contains_key(n)) { + tail_reads.extend(names(o, "in")); + } + } + let sfx = |l: u32| if l == 0 { String::new() } else { format!("#{l}") }; + let up = |src: String, dst: String| serde_json::json!({"op": "interpolate", "in": [src], "out": [dst], "scale": 2, "mode": "bilinear", "align_corners": false}); + let mut out = Vec::new(); + for l in 1..levels { + for img in ["A", "B"] { + out.push(serde_json::json!({"op": "interpolate", "in": [format!("{img}{}", sfx(l - 1))], "out": [format!("{img}{}", sfx(l))], + "scale": 0.5, "mode": "bilinear", "align_corners": false, "antialias": true})); + } + } + let top = levels - 1; + out.extend(ops[..=rep].iter().map(|o| renamed(o, &sfx(top)))); + for l in (0..top).rev() { + let keep = closure(if l == 0 { &tail_reads } else { &body_reads }); + out.extend(ops[..rep].iter().zip(&keep).filter(|(_, k)| **k).map(|(o, _)| renamed(o, &sfx(l)))); + for n in &carry { + let (below, here) = (format!("{n}{}", sfx(l + 1)), format!("{n}{}", sfx(l))); + if n == "flow" { + // flow is in pixels of its own level: twice the grid, twice the value + out.push(up(below, format!("{here}:up"))); + out.push( + serde_json::json!({"op": "scale", "in": [format!("{here}:up")], "out": [here], "factor": 2.0}), + ); + } else { + out.push(up(below, here)); + } + } + let mut r = renamed(&ops[rep], &sfx(l)); + let step = (top - 1 - l) as usize; + r["times"] = Value::from(fine_iters[step.min(fine_iters.len() - 1)].max(1)); + out.push(r); + } + out.extend(ops[rep + 1..].iter().cloned()); + Some(out) +} + // the first op of the t-dependent (per generated frame) stage: everything from here on is rebuilt per mid in a split plan pub fn mid_start(ops: &[Value]) -> usize { ops.iter() diff --git a/crates/flyhigh-engine/src/net/build.rs b/crates/flyhigh-engine/src/net/build.rs index 73717af..442d534 100644 --- a/crates/flyhigh-engine/src/net/build.rs +++ b/crates/flyhigh-engine/src/net/build.rs @@ -172,7 +172,7 @@ pub fn build(ops: &[Value], env: &mut HashMap, plan: &mut Vec { let body: Vec = op["body"].as_array().unwrap().clone(); - for _ in 0..cx.k { + for _ in 0..crate::model::graph::repeat_times(op, cx.k) { build(&body, env, plan, cx); } continue; @@ -507,6 +507,13 @@ pub fn build_network_ex( let (hp, wp) = (a.h, a.w); let (ihp, iwp) = (ih.div_ceil(32) * 32, iw.div_ceil(32) * 32); let internal = (iwp, ihp) != (wp, hp); + // cascade: as many of the requested levels as the network size divides into (the coarsest level needs /16) + let mut levels = if net.fast_render { 1 } else { net.cascade.max(1) }; + while levels > 1 && (ihp % (16 << (levels - 1)) != 0 || iwp % (16 << (levels - 1)) != 0) { + levels -= 1; + } + let cascaded = (levels > 1).then(|| crate::model::graph::cascade(ops, levels, &net.cascade_iters)).flatten(); + let ops: &[Value] = cascaded.as_deref().unwrap_or(ops); let arena_was = if std::env::var_os("FH_NO_ARENA").is_some() { gpu.arena.borrow_mut().take() } else { @@ -530,9 +537,9 @@ pub fn build_network_ex( let static_p = StaticParams::new(net, wp, hp, wp, hp); let static_age = net.protects().then(|| gpu.buf(&vec![0.0; (wp * hp) as usize])); // the age pass's own counters (keep_static_age.wgsl): unchanged pairs, and those with the surroundings moving - let static_state = static_age.as_ref().map(|_| gpu.buf(&vec![0.0; (2 * wp * hp) as usize])); + let static_state = static_age.as_ref().map(|_| gpu.buf(&vec![0.0; (3 * wp * hp) as usize])); if let Some(st) = &static_state { - env.insert("static:state".into(), Tensor { buf: st.clone(), c: 2, h: hp, w: wp, c0: 0, scale: 1.0 }); + env.insert("static:state".into(), Tensor { buf: st.clone(), c: 3, h: hp, w: wp, c0: 0, scale: 1.0 }); } // the host's mask (`SessionExt::write_mask`); the zero buffer fills the binding when the session has none let mask = net.protection_mask.then(|| gpu.buf(&vec![0.0; (wp * hp) as usize])); diff --git a/crates/flyhigh-engine/src/options.rs b/crates/flyhigh-engine/src/options.rs index 07603a5..fa1b172 100644 --- a/crates/flyhigh-engine/src/options.rs +++ b/crates/flyhigh-engine/src/options.rs @@ -230,6 +230,54 @@ pub struct NetOpts { pub still: bool, /// nearest-source regions after the render: where the warped frames disagree over a region, the nearer frame alone pub near: bool, + /// pyramid levels of the pair stage (`model::graph::cascade`); 0 or 1 = the plain graph + pub cascade: u32, + /// fine iterations at each cascade level above the coarsest, coarse to fine + pub cascade_iters: [u32; 3], +} +/// The default of `SessionOptions::precise_motion`: off, FH_PRECISE=1 turns it on. +pub fn precise_default() -> bool { + std::env::var("FH_PRECISE").is_ok_and(|v| v.trim() == "1") +} +/// Whether precise motion applies to a frame: above 1280 px on the long side only, where its 800-line rung is +/// smaller than the frame. Measured at 1080p; smaller frames keep the plain pair stage. +pub fn precise_applies(w: u32, h: u32) -> bool { + w.max(h) > 1280 +} +/// The internal size of precise motion when the host gives none: the 800-line rung (aspect kept, multiples of 32), +/// never above the frame. Half of it is the level the motion is found on, so the search range stays that of the +/// 400-line picture while the result is refined on the 800-line one. +pub fn precise_internal(w: u32, h: u32) -> (u32, u32) { + if !precise_applies(w, h) { + return (w, h); + } + let s = (800.0 / h as f64).min(1.0); + let rw = ((w as f64 * s / 32.0).round() as u32 * 32).max(64); + let rh = ((h as f64 * s / 32.0).round() as u32 * 32).max(64); + (rw.min(w), rh.min(h)) +} +/// Pyramid levels and fine iterations of the pair stage: two levels with two iterations for precise motion, else the +/// plain graph. FH_CASCADE= and FH_CASCADE_K= +/// override both for comparisons. +pub fn cascade_for(precise: bool) -> (u32, [u32; 3]) { + match std::env::var_os("FH_CASCADE") { + Some(_) => cascade_default(), + None => (if precise { 2 } else { 0 }, [2; 3]), + } +} +/// The comparison override of `cascade_for`. +pub fn cascade_default() -> (u32, [u32; 3]) { + let levels = std::env::var("FH_CASCADE").ok().and_then(|v| v.trim().parse::().ok()).unwrap_or(0); + let given: Vec = std::env::var("FH_CASCADE_K") + .map(|v| v.split(',').filter_map(|x| x.trim().parse().ok()).collect()) + .unwrap_or_default(); + let mut iters = [2; 3]; + for (i, slot) in iters.iter_mut().enumerate() { + if let Some(&v) = given.get(i).or(given.last()) { + *slot = v; + } + } + (levels, iters) } /// The decisive blend: off by default (on screen its patches at a failed motion were no better than the blend's /// fog); FH_BLEND=decisive turns it on for comparisons. @@ -270,6 +318,8 @@ impl NetOpts { decisive: false, still: false, near: false, + cascade: cascade_for(false).0, + cascade_iters: cascade_for(false).1, } } /// The render's mode word (render.wgsl / render_tex.wgsl): bit 0 decisive blend, bit 1 still test, bit 2 diff --git a/crates/flyhigh-engine/src/session.rs b/crates/flyhigh-engine/src/session.rs index 80ec11b..5b00e2f 100644 --- a/crates/flyhigh-engine/src/session.rs +++ b/crates/flyhigh-engine/src/session.rs @@ -120,6 +120,13 @@ pub struct SessionOptions { /// viewers prefer the hold on fast turns turns it on. The environment variable `FH_NEAR=1` sets the default to on. #[option(default = "off", values = "on, off", c = "none (FH_NEAR=1 in the environment)")] pub nearest_source: bool, + /// Frames above 1280 px on the long side: the motion is found on a 400-line picture and refined on an 800-line one (`internal` + /// `Auto` picks that size), instead of one pass at 480 lines. On captured 1080p 120 fps video (UVG, 14 clips) + /// 33.83 against 33.42 dB and LPIPS 0.159 against 0.165; on synthetic 1080p pans and moving objects 35.41 + /// against 33.35 dB. Costs GPU time: 1.94 against 1.06 ms per generated frame at 1080p on an RTX 5090. No effect + /// on smaller frames or with `fast_render`. The environment variable `FH_PRECISE=1` sets the default to on. + #[option(default = "off", values = "on, off", c = "none (FH_PRECISE=1 in the environment)")] + pub precise_motion: bool, /// Kernel choices from an earlier autotune on this device at this size (the text of a `TuneTable`); without one /// the engine tunes at the first build, or uses the fixed routing when the autotune is off. #[option(default = "none", values = "an autotune table", c = "tune (text, NULL = none)")] @@ -141,12 +148,19 @@ impl SessionOptions { keep_maps: false, protection_mask: false, nearest_source: crate::options::near_default(), + precise_motion: crate::options::precise_default(), tune: None, } } + /// precise motion is in effect for these options (`precise_motion` on a frame above 1280 px, exact render) + pub fn precise(&self) -> bool { + self.precise_motion && !self.fast_render && crate::options::precise_applies(self.width, self.height) + } + pub fn internal_size(&self) -> (u32, u32) { match self.internal { + Internal::Auto if self.precise() => crate::options::precise_internal(self.width, self.height), Internal::Auto => crate::default_internal(self.width, self.height), Internal::Frame => (self.width, self.height), Internal::Size(w, h) => (w.min(self.width), h.min(self.height)), @@ -200,6 +214,8 @@ impl SessionOptions { decisive: crate::options::decisive_default(), still: crate::options::still_default(), near: self.nearest_source, + cascade: crate::options::cascade_for(self.precise()).0, + cascade_iters: crate::options::cascade_for(self.precise()).1, } } @@ -228,6 +244,7 @@ impl SessionOptions { linear_t: self.linear_t, protection_mask: self.protection_mask, near: self.nearest_source, + precise: self.precise(), } } } diff --git a/crates/flyhigh-engine/tests/core.rs b/crates/flyhigh-engine/tests/core.rs index 9edb8b5..5ddb491 100644 --- a/crates/flyhigh-engine/tests/core.rs +++ b/crates/flyhigh-engine/tests/core.rs @@ -17,6 +17,13 @@ fn session_options_validate() { assert!(o.validate().is_ok()); assert_eq!(o.internal_size(), (856, 480), "the 480-line rung above 960 px"); assert_eq!(SessionOptions::new(640, 480).internal_size(), (640, 480)); + let precise = SessionOptions { precise_motion: true, ..SessionOptions::new(1920, 1080) }; + assert!(precise.precise()); + assert_eq!(precise.internal_size(), (1408, 800), "precise motion refines on the 800-line rung"); + let precise_fast = SessionOptions { fast_render: true, ..precise.clone() }; + assert!(!precise_fast.precise()); + assert_eq!(precise_fast.internal_size(), (856, 480)); + assert!(!SessionOptions { precise_motion: true, ..SessionOptions::new(960, 540) }.precise()); let small = SessionOptions::new(32, 32); assert_eq!(small.validate().unwrap_err().field, "width"); let k = SessionOptions { k: 4, ..SessionOptions::new(W, H) }; diff --git a/crates/flyhigh-engine/tests/defaults.rs b/crates/flyhigh-engine/tests/defaults.rs index be881cc..71bc6d1 100644 --- a/crates/flyhigh-engine/tests/defaults.rs +++ b/crates/flyhigh-engine/tests/defaults.rs @@ -13,6 +13,17 @@ fn internal_size_is_the_480_line_rung_above_960_px() { assert_eq!(default_internal(1080, 1920), (272, 480)); // portrait } +#[test] +fn precise_motion_takes_the_800_line_rung_above_960_px() { + use flyhigh_engine::options::{precise_applies, precise_internal}; + assert_eq!(precise_internal(1920, 1080), (1408, 800)); + assert_eq!(precise_internal(2560, 1440), (1408, 800)); + assert_eq!(precise_internal(3840, 2160), (1408, 800)); + assert_eq!(precise_internal(1600, 900), (1408, 800)); + assert_eq!(precise_internal(1280, 720), (1280, 720)); // not applied: the frame itself + assert!(!precise_applies(1280, 720) && precise_applies(1600, 900)); +} + #[test] fn generator_defaults() { let o = Options::new(1920, 1080); diff --git a/crates/flyhigh-engine/tests/golden/rt-1080p-fast-linear-static.portable-f16.txt b/crates/flyhigh-engine/tests/golden/rt-1080p-fast-linear-static.portable-f16.txt index f6f0af2..4fcc727 100644 --- a/crates/flyhigh-engine/tests/golden/rt-1080p-fast-linear-static.portable-f16.txt +++ b/crates/flyhigh-engine/tests/golden/rt-1080p-fast-linear-static.portable-f16.txt @@ -2,7 +2,7 @@ # pair 0 tex_down (108,60,1) [tex1920x1080 buf2488320 params] 00000780 00000438 00000780 00000440 00000360 000001e0 00000000 00000000 1 tex_down (108,60,1) [tex1920x1080 buf2488320 params] 00000780 00000438 00000780 00000440 00000360 000001e0 00000000 00000000 - 2 keep_static_age_tex (240,136,1) [tex1920x1080 tex1920x1080 buf4177920 params buf1024 buf8355840] 00000780 00000438 00000780 00000440 3c909091 41000000 00000000 00000000 + 2 keep_static_age_tex (240,136,1) [tex1920x1080 tex1920x1080 buf4177920 params buf1024 buf12533760] 00000780 00000438 00000780 00000440 3c909091 41000000 00000000 00000000 3 interp (27,15,3) [buf2488320 buf155520 params] 00000003 000001e0 00000360 00000078 000000d8 00000001 00000000 3f800000 4 interp (27,15,3) [buf2488320 buf155520 params] 00000003 000001e0 00000360 00000078 000000d8 00000001 00000000 3f800000 5 tconv (1,102,2) [buf155520 buf4096 buf72 buf72 buf466560 params buf1866240 buf2048] 00000003 00000078 000000d8 0000003c 0000006c 00000002 00000001 0000001b 00000020 00000024 00000040 00001950 00000000 00000002 00000000 00000000 diff --git a/crates/flyhigh-engine/tests/golden/rt-1080p-fast-linear-static.portable-f32.txt b/crates/flyhigh-engine/tests/golden/rt-1080p-fast-linear-static.portable-f32.txt index b2b376c..86573d8 100644 --- a/crates/flyhigh-engine/tests/golden/rt-1080p-fast-linear-static.portable-f32.txt +++ b/crates/flyhigh-engine/tests/golden/rt-1080p-fast-linear-static.portable-f32.txt @@ -2,7 +2,7 @@ # pair 0 tex_down (108,60,1) [tex1920x1080 buf4976640 params] 00000780 00000438 00000780 00000440 00000360 000001e0 00000000 00000000 1 tex_down (108,60,1) [tex1920x1080 buf4976640 params] 00000780 00000438 00000780 00000440 00000360 000001e0 00000000 00000000 - 2 keep_static_age_tex (240,136,1) [tex1920x1080 tex1920x1080 buf8355840 params buf1024 buf16711680] 00000780 00000438 00000780 00000440 3c909091 41000000 00000000 00000000 + 2 keep_static_age_tex (240,136,1) [tex1920x1080 tex1920x1080 buf8355840 params buf1024 buf25067520] 00000780 00000438 00000780 00000440 3c909091 41000000 00000000 00000000 3 interp (27,15,3) [buf4976640 buf311040 params] 00000003 000001e0 00000360 00000078 000000d8 00000001 00000000 3f800000 4 interp (27,15,3) [buf4976640 buf311040 params] 00000003 000001e0 00000360 00000078 000000d8 00000001 00000000 3f800000 5 tconv (1,102,2) [buf311040 buf8192 buf144 buf144 buf933120 params buf1866240 buf4096] 00000003 00000078 000000d8 0000003c 0000006c 00000002 00000001 0000001b 00000020 00000024 00000040 00001950 00000000 00000002 00000000 00000000 diff --git a/crates/flyhigh-engine/tests/golden/rt-1080p-fast-linear-static.tc-f16.txt b/crates/flyhigh-engine/tests/golden/rt-1080p-fast-linear-static.tc-f16.txt index 157bf3e..5220bc9 100644 --- a/crates/flyhigh-engine/tests/golden/rt-1080p-fast-linear-static.tc-f16.txt +++ b/crates/flyhigh-engine/tests/golden/rt-1080p-fast-linear-static.tc-f16.txt @@ -2,7 +2,7 @@ # pair 0 tex_down (108,60,1) [tex1920x1080 buf2488320 params] 00000780 00000438 00000780 00000440 00000360 000001e0 00000000 00000000 1 tex_down (108,60,1) [tex1920x1080 buf2488320 params] 00000780 00000438 00000780 00000440 00000360 000001e0 00000000 00000000 - 2 keep_static_age_tex (240,136,1) [tex1920x1080 tex1920x1080 buf4177920 params buf1024 buf8355840] 00000780 00000438 00000780 00000440 3c909091 41000000 00000000 00000000 + 2 keep_static_age_tex (240,136,1) [tex1920x1080 tex1920x1080 buf4177920 params buf1024 buf12533760] 00000780 00000438 00000780 00000440 3c909091 41000000 00000000 00000000 3 interp (27,15,3) [buf2488320 buf155520 params] 00000003 000001e0 00000360 00000078 000000d8 00000001 00000000 3f800000 4 interp (27,15,3) [buf2488320 buf155520 params] 00000003 000001e0 00000360 00000078 000000d8 00000001 00000000 3f800000 5 fused (1,405,1) [buf155520 buf4096 buf1024 buf72 buf72 buf466560 params] 00000003 00000078 000000d8 0000003c 0000006c 00000002 00000001 0000001b 00000020 00000024 00000040 00001950 00000000 00000000 00000000 00000000 diff --git a/crates/flyhigh-engine/tests/golden/rt-720p-fast-static.portable-f16.txt b/crates/flyhigh-engine/tests/golden/rt-720p-fast-static.portable-f16.txt index d86bb43..d86ea6c 100644 --- a/crates/flyhigh-engine/tests/golden/rt-720p-fast-static.portable-f16.txt +++ b/crates/flyhigh-engine/tests/golden/rt-720p-fast-static.portable-f16.txt @@ -2,7 +2,7 @@ # pair 0 tex_down (108,60,1) [tex1280x720 buf2488320 params] 00000500 000002d0 00000500 000002e0 00000360 000001e0 00000000 00000000 1 tex_down (108,60,1) [tex1280x720 buf2488320 params] 00000500 000002d0 00000500 000002e0 00000360 000001e0 00000000 00000000 - 2 keep_static_age_tex (160,92,1) [tex1280x720 tex1280x720 buf1884160 params buf1024 buf3768320] 00000500 000002d0 00000500 000002e0 3c909091 41000000 00000000 00000000 + 2 keep_static_age_tex (160,92,1) [tex1280x720 tex1280x720 buf1884160 params buf1024 buf5652480] 00000500 000002d0 00000500 000002e0 3c909091 41000000 00000000 00000000 3 interp (27,15,3) [buf2488320 buf155520 params] 00000003 000001e0 00000360 00000078 000000d8 00000001 00000000 3f800000 4 interp (27,15,3) [buf2488320 buf155520 params] 00000003 000001e0 00000360 00000078 000000d8 00000001 00000000 3f800000 5 tconv (1,102,2) [buf155520 buf4096 buf72 buf72 buf466560 params buf1866240 buf2048] 00000003 00000078 000000d8 0000003c 0000006c 00000002 00000001 0000001b 00000020 00000024 00000040 00001950 00000000 00000002 00000000 00000000 diff --git a/crates/flyhigh-engine/tests/golden/rt-720p-fast-static.portable-f32.txt b/crates/flyhigh-engine/tests/golden/rt-720p-fast-static.portable-f32.txt index be4b3ec..4f332da 100644 --- a/crates/flyhigh-engine/tests/golden/rt-720p-fast-static.portable-f32.txt +++ b/crates/flyhigh-engine/tests/golden/rt-720p-fast-static.portable-f32.txt @@ -2,7 +2,7 @@ # pair 0 tex_down (108,60,1) [tex1280x720 buf4976640 params] 00000500 000002d0 00000500 000002e0 00000360 000001e0 00000000 00000000 1 tex_down (108,60,1) [tex1280x720 buf4976640 params] 00000500 000002d0 00000500 000002e0 00000360 000001e0 00000000 00000000 - 2 keep_static_age_tex (160,92,1) [tex1280x720 tex1280x720 buf3768320 params buf1024 buf7536640] 00000500 000002d0 00000500 000002e0 3c909091 41000000 00000000 00000000 + 2 keep_static_age_tex (160,92,1) [tex1280x720 tex1280x720 buf3768320 params buf1024 buf11304960] 00000500 000002d0 00000500 000002e0 3c909091 41000000 00000000 00000000 3 interp (27,15,3) [buf4976640 buf311040 params] 00000003 000001e0 00000360 00000078 000000d8 00000001 00000000 3f800000 4 interp (27,15,3) [buf4976640 buf311040 params] 00000003 000001e0 00000360 00000078 000000d8 00000001 00000000 3f800000 5 tconv (1,102,2) [buf311040 buf8192 buf144 buf144 buf933120 params buf1866240 buf4096] 00000003 00000078 000000d8 0000003c 0000006c 00000002 00000001 0000001b 00000020 00000024 00000040 00001950 00000000 00000002 00000000 00000000 diff --git a/crates/flyhigh-engine/tests/golden/rt-720p-fast-static.tc-f16.txt b/crates/flyhigh-engine/tests/golden/rt-720p-fast-static.tc-f16.txt index 3990838..251e846 100644 --- a/crates/flyhigh-engine/tests/golden/rt-720p-fast-static.tc-f16.txt +++ b/crates/flyhigh-engine/tests/golden/rt-720p-fast-static.tc-f16.txt @@ -2,7 +2,7 @@ # pair 0 tex_down (108,60,1) [tex1280x720 buf2488320 params] 00000500 000002d0 00000500 000002e0 00000360 000001e0 00000000 00000000 1 tex_down (108,60,1) [tex1280x720 buf2488320 params] 00000500 000002d0 00000500 000002e0 00000360 000001e0 00000000 00000000 - 2 keep_static_age_tex (160,92,1) [tex1280x720 tex1280x720 buf1884160 params buf1024 buf3768320] 00000500 000002d0 00000500 000002e0 3c909091 41000000 00000000 00000000 + 2 keep_static_age_tex (160,92,1) [tex1280x720 tex1280x720 buf1884160 params buf1024 buf5652480] 00000500 000002d0 00000500 000002e0 3c909091 41000000 00000000 00000000 3 interp (27,15,3) [buf2488320 buf155520 params] 00000003 000001e0 00000360 00000078 000000d8 00000001 00000000 3f800000 4 interp (27,15,3) [buf2488320 buf155520 params] 00000003 000001e0 00000360 00000078 000000d8 00000001 00000000 3f800000 5 fused (1,405,1) [buf155520 buf4096 buf1024 buf72 buf72 buf466560 params] 00000003 00000078 000000d8 0000003c 0000006c 00000002 00000001 0000001b 00000020 00000024 00000040 00001950 00000000 00000000 00000000 00000000 diff --git a/crates/flyhigh-engine/tests/gpu_smoke.rs b/crates/flyhigh-engine/tests/gpu_smoke.rs index 570a35d..0a72b00 100644 --- a/crates/flyhigh-engine/tests/gpu_smoke.rs +++ b/crates/flyhigh-engine/tests/gpu_smoke.rs @@ -238,9 +238,15 @@ fn keep_static_leaves_line_art_on_a_pan_alone() { last }; let (off, on) = (run(false), run(true)); - let differ = off.chunks(4).zip(on.chunks(4)).filter(|(a, b)| (0..3).any(|c| a[c].abs_diff(b[c]) > 30)).count(); - eprintln!("line art pan: protection on vs off, {differ} pixels differ by more than 30"); - assert!(differ < 20, "protection changed {differ} pixels of moving line art"); + // against the true in-between: with protection the still test waits for three unchanged pairs, so the two + // renders differ where the unprotected one holds a fill pixel for one pair; protection must not add wrong pixels + let truth = line_art(12.5 * 6.0); + let wrong = |img: &[u8]| { + img.chunks(4).zip(truth.chunks(4)).filter(|(a, b)| (0..3).any(|c| a[c].abs_diff(b[c]) > 30)).count() + }; + let (w_off, w_on) = (wrong(&off), wrong(&on)); + eprintln!("line art pan: pixels off the true in-between by more than 30: protection off {w_off}, on {w_on}"); + assert!(w_on <= w_off + 20, "protection left {w_on} wrong pixels of moving line art against {w_off} without it"); } #[test] diff --git a/crates/flyhigh-web/src/core.rs b/crates/flyhigh-web/src/core.rs index 56c647a..f6c8ef8 100644 --- a/crates/flyhigh-web/src/core.rs +++ b/crates/flyhigh-web/src/core.rs @@ -154,7 +154,7 @@ impl EngineOptionsJs { /// `{ width, height, internal: 'auto' | 'frame' | [w, h], k, repair, render: 'model' | 'bilinear' | 'bicubic', /// staticProtection: false | true | { frames, tolerance }, fastRender, linearT, keepMaps, protectionMask, -/// nearestSource, tune }`. +/// nearestSource, preciseMotion, tune }`. #[derive(Deserialize)] #[serde(deny_unknown_fields, rename_all = "camelCase")] struct SessionOptionsJs { @@ -181,6 +181,8 @@ struct SessionOptionsJs { #[serde(default)] nearest_source: Option, #[serde(default)] + precise_motion: Option, + #[serde(default)] tune: Option, } @@ -233,6 +235,7 @@ impl SessionOptionsJs { o.keep_maps = self.keep_maps.unwrap_or(false); o.protection_mask = self.protection_mask.unwrap_or(false); o.nearest_source = self.nearest_source.unwrap_or(false); + o.precise_motion = self.precise_motion.unwrap_or(false); o.tune = self.tune.as_deref().filter(|t| !t.is_empty()).and_then(flyhigh_engine::tune_table_from); o.validate().map_err(err)?; Ok(o) diff --git a/docs/api-js.md b/docs/api-js.md index 6752e28..bc0b3b9 100644 --- a/docs/api-js.md +++ b/docs/api-js.md @@ -43,7 +43,7 @@ console.log(engine.info(), model.info()); // { name, Session options: `{ width, height, internal: 'auto' | 'frame' | [w, h], k, repair, render: 'model' | 'bilinear' | 'bicubic', staticProtection: false | true | { frames, tolerance }, fastRender, linearT, keepMaps, protectionMask, -nearestSource, tune }`. Only `width` and `height` are required; the defaults are those of [options.md](options.md) (`internal` auto = +nearestSource, preciseMotion, tune }`. Only `width` and `height` are required; the defaults are those of [options.md](options.md) (`internal` auto = the 480-line rung above 960 px, `k` 1, `repair` 1). ## `FhSession`: two frames in, one frame out diff --git a/docs/options.md b/docs/options.md index dfdeba4..388f726 100644 --- a/docs/options.md +++ b/docs/options.md @@ -41,6 +41,7 @@ Fixed when a session is built; changing one means a new session (tens of millise | `keep_maps` | off | on, off | Keep the repair stage's maps after the pair: the Stream's cadence re-timing reads them. Raises `repair` to 1. | `keep_maps (0, 1)` | | `protection_mask` | off | on, off | The host paints a mask of pixels never to interpolate (`write_mask`: one byte per pixel, 0 interpolates, anything else protects): a HUD, a crosshair, subtitles whose place the host knows. Protected pixels take the source frames' value like statically protected ones, and the render keeps the motion around them off them. Works with or without `static_protection`; a rewritten mask applies from the next pair. | `protection_mask (0, 1)` | | `nearest_source` | off | on, off | Where the two warped frames disagree over a whole region (a flick the network could not follow), the in-between takes the nearer source frame alone there: a short hold instead of the fog of two misplaced copies. Off by default: on captured 120 fps video it costs 0.53 dB and gains no sharpness; a host whose viewers prefer the hold on fast turns turns it on. The environment variable `FH_NEAR=1` sets the default to on. | `none (FH_NEAR=1 in the environment)` | +| `precise_motion` | off | on, off | Frames above 1280 px on the long side: the motion is found on a 400-line picture and refined on an 800-line one (`internal` `Auto` picks that size), instead of one pass at 480 lines. On captured 1080p 120 fps video (UVG, 14 clips) 33.83 against 33.42 dB and LPIPS 0.159 against 0.165; on synthetic 1080p pans and moving objects 35.41 against 33.35 dB. Costs GPU time: 1.94 against 1.06 ms per generated frame at 1080p on an RTX 5090. No effect on smaller frames or with `fast_render`. The environment variable `FH_PRECISE=1` sets the default to on. | `none (FH_PRECISE=1 in the environment)` | | `tune` | none | an autotune table | Kernel choices from an earlier autotune on this device at this size (the text of a `TuneTable`); without one the engine tunes at the first build, or uses the fixed routing when the autotune is off. | `tune (text, NULL = none)` | ## Stream options diff --git a/shaders/keep_static_age.wgsl b/shaders/keep_static_age.wgsl index 8b6a3aa..001307b 100644 --- a/shaders/keep_static_age.wgsl +++ b/shaders/keep_static_age.wgsl @@ -22,12 +22,17 @@ const FAR = array(4, 8, 16, 32); const VERIFIED: f32 = 2.0; // a pixel flat in both frames is no evidence of standing still (see keep_static_age_tex.wgsl) const FLAT: f32 = 0.03; -fn decide(o: u32, plane: u32, still: bool, moved: bool) -> f16 { +// `same`: the pixel did not change in this pair, flat or not. Its run length goes out negated where the pixel is not +// protected: the render's still test trusts "this pixel stayed" only after it stayed for several pairs, because one +// pair cannot tell an overlay from a moving flat detail that covers the same place in both frames. +fn decide(o: u32, plane: u32, still: bool, moved: bool, same: bool) -> f16 { let raw = select(0.0, min(f32(state[o]) + 1.0, 255.0), still); let seen = select(0.0, min(f32(state[plane + o]) + select(0.0, 1.0, moved), 255.0), still); + let run = select(0.0, min(f32(state[2u * plane + o]) + 1.0, 255.0), same); state[o] = f16(raw); state[plane + o] = f16(seen); - return f16(select(0.0, raw, seen >= VERIFIED)); + state[2u * plane + o] = f16(run); + return f16(select(-run, raw, seen >= VERIFIED)); } fn spread(x: u32, y: u32, plane: u32) -> f32 { @@ -66,6 +71,7 @@ fn age(@builtin(global_invocation_id) gid: vec3) { let xx = u32(clamp(q.x, 0, i32(p.w) - 1)); let yy = u32(clamp(q.y, 0, i32(p.h) - 1)); moved = moved || diff(yy * p.wp + xx, plane) >= p.thr; } } - let aged = decide(o, plane, diff(o, plane) < p.thr && spread(gid.x, gid.y, plane) >= FLAT, moved); + let same = diff(o, plane) < p.thr; + let aged = decide(o, plane, same && spread(gid.x, gid.y, plane) >= FLAT, moved, same); outp[o] = select(aged, f16(255.0), p.use_mask != 0u && f32(mask[o]) > 0.5); } diff --git a/shaders/keep_static_age_tex.wgsl b/shaders/keep_static_age_tex.wgsl index 55fc7c8..fd090ce 100644 --- a/shaders/keep_static_age_tex.wgsl +++ b/shaders/keep_static_age_tex.wgsl @@ -26,12 +26,17 @@ const VERIFIED: f32 = 2.0; // is fill in both, so protecting it erased the in-between's line there (a dotted, broken outline on anime pans). An // overlay has an edge of its own (a crosshair line, text, a panel's border); a flat panel inside needs no protection. const FLAT: f32 = 0.03; -fn decide(o: u32, plane: u32, still: bool, moved: bool) -> f16 { +// `same`: the pixel did not change in this pair, flat or not. Its run length goes out negated where the pixel is not +// protected: the render's still test trusts "this pixel stayed" only after it stayed for several pairs, because one +// pair cannot tell an overlay from a moving flat detail that covers the same place in both frames. +fn decide(o: u32, plane: u32, still: bool, moved: bool, same: bool) -> f16 { let raw = select(0.0, min(f32(state[o]) + 1.0, 255.0), still); let seen = select(0.0, min(f32(state[plane + o]) + select(0.0, 1.0, moved), 255.0), still); + let run = select(0.0, min(f32(state[2u * plane + o]) + 1.0, 255.0), same); state[o] = f16(raw); state[plane + o] = f16(seen); - return f16(select(0.0, raw, seen >= VERIFIED)); + state[2u * plane + o] = f16(run); + return f16(select(-run, raw, seen >= VERIFIED)); } fn spread(t: vec2, hi: vec2) -> f32 { let ca = textureLoad(tex_a, t, 0).rgb; @@ -63,6 +68,7 @@ fn main(@builtin(global_invocation_id) gid: vec3) { moved = moved || change(clamp(t + DIRS[i] * FAR[j], vec2(0), hi)) >= p.thr; } } let o = gid.y * p.wp + gid.x; - let aged = decide(o, p.hp * p.wp, change(t) < p.thr && spread(t, hi) >= FLAT, moved); + let same = change(t) < p.thr; + let aged = decide(o, p.hp * p.wp, same && spread(t, hi) >= FLAT, moved, same); outp[o] = select(aged, f16(255.0), p.use_mask != 0u && f32(mask[o]) > 0.5); } diff --git a/shaders/render.wgsl b/shaders/render.wgsl index 2efdd60..926f5a8 100644 --- a/shaders/render.wgsl +++ b/shaders/render.wgsl @@ -14,6 +14,8 @@ enable f16; // world (HUD text, a crosshair) is explained far better by staying, but the flow lives on an 8 x 8 px grid and drags // it along with the world; where staying wins the in-between keeps the still pixels. Only nearly identical pixels // count as still (0.03 of the range), so a world the network got wrong is not frozen into a blend of two frames. +// unchanged pairs before the still test (static variant) believes a pixel stayed +const STILL_RUN: f32 = 3.0; struct Par { H: u32, W: u32, hq: u32, wq: u32, f0: u32, v0: u32, fsx: f32, fsy: f32, vs: f32, cubic: u32, min_age: u32, decisive: u32 }; // flow_tq / vis_tq may be views; fsx/fsy: flow scale per axis @group(0) @binding(0) var A: array; @group(0) @binding(1) var B: array;