diff --git a/README.md b/README.md index f7bcce3..126ad34 100644 --- a/README.md +++ b/README.md @@ -17,7 +17,7 @@ Use Virtual Desktop or Steam Link with a **[supported rooted Quest Pro](https:// 2. Open **Setup → Set up** in QFT+. 3. Enable eye and face tracking on the headset. For Virtual Desktop, enable **Forward face/eye tracking to PC**. QFT+ sets up Steam Link sharing. 4. Connect through Virtual Desktop or Steam Link. Select **Start**. -5. Open **Calibration**. Use the Steam overlay to see the app in VR. +5. Open **Calibration**. Use the Steam overlay to see the app in VR. Install or update QFT+ Headset for the avatar guide. - After each headset restart, connect USB once to restore Wi-Fi access. - Use a trusted network. Wi-Fi setup allows ADB access to the headset. @@ -53,7 +53,7 @@ These credits cover shipped features. Each row links a source to its use in QFT+ | [VRCFT-SteamLink, danwillm](https://github.com/danwillm/VRCFT-SteamLink); [LinkFT, ykeara](https://github.com/ykeara/LinkFT) | References for [Steam Link support](src/bridges/vrcft/SteamLinkState.cs) and [setup](src/app/SteamVr.cs). | | [OpenCV](https://github.com/opencv/opencv/blob/4.13.0/modules/imgproc/src/contours.cpp) | Adapted [pupil outline detection](src/headset/pupil_contours.hpp). | | [OpenVR, Valve](https://github.com/ValveSoftware/openvr) | [Driver header](src/steamvr/openvr_driver.h) and [API](src/steamvr/driver.cpp). [Input profiles](src/steamvr/qftplus/resources/input) adapt Valve's SteamVR Touch files. | -| [Meta Avatars SDK](https://developers.meta.com/vr/downloads/package/meta-avatars-sdk/) | Adapted [shader](src/headset/calibration_scene.cpp) and [SDK mapping](src/headset/calibration_avatar.hpp). Includes avatar runtime and preset avatars. | +| [Meta Avatars SDK](https://developers.meta.com/vr/downloads/package/meta-avatars-sdk/) | Adapted [shader](src/headset/calibration_render.hpp) and [SDK mapping](src/headset/calibration_avatar.hpp). The headset app includes the runtime and presets; Windows receives rendered frames. | | [1€ filter, Casiez, Roussel and Vogel](https://gery.casiez.net/1euro/) | Method for [Python](src/tracking/eye_signal_filter.py) and [Java](src/headset/app/java/com/qftplus/headset/Convergence.java) eye smoothing. | | [Ava-256, Meta](https://github.com/facebookresearch/ava-256) | Training data for the mouth-camera encoder. | | [GNM, Google](https://github.com/google/GNM) | Used to create the synthetic training dataset. | diff --git a/src/app/AvatarPreview.cs b/src/app/AvatarPreview.cs new file mode 100644 index 0000000..42328b7 --- /dev/null +++ b/src/app/AvatarPreview.cs @@ -0,0 +1,95 @@ +using System.Buffers.Binary; +using System.ComponentModel; +using System.Diagnostics; +using System.IO.Compression; +using System.Net.Sockets; +using System.Windows.Media; +using System.Windows.Media.Imaging; + +namespace QFTPlus; + +internal static class AvatarPreview +{ + internal const int Width = 360, Height = 250; + + internal static async Task Run(Session session, Func pose, Action show, CancellationToken token) + { + if (string.IsNullOrEmpty(session.Config["adbTarget"]?.GetValue())) throw new IOException(Loc.Get("AvatarPreviewUnavailable")); + var assets = await HeadsetApp.SendAsync(session, token, ("avatarWanted", true)); + var library = assets["avatarLibrary"]?.GetValue(); + var preset = assets["avatarPreset"]?.GetValue(); + if (library is null || !library.StartsWith("/data/app/") || !library.EndsWith("/lib/arm64") + || preset is null || !preset.StartsWith("/data/user/0/com.qftplus.headset/files/avatar/") || preset.Contains("..")) + throw new IOException(Loc.Get("AvatarPreviewUnavailable")); + var adb = Adb.Exe(session.Root); + var target = session.Config["adbTarget"]?.GetValue() ?? ""; + var socket = "qft-avatar-" + Guid.NewGuid().ToString("N"); + var command = string.Join(" ", new[] { library + "/libqft_avatar_preview.so", library + "/libovravatar2.so", preset[..preset.LastIndexOf('/')], preset, socket }.Select(Adb.Quote)); + var info = Processes.Info(adb, ["-s", target, "shell", "-T", "su -c " + Adb.Quote(command)]); + info.RedirectStandardOutput = info.RedirectStandardError = true; + using var process = Process.Start(info) ?? throw new IOException(Loc.Get("AvatarPreviewUnavailable")); + using var timeout = CancellationTokenSource.CreateLinkedTokenSource(token); + timeout.CancelAfter(TimeSpan.FromSeconds(30)); + var errors = process.StandardError.ReadToEndAsync(); + string? port = null; + try + { + if (await process.StandardOutput.ReadLineAsync(timeout.Token) != "QFT_AVATAR_READY") + throw new IOException(Loc.Get("AvatarPreviewUnavailable")); + var output = process.StandardOutput.ReadToEndAsync(); + var forwarded = await Processes.RunAsync(adb, ["-s", target, "forward", "tcp:0", "localabstract:" + socket], timeout.Token); + if (forwarded.Code != 0 || !int.TryParse(forwarded.Text.Trim(), out var number) || number is < 1 or > 65535) + throw new IOException(Loc.Get("AvatarPreviewUnavailable")); + port = number.ToString(System.Globalization.CultureInfo.InvariantCulture); + using var client = new TcpClient(); + await client.ConnectAsync("127.0.0.1", number, timeout.Token); + var stream = client.GetStream(); + var header = new byte[8]; + await stream.ReadExactlyAsync(header, timeout.Token); + if (BinaryPrimitives.ReadUInt32LittleEndian(header) != 0x31564651 || BinaryPrimitives.ReadInt32LittleEndian(header.AsSpan(4)) != 0) + throw new IOException(Loc.Get("AvatarPreviewUnavailable")); + while (true) + { + timeout.CancelAfter(TimeSpan.FromSeconds(5)); + var values = pose(); + var packet = new byte[76 * sizeof(float)]; + Buffer.BlockCopy(values, 0, packet, 0, packet.Length); + await stream.WriteAsync(packet, timeout.Token); + await stream.ReadExactlyAsync(header, timeout.Token); + var size = BinaryPrimitives.ReadInt32LittleEndian(header.AsSpan(4)); + if (BinaryPrimitives.ReadUInt32LittleEndian(header) != 0x31564651 || size is < 1 or > Width * Height * 4 + 1024) + throw new IOException(Loc.Get("AvatarPreviewUnavailable")); + var frame = new byte[size]; + await stream.ReadExactlyAsync(frame, timeout.Token); + show(Decode(frame)); + await Task.Delay(80, token); + } + } + finally + { + try { process.Kill(); } catch (Exception error) when (error is InvalidOperationException or Win32Exception) { } + if (port is not null) await Processes.RunAsync(adb, ["-s", target, "forward", "--remove", "tcp:" + port], seconds: 5); + if (errors.IsCompletedSuccessfully && errors.Result.Length > 0) Session.Log(Path.Combine(session.Root, "studio.log"), errors.Result); + } + } + + internal static BitmapSource Decode(byte[] frame) + { + using var zip = new ZLibStream(new MemoryStream(frame), CompressionMode.Decompress); + var rgba = new byte[Width * Height * 4]; + zip.ReadExactly(rgba); + if (zip.ReadByte() != -1) throw new IOException(Loc.Get("AvatarPreviewUnavailable")); + var pixels = new byte[rgba.Length]; + for (var y = 0; y < Height; y++) + for (var x = 0; x < Width; x++) + { + var source = ((Height - 1 - y) * Width + x) * 4; + var dest = (y * Width + x) * 4; + pixels[dest] = rgba[source + 2]; pixels[dest + 1] = rgba[source + 1]; + pixels[dest + 2] = rgba[source]; pixels[dest + 3] = rgba[source + 3]; + } + var image = BitmapSource.Create(Width, Height, 96, 96, PixelFormats.Bgra32, null, pixels, Width * 4); + image.Freeze(); + return image; + } +} diff --git a/src/app/FaceGuide.cs b/src/app/FaceGuide.cs index bcfbf0a..625a7b8 100644 --- a/src/app/FaceGuide.cs +++ b/src/app/FaceGuide.cs @@ -1,265 +1,123 @@ using System.Globalization; using System.Text.Json.Nodes; using System.Windows; +using System.Windows.Automation; using System.Windows.Controls; using System.Windows.Media; -using System.Windows.Media.Animation; +using System.Windows.Media.Imaging; namespace QFTPlus; public sealed class FaceGuide : Control { - static readonly double[] NeutralMouth = [32, 170, 1 / 3.0, 14.67, 14.67], - SmileMouth = [38, 168, 1 / 3.0, 26.67, 26.67], OpenMouth = [22, 176, 1, -20, 20], - FlatMouth = [34, 176, 1 / 3.0, 0, 0], PursedMouth = [12, 176, 1 / 3.0, 2, 2]; - static readonly double[] Neutral = Pose(new(), ""); - static readonly DependencyProperty PhaseProperty = DependencyProperty.Register("Phase", typeof(double), typeof(FaceGuide), new FrameworkPropertyMetadata(1.0, FrameworkPropertyMetadataOptions.AffectsRender)); - static readonly DependencyProperty PulseProperty = DependencyProperty.Register("Pulse", typeof(double), typeof(FaceGuide), new FrameworkPropertyMetadata(0.0, FrameworkPropertyMetadataOptions.AffectsRender)); - public static readonly DependencyProperty AccentProperty = DependencyProperty.Register(nameof(Accent), typeof(Brush), typeof(FaceGuide), new FrameworkPropertyMetadata(Brushes.DodgerBlue, FrameworkPropertyMetadataOptions.AffectsRender)); - public Brush Accent { get => (Brush)GetValue(AccentProperty); set => SetValue(AccentProperty, value); } - double[] from = Neutral, target = Neutral; - bool throughNeutral, pupils, shown; - int index = -1, count; - - static FaceGuide() => ForegroundProperty.OverrideMetadata(typeof(FaceGuide), new FrameworkPropertyMetadata(Brushes.Black, FrameworkPropertyMetadataOptions.Inherits | FrameworkPropertyMetadataOptions.AffectsRender)); - - public FaceGuide() => Unloaded += (_, _) => + Session? session; + CancellationTokenSource? rendering; + BitmapSource? image; + float[] pose = new float[76]; + bool pupils, loading; + public FaceGuide() { - BeginAnimation(PhaseProperty, null); - BeginAnimation(PulseProperty, null); - shown = false; - }; + AutomationProperties.SetName(this, Loc.Get("AvatarPreview")); + Loaded += (_, _) => Start(); + Unloaded += (_, _) => Stop(); + IsVisibleChanged += (_, _) => { if (IsVisible) Start(); else Stop(); }; + } + + internal void Configure(Session? value) { if (!ReferenceEquals(session, value)) Stop(); session = value; Start(); } - public void Show(JsonObject values, string prompt, string kind, bool animate, int poseIndex = -1, int captured = 0) + public void Show(JsonObject values, string condition, string kind, bool animate, int poseIndex = -1, int captured = 0) { - var next = Pose(values, prompt.ToLowerInvariant()); - animate &= SystemParameters.ClientAreaAnimation && kind != "pupils"; - if (animate && poseIndex == index && captured > count) - BeginAnimation(PulseProperty, new DoubleAnimation(1, 0, TimeSpan.FromSeconds(.6))); - var changed = !target.SequenceEqual(next) || poseIndex != index || pupils != (kind == "pupils"); + pose = Pose(values, condition, animate); pupils = kind == "pupils"; - count = captured; - if (changed || !animate) - { - from = CurrentPose(); - throughNeutral = index >= 0 && poseIndex != index || from[5] > .05 && next[5] > .05 && Math.Abs(from[6] - next[6]) > 90; - if (from[5] < .05) from[6] = next[6]; - if (next[5] < .05) next[6] = from[6]; - target = next; - BeginAnimation(PhaseProperty, animate && shown - ? new DoubleAnimation(0, 1, TimeSpan.FromSeconds(throughNeutral ? .7 : .4)) { EasingFunction = new CubicEase { EasingMode = EasingMode.EaseInOut } } - : null); - InvalidateVisual(); - } - index = poseIndex; - shown = true; + if (pupils) Stop(); else Start(); + InvalidateVisual(); } - double[] CurrentPose() - { - var phase = (double)GetValue(PhaseProperty); - var a = throughNeutral && phase >= .5 ? Neutral : from; - var b = throughNeutral && phase < .5 ? Neutral : target; - var t = throughNeutral ? phase < .5 ? phase * 2 : (phase - .5) * 2 : phase; - var pose = a.Zip(b, (x, y) => x + (y - x) * t).ToArray(); - if (throughNeutral) pose[6] = phase < .5 ? from[6] : target[6]; - return pose; - } + void Stop() { rendering?.Cancel(); image = null; } - static double[] Pose(JsonObject targets, string prompt) + async void Start() { - double Value(string key) => double.TryParse(targets[key]?.ToString(), NumberStyles.Float, CultureInfo.InvariantCulture, out var value) ? value : 0; - var faceGoal = new double[29]; - double right = Value("CheekPuffRight"), left = Value("CheekPuffLeft"), tongue = 0, angle = 0, length = 0; - var mouth = NeutralMouth; - if (right > 0 || left > 0 || Value("CheekSuckRight") > 0 || Value("CheekSuckLeft") > 0) - mouth = PursedMouth; - else if (Value("visibility") > 0) + if (!IsLoaded || !IsVisible || pupils || session is not { } active || rendering is not null) return; + using var cancel = new CancellationTokenSource(); + rendering = cancel; + try { - double h = Value("horizontal"), v = Value("vertical"); - mouth = FlatMouth; - tongue = 1; - angle = v > 0 ? 180 + h * 35 : -h * 35; - length = (12 + Value("extension") * 16) * (v > 0 ? .7 : 1 - Math.Min(v, 0) * .1); - } - else if (prompt.Contains("open")) - mouth = OpenMouth; - else if (Value("MouthSmile") > 0 || prompt.Contains("smil")) - mouth = SmileMouth; - else if (Value("LipSuck") > 0) - mouth = FlatMouth; - var open = Math.Min(1, Value("MouthOpen")); - if (open > 0) - mouth = mouth.Zip(OpenMouth, (a, b) => a + open * (b - a)).ToArray(); - right += .35 * Value("TongueBulgeRight") - .5 * Value("CheekSuckRight"); - left += .35 * Value("TongueBulgeLeft") - .5 * Value("CheekSuckLeft"); - double Both(string name) => (Value(name + "Left") + Value(name + "Right")) / 2; - var kiss = Math.Min(Both("LipPuckerUpper"), Both("LipPuckerLower")); - var pout = Both("LipPuckerLower") - kiss; - var m = (double[])mouth.Clone(); - void Shape(double amount, double width, double upper, double lower) - { - m[0] += amount * width; - m[3] += amount * upper; - m[4] += amount * lower; + while (!cancel.IsCancellationRequested) + { + loading = true; InvalidateVisual(); + try { await AvatarPreview.Run(active, () => pose, frame => { image = frame; InvalidateVisual(); }, cancel.Token); } + catch (Exception error) when (error is IOException or OperationCanceledException or System.ComponentModel.Win32Exception or System.Net.Sockets.SocketException) + { + if (!cancel.IsCancellationRequested) Session.Log(Path.Combine(active.Root, "studio.log"), "Avatar preview: " + error.Message); + } + image = null; loading = false; InvalidateVisual(); + await Task.Delay(5000, cancel.Token); + } } + catch (OperationCanceledException) { } + finally { rendering = null; if (IsLoaded && IsVisible && !pupils && session is not null && cancel.IsCancellationRequested) Start(); } + } - Shape(Both("MouthCornerPull"), 6, -4, 12); - Shape(Both("MouthCornerSlant"), 2, 4, 4); - Shape(Both("MouthUpperDeepen"), -2, -10, 0); - Shape(kiss, -18, -20.67, -6.67); - Shape(pout, -4, -22, -16); - Shape(Both("LipSuckCorner"), -6, -11, -11); - Shape(Math.Max(Value("JawClench"), Value("JawMandibleRaise")), 2, -8, -8); - Shape(Value("JawBackward"), -4, -3, -3); - m.CopyTo(faceGoal, 0); - faceGoal[5] = tongue; - faceGoal[6] = angle; - faceGoal[7] = length; - faceGoal[8] = right; - faceGoal[9] = left; - var mouthMoves = new[] - { - "MouthCornerPull", - "MouthCornerSlant", - "MouthUpperDeepen", - "LipPuckerUpper", - "LipPuckerLower", - "LipSuckCorner" - }.Sum(Both) + new[] - { - "JawClench", - "JawMandibleRaise", - "JawBackward", - "MouthUpperLeft", - "MouthUpperRight", - "MouthLowerLeft", - "MouthLowerRight" - }.Sum(Value); - var lift = 3 * Both("MouthCornerSlant") - 2 * pout - 2 * Value("JawBackward"); - new[] + internal static float[] Pose(JsonObject values, string condition, bool animate) + { + float V(string name) => float.TryParse(values[name]?.ToString(), NumberStyles.Float, CultureInfo.InvariantCulture, out var v) && float.IsFinite(v) ? Math.Clamp(v, -1, 1) : 0; + var weights = new float[76]; + void Set(int index, params string[] keys) => weights[index] = Math.Clamp(keys.Max(V), 0, 1); + for (var side = 0; side < 2; side++) { - lift, - 6 * (Value("MouthUpperRight") - Value("MouthUpperLeft")), - 6 * (Value("MouthLowerRight") - Value("MouthLowerLeft")), - Math.Min(1, mouthMoves), - Both("MouthUpperDeepen"), - 8 * Value("BrowLowererLeft"), - 8 * Value("BrowLowererRight"), - 8 * Value("BrowPinchLeft"), - 8 * Value("BrowPinchRight"), - 10 * Value("BrowInnerUpLeft"), - 10 * Value("BrowInnerUpRight"), - 10 * Value("BrowOuterUpLeft"), - 10 * Value("BrowOuterUpRight"), - Value("NasalDilationLeft") - Value("NasalConstrictLeft"), - Value("NasalDilationRight") - Value("NasalConstrictRight"), - Value("JawClench"), - Value("JawBackward"), - Math.Min(1, Math.Max(Value("EyeClosed"), .6 * Value("EyeSquint"))), - Value("LookUp") - }.CopyTo(faceGoal, 10); - return faceGoal; + var suffix = side == 0 ? "Right" : "Left"; + foreach (var (index, name) in new (int, string)[] { (0,"BrowLowerer"), (2,"CheekPuff"), (4,"CheekSquint"), (6,"CheekSuck"), + (10,"MouthDimple"), (22,"BrowInnerUp"), (30,"MouthFrown"), (32,"MouthCornerPull"), (38,"MouthPress"), + (40,"LipPuckerUpper"), (42,"MouthStretch"), (48,"MouthTightener"), (51,"MouthLowerDown"), (55,"NoseSneer"), (57,"BrowOuterUp"), (59,"EyeWide"), (61,"MouthUpperUp") }) + Set(index + side, name + suffix); + Set(side, "BrowLowerer" + suffix, "BrowPinch" + suffix); + Set(12 + side, "EyeClosed", "EyeClosed" + suffix); + Set(28 + side, "EyeSquint", "EyeSquint" + suffix); + Set(32 + side, "MouthSmile", "MouthCornerPull" + suffix, "MouthCornerSlant" + suffix); + Set(38 + side, "MouthPress" + suffix, "JawClench"); + Set(48 + side, "MouthTightener" + suffix, "JawClench"); + Set(55 + side, "NoseSneer" + suffix, "NasalDilation" + suffix, "NasalConstrict" + suffix); + Set(61 + side, "MouthUpperUp" + suffix, "MouthUpperDeepen" + suffix); + weights[2 + side] = Math.Max(weights[2 + side], Math.Max(0, .35f * V("TongueBulge" + suffix))); + Set(40 + side, "LipPuckerUpper" + suffix, "LipPuckerLower" + suffix); + Set(44 + side * 2, "LipSuck", "LipSuckLower" + suffix, "LipSuckCorner" + suffix); + Set(45 + side * 2, "LipSuck", "LipSuckUpper" + suffix, "LipSuckCorner" + suffix); + Set(34 + side * 2, "LipFunnelLower" + suffix); + Set(35 + side * 2, "LipFunnelUpper" + suffix); + weights[(V("LookUp") < 0 ? 14 : 20) + side] = Math.Abs(V("LookUp")); + } + Set(24, "JawOpen", "MouthOpen"); + if (condition is "anchor:jaw_open" or "target:JawOpen") weights[24] = 1; + weights[25] = weights[53] = Math.Max(V("MouthUpperRight"), V("MouthLowerRight")); + weights[26] = weights[54] = Math.Max(V("MouthUpperLeft"), V("MouthLowerLeft")); + weights[27] = Math.Max(0, -V("JawBackward")); + Set(8, "JawMandibleRaise"); + weights[72] = Math.Max(0, V("visibility") * V("extension")); + weights[73] = V("horizontal"); weights[74] = V("vertical"); + if (weights[72] > 0) weights[24] = Math.Max(weights[24], .7f); + weights[75] = animate && SystemParameters.ClientAreaAnimation && V("LookUp") == 0 ? 1 : 0; + return weights; } protected override void OnRender(DrawingContext drawing) { - var scale = Math.Min(ActualWidth / 360, ActualHeight / 250); - var pulse = (double)GetValue(PulseProperty); - var pop = pulse > .5 ? 1 + .04 * Math.Sin(2 * Math.PI * (1 - pulse)) : 1; - var matrix = Matrix.Identity; - matrix.ScaleAt(pop, pop, 180, 130); - matrix.Scale(scale, scale); - matrix.Translate((ActualWidth - 360 * scale) / 2, 0); - drawing.PushTransform(new MatrixTransform(matrix)); - var ink = Foreground; - var accent = Accent; - void Draw(string data, Brush? fill, Pen? pen, Transform? at, double opacity) - { - drawing.PushOpacity(opacity); - drawing.PushTransform(at ?? Transform.Identity); - drawing.DrawGeometry(fill, pen, Geometry.Parse(data)); - drawing.Pop(); - drawing.Pop(); - } - void Line(string data, Brush brush, Transform? at = null, double opacity = 1, double width = 8) => - Draw(data, null, new Pen(brush, width) { StartLineCap = PenLineCap.Round, EndLineCap = PenLineCap.Round, LineJoin = PenLineJoin.Round }, at, opacity); - void Fill(string data, double opacity, Transform? at = null) => Draw(data, accent, null, at, opacity); if (pupils) { - Line("M 160,125 A 20,20 0 1 1 200,125 A 20,20 0 1 1 160,125 M 180,125 L 180,125", ink); - drawing.Pop(); - return; - } - var p = CurrentPose(); - string F(FormattableString value) => FormattableString.Invariant(value); - var chin = 230 - 4 * p[26]; - var head = F($"M 180,22 C 234,22 258,62 258,112 C {258 + p[8] * 30},190 220,{chin} 180,{chin} C 140,{chin} {102 - p[9] * 30},190 102,112 C 102,62 126,22 180,22 Z"); - Line(head, ink); - if (pulse > 0) - Line(head, accent, null, pulse); - var closed = Math.Clamp(p[27], 0, 1); - var look = 6 * p[28]; - foreach (var x in new[] -{ 142.0, 218.0 }) - { - if (closed > .85) - Line(F($"M {x - 9},107 Q {x},112 {x + 9},107"), accent); - else - Line(F($"M {x},{107 - 9 * (1 - closed) - look} V {107 + 9 * (1 - closed) - look}"), closed > .1 || Math.Abs(look) > .5 ? accent : ink); + drawing.DrawEllipse(null, new Pen(Foreground, 5), new Point(ActualWidth / 2, ActualHeight / 2), 20, 20); + drawing.DrawEllipse(Foreground, null, new Point(ActualWidth / 2, ActualHeight / 2), 3, 3); } - - var flare = (p[23] + p[24]) / 2; - if (Math.Abs(flare) > .02) + else if (image is not null) { - Line("M 180,98 V 124", ink); - Line(F($"M {171 - 4 * flare},135 Q {173 - 3 * flare},127 180,129 Q {187 + 3 * flare},127 {189 + 4 * flare},135"), Math.Abs(flare) > .05 ? accent : ink, null, 1, 6); + var scale = Math.Min(ActualWidth / AvatarPreview.Width, ActualHeight / AvatarPreview.Height); + var width = AvatarPreview.Width * scale; var height = AvatarPreview.Height * scale; + drawing.DrawImage(image, new Rect((ActualWidth - width) / 2, (ActualHeight - height) / 2, width, height)); } else - Line("M 180,98 V 124 Q 180,134 170,134", ink); - foreach (var (side, down, pinch, inner, outer) in new[] -{ (-1.0, p[15], p[17], p[19], p[21]), (1.0, p[16], p[18], p[20], p[22]) }) - { - double ix = 180 + side * (20 - pinch), ox = 180 + side * (56 - pinch * .3), iy = 86 + down - inner, oy = 84 + down - outer; - Line(F($"M {ox},{oy} Q {(ix + ox) / 2},{Math.Min(iy, oy) - 5} {ix},{iy}"), down + pinch + inner + outer > .1 ? accent : ink); - } - - if (p[14] > .01) - Line("M 173,109 L 187,109 M 174,116 L 186,116", accent, null, Math.Min(1, p[14]), 3); - foreach (var (puff, x, outward) in new[] -{ (p[9], 136.0, -1.0), (p[8], 224.0, 1.0) }) - if (puff < -.02) - Line(F($"M {x + outward * 8},148 Q {x - outward * (6 + 24 * -puff)},168 {x + outward * 8},188"), accent, null, Math.Min(1, -puff * 3), 5); - else if (puff > .01) - { - var rx = 8 + 22 * puff; - var ry = 6 + 16 * puff; - var cx = x + outward * 6 * puff; - Fill(F($"M {cx - rx},168 A {rx},{ry} 0 1 0 {cx + rx},168 A {rx},{ry} 0 1 0 {cx - rx},168 Z"), .15 + .3 * Math.Min(puff, 1)); - } - - if (p[5] > .01) { - var at = new TransformGroup(); - at.Children.Add(new ScaleTransform(p[5], p[5], 180, 176)); - at.Children.Add(new RotateTransform(p[6], 180, 176)); - var lean = Math.Clamp(p[6] - 180 * Math.Round(p[6] / 180), -60, 60); - var trim = 15 * Math.Tan(lean * Math.PI / 180); - var tongue = F($"M 165,{176 + trim} V {176 + p[7]} A 15,15 0 0 0 195,{176 + p[7]} V {176 - trim}"); - Fill(tongue + " Z", .2, at); - Line(tongue, accent, at); + var text = new FormattedText(Loc.Get(loading ? "AvatarPreviewLoading" : "AvatarPreviewUnavailable"), CultureInfo.CurrentUICulture, + FlowDirection, new Typeface(FontFamily, FontStyle, FontWeight, FontStretch), 14, Foreground, VisualTreeHelper.GetDpi(this).PixelsPerDip) + { MaxTextWidth = Math.Max(1, ActualWidth - 32), TextAlignment = TextAlignment.Center }; + drawing.DrawText(text, new Point(16, Math.Max(0, (ActualHeight - text.Height) / 2))); } - - var shift = (p[11] + p[12]) / 2; - double w = p[0], y = p[1], k = p[2] * p[0], leftX = 180 - w + shift, rightX = 180 + w + shift, corner = y - p[10]; - var bite = p[25]; - if (bite > .05) - { - double h = 7 * bite, third = (rightX - leftX) / 3; - Line(F($"M {leftX},{y - h} H {rightX} V {y + h} H {leftX} Z M {leftX},{y} H {rightX} M {leftX + third},{y - h} V {y + h} M {rightX - third},{y - h} V {y + h}"), accent, null, 1, 5); - } - else - Line(F($"M {leftX},{corner} C {180 - k + p[11]},{y + p[3]} {180 + k + p[11]},{y + p[3]} {rightX},{corner} C {180 + k + p[12]},{y + p[4]} {180 - k + p[12]},{y + p[4]} {leftX},{corner} Z"), p[13] > .1 ? accent : ink); - drawing.Pop(); } } diff --git a/src/app/Localization/Strings.resx b/src/app/Localization/Strings.resx index ce3a276..350d0aa 100644 --- a/src/app/Localization/Strings.resx +++ b/src/app/Localization/Strings.resx @@ -1114,6 +1114,15 @@ Input minimum QFT+ Windows app. StudioControls.cs | StudioSearch.cs + + Loading the headset avatar… + + + Connect your headset and update QFT+ Headset to see the avatar. You can still follow the instructions. + + + Calibration avatar + Input minimum must be below input maximum. Output minimum cannot exceed output maximum. Changes are not saved until the ranges are valid. QFT+ Windows app. StudioControls.cs diff --git a/src/app/StudioCalibration.cs b/src/app/StudioCalibration.cs index 4a97d14..ed255b2 100644 --- a/src/app/StudioCalibration.cs +++ b/src/app/StudioCalibration.cs @@ -24,8 +24,11 @@ public partial class StudioWindow bool FreshState() => state["time"]?.GetValue() is { } time && Math.Abs(DateTimeOffset.UtcNow.ToUnixTimeMilliseconds() / 1000.0 - time) < 3; static string GroupTitle(string kind) => kind == "pupils" ? Loc.Get("Pupils") : kind.StartsWith("benchmark") ? Loc.Get("Benchmark") : Loc.Get("Face"); - void DrawFace(JsonObject targets) => - face.Show(targets, poseTitle.Text, kind, !preview, recording ? state["index"]?.GetValue() ?? -1 : -1, state["count"]?.GetValue() ?? 0); + void DrawFace(JsonObject targets) + { + face.Configure(preview ? null : session); + face.Show(targets, state["condition"]?.GetValue() ?? "", kind, !preview, recording ? state["index"]?.GetValue() ?? -1 : -1, state["count"]?.GetValue() ?? 0); + } void ResizeGuide() { diff --git a/src/app/StudioWindow.xaml b/src/app/StudioWindow.xaml index d0d46dc..604d4b6 100644 --- a/src/app/StudioWindow.xaml +++ b/src/app/StudioWindow.xaml @@ -255,7 +255,7 @@ - + diff --git a/src/headset/app/java/com/qftplus/headset/AvatarAssets.java b/src/headset/app/java/com/qftplus/headset/AvatarAssets.java new file mode 100644 index 0000000..a0803ea --- /dev/null +++ b/src/headset/app/java/com/qftplus/headset/AvatarAssets.java @@ -0,0 +1,35 @@ +package com.qftplus.headset; + +import android.content.Context; +import java.io.*; +import java.nio.file.*; +import java.util.Arrays; +import java.util.concurrent.ThreadLocalRandom; + +final class AvatarAssets { + static synchronized File prepare(Context context, boolean choose) throws IOException { + File folder = new File(context.getFilesDir(), "avatar"); + Files.createDirectories(folder.toPath()); + copy(context, "avatar/OvrAvatar2Assets.zip", new File(folder, "OvrAvatar2Assets.zip")); + String[] presets = context.getAssets().list("avatar/presets"); + if (presets == null || presets.length == 0) throw new IOException(context.getString(R.string.avatar_preview_unavailable)); + var settings = context.getSharedPreferences("settings", 0); + String selected = settings.getString("avatarPreset", ""); + if (choose || !Arrays.asList(presets).contains(selected)) { + selected = presets[ThreadLocalRandom.current().nextInt(presets.length)]; + settings.edit().putString("avatarPreset", selected).apply(); + } + File preset = new File(folder, selected); + copy(context, "avatar/presets/" + selected, preset); + return preset; + } + + private static void copy(Context context, String asset, File destination) throws IOException { + if (destination.isFile()) return; + File temporary = new File(destination.getPath() + ".tmp"); + try (InputStream input = context.getAssets().open(asset)) { + Files.copy(input, temporary.toPath(), StandardCopyOption.REPLACE_EXISTING); + Files.move(temporary.toPath(), destination.toPath(), StandardCopyOption.REPLACE_EXISTING); + } finally { temporary.delete(); } + } +} diff --git a/src/headset/app/java/com/qftplus/headset/CalibrationActivity.kt b/src/headset/app/java/com/qftplus/headset/CalibrationActivity.kt index 51d1609..76f8560 100644 --- a/src/headset/app/java/com/qftplus/headset/CalibrationActivity.kt +++ b/src/headset/app/java/com/qftplus/headset/CalibrationActivity.kt @@ -108,10 +108,8 @@ class CalibrationActivity : ComponentActivity() { renderer = Thread({ val error = try { val folder = File(filesDir, "avatar").apply { mkdirs() } - val core = File(folder, "OvrAvatar2Assets.zip") - if (!pupils && !core.exists()) assets.open("avatar/OvrAvatar2Assets.zip").use { input -> File(folder, "core.tmp").also { t -> t.outputStream().use { input.copyTo(it) }; t.renameTo(core) } } - val presets = if (pupils) emptyArray() else assets.list("avatar/presets").orEmpty() - val preset = presets.randomOrNull()?.let { name -> Log.i("QFTCalibration", "Avatar preset $name"); assets.open("avatar/presets/$name").use { it.readBytes() } } + val preset = if (pupils) null else runCatching { AvatarAssets.prepare(this, true).readBytes() } + .onFailure { Log.w("QFTCalibration", "Avatar unavailable", it) }.getOrNull() val space = if (pupils) null else runCatching { SetupSpace.read().toTypedArray() }.onFailure { Log.w("QFTCalibration", "Setup space unavailable", it) }.getOrNull() if (!pupils && preset == null) avatar = 2 runScene(applicationInfo.nativeLibraryDir + "/libovravatar2.so", folder.path, preset, scene, pupils, space) @@ -120,7 +118,10 @@ class CalibrationActivity : ComponentActivity() { while (calibration.done && !calibration.cancelled() && TrackingService.calibration === calibration && TrackingService.running && SystemClock.elapsedRealtime() < fitting) Thread.sleep(100) if (!calibration.done || TrackingService.calibration === calibration) - calibration.fail(if (error.isEmpty()) getString(R.string.calibration_cancelled_previous_calibration_retained) else error) + { + if (error.isNotEmpty()) Log.e("QFTCalibration", error) + calibration.fail(getString(if (error.isEmpty()) R.string.calibration_cancelled_previous_calibration_retained else R.string.calibration_scene_failed)) + } if (TrackingService.calibration === calibration) TrackingService.calibration = null Log.i("QFTCalibration", calibration.message) TrackingService.lastCalibration = calibration.message; TrackingService.calibrating = false @@ -129,6 +130,10 @@ class CalibrationActivity : ComponentActivity() { }, "qft-calibration-scene").also { it.start() } } + @Suppress("unused") private fun inputLabel(index: Int): String = getString(intArrayOf( + R.string.calibration, R.string.calibration_input_leave, R.string.calibration_input_start, + R.string.calibration_input_select, R.string.calibration_input_point)[index]) + @Suppress("unused") private fun updateScene(focused: Boolean): Int { val now = SystemClock.elapsedRealtime() if (!resumed) return -1 diff --git a/src/headset/app/java/com/qftplus/headset/ControlReceiver.java b/src/headset/app/java/com/qftplus/headset/ControlReceiver.java index 2c0651f..0db47d8 100644 --- a/src/headset/app/java/com/qftplus/headset/ControlReceiver.java +++ b/src/headset/app/java/com/qftplus/headset/ControlReceiver.java @@ -9,6 +9,21 @@ public final class ControlReceiver extends BroadcastReceiver { static final int[] RATES = {0, 30, 20}; public void onReceive(Context context, Intent intent) { + if (intent.getBooleanExtra("avatarWanted", false)) { + PendingResult reply = goAsync(); + new Thread(() -> { + try { + File preset = AvatarAssets.prepare(context, false); + reply.setResultCode(0); + reply.setResultData(new JSONObject().put("avatarPreset", preset.getPath()) + .put("avatarLibrary", context.getApplicationInfo().nativeLibraryDir).toString()); + } catch (IOException | JSONException error) { + android.util.Log.w("QFTCalibration", "Prepare avatar preview", error); + reply.setResultCode(2); reply.setResultData(context.getString(R.string.avatar_preview_unavailable)); + } finally { reply.finish(); } + }, "qft-avatar-assets").start(); + return; + } SharedPreferences settings = context.getSharedPreferences("settings", 0); int capabilities = capabilities(context); String problem = intent.hasExtra("rate") && java.util.Arrays.stream(RATES).noneMatch(r -> r == intent.getIntExtra("rate", -1)) ? context.getString(R.string.choose_an_update_rate_of_0_fastest_30_or_20) diff --git a/src/headset/app/res/values/strings.xml b/src/headset/app/res/values/strings.xml index 362bcd2..f2aa74c 100644 --- a/src/headset/app/res/values/strings.xml +++ b/src/headset/app/res/values/strings.xml @@ -131,6 +131,12 @@ Offset on off start stop status Open your mouth wide + The avatar could not load. You can still follow the calibration instructions. + Calibration could not open. Restart QFT+ Headset and try again. Your previous calibration is unchanged. + Leave calibration + Start calibration + Select + Point OSC apps Other receivers Output diff --git a/src/headset/avatar_preview.cpp b/src/headset/avatar_preview.cpp new file mode 100644 index 0000000..5890041 --- /dev/null +++ b/src/headset/avatar_preview.cpp @@ -0,0 +1,111 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +static void require(bool ok, const char* what) { if (!ok) throw std::runtime_error(what); } +#include "calibration_avatar.hpp" +#include "calibration_render.hpp" + +int main(int argc, char** argv) { + if (argc != 5 || std::strlen(argv[4]) > 80) return 2; + try { + int server = socket(AF_UNIX, SOCK_STREAM, 0); + sockaddr_un address{}; address.sun_family = AF_UNIX; std::strcpy(address.sun_path + 1, argv[4]); + require(bind(server, reinterpret_cast(&address), offsetof(sockaddr_un, sun_path) + 1 + std::strlen(argv[4])) == 0 && listen(server, 1) == 0, "Open avatar preview socket"); + fputs("QFT_AVATAR_READY\n", stdout); fflush(stdout); + pollfd connection{server, POLLIN, 0}; + require(poll(&connection, 1, 30000) > 0, "Wait for avatar preview connection"); + int client = accept(server, nullptr, nullptr); close(server); + require(client >= 0, "Connect avatar preview"); + timeval timeout{10, 0}; setsockopt(client, SOL_SOCKET, SO_SNDTIMEO, &timeout, sizeof timeout); + FILE* output = fdopen(dup(client), "wb"); + require(output, "Open avatar frame stream"); + constexpr int width = 360, height = 250; + EGLDisplay display = eglGetDisplay(EGL_DEFAULT_DISPLAY); + require(eglInitialize(display, nullptr, nullptr), "Initialize avatar graphics"); + EGLint attributes[] = {EGL_RENDERABLE_TYPE, EGL_OPENGL_ES3_BIT, EGL_SURFACE_TYPE, EGL_PBUFFER_BIT, EGL_NONE}; + EGLConfig config; EGLint count = 0; + require(eglChooseConfig(display, attributes, &config, 1, &count) && count, "Choose avatar graphics format"); + EGLint ca[] = {EGL_CONTEXT_CLIENT_VERSION, 3, EGL_NONE}, pa[] = {EGL_WIDTH, width, EGL_HEIGHT, height, EGL_NONE}; + EGLContext context = eglCreateContext(display, config, EGL_NO_CONTEXT, ca); + EGLSurface surface = eglCreatePbufferSurface(display, config, pa); + require(eglMakeCurrent(display, surface, surface, context), "Open avatar graphics context"); + GLuint shader = program(avatarVertex, avatarFragment), framebuffer, color, depth; + glUniformBlockBinding(shader, glGetUniformBlockIndex(shader, "SkinBlock"), 0); + glGenFramebuffers(1, &framebuffer); glBindFramebuffer(GL_FRAMEBUFFER, framebuffer); + glGenRenderbuffers(1, &color); glBindRenderbuffer(GL_RENDERBUFFER, color); + glRenderbufferStorage(GL_RENDERBUFFER, GL_SRGB8_ALPHA8, width, height); + glFramebufferRenderbuffer(GL_FRAMEBUFFER, GL_COLOR_ATTACHMENT0, GL_RENDERBUFFER, color); + glGenRenderbuffers(1, &depth); glBindRenderbuffer(GL_RENDERBUFFER, depth); + glRenderbufferStorage(GL_RENDERBUFFER, GL_DEPTH_COMPONENT24, width, height); + glFramebufferRenderbuffer(GL_FRAMEBUFFER, GL_DEPTH_ATTACHMENT, GL_RENDERBUFFER, depth); + require(glCheckFramebufferStatus(GL_FRAMEBUFFER) == GL_FRAMEBUFFER_COMPLETE, "Create avatar frame"); + std::ifstream file(argv[3], std::ios::binary); + std::vector preset((std::istreambuf_iterator(file)), {}); + require(!preset.empty() && preset.size() < 32 * 1024 * 1024, "Read avatar preset"); + avatar::Guide guide; guide.start(argv[1], argv[2], std::move(preset)); + auto until = std::chrono::steady_clock::now() + std::chrono::seconds(20); + while (!guide.ready() && !guide.failed() && std::chrono::steady_clock::now() < until) { + guide.update(1 / 60.f); std::this_thread::sleep_for(std::chrono::milliseconds(16)); + } + require(guide.ready(), "Load avatar preview"); + uint32_t ready[] = {0x31564651, 0}; + fwrite(ready, sizeof ready, 1, output); fflush(output); + glViewport(0, 0, width, height); glEnable(GL_DEPTH_TEST); glEnable(GL_CULL_FACE); + glUseProgram(shader); + auto light = avatar::normalize({.25f, .8f, .55f}); + glUniform3f(glGetUniformLocation(shader, "u_LightDir"), light.x, light.y, light.z); + glUniform3f(glGetUniformLocation(shader, "u_CameraPos"), 0, 0, .8f); + glUniform3f(glGetUniformLocation(shader, "u_LightColor"), 3.93f, 3.77f, 3.61f); + glUniform3f(glGetUniformLocation(shader, "u_SkyColor"), .95f, 1.f, 1.1f); + glUniform3f(glGetUniformLocation(shader, "u_GroundColor"), .5f, .48f, .46f); + avatar::Mat projection{}; + projection.m[0] = 2.f / .62f; projection.m[5] = 2.f / .43f; + projection.m[10] = -1; projection.m[15] = 1; + glUniformMatrix4fv(glGetUniformLocation(shader, "u_ViewProj"), 1, GL_FALSE, projection.m); + std::vector pixels(width * height * 4), compressed(compressBound(pixels.size())); + float pose[76]; + while (true) { + size_t received = 0; + while (received < sizeof pose) { + pollfd input{client, POLLIN, 0}; + if (poll(&input, 1, 10000) <= 0) return 0; + auto n = read(client, reinterpret_cast(pose) + received, sizeof pose - received); + if (n <= 0) return 0; + received += size_t(n); + } + for (float v : pose) require(std::isfinite(v) && v >= -1 && v <= 1, "Invalid avatar pose"); + std::copy(pose, pose + 72, guide.pose.weights); + std::fill(std::begin(guide.pose.confidence), std::end(guide.pose.confidence), 1.f); + guide.tongue = {pose[72], pose[73], pose[74]}; guide.idle = pose[75] > 0; + guide.update(1 / 12.f); + auto model = avatar::translation({-guide.head.x, -(guide.head.y + .055f), -guide.head.z}); + glUniformMatrix4fv(glGetUniformLocation(shader, "u_Model"), 1, GL_FALSE, model.m); + glUniform2f(glGetUniformLocation(shader, "u_Bust"), guide.head.y - .22f, guide.head.y - .12f); + glClearColor(0, 0, 0, 0); glClear(GL_COLOR_BUFFER_BIT | GL_DEPTH_BUFFER_BIT); guide.draw(shader); + glReadPixels(0, 0, width, height, GL_RGBA, GL_UNSIGNED_BYTE, pixels.data()); + require(glGetError() == GL_NO_ERROR, "Read avatar frame"); + uLongf size = compressed.size(); + require(compress2(compressed.data(), &size, pixels.data(), pixels.size(), 1) == Z_OK, "Compress avatar frame"); + uint32_t header[] = {0x31564651, uint32_t(size)}; + if (fwrite(header, sizeof header, 1, output) != 1 || fwrite(compressed.data(), size, 1, output) != 1 || fflush(output)) return 0; + } + } catch (const std::exception& error) { fprintf(stderr, "Avatar preview: %s\n", error.what()); return 1; } +} diff --git a/src/headset/calibration_render.hpp b/src/headset/calibration_render.hpp new file mode 100644 index 0000000..02d2017 --- /dev/null +++ b/src/headset/calibration_render.hpp @@ -0,0 +1,147 @@ +#pragma once + +GLuint program(const char* vertex, const char* fragment) { + GLuint shaders[2] = {glCreateShader(GL_VERTEX_SHADER), glCreateShader(GL_FRAGMENT_SHADER)}; + GLuint result = glCreateProgram(); + try { + for (int i = 0; i < 2; i++) { + const char* source = i ? fragment : vertex; glShaderSource(shaders[i], 1, &source, nullptr); glCompileShader(shaders[i]); + GLint valid = 0; glGetShaderiv(shaders[i], GL_COMPILE_STATUS, &valid); + if (!valid) { char log[1024] = {}; glGetShaderInfoLog(shaders[i], sizeof log, nullptr, log); __android_log_print(ANDROID_LOG_ERROR, "QFTCalibration", "%s", log); } + require(valid, "Calibration shader compilation failed"); glAttachShader(result, shaders[i]); + } + glLinkProgram(result); GLint valid = 0; glGetProgramiv(result, GL_LINK_STATUS, &valid); require(valid, "Calibration shader link failed"); + } catch (...) { glDeleteProgram(result); for (auto shader : shaders) glDeleteShader(shader); throw; } + for (auto shader : shaders) glDeleteShader(shader); + return result; +} + +const char* avatarVertex = R"glsl(#version 300 es +layout(location = 0) in vec3 a_Position; +layout(location = 1) in vec3 a_Normal; +layout(location = 2) in vec3 a_Tangent; +layout(location = 3) in uvec4 a_Joints; +layout(location = 4) in vec4 a_Weights; +layout(location = 5) in vec4 a_Color; +layout(location = 6) in vec2 a_UV; +layout(location = 7) in vec4 a_ORMT; +layout(location = 8) in float a_Curvature; +layout(std140) uniform SkinBlock { mat4 u_Skin[256]; }; +uniform mat4 u_Model, u_ViewProj; +out vec3 v_WorldPos, v_Normal, v_Tangent; out vec2 v_UV; out vec4 v_Color, v_ORMT; out float v_Curvature, v_Height; +void main() { + ivec4 j = ivec4(a_Joints); + mat4 skin = (u_Skin[j.x] * a_Weights.x + u_Skin[j.y] * a_Weights.y + u_Skin[j.z] * a_Weights.z + u_Skin[j.w] * a_Weights.w) / dot(a_Weights, vec4(1.0)); + mat4 m = u_Model * skin; + vec4 world = m * vec4(a_Position, 1.0); + v_WorldPos = world.xyz; v_Normal = normalize(mat3(m) * a_Normal); v_Tangent = normalize(mat3(m) * a_Tangent); + v_UV = a_UV; v_Color = a_Color; v_ORMT = a_ORMT; v_Curvature = a_Curvature; v_Height = (skin * vec4(a_Position, 1.0)).y; + gl_Position = u_ViewProj * world; +})glsl"; +const char* avatarFragment = R"glsl(#version 300 es +precision highp float; +in vec3 v_WorldPos, v_Normal, v_Tangent; in vec2 v_UV; in vec4 v_Color, v_ORMT; in float v_Curvature, v_Height; +uniform sampler2D u_BaseColor, u_Normal, u_ORM; +uniform vec3 u_CameraPos, u_LightDir, u_LightColor, u_SkyColor, u_GroundColor; +uniform vec4 u_Hair[2], u_FacialHair[2]; +uniform vec2 u_Bust; +out vec4 o_Color; +const float PI = 3.141592653589793, EPS = 0.001; +vec3 safeNormalize(vec3 v) { return v * inversesqrt(max(EPS, dot(v, v))); } +float avg3(vec3 v) { return (v.x + v.y + v.z) / 3.0; } +vec3 ambient(vec3 n) { return mix(u_GroundColor, u_SkyColor, n.y * 0.5 + 0.5); } +vec3 ambientSpecular(vec3 n, vec3 V, float rough, float metal, vec3 base, float occ) { + vec3 env = mix(ambient(reflect(-V, n)), 0.5 * (u_SkyColor + u_GroundColor), rough) * occ; + float r2 = max(rough * rough, 0.0078125); r2 *= r2; + float grazing = clamp(2.0 - rough - 0.96 * (1.0 - metal), 0.0, 1.0); + float fresnel = pow(1.0 - clamp(dot(n, V), 0.0, 1.0), 4.0); + return env * (1.0 + mix(mix(vec3(0.04), base, metal), vec3(grazing), fresnel) / (r2 + 1.0)); +} +vec3 dfgTerm(vec3 f0, float gloss, float NdotV) { + float x = gloss, y = NdotV; + float bias = clamp(min(-0.1688 * x + 1.895 * x * x, 0.9903 - 4.853 * y + 8.404 * y * y - 5.069 * y * y * y), 0.0, 1.0); + float delta = clamp(0.6045 + 1.699 * x - 0.5228 * y - 3.603 * x * x + 1.404 * x * y + 0.1939 * y * y + 2.661 * x * x * x, 0.0, 1.0); + return f0 * (delta - bias) + bias * clamp(50.0 * f0.y, 0.0, 1.0); +} +vec3 ggx(vec3 N, vec3 V, vec3 L, vec3 f0, float NdotL, float NdotV, float alphaG, vec3 dlc) { + vec3 H = normalize(L + V); NdotL = clamp(NdotL, 0.0, 1.0); + vec3 F = f0 + (1.0 - f0) * pow(1.0 - max(clamp(dot(V, H), 0.0, 1.0), 0.5), 5.0); + vec3 NxH = cross(N, H); float c = 1.0 - dot(NxH, NxH), a2 = alphaG * alphaG; + float d = max(c * c * (a2 - 1.0) + 1.0, 1e-10), D = min(a2 / (d * d), 55500.0); + float Vis = 0.5 / max(0.05, mix(2.0 * NdotL * NdotV, NdotL + NdotV, alphaG)); + return F * D * Vis * NdotL * dlc; +} +vec3 sssPunctual(float NdotL, float c) { + c = clamp(c, 0.0, 1.0); + vec3 t = NdotL * (vec3(-0.00808314, 0.0835151, 0.0920432) * c + vec3(0.355864, 0.418737, 0.407632)) + (vec3(0.0241138, -0.0446712, -0.0382355) * c + vec3(0.622981, 0.562233, 0.546097)); + vec3 fade = clamp(vec3(-0.915144, -0.366103, -0.207718) * c + vec3(0.9084, 0.315669, 0.167824), 0.0, 1.0); + return t * t * t * fade + clamp(NdotL, 0.0, 1.0) * (1.0 - fade); +} +vec3 sssAmbient(vec3 n, float c) { + c = clamp(c, 0.0, 1.0); + vec3 zh1 = vec3(c * c * c * 0.29126806 - c * c * 0.3900409 - c * 0.11944291 + 0.6474976, -c * c * 0.03049119 - c * 0.00234933 + 0.64157382, -c * c * 0.0137731528 + c * 0.000875828145 + 0.641376033); + vec3 l0 = 0.5 * (u_SkyColor + u_GroundColor), l1 = 0.5 * (u_SkyColor - u_GroundColor); + return max(l0 * 0.282095 + l1 * (0.488603 * zh1 * n.y), 0.0) * PI; +} +float hairLobe(float aniso, float rough, vec3 L, vec3 E, vec3 N, vec3 T, vec3 B, float offset) { + vec3 H = E + L; N = normalize(N + B * offset); + H = normalize(mix(normalize(H), normalize(H - dot(H, T) * T), aniso)); + return pow(max(0.0, dot(N, H)), 1.0 / (rough * rough + EPS)); +} +void main() { + float type = floor(v_Color.a * 255.0 + 0.5); + bool isEye = type == 2.0, isSkin = type == 3.0, isFacialHair = type == 5.0, isHair = type == 4.0 || isFacialHair; + vec3 N0 = normalize(v_Normal), T = normalize(v_Tangent), B = normalize(cross(T, N0)); + vec3 V = normalize(u_CameraPos - v_WorldPos), L = u_LightDir; + vec3 base = texture(u_BaseColor, v_UV).rgb; vec4 orm = texture(u_ORM, v_UV); + vec4 nm = texture(u_Normal, v_UV); vec2 xy = vec2(nm.g, 1.0 - nm.a) * 2.0 - 1.0; + vec3 N = normalize(xy.x * T + xy.y * B + sqrt(max(1.0 - dot(xy, xy), 0.0)) * N0); + float occ = orm.r, rough = orm.g, metal = orm.b, curvature = v_Curvature * 1.284 + 0.0186; + vec4 h0 = isFacialHair ? u_FacialHair[0] : u_Hair[0], h1 = isFacialHair ? u_FacialHair[1] : u_Hair[1]; + vec3 hairColor = safeNormalize(base * base + EPS) * max(0.1, length(base)); + float hairBlend = 0.0, hairAniso = 0.0, flow = 0.0; + if (isHair) { + flow = ((1.0 - v_ORMT.g) - 0.25) * 2.0 * PI; hairBlend = clamp(orm.a * 2.0, 0.0, 1.0); hairAniso = clamp((orm.a - 0.5) * 2.0, 0.0, 1.0); + rough = 0.4; metal = 0.0; N = mix(N0, N, hairBlend); + } + vec3 hT = normalize(T * cos(flow) + B * sin(flow)), hN = mix(N0, N, h1.y), hB = normalize(cross(hT, hN)); + float NdotV = clamp(dot(N, V), 0.0, 1.0), NdotL = dot(N, L); + vec3 f0 = mix(vec3(0.04), base, metal); float alphaG = rough * rough + EPS; + vec3 dfg = dfgTerm(f0, 1.0 - pow(alphaG, 0.25), NdotV); + vec3 glint = vec3(0.0); + if (isEye) glint = vec3(10.0 * pow(clamp(dot(N0, normalize(V + vec3(0.0, 0.25, 0.0) + 0.25 * cross(vec3(0.0, 1.0, 0.0), V))), 0.0, 1.0), 2000.0)); + vec3 dlc = u_LightColor / PI * clamp((dot(V, L) + 1.0) / 0.75, 0.0, 1.0); + vec3 punctualDiffuse = clamp(NdotL, 0.0, 1.0) * (1.0 - avg3(dfg)) * dlc, punctualSSS = vec3(0.0), rim = vec3(0.0); + if (isSkin) punctualSSS = sssPunctual(NdotL, curvature) * dlc; + if (isHair) { float w = smoothstep(-0.5, 1.0, NdotL); punctualSSS = w * dlc / (1.0 + abs(NdotL - w)); } + vec3 punctualSpec = ggx(N, V, L, f0, NdotL, NdotV, alphaG, dlc); + vec3 ambDiffuse = ambient(N) * occ, ambSpec = ambientSpecular(N, V, rough, metal, base, occ); + if (isHair) { + float anisotropy = u_Hair[0].x * hairAniso, offset = u_Hair[1].z * (orm.b - 0.5) * hairAniso; + vec3 dir = anisotropy >= 0.0 ? hT : hB, across = cross(cross(dir, V), dir); + vec3 bent = normalize(mix(N, normalize(across + dir * offset), anisotropy * 0.75)); + vec3 spec = ambientSpecular(bent, V, u_Hair[0].z, 0.0, base, occ) * h1.w; + bent = normalize(mix(N, normalize(across + dir * (offset + h1.x)), anisotropy * 0.75)); + spec += ambientSpecular(bent, V, u_Hair[0].y, 0.0, base, occ) * hairColor * h0.w; + ambSpec = spec * hairBlend; + } + vec3 dc = (1.0 - metal) * base; + ambDiffuse *= occ / ((1.0 - dc) + dc * occ); + ambSpec *= clamp(pow(abs(NdotV + occ), exp2(-16.0 * rough - 1.0)) - 1.0 + occ, 0.0, 1.0) * dfg; + if (isHair) { + vec3 R = reflect(-V, N), toRay = dot(L, R) * R - L; + vec3 Ls = normalize(L + toRay * clamp(0.174108138 / length(toRay), 0.0, 1.0)); + float aniso = h0.x * hairAniso, offset = h1.z * (orm.b - 0.5) * hairAniso; + float white = hairLobe(aniso, h0.z, Ls, V, hN, hB, hT, offset) * h1.w, color = hairLobe(aniso, h0.y, Ls, V, hN, hB, hT, h1.x + offset) * h0.w; + vec3 hairSpec = (white + hairColor * color) * orm.r * clamp(dot(L, N), 0.0, 1.0) * dlc; + punctualSpec = mix(punctualSpec, hairSpec, hairBlend); + punctualDiffuse = punctualSSS * hairColor + punctualDiffuse * (1.0 - hairColor); + } + vec3 diffuseColor = (isEye ? 1.0 : 1.0 - metal) * base; + vec3 diffuse = isSkin ? (sssAmbient(N, curvature) * occ + punctualSSS) * diffuseColor + : isHair ? (punctualDiffuse + clamp(ambient(N) * occ, 0.0, 1.0) * occ) * diffuseColor + : (punctualDiffuse + ambDiffuse) * diffuseColor; + float bust = smoothstep(u_Bust.x, u_Bust.y, v_Height); + if (bust < 0.004) discard; + o_Color = vec4(diffuse + rim + punctualSpec + ambSpec + glint, bust); +})glsl"; diff --git a/src/headset/calibration_scene.cpp b/src/headset/calibration_scene.cpp index 80818e9..50dd5f3 100644 --- a/src/headset/calibration_scene.cpp +++ b/src/headset/calibration_scene.cpp @@ -33,151 +33,8 @@ template T function(XrInstance instance, const char* name) { namespace { using avatar::Mat; using avatar::Vec3; -GLuint program(const char* vertex, const char* fragment) { - GLuint shaders[2] = {glCreateShader(GL_VERTEX_SHADER), glCreateShader(GL_FRAGMENT_SHADER)}; - GLuint result = glCreateProgram(); - try { - for (int i = 0; i < 2; i++) { - const char* source = i ? fragment : vertex; glShaderSource(shaders[i], 1, &source, nullptr); glCompileShader(shaders[i]); - GLint valid = 0; glGetShaderiv(shaders[i], GL_COMPILE_STATUS, &valid); - if (!valid) { char log[1024] = {}; glGetShaderInfoLog(shaders[i], sizeof log, nullptr, log); __android_log_print(ANDROID_LOG_ERROR, "QFTCalibration", "%s", log); } - require(valid, "Calibration shader compilation failed"); glAttachShader(result, shaders[i]); - } - glLinkProgram(result); GLint valid = 0; glGetProgramiv(result, GL_LINK_STATUS, &valid); require(valid, "Calibration shader link failed"); - } catch (...) { glDeleteProgram(result); for (auto shader : shaders) glDeleteShader(shader); throw; } - for (auto shader : shaders) glDeleteShader(shader); - return result; -} +#include "calibration_render.hpp" -const char* avatarVertex = R"glsl(#version 300 es -layout(location = 0) in vec3 a_Position; -layout(location = 1) in vec3 a_Normal; -layout(location = 2) in vec3 a_Tangent; -layout(location = 3) in uvec4 a_Joints; -layout(location = 4) in vec4 a_Weights; -layout(location = 5) in vec4 a_Color; -layout(location = 6) in vec2 a_UV; -layout(location = 7) in vec4 a_ORMT; -layout(location = 8) in float a_Curvature; -layout(std140) uniform SkinBlock { mat4 u_Skin[256]; }; -uniform mat4 u_Model, u_ViewProj; -out vec3 v_WorldPos, v_Normal, v_Tangent; out vec2 v_UV; out vec4 v_Color, v_ORMT; out float v_Curvature, v_Height; -void main() { - ivec4 j = ivec4(a_Joints); - mat4 skin = (u_Skin[j.x] * a_Weights.x + u_Skin[j.y] * a_Weights.y + u_Skin[j.z] * a_Weights.z + u_Skin[j.w] * a_Weights.w) / dot(a_Weights, vec4(1.0)); - mat4 m = u_Model * skin; - vec4 world = m * vec4(a_Position, 1.0); - v_WorldPos = world.xyz; v_Normal = normalize(mat3(m) * a_Normal); v_Tangent = normalize(mat3(m) * a_Tangent); - v_UV = a_UV; v_Color = a_Color; v_ORMT = a_ORMT; v_Curvature = a_Curvature; v_Height = (skin * vec4(a_Position, 1.0)).y; - gl_Position = u_ViewProj * world; -})glsl"; -const char* avatarFragment = R"glsl(#version 300 es -precision highp float; -in vec3 v_WorldPos, v_Normal, v_Tangent; in vec2 v_UV; in vec4 v_Color, v_ORMT; in float v_Curvature, v_Height; -uniform sampler2D u_BaseColor, u_Normal, u_ORM; -uniform vec3 u_CameraPos, u_LightDir, u_LightColor, u_SkyColor, u_GroundColor; -uniform vec4 u_Hair[2], u_FacialHair[2]; -uniform vec2 u_Bust; -out vec4 o_Color; -const float PI = 3.141592653589793, EPS = 0.001; -vec3 safeNormalize(vec3 v) { return v * inversesqrt(max(EPS, dot(v, v))); } -float avg3(vec3 v) { return (v.x + v.y + v.z) / 3.0; } -vec3 ambient(vec3 n) { return mix(u_GroundColor, u_SkyColor, n.y * 0.5 + 0.5); } -vec3 ambientSpecular(vec3 n, vec3 V, float rough, float metal, vec3 base, float occ) { - vec3 env = mix(ambient(reflect(-V, n)), 0.5 * (u_SkyColor + u_GroundColor), rough) * occ; - float r2 = max(rough * rough, 0.0078125); r2 *= r2; - float grazing = clamp(2.0 - rough - 0.96 * (1.0 - metal), 0.0, 1.0); - float fresnel = pow(1.0 - clamp(dot(n, V), 0.0, 1.0), 4.0); - return env * (1.0 + mix(mix(vec3(0.04), base, metal), vec3(grazing), fresnel) / (r2 + 1.0)); -} -vec3 dfgTerm(vec3 f0, float gloss, float NdotV) { - float x = gloss, y = NdotV; - float bias = clamp(min(-0.1688 * x + 1.895 * x * x, 0.9903 - 4.853 * y + 8.404 * y * y - 5.069 * y * y * y), 0.0, 1.0); - float delta = clamp(0.6045 + 1.699 * x - 0.5228 * y - 3.603 * x * x + 1.404 * x * y + 0.1939 * y * y + 2.661 * x * x * x, 0.0, 1.0); - return f0 * (delta - bias) + bias * clamp(50.0 * f0.y, 0.0, 1.0); -} -vec3 ggx(vec3 N, vec3 V, vec3 L, vec3 f0, float NdotL, float NdotV, float alphaG, vec3 dlc) { - vec3 H = normalize(L + V); NdotL = clamp(NdotL, 0.0, 1.0); - vec3 F = f0 + (1.0 - f0) * pow(1.0 - max(clamp(dot(V, H), 0.0, 1.0), 0.5), 5.0); - vec3 NxH = cross(N, H); float c = 1.0 - dot(NxH, NxH), a2 = alphaG * alphaG; - float d = max(c * c * (a2 - 1.0) + 1.0, 1e-10), D = min(a2 / (d * d), 55500.0); - float Vis = 0.5 / max(0.05, mix(2.0 * NdotL * NdotV, NdotL + NdotV, alphaG)); - return F * D * Vis * NdotL * dlc; -} -vec3 sssPunctual(float NdotL, float c) { - c = clamp(c, 0.0, 1.0); - vec3 t = NdotL * (vec3(-0.00808314, 0.0835151, 0.0920432) * c + vec3(0.355864, 0.418737, 0.407632)) + (vec3(0.0241138, -0.0446712, -0.0382355) * c + vec3(0.622981, 0.562233, 0.546097)); - vec3 fade = clamp(vec3(-0.915144, -0.366103, -0.207718) * c + vec3(0.9084, 0.315669, 0.167824), 0.0, 1.0); - return t * t * t * fade + clamp(NdotL, 0.0, 1.0) * (1.0 - fade); -} -vec3 sssAmbient(vec3 n, float c) { - c = clamp(c, 0.0, 1.0); - vec3 zh1 = vec3(c * c * c * 0.29126806 - c * c * 0.3900409 - c * 0.11944291 + 0.6474976, -c * c * 0.03049119 - c * 0.00234933 + 0.64157382, -c * c * 0.0137731528 + c * 0.000875828145 + 0.641376033); - vec3 l0 = 0.5 * (u_SkyColor + u_GroundColor), l1 = 0.5 * (u_SkyColor - u_GroundColor); - return max(l0 * 0.282095 + l1 * (0.488603 * zh1 * n.y), 0.0) * PI; -} -float hairLobe(float aniso, float rough, vec3 L, vec3 E, vec3 N, vec3 T, vec3 B, float offset) { - vec3 H = E + L; N = normalize(N + B * offset); - H = normalize(mix(normalize(H), normalize(H - dot(H, T) * T), aniso)); - return pow(max(0.0, dot(N, H)), 1.0 / (rough * rough + EPS)); -} -void main() { - float type = floor(v_Color.a * 255.0 + 0.5); - bool isEye = type == 2.0, isSkin = type == 3.0, isFacialHair = type == 5.0, isHair = type == 4.0 || isFacialHair; - vec3 N0 = normalize(v_Normal), T = normalize(v_Tangent), B = normalize(cross(T, N0)); - vec3 V = normalize(u_CameraPos - v_WorldPos), L = u_LightDir; - vec3 base = texture(u_BaseColor, v_UV).rgb; vec4 orm = texture(u_ORM, v_UV); - vec4 nm = texture(u_Normal, v_UV); vec2 xy = vec2(nm.g, 1.0 - nm.a) * 2.0 - 1.0; - vec3 N = normalize(xy.x * T + xy.y * B + sqrt(max(1.0 - dot(xy, xy), 0.0)) * N0); - float occ = orm.r, rough = orm.g, metal = orm.b, curvature = v_Curvature * 1.284 + 0.0186; - vec4 h0 = isFacialHair ? u_FacialHair[0] : u_Hair[0], h1 = isFacialHair ? u_FacialHair[1] : u_Hair[1]; - vec3 hairColor = safeNormalize(base * base + EPS) * max(0.1, length(base)); - float hairBlend = 0.0, hairAniso = 0.0, flow = 0.0; - if (isHair) { - flow = ((1.0 - v_ORMT.g) - 0.25) * 2.0 * PI; hairBlend = clamp(orm.a * 2.0, 0.0, 1.0); hairAniso = clamp((orm.a - 0.5) * 2.0, 0.0, 1.0); - rough = 0.4; metal = 0.0; N = mix(N0, N, hairBlend); - } - vec3 hT = normalize(T * cos(flow) + B * sin(flow)), hN = mix(N0, N, h1.y), hB = normalize(cross(hT, hN)); - float NdotV = clamp(dot(N, V), 0.0, 1.0), NdotL = dot(N, L); - vec3 f0 = mix(vec3(0.04), base, metal); float alphaG = rough * rough + EPS; - vec3 dfg = dfgTerm(f0, 1.0 - pow(alphaG, 0.25), NdotV); - vec3 glint = vec3(0.0); - if (isEye) glint = vec3(10.0 * pow(clamp(dot(N0, normalize(V + vec3(0.0, 0.25, 0.0) + 0.25 * cross(vec3(0.0, 1.0, 0.0), V))), 0.0, 1.0), 2000.0)); - vec3 dlc = u_LightColor / PI * clamp((dot(V, L) + 1.0) / 0.75, 0.0, 1.0); - vec3 punctualDiffuse = clamp(NdotL, 0.0, 1.0) * (1.0 - avg3(dfg)) * dlc, punctualSSS = vec3(0.0), rim = vec3(0.0); - if (isSkin) punctualSSS = sssPunctual(NdotL, curvature) * dlc; - if (isHair) { float w = smoothstep(-0.5, 1.0, NdotL); punctualSSS = w * dlc / (1.0 + abs(NdotL - w)); } - vec3 punctualSpec = ggx(N, V, L, f0, NdotL, NdotV, alphaG, dlc); - vec3 ambDiffuse = ambient(N) * occ, ambSpec = ambientSpecular(N, V, rough, metal, base, occ); - if (isHair) { - float anisotropy = u_Hair[0].x * hairAniso, offset = u_Hair[1].z * (orm.b - 0.5) * hairAniso; - vec3 dir = anisotropy >= 0.0 ? hT : hB, across = cross(cross(dir, V), dir); - vec3 bent = normalize(mix(N, normalize(across + dir * offset), anisotropy * 0.75)); - vec3 spec = ambientSpecular(bent, V, u_Hair[0].z, 0.0, base, occ) * h1.w; - bent = normalize(mix(N, normalize(across + dir * (offset + h1.x)), anisotropy * 0.75)); - spec += ambientSpecular(bent, V, u_Hair[0].y, 0.0, base, occ) * hairColor * h0.w; - ambSpec = spec * hairBlend; - } - vec3 dc = (1.0 - metal) * base; - ambDiffuse *= occ / ((1.0 - dc) + dc * occ); - ambSpec *= clamp(pow(abs(NdotV + occ), exp2(-16.0 * rough - 1.0)) - 1.0 + occ, 0.0, 1.0) * dfg; - if (isHair) { - vec3 R = reflect(-V, N), toRay = dot(L, R) * R - L; - vec3 Ls = normalize(L + toRay * clamp(0.174108138 / length(toRay), 0.0, 1.0)); - float aniso = h0.x * hairAniso, offset = h1.z * (orm.b - 0.5) * hairAniso; - float white = hairLobe(aniso, h0.z, Ls, V, hN, hB, hT, offset) * h1.w, color = hairLobe(aniso, h0.y, Ls, V, hN, hB, hT, h1.x + offset) * h0.w; - vec3 hairSpec = (white + hairColor * color) * orm.r * clamp(dot(L, N), 0.0, 1.0) * dlc; - punctualSpec = mix(punctualSpec, hairSpec, hairBlend); - punctualDiffuse = punctualSSS * hairColor + punctualDiffuse * (1.0 - hairColor); - } - vec3 diffuseColor = (isEye ? 1.0 : 1.0 - metal) * base; - vec3 diffuse = isSkin ? (sssAmbient(N, curvature) * occ + punctualSSS) * diffuseColor - : isHair ? (punctualDiffuse + clamp(ambient(N) * occ, 0.0, 1.0) * occ) * diffuseColor - : (punctualDiffuse + ambDiffuse) * diffuseColor; - float bust = smoothstep(u_Bust.x, u_Bust.y, v_Height); - if (bust < 0.004) discard; - o_Color = vec4(diffuse + rim + punctualSpec + ambSpec + glint, bust); -})glsl"; const char* spaceVertex = R"glsl(#version 300 es layout(location = 0) in vec3 p; layout(location = 1) in vec2 uv; uniform mat4 u_ModelViewProj; out vec2 v_UV; void main() { v_UV = uv; gl_Position = u_ModelViewProj * vec4(p, 1.0); })glsl"; @@ -265,6 +122,22 @@ struct Scene { } void setup(JNIEnv* jni, jobject activity, bool pupils) { env = jni; JavaVM* vm; env->GetJavaVM(&vm); + jclass activityClass = env->GetObjectClass(activity); + jmethodID inputLabel = env->GetMethodID(activityClass, "inputLabel", "(I)Ljava/lang/String;"); + env->DeleteLocalRef(activityClass); + require(inputLabel, "Calibration input labels missing"); + auto label = [&](int index) { + auto text = static_cast(env->CallObjectMethod(activity, inputLabel, index)); + require(text && !env->ExceptionCheck(), "Read calibration input label"); + const char* chars = env->GetStringUTFChars(text, nullptr); + std::string value(chars); env->ReleaseStringUTFChars(text, chars); env->DeleteLocalRef(text); + if (value.size() >= XR_MAX_LOCALIZED_ACTION_NAME_SIZE) { + size_t length = XR_MAX_LOCALIZED_ACTION_NAME_SIZE - 1; + while ((static_cast(value[length]) & 0xc0) == 0x80) --length; + value.resize(length); + } + return value; + }; auto initialize = function(XR_NULL_HANDLE, "xrInitializeLoaderKHR"); XrLoaderInitInfoAndroidKHR loader{XR_TYPE_LOADER_INIT_INFO_ANDROID_KHR}; loader.applicationVM = vm; loader.applicationContext = activity; xr(initialize(reinterpret_cast(&loader)), "Initialize OpenXR"); @@ -306,13 +179,13 @@ struct Scene { XrEyeTrackerCreateInfoFB ec{XR_TYPE_EYE_TRACKER_CREATE_INFO_FB}; xr(function(instance, "xrCreateEyeTrackerFB")(session, &ec, &eyes), "Start eye tracker"); - XrActionSetCreateInfo ac{XR_TYPE_ACTION_SET_CREATE_INFO}; std::strcpy(ac.actionSetName, "calibration"); std::strcpy(ac.localizedActionSetName, "Calibration"); + XrActionSetCreateInfo ac{XR_TYPE_ACTION_SET_CREATE_INFO}; std::strcpy(ac.actionSetName, "calibration"); std::strcpy(ac.localizedActionSetName, label(0).c_str()); xr(xrCreateActionSet(instance, &ac, &actions), "Create calibration input"); hands[0] = path("/user/hand/left"); hands[1] = path("/user/hand/right"); - cancel = action("cancel", "Leave calibration", XR_ACTION_TYPE_BOOLEAN_INPUT, false); - start = action("start", "Start calibration", XR_ACTION_TYPE_BOOLEAN_INPUT, false); - select = action("select", "Select", XR_ACTION_TYPE_BOOLEAN_INPUT, true); - pose = action("aim", "Point", XR_ACTION_TYPE_POSE_INPUT, true); + cancel = action("cancel", label(1).c_str(), XR_ACTION_TYPE_BOOLEAN_INPUT, false); + start = action("start", label(2).c_str(), XR_ACTION_TYPE_BOOLEAN_INPUT, false); + select = action("select", label(3).c_str(), XR_ACTION_TYPE_BOOLEAN_INPUT, true); + pose = action("aim", label(4).c_str(), XR_ACTION_TYPE_POSE_INPUT, true); XrActionSuggestedBinding bindings[] = {{cancel, path("/user/hand/right/input/b/click")}, {cancel, path("/user/hand/left/input/y/click")}, {start, path("/user/hand/right/input/a/click")}, {start, path("/user/hand/left/input/x/click")}, {select, path("/user/hand/left/input/trigger/value")}, {select, path("/user/hand/right/input/trigger/value")}, diff --git a/src/tracking/guided_session.py b/src/tracking/guided_session.py index aac3c92..e346145 100644 --- a/src/tracking/guided_session.py +++ b/src/tracking/guided_session.py @@ -175,7 +175,7 @@ def ui(self): s = self.current forming = 1. if s.ramp <= 0 else min(1., self.elapsed / s.ramp) remaining = max(0., s.seconds - self.elapsed) - return {"title": s.name, "instruction": s.instruction, "index": self.index, "total": len(self.steps), + return {"title": s.name, "instruction": s.instruction, "condition": s.condition, "index": self.index, "total": len(self.steps), "targets": {k: (v if k in ("horizontal", "vertical", "visibility") else v * forming) for k, v in {**GUIDE.get(s.condition, {}), **s.targets}.items()}, "progress": (self.index + min(1., self.elapsed / s.seconds)) / len(self.steps), "remaining": remaining,