Encode a share or camera with the machine's own hardware encoder first

crates/mediad/Cargo.toml

@@ -91,8 +91,9 @@ webpki-roots = "1"
 # Already in the tree through cpal and mio.
 windows-sys = { version = "0.61", features = ["Win32_Media", "Win32_System_Threading", "Win32_Foundation", "Win32_Graphics_Gdi", "Win32_Graphics_Dwm", "Win32_Storage_Xps", "Win32_UI_HiDpi", "Win32_UI_WindowsAndMessaging"] }
 # A screen share through DXGI Desktop Duplication (`picture/duplicate`): Direct3D 11 and DXGI are
-# COM, which the `windows` crate wraps. The version notifier already builds.
-windows = { version = "0.62", features = ["Win32_Foundation", "Win32_Graphics_Direct3D", "Win32_Graphics_Direct3D11", "Win32_Graphics_Dxgi", "Win32_Graphics_Dxgi_Common", "Win32_Graphics_Gdi"] }
+# COM, which the `windows` crate wraps. The version notifier already builds. Media Foundation's
+# hardware H.264 encoder (`mf_encoder`) is COM too, with its variants (`Variant`) beside it.
+windows = { version = "0.62", features = ["Win32_Foundation", "Win32_Graphics_Direct3D", "Win32_Graphics_Direct3D11", "Win32_Graphics_Dxgi", "Win32_Graphics_Dxgi_Common", "Win32_Graphics_Gdi", "Win32_Media_MediaFoundation", "Win32_System_Com", "Win32_System_Ole", "Win32_System_Variant"] }
 
 [target.'cfg(unix)'.dependencies]
 # The priority of video's threads. Already in the tree through mio and cpal.

crates/mediad/src/lib.rs

@@ -37,6 +37,9 @@ pub mod hear;
 pub mod inproc;
 pub mod jitter;
 pub mod media;
+/// The machine's own H.264 hardware encoder, ahead of the OpenH264 download. Windows only.
+#[cfg(windows)]
+mod mf_encoder;
 pub mod ogg;
 pub mod opus;
 pub mod picture;

docs/product/06-calls.md

@@ -12,7 +12,7 @@ A Tiny room joins with sound when the server's audio is up: one send-only Opus m
 
 ## Pictures, on a Tiny room
 
-A Tiny call can send a camera and a screen, and it receives one H.264 picture. The codec is OpenH264 constrained baseline, downloaded from Cisco the first time a picture is encoded. If that download fails, the call stays voice.
+A Tiny call can send a camera and a screen, and it receives one H.264 picture. On a desktop, what it sends is encoded first by the machine's own hardware: on Windows, Media Foundation lists the hardware H.264 encoders (NVIDIA, Intel, AMD), and the first one that takes the job runs it at the size, frame rate and bitrate the share chose, at constrained baseline, with no download and hardly any CPU. A machine that lists no such encoder, or one that refuses its setup, falls back to OpenH264 constrained baseline, downloaded from Cisco the first time a picture is encoded. If that download fails, the call stays voice. macOS and Linux send through OpenH264 as they always have; whatever arrives is decoded through OpenH264 on every desktop.
 
 The room forwards one picture: a share while someone else is sharing, otherwise one layer of another person's camera. The listener does not get their own picture. The client offers three video lines, the first two inactive until asked: a camera as two layers of one picture (640×360 and 320×180), one share fitted inside 1280×720, and one receive line for the picture the room forwards. Both the camera and the share run at 15 frames a second. Nothing plays until someone chooses Watch.
 

e2e/scenarios/hardware-encoder-fallback/README.md

@@ -0,0 +1,22 @@
+# Hardware encoder fallback
+
+Run `sh e2e/scenarios/hardware-encoder-fallback/run.sh`. It needs no devices, no encoder
+hardware and no network: the scenario drives the tests of
+`crates/mediad/src/mf_encoder.rs` and the fallback seam in `picture/desktop.rs` — the
+encoder selection, the fake-MFT seam that holds the open (the H.264 output type refused,
+a driver that kept Main/High or named no profile, a keyframe that only counts when the
+sample says `MFSampleExtension_CleanPoint`, the frame loop with its asks), the pure NV12
+and Annex B helpers, and the place a refusal of the hardware becomes the OpenH264
+fallback.
+
+On the base the scenario fails: the module does not exist, so cargo reports no matching
+tests. With the branch the Media Foundation hardware encoder is tried first (the output
+type negotiated first, at the constrained baseline, and what the driver kept read back
+and held to that), and the OpenH264 runtime download stays the fallback with its own
+covered path.
+
+The on-hardware record (the lab driving a real share over a call) needs a machine with a
+hardware MFT; that recording is the released-client proof this scenario points to. What
+this scenario can say without that machine: a driver that would not keep the promised
+wire is refused in its own words and OpenH264 takes the job, and nothing else changed
+about what goes out.

e2e/scenarios/hardware-encoder-fallback/run.sh

@@ -0,0 +1,40 @@
+#!/bin/sh
+# The Media Foundation hardware encoder's tests, unit-tested on Windows (the module is
+# cfg(windows), so this driver runs where it compiles): the selection logic, the fake-MFT
+# seam that holds the open, the profile refusal, the frame loop and the keyframe flag
+# without hardware, and the pure NV12/Annex B helpers. No network, no devices, no encoder
+# hardware.
+set -eu
+cd "$(dirname "$0")/../../.."
+log=$(mktemp)
+trap 'rm -f "$log"' EXIT HUP INT TERM
+run() {
+    name=$1
+    printf '%s\n' "media foundation encoder: $name"
+    if ! cargo test -p mediad --lib "$name" -- --exact --nocapture --test-threads=1 >"$log" 2>&1; then
+        cat "$log"
+        return 1
+    fi
+    cat "$log"
+    grep -Fq "running 1 test" "$log"
+    grep -Fq "test $name ... ok" "$log"
+    if grep -Eq '(^|[[:space:]])skipped:' "$log"; then
+        echo 'FAIL: a fixture skipped' >&2
+        return 1
+    fi
+}
+run mf_encoder::tests::a_clean_probe_means_hardware_and_any_failure_means_the_download
+run mf_encoder::tests::an_encoder_that_refuses_the_output_type_is_failed_in_its_own_words
+run mf_encoder::tests::a_profile_other_than_the_baseline_promised_is_refused
+run mf_encoder::tests::a_frame_is_a_keyframe_only_when_the_sample_says_it_is_a_clean_point
+run mf_encoder::tests::an_open_encoder_takes_frames_and_holds_the_asks
+run mf_encoder::tests::a_closed_encoder_ends_its_stream
+run mf_encoder::tests::the_probe_answers_on_a_thread_of_its_own
+run mf_encoder::tests::nv12_is_the_y_plane_and_the_two_chroma_planes_interleaved
+run mf_encoder::tests::nv12_of_a_short_buffer_writes_nothing_past_it
+run mf_encoder::tests::annex_b_already_annex_b_is_untouched
+run mf_encoder::tests::length_prefixed_units_come_out_behind_start_codes
+run mf_encoder::tests::a_truncated_or_empty_stream_never_runs_past_itself
+run mf_encoder::tests::sizes_and_rates_are_packed_width_over_height_and_thousandths
+run picture::desktop::tests::a_refused_hardware_encoder_settles_on_the_software_one
+printf '%s\n' 'PASS: the hardware encoder is chosen when present, refuses a wire it was not promised, and OpenH264 stays the fallback.'

e2e/scenarios/hardware-encoder-fallback/scenario.json

@@ -0,0 +1,10 @@
+{
+  "id": "hardware-encoder-fallback",
+  "title": "A camera or screen share encodes through the machine's hardware encoder when present and falls back to the OpenH264 download otherwise",
+  "platforms": ["windows"],
+  "needs": [],
+  "release": false,
+  "driver": "run.sh",
+  "proof_scope": "encoder selection, the fake-MFT open/refusal/keyframe path, and the NV12/Annex B helpers, unit-level; the on-hardware record needs a machine with a hardware MFT",
+  "native_client_proof": "the released client shows screen share and camera running on the machine's own hardware encoder, with the OpenH264 download as the fallback"
+}