NetDesk 1.5.0 — source release

native/core/src/decode.rs

@@ -0,0 +1,254 @@
+//! H.264 decode. Phase 2 uses openh264 software decode (portable across all five
+//! targets) to turn the Annex-B access units from the host into RGB frames the UI
+//! can paint. Hardware decoders (Media Foundation / VideoToolbox / MediaCodec) are
+//! layered in later behind this same `Frame` output.
+
+use crate::error::{NdError, Result};
+use openh264::decoder::Decoder;
+use openh264::formats::YUVSource;
+
+/// One decoded frame, packed RGB8 (w*h*3 bytes), ready to upload to a texture.
+pub struct Frame {
+    pub w: usize,
+    pub h: usize,
+    pub rgb: Vec<u8>,
+}
+
+/// Object-safe decode interface shared by every backend.
+///
+/// Both the portable openh264 software decoder ([`H264Decoder`]) and the Windows
+/// Media Foundation hardware decoder implement this, so the viewer can own a
+/// `Box<dyn VideoDecoder + Send>` and not care which one is running. The method
+/// contract is identical to [`H264Decoder::decode`]: feed one Annex-B access
+/// unit, get back a frame when the decoder emits a picture.
+pub trait VideoDecoder {
+    fn decode(&mut self, au: &[u8]) -> Result<Option<Frame>>;
+}
+
+pub struct H264Decoder {
+    inner: Decoder,
+}
+
+impl H264Decoder {
+    pub fn new() -> Result<Self> {
+        let inner =
+            Decoder::new().map_err(|e| NdError::Other(anyhow::anyhow!("openh264 init: {e}")))?;
+        Ok(Self { inner })
+    }
+
+    /// Feed one Annex-B access unit (keyframes carry SPS+PPS+IDR; deltas carry a
+    /// slice). Returns the decoded RGB frame when the decoder emits a picture.
+    pub fn decode(&mut self, au: &[u8]) -> Result<Option<Frame>> {
+        match self.inner.decode(au) {
+            Ok(Some(yuv)) => {
+                let (w, h) = yuv.dimensions();
+                let mut rgb = vec![0u8; w * h * 3];
+                yuv.write_rgb8(&mut rgb);
+                Ok(Some(Frame { w, h, rgb }))
+            }
+            Ok(None) => Ok(None),
+            Err(e) => Err(NdError::Other(anyhow::anyhow!("decode: {e}"))),
+        }
+    }
+}
+
+impl VideoDecoder for H264Decoder {
+    fn decode(&mut self, au: &[u8]) -> Result<Option<Frame>> {
+        // Reuse the existing inherent method so both call sites stay in lockstep.
+        H264Decoder::decode(self, au)
+    }
+}
+
+// Per-platform hardware decoders. Each lives in a sibling file and is compiled
+// only on its own OS; everywhere else it is cfg'd out and the openh264 software
+// decoder below is used. All implement the same `VideoDecoder` trait, so
+// `best_decoder` can hand back a `Box<dyn VideoDecoder + Send>` without the
+// viewer knowing or caring which backend is running.
+
+// Windows: DXVA H.264 via Media Foundation.
+#[cfg(windows)]
+#[path = "decode_mf.rs"]
+mod mf;
+
+// macOS / iOS: VideoToolbox (VTDecompressionSession).
+#[cfg(any(target_os = "macos", target_os = "ios"))]
+#[path = "decode_vt.rs"]
+mod vt;
+
+// Android: MediaCodec (NDK AMediaCodec).
+#[cfg(target_os = "android")]
+#[path = "decode_mediacodec.rs"]
+mod mediacodec;
+
+// Linux: VAAPI via FFmpeg/libavcodec.
+#[cfg(target_os = "linux")]
+#[path = "decode_vaapi.rs"]
+mod vaapi;
+
+/// Last-resort decoder that drops every access unit. Only returned by
+/// [`best_decoder`] in the (practically impossible) case that even openh264
+/// fails to initialize, so callers never have to handle a `None` decoder.
+struct NullDecoder;
+
+impl VideoDecoder for NullDecoder {
+    fn decode(&mut self, _au: &[u8]) -> Result<Option<Frame>> {
+        Ok(None)
+    }
+}
+
+/// Pick the fastest decoder available on this platform.
+///
+/// Each OS gets its native hardware H.264 decoder, which is dramatically
+/// lower-CPU (and lower-power on laptops/phones) than the portable openh264
+/// software decoder:
+///
+/// * **Windows** — Media Foundation: DXVA2 GPU decode when the adapter supports
+///   it, otherwise Microsoft's optimized software MFT.
+/// * **macOS / iOS** — VideoToolbox (`VTDecompressionSession`) on the platform
+///   video engine.
+/// * **Android** — `MediaCodec` (`AMediaCodec`), i.e. the SoC video decoder.
+/// * **Linux** — VAAPI through FFmpeg/libavcodec (Intel/AMD GPUs via `/dev/dri`).
+///
+/// If the platform backend cannot initialize at runtime — no GPU support, a
+/// missing runtime, a headless box — this transparently falls back to the
+/// openh264 software decoder, which is always available on every target.
+///
+/// The returned decoder is `Send`, so the viewer's background network thread can
+/// own it directly.
+pub fn best_decoder() -> Box<dyn VideoDecoder + Send> {
+    #[cfg(windows)]
+    {
+        match mf::MediaFoundationDecoder::new() {
+            Ok(d) => {
+                tracing::info!(
+                    "decode: Media Foundation H.264 ({})",
+                    if d.is_hardware() {
+                        "DXVA hardware"
+                    } else {
+                        "MS software MFT"
+                    }
+                );
+                return Box::new(d);
+            }
+            Err(e) => {
+                tracing::warn!(
+                    "decode: Media Foundation init failed ({e}); using openh264 software decode"
+                );
+            }
+        }
+    }
+
+    #[cfg(any(target_os = "macos", target_os = "ios"))]
+    {
+        match vt::VideoToolboxDecoder::new() {
+            Ok(d) => {
+                tracing::info!("decode: VideoToolbox H.264 (hardware)");
+                return Box::new(d);
+            }
+            Err(e) => {
+                tracing::warn!(
+                    "decode: VideoToolbox init failed ({e}); using openh264 software decode"
+                );
+            }
+        }
+    }
+
+    #[cfg(target_os = "android")]
+    {
+        match mediacodec::MediaCodecDecoder::new() {
+            Ok(d) => {
+                tracing::info!("decode: MediaCodec H.264 (hardware)");
+                return Box::new(d);
+            }
+            Err(e) => {
+                tracing::warn!(
+                    "decode: MediaCodec init failed ({e}); using openh264 software decode"
+                );
+            }
+        }
+    }
+
+    #[cfg(target_os = "linux")]
+    {
+        match vaapi::VaapiDecoder::new() {
+            Ok(d) => {
+                tracing::info!("decode: VAAPI H.264 (FFmpeg hardware)");
+                return Box::new(d);
+            }
+            Err(e) => {
+                tracing::warn!("decode: VAAPI init failed ({e}); using openh264 software decode");
+            }
+        }
+    }
+
+    match H264Decoder::new() {
+        Ok(d) => Box::new(d),
+        Err(e) => {
+            tracing::error!("decode: openh264 init failed ({e}); video will not render");
+            Box::new(NullDecoder)
+        }
+    }
+}
+
+/// VP9 software decoder (libvpx via the netdesk-vpx crate). Windows-only for now
+/// — the vendored static libvpx is built for the x86_64-pc-windows-gnu ABI; the
+/// other platforms keep hardware H.264 until their libvpx is vendored too.
+#[cfg(windows)]
+pub struct Vp9Decoder {
+    inner: netdesk_vpx::Decoder,
+}
+
+#[cfg(windows)]
+impl Vp9Decoder {
+    pub fn new() -> Result<Self> {
+        let inner = netdesk_vpx::Decoder::new(netdesk_vpx::VpxCodec::Vp9)
+            .map_err(|e| NdError::Other(anyhow::anyhow!("vp9 decoder init: {e}")))?;
+        Ok(Self { inner })
+    }
+}
+
+#[cfg(windows)]
+impl VideoDecoder for Vp9Decoder {
+    fn decode(&mut self, au: &[u8]) -> Result<Option<Frame>> {
+        let frames = self
+            .inner
+            .decode_to_bgra(au)
+            .map_err(|e| NdError::Other(anyhow::anyhow!("vp9 decode: {e}")))?;
+        // One compressed packet yields 0 or 1 pictures; take it.
+        let Some(f) = frames.into_iter().next() else {
+            return Ok(None);
+        };
+        let (w, h) = (f.width as usize, f.height as usize);
+        let mut rgb = vec![0u8; w * h * 3];
+        for i in 0..(w * h) {
+            // BGRA -> RGB (drop alpha, swap B/R)
+            rgb[i * 3] = f.bgra[i * 4 + 2];
+            rgb[i * 3 + 1] = f.bgra[i * 4 + 1];
+            rgb[i * 3 + 2] = f.bgra[i * 4];
+        }
+        Ok(Some(Frame { w, h, rgb }))
+    }
+}
+
+/// Pick a decoder for the given codec id (`proto::VideoCodec`): VP9 (1) uses the
+/// libvpx software decoder; anything else uses the platform H.264 decoder from
+/// [`best_decoder`]. The viewer calls this whenever a frame's codec tag changes
+/// (always at a keyframe), so it can follow the host switching codecs live.
+pub fn best_decoder_for(codec: u8) -> Box<dyn VideoDecoder + Send> {
+    #[cfg(windows)]
+    {
+        if codec == crate::proto::VideoCodec::Vp9 as u8 {
+            match Vp9Decoder::new() {
+                Ok(d) => {
+                    tracing::info!("decode: VP9 (libvpx software)");
+                    return Box::new(d);
+                }
+                Err(e) => {
+                    tracing::warn!("decode: VP9 init failed ({e}); falling back to H.264");
+                }
+            }
+        }
+    }
+    let _ = codec;
+    best_decoder()
+}