//! H.264 decode. Phase 2 uses openh264 software decode (portable across all five //! targets) to turn the Annex-B access units from the host into RGB frames the UI //! can paint. Hardware decoders (Media Foundation / VideoToolbox / MediaCodec) are //! layered in later behind this same `Frame` output. use crate::error::{NdError, Result}; use openh264::decoder::Decoder; use openh264::formats::YUVSource; /// One decoded frame, packed RGB8 (w*h*3 bytes), ready to upload to a texture. pub struct Frame { pub w: usize, pub h: usize, pub rgb: Vec, } /// Object-safe decode interface shared by every backend. /// /// Both the portable openh264 software decoder ([`H264Decoder`]) and the Windows /// Media Foundation hardware decoder implement this, so the viewer can own a /// `Box` and not care which one is running. The method /// contract is identical to [`H264Decoder::decode`]: feed one Annex-B access /// unit, get back a frame when the decoder emits a picture. pub trait VideoDecoder { fn decode(&mut self, au: &[u8]) -> Result>; } pub struct H264Decoder { inner: Decoder, } impl H264Decoder { pub fn new() -> Result { let inner = Decoder::new().map_err(|e| NdError::Other(anyhow::anyhow!("openh264 init: {e}")))?; Ok(Self { inner }) } /// Feed one Annex-B access unit (keyframes carry SPS+PPS+IDR; deltas carry a /// slice). Returns the decoded RGB frame when the decoder emits a picture. pub fn decode(&mut self, au: &[u8]) -> Result> { match self.inner.decode(au) { Ok(Some(yuv)) => { let (w, h) = yuv.dimensions(); let mut rgb = vec![0u8; w * h * 3]; yuv.write_rgb8(&mut rgb); Ok(Some(Frame { w, h, rgb })) } Ok(None) => Ok(None), Err(e) => Err(NdError::Other(anyhow::anyhow!("decode: {e}"))), } } } impl VideoDecoder for H264Decoder { fn decode(&mut self, au: &[u8]) -> Result> { // Reuse the existing inherent method so both call sites stay in lockstep. H264Decoder::decode(self, au) } } // Per-platform hardware decoders. Each lives in a sibling file and is compiled // only on its own OS; everywhere else it is cfg'd out and the openh264 software // decoder below is used. All implement the same `VideoDecoder` trait, so // `best_decoder` can hand back a `Box` without the // viewer knowing or caring which backend is running. // Windows: DXVA H.264 via Media Foundation. #[cfg(windows)] #[path = "decode_mf.rs"] mod mf; // macOS / iOS: VideoToolbox (VTDecompressionSession). #[cfg(any(target_os = "macos", target_os = "ios"))] #[path = "decode_vt.rs"] mod vt; // Android: MediaCodec (NDK AMediaCodec). #[cfg(target_os = "android")] #[path = "decode_mediacodec.rs"] mod mediacodec; // Linux: VAAPI via FFmpeg/libavcodec. #[cfg(target_os = "linux")] #[path = "decode_vaapi.rs"] mod vaapi; /// Last-resort decoder that drops every access unit. Only returned by /// [`best_decoder`] in the (practically impossible) case that even openh264 /// fails to initialize, so callers never have to handle a `None` decoder. struct NullDecoder; impl VideoDecoder for NullDecoder { fn decode(&mut self, _au: &[u8]) -> Result> { Ok(None) } } /// Pick the fastest decoder available on this platform. /// /// Each OS gets its native hardware H.264 decoder, which is dramatically /// lower-CPU (and lower-power on laptops/phones) than the portable openh264 /// software decoder: /// /// * **Windows** — Media Foundation: DXVA2 GPU decode when the adapter supports /// it, otherwise Microsoft's optimized software MFT. /// * **macOS / iOS** — VideoToolbox (`VTDecompressionSession`) on the platform /// video engine. /// * **Android** — `MediaCodec` (`AMediaCodec`), i.e. the SoC video decoder. /// * **Linux** — VAAPI through FFmpeg/libavcodec (Intel/AMD GPUs via `/dev/dri`). /// /// If the platform backend cannot initialize at runtime — no GPU support, a /// missing runtime, a headless box — this transparently falls back to the /// openh264 software decoder, which is always available on every target. /// /// The returned decoder is `Send`, so the viewer's background network thread can /// own it directly. pub fn best_decoder() -> Box { #[cfg(windows)] { match mf::MediaFoundationDecoder::new() { Ok(d) => { tracing::info!( "decode: Media Foundation H.264 ({})", if d.is_hardware() { "DXVA hardware" } else { "MS software MFT" } ); return Box::new(d); } Err(e) => { tracing::warn!( "decode: Media Foundation init failed ({e}); using openh264 software decode" ); } } } #[cfg(any(target_os = "macos", target_os = "ios"))] { match vt::VideoToolboxDecoder::new() { Ok(d) => { tracing::info!("decode: VideoToolbox H.264 (hardware)"); return Box::new(d); } Err(e) => { tracing::warn!( "decode: VideoToolbox init failed ({e}); using openh264 software decode" ); } } } #[cfg(target_os = "android")] { match mediacodec::MediaCodecDecoder::new() { Ok(d) => { tracing::info!("decode: MediaCodec H.264 (hardware)"); return Box::new(d); } Err(e) => { tracing::warn!( "decode: MediaCodec init failed ({e}); using openh264 software decode" ); } } } #[cfg(target_os = "linux")] { match vaapi::VaapiDecoder::new() { Ok(d) => { tracing::info!("decode: VAAPI H.264 (FFmpeg hardware)"); return Box::new(d); } Err(e) => { tracing::warn!("decode: VAAPI init failed ({e}); using openh264 software decode"); } } } match H264Decoder::new() { Ok(d) => Box::new(d), Err(e) => { tracing::error!("decode: openh264 init failed ({e}); video will not render"); Box::new(NullDecoder) } } } /// VP9 software decoder (libvpx via the netdesk-vpx crate). Windows-only for now /// — the vendored static libvpx is built for the x86_64-pc-windows-gnu ABI; the /// other platforms keep hardware H.264 until their libvpx is vendored too. #[cfg(windows)] pub struct Vp9Decoder { inner: netdesk_vpx::Decoder, } #[cfg(windows)] impl Vp9Decoder { pub fn new() -> Result { let inner = netdesk_vpx::Decoder::new(netdesk_vpx::VpxCodec::Vp9) .map_err(|e| NdError::Other(anyhow::anyhow!("vp9 decoder init: {e}")))?; Ok(Self { inner }) } } #[cfg(windows)] impl VideoDecoder for Vp9Decoder { fn decode(&mut self, au: &[u8]) -> Result> { let frames = self .inner .decode_to_bgra(au) .map_err(|e| NdError::Other(anyhow::anyhow!("vp9 decode: {e}")))?; // One compressed packet yields 0 or 1 pictures; take it. let Some(f) = frames.into_iter().next() else { return Ok(None); }; let (w, h) = (f.width as usize, f.height as usize); let mut rgb = vec![0u8; w * h * 3]; for i in 0..(w * h) { // BGRA -> RGB (drop alpha, swap B/R) rgb[i * 3] = f.bgra[i * 4 + 2]; rgb[i * 3 + 1] = f.bgra[i * 4 + 1]; rgb[i * 3 + 2] = f.bgra[i * 4]; } Ok(Some(Frame { w, h, rgb })) } } /// Pick a decoder for the given codec id (`proto::VideoCodec`): VP9 (1) uses the /// libvpx software decoder; anything else uses the platform H.264 decoder from /// [`best_decoder`]. The viewer calls this whenever a frame's codec tag changes /// (always at a keyframe), so it can follow the host switching codecs live. pub fn best_decoder_for(codec: u8) -> Box { #[cfg(windows)] { if codec == crate::proto::VideoCodec::Vp9 as u8 { match Vp9Decoder::new() { Ok(d) => { tracing::info!("decode: VP9 (libvpx software)"); return Box::new(d); } Err(e) => { tracing::warn!("decode: VP9 init failed ({e}); falling back to H.264"); } } } } let _ = codec; best_decoder() }