netdesk/native/core/src/decode.rs

//! H.264 decode. Phase 2 uses openh264 software decode (portable across all five
//! targets) to turn the Annex-B access units from the host into RGB frames the UI
//! can paint. Hardware decoders (Media Foundation / VideoToolbox / MediaCodec) are
//! layered in later behind this same `Frame` output.

use crate::error::{NdError, Result};
use openh264::decoder::Decoder;
use openh264::formats::YUVSource;

/// One decoded frame, packed RGB8 (w*h*3 bytes), ready to upload to a texture.
pub struct Frame {
    pub w: usize,
    pub h: usize,
    pub rgb: Vec<u8>,
}

/// Object-safe decode interface shared by every backend.
///
/// Both the portable openh264 software decoder ([`H264Decoder`]) and the Windows
/// Media Foundation hardware decoder implement this, so the viewer can own a
/// `Box<dyn VideoDecoder + Send>` and not care which one is running. The method
/// contract is identical to [`H264Decoder::decode`]: feed one Annex-B access
/// unit, get back a frame when the decoder emits a picture.
pub trait VideoDecoder {
    fn decode(&mut self, au: &[u8]) -> Result<Option<Frame>>;
}

pub struct H264Decoder {
    inner: Decoder,
}

impl H264Decoder {
    pub fn new() -> Result<Self> {
        let inner =
            Decoder::new().map_err(|e| NdError::Other(anyhow::anyhow!("openh264 init: {e}")))?;
        Ok(Self { inner })
    }

    /// Feed one Annex-B access unit (keyframes carry SPS+PPS+IDR; deltas carry a
    /// slice). Returns the decoded RGB frame when the decoder emits a picture.
    pub fn decode(&mut self, au: &[u8]) -> Result<Option<Frame>> {
        match self.inner.decode(au) {
            Ok(Some(yuv)) => {
                let (w, h) = yuv.dimensions();
                let mut rgb = vec![0u8; w * h * 3];
                yuv.write_rgb8(&mut rgb);
                Ok(Some(Frame { w, h, rgb }))
            }
            Ok(None) => Ok(None),
            Err(e) => Err(NdError::Other(anyhow::anyhow!("decode: {e}"))),
        }
    }
}

impl VideoDecoder for H264Decoder {
    fn decode(&mut self, au: &[u8]) -> Result<Option<Frame>> {
        // Reuse the existing inherent method so both call sites stay in lockstep.
        H264Decoder::decode(self, au)
    }
}

// Per-platform hardware decoders. Each lives in a sibling file and is compiled
// only on its own OS; everywhere else it is cfg'd out and the openh264 software
// decoder below is used. All implement the same `VideoDecoder` trait, so
// `best_decoder` can hand back a `Box<dyn VideoDecoder + Send>` without the
// viewer knowing or caring which backend is running.

// Windows: DXVA H.264 via Media Foundation.
#[cfg(windows)]
#[path = "decode_mf.rs"]
mod mf;

// macOS / iOS: VideoToolbox (VTDecompressionSession).
#[cfg(any(target_os = "macos", target_os = "ios"))]
#[path = "decode_vt.rs"]
mod vt;

// Android: MediaCodec (NDK AMediaCodec).
#[cfg(target_os = "android")]
#[path = "decode_mediacodec.rs"]
mod mediacodec;

// Linux: VAAPI via FFmpeg/libavcodec.
#[cfg(target_os = "linux")]
#[path = "decode_vaapi.rs"]
mod vaapi;

/// Last-resort decoder that drops every access unit. Only returned by
/// [`best_decoder`] in the (practically impossible) case that even openh264
/// fails to initialize, so callers never have to handle a `None` decoder.
struct NullDecoder;

impl VideoDecoder for NullDecoder {
    fn decode(&mut self, _au: &[u8]) -> Result<Option<Frame>> {
        Ok(None)
    }
}

/// Pick the fastest decoder available on this platform.
///
/// Each OS gets its native hardware H.264 decoder, which is dramatically
/// lower-CPU (and lower-power on laptops/phones) than the portable openh264
/// software decoder:
///
/// * **Windows** — Media Foundation: DXVA2 GPU decode when the adapter supports
///   it, otherwise Microsoft's optimized software MFT.
/// * **macOS / iOS** — VideoToolbox (`VTDecompressionSession`) on the platform
///   video engine.
/// * **Android** — `MediaCodec` (`AMediaCodec`), i.e. the SoC video decoder.
/// * **Linux** — VAAPI through FFmpeg/libavcodec (Intel/AMD GPUs via `/dev/dri`).
///
/// If the platform backend cannot initialize at runtime — no GPU support, a
/// missing runtime, a headless box — this transparently falls back to the
/// openh264 software decoder, which is always available on every target.
///
/// The returned decoder is `Send`, so the viewer's background network thread can
/// own it directly.
pub fn best_decoder() -> Box<dyn VideoDecoder + Send> {
    #[cfg(windows)]
    {
        match mf::MediaFoundationDecoder::new() {
            Ok(d) => {
                tracing::info!(
                    "decode: Media Foundation H.264 ({})",
                    if d.is_hardware() {
                        "DXVA hardware"
                    } else {
                        "MS software MFT"
                    }
                );
                return Box::new(d);
            }
            Err(e) => {
                tracing::warn!(
                    "decode: Media Foundation init failed ({e}); using openh264 software decode"
                );
            }
        }
    }

    #[cfg(any(target_os = "macos", target_os = "ios"))]
    {
        match vt::VideoToolboxDecoder::new() {
            Ok(d) => {
                tracing::info!("decode: VideoToolbox H.264 (hardware)");
                return Box::new(d);
            }
            Err(e) => {
                tracing::warn!(
                    "decode: VideoToolbox init failed ({e}); using openh264 software decode"
                );
            }
        }
    }

    #[cfg(target_os = "android")]
    {
        match mediacodec::MediaCodecDecoder::new() {
            Ok(d) => {
                tracing::info!("decode: MediaCodec H.264 (hardware)");
                return Box::new(d);
            }
            Err(e) => {
                tracing::warn!(
                    "decode: MediaCodec init failed ({e}); using openh264 software decode"
                );
            }
        }
    }

    #[cfg(target_os = "linux")]
    {
        match vaapi::VaapiDecoder::new() {
            Ok(d) => {
                tracing::info!("decode: VAAPI H.264 (FFmpeg hardware)");
                return Box::new(d);
            }
            Err(e) => {
                tracing::warn!("decode: VAAPI init failed ({e}); using openh264 software decode");
            }
        }
    }

    match H264Decoder::new() {
        Ok(d) => Box::new(d),
        Err(e) => {
            tracing::error!("decode: openh264 init failed ({e}); video will not render");
            Box::new(NullDecoder)
        }
    }
}

/// VP9 software decoder (libvpx via the netdesk-vpx crate). Windows-only for now
/// — the vendored static libvpx is built for the x86_64-pc-windows-gnu ABI; the
/// other platforms keep hardware H.264 until their libvpx is vendored too.
#[cfg(windows)]
pub struct Vp9Decoder {
    inner: netdesk_vpx::Decoder,
}

#[cfg(windows)]
impl Vp9Decoder {
    pub fn new() -> Result<Self> {
        let inner = netdesk_vpx::Decoder::new(netdesk_vpx::VpxCodec::Vp9)
            .map_err(|e| NdError::Other(anyhow::anyhow!("vp9 decoder init: {e}")))?;
        Ok(Self { inner })
    }
}

#[cfg(windows)]
impl VideoDecoder for Vp9Decoder {
    fn decode(&mut self, au: &[u8]) -> Result<Option<Frame>> {
        let frames = self
            .inner
            .decode_to_bgra(au)
            .map_err(|e| NdError::Other(anyhow::anyhow!("vp9 decode: {e}")))?;
        // One compressed packet yields 0 or 1 pictures; take it.
        let Some(f) = frames.into_iter().next() else {
            return Ok(None);
        };
        let (w, h) = (f.width as usize, f.height as usize);
        let mut rgb = vec![0u8; w * h * 3];
        for i in 0..(w * h) {
            // BGRA -> RGB (drop alpha, swap B/R)
            rgb[i * 3] = f.bgra[i * 4 + 2];
            rgb[i * 3 + 1] = f.bgra[i * 4 + 1];
            rgb[i * 3 + 2] = f.bgra[i * 4];
        }
        Ok(Some(Frame { w, h, rgb }))
    }
}

/// Pick a decoder for the given codec id (`proto::VideoCodec`): VP9 (1) uses the
/// libvpx software decoder; anything else uses the platform H.264 decoder from
/// [`best_decoder`]. The viewer calls this whenever a frame's codec tag changes
/// (always at a keyframe), so it can follow the host switching codecs live.
pub fn best_decoder_for(codec: u8) -> Box<dyn VideoDecoder + Send> {
    #[cfg(windows)]
    {
        if codec == crate::proto::VideoCodec::Vp9 as u8 {
            match Vp9Decoder::new() {
                Ok(d) => {
                    tracing::info!("decode: VP9 (libvpx software)");
                    return Box::new(d);
                }
                Err(e) => {
                    tracing::warn!("decode: VP9 init failed ({e}); falling back to H.264");
                }
            }
        }
    }
    let _ = codec;
    best_decoder()
}