Tag encoded video as BT.709 and convert to match

InstantReplay.Externals/unienc/crates/unienc_android_mc/src/config.rs

@@ -24,6 +24,9 @@ pub mod format_keys {
     pub const KEY_LEVEL: &str = "level";
     pub const KEY_PRIORITY: &str = "priority";
     pub const KEY_OPERATING_RATE: &str = "operating-rate";
+    pub const KEY_COLOR_STANDARD: &str = "color-standard";
+    pub const KEY_COLOR_TRANSFER: &str = "color-transfer";
+    pub const KEY_COLOR_RANGE: &str = "color-range";
 
     // Audio keys
     pub const KEY_SAMPLE_RATE: &str = "sample-rate";
@@ -33,6 +36,13 @@ pub mod format_keys {
 
 pub const COLOR_FORMAT_SURFACE: jint = 0x7F000789;
 pub const COLOR_FORMAT_YUV420_FLEXIBLE: jint = 0x7F420888;
+
+/// `MediaFormat.COLOR_STANDARD_BT709`, `COLOR_TRANSFER_SDR_VIDEO` and `COLOR_RANGE_LIMITED`.
+/// All three were introduced in API 24, below the minimum this crate targets, so setting them
+/// needs no version guard.
+pub const COLOR_STANDARD_BT709: jint = 1;
+pub const COLOR_TRANSFER_SDR_VIDEO: jint = 3;
+pub const COLOR_RANGE_LIMITED: jint = 2;
 pub const AAC_OBJECT_TYPE_AAC_LC: jint = 2;
 
 pub const MUXER_OUTPUT_FORMAT_MPEG_4: jint = 0;

InstantReplay.Externals/unienc/crates/unienc_android_mc/src/mux/mod.rs

@@ -5,7 +5,10 @@ use unienc_common::{CompletionHandle, Muxer, MuxerInput};
 
 use crate::bindings;
 use crate::common::*;
-use crate::config::MUXER_OUTPUT_FORMAT_MPEG_4;
+use crate::config::{
+    COLOR_RANGE_LIMITED, COLOR_STANDARD_BT709, COLOR_TRANSFER_SDR_VIDEO,
+    MUXER_OUTPUT_FORMAT_MPEG_4, format_keys,
+};
 use crate::error::{AndroidError, Result};
 use crate::java::*;
 
@@ -128,6 +131,23 @@ async fn push(
                     "height".to_string(),
                     crate::common::MediaFormatValue::Integer(height as i32),
                 );
+
+                // The track format is what MediaMuxer writes the `colr` box from. These keys are
+                // set on the encoder and most codecs echo them back in their output format, but
+                // that is not guaranteed across vendors, so state them here as well. The values
+                // must match the ones `create_video_format` configures the encoder with.
+                map.insert(
+                    format_keys::KEY_COLOR_STANDARD.to_string(),
+                    crate::common::MediaFormatValue::Integer(COLOR_STANDARD_BT709),
+                );
+                map.insert(
+                    format_keys::KEY_COLOR_TRANSFER.to_string(),
+                    crate::common::MediaFormatValue::Integer(COLOR_TRANSFER_SDR_VIDEO),
+                );
+                map.insert(
+                    format_keys::KEY_COLOR_RANGE.to_string(),
+                    crate::common::MediaFormatValue::Integer(COLOR_RANGE_LIMITED),
+                );
             }
 
             let mut shared_state_lock = shared_state.write().await;

InstantReplay.Externals/unienc/crates/unienc_android_mc/src/video/mod.rs

@@ -401,5 +401,19 @@ fn create_video_format_raw(
     set_format_integer(env, &format_obj, KEY_PRIORITY, 0)?;
     set_format_integer(env, &format_obj, KEY_OPERATING_RATE, fps_hint as jint)?;
 
+    // Tag the stream as BT.709 limited range. Without these the encoder leaves the color
+    // information unspecified, so the muxed file gives players nothing to go on. The values match
+    // the conversion `VideoFrameBgra32::to_yuv420_planes` performs on the buffer input path, and
+    // on the surface input path they additionally tell MediaCodec which coefficients to use when
+    // it converts the frames the Vulkan preprocessor hands it.
+    set_format_integer(env, &format_obj, KEY_COLOR_STANDARD, COLOR_STANDARD_BT709)?;
+    set_format_integer(
+        env,
+        &format_obj,
+        KEY_COLOR_TRANSFER,
+        COLOR_TRANSFER_SDR_VIDEO,
+    )?;
+    set_format_integer(env, &format_obj, KEY_COLOR_RANGE, COLOR_RANGE_LIMITED)?;
+
     SafeGlobalRef::new(env, format_obj)
 }

InstantReplay.Externals/unienc/crates/unienc_apple_vt/src/mux/mod.rs

@@ -16,10 +16,14 @@ use objc2_av_foundation::{
 use objc2_core_audio_types::{
     AudioStreamBasicDescription, AudioStreamPacketDescription, MPEG4ObjectID, kAudioFormatMPEG4AAC,
 };
+use objc2_core_foundation::{CFDictionary, CFString, CFType};
 use objc2_core_media::{
     CMAudioFormatDescriptionCreate, CMAudioSampleBufferCreateReadyWithPacketDescriptions,
     CMBlockBuffer, CMFormatDescription, CMSampleBuffer, CMTime, CMVideoFormatDescriptionCreate,
-    kCMBlockBufferAssureMemoryNowFlag, kCMTimeZero, kCMVideoCodecType_H264,
+    kCMBlockBufferAssureMemoryNowFlag, kCMFormatDescriptionColorPrimaries_ITU_R_709_2,
+    kCMFormatDescriptionExtension_ColorPrimaries, kCMFormatDescriptionExtension_TransferFunction,
+    kCMFormatDescriptionExtension_YCbCrMatrix, kCMFormatDescriptionTransferFunction_ITU_R_709_2,
+    kCMFormatDescriptionYCbCrMatrix_ITU_R_709_2, kCMTimeZero, kCMVideoCodecType_H264,
 };
 use objc2_foundation::{NSString, NSURL};
 use tokio::sync::{mpsc, oneshot};
@@ -165,14 +169,37 @@ impl AVFMuxer {
             objc2_av_foundation::AVAssetWriter::assetWriterWithURL_fileType_error(&url, file_type)?
         };
 
+        // AVAssetWriter writes the `colr` box from the format description it is given, not from
+        // the one the encoder produced: the encoded frames travel back through the caller as
+        // opaque buffers, so this muxer never sees the encoder's format description. Without
+        // these extensions the output carries no color information at all, which makes players
+        // guess at the color space. The values match the tags the encoder writes into the SPS.
+        let color_extensions = {
+            let keys: [&CFString; 3] = unsafe {
+                [
+                    kCMFormatDescriptionExtension_ColorPrimaries,
+                    kCMFormatDescriptionExtension_TransferFunction,
+                    kCMFormatDescriptionExtension_YCbCrMatrix,
+                ]
+            };
+            let values: [&CFType; 3] = unsafe {
+                [
+                    kCMFormatDescriptionColorPrimaries_ITU_R_709_2,
+                    kCMFormatDescriptionTransferFunction_ITU_R_709_2,
+                    kCMFormatDescriptionYCbCrMatrix_ITU_R_709_2,
+                ]
+            };
+            CFDictionary::from_slices(&keys, &values)
+        };
+
         let source_format_hint = unsafe {
             let mut format_desc: *const CMFormatDescription = std::ptr::null();
             CMVideoFormatDescriptionCreate(
                 allocator::default(),
                 kCMVideoCodecType_H264,
                 video_options.width() as i32,
                 video_options.height() as i32,
-                None,
+                Some(color_extensions.as_opaque()),
                 NonNull::new(&mut format_desc).unwrap(),
             )
             .to_result()?;

InstantReplay.Externals/unienc/crates/unienc_apple_vt/src/video/mod.rs

@@ -8,13 +8,17 @@ use objc2_core_foundation::{
     CFBoolean, CFDictionary, CFNumber, CFString, CFType, kCFBooleanFalse, kCFBooleanTrue,
 };
 use objc2_core_media::{
-    CMSampleBuffer, CMTime, kCMSampleAttachmentKey_NotSync, kCMTimeInvalid, kCMVideoCodecType_H264,
+    CMSampleBuffer, CMTime, kCMFormatDescriptionColorPrimaries_ITU_R_709_2,
+    kCMFormatDescriptionTransferFunction_ITU_R_709_2, kCMFormatDescriptionYCbCrMatrix_ITU_R_709_2,
+    kCMSampleAttachmentKey_NotSync, kCMTimeInvalid, kCMVideoCodecType_H264,
 };
 use objc2_core_video::{CVPixelBuffer, CVPixelBufferCreateWithBytes, kCVPixelFormatType_32BGRA};
 use objc2_video_toolbox::{
     VTCompressionSession, VTEncodeInfoFlags, VTSessionSetProperty,
     kVTCompressionPropertyKey_AllowFrameReordering, kVTCompressionPropertyKey_AverageBitRate,
-    kVTCompressionPropertyKey_RealTime, kVTInvalidSessionErr,
+    kVTCompressionPropertyKey_ColorPrimaries, kVTCompressionPropertyKey_RealTime,
+    kVTCompressionPropertyKey_TransferFunction, kVTCompressionPropertyKey_YCbCrMatrix,
+    kVTInvalidSessionErr,
 };
 use tokio::sync::mpsc;
 use unienc_common::{
@@ -383,6 +387,36 @@ impl CompressionSession {
         }
         .to_result()?;
 
+        // Tag the stream as BT.709 limited range. Without these the encoder leaves the color
+        // information unspecified, so the muxed file carries no `colr` box and players have to
+        // guess. The values must match the conversion `VideoFrameBgra32::to_yuv420_planes`
+        // performs on the readback path, and the matrix additionally tells VideoToolbox which
+        // coefficients to use when it converts the BGRA pixel buffers of the blit path itself.
+        unsafe {
+            VTSessionSetProperty(
+                &session,
+                kVTCompressionPropertyKey_ColorPrimaries,
+                Some(kCMFormatDescriptionColorPrimaries_ITU_R_709_2 as &CFType),
+            )
+        }
+        .to_result()?;
+        unsafe {
+            VTSessionSetProperty(
+                &session,
+                kVTCompressionPropertyKey_TransferFunction,
+                Some(kCMFormatDescriptionTransferFunction_ITU_R_709_2 as &CFType),
+            )
+        }
+        .to_result()?;
+        unsafe {
+            VTSessionSetProperty(
+                &session,
+                kVTCompressionPropertyKey_YCbCrMatrix,
+                Some(kCMFormatDescriptionYCbCrMatrix_ITU_R_709_2 as &CFType),
+            )
+        }
+        .to_result()?;
+
         Ok(CompressionSession { inner: session })
     }
 }

InstantReplay.Externals/unienc/crates/unienc_common/src/lib.rs

@@ -147,23 +147,40 @@ impl VideoFrameBgra32 {
         let mut u_data = vec![128u8; padded_uv_size]; // Neutral for U
         let mut v_data = vec![128u8; padded_uv_size]; // Neutral for V
 
-        // Convert ARGB to YUV for the original image area only
+        // Convert ARGB to YUV for the original image area only.
+        //
+        // BT.709 limited range ("studio swing"), 8-bit fixed point with a denominator of 256:
+        //
+        //     Y  = 16  + (219/255) * ( 0.2126 R + 0.7152 G + 0.0722 B )
+        //     Cb = 128 + (224/255) * ( B - Y ) / 1.8556
+        //     Cr = 128 + (224/255) * ( R - Y ) / 1.5748
+        //
+        // Scaling those factors by 256 gives (46.74, 157.24, 15.87) for Y, (-25.76, -86.67,
+        // 112.43) for Cb and (112.43, -102.13, -10.30) for Cr. The Cb row is rounded to
+        // (-26, -86, 112) instead of to the nearest integers so that every row sums to the value
+        // the reference formula requires: 220 for Y, which maps white to 235 and black to 16, and
+        // 0 for Cb and Cr, which maps neutral colors to exactly 128. That also keeps both chroma
+        // rows inside the nominal 16..240 range at the primary extremes, as the BT.601
+        // coefficients this replaces did. The results therefore always fit in u8 without clamping.
+        //
+        // The color tags written by each platform encoder must stay in sync with these
+        // coefficients.
         for y in 0..self.height {
             for x in 0..self.width {
                 let bgra_idx = ((y * self.width + x) * 4) as usize;
                 let r = data[bgra_idx + 2] as i32;
                 let g = data[bgra_idx + 1] as i32;
                 let b = data[bgra_idx] as i32;
 
-                let y_val = (((66 * r + 129 * g + 25 * b + 128) >> 8) + 16) as u8;
+                let y_val = (((47 * r + 157 * g + 16 * b + 128) >> 8) + 16) as u8;
 
                 let y_idx = (y * w + x) as usize;
                 y_data[y_idx] = y_val;
 
                 // Sample U and V for every 2x2 block (4:2:0 subsampling)
                 if x % 2 == 0 && y % 2 == 0 {
-                    let u_val = (((-38 * r - 74 * g + 112 * b + 128) >> 8) + 128) as u8;
-                    let v_val = (((112 * r - 94 * g - 18 * b + 128) >> 8) + 128) as u8;
+                    let u_val = (((-26 * r - 86 * g + 112 * b + 128) >> 8) + 128) as u8;
+                    let v_val = (((112 * r - 102 * g - 10 * b + 128) >> 8) + 128) as u8;
 
                     let uv_idx = ((y / 2) * (w / 2) + (x / 2)) as usize;
                     u_data[uv_idx] = u_val;

InstantReplay.Externals/unienc/crates/unienc_ffmpeg/src/video/mod.rs

@@ -167,6 +167,21 @@ impl<R: Runtime + 'static> FFmpegVideoEncoder<R> {
                     &format!("{}", options.bitrate()),
                     "-force_key_frames",
                     "expr:gte(t,n_forced*1)",
+                    // Convert to and tag BT.709 limited range. Tagging the frames makes the
+                    // auto-inserted scale filter use the BT.709 matrix for the BGRA to YUV
+                    // conversion (FFmpeg otherwise derives the matrix from the frame size, which
+                    // yields BT.601 at the sizes this library records) and makes the encoder write
+                    // the color information into the SPS VUI. The muxer stream-copies this
+                    // elementary stream, so the VUI is what carries the tags into the MP4.
+                    //
+                    // The `-color_primaries` / `-color_trc` output options would be the more
+                    // obvious spelling, but FFmpeg does not reliably forward them to the encoder:
+                    // as of 8.0.1 only the matrix and the range reach the VUI that way, leaving
+                    // the primaries and the transfer function unspecified. Setting them on the
+                    // frames is honored by every encoder because it does not depend on the
+                    // encoder wrapper reading them off the codec context.
+                    "-vf",
+                    "setparams=color_primaries=bt709:color_trc=bt709:colorspace=bt709:range=tv",
                 ],
                 ffmpeg::Destination::Stdout,
             )?;

InstantReplay.Externals/unienc/crates/unienc_webcodecs/src/js/library.ts

@@ -225,7 +225,18 @@ window["unienc_webcodecs"] = {
                         offset: 0,
                         stride: options.width * 4  // BGRA = 4 bytes per pixel
                     }
-                ]
+                ],
+                // Describe the source buffer as sRGB. WebCodecs has no way to ask the encoder for
+                // a particular output color space, so this is the only lever available: the user
+                // agent converts to the encoder's YUV space itself and tags the stream from what
+                // the source frame declares. Left unset it has to guess, which is what makes the
+                // output carry no color information today.
+                colorSpace: {
+                    primaries: "bt709",
+                    transfer: "iec61966-2-1",
+                    matrix: "rgb",
+                    fullRange: true
+                }
             };
             const frame = new VideoFrame(data, init);
             encoder.encode(frame, {

InstantReplay.Externals/unienc/crates/unienc_windows_mf/src/video/mod.rs

@@ -16,6 +16,22 @@ pub struct MediaFoundationVideoEncoder {
     fps_hint: f64,
 }
 
+/// Describes a media type as BT.709 limited range.
+///
+/// On the input type this states what the NV12 buffers `VideoFrameBgra32::to_yuv420_planes`
+/// produces actually contain, and on the output type it asks the encoder to record the same in the
+/// H.264 VUI. Without it the color information stays unspecified all the way into the file and
+/// players have to guess at the color space.
+fn set_bt709_color_attributes(media_type: &IMFMediaType) -> Result<()> {
+    unsafe {
+        media_type.SetUINT32(&MF_MT_VIDEO_PRIMARIES, MFVideoPrimaries_BT709.0 as u32)?;
+        media_type.SetUINT32(&MF_MT_TRANSFER_FUNCTION, MFVideoTransFunc_709.0 as u32)?;
+        media_type.SetUINT32(&MF_MT_YUV_MATRIX, MFVideoTransferMatrix_BT709.0 as u32)?;
+        media_type.SetUINT32(&MF_MT_VIDEO_NOMINAL_RANGE, MFNominalRange_16_235.0 as u32)?;
+    }
+    Ok(())
+}
+
 impl MediaFoundationVideoEncoder {
     pub fn new<V: VideoEncoderOptions>(options: &V, runtime: &impl Runtime) -> Result<Self> {
         let input_type = unsafe {
@@ -30,6 +46,7 @@ impl MediaFoundationVideoEncoder {
             )?;
 
             input_type.SetUINT64(&MF_MT_FRAME_RATE, ((options.fps_hint() as u64) << 32) + 1)?;
+            set_bt709_color_attributes(&input_type)?;
             input_type
         };
 
@@ -45,6 +62,7 @@ impl MediaFoundationVideoEncoder {
             )?;
             output_type.SetUINT32(&MF_MT_INTERLACE_MODE, MFVideoInterlace_Progressive.0 as u32)?;
             output_type.SetUINT32(&MF_MT_MPEG2_PROFILE, eAVEncH264VProfile_Base.0 as u32)?;
+            set_bt709_color_attributes(&output_type)?;
             output_type
         };