Fix corrupted luma from biased subtract in the x86 residue blocks

src/avx512bw/y_to_rgb.rs

@@ -104,7 +104,7 @@ unsafe fn avx512_y_to_rgb_row_impl<const DESTINATION_CHANNELS: u8, const HAS_VBM
     let v_alpha = _mm512_set1_epi8(255u8 as i8);
 
     while cx + 64 < width {
-        let y_s = _mm512_subs_epi8(_mm512_loadu_si512(y_ptr.add(cx) as *const _), y_corr);
+        let y_s = _mm512_subs_epu8(_mm512_loadu_si512(y_ptr.add(cx) as *const _), y_corr);
 
         let y10 = _mm512_expand8_unordered_to_10(y_s);
 
@@ -143,7 +143,7 @@ unsafe fn avx512_y_to_rgb_row_impl<const DESTINATION_CHANNELS: u8, const HAS_VBM
             diff,
         );
 
-        let y_s = _mm512_subs_epi8(_mm512_loadu_si512(y_buffer.as_ptr() as *const _), y_corr);
+        let y_s = _mm512_subs_epu8(_mm512_loadu_si512(y_buffer.as_ptr() as *const _), y_corr);
 
         let y10 = _mm512_expand8_unordered_to_10(y_s);
 

src/sse/y_to_rgba.rs

@@ -101,7 +101,7 @@ unsafe fn sse_y_to_rgba_row_impl<const DESTINATION_CHANNELS: u8>(
     }
 
     while cx + 8 < width {
-        let y_values = _mm_subs_epi8(_mm_loadu_si64(y_ptr.add(cx)), y_corr);
+        let y_values = _mm_subs_epu8(_mm_loadu_si64(y_ptr.add(cx)), y_corr);
 
         let v_low = _mm_mulhrs_epi16(_mm_expand8_lo_to_10(y_values), v_luma_coeff);
 
@@ -134,7 +134,7 @@ unsafe fn sse_y_to_rgba_row_impl<const DESTINATION_CHANNELS: u8>(
             diff,
         );
 
-        let y_values = _mm_subs_epi8(_mm_loadu_si64(y_buffer.as_ptr().cast()), y_corr);
+        let y_values = _mm_subs_epu8(_mm_loadu_si64(y_buffer.as_ptr().cast()), y_corr);
 
         let v_low = _mm_mulhrs_epi16(_mm_expand8_lo_to_10(y_values), v_luma_coeff);
 

src/sse/y_to_rgba_alpha.rs

@@ -104,7 +104,7 @@ unsafe fn sse_y_to_rgba_alpha_row_impl<const DESTINATION_CHANNELS: u8>(
     }
 
     while cx + 8 < width {
-        let y_values = _mm_subs_epi8(_mm_loadu_si64(y_ptr.add(cx)), y_corr);
+        let y_values = _mm_subs_epu8(_mm_loadu_si64(y_ptr.add(cx)), y_corr);
         let a_values = _mm_loadu_si64(a_plane.get_unchecked(cx..).as_ptr());
 
         let v_low = _mm_mulhrs_epi16(_mm_expand8_lo_to_10(y_values), v_luma_coeff);

src/sse/yuv_nv_to_rgba.rs

@@ -264,7 +264,7 @@ unsafe fn sse_yuv_nv_to_rgba_impl<
             }
         }
 
-        let y_values = _mm_subs_epi8(y_vl0, y_corr);
+        let y_values = _mm_subs_epu8(y_vl0, y_corr);
         let u_low = _mm_sub_epi16(u_low_u16, uv_corr);
         let v_low = _mm_sub_epi16(v_low_u16, uv_corr);
         let y_low = _mm_mulhrs_epi16(_mm_expand8_lo_to_10(y_values), v_luma_coeff);
@@ -383,7 +383,7 @@ unsafe fn sse_yuv_nv_to_rgba_impl<
             }
         }
 
-        let y_values = _mm_subs_epi8(y_vl0, y_corr);
+        let y_values = _mm_subs_epu8(y_vl0, y_corr);
         let u_low = _mm_sub_epi16(u_low_u16, uv_corr);
         let v_low = _mm_sub_epi16(v_low_u16, uv_corr);
         let y_low = _mm_mulhrs_epi16(_mm_expand8_lo_to_10(y_values), v_luma_coeff);

src/sse/yuv_nv_to_rgba420.rs

@@ -190,8 +190,8 @@ unsafe fn sse_yuv_nv_to_rgba_impl420<const UV_ORDER: u8, const DESTINATION_CHANN
 
         let uv_values_ = _mm_loadu_si64(uv_ptr.add(uv_x));
 
-        let y_values0 = _mm_subs_epi8(y_vl0, y_corr);
-        let y_values1 = _mm_subs_epi8(y_vl1, y_corr);
+        let y_values0 = _mm_subs_epu8(y_vl0, y_corr);
+        let y_values1 = _mm_subs_epu8(y_vl1, y_corr);
         let (mut u, mut v) = _mm_deinterleave_x2_epi8(uv_values_, zeros);
 
         if order == YuvNVOrder::VU {
@@ -291,8 +291,8 @@ unsafe fn sse_yuv_nv_to_rgba_impl420<const UV_ORDER: u8, const DESTINATION_CHANN
 
         let uv_values_ = _mm_loadu_si64(uv_buffer.as_ptr().cast());
 
-        let y_values0 = _mm_subs_epi8(y_vl0, y_corr);
-        let y_values1 = _mm_subs_epi8(y_vl1, y_corr);
+        let y_values0 = _mm_subs_epu8(y_vl0, y_corr);
+        let y_values1 = _mm_subs_epu8(y_vl1, y_corr);
         let (mut u, mut v) = _mm_deinterleave_x2_epi8(uv_values_, zeros);
 
         if order == YuvNVOrder::VU {

src/sse/yuv_nv_to_rgba422.rs

@@ -163,7 +163,7 @@ unsafe fn sse_yuv_nv_to_rgba_impl422<const UV_ORDER: u8, const DESTINATION_CHANN
 
         let distribute_shuffle = _mm_setr_epi8(0, 0, 0, 0, 1, 1, 1, 1, 2, 2, 2, 2, 3, 3, 3, 3);
 
-        let y_values = _mm_subs_epi8(y_vl0, y_corr);
+        let y_values = _mm_subs_epu8(y_vl0, y_corr);
 
         u = _mm_shuffle_epi8(u, distribute_shuffle);
         v = _mm_shuffle_epi8(v, distribute_shuffle);
@@ -238,7 +238,7 @@ unsafe fn sse_yuv_nv_to_rgba_impl422<const UV_ORDER: u8, const DESTINATION_CHANN
 
         let distribute_shuffle = _mm_setr_epi8(0, 0, 0, 0, 1, 1, 1, 1, 2, 2, 2, 2, 3, 3, 3, 3);
 
-        let y_values = _mm_subs_epi8(y_vl0, y_corr);
+        let y_values = _mm_subs_epu8(y_vl0, y_corr);
 
         u = _mm_shuffle_epi8(u, distribute_shuffle);
         v = _mm_shuffle_epi8(v, distribute_shuffle);

src/sse/yuv_to_rgba.rs

@@ -176,7 +176,7 @@ unsafe fn sse_yuv_to_rgba_row_impl<const DESTINATION_CHANNELS: u8, const SAMPLIN
     }
 
     while cx + 8 < width {
-        let y_values = _mm_subs_epi8(_xx_load_si64(y_ptr.add(cx)), y_corr);
+        let y_values = _mm_subs_epu8(_xx_load_si64(y_ptr.add(cx)), y_corr);
 
         let (u_low_u16, v_low_u16);
 

src/sse/yuv_to_rgba422.rs

@@ -157,7 +157,7 @@ unsafe fn sse_yuv_to_rgba_row_impl422<const DESTINATION_CHANNELS: u8>(
     }
 
     while cx + 8 < width {
-        let y_values = _mm_subs_epi8(_xx_load_si64(y_ptr.add(cx)), y_corr);
+        let y_values = _mm_subs_epu8(_xx_load_si64(y_ptr.add(cx)), y_corr);
 
         let reshuffle = _mm_setr_epi8(0, 0, 0, 0, 1, 1, 1, 1, 2, 2, 2, 2, 3, 3, 3, 3);
 

src/yuv_to_rgba.rs

@@ -1192,6 +1192,57 @@ mod tests {
     use crate::{rgb_to_yuv420, rgb_to_yuv422, rgb_to_yuv444, yuv444_to_rgb, YuvPlanarImageMut};
     use rand::RngExt;
 
+    #[test]
+    fn test_yuv444_limited_range_luma_bias_in_residue_block() {
+        for width in [64usize, 128, 256, 67] {
+            for luma in [0usize, 128] {
+                let y: Vec<u8> = (0..width)
+                    .map(|x| {
+                        if luma == 0 {
+                            (x % 16) as u8
+                        } else {
+                            (128 + x % 8) as u8
+                        }
+                    })
+                    .collect();
+                let u = vec![128u8; width];
+                let v = vec![128u8; width];
+                let image = crate::YuvPlanarImage {
+                    y_plane: &y,
+                    y_stride: width as u32,
+                    u_plane: &u,
+                    u_stride: width as u32,
+                    v_plane: &v,
+                    v_stride: width as u32,
+                    width: width as u32,
+                    height: 1,
+                };
+                let mut rgb = vec![0u8; width * 3];
+                yuv444_to_rgb(
+                    &image,
+                    &mut rgb,
+                    (width * 3) as u32,
+                    YuvRange::Limited,
+                    YuvStandardMatrix::Bt601,
+                )
+                .unwrap();
+
+                for (x, &y_value) in y.iter().enumerate() {
+                    let expect = (1.164 * (f64::from(y_value) - 16.0))
+                        .round()
+                        .clamp(0.0, 255.0) as i32;
+                    for channel in 0..3 {
+                        let got = i32::from(rgb[x * 3 + channel]);
+                        assert!(
+                            (got - expect).abs() <= 3,
+                            "width {width}, luma {y_value}, x {x}, channel {channel}: got {got}, expected ~{expect}"
+                        );
+                    }
+                }
+            }
+        }
+    }
+
     #[test]
     fn test_yuv444_round_trip_full_range() {
         fn matrix(yuv_accuracy: YuvConversionMode, max_diff: i32) {