From c74f23784c262b4bfa165a40e586bd9712c50f31 Mon Sep 17 00:00:00 2001 From: "Sergey \"Shnatsel\" Davidoff" Date: Thu, 26 Feb 2026 15:45:21 +0000 Subject: [PATCH] Vectorize byte-level horizontal differencing in fp_predict Process the difference step in reverse 16-byte chunks, reading both curr and prev into local arrays before writing. This makes all 16 subtractions independent, enabling autovectorization (confirmed with SSE2: movdqu + psubb). Co-Authored-By: Claude --- src/encoder/colortype.rs | 57 ++++++++++++++++++++++++++++++++++++---- 1 file changed, 52 insertions(+), 5 deletions(-) diff --git a/src/encoder/colortype.rs b/src/encoder/colortype.rs index b717669..ef4b9a8 100644 --- a/src/encoder/colortype.rs +++ b/src/encoder/colortype.rs @@ -30,12 +30,59 @@ macro_rules! fp_predict { } } - // Step 3: Apply byte-level horizontal difference across the entire buffer - // We iterate backwards so that when we compute output[i] - output[i-samples], - // output[i-samples] still has its original value. + // Step 3: Apply byte-level horizontal difference across the entire buffer. // The prediction spans across quarter boundaries, just like the decoder's cumulative sum. - let output = &mut result[start_len..]; - for i in (samples..byte_count).rev() { + // + // We process elements in reverse in batches of 16 to enable autovectorization. + // Reverse order ensures that when we compute output[i] - output[i-samples], + // the value at output[i-samples] hasn't been overwritten yet (critical when + // samples < 16, which is the common case: 1, 3, 4, 5). + // + // 8 bits * 16 = 128 bits, the width of a typical SIMD register. + // Even if the target has no vector instructions, this still benefits from + // instruction-level parallelism, since unlike in decoding, all subtractions + // within a chunk are truly independent of each other. + let output = &mut result[start_len..byte_count + start_len]; + let diff_len = byte_count - samples; + let remainder_len = diff_len % 16; + // Process full 16-byte chunks in reverse. + // We iterate from the end so that predecessor values (at lower indices) + // are still in their original state when we read them. + let mut chunk_end = byte_count; + while chunk_end >= samples + remainder_len + 16 { + let chunk_start = chunk_end - 16; + let prev_start = chunk_start - samples; + + // Read all values in the chunk and their predecessors first. + // This makes all subtractions below independent, enabling vectorization. + let curr: [u8; 16] = output[chunk_start..chunk_end].try_into().unwrap(); + let prev: [u8; 16] = output[prev_start..prev_start + 16].try_into().unwrap(); + + let chunk = &mut output[chunk_start..chunk_end]; + chunk[0] = curr[0].wrapping_sub(prev[0]); + chunk[1] = curr[1].wrapping_sub(prev[1]); + chunk[2] = curr[2].wrapping_sub(prev[2]); + chunk[3] = curr[3].wrapping_sub(prev[3]); + chunk[4] = curr[4].wrapping_sub(prev[4]); + chunk[5] = curr[5].wrapping_sub(prev[5]); + chunk[6] = curr[6].wrapping_sub(prev[6]); + chunk[7] = curr[7].wrapping_sub(prev[7]); + chunk[8] = curr[8].wrapping_sub(prev[8]); + chunk[9] = curr[9].wrapping_sub(prev[9]); + chunk[10] = curr[10].wrapping_sub(prev[10]); + chunk[11] = curr[11].wrapping_sub(prev[11]); + chunk[12] = curr[12].wrapping_sub(prev[12]); + chunk[13] = curr[13].wrapping_sub(prev[13]); + chunk[14] = curr[14].wrapping_sub(prev[14]); + chunk[15] = curr[15].wrapping_sub(prev[15]); + + chunk_end = chunk_start; + } + // Handle the leading remainder that doesn't fill a complete chunk of 16. + // This must be done last (after the reverse chunk loop) because these are + // the lowest indices, and the chunks above may read predecessor values + // from this range. + for i in (samples..samples + remainder_len).rev() { output[i] = output[i].wrapping_sub(output[i - samples]); } }