From dc94cd44368ef557c0bb1424c9e4142acb5182ff Mon Sep 17 00:00:00 2001 From: Lilith River Date: Sun, 5 Apr 2026 22:00:22 -0600 Subject: [PATCH 1/2] bench: add comprehensive decode benchmarks Add decode benchmark covering 16 TIFF configurations with both a real photo corpus (if available) and synthetic in-memory images: Strip: uncompressed, PackBits, LZW, LZW+hpred, deflate+hpred, RGB16 deflate+hpred, gray8 LZW, gray8 deflate+hpred, RGB f32 deflate+fpred, gray f32 deflate+fpred, palette8 LZW, bilevel fax4. Tiled: RGB8 LZW/deflate/JPEG 256, gray f32 deflate+fpred 512. Synthetic benchmarks isolate predictor and compression overhead with controlled 2048x2048 images. A callgrind profiling example is included for instruction-level analysis. --- .gitignore | 1 + Cargo.toml | 4 + benches/decode.rs | 461 +++++++++++++++++++++++++++++++++++++ examples/profile_decode.rs | 130 +++++++++++ 4 files changed, 596 insertions(+) create mode 100644 benches/decode.rs create mode 100644 examples/profile_decode.rs diff --git a/.gitignore b/.gitignore index 143b1ca..4ad8ac2 100644 --- a/.gitignore +++ b/.gitignore @@ -2,3 +2,4 @@ /target/ **/*.rs.bk Cargo.lock +tests/benches/corpus/ diff --git a/Cargo.toml b/Cargo.toml index 752c199..0df78ad 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -54,6 +54,10 @@ harness = false name = "encode" harness = false +[[bench]] +name = "decode" +harness = false + [[test]] name = "decode_libtiffpic" diff --git a/benches/decode.rs b/benches/decode.rs new file mode 100644 index 0000000..8b0a68f --- /dev/null +++ b/benches/decode.rs @@ -0,0 +1,461 @@ +extern crate criterion; +extern crate tiff; + +use criterion::{black_box, Criterion, Throughput}; +use std::io::Cursor; +use tiff::decoder::Decoder; +use tiff::encoder::colortype::{Gray16, Gray8, RGB32Float, RGB16, RGB8, RGBA8}; +use tiff::encoder::{Compression, Predictor, TiffEncoder}; + +const WIDTH: u32 = 2048; +const HEIGHT: u32 = 2048; + +// === Corpus paths (real photo-based TIFFs) === +// Set TIFF_BENCH_DIR / TIFF_GEOTIFF_DIR env vars to a directory containing the corpus files. +// If the files are not found, the corresponding benchmarks are silently skipped. +// The corpus can be generated with vips/imagemagick from any large photo; see the benchmark +// names for the expected filenames and compression settings. +const CORPUS_DIR: &str = "tests/benches/corpus"; + +fn bench_dir() -> String { + std::env::var("TIFF_BENCH_DIR").unwrap_or_else(|_| CORPUS_DIR.to_string()) +} + +fn geotiff_dir() -> String { + std::env::var("TIFF_GEOTIFF_DIR").unwrap_or_else(|_| CORPUS_DIR.to_string()) +} + +/// Load a file from the corpus directory, returning None if not found. +fn load_corpus(dir: &str, name: &str) -> Option> { + let path = format!("{}/{}", dir, name); + std::fs::read(&path).ok() +} + +fn decode_tiff_bytes(data: &[u8]) { + let cursor = Cursor::new(data); + let mut decoder = Decoder::open(black_box(cursor)).unwrap(); + decoder.next_image().unwrap(); + decoder.read_image().unwrap(); +} + +// === Synthetic image generators === + +fn generate_rgb8(w: u32, h: u32) -> Vec { + (0..w * h * 3) + .map(|i| ((i * 7 + i / w * 3) & 0xFF) as u8) + .collect() +} + +fn generate_rgba8(w: u32, h: u32) -> Vec { + (0..w * h * 4) + .map(|i| ((i * 11 + i / w * 5) & 0xFF) as u8) + .collect() +} + +fn generate_gray8(w: u32, h: u32) -> Vec { + (0..w * h) + .map(|i| ((i * 7 + i / w * 3) & 0xFF) as u8) + .collect() +} + +fn generate_rgb16(w: u32, h: u32) -> Vec { + (0..w * h * 3) + .map(|i| ((i * 257 + i / w * 131) & 0xFFFF) as u16) + .collect() +} + +fn generate_gray16(w: u32, h: u32) -> Vec { + (0..w * h) + .map(|i| ((i * 257 + i / w * 131) & 0xFFFF) as u16) + .collect() +} + +fn generate_rgb_f32(w: u32, h: u32) -> Vec { + let mut data = Vec::with_capacity((w * h * 3) as usize); + for y in 0..h { + for x in 0..w { + let fx = x as f32 / w as f32; + let fy = y as f32 / h as f32; + data.push(fx * 0.5 + fy * 0.3); + data.push(fy * 0.7 + fx * 0.2); + data.push((fx + fy) * 0.25); + } + } + data +} + +// === Synthetic encoders (strip-only, encoder doesn't support tiles) === + +fn encode( + data: &[C::Inner], + w: u32, + h: u32, + compression: Compression, + predictor: Predictor, +) -> Vec +where + [C::Inner]: tiff::encoder::TiffValue, +{ + let mut buf = Cursor::new(Vec::new()); + TiffEncoder::new(&mut buf) + .unwrap() + .with_predictor(predictor) + .with_compression(compression) + .write_image::(w, h, data) + .unwrap(); + buf.into_inner() +} + +fn main() { + let mut c = Criterion::default().configure_from_args(); + let bd = bench_dir(); + let gd = geotiff_dir(); + + // ======================================================================== + // PART 1: Real photo corpus (5616x3744, ~21MP) + // ======================================================================== + // Raw pixel sizes for throughput calculation: + // RGB8: 5616*3744*3 = 63,078,912 + // RGB16: 5616*3744*6 = 126,157,824 + // F32x3: 5616*3744*12 = 252,315,648 + // Gray8: 5616*3744*1 = 21,026,304 + // GrayF32: 5616*3744*4 = 84,105,216 + // Palette8: 5616*3744*1 = 21,026,304 (output) + + struct CorpusBench { + file: &'static str, + name: &'static str, + raw_bytes: u64, + } + + let photo_benches = [ + // Strip-based + CorpusBench { + file: "photo-rgb8-none.tif", + name: "strip-rgb8-none", + raw_bytes: 63_078_912, + }, + CorpusBench { + file: "photo-rgb8-packbits.tif", + name: "strip-rgb8-packbits", + raw_bytes: 63_078_912, + }, + CorpusBench { + file: "photo-rgb8-lzw.tif", + name: "strip-rgb8-lzw", + raw_bytes: 63_078_912, + }, + CorpusBench { + file: "photo-rgb8-lzw-hpred.tif", + name: "strip-rgb8-lzw-hpred", + raw_bytes: 63_078_912, + }, + CorpusBench { + file: "photo-rgb8-deflate-hpred.tif", + name: "strip-rgb8-deflate-hpred", + raw_bytes: 63_078_912, + }, + CorpusBench { + file: "photo-rgb16-deflate-hpred.tif", + name: "strip-rgb16-deflate-hpred", + raw_bytes: 126_157_824, + }, + CorpusBench { + file: "photo-gray8-lzw.tif", + name: "strip-gray8-lzw", + raw_bytes: 21_026_304, + }, + CorpusBench { + file: "photo-gray8-deflate-hpred.tif", + name: "strip-gray8-deflate-hpred", + raw_bytes: 21_026_304, + }, + CorpusBench { + file: "photo-rgb-f32-deflate-fpred.tif", + name: "strip-rgb-f32-deflate-fpred", + raw_bytes: 252_315_648, + }, + CorpusBench { + file: "photo-gray-f32-deflate-fpred.tif", + name: "strip-gray-f32-deflate-fpred", + raw_bytes: 84_105_216, + }, + CorpusBench { + file: "photo-palette8-lzw.tif", + name: "strip-palette8-lzw", + raw_bytes: 21_026_304, + }, + CorpusBench { + file: "scan-bilevel-fax4.tif", + name: "strip-bilevel-fax4", + raw_bytes: 5616 * 3744 / 8, + }, + // Tiled + CorpusBench { + file: "photo-rgb8-lzw-hpred-tiled256.tif", + name: "tiled256-rgb8-lzw-hpred", + raw_bytes: 63_078_912, + }, + CorpusBench { + file: "photo-rgb8-deflate-hpred-tiled256.tif", + name: "tiled256-rgb8-deflate-hpred", + raw_bytes: 63_078_912, + }, + CorpusBench { + file: "photo-rgb8-jpeg-tiled256.tif", + name: "tiled256-rgb8-jpeg", + raw_bytes: 63_078_912, + }, + CorpusBench { + file: "photo-gray-f32-deflate-fpred-tiled512.tif", + name: "tiled512-gray-f32-deflate-fpred", + raw_bytes: 84_105_216, + }, + ]; + + { + let mut group = c.benchmark_group("photo-21mp"); + group.sample_size(10); + + for bench in &photo_benches { + if let Some(data) = load_corpus(&bd, bench.file) { + group.throughput(Throughput::Bytes(bench.raw_bytes)); + let data_ref = &data; + group.bench_function(bench.name, |b| b.iter(|| decode_tiff_bytes(data_ref))); + } + } + + group.finish(); + } + + // ======================================================================== + // PART 2: GeoTIFF (real tiled DEM data) + // ======================================================================== + + let geotiff_benches = [ + // Copernicus DEM: 3600x3600, tiled 1024x1024, Gray f32, deflate + fpredict + CorpusBench { + file: "Copernicus_DSM_COG_10_N40_00_W075_00_DEM.tif", + name: "copernicus-dem-f32-deflate-fpred-1024tile", + raw_bytes: 3600 * 3600 * 4, + }, + // USGS DEM: 3612x3612, tiled 512x512, Gray f32, LZW + fpredict + CorpusBench { + file: "USGS_1_n40w075.tif", + name: "usgs-dem-f32-lzw-fpred-512tile", + raw_bytes: 3612 * 3612 * 4, + }, + ]; + + { + let mut group = c.benchmark_group("geotiff"); + group.sample_size(10); + + for bench in &geotiff_benches { + if let Some(data) = load_corpus(&gd, bench.file) { + group.throughput(Throughput::Bytes(bench.raw_bytes)); + let data_ref = &data; + group.bench_function(bench.name, |b| b.iter(|| decode_tiff_bytes(data_ref))); + } + } + + group.finish(); + } + + // ======================================================================== + // PART 3: Synthetic (2048x2048, controlled comparisons) + // ======================================================================== + + let rgb8 = generate_rgb8(WIDTH, HEIGHT); + let rgba8 = generate_rgba8(WIDTH, HEIGHT); + let gray8 = generate_gray8(WIDTH, HEIGHT); + let rgb16 = generate_rgb16(WIDTH, HEIGHT); + let gray16 = generate_gray16(WIDTH, HEIGHT); + let rgb_f32 = generate_rgb_f32(WIDTH, HEIGHT); + + let raw_rgb8 = (WIDTH * HEIGHT * 3) as u64; + let raw_rgba8 = (WIDTH * HEIGHT * 4) as u64; + let raw_gray8 = (WIDTH * HEIGHT) as u64; + let raw_rgb16 = (WIDTH * HEIGHT * 6) as u64; + let raw_gray16 = (WIDTH * HEIGHT * 2) as u64; + let raw_f32x3 = (WIDTH * HEIGHT * 12) as u64; + + // --- Uncompressed baseline --- + { + let mut group = c.benchmark_group("synth-uncompressed"); + group.sample_size(10); + + let t = encode::( + &rgb8, + WIDTH, + HEIGHT, + Compression::Uncompressed, + Predictor::None, + ); + group + .throughput(Throughput::Bytes(raw_rgb8)) + .bench_function("rgb8", |b| b.iter(|| decode_tiff_bytes(&t))); + + let t = encode::( + &rgba8, + WIDTH, + HEIGHT, + Compression::Uncompressed, + Predictor::None, + ); + group + .throughput(Throughput::Bytes(raw_rgba8)) + .bench_function("rgba8", |b| b.iter(|| decode_tiff_bytes(&t))); + + let t = encode::( + &gray8, + WIDTH, + HEIGHT, + Compression::Uncompressed, + Predictor::None, + ); + group + .throughput(Throughput::Bytes(raw_gray8)) + .bench_function("gray8", |b| b.iter(|| decode_tiff_bytes(&t))); + + let t = encode::( + &rgb16, + WIDTH, + HEIGHT, + Compression::Uncompressed, + Predictor::None, + ); + group + .throughput(Throughput::Bytes(raw_rgb16)) + .bench_function("rgb16", |b| b.iter(|| decode_tiff_bytes(&t))); + + let t = encode::( + &gray16, + WIDTH, + HEIGHT, + Compression::Uncompressed, + Predictor::None, + ); + group + .throughput(Throughput::Bytes(raw_gray16)) + .bench_function("gray16", |b| b.iter(|| decode_tiff_bytes(&t))); + + group.finish(); + } + + // --- Predictor overhead isolation (deflate-1, with vs without predictor) --- + { + let mut group = c.benchmark_group("synth-predictor"); + group.sample_size(10); + + let t = encode::( + &rgb8, + WIDTH, + HEIGHT, + Compression::Deflate(1), + Predictor::None, + ); + group + .throughput(Throughput::Bytes(raw_rgb8)) + .bench_function("rgb8-deflate-none", |b| b.iter(|| decode_tiff_bytes(&t))); + + let t = encode::( + &rgb8, + WIDTH, + HEIGHT, + Compression::Deflate(1), + Predictor::Horizontal, + ); + group + .throughput(Throughput::Bytes(raw_rgb8)) + .bench_function("rgb8-deflate-hpred", |b| b.iter(|| decode_tiff_bytes(&t))); + + let t = encode::( + &rgb16, + WIDTH, + HEIGHT, + Compression::Deflate(1), + Predictor::None, + ); + group + .throughput(Throughput::Bytes(raw_rgb16)) + .bench_function("rgb16-deflate-none", |b| b.iter(|| decode_tiff_bytes(&t))); + + let t = encode::( + &rgb16, + WIDTH, + HEIGHT, + Compression::Deflate(1), + Predictor::Horizontal, + ); + group + .throughput(Throughput::Bytes(raw_rgb16)) + .bench_function("rgb16-deflate-hpred", |b| b.iter(|| decode_tiff_bytes(&t))); + + let t = encode::( + &rgb_f32, + WIDTH, + HEIGHT, + Compression::Deflate(1), + Predictor::None, + ); + group + .throughput(Throughput::Bytes(raw_f32x3)) + .bench_function("rgb-f32-deflate-none", |b| b.iter(|| decode_tiff_bytes(&t))); + + let t = encode::( + &rgb_f32, + WIDTH, + HEIGHT, + Compression::Deflate(1), + Predictor::FloatingPoint, + ); + group + .throughput(Throughput::Bytes(raw_f32x3)) + .bench_function("rgb-f32-deflate-fpred", |b| { + b.iter(|| decode_tiff_bytes(&t)) + }); + + group.finish(); + } + + // --- LZW vs Deflate comparison --- + { + let mut group = c.benchmark_group("synth-compression"); + group.sample_size(10); + + let t = encode::( + &rgb8, + WIDTH, + HEIGHT, + Compression::Deflate(1), + Predictor::Horizontal, + ); + group + .throughput(Throughput::Bytes(raw_rgb8)) + .bench_function("rgb8-deflate-hpred", |b| b.iter(|| decode_tiff_bytes(&t))); + + let t = encode::( + &rgb8, + WIDTH, + HEIGHT, + Compression::Lzw, + Predictor::Horizontal, + ); + group + .throughput(Throughput::Bytes(raw_rgb8)) + .bench_function("rgb8-lzw-hpred", |b| b.iter(|| decode_tiff_bytes(&t))); + + let t = encode::(&rgb8, WIDTH, HEIGHT, Compression::Lzw, Predictor::None); + group + .throughput(Throughput::Bytes(raw_rgb8)) + .bench_function("rgb8-lzw-none", |b| b.iter(|| decode_tiff_bytes(&t))); + + let t = encode::(&rgb8, WIDTH, HEIGHT, Compression::Packbits, Predictor::None); + group + .throughput(Throughput::Bytes(raw_rgb8)) + .bench_function("rgb8-packbits", |b| b.iter(|| decode_tiff_bytes(&t))); + + group.finish(); + } +} diff --git a/examples/profile_decode.rs b/examples/profile_decode.rs new file mode 100644 index 0000000..8a483ac --- /dev/null +++ b/examples/profile_decode.rs @@ -0,0 +1,130 @@ +//! Profiling helper: encodes and decodes several TIFF configurations. +//! Run under callgrind: `valgrind --tool=callgrind target/release/examples/profile_decode` +use std::io::Cursor; +use tiff::decoder::Decoder; +use tiff::encoder::colortype::{Gray8, RGB32Float, RGB16, RGB8}; +use tiff::encoder::{Compression, Predictor, TiffEncoder}; + +const WIDTH: u32 = 2048; +const HEIGHT: u32 = 2048; +const ITERS: usize = 3; + +fn generate_rgb8(w: u32, h: u32) -> Vec { + (0..w * h * 3).map(|i| (i % 256) as u8).collect() +} + +fn generate_rgb16(w: u32, h: u32) -> Vec { + (0..w * h * 3).map(|i| (i % 65536) as u16).collect() +} + +fn generate_rgb_f32(w: u32, h: u32) -> Vec { + (0..w * h * 3) + .map(|i| (i as f32) / (w * h * 3) as f32) + .collect() +} + +fn generate_gray8(w: u32, h: u32) -> Vec { + (0..w * h).map(|i| (i % 256) as u8).collect() +} + +fn encode( + data: &[C::Inner], + w: u32, + h: u32, + compression: Compression, + predictor: Predictor, +) -> Vec +where + [C::Inner]: tiff::encoder::TiffValue, +{ + let mut buf = Cursor::new(Vec::new()); + TiffEncoder::new(&mut buf) + .unwrap() + .with_predictor(predictor) + .with_compression(compression) + .write_image::(w, h, data) + .unwrap(); + buf.into_inner() +} + +fn decode(data: &[u8]) { + let cursor = Cursor::new(data); + let mut decoder = Decoder::open(cursor).unwrap(); + decoder.next_image().unwrap(); + let _ = decoder.read_image().unwrap(); +} + +fn main() { + let rgb8 = generate_rgb8(WIDTH, HEIGHT); + let rgb16 = generate_rgb16(WIDTH, HEIGHT); + let rgb_f32 = generate_rgb_f32(WIDTH, HEIGHT); + let gray8 = generate_gray8(WIDTH, HEIGHT); + + let cases: Vec<(&str, Vec)> = vec![ + ( + "deflate-rgb8-hpredict", + encode::( + &rgb8, + WIDTH, + HEIGHT, + Compression::Deflate(1), + Predictor::Horizontal, + ), + ), + ( + "deflate-gray8-hpredict", + encode::( + &gray8, + WIDTH, + HEIGHT, + Compression::Deflate(1), + Predictor::Horizontal, + ), + ), + ( + "deflate-rgb16-hpredict", + encode::( + &rgb16, + WIDTH, + HEIGHT, + Compression::Deflate(1), + Predictor::Horizontal, + ), + ), + ( + "deflate-rgb-f32-fpredict", + encode::( + &rgb_f32, + WIDTH, + HEIGHT, + Compression::Deflate(1), + Predictor::FloatingPoint, + ), + ), + ( + "lzw-rgb8-no-pred", + encode::(&rgb8, WIDTH, HEIGHT, Compression::Lzw, Predictor::None), + ), + ( + "lzw-rgb8-hpredict", + encode::( + &rgb8, + WIDTH, + HEIGHT, + Compression::Lzw, + Predictor::Horizontal, + ), + ), + ]; + + for (name, tiff_data) in &cases { + eprintln!( + "=== Profiling: {} ({} bytes encoded) ===", + name, + tiff_data.len() + ); + for _ in 0..ITERS { + decode(tiff_data); + } + } +} From fa8a911f792486e6c242e679826f9285cfed877b Mon Sep 17 00:00:00 2001 From: Lilith River Date: Sun, 5 Apr 2026 22:00:22 -0600 Subject: [PATCH 2/2] perf: optimize horizontal and float predictor decode Horizontal predictor (one_byte_predict, all integer TIFFs with Predictor=2): rewrite with explicit [u8; N] accumulator so LLVM unrolls and keeps each channel in a register. For N=3 (RGB8), this replaces a scalar 1-byte-per-iteration loop with a 2x-unrolled 3-register loop. Float predictor (Predictor=3, f16/f32/f64): split deinterleave into per-format functions with plane slicing up front; f32 batches 4 samples through a u128 wide store to eliminate per-sample bounds checks. Thread a reusable scratch buffer through expand_chunk to eliminate per-row heap allocation. Benchmarked on real 21MP photos and GeoTIFF DEMs (vs upstream main): Strip RGB8 deflate+hpred: 253ms -> 224ms (-11%) Strip RGB8 LZW: 405ms -> 370ms (-9%) Tiled RGB8 deflate+hpred: 241ms -> 222ms (-8%) Strip RGB f32 deflate+fpred: 726ms -> 527ms (-27%) Gray f32 tiled 512: 290ms -> 166ms (-43%) Copernicus DEM GeoTIFF: 239ms -> 152ms (-37%) USGS DEM GeoTIFF: 284ms -> 198ms (-30%) --- src/decoder/image.rs | 5 + src/decoder/mod.rs | 250 +++++++++++++++++++++++++++++++++---------- 2 files changed, 197 insertions(+), 58 deletions(-) diff --git a/src/decoder/image.rs b/src/decoder/image.rs index 3a63750..6815f91 100644 --- a/src/decoder/image.rs +++ b/src/decoder/image.rs @@ -1009,6 +1009,7 @@ impl Image { buf: &mut [u8], layout: &ReadoutLayout, chunk_index: u32, + scratch: &mut Vec, ) -> TiffResult<()> { let ValueReader { reader, @@ -1191,6 +1192,7 @@ impl Image { samples, ByteOrder::native(), predictor, + scratch, ); if photometric_interpretation == PhotometricInterpretation::WhiteIsZero { @@ -1209,6 +1211,7 @@ impl Image { samples, decompressed_byte_order, predictor, + scratch, ); } @@ -1255,6 +1258,7 @@ impl Image { samples, decompressed_byte_order, predictor, + scratch, ); if photometric_interpretation == PhotometricInterpretation::WhiteIsZero { @@ -1296,6 +1300,7 @@ impl Image { samples, decompressed_byte_order, predictor, + scratch, ); if photometric_interpretation == PhotometricInterpretation::WhiteIsZero { diff --git a/src/decoder/mod.rs b/src/decoder/mod.rs index bd05d48..6b56c59 100644 --- a/src/decoder/mod.rs +++ b/src/decoder/mod.rs @@ -254,6 +254,9 @@ pub struct Decoder { /// Map from the ifd into the `ifd_offsets` ordered list. seen_ifds: cycles::IfdCycles, image: Option, + /// Reusable scratch buffer for the floating-point predictor. Stored here so that the + /// allocation persists across strips/tiles and images, avoiding per-row heap allocation. + scratch: Vec, } /// All the information needed to read and interpret byte slices from the underlying file, i.e. to @@ -273,13 +276,37 @@ pub struct IfdDecoder<'lt> { inner: tag_reader::TagReader<'lt, dyn tag_reader::EntryDecoder + 'lt>, } -fn rev_hpredict_nsamp(buf: &mut [u8], bit_depth: u8, samples: u16) { - fn one_byte_predict(buf: &mut [u8]) { - for i in N..buf.len() { - buf[i] = buf[i].wrapping_add(buf[i - N]); +/// Byte-level prefix sum with compile-time stride. Each element is the wrapping sum of itself and +/// the element N positions earlier. For N >= 2, the N interleaved channels are independent streams. +/// +/// For N=3 (RGB) and N=4 (RGBA), we use an explicit `[u8; N]` accumulator so LLVM sees each +/// channel as an independent scalar chain and can unroll effectively. Without this, LLVM emits a +/// scalar byte-at-a-time loop for N=3 because 3 doesn't map cleanly to SIMD widths. +fn one_byte_predict(buf: &mut [u8]) { + if N == 0 || buf.len() <= N { + return; + } + + // Copy the first N bytes as the initial accumulator state. + let mut acc = [0u8; N]; + acc.copy_from_slice(&buf[..N]); + + // Process N bytes per iteration. Each channel's prefix sum is independent. + let mut chunks = buf[N..].chunks_exact_mut(N); + for chunk in &mut chunks { + for c in 0..N { + acc[c] = chunk[c].wrapping_add(acc[c]); + chunk[c] = acc[c]; } } + // Handle trailing bytes (when buf.len() - N is not a multiple of N). + for (c, byte) in chunks.into_remainder().iter_mut().enumerate() { + *byte = byte.wrapping_add(acc[c]); + } +} + +fn rev_hpredict_nsamp(buf: &mut [u8], bit_depth: u8, samples: u16) { fn two_bytes_predict(buf: &mut [u8]) { for i in (2 * N..buf.len()).step_by(2) { let v = u16::from_ne_bytes(buf[i..][..2].try_into().unwrap()); @@ -364,65 +391,150 @@ fn rev_hpredict_nsamp(buf: &mut [u8], bit_depth: u8, samples: u16) { } } +/// Undo the floating-point predictor: byte-level prefix sum, then de-interleave byte planes back +/// into native-endian samples. +/// +/// The TIFF float predictor splits each N-byte sample into N separate byte planes within the row, +/// then delta-encodes each plane. Decoding reverses this: prefix-sum to undo the deltas, then +/// transpose the N planes back into packed samples. fn predict_f32(input: &mut [u8], output: &mut [u8], samples: u16) { - let samples = usize::from(samples); - - for i in samples..input.len() { - input[i] = input[i].wrapping_add(input[i - samples]); - } - - for (i, chunk) in output.chunks_mut(4).enumerate() { - chunk.copy_from_slice(&u32::to_ne_bytes(u32::from_be_bytes([ - input[i], - input[input.len() / 4 + i], - input[input.len() / 4 * 2 + i], - input[input.len() / 4 * 3 + i], - ]))); - } + one_byte_predict_generic(input, usize::from(samples)); + deinterleave_f32_planes(input, output); } fn predict_f16(input: &mut [u8], output: &mut [u8], samples: u16) { - let samples = usize::from(samples); - - for i in samples..input.len() { - input[i] = input[i].wrapping_add(input[i - samples]); - } - - for (i, chunk) in output.chunks_mut(2).enumerate() { - chunk.copy_from_slice(&u16::to_ne_bytes(u16::from_be_bytes([ - input[i], - input[input.len() / 2 + i], - ]))); - } + one_byte_predict_generic(input, usize::from(samples)); + deinterleave_f16_planes(input, output); } fn predict_f64(input: &mut [u8], output: &mut [u8], samples: u16) { - let samples = usize::from(samples); + one_byte_predict_generic(input, usize::from(samples)); + deinterleave_f64_planes(input, output); +} - for i in samples..input.len() { - input[i] = input[i].wrapping_add(input[i - samples]); - } - - for (i, chunk) in output.chunks_mut(8).enumerate() { - chunk.copy_from_slice(&u64::to_ne_bytes(u64::from_be_bytes([ - input[i], - input[input.len() / 8 + i], - input[input.len() / 8 * 2 + i], - input[input.len() / 8 * 3 + i], - input[input.len() / 8 * 4 + i], - input[input.len() / 8 * 5 + i], - input[input.len() / 8 * 6 + i], - input[input.len() / 8 * 7 + i], - ]))); +/// Byte-level prefix sum: `buf[i] += buf[i - stride]` for all `i >= stride`. +/// +/// Same operation as `one_byte_predict` but dispatched at runtime. The most common float +/// channel counts (1, 3, 4) get dedicated loops so LLVM can see the independent channel streams +/// and vectorize each one separately. +fn one_byte_predict_generic(buf: &mut [u8], stride: usize) { + match stride { + 1 => one_byte_predict::<1>(buf), + 2 => one_byte_predict::<2>(buf), + 3 => one_byte_predict::<3>(buf), + 4 => one_byte_predict::<4>(buf), + _ => { + for i in stride..buf.len() { + buf[i] = buf[i].wrapping_add(buf[i - stride]); + } + } } } +/// Transpose byte-planes back into packed f16/f32/f64 samples with big-endian to native conversion. +/// +/// Input layout: `[plane0: n bytes][plane1: n bytes]...[planeP: n bytes]` where each plane holds +/// one byte position of each sample. Output: packed native-endian samples. +/// +/// Each function splits the input into equal-sized plane slices, then gathers one byte per plane +/// per sample. Using `from_be_bytes` for endian conversion lets LLVM recognise the byte-swap idiom +/// and emit efficient code (bswap or movbe). +fn deinterleave_f32_planes(input: &[u8], output: &mut [u8]) { + let n = output.len() / 4; + if n == 0 || input.len() < n * 4 { + return; + } + let (p0, rest) = input.split_at(n); + let (p1, rest) = rest.split_at(n); + let (p2, p3) = rest.split_at(n); + + // Batch 4 samples (16 output bytes) per iteration. Fixed-size array slicing eliminates + // bounds checks in the inner loop. + let (out_chunks, out_rest) = output.as_chunks_mut::<16>(); + let (p0_chunks, p0_rest) = p0.as_chunks::<4>(); + let (p1_chunks, p1_rest) = p1.as_chunks::<4>(); + let (p2_chunks, p2_rest) = p2.as_chunks::<4>(); + let (p3_chunks, p3_rest) = p3.as_chunks::<4>(); + + for ((((out, b0), b1), b2), b3) in out_chunks + .iter_mut() + .zip(p0_chunks) + .zip(p1_chunks) + .zip(p2_chunks) + .zip(p3_chunks) + { + let s0 = u32::from_be_bytes([b0[0], b1[0], b2[0], b3[0]]).to_ne_bytes(); + let s1 = u32::from_be_bytes([b0[1], b1[1], b2[1], b3[1]]).to_ne_bytes(); + let s2 = u32::from_be_bytes([b0[2], b1[2], b2[2], b3[2]]).to_ne_bytes(); + let s3 = u32::from_be_bytes([b0[3], b1[3], b2[3], b3[3]]).to_ne_bytes(); + *out = [ + s0[0], s0[1], s0[2], s0[3], s1[0], s1[1], s1[2], s1[3], s2[0], s2[1], s2[2], s2[3], + s3[0], s3[1], s3[2], s3[3], + ]; + } + + // Remaining 0–3 samples. + for (j, out) in out_rest.chunks_exact_mut(4).enumerate() { + out.copy_from_slice( + &u32::from_be_bytes([p0_rest[j], p1_rest[j], p2_rest[j], p3_rest[j]]).to_ne_bytes(), + ); + } +} + +fn deinterleave_f16_planes(input: &[u8], output: &mut [u8]) { + let n = output.len() / 2; + if n == 0 || input.len() < n * 2 { + return; + } + let (p0, p1) = input.split_at(n); + + for (out, (b0, b1)) in output.chunks_exact_mut(2).zip(p0.iter().zip(p1)) { + out.copy_from_slice(&u16::from_be_bytes([*b0, *b1]).to_ne_bytes()); + } +} + +fn deinterleave_f64_planes(input: &[u8], output: &mut [u8]) { + let n = output.len() / 8; + if n == 0 || input.len() < n * 8 { + return; + } + let (p0, rest) = input.split_at(n); + let (p1, rest) = rest.split_at(n); + let (p2, rest) = rest.split_at(n); + let (p3, rest) = rest.split_at(n); + let (p4, rest) = rest.split_at(n); + let (p5, rest) = rest.split_at(n); + let (p6, p7) = rest.split_at(n); + + // Zip all 8 planes with the output to avoid indexed access and bounds checks. + let iter = output.chunks_exact_mut(8).zip( + p0.iter() + .zip(p1) + .zip(p2) + .zip(p3) + .zip(p4) + .zip(p5) + .zip(p6) + .zip(p7), + ); + for (out, (((((((b0, b1), b2), b3), b4), b5), b6), b7)) in iter { + out.copy_from_slice( + &u64::from_be_bytes([*b0, *b1, *b2, *b3, *b4, *b5, *b6, *b7]).to_ne_bytes(), + ); + } +} + +/// Fix byte order and apply the inverse predictor for one row of decoded data. +/// +/// `scratch` is a reusable buffer for the floating-point predictor (which needs a working copy of +/// the input). Callers should pass the same buffer across rows to avoid per-row heap allocation. fn fix_endianness_and_predict( buf: &mut [u8], bit_depth: u8, samples: u16, byte_order: ByteOrder, predictor: Predictor, + scratch: &mut Vec, ) { match predictor { Predictor::None => { @@ -433,11 +545,14 @@ fn fix_endianness_and_predict( rev_hpredict_nsamp(buf, bit_depth, samples); } Predictor::FloatingPoint => { - let mut buffer_copy = buf.to_vec(); + // The float predictor reads from scattered byte planes, so it needs a separate input + // buffer. + scratch.resize(buf.len(), 0); + scratch.copy_from_slice(buf); match bit_depth { - 16 => predict_f16(&mut buffer_copy, buf, samples), - 32 => predict_f32(&mut buffer_copy, buf, samples), - 64 => predict_f64(&mut buffer_copy, buf, samples), + 16 => predict_f16(scratch, buf, samples), + 32 => predict_f32(scratch, buf, samples), + 64 => predict_f64(scratch, buf, samples), _ => unreachable!("Caller should have validated arguments. Please file a bug."), } } @@ -627,6 +742,7 @@ impl TiffHeader { }, seen_ifds: cycles::IfdCycles::new(), image: None, + scratch: Vec::new(), } } } @@ -712,13 +828,23 @@ impl Decoder { // FIXME: this tuple would be a good candidate for a proxy, like IfdDecoder. That has been // requested in some form or another in a number of issues. #[expect(clippy::unnecessary_unwrap)] // Lifetimes would overlap due to return. - fn ensure_image_and_reader(&mut self) -> TiffResult<(&mut Image, &mut ValueReader)> { + fn ensure_image_and_reader( + &mut self, + ) -> TiffResult<(&mut Image, &mut ValueReader, &mut Vec)> { if self.image.is_some() { - return Ok((self.image.as_mut().unwrap(), &mut self.value_reader)); + return Ok(( + self.image.as_mut().unwrap(), + &mut self.value_reader, + &mut self.scratch, + )); } let image = Image::from_reader(&mut self.value_reader, &self.directory)?; - Ok((self.image.insert(image), &mut self.value_reader)) + Ok(( + self.image.insert(image), + &mut self.value_reader, + &mut self.scratch, + )) } fn ensure_image(&mut self) -> TiffResult<&mut Image> { @@ -848,6 +974,7 @@ impl Decoder { directory: self.directory.clone(), current_ifd_pointer: self.current_ifd_pointer, image: core::mem::take(&mut self.image), + scratch: core::mem::take(&mut self.scratch), } } @@ -1323,10 +1450,10 @@ impl Decoder { chunk_index: u32, layout: &image::ReadoutLayout, ) -> TiffResult<()> { - let (image, value_reader) = self.ensure_image_and_reader()?; + let (image, value_reader, scratch) = self.ensure_image_and_reader()?; let offset = image.chunk_file_range(chunk_index)?.0; value_reader.reader.goto_offset(offset)?; - image.expand_chunk(value_reader, buffer, layout, chunk_index)?; + image.expand_chunk(value_reader, buffer, layout, chunk_index, scratch)?; Ok(()) } @@ -1478,7 +1605,7 @@ impl Decoder { slice: TiffCodingUnit, buffer: &mut [u8], ) -> TiffResult<()> { - let (image, value_reader) = self.ensure_image_and_reader()?; + let (image, value_reader, scratch) = self.ensure_image_and_reader()?; let (width, height) = image.chunk_data_dimensions(slice.0)?; let readout = image.readout_for_size(width, height)?; @@ -1505,7 +1632,13 @@ impl Decoder { let chunk = slice.0 + idx as u32 * readout.chunks_per_plane; let offset = image.chunk_offsets[chunk as usize]; value_reader.reader.goto_offset(offset)?; - image.expand_chunk(value_reader, &mut buffer[plane_offset..], readout, chunk)?; + image.expand_chunk( + value_reader, + &mut buffer[plane_offset..], + readout, + chunk, + scratch, + )?; } Ok(()) @@ -1676,7 +1809,7 @@ impl Decoder { /// Returns an error if the buffer fits less than one plane. In particular, for non-planar /// images returns an error if the buffer does not fit the required size. pub fn read_image_bytes(&mut self, buffer: &mut [u8]) -> TiffResult<()> { - let (image, value_reader) = self.ensure_image_and_reader()?; + let (image, value_reader, scratch) = self.ensure_image_and_reader()?; let readout = image.readout_for_image()?; let ref layout @ image::PlaneLayout { @@ -1709,6 +1842,7 @@ impl Decoder { &mut buffer[plane_offset..][buffer_offset..], readout, chunk, + scratch, )?; } }