Build Directory FromIterator

This enables a test for the encoder's DirectoryOffset but it seems to
fit nicely with the current design, too.

CHANGES.md

@@ -1,3 +1,7 @@
+# Version 0.10.4 (unreleased)
+
+- `Directory` now implements `FromIterator<(Tag, Value)>`.
+
 # Version 0.10.3
 
 New features:

src/decoder/ifd.rs

@@ -332,6 +332,12 @@ impl Value {
     }
 }
 
+/// A combination of type, count, and offset.
+///
+/// In a TIFF the data offset portion of an entry is used for inline data in case the length of the
+/// encoded value does not exceed the size of the offset field. Since the size of the offset field
+/// depends on the file kind (4 bytes for standard TIFF, 8 bytes for BigTIFF) the interpretation of
+/// this struct is only complete in combination with file metadata.
 #[derive(Clone)]
 pub struct Entry {
     type_: Type,
@@ -349,12 +355,14 @@ impl ::std::fmt::Debug for Entry {
 }
 
 impl Entry {
+    /// Create a new entry fit to be added to a standard TIFF IFD.
     pub fn new(type_: Type, count: u32, offset: [u8; 4]) -> Entry {
         let mut entry_off = [0u8; 8];
         entry_off[..4].copy_from_slice(&offset);
         Entry::new_u64(type_, count.into(), entry_off)
     }
 
+    /// Create a new entry with data for a Big TIFF IFD.
     pub fn new_u64(type_: Type, count: u64, offset: [u8; 8]) -> Entry {
         Entry {
             type_,

src/decoder/mod.rs

@@ -368,27 +368,82 @@ pub struct IfdDecoder<'lt> {
 }
 
 fn rev_hpredict_nsamp(buf: &mut [u8], bit_depth: u8, samples: usize) {
-    match bit_depth {
+    fn one_byte_predict<const N: usize>(buf: &mut [u8]) {
-        0..=8 => {
+        for i in N..buf.len() {
+            buf[i] = buf[i].wrapping_add(buf[i - N]);
+        }
+    }
+
+    fn two_bytes_predict<const N: usize>(buf: &mut [u8]) {
+        for i in (2 * N..buf.len()).step_by(2) {
+            let v = u16::from_ne_bytes(buf[i..][..2].try_into().unwrap());
+            let p = u16::from_ne_bytes(buf[i - 2 * N..][..2].try_into().unwrap());
+            buf[i..][..2].copy_from_slice(&(v.wrapping_add(p)).to_ne_bytes());
+        }
+    }
+
+    fn four_bytes_predict<const N: usize>(buf: &mut [u8]) {
+        for i in (N * 4..buf.len()).step_by(4) {
+            let v = u32::from_ne_bytes(buf[i..][..4].try_into().unwrap());
+            let p = u32::from_ne_bytes(buf[i - 4 * N..][..4].try_into().unwrap());
+            buf[i..][..4].copy_from_slice(&(v.wrapping_add(p)).to_ne_bytes());
+        }
+    }
+
+    match (bit_depth, samples) {
+        // Note we can't use `windows` or so due to the overlap between each iteration. We split
+        // the cases by the samples / lookback constant so that each is optimized individually.
+        // This is more code generated but each loop can then have a different vectorization
+        // strategy.
+        (0..=8, 1) => one_byte_predict::<1>(buf),
+        (0..=8, 2) => one_byte_predict::<2>(buf),
+        (0..=8, 3) => one_byte_predict::<3>(buf),
+        (0..=8, 4) => one_byte_predict::<4>(buf),
+        // The generic, sub-optimal case for the above.
+        (0..=8, _) => {
             for i in samples..buf.len() {
                 buf[i] = buf[i].wrapping_add(buf[i - samples]);
             }
         }
-        9..=16 => {
+        (9..=16, 1) => {
+            two_bytes_predict::<1>(buf);
+        }
+        (9..=16, 2) => {
+            two_bytes_predict::<2>(buf);
+        }
+        (9..=16, 3) => {
+            two_bytes_predict::<3>(buf);
+        }
+        (9..=16, 4) => {
+            two_bytes_predict::<4>(buf);
+        }
+        (9..=16, _) => {
             for i in (samples * 2..buf.len()).step_by(2) {
                 let v = u16::from_ne_bytes(buf[i..][..2].try_into().unwrap());
                 let p = u16::from_ne_bytes(buf[i - 2 * samples..][..2].try_into().unwrap());
                 buf[i..][..2].copy_from_slice(&(v.wrapping_add(p)).to_ne_bytes());
             }
         }
-        17..=32 => {
+        (17..=32, 1) => {
+            four_bytes_predict::<1>(buf);
+        }
+        (17..=32, 2) => {
+            four_bytes_predict::<2>(buf);
+        }
+        (17..=32, 3) => {
+            four_bytes_predict::<3>(buf);
+        }
+        (17..=32, 4) => {
+            four_bytes_predict::<4>(buf);
+        }
+        (17..=32, _) => {
             for i in (samples * 4..buf.len()).step_by(4) {
                 let v = u32::from_ne_bytes(buf[i..][..4].try_into().unwrap());
                 let p = u32::from_ne_bytes(buf[i - 4 * samples..][..4].try_into().unwrap());
                 buf[i..][..4].copy_from_slice(&(v.wrapping_add(p)).to_ne_bytes());
             }
         }
-        33..=64 => {
+        (33..=64, _) => {
             for i in (samples * 8..buf.len()).step_by(8) {
                 let v = u64::from_ne_bytes(buf[i..][..8].try_into().unwrap());
                 let p = u64::from_ne_bytes(buf[i - 8 * samples..][..8].try_into().unwrap());

src/decoder/stream.rs

@@ -168,9 +168,6 @@ impl<R: Read> Read for LZWReader<R> {
                     }
                 }
                 Ok(weezl::LzwStatus::NoProgress) => {
-                    assert_eq!(result.consumed_in, 0);
-                    assert_eq!(result.consumed_out, 0);
-                    assert!(self.reader.buffer().is_empty());
                     return Err(io::Error::new(
                         io::ErrorKind::UnexpectedEof,
                         "no lzw end code found",

src/directory.rs

@@ -104,6 +104,28 @@ impl Directory {
             }
         }
     }
+
+    pub(crate) fn encoded_len<K: crate::encoder::TiffKind>(&self) -> u64 {
+        let (len_count, len_offset);
+
+        match std::mem::size_of::<K::OffsetType>() {
+            offset @ 4 => {
+                // non-BigTiff uses a `u16` here.
+                len_count = 2;
+                len_offset = offset as u64;
+            }
+            offset => {
+                let offset = offset as u64;
+                // assuming BigTIFF
+                len_count = offset;
+                len_offset = offset;
+            }
+        }
+
+        // Each entry is 12 bytes in classic TIFF, 20 bytes in BigTIFF.
+        let entry_bytes = 4 + 2 * len_offset;
+        len_count + entry_bytes * self.entries.len() as u64 + len_offset
+    }
 }
 
 impl fmt::Debug for Directory {
@@ -118,6 +140,14 @@ impl fmt::Debug for Directory {
     }
 }
 
+impl core::iter::FromIterator<(Tag, Entry)> for Directory {
+    fn from_iter<T: IntoIterator<Item = (Tag, Entry)>>(iter: T) -> Self {
+        let mut dir = Directory::empty();
+        dir.extend(iter);
+        dir
+    }
+}
+
 #[cfg(test)]
 mod tests {
     use super::Directory;

src/encoder/compression/uncompressed.rs

@@ -14,7 +14,7 @@ impl Compression for Uncompressed {
 
 impl CompressionAlgorithm for Uncompressed {
     fn write_to<W: Write>(&mut self, writer: &mut W, bytes: &[u8]) -> Result<u64, io::Error> {
-        writer.write(bytes).map(|byte_count| byte_count as u64)
+        writer.write_all(bytes).map(|()| bytes.len() as u64)
     }
 }
 

src/encoder/mod.rs

@@ -74,10 +74,10 @@ impl Compression {
 
 /// Encoder for Tiff and BigTiff files.
 ///
-/// With this type you can get a `DirectoryEncoder` or a `ImageEncoder`
+/// With this type you can get a [`DirectoryEncoder`] or a [`ImageEncoder`]
 /// to encode Tiff/BigTiff ifd directories with images.
 ///
-/// See `DirectoryEncoder` and `ImageEncoder`.
+/// See [`DirectoryEncoder`] and [`ImageEncoder`].
 ///
 /// # Examples
 /// ```
@@ -171,7 +171,7 @@ impl<W: Write + Seek, K: TiffKind> TiffEncoder<W, K> {
         Self::chain_directory(&mut self.writer, &mut self.last_ifd_chain)
     }
 
-    /// Create a [`DirectoryEncoder`] to encode an ifd directory.
+    /// Create a [`DirectoryEncoder`] to encode an ifd (directory).
     ///
     /// The caller is responsible for ensuring that the directory is a valid image in the main TIFF
     /// IFD sequence. To encode additional directories that are not linked into the sequence, use
@@ -180,7 +180,7 @@ impl<W: Write + Seek, K: TiffKind> TiffEncoder<W, K> {
         Self::chain_directory(&mut self.writer, &mut self.last_ifd_chain)
     }
 
-    /// Create a [`DirectoryEncoder`] to encode an ifd directory.
+    /// Create a [`DirectoryEncoder`] to encode a non-image directory.
     ///
     /// The directory is not linked into the sequence of directories. For instance, encode Exif
     /// directories or SubIfd directories with this method.
@@ -244,6 +244,12 @@ pub struct DirectoryEncoder<'a, W: 'a + Write + Seek, K: TiffKind> {
 }
 
 /// The offset of an encoded directory in the file.
+///
+/// Both the `offset` and `pointer` field are a complete description of the start offset of the
+/// dictionary which is used to point to it. The struct also describes the extent covered by the
+/// encoding of the directory. The constructors [`new`][DirectoryOffset::new] allow the checked
+/// conversion. Note that even though the offsets are public do not expect their information to
+/// propagate when modified.
 #[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
 pub struct DirectoryOffset<K: TiffKind> {
     /// The start of the directory as a Tiff value.
@@ -255,7 +261,9 @@ pub struct DirectoryOffset<K: TiffKind> {
     pub offset: K::OffsetType,
     /// The start of the directory as a pure offset.
     pub pointer: IfdPointer,
-    /// The offset of its sequence link field, in our private representation.
+    /// The offset of its sequence link field, in our private representation. Before exposing this
+    /// make sure that we don't want any invariants (e.g. pointer being smaller than this). The
+    /// type should be fine.
     ifd_chain: NonZeroU64,
     /// The kind of Tiff file the offset is for.
     kind: PhantomData<K>,
@@ -313,6 +321,17 @@ impl<'a, W: 'a + Write + Seek, K: TiffKind> DirectoryEncoder<'a, W, K> {
         Ok(offset)
     }
 
+    /// Insert previously written key-value entries.
+    ///
+    /// It is the caller's responsibility to ensure that the data referred to by the entries is
+    /// actually (or will be) written to the file. Note that small values are necessarily inline in
+    /// the entry, the size limit depends on the Tiff kind. So do not confuse a directory from a
+    /// BigTiff in a standard tiff or the other way around.
+    pub fn extend_from(&mut self, dir: &Directory) {
+        let entries = dir.iter().map(|(tag, val)| (tag, val.clone()));
+        self.directory.extend(entries);
+    }
+
     /// Define the parent directory.
     ///
     /// Each directory has an offset based link to its successor, forming a linked list in the
@@ -329,6 +348,11 @@ impl<'a, W: 'a + Write + Seek, K: TiffKind> DirectoryEncoder<'a, W, K> {
         Ok(())
     }
 
+    /// Write the IFD to the file and return the offset it is written to.
+    ///
+    /// The offset can be used as the value, or as part of a list of values, in the entry of
+    /// another directory. If you're constructing the entry directly you'll want to use an
+    /// appropriate type variant for this such as [`Type::IFD`].
     pub fn finish_with_offsets(mut self) -> TiffResult<DirectoryOffset<K>> {
         self.finish_internal()
     }
@@ -786,6 +810,71 @@ pub trait TiffKind {
     fn convert_slice(slice: &[Self::OffsetType]) -> &Self::OffsetArrayType;
 }
 
+impl<K: TiffKind> DirectoryOffset<K> {
+    /// Construct the directory pointer information from its raw offset.
+    ///
+    /// Fails in these cases:
+    ///
+    /// - if start offset is too large to be represented with the chosen TIFF format, i.e. a value
+    ///   larger than [`u32::MAX`] with a standard tiff which is only valid in a BigTiff. This
+    ///   returns [`TiffError::IntSizeError`].
+    /// - the extent of the directory would overflow the file length. This returns a
+    ///   [`TiffError::UsageError`].
+    ///
+    /// The library does *not* validate the contents or offset in any way, it just performs the
+    /// necessary math assuming the data is an accurate representation of existing content.
+    ///
+    /// # Examples
+    ///
+    /// If you have any custom way of writing a directory to a file that can not go through the
+    /// usual [`TiffEncoder::extra_directory`] then you can reconstruct the offset information as
+    /// long as you provide all the necessary data to the library. You can then use it as a parent
+    /// directory, i.e. have the library overwrite its `next` field.
+    ///
+    /// ```
+    /// use tiff::{
+    ///     encoder::{TiffEncoder, DirectoryOffset},
+    ///     Directory,
+    ///     tags::IfdPointer,
+    /// };
+    ///
+    /// # let mut file = std::io::Cursor::new(Vec::new());
+    /// let mut tiff = TiffEncoder::new(&mut file)?;
+    ///
+    /// // … some custom data writes, assume we know where a directory was written
+    /// # fn reconstruction_of_your_directory() -> Directory { Directory::from_iter([]) }
+    /// let known_offset = IfdPointer(1024);
+    /// let known_contents: Directory = reconstruction_of_your_directory();
+    ///
+    /// let reconstructed = DirectoryOffset::new(known_offset, &known_contents)?;
+    ///
+    /// // Now we can use it as if the directory was written by the `TiffEncoder` itself.
+    /// let mut chain = tiff.extra_directory()?;
+    /// chain.set_parent(&reconstructed);
+    /// chain.finish_with_offsets()?;
+    ///
+    /// # Ok::<_, tiff::TiffError>(())
+    /// ```
+    pub fn new(pointer: IfdPointer, dir: &Directory) -> TiffResult<Self> {
+        let offset = K::convert_offset(pointer.0)?;
+        let encoded = dir.encoded_len::<K>();
+        let offset_field_offset = encoded - mem::size_of_val(&offset) as u64;
+
+        let ifd_chain = pointer
+            .0
+            .checked_add(offset_field_offset)
+            .and_then(NonZeroU64::new)
+            .ok_or(TiffError::UsageError(UsageError::ZeroIfdPointer))?;
+
+        Ok(DirectoryOffset {
+            pointer,
+            offset,
+            ifd_chain,
+            kind: PhantomData,
+        })
+    }
+}
+
 /// Create a standard Tiff file.
 pub struct TiffKindStandard;
 
@@ -854,3 +943,33 @@ impl TiffKind for TiffKindBig {
         slice
     }
 }
+
+#[test]
+fn directory_offset_new_equivalent_to_writing() {
+    type K = TiffKindStandard;
+
+    let dir = Directory::from_iter(vec![
+        (
+            Tag::ImageWidth,
+            Entry::new(Type::LONG, 1, 100u32.to_ne_bytes()),
+        ),
+        (
+            Tag::ImageLength,
+            Entry::new(Type::LONG, 1, 200u32.to_ne_bytes()),
+        ),
+    ]);
+
+    let data = std::io::Cursor::new(vec![]);
+    let mut file = TiffEncoder::new(data).unwrap();
+
+    let mut as_dir_encoder = file.extra_directory().unwrap();
+    as_dir_encoder.extend_from(&dir);
+    let dir_extent = as_dir_encoder.finish_with_offsets().unwrap();
+
+    // Check we can recreate the extent had we written ourselves.
+    let synth = DirectoryOffset::<K>::new(dir_extent.pointer, &dir).unwrap();
+
+    assert_eq!(synth.pointer, dir_extent.pointer);
+    assert_eq!(synth.offset, dir_extent.offset);
+    assert_eq!(synth.ifd_chain, dir_extent.ifd_chain);
+}

src/encoder/writer.rs

@@ -190,3 +190,58 @@ impl<W: Seek> TiffWriter<W> {
         Ok(())
     }
 }
+
+#[test]
+fn encoder_on_short_writes() {
+    struct ShortWriter<T> {
+        inner: T,
+    }
+
+    impl<T: std::io::Write> std::io::Write for ShortWriter<T> {
+        fn write(&mut self, buf: &[u8]) -> std::io::Result<usize> {
+            let lower_bound = buf.len().min(1);
+            let lower = buf.len().saturating_sub(1).max(lower_bound);
+
+            self.inner.write(&buf[..lower])
+        }
+
+        fn flush(&mut self) -> std::io::Result<()> {
+            self.inner.flush()
+        }
+    }
+
+    impl<T: std::io::Seek> std::io::Seek for ShortWriter<T> {
+        fn seek(&mut self, pos: std::io::SeekFrom) -> std::io::Result<u64> {
+            self.inner.seek(pos)
+        }
+    }
+
+    let mut write = ShortWriter {
+        inner: std::io::Cursor::new(vec![0u8; 0]),
+    };
+
+    let imgdata = vec![42u8; 100 * 100];
+    for comp in [
+        super::Compression::Uncompressed,
+        super::Compression::Packbits,
+    ] {
+        write.inner.set_position(0);
+        write.inner.get_mut().clear();
+
+        {
+            let mut tiff_writer = super::TiffEncoder::new(&mut write)
+                .unwrap()
+                .with_compression(comp);
+
+            tiff_writer
+                .write_image::<super::Gray8>(100, 100, &imgdata)
+                .unwrap();
+        }
+
+        write.inner.set_position(0);
+        let mut tiff_reader = crate::decoder::Decoder::new(&mut write.inner).unwrap();
+        let data = tiff_reader.read_image().unwrap();
+
+        assert!(matches!(data, crate::decoder::DecodingResult::U8(v) if v == imgdata));
+    }
+}

src/error.rs

@@ -187,6 +187,9 @@ quick_error! {
         InsufficientOutputBufferSize { needed: usize, provided: usize } {
             display("the borrowed output buffer is not large enough for the decoded data, needed {needed} but have {provided}")
         }
+        ZeroIfdPointer {
+            display("the offset 0 can not point to a valid IFD")
+        }
     }
 }