diff --git a/Cargo.toml b/Cargo.toml
index f1b411f..affd254 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -25,7 +25,6 @@ flate2 = { version = "1.0.20", optional = true }
 weezl = { version = "0.1.10", optional = true }
 zstd = { version = "0.13", optional = true }
 zune-jpeg = { version = "0.4.17", optional = true }
-bitvec = "1.0.1"
 
 [dev-dependencies]
 criterion = "0.3.1"
diff --git a/src/decoder/stream.rs b/src/decoder/stream.rs
index 4ecbd5e..96a2837 100644
--- a/src/decoder/stream.rs
+++ b/src/decoder/stream.rs
@@ -1,7 +1,4 @@
 //! All IO functionality needed for TIFF decoding
-
-#[cfg(feature = "fax")]
-use std::collections::VecDeque;
 use std::io::{self, BufRead, BufReader, Read, Seek, Take};
 
 /// Byte order of the TIFF file.
@@ -281,8 +278,7 @@ impl<R: Read> Read for PackBitsReader<R> {
 #[cfg(feature = "fax")]
 pub struct Group4Reader<R: Read> {
     decoder: fax::decoder::Group4Decoder<io::Bytes<io::Take<R>>>,
-    bits: bitvec::vec::BitVec<u8, bitvec::prelude::Msb0>,
-    byte_buf: VecDeque<u8>,
+    line_buf: io::Cursor<Vec<u8>>,
     height: u16,
     width: u16,
     y: u16,
@@ -303,8 +299,7 @@ impl<R: Read> Group4Reader<R> {
                 reader.take(compressed_length).bytes(),
                 width,
             )?,
-            bits: bitvec::vec::BitVec::new(),
-            byte_buf: VecDeque::new(),
+            line_buf: io::Cursor::new(Vec::with_capacity(width.into())),
             width: width,
             height: height,
             y: 0,
@@ -315,27 +310,47 @@ impl<R: Read> Group4Reader<R> {
 #[cfg(feature = "fax")]
 impl<R: Read> Read for Group4Reader<R> {
     fn read(&mut self, buf: &mut [u8]) -> io::Result<usize> {
-        if self.byte_buf.is_empty() && self.y < self.height {
-            let next = self
-                .decoder
-                .advance()
-                .map_err(|e| std::io::Error::new(std::io::ErrorKind::Other, e))?;
+        // Either we have not read any line or we are at the end of a line.
+        if self.line_buf.position() as usize == self.line_buf.get_ref().len()
+            && self.y < self.height
+        {
+            let next = self.decoder.advance().map_err(std::io::Error::other)?;
 
             match next {
                 fax::decoder::DecodeStatus::End => (),
                 fax::decoder::DecodeStatus::Incomplete => {
                     self.y += 1;
+
+                    // We known `transitions` yields exactly `self.width` items (per doc).
                     let transitions = fax::decoder::pels(self.decoder.transition(), self.width);
-                    self.bits.extend(transitions.map(|c| match c {
+
+                    let buffer = self.line_buf.get_mut();
+                    buffer.resize(usize::from(self.width).div_ceil(8), 0u8);
+
+                    let target = &mut buffer[..];
+
+                    let mut bits = transitions.map(|c| match c {
                         fax::Color::Black => true,
                         fax::Color::White => false,
-                    }));
-                    self.byte_buf.extend(self.bits.as_raw_slice());
-                    self.bits.clear();
+                    });
+
+                    // Assemble bits in MSB as per our library representation for buffer.
+                    for byte in target {
+                        let mut val = 0;
+
+                        for (idx, bit) in bits.by_ref().take(8).enumerate() {
+                            val |= u8::from(bit) << (7 - idx % 8);
+                        }
+
+                        *byte = val;
+                    }
+
+                    self.line_buf.set_position(0);
                 }
             }
         }
-        self.byte_buf.read(buf)
+
+        self.line_buf.read(buf)
     }
 }