API to undo make_lzw_pre_encoded

Cargo.toml

@@ -22,6 +22,7 @@ color_quant = { version = "1.1", optional = true }
 glob = "0.3"
 criterion = "0.5.1"
 png = "0.17.10"
+rayon = "1.8.0" # for parallel reencoding example
 
 [features]
 default = ["raii_no_panic", "std", "color_quant"]

Changes.md

@@ -7,6 +7,8 @@ Features:
    It works together with `write_lzw_pre_encoded_frame` for quick rewriting of GIF files.
  - Added support pre-allocated `Vec`s in `from_palette_pixels`
  - Added ability to recover the `io::Read`er after decoding.
+ - Added support for decompressing `Frame.buffer` with LZW data,
+   which enables fully parallel GIF re-encoding (see examples/parallel.rs),
 
 Optimization:
  - Less buffering, copying, and lower peak memory usage.

examples/parallel.rs

@@ -0,0 +1,75 @@
+//! Reencodes GIF in parallel
+
+use gif::streaming_decoder::FrameDecoder;
+use gif::DecodeOptions;
+use rayon::iter::ParallelBridge;
+use rayon::iter::ParallelIterator;
+use std::env;
+use std::fs::File;
+use std::io::BufWriter;
+use std::path::PathBuf;
+
+fn main() -> Result<(), Box<dyn std::error::Error>> {
+    let input_path = PathBuf::from(
+        env::args_os()
+            .nth(1)
+            .ok_or("Specify a GIF path as the first argument")?,
+    );
+
+    let input = std::fs::read(&input_path)?;
+    let input_size = input.len();
+
+    let start = std::time::Instant::now();
+
+    let mut options = DecodeOptions::new();
+    options.skip_frame_decoding(true); // This gives LZW frames
+
+    let decoder = options.read_info(std::io::Cursor::new(input))?;
+    let repeat = decoder.repeat();
+    let screen_width = decoder.width();
+    let screen_height = decoder.height();
+    let global_pal = decoder.global_palette().unwrap_or_default().to_vec();
+
+    let output_file = format!(
+        "{}-reencoded.gif",
+        input_path.file_stem().unwrap().to_str().unwrap()
+    );
+    let output = BufWriter::new(File::create(output_file)?);
+    let mut encoder = gif::Encoder::new(output, screen_width, screen_height, &global_pal)?;
+    encoder.set_repeat(repeat)?;
+
+    let (send, recv) = std::sync::mpsc::channel();
+
+    decoder.into_iter().enumerate().par_bridge().try_for_each(move |(frame_number, frame)| {
+        let mut frame = frame?;
+        FrameDecoder::new(DecodeOptions::new())
+            .decode_lzw_encoded_frame(&mut frame)
+            .unwrap();
+        // frame is now pixels
+        frame.make_lzw_pre_encoded();
+        // frame is now LZW again, re-encoded
+        send.send((frame_number, frame)).unwrap();
+        Ok::<_, gif::DecodingError>(())
+    })?;
+
+    // Decoding and encoding can happen in parallel, but writing to the GIF file is sequential
+    let mut next_frame_number = 0;
+    let mut frames_to_process = Vec::new();
+    for (frame_number, frame) in recv {
+        // frames can arrive in any order, since they're processed in parallel,
+        // so they have to be stored in a queue
+        frames_to_process.push((frame_number, frame));
+        while let Some(index) = frames_to_process.iter().position(|&(num, _)| num == next_frame_number) {
+            let frame = frames_to_process.remove(index).1;
+            encoder.write_lzw_pre_encoded_frame(&frame)?;
+            next_frame_number += 1;
+        }
+    }
+    encoder.into_inner()?;
+
+    let seconds = start.elapsed().as_millis() as f64 / 1000.;
+    let rate = (input_size / 1024 / 1024) as f64 / seconds;
+
+    eprintln!("Finished in {seconds:0.2}s, {rate:0.0}MiB/s {}", if cfg!(debug_assertions) { ". Run with --release for more speed." } else { "" });
+    Ok(())
+}

src/lib.rs

@@ -128,7 +128,7 @@ pub use crate::encoder::{Encoder, ExtensionData, Repeat, EncodingError};
 /// Low-level, advanced decoder. Prefer [`Decoder`] instead, which can stream frames too.
 pub mod streaming_decoder {
     pub use crate::common::Block;
-    pub use crate::reader::{StreamingDecoder, OutputBuffer, Decoded, FrameDataType};
+    pub use crate::reader::{Decoded, FrameDataType, FrameDecoder, OutputBuffer, StreamingDecoder};
 }
 
 macro_rules! insert_as_doc {

src/reader/converter.rs

@@ -45,7 +45,7 @@ impl PixelConverter {
         }
     }
 
-    pub(crate) fn read_frame(&mut self, frame: &mut Frame<'_>, data_callback: FillBufferCallback<'_>) -> Result<(), DecodingError> {
+    pub(crate) fn check_buffer_size(&mut self, frame: &Frame<'_>) -> Result<usize, DecodingError> {
         let pixel_bytes = self.memory_limit
             .buffer_size(self.color_output, frame.width, frame.height)
             .ok_or_else(|| io::Error::new(io::ErrorKind::OutOfMemory, "image is too large"))?;
@@ -54,7 +54,12 @@ impl PixelConverter {
             pixel_bytes, self.buffer_size(frame),
             "Checked computation diverges from required buffer size"
         );
+        Ok(pixel_bytes)
+    }
 
+    #[inline]
+    pub(crate) fn read_frame(&mut self, frame: &mut Frame<'_>, data_callback: FillBufferCallback<'_>) -> Result<(), DecodingError> {
+        let pixel_bytes = self.check_buffer_size(frame)?;
         let mut vec = match mem::replace(&mut frame.buffer, Cow::Borrowed(&[])) {
             // reuse buffer if possible without reallocating
             Cow::Owned(mut vec) if vec.capacity() >= pixel_bytes => {
@@ -82,7 +87,9 @@ impl PixelConverter {
         }
     }
 
-    pub(crate) fn fill_buffer(&mut self, current_frame: &mut Frame<'_>, mut buf: &mut [u8], data_callback: FillBufferCallback<'_>) -> Result<bool, DecodingError> {
+    /// Use `read_into_buffer` to deinterlace
+    #[inline(never)]
+    pub(crate) fn fill_buffer(&mut self, current_frame: &Frame<'_>, mut buf: &mut [u8], data_callback: FillBufferCallback<'_>) -> Result<bool, DecodingError> {
         loop {
             let decode_into = match self.color_output {
                 // When decoding indexed data, LZW can write the pixels directly
@@ -151,7 +158,10 @@ impl PixelConverter {
         };
     }
 
-    pub(crate) fn read_into_buffer(&mut self, frame: &mut Frame<'_>, buf: &mut [u8], data_callback: FillBufferCallback<'_>) -> Result<(), DecodingError> {
+    /// Applies deinterlacing
+    ///
+    /// Set `frame.interlaced = false` afterwards if you're putting the buffer back into the `Frame`
+    pub(crate) fn read_into_buffer(&mut self, frame: &Frame<'_>, buf: &mut [u8], data_callback: FillBufferCallback<'_>) -> Result<(), DecodingError> {
         if frame.interlaced {
             let width = self.line_length(frame);
             let height = frame.height as usize;

src/reader/decoder.rs

@@ -1,3 +1,4 @@
+use std::borrow::Cow;
 use std::cmp;
 use std::error;
 use std::fmt;
@@ -95,7 +96,7 @@ impl From<DecodingFormatError> for DecodingError {
     }
 }
 
-/// Varies depending on skip_frame_decoding
+/// Varies depending on `skip_frame_decoding`
 #[derive(Debug, Copy, Clone)]
 pub enum FrameDataType {
     /// `Frame.buffer` will be regular pixel data
@@ -178,6 +179,8 @@ enum State {
 }
 use self::State::*;
 
+use super::converter::PixelConverter;
+
 /// U16 values that may occur in a GIF image
 #[derive(Debug, Copy, Clone)]
 enum U16Value {
@@ -209,6 +212,69 @@ enum ByteValue {
     CodeSize,
 }
 
+/// Decoder for `Frame::make_lzw_pre_encoded`
+pub struct FrameDecoder {
+    lzw_reader: LzwReader,
+    pixel_converter: PixelConverter,
+}
+
+impl FrameDecoder {
+    /// See also `set_global_palette`
+    #[inline]
+    #[must_use]
+    pub fn new(options: DecodeOptions) -> Self {
+        Self {
+            lzw_reader: LzwReader::new(options.check_for_end_code),
+            pixel_converter: PixelConverter::new(options.color_output, options.memory_limit),
+        }
+    }
+
+    /// Palette used for RGBA conversion
+    #[inline]
+    pub fn set_global_palette(&mut self, palette: Vec<u8>) {
+        self.pixel_converter.set_global_palette(palette);
+    }
+
+    /// Converts the frame in-place, replacing its LZW buffer with pixels.
+    ///
+    /// If you get an error about invalid min code size, the buffer was probably pixels, not compressed data.
+    #[inline]
+    pub fn decode_lzw_encoded_frame(&mut self, frame: &mut Frame<'_>) -> Result<(), DecodingError> {
+        let pixel_bytes = self.pixel_converter.check_buffer_size(frame)?;
+        let mut vec = vec![0; pixel_bytes];
+        self.decode_lzw_encoded_frame_into_buffer(frame, &mut vec)?;
+        frame.buffer = Cow::Owned(vec);
+        frame.interlaced = false;
+        Ok(())
+    }
+
+    /// Converts into the given buffer. It must be [`buffer_size()`] bytes large.
+    ///
+    /// Pixels are always deinterlaced, so update `frame.interlaced` afterwards if you're putting the buffer back into the frame.
+    pub fn decode_lzw_encoded_frame_into_buffer(&mut self, frame: &Frame<'_>, buf: &mut [u8]) -> Result<(), DecodingError> {
+        let (&min_code_size, mut data) = frame.buffer.split_first().unwrap_or((&2, &[]));
+        self.lzw_reader.reset(min_code_size)?;
+        let lzw_reader = &mut self.lzw_reader;
+        self.pixel_converter.read_into_buffer(frame, buf, &mut move |out| {
+            loop {
+                let (bytes_read, bytes_written) = lzw_reader.decode_bytes(data, out)?;
+                data = &data.get(bytes_read..).unwrap_or_default();
+                if bytes_written > 0 || bytes_read == 0 || data.is_empty() {
+                    return Ok(bytes_written)
+                }
+            }
+        })?;
+        Ok(())
+    }
+
+    /// Number of bytes required for `decode_lzw_encoded_frame_into_buffer`
+    #[inline]
+    #[must_use]
+    pub fn buffer_size(&self, frame: &Frame<'_>) -> usize {
+        self.pixel_converter.buffer_size(frame)
+    }
+}
+
 struct LzwReader {
     decoder: Option<LzwDecoder>,
     min_code_size: u8,
@@ -225,8 +291,9 @@ impl LzwReader {
     }
 
     pub fn reset(&mut self, min_code_size: u8) -> Result<(), DecodingError> {
-        // LZW spec: max 12 bits per code
-        if min_code_size > 11 {
+        // LZW spec: max 12 bits per code. This check helps catch confusion
+        // between LZW-compressed buffers and raw pixel data
+        if min_code_size > 11 || min_code_size < 1 {
             return Err(DecodingError::format("invalid minimal code size"));
         }
 

src/reader/mod.rs

@@ -15,7 +15,7 @@ mod converter;
 
 pub use self::decoder::{
     PLTE_CHANNELS, StreamingDecoder, Decoded, DecodingError, DecodingFormatError,
-    Version, FrameDataType, OutputBuffer
+    Version, FrameDataType, OutputBuffer, FrameDecoder
 };
 
 use self::converter::PixelConverter;
@@ -230,9 +230,9 @@ impl<R: Read> ReadDecoder<R> {
 
     fn decode_next_bytes(&mut self, out: &mut OutputBuffer<'_>) -> Result<usize, DecodingError> {
         match self.decode_next(out)? {
-            Some(Decoded::BytesDecoded(len)) => return Ok(len.get()),
-            Some(Decoded::DataEnd) => return Ok(0),
-            _ => return Err(DecodingError::format("unexpected data")),
+            Some(Decoded::BytesDecoded(len)) => Ok(len.get()),
+            Some(Decoded::DataEnd) => Ok(0),
+            _ => Err(DecodingError::format("unexpected data")),
         }
     }
 }