diff --git a/Cargo.toml b/Cargo.toml index 71b1ac4..030d393 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -22,6 +22,7 @@ color_quant = { version = "1.1", optional = true } glob = "0.3" criterion = "0.5.1" png = "0.17.10" +rayon = "1.8.0" # for parallel reencoding example [features] default = ["raii_no_panic", "std", "color_quant"] diff --git a/Changes.md b/Changes.md index a62c530..ff67e81 100644 --- a/Changes.md +++ b/Changes.md @@ -7,6 +7,8 @@ Features: It works together with `write_lzw_pre_encoded_frame` for quick rewriting of GIF files. - Added support pre-allocated `Vec`s in `from_palette_pixels` - Added ability to recover the `io::Read`er after decoding. + - Added support for decompressing `Frame.buffer` with LZW data, + which enables fully parallel GIF re-encoding (see examples/parallel.rs), Optimization: - Less buffering, copying, and lower peak memory usage. diff --git a/examples/parallel.rs b/examples/parallel.rs new file mode 100644 index 0000000..893368c --- /dev/null +++ b/examples/parallel.rs @@ -0,0 +1,75 @@ +//! Reencodes GIF in parallel + +use gif::streaming_decoder::FrameDecoder; +use gif::DecodeOptions; +use rayon::iter::ParallelBridge; +use rayon::iter::ParallelIterator; +use std::env; +use std::fs::File; +use std::io::BufWriter; +use std::path::PathBuf; + +fn main() -> Result<(), Box> { + let input_path = PathBuf::from( + env::args_os() + .nth(1) + .ok_or("Specify a GIF path as the first argument")?, + ); + + let input = std::fs::read(&input_path)?; + let input_size = input.len(); + + let start = std::time::Instant::now(); + + let mut options = DecodeOptions::new(); + options.skip_frame_decoding(true); // This gives LZW frames + + let decoder = options.read_info(std::io::Cursor::new(input))?; + let repeat = decoder.repeat(); + let screen_width = decoder.width(); + let screen_height = decoder.height(); + let global_pal = decoder.global_palette().unwrap_or_default().to_vec(); + + let output_file = format!( + "{}-reencoded.gif", + input_path.file_stem().unwrap().to_str().unwrap() + ); + let output = BufWriter::new(File::create(output_file)?); + let mut encoder = gif::Encoder::new(output, screen_width, screen_height, &global_pal)?; + encoder.set_repeat(repeat)?; + + let (send, recv) = std::sync::mpsc::channel(); + + decoder.into_iter().enumerate().par_bridge().try_for_each(move |(frame_number, frame)| { + let mut frame = frame?; + FrameDecoder::new(DecodeOptions::new()) + .decode_lzw_encoded_frame(&mut frame) + .unwrap(); + // frame is now pixels + frame.make_lzw_pre_encoded(); + // frame is now LZW again, re-encoded + send.send((frame_number, frame)).unwrap(); + Ok::<_, gif::DecodingError>(()) + })?; + + // Decoding and encoding can happen in parallel, but writing to the GIF file is sequential + let mut next_frame_number = 0; + let mut frames_to_process = Vec::new(); + for (frame_number, frame) in recv { + // frames can arrive in any order, since they're processed in parallel, + // so they have to be stored in a queue + frames_to_process.push((frame_number, frame)); + while let Some(index) = frames_to_process.iter().position(|&(num, _)| num == next_frame_number) { + let frame = frames_to_process.remove(index).1; + encoder.write_lzw_pre_encoded_frame(&frame)?; + next_frame_number += 1; + } + } + encoder.into_inner()?; + + let seconds = start.elapsed().as_millis() as f64 / 1000.; + let rate = (input_size / 1024 / 1024) as f64 / seconds; + + eprintln!("Finished in {seconds:0.2}s, {rate:0.0}MiB/s {}", if cfg!(debug_assertions) { ". Run with --release for more speed." } else { "" }); + Ok(()) +} diff --git a/src/lib.rs b/src/lib.rs index 1a845e7..b883669 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -128,7 +128,7 @@ pub use crate::encoder::{Encoder, ExtensionData, Repeat, EncodingError}; /// Low-level, advanced decoder. Prefer [`Decoder`] instead, which can stream frames too. pub mod streaming_decoder { pub use crate::common::Block; - pub use crate::reader::{StreamingDecoder, OutputBuffer, Decoded, FrameDataType}; + pub use crate::reader::{Decoded, FrameDataType, FrameDecoder, OutputBuffer, StreamingDecoder}; } macro_rules! insert_as_doc { diff --git a/src/reader/converter.rs b/src/reader/converter.rs index bf0ea13..f6183f1 100644 --- a/src/reader/converter.rs +++ b/src/reader/converter.rs @@ -45,7 +45,7 @@ impl PixelConverter { } } - pub(crate) fn read_frame(&mut self, frame: &mut Frame<'_>, data_callback: FillBufferCallback<'_>) -> Result<(), DecodingError> { + pub(crate) fn check_buffer_size(&mut self, frame: &Frame<'_>) -> Result { let pixel_bytes = self.memory_limit .buffer_size(self.color_output, frame.width, frame.height) .ok_or_else(|| io::Error::new(io::ErrorKind::OutOfMemory, "image is too large"))?; @@ -54,7 +54,12 @@ impl PixelConverter { pixel_bytes, self.buffer_size(frame), "Checked computation diverges from required buffer size" ); + Ok(pixel_bytes) + } + #[inline] + pub(crate) fn read_frame(&mut self, frame: &mut Frame<'_>, data_callback: FillBufferCallback<'_>) -> Result<(), DecodingError> { + let pixel_bytes = self.check_buffer_size(frame)?; let mut vec = match mem::replace(&mut frame.buffer, Cow::Borrowed(&[])) { // reuse buffer if possible without reallocating Cow::Owned(mut vec) if vec.capacity() >= pixel_bytes => { @@ -82,7 +87,9 @@ impl PixelConverter { } } - pub(crate) fn fill_buffer(&mut self, current_frame: &mut Frame<'_>, mut buf: &mut [u8], data_callback: FillBufferCallback<'_>) -> Result { + /// Use `read_into_buffer` to deinterlace + #[inline(never)] + pub(crate) fn fill_buffer(&mut self, current_frame: &Frame<'_>, mut buf: &mut [u8], data_callback: FillBufferCallback<'_>) -> Result { loop { let decode_into = match self.color_output { // When decoding indexed data, LZW can write the pixels directly @@ -151,7 +158,10 @@ impl PixelConverter { }; } - pub(crate) fn read_into_buffer(&mut self, frame: &mut Frame<'_>, buf: &mut [u8], data_callback: FillBufferCallback<'_>) -> Result<(), DecodingError> { + /// Applies deinterlacing + /// + /// Set `frame.interlaced = false` afterwards if you're putting the buffer back into the `Frame` + pub(crate) fn read_into_buffer(&mut self, frame: &Frame<'_>, buf: &mut [u8], data_callback: FillBufferCallback<'_>) -> Result<(), DecodingError> { if frame.interlaced { let width = self.line_length(frame); let height = frame.height as usize; diff --git a/src/reader/decoder.rs b/src/reader/decoder.rs index aa0f551..1020dbd 100644 --- a/src/reader/decoder.rs +++ b/src/reader/decoder.rs @@ -1,3 +1,4 @@ +use std::borrow::Cow; use std::cmp; use std::error; use std::fmt; @@ -95,7 +96,7 @@ impl From for DecodingError { } } -/// Varies depending on skip_frame_decoding +/// Varies depending on `skip_frame_decoding` #[derive(Debug, Copy, Clone)] pub enum FrameDataType { /// `Frame.buffer` will be regular pixel data @@ -178,6 +179,8 @@ enum State { } use self::State::*; +use super::converter::PixelConverter; + /// U16 values that may occur in a GIF image #[derive(Debug, Copy, Clone)] enum U16Value { @@ -209,6 +212,69 @@ enum ByteValue { CodeSize, } +/// Decoder for `Frame::make_lzw_pre_encoded` +pub struct FrameDecoder { + lzw_reader: LzwReader, + pixel_converter: PixelConverter, +} + +impl FrameDecoder { + /// See also `set_global_palette` + #[inline] + #[must_use] + pub fn new(options: DecodeOptions) -> Self { + Self { + lzw_reader: LzwReader::new(options.check_for_end_code), + pixel_converter: PixelConverter::new(options.color_output, options.memory_limit), + } + } + + /// Palette used for RGBA conversion + #[inline] + pub fn set_global_palette(&mut self, palette: Vec) { + self.pixel_converter.set_global_palette(palette); + } + + /// Converts the frame in-place, replacing its LZW buffer with pixels. + /// + /// If you get an error about invalid min code size, the buffer was probably pixels, not compressed data. + #[inline] + pub fn decode_lzw_encoded_frame(&mut self, frame: &mut Frame<'_>) -> Result<(), DecodingError> { + let pixel_bytes = self.pixel_converter.check_buffer_size(frame)?; + let mut vec = vec![0; pixel_bytes]; + self.decode_lzw_encoded_frame_into_buffer(frame, &mut vec)?; + frame.buffer = Cow::Owned(vec); + frame.interlaced = false; + Ok(()) + } + + /// Converts into the given buffer. It must be [`buffer_size()`] bytes large. + /// + /// Pixels are always deinterlaced, so update `frame.interlaced` afterwards if you're putting the buffer back into the frame. + pub fn decode_lzw_encoded_frame_into_buffer(&mut self, frame: &Frame<'_>, buf: &mut [u8]) -> Result<(), DecodingError> { + let (&min_code_size, mut data) = frame.buffer.split_first().unwrap_or((&2, &[])); + self.lzw_reader.reset(min_code_size)?; + let lzw_reader = &mut self.lzw_reader; + self.pixel_converter.read_into_buffer(frame, buf, &mut move |out| { + loop { + let (bytes_read, bytes_written) = lzw_reader.decode_bytes(data, out)?; + data = &data.get(bytes_read..).unwrap_or_default(); + if bytes_written > 0 || bytes_read == 0 || data.is_empty() { + return Ok(bytes_written) + } + } + })?; + Ok(()) + } + + /// Number of bytes required for `decode_lzw_encoded_frame_into_buffer` + #[inline] + #[must_use] + pub fn buffer_size(&self, frame: &Frame<'_>) -> usize { + self.pixel_converter.buffer_size(frame) + } +} + struct LzwReader { decoder: Option, min_code_size: u8, @@ -225,8 +291,9 @@ impl LzwReader { } pub fn reset(&mut self, min_code_size: u8) -> Result<(), DecodingError> { - // LZW spec: max 12 bits per code - if min_code_size > 11 { + // LZW spec: max 12 bits per code. This check helps catch confusion + // between LZW-compressed buffers and raw pixel data + if min_code_size > 11 || min_code_size < 1 { return Err(DecodingError::format("invalid minimal code size")); } diff --git a/src/reader/mod.rs b/src/reader/mod.rs index 5e06a17..a72e462 100644 --- a/src/reader/mod.rs +++ b/src/reader/mod.rs @@ -15,7 +15,7 @@ mod converter; pub use self::decoder::{ PLTE_CHANNELS, StreamingDecoder, Decoded, DecodingError, DecodingFormatError, - Version, FrameDataType, OutputBuffer + Version, FrameDataType, OutputBuffer, FrameDecoder }; use self::converter::PixelConverter; @@ -230,9 +230,9 @@ impl ReadDecoder { fn decode_next_bytes(&mut self, out: &mut OutputBuffer<'_>) -> Result { match self.decode_next(out)? { - Some(Decoded::BytesDecoded(len)) => return Ok(len.get()), - Some(Decoded::DataEnd) => return Ok(0), - _ => return Err(DecodingError::format("unexpected data")), + Some(Decoded::BytesDecoded(len)) => Ok(len.get()), + Some(Decoded::DataEnd) => Ok(0), + _ => Err(DecodingError::format("unexpected data")), } } }