Make form data streamable

This is a squashed commit of [PR#12][] with some tiny cleanups
applied on top of that.

[PR#12]: #12

Allow any IO object in FormData::File
-------------------------------------

Previously we allowed only File and StringIO objects as an input
to `FormData::File`, but we can generalize that to any IO object that
responds to `#read` and `#size` (which includes `Tempfile`,
`ActionDispatch::Http::UploadedFile` etc).

Open File for given path in binary mode
---------------------------------------

That way different operating systems won't attempt to convert newline
characters to their internal representation, instead the file content
will always be retrieved byte-for-byte as is.

Officially support Pathname in FormData::File.new
-------------------------------------------------

Previously Pathname was implicitly supported, though extracting
filename wasn't working. With the recent refactoring this stopped
working, so we make the Pathname support explicit.

Make all components into IO objects
-----------------------------------

By changing all components to use an IO object as a base, we can
implement a common IO interface for all components, which delegates to
the underlying IO object.

This enables streaming multipart data into the request body, avoiding
loading the whole multipart data into memory when File parts are backed
by File objects.

See httprb/http#409 for the new streaming API.

Make CompositeIO convert strings to StringIOs
---------------------------------------------

By delegating handling strings to CompositeIO we can remove a lot of the
StringIO.new clutter when instantiating CompositeIO objects.

Use a buffer when reading IO files in CompositeIO
-------------------------------------------------

This way we're not creating a new string for each chunk read, instead
each chunk will be read into an existing string object (a "buffer"),
replacing any previous content.

lib/http/form_data/composite_io.rb

@@ -0,0 +1,73 @@
+# frozen_string_literal: true
+
+require "stringio"
+
+module HTTP
+  module FormData
+    # Provides IO interface across multiple IO objects.
+    class CompositeIO
+      # @param [Array<IO>] ios Array of IO objects
+      def initialize(ios)
+        @index  = 0
+        @buffer = String.new
+        @ios    = ios.map do |io|
+          if io.is_a?(String)
+            StringIO.new(io)
+          elsif io.respond_to?(:read)
+            io
+          else
+            raise ArgumentError,
+              "#{io.inspect} is neither a String nor an IO object"
+          end
+        end
+      end
+
+      # Reads and returns partial content acrosss multiple IO objects.
+      #
+      # @param [Integer] length Number of bytes to retrieve
+      # @param [String] outbuf String to be replaced with retrieved data
+      #
+      # @return [String, nil]
+      def read(length = nil, outbuf = nil)
+        outbuf = outbuf.to_s.replace("")
+
+        while current_io
+          current_io.read(length, @buffer)
+          outbuf << @buffer
+
+          if length
+            length -= @buffer.length
+            break if length.zero?
+          end
+
+          advance_io
+        end
+
+        outbuf unless length && outbuf.empty?
+      end
+
+      # Returns sum of all IO sizes.
+      def size
+        @size ||= @ios.map(&:size).inject(0, :+)
+      end
+
+      # Rewinds all IO objects and set cursor to the first IO object.
+      def rewind
+        @ios.each(&:rewind)
+        @index = 0
+      end
+
+      private
+
+      # Returns IO object under the cursor.
+      def current_io
+        @ios[@index]
+      end
+
+      # Advances cursor to the next IO object.
+      def advance_io
+        @index += 1
+      end
+    end
+  end
+end

lib/http/form_data/file.rb

@@ -26,56 +26,43 @@ class File < Part
       alias mime_type content_type
 
       # @see DEFAULT_MIME
-      # @param [String, StringIO, File] file_or_io Filename or IO instance.
+      # @param [String, Pathname, IO] path_or_io Filename or IO instance.
       # @param [#to_h] opts
       # @option opts [#to_s] :content_type (DEFAULT_MIME)
       #   Value of Content-Type header
       # @option opts [#to_s] :filename
-      #   When `file` is a String, defaults to basename of `file`.
-      #   When `file` is a File, defaults to basename of `file`.
-      #   When `file` is a StringIO, defaults to `"stream-{object_id}"`
-      def initialize(file_or_io, opts = {})
+      #   When `path_or_io` is a String, Pathname or File, defaults to basename.
+      #   When `path_or_io` is a IO, defaults to `"stream-{object_id}"`.
+      def initialize(path_or_io, opts = {})
         opts = FormData.ensure_hash(opts)
 
         if opts.key? :mime_type
           warn "[DEPRECATED] :mime_type option deprecated, use :content_type"
           opts[:content_type] = opts[:mime_type]
         end
 
-        @file_or_io   = file_or_io
+        @io           = make_io(path_or_io)
         @content_type = opts.fetch(:content_type, DEFAULT_MIME).to_s
-        @filename     = opts.fetch :filename do
-          case file_or_io
-          when String then ::File.basename file_or_io
-          when ::File then ::File.basename file_or_io.path
-          else             "stream-#{file_or_io.object_id}"
-          end
-        end
+        @filename     = opts.fetch(:filename, filename_for(@io))
       end
 
-      # Returns content size.
-      #
-      # @return [Integer]
-      def size
-        with_io(&:size)
-      end
+      private
 
-      # Returns content of a file of IO.
-      #
-      # @return [String]
-      def to_s
-        with_io(&:read)
+      def make_io(path_or_io)
+        if path_or_io.is_a?(String)
+          ::File.open(path_or_io, :binmode => true)
+        elsif defined?(Pathname) && path_or_io.is_a?(Pathname)
+          path_or_io.open(:binmode => true)
+        else
+          path_or_io
+        end
       end
 
-      private
-
-      # @yield [io] Gives IO instance to the block
-      # @return result of yielded block
-      def with_io
-        if @file_or_io.is_a?(::File) || @file_or_io.is_a?(StringIO)
-          yield @file_or_io
+      def filename_for(io)
+        if io.respond_to?(:path)
+          ::File.basename io.path
         else
-          ::File.open(@file_or_io, "rb") { |io| yield io }
+          "stream-#{io.object_id}"
         end
       end
     end

lib/http/form_data/multipart.rb

@@ -3,23 +3,21 @@
 require "securerandom"
 
 require "http/form_data/multipart/param"
+require "http/form_data/readable"
+require "http/form_data/composite_io"
 
 module HTTP
   module FormData
     # `multipart/form-data` form data.
     class Multipart
+      include Readable
+
       # @param [#to_h, Hash] data form data key-value Hash
       def initialize(data)
-        @parts          = Param.coerce FormData.ensure_hash data
-        @boundary       = (Array.new(21, "-") << SecureRandom.hex(21)).join("")
-        @content_length = nil
-      end
+        parts = Param.coerce FormData.ensure_hash data
 
-      # Returns content to be used for HTTP request body.
-      #
-      # @return [String]
-      def to_s
-        head + @parts.map(&:to_s).join(glue) + tail
+        @boundary = ("-" * 21) << SecureRandom.hex(21)
+        @io = CompositeIO.new [*parts.flat_map { |part| [glue, part] }, tail]
       end
 
       # Returns MIME type to be used for HTTP request `Content-Type` header.
@@ -33,31 +31,18 @@ def content_type
       # `Content-Length` header.
       #
       # @return [Integer]
-      def content_length
-        unless @content_length
-          @content_length  = head.bytesize + tail.bytesize
-          @content_length += @parts.map(&:size).reduce(:+)
-          @content_length += (glue.bytesize * (@parts.count - 1))
-        end
-
-        @content_length
-      end
+      alias content_length size
 
       private
 
-      # @return [String]
-      def head
-        @head ||= "--#{@boundary}#{CRLF}"
-      end
-
       # @return [String]
       def glue
-        @glue ||= "#{CRLF}--#{@boundary}#{CRLF}"
+        @glue ||= "--#{@boundary}#{CRLF}"
       end
 
       # @return [String]
       def tail
-        @tail ||= "#{CRLF}--#{@boundary}--"
+        @tail ||= "--#{@boundary}--"
       end
     end
   end

lib/http/form_data/multipart/param.rb

@@ -1,34 +1,16 @@
 # frozen_string_literal: true
 
+require "http/form_data/readable"
+require "http/form_data/composite_io"
+
 module HTTP
   module FormData
     class Multipart
       # Utility class to represent multi-part chunks
       class Param
-        # @param [#to_s] name
-        # @param [FormData::File, FormData::Part, #to_s] value
-        def initialize(name, value)
-          @name = name.to_s
-
-          @part =
-            if value.is_a?(FormData::Part)
-              value
-            else
-              FormData::Part.new(value)
-            end
-
-          parameters = { :name => @name }
-          parameters[:filename] = @part.filename if @part.filename
-          parameters = parameters.map { |k, v| "#{k}=#{v.inspect}" }.join("; ")
+        include Readable
 
-          @header = "Content-Disposition: form-data; #{parameters}"
-
-          return unless @part.content_type
-
-          @header += "#{CRLF}Content-Type: #{@part.content_type}"
-        end
-
-        # Returns body part with headers and data.
+        # Initializes body part with headers and data.
         #
         # @example With {FormData::File} value
         #
@@ -44,15 +26,19 @@ def initialize(name, value)
         #   ixti
         #
         # @return [String]
-        def to_s
-          "#{@header}#{CRLF * 2}#{@part}"
-        end
+        # @param [#to_s] name
+        # @param [FormData::File, FormData::Part, #to_s] value
+        def initialize(name, value)
+          @name = name.to_s
 
-        # Calculates size of a part (headers + body).
-        #
-        # @return [Integer]
-        def size
-          @header.bytesize + (CRLF.bytesize * 2) + @part.size
+          @part =
+            if value.is_a?(FormData::Part)
+              value
+            else
+              FormData::Part.new(value)
+            end
+
+          @io = CompositeIO.new [header, @part, footer]
         end
 
         # Flattens given `data` Hash into an array of `Param`'s.
@@ -72,6 +58,34 @@ def self.coerce(data)
 
           params
         end
+
+        private
+
+        def header
+          header = String.new
+          header << "Content-Disposition: form-data; #{parameters}#{CRLF}"
+          header << "Content-Type: #{content_type}#{CRLF}" if content_type
+          header << CRLF
+          header
+        end
+
+        def parameters
+          parameters = { :name => @name }
+          parameters[:filename] = filename if filename
+          parameters.map { |k, v| "#{k}=#{v.inspect}" }.join("; ")
+        end
+
+        def content_type
+          @part.content_type
+        end
+
+        def filename
+          @part.filename
+        end
+
+        def footer
+          CRLF.dup
+        end
       end
     end
   end

lib/http/form_data/part.rb

@@ -1,5 +1,9 @@
 # frozen_string_literal: true
 
+require "stringio"
+
+require "http/form_data/readable"
+
 module HTTP
   module FormData
     # Represents a body part of multipart/form-data request.
@@ -9,30 +13,18 @@ module FormData
     #  body = "Message"
     #  FormData::Part.new body, :content_type => 'foobar.txt; charset="UTF-8"'
     class Part
+      include Readable
+
       attr_reader :content_type, :filename
 
       # @param [#to_s] body
       # @param [String] content_type Value of Content-Type header
       # @param [String] filename     Value of filename parameter
       def initialize(body, content_type: nil, filename: nil)
-        @body = body.to_s
+        @io = StringIO.new(body.to_s)
         @content_type = content_type
         @filename = filename
       end
-
-      # Returns content size.
-      #
-      # @return [Integer]
-      def size
-        @body.bytesize
-      end
-
-      # Returns content of a file of IO.
-      #
-      # @return [String]
-      def to_s
-        @body
-      end
     end
   end
 end