Permalink
Browse files

Split Fixed uploads into chunks

Instead of reading the entire upload into a single string, we now
return one chunk each time the underlying [read] call returns some
data. This avoids using large amounts of RAM, hitting the 16 MB
string limit on 32-bit systems. It also allows the application to
begin processing the data before it has all arrived.

Fixes mirage#153.
  • Loading branch information...
talex5 committed Jul 23, 2014
1 parent 86394ac commit 030969c32d310d690b8df5929d9a5f32caee6eb5
Showing with 8 additions and 7 deletions.
  1. +1 −1 cohttp/request.ml
  2. +7 −6 cohttp/transfer_io.ml
@@ -102,7 +102,7 @@ module Make(IO : S.IO) = struct
return (`Ok { headers; meth; uri; version; encoding })

let has_body req = Transfer.has_body req.encoding
let read_body_chunk req ic = Transfer_IO.read req.encoding ic
let read_body_chunk req = Transfer_IO.read req.encoding

let write_header req oc =
let fst_line = Printf.sprintf "%s %s %s\r\n" (Code.string_of_method req.meth)
@@ -53,14 +53,15 @@ module Make(IO : S.IO) = struct
end

module Fixed = struct
let read ~len ic =
let read ~remaining ic =
(* TODO functorise string to a bigbuffer *)
match len with
match !remaining with
|0 -> return Done
|len ->
read_exactly ic len >>= function
|None -> return Done
|Some buf -> return (Final_chunk buf)
read ic len >>= fun buf ->
remaining := !remaining - String.length buf;
if !remaining = 0 then return (Final_chunk buf)
else return (Chunk buf)

(* TODO enforce that the correct length is written? *)
let write oc buf =
@@ -80,7 +81,7 @@ module Make(IO : S.IO) = struct
let read =
function
| Chunked -> Chunked.read
| Fixed len -> Fixed.read ~len
| Fixed len -> Fixed.read ~remaining:(ref len)
| Unknown -> Unknown.read

let write =

1 comment on commit 030969c

@avsm

This comment has been minimized.

avsm commented on 030969c Jul 23, 2014

Makes sense; should match the Chunked behaviour.

Please sign in to comment.