! can: make Content-Length a long value, fixes #443

API changes:
  * `Content-Length`.length is now a Long
  * `max-content-length` setting is now also a Long
  * `incoming-auto-chunking-threshold-size` is now also a Long

This change makes only sense in combination with (auto-)chunking,
because we are not able to support byte arrays (HttpEntities) bigger
than Int.MaxValue.

examples/spray-can/simple-http-server/src/main/resources/application.conf

@@ -7,11 +7,11 @@ akka {
 spray.can.server {
   # uncomment the next line for making this an HTTPS example
   # ssl-encryption = on
-  idle-timeout = 5 s
-  request-timeout = 2 s
+  idle-timeout = 120 s
+  request-timeout = 60 s
 
   request-chunk-aggregation-limit = 0
 
-  parsing.max-content-length = 100m
+  parsing.max-content-length = 5g
   parsing.incoming-auto-chunking-threshold-size = 45k
 }

examples/spray-can/simple-http-server/src/main/scala/spray/examples/FileUploadHandler.scala

@@ -2,7 +2,7 @@ package spray.examples
 
 import akka.actor._
 import scala.concurrent.duration._
-import java.io.{FileInputStream, FileOutputStream, File}
+import java.io.{InputStream, FileInputStream, FileOutputStream, File}
 import org.jvnet.mimepull.{MIMEPart, MIMEMessage}
 import org.parboiled.common.FileUtils
 import spray.http._
@@ -12,6 +12,7 @@ import parser.HttpParser
 import HttpHeaders.RawHeader
 import spray.io.CommandWrapper
 import spray.util.SprayActorLogging
+import scala.annotation.tailrec
 
 
 class FileUploadHandler(client: ActorRef, start: ChunkedRequestStart) extends Actor with SprayActorLogging {
@@ -25,11 +26,11 @@ class FileUploadHandler(client: ActorRef, start: ChunkedRequestStart) extends Ac
   val boundary = multipart.parameters("boundary")
 
   log.info(s"Got start of chunked request $method $uri with multipart boundary '$boundary' writing to $tmpFile")
-  var bytesWritten = 0
+  var bytesWritten = 0L
 
   def receive = {
     case c: MessageChunk =>
-      log.info(s"Got ${c.body.size} bytes of chunked request $method $uri")
+      log.debug(s"Got ${c.body.size} bytes of chunked request $method $uri")
 
       output.write(c.body)
       bytesWritten += c.body.size
@@ -47,6 +48,9 @@ class FileUploadHandler(client: ActorRef, start: ChunkedRequestStart) extends Ac
   import collection.JavaConverters._
   def renderResult(): HttpEntity = {
     val message = new MIMEMessage(new FileInputStream(tmpFile), boundary)
+    // caution: the next line will read the complete file regardless of its size
+    // In the end the mime pull parser is not a decent way of parsing multipart attachments
+    // properly
     val parts = message.getAttachments.asScala.toSeq
 
     HttpEntity(`text/html`,
@@ -57,7 +61,7 @@ class FileUploadHandler(client: ActorRef, start: ChunkedRequestStart) extends Ac
           {
             parts.map { part =>
               val name = fileNameForPart(part).getOrElse("<unknown>")
-              <div>{name}: {part.getContentType} of size {FileUtils.readAllBytes(part.read()).size}</div>
+              <div>{name}: {part.getContentType} of size {sizeOf(part.readOnce())}</div>
             }
           }
         </body>
@@ -70,4 +74,16 @@ class FileUploadHandler(client: ActorRef, start: ChunkedRequestStart) extends Ac
       Right(disp: `Content-Disposition`) = HttpParser.parseHeader(RawHeader("Content-Disposition", dispHeader))
       name <- disp.parameters.get("filename")
     } yield name
+
+  def sizeOf(is: InputStream): Long = {
+    val buffer = new Array[Byte](65000)
+
+    @tailrec def inner(cur: Long): Long = {
+      val read = is.read(buffer)
+      if (read > 0) inner(cur + read)
+      else cur
+    }
+
+    inner(0)
+  }
 }

spray-can-tests/src/test/scala/spray/can/parsing/ContentLengthHeaderParserSpec.scala

@@ -0,0 +1,32 @@
+package spray.can.parsing
+
+import org.specs2.mutable.Specification
+import java.math.BigInteger
+import spray.can.parsing.SpecializedHeaderValueParsers.ContentLengthParser
+import akka.util.ByteString
+import spray.http.HttpHeaders.`Content-Length`
+
+class ContentLengthHeaderParserSpec extends Specification {
+  "specialized ContentLength parser" should {
+    "accept zero" in {
+      parse("0") === 0L
+    }
+    "accept positive value" in {
+      parse("43234398") === 43234398L
+    }
+    "accept positive value > Int.MaxValue <= Long.MaxValue" in {
+      parse("274877906944") === 274877906944L
+      parse("9223372036854775807") === 9223372036854775807L // Long.MaxValue
+    }
+    "don't accept positive value > Long.MaxValue" in {
+      parse("9223372036854775808") must throwA[ParsingException] // Long.MaxValue + 1
+      parse("92233720368547758070") must throwA[ParsingException] // Long.MaxValue * 10 which is 0 taken overflow into account
+      parse("92233720368547758080") must throwA[ParsingException] // (Long.MaxValue + 1) * 10 which is 0 taken overflow into account
+    }
+  }
+
+  def parse(bigint: String): Long = {
+    val (`Content-Length`(length), _) = ContentLengthParser(ByteString(bigint + "\r\n").compact, 0, _ ⇒ ())
+    length
+  }
+}

spray-can-tests/src/test/scala/spray/can/parsing/RequestParserSpec.scala

@@ -34,6 +34,7 @@ class RequestParserSpec extends Specification {
     akka.loglevel = WARNING
     spray.can.parsing.max-header-value-length = 32
     spray.can.parsing.max-uri-length = 20
+    spray.can.parsing.max-content-length = 4000000000
     spray.can.parsing.incoming-auto-chunking-threshold-size = 20""")
   val system = ActorSystem(Utils.actorSystemNameFrom(getClass), testConf)
 
@@ -117,9 +118,19 @@ class RequestParserSpec extends Specification {
         parse(parser)("DEFGH") === (PUT, Uri("/resource/yes"), `HTTP/1.1`, List(Host("x"), `Content-Length`(4)),
           "ABCD", "EFGH", false)
       }
+      "reject requests with Content-Length > Int.MaxSize" in {
+        val request =
+          """PUT /resource/yes HTTP/1.1
+            |Content-length:    2147483649
+            |Host: x
+            |
+            |"""
+        val parser = new HttpRequestPartParser(ParserSettings(system).copy(autoChunkingThreshold = Long.MaxValue))()
+        parse(parser)(request) === (400: StatusCode, "Content-Length > Int.MaxSize not supported for non-(auto)-chunked requests")
+      }
     }
 
-    "properly parse a chunked" in {
+    "properly parse a chunked request" in {
       val start =
         """PATCH /data HTTP/1.1
           |Transfer-Encoding: chunked
@@ -212,6 +223,27 @@ class RequestParserSpec extends Specification {
         parse(parser)("\0GET /data HTTP/1.1") === ("", Nil, "GET /data HTTP/1.1", false) // next parse run produced end
         parse(parser)("GET /data HTTP/1.1") === Result.NeedMoreData // start of next request
       }
+      "don't reject requests with Content-Length > Int.MaxSize" in {
+        val request =
+          """PUT /resource/yes HTTP/1.1
+            |Content-length:    2147483649
+            |Host: x
+            |
+            |"""
+        val parser = newParser
+        parse(parser)(request) === (PUT, Uri("/resource/yes"), `HTTP/1.1`, List(Host("x"), `Content-Length`(2147483649L)), "", false)
+      }
+      "reject requests with Content-Length > Long.MaxSize" in {
+        // content-length = (Long.MaxValue + 1) * 10, which is 0 when calculated overflow
+        val request =
+          """PUT /resource/yes HTTP/1.1
+            |Content-length: 92233720368547758080
+            |Host: x
+            |
+            |"""
+        val parser = newParser
+        parse(parser)(request) === (400: StatusCode, "Illegal `Content-Length` header value")
+      }
     }
 
     "reject a message chunk with" in {

spray-can/src/main/scala/spray/can/parsing/HttpMessagePartParser.scala

@@ -117,16 +117,18 @@ private[parsing] abstract class HttpMessagePartParser[Part <: HttpMessagePart](v
                   cth: Option[`Content-Type`], teh: Option[`Transfer-Encoding`], hostHeaderPresent: Boolean,
                   closeAfterResponseCompletion: Boolean): Result[Part]
 
-  def parseFixedLengthBody(headers: List[HttpHeader], input: ByteString, bodyStart: Int, length: Int,
+  def parseFixedLengthBody(headers: List[HttpHeader], input: ByteString, bodyStart: Int, length: Long,
                            cth: Option[`Content-Type`], closeAfterResponseCompletion: Boolean): Result[Part] =
     if (length >= settings.autoChunkingThreshold) {
       parse = parseAutoChunk(length, closeAfterResponseCompletion)
       val part = chunkStartMessage(headers)
       Result.Ok(part, drop(input, bodyStart), closeAfterResponseCompletion)
-    } else if (bodyStart + length <= input.length) {
+    } else if (length > Int.MaxValue) fail(s"Content-Length > Int.MaxSize not supported for non-(auto)-chunked requests")
+    else if (bodyStart.toLong + length <= input.length) {
+      val intLength = length.toInt
       parse = this
-      val part = message(headers, entity(cth, input.iterator.slice(bodyStart, bodyStart + length).toArray[Byte]))
-      Result.Ok(part, drop(input, bodyStart + length), closeAfterResponseCompletion)
+      val part = message(headers, entity(cth, input.iterator.slice(bodyStart, bodyStart + intLength).toArray[Byte]))
+      Result.Ok(part, drop(input, bodyStart + intLength), closeAfterResponseCompletion)
     } else {
       parse = more ⇒ parseFixedLengthBody(headers, input ++ more, bodyStart, length, cth, closeAfterResponseCompletion)
       Result.NeedMoreData
@@ -192,14 +194,14 @@ private[parsing] abstract class HttpMessagePartParser[Part <: HttpMessagePart](v
     }
   }
 
-  def parseAutoChunk(remainingBytes: Int, closeAfterResponseCompletion: Boolean)(input: CompactByteString): Result[Part] = {
+  def parseAutoChunk(remainingBytes: Long, closeAfterResponseCompletion: Boolean)(input: CompactByteString): Result[Part] = {
 
     def finishAutoChunking(input: CompactByteString): Result[Part] = {
       parse = this
       Result.Ok(ChunkedMessageEnd.asInstanceOf[Part], input.drop(1).compact, closeAfterResponseCompletion)
     }
 
-    val consumed = math.min(remainingBytes, input.size)
+    val consumed = math.min(remainingBytes, input.size).toInt // safe conversion because input.size returns an Int
     val chunk = MessageChunk(input.take(consumed).toArray[Byte])
 
     val remaining =

spray-can/src/main/scala/spray/can/parsing/ParserSettings.scala

@@ -27,10 +27,10 @@ case class ParserSettings(
     maxHeaderNameLength: Int,
     maxHeaderValueLength: Int,
     maxHeaderCount: Int,
-    maxContentLength: Int,
+    maxContentLength: Long,
     maxChunkExtLength: Int,
     maxChunkSize: Int,
-    autoChunkingThreshold: Int,
+    autoChunkingThreshold: Long,
     uriParsingMode: Uri.ParsingMode,
     illegalHeaderWarnings: Boolean,
     headerValueCacheLimits: Map[String, Int]) {
@@ -61,10 +61,10 @@ object ParserSettings extends SettingsCompanion[ParserSettings]("spray.can.parsi
       c getIntBytes "max-header-name-length",
       c getIntBytes "max-header-value-length",
       c getIntBytes "max-header-count",
-      c getIntBytes "max-content-length",
+      c getBytes "max-content-length",
       c getIntBytes "max-chunk-ext-length",
       c getIntBytes "max-chunk-size",
-      c getPossiblyInfiniteIntBytes "incoming-auto-chunking-threshold-size",
+      c getPossiblyInfiniteLongBytes "incoming-auto-chunking-threshold-size",
       Uri.ParsingMode(c getString "uri-parsing-mode"),
       c getBoolean "illegal-header-warnings",
       cacheConfig.entrySet.asScala.map(kvp ⇒ kvp.getKey -> cacheConfig.getInt(kvp.getKey))(collection.breakOut))

spray-can/src/main/scala/spray/can/parsing/SpecializedHeaderValueParsers.scala

@@ -26,16 +26,15 @@ import ProtectedHeaderCreation.enable
 private[parsing] object SpecializedHeaderValueParsers {
   import HttpHeaderParser._
 
-  def specializedHeaderValueParsers = Seq(
-    ContentLengthParser)
+  def specializedHeaderValueParsers = Seq(ContentLengthParser)
 
   object ContentLengthParser extends HeaderValueParser("Content-Length", maxValueCount = 1) {
     def apply(input: CompactByteString, valueStart: Int, warnOnIllegalHeader: ErrorInfo ⇒ Unit): (HttpHeader, Int) = {
       @tailrec def recurse(ix: Int = valueStart, result: Long = 0): (HttpHeader, Int) = {
         val c = byteChar(input, ix)
-        if (isDigit(c)) recurse(ix + 1, result * 10 + c - '0')
+        if (isDigit(c) && result >= 0) recurse(ix + 1, result * 10 + c - '0')
         else if (isWhitespace(c)) recurse(ix + 1, result)
-        else if (c == '\r' && byteChar(input, ix + 1) == '\n' && result < Int.MaxValue) (`Content-Length`(result.toInt), ix + 2)
+        else if (c == '\r' && byteChar(input, ix + 1) == '\n' && result >= 0) (`Content-Length`(result), ix + 2)
         else fail("Illegal `Content-Length` header value")
       }
       recurse()

spray-http/src/main/scala/spray/http/HttpHeader.scala

@@ -217,7 +217,7 @@ object HttpHeaders {
   }
 
   object `Content-Length` extends ModeledCompanion
-  case class `Content-Length`(length: Int)(implicit ev: ProtectedHeaderCreation.Enabled) extends ModeledHeader {
+  case class `Content-Length`(length: Long)(implicit ev: ProtectedHeaderCreation.Enabled) extends ModeledHeader {
     def renderValue[R <: Rendering](r: R): r.type = r ~~ length
     protected def companion = `Content-Length`
   }

spray-http/src/main/scala/spray/http/parser/SimpleHeaders.scala

@@ -32,7 +32,7 @@ private[parser] trait SimpleHeaders {
     oneOrMore(Token, separator = ListSep) ~ EOI ~~> (HttpHeaders.Connection(_)))
 
   def `*Content-Length` = rule {
-    oneOrMore(Digit) ~> (s ⇒ `Content-Length`(s.toInt)) ~ EOI
+    oneOrMore(Digit) ~> (s ⇒ `Content-Length`(s.toLong)) ~ EOI
   }
 
   def `*Content-Disposition` = rule {

spray-util/src/main/scala/spray/util/pimps/PimpedConfig.scala

@@ -41,4 +41,8 @@ class PimpedConfig(underlying: Config) {
     case "infinite" ⇒ Int.MaxValue
     case x          ⇒ getIntBytes(path)
   }
+  def getPossiblyInfiniteLongBytes(path: String): Long = underlying.getString(path) match {
+    case "infinite" ⇒ Long.MaxValue
+    case x          ⇒ getIntBytes(path)
+  }
 }