Permalink
Browse files

! can: make Content-Length a long value, fixes #443

API changes:
  * `Content-Length`.length is now a Long
  * `max-content-length` setting is now also a Long
  * `incoming-auto-chunking-threshold-size` is now also a Long

This change makes only sense in combination with (auto-)chunking,
because we are not able to support byte arrays (HttpEntities) bigger
than Int.MaxValue.
  • Loading branch information...
jrudolph committed Aug 22, 2013
1 parent 068de92 commit b2fee8de3b57bd43cad3aa81f0ea0d4d9b754255
@@ -7,11 +7,11 @@ akka {
spray.can.server {
# uncomment the next line for making this an HTTPS example
# ssl-encryption = on
- idle-timeout = 5 s
- request-timeout = 2 s
+ idle-timeout = 120 s
+ request-timeout = 60 s
request-chunk-aggregation-limit = 0
- parsing.max-content-length = 100m
+ parsing.max-content-length = 5g
parsing.incoming-auto-chunking-threshold-size = 45k
}
@@ -2,7 +2,7 @@ package spray.examples
import akka.actor._
import scala.concurrent.duration._
-import java.io.{FileInputStream, FileOutputStream, File}
+import java.io.{InputStream, FileInputStream, FileOutputStream, File}
import org.jvnet.mimepull.{MIMEPart, MIMEMessage}
import org.parboiled.common.FileUtils
import spray.http._
@@ -12,6 +12,7 @@ import parser.HttpParser
import HttpHeaders.RawHeader
import spray.io.CommandWrapper
import spray.util.SprayActorLogging
+import scala.annotation.tailrec
class FileUploadHandler(client: ActorRef, start: ChunkedRequestStart) extends Actor with SprayActorLogging {
@@ -25,11 +26,11 @@ class FileUploadHandler(client: ActorRef, start: ChunkedRequestStart) extends Ac
val boundary = multipart.parameters("boundary")
log.info(s"Got start of chunked request $method $uri with multipart boundary '$boundary' writing to $tmpFile")
- var bytesWritten = 0
+ var bytesWritten = 0L
def receive = {
case c: MessageChunk =>
- log.info(s"Got ${c.body.size} bytes of chunked request $method $uri")
+ log.debug(s"Got ${c.body.size} bytes of chunked request $method $uri")
output.write(c.body)
bytesWritten += c.body.size
@@ -47,6 +48,9 @@ class FileUploadHandler(client: ActorRef, start: ChunkedRequestStart) extends Ac
import collection.JavaConverters._
def renderResult(): HttpEntity = {
val message = new MIMEMessage(new FileInputStream(tmpFile), boundary)
+ // caution: the next line will read the complete file regardless of its size
+ // In the end the mime pull parser is not a decent way of parsing multipart attachments
+ // properly
val parts = message.getAttachments.asScala.toSeq
HttpEntity(`text/html`,
@@ -57,7 +61,7 @@ class FileUploadHandler(client: ActorRef, start: ChunkedRequestStart) extends Ac
{
parts.map { part =>
val name = fileNameForPart(part).getOrElse("<unknown>")
- <div>{name}: {part.getContentType} of size {FileUtils.readAllBytes(part.read()).size}</div>
+ <div>{name}: {part.getContentType} of size {sizeOf(part.readOnce())}</div>
}
}
</body>
@@ -70,4 +74,16 @@ class FileUploadHandler(client: ActorRef, start: ChunkedRequestStart) extends Ac
Right(disp: `Content-Disposition`) = HttpParser.parseHeader(RawHeader("Content-Disposition", dispHeader))
name <- disp.parameters.get("filename")
} yield name
+
+ def sizeOf(is: InputStream): Long = {
+ val buffer = new Array[Byte](65000)
+
+ @tailrec def inner(cur: Long): Long = {
+ val read = is.read(buffer)
+ if (read > 0) inner(cur + read)
+ else cur
+ }
+
+ inner(0)
+ }
}
@@ -0,0 +1,32 @@
+package spray.can.parsing
+
+import org.specs2.mutable.Specification
+import java.math.BigInteger
+import spray.can.parsing.SpecializedHeaderValueParsers.ContentLengthParser
+import akka.util.ByteString
+import spray.http.HttpHeaders.`Content-Length`
+
+class ContentLengthHeaderParserSpec extends Specification {
+ "specialized ContentLength parser" should {
+ "accept zero" in {
+ parse("0") === 0L
+ }
+ "accept positive value" in {
+ parse("43234398") === 43234398L
+ }
+ "accept positive value > Int.MaxValue <= Long.MaxValue" in {
+ parse("274877906944") === 274877906944L
+ parse("9223372036854775807") === 9223372036854775807L // Long.MaxValue
+ }
+ "don't accept positive value > Long.MaxValue" in {
+ parse("9223372036854775808") must throwA[ParsingException] // Long.MaxValue + 1
+ parse("92233720368547758070") must throwA[ParsingException] // Long.MaxValue * 10 which is 0 taken overflow into account
+ parse("92233720368547758080") must throwA[ParsingException] // (Long.MaxValue + 1) * 10 which is 0 taken overflow into account
+ }
+ }
+
+ def parse(bigint: String): Long = {
+ val (`Content-Length`(length), _) = ContentLengthParser(ByteString(bigint + "\r\n").compact, 0, _ ())
+ length
+ }
+}
@@ -34,6 +34,7 @@ class RequestParserSpec extends Specification {
akka.loglevel = WARNING
spray.can.parsing.max-header-value-length = 32
spray.can.parsing.max-uri-length = 20
+ spray.can.parsing.max-content-length = 4000000000
spray.can.parsing.incoming-auto-chunking-threshold-size = 20""")
val system = ActorSystem(Utils.actorSystemNameFrom(getClass), testConf)
@@ -117,9 +118,19 @@ class RequestParserSpec extends Specification {
parse(parser)("DEFGH") === (PUT, Uri("/resource/yes"), `HTTP/1.1`, List(Host("x"), `Content-Length`(4)),
"ABCD", "EFGH", false)
}
+ "reject requests with Content-Length > Int.MaxSize" in {
+ val request =
+ """PUT /resource/yes HTTP/1.1
+ |Content-length: 2147483649
+ |Host: x
+ |
+ |"""
+ val parser = new HttpRequestPartParser(ParserSettings(system).copy(autoChunkingThreshold = Long.MaxValue))()
+ parse(parser)(request) === (400: StatusCode, "Content-Length > Int.MaxSize not supported for non-(auto)-chunked requests")
+ }
}
- "properly parse a chunked" in {
+ "properly parse a chunked request" in {
val start =
"""PATCH /data HTTP/1.1
|Transfer-Encoding: chunked
@@ -212,6 +223,27 @@ class RequestParserSpec extends Specification {
parse(parser)("\0GET /data HTTP/1.1") === ("", Nil, "GET /data HTTP/1.1", false) // next parse run produced end
parse(parser)("GET /data HTTP/1.1") === Result.NeedMoreData // start of next request
}
+ "don't reject requests with Content-Length > Int.MaxSize" in {
+ val request =
+ """PUT /resource/yes HTTP/1.1
+ |Content-length: 2147483649
+ |Host: x
+ |
+ |"""
+ val parser = newParser
+ parse(parser)(request) === (PUT, Uri("/resource/yes"), `HTTP/1.1`, List(Host("x"), `Content-Length`(2147483649L)), "", false)
+ }
+ "reject requests with Content-Length > Long.MaxSize" in {
+ // content-length = (Long.MaxValue + 1) * 10, which is 0 when calculated overflow
+ val request =
+ """PUT /resource/yes HTTP/1.1
+ |Content-length: 92233720368547758080
+ |Host: x
+ |
+ |"""
+ val parser = newParser
+ parse(parser)(request) === (400: StatusCode, "Illegal `Content-Length` header value")
+ }
}
"reject a message chunk with" in {
@@ -117,16 +117,18 @@ private[parsing] abstract class HttpMessagePartParser[Part <: HttpMessagePart](v
cth: Option[`Content-Type`], teh: Option[`Transfer-Encoding`], hostHeaderPresent: Boolean,
closeAfterResponseCompletion: Boolean): Result[Part]
- def parseFixedLengthBody(headers: List[HttpHeader], input: ByteString, bodyStart: Int, length: Int,
+ def parseFixedLengthBody(headers: List[HttpHeader], input: ByteString, bodyStart: Int, length: Long,
cth: Option[`Content-Type`], closeAfterResponseCompletion: Boolean): Result[Part] =
if (length >= settings.autoChunkingThreshold) {
parse = parseAutoChunk(length, closeAfterResponseCompletion)
val part = chunkStartMessage(headers)
Result.Ok(part, drop(input, bodyStart), closeAfterResponseCompletion)
- } else if (bodyStart + length <= input.length) {
+ } else if (length > Int.MaxValue) fail(s"Content-Length > Int.MaxSize not supported for non-(auto)-chunked requests")
+ else if (bodyStart.toLong + length <= input.length) {
+ val intLength = length.toInt
parse = this
- val part = message(headers, entity(cth, input.iterator.slice(bodyStart, bodyStart + length).toArray[Byte]))
- Result.Ok(part, drop(input, bodyStart + length), closeAfterResponseCompletion)
+ val part = message(headers, entity(cth, input.iterator.slice(bodyStart, bodyStart + intLength).toArray[Byte]))
+ Result.Ok(part, drop(input, bodyStart + intLength), closeAfterResponseCompletion)
} else {
parse = more parseFixedLengthBody(headers, input ++ more, bodyStart, length, cth, closeAfterResponseCompletion)
Result.NeedMoreData
@@ -192,14 +194,14 @@ private[parsing] abstract class HttpMessagePartParser[Part <: HttpMessagePart](v
}
}
- def parseAutoChunk(remainingBytes: Int, closeAfterResponseCompletion: Boolean)(input: CompactByteString): Result[Part] = {
+ def parseAutoChunk(remainingBytes: Long, closeAfterResponseCompletion: Boolean)(input: CompactByteString): Result[Part] = {
def finishAutoChunking(input: CompactByteString): Result[Part] = {
parse = this
Result.Ok(ChunkedMessageEnd.asInstanceOf[Part], input.drop(1).compact, closeAfterResponseCompletion)
}
- val consumed = math.min(remainingBytes, input.size)
+ val consumed = math.min(remainingBytes, input.size).toInt // safe conversion because input.size returns an Int
val chunk = MessageChunk(input.take(consumed).toArray[Byte])
val remaining =
@@ -27,10 +27,10 @@ case class ParserSettings(
maxHeaderNameLength: Int,
maxHeaderValueLength: Int,
maxHeaderCount: Int,
- maxContentLength: Int,
+ maxContentLength: Long,
maxChunkExtLength: Int,
maxChunkSize: Int,
- autoChunkingThreshold: Int,
+ autoChunkingThreshold: Long,
uriParsingMode: Uri.ParsingMode,
illegalHeaderWarnings: Boolean,
headerValueCacheLimits: Map[String, Int]) {
@@ -61,10 +61,10 @@ object ParserSettings extends SettingsCompanion[ParserSettings]("spray.can.parsi
c getIntBytes "max-header-name-length",
c getIntBytes "max-header-value-length",
c getIntBytes "max-header-count",
- c getIntBytes "max-content-length",
+ c getBytes "max-content-length",
c getIntBytes "max-chunk-ext-length",
c getIntBytes "max-chunk-size",
- c getPossiblyInfiniteIntBytes "incoming-auto-chunking-threshold-size",
+ c getPossiblyInfiniteLongBytes "incoming-auto-chunking-threshold-size",
Uri.ParsingMode(c getString "uri-parsing-mode"),
c getBoolean "illegal-header-warnings",
cacheConfig.entrySet.asScala.map(kvp kvp.getKey -> cacheConfig.getInt(kvp.getKey))(collection.breakOut))
@@ -26,16 +26,15 @@ import ProtectedHeaderCreation.enable
private[parsing] object SpecializedHeaderValueParsers {
import HttpHeaderParser._
- def specializedHeaderValueParsers = Seq(
- ContentLengthParser)
+ def specializedHeaderValueParsers = Seq(ContentLengthParser)
object ContentLengthParser extends HeaderValueParser("Content-Length", maxValueCount = 1) {
def apply(input: CompactByteString, valueStart: Int, warnOnIllegalHeader: ErrorInfo Unit): (HttpHeader, Int) = {
@tailrec def recurse(ix: Int = valueStart, result: Long = 0): (HttpHeader, Int) = {
val c = byteChar(input, ix)
- if (isDigit(c)) recurse(ix + 1, result * 10 + c - '0')
+ if (isDigit(c) && result >= 0) recurse(ix + 1, result * 10 + c - '0')
else if (isWhitespace(c)) recurse(ix + 1, result)
- else if (c == '\r' && byteChar(input, ix + 1) == '\n' && result < Int.MaxValue) (`Content-Length`(result.toInt), ix + 2)
+ else if (c == '\r' && byteChar(input, ix + 1) == '\n' && result >= 0) (`Content-Length`(result), ix + 2)
else fail("Illegal `Content-Length` header value")
}
recurse()
@@ -217,7 +217,7 @@ object HttpHeaders {
}
object `Content-Length` extends ModeledCompanion
- case class `Content-Length`(length: Int)(implicit ev: ProtectedHeaderCreation.Enabled) extends ModeledHeader {
+ case class `Content-Length`(length: Long)(implicit ev: ProtectedHeaderCreation.Enabled) extends ModeledHeader {
def renderValue[R <: Rendering](r: R): r.type = r ~~ length
protected def companion = `Content-Length`
}
@@ -32,7 +32,7 @@ private[parser] trait SimpleHeaders {
oneOrMore(Token, separator = ListSep) ~ EOI ~~> (HttpHeaders.Connection(_)))
def `*Content-Length` = rule {
- oneOrMore(Digit) ~> (s `Content-Length`(s.toInt)) ~ EOI
+ oneOrMore(Digit) ~> (s `Content-Length`(s.toLong)) ~ EOI
}
def `*Content-Disposition` = rule {
@@ -41,4 +41,8 @@ class PimpedConfig(underlying: Config) {
case "infinite" Int.MaxValue
case x getIntBytes(path)
}
+ def getPossiblyInfiniteLongBytes(path: String): Long = underlying.getString(path) match {
+ case "infinite" Long.MaxValue
+ case x getIntBytes(path)
+ }
}

0 comments on commit b2fee8d

Please sign in to comment.