From d0bc1ab43b38a93ec45371cd26da41b76291565c Mon Sep 17 00:00:00 2001 From: matkob Date: Wed, 17 May 2023 10:14:05 +0200 Subject: [PATCH] Add request decompression to server GZip middleware --- docs/docs/gzip.md | 85 ++++++++++++++++++- .../org/http4s/server/middleware/GZip.scala | 33 ++++++- .../http4s/server/middleware/GZipSuite.scala | 39 +++++++++ 3 files changed, 150 insertions(+), 7 deletions(-) diff --git a/docs/docs/gzip.md b/docs/docs/gzip.md index 7e22f924ed3..067ec3a17cb 100644 --- a/docs/docs/gzip.md +++ b/docs/docs/gzip.md @@ -1,7 +1,7 @@ -# GZip Compression +# GZip Http4s provides [Middleware], named `GZip`, for allowing for the compression of the `Response` -body using GZip. +body and for the decompression of the incoming `Request` using `GZip`. Examples in this document have the following dependencies. @@ -28,6 +28,8 @@ import cats.effect.unsafe.IORuntime implicit val runtime: IORuntime = cats.effect.unsafe.IORuntime.global ``` +## Compressing Response + Let's start by making a simple service that returns a (relatively) large string in its body. We'll use `as[String]` to examine the body. @@ -63,7 +65,7 @@ bodyNormal.length So far, there was no change. That's because the caller needs to inform us that they will accept GZipped responses via an `Accept-Encoding` header. Acceptable -values for the `Accept-Encoding` header are **"gzip"**, **"x-gzip"**, and **"*"**. +values for the `Accept-Encoding` header are **"gzip"**, **"x-gzip"**, and **"\*"**. ```scala mdoc val requestZip = request.putHeaders("Accept-Encoding" -> "gzip") @@ -79,7 +81,82 @@ length. Also, there is a `Content-Encoding` header in the response with a value of **"gzip"**. As described in [Middleware], services and middleware can be composed such -that only some of your endpoints are GZip enabled. +that only some of your endpoints are `GZip` enabled. + +## Decompressing Request + +Analogically to compressing response, `GZip` middleware supports decompressing request. +Let's see how it works with a simple echo service, returning the body of incoming requests. + +```scala mdoc:reset:invisible +import cats.effect._ +import org.http4s._ +import org.http4s.dsl.io._ +import org.http4s.implicits._ + +import cats.effect.unsafe.IORuntime +implicit val runtime: IORuntime = cats.effect.unsafe.IORuntime.global +``` + +```scala mdoc:silent +val service = HttpRoutes.of[IO] { + case req => + Ok(req.body) +} + +val request = Request[IO](Method.POST, uri"/").withEntity("echo") +``` + +```scala mdoc +// Do not call 'unsafeRun' in your code - see note at bottom. +val response = service.orNotFound(request).unsafeRunSync() +val body = response.as[String].unsafeRunSync() +body.length +``` + +Now let's see what happens when we wrap the service with `GZip` middleware. +For the purpose of this example, let's create a compressed body using +`Compression` utility from [fs2](https://fs2.io) library. + +```scala mdoc:silent +import fs2._ +import fs2.compression._ + +val compressedEntity = + Stream + .emits(("I repeat myself when I'm under stress. " * 3).getBytes()) + .through(Compression[IO].gzip()) + +val compressedRequest = request.withEntity(compressedEntity) +``` + +Now, similarly to before, let's wrap the service with the `GZip` middleware. + +```scala mdoc:silent +import org.http4s.server.middleware._ +val serviceZip = GZip(service) +``` + +```scala mdoc +// Do not call 'unsafeRun' in your code - see note at bottom. +val respNormal = serviceZip.orNotFound(compressedRequest).unsafeRunSync() +val bodyNormal = respNormal.as[String].unsafeRunSync() +bodyNormal.length +``` + +We can clearly see the middleware didn't do much and that's because the decompression will only be +triggered if the request contains `Content-Encoding` header with a value of **"gzip"** or **"x-gzip"**. + +```scala mdoc +// Do not call 'unsafeRun' in your code - see note at bottom. +val validRequest = compressedRequest.putHeaders("Content-Encoding" -> "gzip") + +val respDecompressed = serviceZip.orNotFound(validRequest).unsafeRunSync() +val bodyDecompressed = respDecompressed.as[String].unsafeRunSync() +bodyDecompressed.length +``` + +This time we can see that the request got decompressed as expected. **NOTE:** In this documentation, we are calling `unsafeRunSync` to extract values out of a service or middleware code. You can work with values while keeping them inside the diff --git a/server/shared/src/main/scala/org/http4s/server/middleware/GZip.scala b/server/shared/src/main/scala/org/http4s/server/middleware/GZip.scala index 388262feabb..827443ba21e 100644 --- a/server/shared/src/main/scala/org/http4s/server/middleware/GZip.scala +++ b/server/shared/src/main/scala/org/http4s/server/middleware/GZip.scala @@ -36,10 +36,11 @@ object GZip extends GZipPlatform { isZippable: Response[G] => Boolean = defaultIsZippable[G](_: Response[G]), ): Http[F, G] = Kleisli { (req: Request[G]) => - req.headers.get[`Accept-Encoding`] match { + val unzippedRequest = unzipOrPass(req, bufferSize) + unzippedRequest.headers.get[`Accept-Encoding`] match { case Some(acceptEncoding) if satisfiedByGzip(acceptEncoding) => - http(req).map(zipOrPass(_, bufferSize, level, isZippable)) - case _ => http(req) + http(unzippedRequest).map(zipOrPass(_, bufferSize, level, isZippable)) + case _ => http(unzippedRequest) } } @@ -51,6 +52,11 @@ object GZip extends GZipPlatform { (contentType.get.mediaType == MediaType.application.`octet-stream`)) } + private def isZipped[F[_]](req: Request[F]): Boolean = + req.headers.get[`Content-Encoding`].map(_.contentCoding).exists { coding => + coding === ContentCoding.gzip || coding === ContentCoding.`x-gzip` + } + private def satisfiedByGzip(acceptEncoding: `Accept-Encoding`) = acceptEncoding.satisfiedBy(ContentCoding.gzip) || acceptEncoding.satisfiedBy( ContentCoding.`x-gzip` @@ -67,6 +73,15 @@ object GZip extends GZipPlatform { case resp => resp // Don't touch it, Content-Encoding already set } + private def unzipOrPass[F[_]: Compression]( + request: Request[F], + bufferSize: Int, + ): Request[F] = + request match { + case req if isZipped(req) => unzipRequest(bufferSize, req) + case req => req + } + private def zipResponse[F[_]: Compression]( bufferSize: Int, level: DeflateParams.Level, @@ -89,4 +104,16 @@ object GZip extends GZipPlatform { .putHeaders(`Content-Encoding`(ContentCoding.gzip)) .pipeBodyThrough(compressPipe) } + + private def unzipRequest[F[_]: Compression]( + bufferSize: Int, + req: Request[F], + ): Request[F] = { + val decompressPipe = Compression[F].gunzip(bufferSize = bufferSize).andThenF(_.content) + logger.trace("GZip middleware decoding content").unsafeRunSync() + req + .removeHeader[`Content-Length`] + .removeHeader[`Content-Encoding`] + .pipeBodyThrough(decompressPipe) + } } diff --git a/server/shared/src/test/scala/org/http4s/server/middleware/GZipSuite.scala b/server/shared/src/test/scala/org/http4s/server/middleware/GZipSuite.scala index 49997df768b..be13ce68a03 100644 --- a/server/shared/src/test/scala/org/http4s/server/middleware/GZipSuite.scala +++ b/server/shared/src/test/scala/org/http4s/server/middleware/GZipSuite.scala @@ -29,6 +29,7 @@ import org.http4s.syntax.all._ import org.scalacheck.Arbitrary.arbitrary import org.scalacheck.Gen import org.scalacheck.effect.PropF +import org.typelevel.ci._ import java.util.Arrays @@ -88,6 +89,44 @@ class GZipSuite extends Http4sSuite { resp.map(!_.headers.contains[`Content-Encoding`]).assert } + test("decodes random content-type if content-encoding allows it") { + val request = "Request string" + val routes: HttpRoutes[IO] = HttpRoutes.of[IO] { case req @ POST -> Root => Ok(req.body) } + val gzipRoutes: HttpRoutes[IO] = GZip(routes, isZippable = _ => false) + + val req: Request[IO] = Request[IO](Method.POST, uri"/") + .putHeaders(Header.Raw(ci"Content-Encoding", "gzip")) + .withBodyStream(Stream.emits(request.getBytes()).through(Compression[IO].gzip())) + + gzipRoutes.orNotFound(req).flatMap { response => + response.body.compile + .to(Chunk) + .map { decoded => + Arrays.equals(request.getBytes(), decoded.toArray) + } + .assert + } + } + + test("doesn't decode request if content-encoding doesn't allow it") { + val request = "Request string" + val routes: HttpRoutes[IO] = HttpRoutes.of[IO] { case req @ POST -> Root => Ok(req.body) } + val gzipRoutes: HttpRoutes[IO] = GZip(routes, isZippable = _ => false) + + val req: Request[IO] = Request[IO](Method.POST, uri"/") + .putHeaders(`Content-Encoding`(ContentCoding.identity)) + .withBodyStream(Stream.emits(request.getBytes())) + + gzipRoutes.orNotFound(req).flatMap { response => + response.body.compile + .to(Chunk) + .map { decoded => + Arrays.equals(request.getBytes(), decoded.toArray) + } + .assert + } + } + test("encoding") { val genByteArray = Gen.poisson(10).flatMap(n => Gen.buildableOfN[Array[Byte], Byte](n, arbitrary[Byte]))