From 24454f7f09260a990c3055c0f5d4eb35e1814cab Mon Sep 17 00:00:00 2001 From: Konrad Malawski Date: Tue, 24 May 2016 10:20:28 +0200 Subject: [PATCH 01/10] +htp #18837 JSON framing and framed entity streaming directives --- .../akka/stream/JsonFramingBenchmark.scala | 61 +++ .../FramedEntityStreamingExamplesTest.java | 33 ++ .../JsonStreamingExamplesSpec.scala | 150 ++++++ .../routing-dsl/json-streaming-support.rst | 96 ++++ .../sprayjson/SprayJsonSupport.scala | 15 +- .../FramedEntityStreamingDirectives.scala | 51 ++ .../http/scaladsl/server/Directives.scala | 1 + .../scaladsl/server/JsonEntityStreaming.scala | 168 +++++++ .../FramedEntityStreamingDirectives.scala | 137 ++++++ .../stream/scaladsl/JsonFramingSpec.scala | 444 ++++++++++++++++++ .../stream/impl/JsonBracketCounting.scala | 150 ++++++ .../akka/stream/javadsl/JsonFraming.scala | 40 ++ .../scala/akka/stream/scaladsl/Framing.scala | 8 + .../akka/stream/scaladsl/JsonFraming.scala | 72 +++ 14 files changed, 1423 insertions(+), 3 deletions(-) create mode 100644 akka-bench-jmh/src/main/scala/akka/stream/JsonFramingBenchmark.scala create mode 100644 akka-docs/rst/java/code/docs/http/javadsl/server/directives/FramedEntityStreamingExamplesTest.java create mode 100644 akka-docs/rst/scala/code/docs/http/scaladsl/server/directives/JsonStreamingExamplesSpec.scala create mode 100644 akka-docs/rst/scala/http/routing-dsl/json-streaming-support.rst create mode 100644 akka-http/src/main/scala/akka/http/javadsl/server/directives/FramedEntityStreamingDirectives.scala create mode 100644 akka-http/src/main/scala/akka/http/scaladsl/server/JsonEntityStreaming.scala create mode 100644 akka-http/src/main/scala/akka/http/scaladsl/server/directives/FramedEntityStreamingDirectives.scala create mode 100644 akka-stream-tests/src/test/scala/akka/stream/scaladsl/JsonFramingSpec.scala create mode 100644 akka-stream/src/main/scala/akka/stream/impl/JsonBracketCounting.scala create mode 100644 akka-stream/src/main/scala/akka/stream/javadsl/JsonFraming.scala create mode 100644 akka-stream/src/main/scala/akka/stream/scaladsl/JsonFraming.scala diff --git a/akka-bench-jmh/src/main/scala/akka/stream/JsonFramingBenchmark.scala b/akka-bench-jmh/src/main/scala/akka/stream/JsonFramingBenchmark.scala new file mode 100644 index 00000000000..6f353ed4d9e --- /dev/null +++ b/akka-bench-jmh/src/main/scala/akka/stream/JsonFramingBenchmark.scala @@ -0,0 +1,61 @@ +/* + * Copyright (C) 2009-2015 Typesafe Inc. + */ +package akka.stream + +import java.util.concurrent.TimeUnit + +import akka.stream.impl.JsonBracketCounting +import akka.util.ByteString +import org.openjdk.jmh.annotations._ + +@State(Scope.Benchmark) +@OutputTimeUnit(TimeUnit.SECONDS) +@BenchmarkMode(Array(Mode.Throughput)) +class JsonFramingBenchmark { + + /* + Benchmark Mode Cnt Score Error Units + // old + JsonFramingBenchmark.collecting_1 thrpt 20 81.476 ± 14.793 ops/s + JsonFramingBenchmark.collecting_offer_5 thrpt 20 20.187 ± 2.291 ops/s + + // new + JsonFramingBenchmark.counting_1 thrpt 20 10766.738 ± 1278.300 ops/s + JsonFramingBenchmark.counting_offer_5 thrpt 20 28798.255 ± 2670.163 ops/s + */ + + val json = + ByteString( + """|{"fname":"Frank","name":"Smith","age":42,"id":1337,"boardMember":false}, + |{"fname":"Bob","name":"Smith","age":42,"id":1337,"boardMember":false}, + |{"fname":"Bob","name":"Smith","age":42,"id":1337,"boardMember":false}, + |{"fname":"Bob","name":"Smith","age":42,"id":1337,"boardMember":false}, + |{"fname":"Bob","name":"Smith","age":42,"id":1337,"boardMember":false}, + |{"fname":"Bob","name":"Smith","age":42,"id":1337,"boardMember":false}, + |{"fname":"Hank","name":"Smith","age":42,"id":1337,"boardMember":false}""".stripMargin) + + val bracket = new JsonBracketCounting + + @Setup(Level.Invocation) + def init(): Unit = { + bracket.offer(json) + } + + @Benchmark + def counting_1: ByteString = + bracket.poll().get + + @Benchmark + @OperationsPerInvocation(5) + def counting_offer_5: ByteString = { + bracket.offer(json) + bracket.poll().get + bracket.poll().get + bracket.poll().get + bracket.poll().get + bracket.poll().get + bracket.poll().get + } + +} diff --git a/akka-docs/rst/java/code/docs/http/javadsl/server/directives/FramedEntityStreamingExamplesTest.java b/akka-docs/rst/java/code/docs/http/javadsl/server/directives/FramedEntityStreamingExamplesTest.java new file mode 100644 index 00000000000..aee10b19359 --- /dev/null +++ b/akka-docs/rst/java/code/docs/http/javadsl/server/directives/FramedEntityStreamingExamplesTest.java @@ -0,0 +1,33 @@ +/* + * Copyright (C) 2015-2016 Lightbend Inc. + */ + +package docs.http.javadsl.server.directives; + +import akka.http.javadsl.model.HttpRequest; +import akka.http.javadsl.model.HttpResponse; +import akka.http.javadsl.server.Route; +import akka.http.javadsl.server.directives.FramedEntityStreamingDirectives; +import akka.http.javadsl.server.directives.LogEntry; +import akka.http.javadsl.testkit.JUnitRouteTest; +import akka.http.scaladsl.server.Rejection; +import org.junit.Test; + +import java.util.List; +import java.util.Optional; +import java.util.function.BiFunction; +import java.util.function.Function; +import java.util.stream.Collectors; + +import static akka.event.Logging.InfoLevel; + +import static akka.http.javadsl.server.directives.FramedEntityStreamingDirectives.*; + +public class FramedEntityStreamingExamplesTest extends JUnitRouteTest { + + @Test + public void testRenderSource() { + FramedEntityStreamingDirectives. + } + +} diff --git a/akka-docs/rst/scala/code/docs/http/scaladsl/server/directives/JsonStreamingExamplesSpec.scala b/akka-docs/rst/scala/code/docs/http/scaladsl/server/directives/JsonStreamingExamplesSpec.scala new file mode 100644 index 00000000000..6a6fffe86fe --- /dev/null +++ b/akka-docs/rst/scala/code/docs/http/scaladsl/server/directives/JsonStreamingExamplesSpec.scala @@ -0,0 +1,150 @@ +/* + * Copyright (C) 2009-2015 Typesafe Inc. + */ + +package docs.http.scaladsl.server.directives + +import akka.NotUsed +import akka.http.scaladsl.marshalling.ToResponseMarshallable +import akka.http.scaladsl.model._ +import akka.http.scaladsl.model.headers.Accept +import akka.http.scaladsl.server.{ UnsupportedRequestContentTypeRejection, UnacceptedResponseContentTypeRejection, JsonSourceRenderingMode } +import akka.stream.scaladsl.{ Flow, Source } +import docs.http.scaladsl.server.RoutingSpec +import spray.json.{ JsValue, JsObject, DefaultJsonProtocol } + +import scala.concurrent.Future + +class JsonStreamingExamplesSpec extends RoutingSpec { + + //#models + case class Tweet(uid: Int, txt: String) + case class Measurement(id: String, value: Int) + //# + + def getTweets() = + Source(List( + Tweet(1, "#Akka rocks!"), + Tweet(2, "Streaming is so hot right now!"), + Tweet(3, "You cannot enter the same river twice."))) + + //#formats + object MyJsonProtocol extends spray.json.DefaultJsonProtocol { + implicit val userFormat = jsonFormat2(Tweet.apply) + implicit val measurementFormat = jsonFormat2(Measurement.apply) + } + //# + + "spray-json-response-streaming" in { + // [1] import generic spray-json marshallers support: + import akka.http.scaladsl.marshallers.sprayjson.SprayJsonSupport._ + + // [2] import "my protocol", for marshalling Tweet objects: + import MyJsonProtocol._ + + // [3] pick json rendering mode: + implicit val jsonRenderingMode = JsonSourceRenderingMode.LineByLine + + val route = + path("users") { + val users: Source[Tweet, NotUsed] = getTweets() + complete(ToResponseMarshallable(users)) + } + + // tests: + val AcceptJson = Accept(MediaRange(MediaTypes.`application/json`)) + val AcceptXml = Accept(MediaRange(MediaTypes.`text/xml`)) + + Get("/users").withHeaders(AcceptJson) ~> route ~> check { + responseAs[String] shouldEqual + """{"uid":1,"txt":"#Akka rocks!"}""" + "\n" + + """{"uid":2,"txt":"Streaming is so hot right now!"}""" + "\n" + + """{"uid":3,"txt":"You cannot enter the same river twice."}""" + } + + // endpoint can only marshal Json, so it will *reject* requests for application/xml: + Get("/users").withHeaders(AcceptXml) ~> route ~> check { + handled should ===(false) + rejection should ===(UnacceptedResponseContentTypeRejection(Set(ContentTypes.`application/json`))) + } + } + + "response-streaming-modes" in { + import akka.http.scaladsl.marshallers.sprayjson.SprayJsonSupport._ + import MyJsonProtocol._ + implicit val jsonRenderingMode = JsonSourceRenderingMode.LineByLine + + //#async-rendering + path("users") { + val users: Source[Tweet, NotUsed] = getTweets() + complete(users.renderAsync(parallelism = 8)) + } + //# + + //#async-unordered-rendering + path("users" / "unordered") { + val users: Source[Tweet, NotUsed] = getTweets() + complete(users.renderAsyncUnordered(parallelism = 8)) + } + //# + } + + "spray-json-request-streaming" in { + // [1] import generic spray-json (un)marshallers support: + import akka.http.scaladsl.marshallers.sprayjson.SprayJsonSupport._ + + // [1.1] import framing mode + implicit val jsonFramingMode = akka.http.scaladsl.server.JsonEntityFramingSupport.bracketCountingJsonFraming(Int.MaxValue) + + // [2] import "my protocol", for unmarshalling Measurement objects: + import MyJsonProtocol._ + + // [3] prepareyour persisting logic here + val persistMetrics = Flow[Measurement] + + val route = + path("metrics") { + // [4] extract Source[Measurement, _] + entity(stream[Measurement]) { measurements => + println("measurements = " + measurements) + val measurementsSubmitted: Future[Int] = + measurements + .via(persistMetrics) + .runFold(0) { (cnt, _) => + println("cnt = " + cnt) + cnt + 1 + } + + complete { + measurementsSubmitted.map(n => Map("msg" -> s"""Total metrics received: $n""")) + } + } + } + + // tests: + val data = HttpEntity( + ContentTypes.`application/json`, + """ + |{"id":"temp","value":32} + |{"id":"temp","value":31} + | + """.stripMargin) + + Post("/metrics", entity = data) ~> route ~> check { + status should ===(StatusCodes.OK) + responseAs[String] should ===("""{"msg":"Total metrics received: 2"}""") + } + + // the FramingWithContentType will reject any content type that it does not understand: + val xmlData = HttpEntity( + ContentTypes.`text/xml(UTF-8)`, + """| + |""".stripMargin) + + Post("/metrics", entity = xmlData) ~> route ~> check { + handled should ===(false) + rejection should ===(UnsupportedRequestContentTypeRejection(Set(ContentTypes.`application/json`))) + } + } + +} diff --git a/akka-docs/rst/scala/http/routing-dsl/json-streaming-support.rst b/akka-docs/rst/scala/http/routing-dsl/json-streaming-support.rst new file mode 100644 index 00000000000..34dda6235b0 --- /dev/null +++ b/akka-docs/rst/scala/http/routing-dsl/json-streaming-support.rst @@ -0,0 +1,96 @@ +.. _json-streaming-scala: + +JSON Streaming +============== + +`JSON Streaming`_ is a term refering to streaming a (possibly infinite) stream of element as independent JSON +objects onto one continious HTTP connection. The elements are most often separated using newlines, +however do not have to be and concatenating elements side-by-side or emitting "very long" JSON array is also another +use case. + +In the below examples, we'll be refering to the ``User`` and ``Measurement`` case classes as our model, which are defined as: + +.. includecode2:: ../../code/docs/http/scaladsl/server/directives/JsonStreamingExamplesSpec.scala + :snippet: models + +And as always with spray-json, we provide our (Un)Marshaller instances as implicit values uding the ``jsonFormat##`` +method to generate them statically: + +.. includecode2:: ../../code/docs/http/scaladsl/server/directives/JsonStreamingExamplesSpec.scala + :snippet: formats + +.. _Json Streaming: https://en.wikipedia.org/wiki/JSON_Streaming + +Responding with JSON Streams +---------------------------- + +In this example we implement an API representing an infinite stream of tweets, very much like Twitter's `Streaming API`_. + +Firstly, we'll need to get some additional marshalling infrastructure set up, that is able to marshal to and from an +Akka Streams ``Source[T,_]``. One such trait, containing the needed marshallers is ``SprayJsonSupport``, which uses +spray-json (a high performance json parser library), and is shipped as part of Akka HTTP in the +``akka-http-spray-json-experimental`` module. +to and from ``Source[T,_]`` by using spray-json provided + +Next we import our model's marshallers, generated by spray-json. + +The last bit of setup, before we can render a streaming json response + +.. includecode2:: ../../code/docs/http/scaladsl/server/directives/JsonStreamingExamplesSpec.scala + :snippet: spray-json-response-streaming + +.. _Streaming API: https://dev.twitter.com/streaming/overview + +Customising response rendering mode +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +The mode in which a response is marshalled and then rendered to the HttpResponse from the provided ``Source[T,_]`` +is customisable (thanks to conversions originating from ``Directives`` via ``EntityStreamingDirectives``). + +Since Marshalling is a potentially asynchronous operation in Akka HTTP (because transforming ``T`` to ``JsValue`` may +potentially take a long time (depending on your definition of "long time"), we allow to run marshalling concurrently +(up to ``parallelism`` concurrent marshallings) by using the ``renderAsync(parallelism)`` mode: + +.. includecode2:: ../../code/docs/http/scaladsl/server/directives/JsonStreamingExamplesSpec.scala + :snippet: async-rendering + +The ``renderAsync`` mode perserves ordering of the Source's elements, which may sometimes be a required property, +for example when streaming a strictly ordered dataset. Sometimes the contept of strict-order does not apply to the +data being streamed though, which allows us to explit this property and use ``renderAsyncUnordered(parallelism)``, +which will concurrently marshall up to ``parallelism`` elements and emit the first which is marshalled onto +the HttpResponse: + +.. includecode2:: ../../code/docs/http/scaladsl/server/directives/JsonStreamingExamplesSpec.scala + :snippet: async-unordered-rendering + +This allows us to _potentially_ render elements faster onto the HttpResponse, since it can avoid "head of line blocking", +in case one element in front of the stream takes a long time to marshall, yet others after it are very quick to marshall. + +Consuming JSON Streaming uploads +-------------------------------- + +Sometimes the client may be sending in a streaming request, for example an embedded device initiated a connection with +the server and is feeding it with one line of measurement data. + +In this example, we want to consume this data in a streaming fashion from the request entity, and also apply +back-pressure to the underlying TCP connection, if the server can not cope with the rate of incoming data (back-pressure +will be applied automatically thanks to using Akka HTTP/Streams). + + +.. includecode2:: ../../code/docs/http/scaladsl/server/directives/JsonStreamingExamplesSpec.scala + :snippet: spray-json-request-streaming + +Implementing custom (Un)Marshaller support for JSON streaming +------------------------------------------------------------- + +While not provided by Akka HTTP directly, the infrastructure is extensible and by investigating how ``SprayJsonSupport`` +is implemented it is certainly possible to provide the same infrastructure for other marshaller implementations (such as +Play JSON, or Jackson directly for example). Such support traits will want to extend the ``JsonEntityStreamingSupport`` trait. + +The following types that may need to be implemented by a custom framed-streaming support library are: + +- ``SourceRenderingMode`` which can customise how to render the begining / between-elements and ending of such stream (while writing a response, i.e. by calling ``complete(source)``). + Implementations for JSON are available in ``akka.http.scaladsl.server.JsonSourceRenderingMode``. +- ``FramingWithContentType`` which is needed to be able to split incoming ``ByteString`` chunks into frames + of the higher-level data type format that is understood by the provided unmarshallers. + In the case of JSON it means chunking up ByteStrings such that each emitted element corresponds to exactly one JSON object, + this framing is implemented in ``JsonEntityStreamingSupport``. diff --git a/akka-http-marshallers-scala/akka-http-spray-json/src/main/scala/akka/http/scaladsl/marshallers/sprayjson/SprayJsonSupport.scala b/akka-http-marshallers-scala/akka-http-spray-json/src/main/scala/akka/http/scaladsl/marshallers/sprayjson/SprayJsonSupport.scala index 99bbb2c2fc2..c62f51314f6 100644 --- a/akka-http-marshallers-scala/akka-http-spray-json/src/main/scala/akka/http/scaladsl/marshallers/sprayjson/SprayJsonSupport.scala +++ b/akka-http-marshallers-scala/akka-http-spray-json/src/main/scala/akka/http/scaladsl/marshallers/sprayjson/SprayJsonSupport.scala @@ -4,10 +4,13 @@ package akka.http.scaladsl.marshallers.sprayjson +import akka.http.scaladsl.util.FastFuture +import akka.util.ByteString + import scala.language.implicitConversions import akka.http.scaladsl.marshalling.{ ToEntityMarshaller, Marshaller } import akka.http.scaladsl.unmarshalling.{ FromEntityUnmarshaller, Unmarshaller } -import akka.http.scaladsl.model.{ MediaTypes, HttpCharsets } +import akka.http.scaladsl.model.{ ContentTypes, MediaTypes, HttpCharsets } import akka.http.scaladsl.model.MediaTypes.`application/json` import spray.json._ @@ -19,6 +22,10 @@ trait SprayJsonSupport { sprayJsonUnmarshaller(reader) implicit def sprayJsonUnmarshaller[T](implicit reader: RootJsonReader[T]): FromEntityUnmarshaller[T] = sprayJsValueUnmarshaller.map(jsonReader[T].read) + implicit def sprayJsonByteStringUnmarshaller[T](implicit reader: RootJsonReader[T]): Unmarshaller[ByteString, T] = + Unmarshaller.withMaterializer[ByteString, JsValue](_ ⇒ implicit mat ⇒ { bs ⇒ + FastFuture.successful(JsonParser(bs.toArray[Byte])) + }).map(jsonReader[T].read) implicit def sprayJsValueUnmarshaller: FromEntityUnmarshaller[JsValue] = Unmarshaller.byteStringUnmarshaller.forContentTypes(`application/json`).mapWithCharset { (data, charset) ⇒ val input = @@ -29,9 +36,11 @@ trait SprayJsonSupport { implicit def sprayJsonMarshallerConverter[T](writer: RootJsonWriter[T])(implicit printer: JsonPrinter = PrettyPrinter): ToEntityMarshaller[T] = sprayJsonMarshaller[T](writer, printer) - implicit def sprayJsonMarshaller[T](implicit writer: RootJsonWriter[T], printer: JsonPrinter = PrettyPrinter): ToEntityMarshaller[T] = + implicit def sprayJsonMarshaller[T](implicit writer: RootJsonWriter[T], printer: JsonPrinter = CompactPrinter): ToEntityMarshaller[T] = sprayJsValueMarshaller compose writer.write implicit def sprayJsValueMarshaller(implicit printer: JsonPrinter = PrettyPrinter): ToEntityMarshaller[JsValue] = Marshaller.StringMarshaller.wrap(MediaTypes.`application/json`)(printer) + implicit def sprayByteStringMarshaller[T](implicit writer: RootJsonFormat[T], printer: JsonPrinter = CompactPrinter): Marshaller[T, ByteString] = + sprayJsValueMarshaller.map(s ⇒ ByteString(s.toString)) compose writer.write } -object SprayJsonSupport extends SprayJsonSupport \ No newline at end of file +object SprayJsonSupport extends SprayJsonSupport diff --git a/akka-http/src/main/scala/akka/http/javadsl/server/directives/FramedEntityStreamingDirectives.scala b/akka-http/src/main/scala/akka/http/javadsl/server/directives/FramedEntityStreamingDirectives.scala new file mode 100644 index 00000000000..ec05f046e8c --- /dev/null +++ b/akka-http/src/main/scala/akka/http/javadsl/server/directives/FramedEntityStreamingDirectives.scala @@ -0,0 +1,51 @@ +/* + * Copyright (C) 2009-2015 Typesafe Inc. + */ +package akka.http.javadsl.server.directives + +import akka.http.javadsl.model.{ContentType, HttpEntity} +import akka.util.ByteString +import java.util.{List => JList, Map => JMap} +import java.util.AbstractMap.SimpleImmutableEntry +import java.util.Optional +import java.util.function.{Function => JFunction} + +import akka.NotUsed + +import scala.collection.JavaConverters._ +import akka.http.impl.util.JavaMapping.Implicits._ +import akka.http.javadsl.server.directives.FramedEntityStreamingDirectives.SourceRenderingMode +import akka.http.javadsl.server.{Route, Unmarshaller} +import akka.http.scaladsl.marshalling.ToResponseMarshallable +import akka.http.scaladsl.server.{FramingWithContentType, Directives => D} +import akka.http.scaladsl.server.directives.ParameterDirectives._ +import akka.stream.javadsl.Source + +import scala.compat.java8.OptionConverters + +/** EXPERIMENTAL API */ +trait FramedEntityStreamingDirectives { + + def entityAsStream[T](clazz: Class[T], um: Unmarshaller[_ >: HttpEntity, T], framing: FramingWithContentType, + inner: java.util.function.Function[Source[T, NotUsed], Route]): Route = RouteAdapter { + D.entity[T](D.stream[T](um, framing)) { s => + + } + ??? + } + + def completeWithSource[T](source: Source[T, Any], rendering: SourceRenderingMode): Route = RouteAdapter { + val response = ToResponseMarshallable(source) + D.complete(response) + } +} + +object FramedEntityStreamingDirectives extends FramedEntityStreamingDirectives { + trait SourceRenderingMode { + def getContentType: ContentType + + def start: ByteString + def between: ByteString + def end: ByteString + } +} diff --git a/akka-http/src/main/scala/akka/http/scaladsl/server/Directives.scala b/akka-http/src/main/scala/akka/http/scaladsl/server/Directives.scala index b92bb8d97ad..489ea7375a6 100644 --- a/akka-http/src/main/scala/akka/http/scaladsl/server/Directives.scala +++ b/akka-http/src/main/scala/akka/http/scaladsl/server/Directives.scala @@ -36,5 +36,6 @@ trait Directives extends RouteConcatenation with SchemeDirectives with SecurityDirectives with WebSocketDirectives + with FramedEntityStreamingDirectives object Directives extends Directives diff --git a/akka-http/src/main/scala/akka/http/scaladsl/server/JsonEntityStreaming.scala b/akka-http/src/main/scala/akka/http/scaladsl/server/JsonEntityStreaming.scala new file mode 100644 index 00000000000..228aaf96a82 --- /dev/null +++ b/akka-http/src/main/scala/akka/http/scaladsl/server/JsonEntityStreaming.scala @@ -0,0 +1,168 @@ +/* + * Copyright (C) 2009-2015 Typesafe Inc. + */ +package akka.http.scaladsl.server + +import akka.NotUsed +import akka.actor.ActorSystem +import akka.http.impl.util.JavaMapping +import akka.http.scaladsl.model.{ ContentType, ContentTypes } +import akka.http.scaladsl.server.directives.FramedEntityStreamingDirectives.SourceRenderingMode +import akka.stream.scaladsl.{ Flow, Framing } +import akka.util.ByteString +import com.typesafe.config.Config + +import scala.collection.immutable + +/** + * Same as [[akka.stream.scaladsl.Framing]] but additionally can express which [[ContentType]] it supports, + * which can be used to reject routes if content type does not match used framing. + */ +abstract class FramingWithContentType extends Framing { + def flow: Flow[ByteString, ByteString, NotUsed] + def supported: immutable.Set[ContentType] + def isSupported(ct: akka.http.javadsl.model.ContentType): Boolean = supported(JavaMapping.ContentType.toScala(ct)) +} +object FramingWithContentType { + def apply(framing: Flow[ByteString, ByteString, NotUsed], contentType: ContentType, moreContentTypes: ContentType*) = + new FramingWithContentType { + override def flow = framing + + override val supported: immutable.Set[ContentType] = + if (moreContentTypes.isEmpty) Set(contentType) + else Set(contentType) ++ moreContentTypes + } +} + +/** + * Json entity streaming support, independent of used Json parsing library. + * + * Can be extended by various Support traits (e.g. "SprayJsonSupport"), + * in order to provide users with both `framing` (this trait) and `marshalling` + * (implemented by a library) by using a single trait. + */ +trait JsonEntityFramingSupport { + + /** `application/json` specific Framing implementation */ + def bracketCountingJsonFraming(maximumObjectLength: Int) = new FramingWithContentType { + override final val flow = Flow[ByteString].via(akka.stream.scaladsl.JsonFraming.bracketCounting(maximumObjectLength)) + + override val supported: immutable.Set[ContentType] = Set(ContentTypes.`application/json`) + } +} +object JsonEntityFramingSupport extends JsonEntityFramingSupport + +/** + * Specialised rendering mode for streaming elements as JSON. + * + * See also: JSON Streaming on Wikipedia. + */ +trait JsonSourceRenderingMode extends SourceRenderingMode { + override val contentType = ContentTypes.`application/json` +} + +object JsonSourceRenderingMode { + + /** + * Most compact rendering mode + * It does not intersperse any separator between the signalled elements. + * + * {{{ + * {"id":42}{"id":43}{"id":44} + * }}} + */ + object Compact extends JsonSourceRenderingMode { + override val start: ByteString = ByteString.empty + override val between: ByteString = ByteString.empty + override val end: ByteString = ByteString.empty + } + + /** + * Simple rendering mode, similar to [[Compact]] however interspersing elements with a `\n` character. + * + * {{{ + * {"id":42},{"id":43},{"id":44} + * }}} + */ + object CompactCommaSeparated extends JsonSourceRenderingMode { + override val start: ByteString = ByteString.empty + override val between: ByteString = ByteString(",") + override val end: ByteString = ByteString.empty + } + + /** + * Rendering mode useful when the receiving end expects a valid JSON Array. + * It can be useful when the client wants to detect when the stream has been successfully received in-full, + * which it can determine by seeing the terminating `]` character. + * + * The framing's terminal `]` will ONLY be emitted if the stream has completed successfully, + * in other words - the stream has been emitted completely, without errors occuring before the final element has been signaled. + * + * {{{ + * [{"id":42},{"id":43},{"id":44}] + * }}} + */ + object CompactArray extends JsonSourceRenderingMode { + override val start: ByteString = ByteString("[") + override val between: ByteString = ByteString(",") + override val end: ByteString = ByteString("]") + } + + /** + * Recommended rendering mode. + * + * It is a nice balance between valid and human-readable as well as resonably small size overhead (just the `\n` between elements). + * A good example of API's using this syntax is Twitter's Firehose (last verified at 1.1 version of that API). + * + * {{{ + * {"id":42} + * {"id":43} + * {"id":44} + * }}} + */ + object LineByLine extends JsonSourceRenderingMode { + override val start: ByteString = ByteString.empty + override val between: ByteString = ByteString("\n") + override val end: ByteString = ByteString.empty + } + + /** + * Simple rendering mode interspersing each pair of elements with both `,\n`. + * Picking the [[LineByLine]] format may be preferable, as it is slightly simpler to parse - each line being a valid json object (no need to trim the comma). + * + * {{{ + * {"id":42}, + * {"id":43}, + * {"id":44} + * }}} + */ + object LineByLineCommaSeparated extends JsonSourceRenderingMode { + override val start: ByteString = ByteString.empty + override val between: ByteString = ByteString(",\n") + override val end: ByteString = ByteString.empty + } + +} + +object JsonStreamingSettings { + + def apply(sys: ActorSystem): JsonStreamingSettings = + apply(sys.settings.config.getConfig("akka.http.json-streaming")) + + def apply(c: Config): JsonStreamingSettings = { + JsonStreamingSettings( + c.getInt("max-object-size"), + renderingMode(c.getString("rendering-mode"))) + } + + def renderingMode(name: String): SourceRenderingMode = name match { + case "line-by-line" ⇒ JsonSourceRenderingMode.LineByLine // the default + case "line-by-line-comma-separated" ⇒ JsonSourceRenderingMode.LineByLineCommaSeparated + case "compact" ⇒ JsonSourceRenderingMode.Compact + case "compact-comma-separated" ⇒ JsonSourceRenderingMode.CompactCommaSeparated + case "compact-array" ⇒ JsonSourceRenderingMode.CompactArray + } +} +final case class JsonStreamingSettings( + maxObjectSize: Int, + style: SourceRenderingMode) diff --git a/akka-http/src/main/scala/akka/http/scaladsl/server/directives/FramedEntityStreamingDirectives.scala b/akka-http/src/main/scala/akka/http/scaladsl/server/directives/FramedEntityStreamingDirectives.scala new file mode 100644 index 00000000000..e9a6996a966 --- /dev/null +++ b/akka-http/src/main/scala/akka/http/scaladsl/server/directives/FramedEntityStreamingDirectives.scala @@ -0,0 +1,137 @@ +/* + * Copyright (C) 2009-2015 Typesafe Inc. + */ +package akka.http.scaladsl.server.directives + +import akka.NotUsed +import akka.http.scaladsl.marshalling._ +import akka.http.scaladsl.model._ +import akka.http.scaladsl.server.FramingWithContentType +import akka.http.scaladsl.unmarshalling.{ Unmarshal, Unmarshaller, _ } +import akka.http.scaladsl.util.FastFuture +import akka.stream.Materializer +import akka.stream.impl.ConstantFun +import akka.stream.scaladsl.{ Flow, Source } +import akka.util.ByteString + +import scala.concurrent.ExecutionContext +import scala.language.implicitConversions + +/** + * Allows the [[MarshallingDirectives.entity]] directive to extract a `stream[T]` for framed messages. + * See `JsonEntityStreamingSupport` and classes extending it, such as `SprayJsonSupport` to get marshallers. + */ +trait FramedEntityStreamingDirectives extends MarshallingDirectives { + import FramedEntityStreamingDirectives._ + + type RequestToSourceUnmarshaller[T] = FromRequestUnmarshaller[Source[T, NotUsed]] + + // TODO DOCS + + final def stream[T](implicit um: Unmarshaller[ByteString, T], framing: FramingWithContentType): RequestToSourceUnmarshaller[T] = + streamAsync(1)(um, framing) + final def stream[T](framing: FramingWithContentType)(implicit um: Unmarshaller[ByteString, T]): RequestToSourceUnmarshaller[T] = + streamAsync(1)(um, framing) + + final def streamAsync[T](parallelism: Int)(implicit um: Unmarshaller[ByteString, T], framing: FramingWithContentType): RequestToSourceUnmarshaller[T] = + streamInternal[T](framing, (ec, mat) ⇒ Flow[ByteString].mapAsync(parallelism)(Unmarshal(_).to[T](um, ec, mat))) + final def streamAsync[T](parallelism: Int, framing: FramingWithContentType)(implicit um: Unmarshaller[ByteString, T]): RequestToSourceUnmarshaller[T] = + streamAsync(parallelism)(um, framing) + + final def streamAsyncUnordered[T](parallelism: Int)(implicit um: Unmarshaller[ByteString, T], framing: FramingWithContentType): RequestToSourceUnmarshaller[T] = + streamInternal[T](framing, (ec, mat) ⇒ Flow[ByteString].mapAsyncUnordered(parallelism)(Unmarshal(_).to[T](um, ec, mat))) + final def streamAsyncUnordered[T](parallelism: Int, framing: FramingWithContentType)(implicit um: Unmarshaller[ByteString, T]): RequestToSourceUnmarshaller[T] = + streamAsyncUnordered(parallelism)(um, framing) + + // TODO materialized value may want to be "drain/cancel" or something like it? + // TODO could expose `streamMat`, for more fine grained picking of Marshaller + + // format: OFF + private def streamInternal[T](framing: FramingWithContentType, marshalling: (ExecutionContext, Materializer) => Flow[ByteString, ByteString, NotUsed]#ReprMat[T, NotUsed]): RequestToSourceUnmarshaller[T] = + Unmarshaller.withMaterializer[HttpRequest, Source[T, NotUsed]] { implicit ec ⇒ implicit mat ⇒ req ⇒ + val entity = req.entity + if (!framing.supported(entity.contentType)) { + val supportedContentTypes = framing.supported.map(ContentTypeRange(_)) + FastFuture.failed(Unmarshaller.UnsupportedContentTypeException(supportedContentTypes)) + } else { + val stream = entity.dataBytes.via(framing.flow).via(marshalling(ec, mat)).mapMaterializedValue(_ => NotUsed) + FastFuture.successful(stream) + } + } + // format: ON + + // TODO note to self - we need the same of ease of streaming stuff for the client side - i.e. the twitter firehose case. + + implicit def _sourceMarshaller[T, M](implicit m: ToEntityMarshaller[T], mode: SourceRenderingMode): ToResponseMarshaller[Source[T, M]] = + Marshaller[Source[T, M], HttpResponse] { implicit ec ⇒ source ⇒ + FastFuture successful { + Marshalling.WithFixedContentType(mode.contentType, () ⇒ { // TODO charset? + val bytes = source + .mapAsync(1)(t ⇒ Marshal(t).to[HttpEntity]) + .map(_.dataBytes) + .flatMapConcat(ConstantFun.scalaIdentityFunction) + .intersperse(mode.start, mode.between, mode.end) + HttpResponse(entity = HttpEntity(mode.contentType, bytes)) + }) :: Nil + } + } + + implicit def _sourceParallelismMarshaller[T](implicit m: ToEntityMarshaller[T], mode: SourceRenderingMode): ToResponseMarshaller[AsyncRenderingOf[T]] = + Marshaller[AsyncRenderingOf[T], HttpResponse] { implicit ec ⇒ rendering ⇒ + FastFuture successful { + Marshalling.WithFixedContentType(mode.contentType, () ⇒ { // TODO charset? + val bytes = rendering.source + .mapAsync(rendering.parallelism)(t ⇒ Marshal(t).to[HttpEntity]) + .map(_.dataBytes) + .flatMapConcat(ConstantFun.scalaIdentityFunction) + .intersperse(mode.start, mode.between, mode.end) + HttpResponse(entity = HttpEntity(mode.contentType, bytes)) + }) :: Nil + } + } + + implicit def _sourceUnorderedMarshaller[T](implicit m: ToEntityMarshaller[T], mode: SourceRenderingMode): ToResponseMarshaller[AsyncUnorderedRenderingOf[T]] = + Marshaller[AsyncUnorderedRenderingOf[T], HttpResponse] { implicit ec ⇒ rendering ⇒ + FastFuture successful { + Marshalling.WithFixedContentType(mode.contentType, () ⇒ { // TODO charset? + val bytes = rendering.source + .mapAsync(rendering.parallelism)(t ⇒ Marshal(t).to[HttpEntity]) + .map(_.dataBytes) + .flatMapConcat(ConstantFun.scalaIdentityFunction) + .intersperse(mode.start, mode.between, mode.end) + HttpResponse(entity = HttpEntity(mode.contentType, bytes)) + }) :: Nil + } + } + + // special rendering modes + + implicit def enableSpecialSourceRenderingModes[T](source: Source[T, Any]): EnableSpecialSourceRenderingModes[T] = + new EnableSpecialSourceRenderingModes(source) + +} +object FramedEntityStreamingDirectives extends FramedEntityStreamingDirectives { + /** + * Defines ByteStrings to be injected before the first, between, and after all elements of a [[Source]], + * when used to complete a request. + * + * A typical example would be rendering a ``Source[T, _]`` as JSON array, + * where start is `[`, between is `,`, and end is `]` - which procudes a valid json array, assuming each element can + * be properly marshalled as JSON object. + * + * The corresponding values will typically be put into an [[Source.intersperse]] call on the to-be-rendered Source. + */ + trait SourceRenderingMode extends akka.http.javadsl.server.directives.FramedEntityStreamingDirectives.SourceRenderingMode { + override final def getContentType = contentType + def contentType: ContentType + } + + final class AsyncRenderingOf[T](val source: Source[T, Any], val parallelism: Int) + final class AsyncUnorderedRenderingOf[T](val source: Source[T, Any], val parallelism: Int) + +} + +final class EnableSpecialSourceRenderingModes[T](val source: Source[T, Any]) extends AnyVal { + def renderAsync(parallelism: Int) = new FramedEntityStreamingDirectives.AsyncRenderingOf(source, parallelism) + def renderAsyncUnordered(parallelism: Int) = new FramedEntityStreamingDirectives.AsyncUnorderedRenderingOf(source, parallelism) +} diff --git a/akka-stream-tests/src/test/scala/akka/stream/scaladsl/JsonFramingSpec.scala b/akka-stream-tests/src/test/scala/akka/stream/scaladsl/JsonFramingSpec.scala new file mode 100644 index 00000000000..ccd58690dde --- /dev/null +++ b/akka-stream-tests/src/test/scala/akka/stream/scaladsl/JsonFramingSpec.scala @@ -0,0 +1,444 @@ +/* + * Copyright (C) 2009-2015 Typesafe Inc. + */ +package akka.stream.scaladsl + +import akka.stream.ActorMaterializer +import akka.stream.impl.JsonBracketCounting +import akka.stream.scaladsl.Framing.FramingException +import akka.stream.scaladsl.{ JsonFraming, Framing, Source } +import akka.stream.testkit.scaladsl.TestSink +import akka.testkit.AkkaSpec +import akka.util.ByteString +import org.scalatest.concurrent.ScalaFutures + +import scala.collection.immutable.Seq +import scala.concurrent.Await +import scala.concurrent.duration._ + +class JsonFramingSpec extends AkkaSpec { + + implicit val mat = ActorMaterializer() + + "collecting multiple json" should { + "xoxo parse json array" in { + val input = + """ + |[ + | { "name" : "john" }, + | { "name" : "jack" }, + | { "name" : "katie" } + |] + |""".stripMargin // also should complete once notices end of array + + val result = Source.single(ByteString(input)) + .via(JsonFraming.bracketCounting(Int.MaxValue)) + .runFold(Seq.empty[String]) { + case (acc, entry) ⇒ acc ++ Seq(entry.utf8String) + } + + result.futureValue shouldBe Seq( + """{ "name" : "john" }""".stripMargin, + """{ "name" : "jack" }""".stripMargin, + """{ "name" : "katie" }""".stripMargin) + } + + "emit single json element from string" in { + val input = + """| { "name": "john" } + | { "name": "jack" } + """.stripMargin + + val result = Source.single(ByteString(input)) + .via(JsonFraming.bracketCounting(Int.MaxValue)) + .take(1) + .runFold(Seq.empty[String]) { + case (acc, entry) ⇒ acc ++ Seq(entry.utf8String) + } + + Await.result(result, 3.seconds) shouldBe Seq("""{ "name": "john" }""".stripMargin) + } + + "parse line delimited" in { + val input = + """| { "name": "john" } + | { "name": "jack" } + | { "name": "katie" } + """.stripMargin + + val result = Source.single(ByteString(input)) + .via(JsonFraming.bracketCounting(Int.MaxValue)) + .runFold(Seq.empty[String]) { + case (acc, entry) ⇒ acc ++ Seq(entry.utf8String) + } + + Await.result(result, 3.seconds) shouldBe Seq( + """{ "name": "john" }""".stripMargin, + """{ "name": "jack" }""".stripMargin, + """{ "name": "katie" }""".stripMargin) + } + + "parse comma delimited" in { + val input = + """ + | { "name": "john" }, { "name": "jack" }, { "name": "katie" } + """.stripMargin + + val result = Source.single(ByteString(input)) + .via(JsonFraming.bracketCounting(Int.MaxValue)) + .runFold(Seq.empty[String]) { + case (acc, entry) ⇒ acc ++ Seq(entry.utf8String) + } + + result.futureValue shouldBe Seq( + """{ "name": "john" }""".stripMargin, + """{ "name": "jack" }""", + """{ "name": "katie" }""") + } + + "parse chunks successfully" in { + val input: Seq[ByteString] = Seq( + """ + |[ + | { "name": "john"""".stripMargin, + """ + |}, + """.stripMargin, + """{ "na""", + """me": "jack""", + """"}]"""").map(ByteString(_)) + + val result = Source.apply(input) + .via(JsonFraming.bracketCounting(Int.MaxValue)) + .runFold(Seq.empty[String]) { + case (acc, entry) ⇒ acc ++ Seq(entry.utf8String) + } + + result.futureValue shouldBe Seq( + """{ "name": "john" + |}""".stripMargin, + """{ "name": "jack"}""") + } + } + + // TODO fold these specs into the previous section + "collecting json buffer" when { + "nothing is supplied" should { + "return nothing" in { + val buffer = new JsonBracketCounting() + buffer.poll() should ===(None) + } + } + + "valid json is supplied" which { + "has one object" should { + "successfully parse empty object" in { + val buffer = new JsonBracketCounting() + buffer.offer(ByteString("""{}""")) + buffer.poll().get.utf8String shouldBe """{}""" + } + + "successfully parse single field having string value" in { + val buffer = new JsonBracketCounting() + buffer.offer(ByteString("""{ "name": "john"}""")) + buffer.poll().get.utf8String shouldBe """{ "name": "john"}""" + } + + "successfully parse single field having string value containing space" in { + val buffer = new JsonBracketCounting() + buffer.offer(ByteString("""{ "name": "john doe"}""")) + buffer.poll().get.utf8String shouldBe """{ "name": "john doe"}""" + } + + "successfully parse single field having string value containing curly brace" in { + val buffer = new JsonBracketCounting() + + buffer.offer(ByteString("""{ "name": "john{""")) + buffer.offer(ByteString("}")) + buffer.offer(ByteString("\"")) + buffer.offer(ByteString("}")) + + buffer.poll().get.utf8String shouldBe """{ "name": "john{}"}""" + } + + "successfully parse single field having string value containing curly brace and escape character" in { + val buffer = new JsonBracketCounting() + + buffer.offer(ByteString("""{ "name": "john""")) + buffer.offer(ByteString("\\\"")) + buffer.offer(ByteString("{")) + buffer.offer(ByteString("}")) + buffer.offer(ByteString("\\\"")) + buffer.offer(ByteString(" ")) + buffer.offer(ByteString("hey")) + buffer.offer(ByteString("\"")) + + buffer.offer(ByteString("}")) + buffer.poll().get.utf8String shouldBe """{ "name": "john\"{}\" hey"}""" + } + + "successfully parse single field having integer value" in { + val buffer = new JsonBracketCounting() + buffer.offer(ByteString("""{ "age": 101}""")) + buffer.poll().get.utf8String shouldBe """{ "age": 101}""" + } + + "successfully parse single field having decimal value" in { + val buffer = new JsonBracketCounting() + buffer.offer(ByteString("""{ "age": 101}""")) + buffer.poll().get.utf8String shouldBe """{ "age": 101}""" + } + + "successfully parse single field having nested object" in { + val buffer = new JsonBracketCounting() + buffer.offer(ByteString( + """ + |{ "name": "john", + | "age": 101, + | "address": { + | "street": "Straight Street", + | "postcode": 1234 + | } + |} + | """.stripMargin)) + buffer.poll().get.utf8String shouldBe """{ "name": "john", + | "age": 101, + | "address": { + | "street": "Straight Street", + | "postcode": 1234 + | } + |}""".stripMargin + } + + "successfully parse single field having multiple level of nested object" in { + val buffer = new JsonBracketCounting() + buffer.offer(ByteString( + """ + |{ "name": "john", + | "age": 101, + | "address": { + | "street": { + | "name": "Straight", + | "type": "Avenue" + | }, + | "postcode": 1234 + | } + |} + | """.stripMargin)) + buffer.poll().get.utf8String shouldBe """{ "name": "john", + | "age": 101, + | "address": { + | "street": { + | "name": "Straight", + | "type": "Avenue" + | }, + | "postcode": 1234 + | } + |}""".stripMargin + } + } + + "has nested array" should { + "successfully parse" in { + val buffer = new JsonBracketCounting() + buffer.offer(ByteString( + """ + |{ "name": "john", + | "things": [ + | 1, + | "hey", + | 3, + | "there" + | ] + |} + | """.stripMargin)) + buffer.poll().get.utf8String shouldBe """{ "name": "john", + | "things": [ + | 1, + | "hey", + | 3, + | "there" + | ] + |}""".stripMargin + } + } + + "has complex object graph" should { + "successfully parse" in { + val buffer = new JsonBracketCounting() + buffer.offer(ByteString( + """ + |{ + | "name": "john", + | "addresses": [ + | { + | "street": "3 Hopson Street", + | "postcode": "ABC-123", + | "tags": ["work", "office"], + | "contactTime": [ + | {"time": "0900-1800", "timezone", "UTC"} + | ] + | }, + | { + | "street": "12 Adielie Road", + | "postcode": "ZZY-888", + | "tags": ["home"], + | "contactTime": [ + | {"time": "0800-0830", "timezone", "UTC"}, + | {"time": "1800-2000", "timezone", "UTC"} + | ] + | } + | ] + |} + | """.stripMargin)) + + buffer.poll().get.utf8String shouldBe """{ + | "name": "john", + | "addresses": [ + | { + | "street": "3 Hopson Street", + | "postcode": "ABC-123", + | "tags": ["work", "office"], + | "contactTime": [ + | {"time": "0900-1800", "timezone", "UTC"} + | ] + | }, + | { + | "street": "12 Adielie Road", + | "postcode": "ZZY-888", + | "tags": ["home"], + | "contactTime": [ + | {"time": "0800-0830", "timezone", "UTC"}, + | {"time": "1800-2000", "timezone", "UTC"} + | ] + | } + | ] + |}""".stripMargin + } + } + + "has multiple fields" should { + "parse successfully" in { + val buffer = new JsonBracketCounting() + buffer.offer(ByteString("""{ "name": "john", "age": 101}""")) + buffer.poll().get.utf8String shouldBe """{ "name": "john", "age": 101}""" + } + + "parse successfully despite valid whitespaces around json" in { + val buffer = new JsonBracketCounting() + buffer.offer(ByteString( + """ + | + | + |{"name": "john" + |, "age": 101}""".stripMargin)) + buffer.poll().get.utf8String shouldBe + """{"name": "john" + |, "age": 101}""".stripMargin + } + } + + "has multiple objects" should { + "pops the right object as buffer is filled" in { + val input = + """ + | { + | "name": "john", + | "age": 32 + | }, + | { + | "name": "katie", + | "age": 25 + | } + """.stripMargin + + val buffer = new JsonBracketCounting() + buffer.offer(ByteString(input)) + + buffer.poll().get.utf8String shouldBe + """{ + | "name": "john", + | "age": 32 + | }""".stripMargin + buffer.poll().get.utf8String shouldBe + """{ + | "name": "katie", + | "age": 25 + | }""".stripMargin + buffer.poll() should ===(None) + + buffer.offer(ByteString("""{"name":"jenkins","age": """)) + buffer.poll() should ===(None) + + buffer.offer(ByteString("65 }")) + buffer.poll().get.utf8String shouldBe """{"name":"jenkins","age": 65 }""" + } + } + + "returns none until valid json is encountered" in { + val buffer = new JsonBracketCounting() + + """{ "name": "john"""".stripMargin.foreach { + c ⇒ + buffer.offer(ByteString(c)) + buffer.poll() should ===(None) + } + + buffer.offer(ByteString("}")) + buffer.poll().get.utf8String shouldBe """{ "name": "john"}""" + } + + "invalid json is supplied" should { + "fail if it's broken from the start" in { + val buffer = new JsonBracketCounting() + buffer.offer(ByteString("""THIS IS NOT VALID { "name": "john"}""")) + a[FramingException] shouldBe thrownBy { buffer.poll() } + } + + "fail if it's broken at the end" in { + val buffer = new JsonBracketCounting() + buffer.offer(ByteString("""{ "name": "john"} THIS IS NOT VALID""")) + buffer.poll() // first emitting the valid element + a[FramingException] shouldBe thrownBy { buffer.poll() } + } + } + } + + "fail on too large initial object" in { + val input = + """ + | { "name": "john" }, { "name": "jack" } + """.stripMargin + + val result = Source.single(ByteString(input)) + .via(JsonFraming.bracketCounting(5)).map(_.utf8String) + .runFold(Seq.empty[String]) { + case (acc, entry) ⇒ acc ++ Seq(entry) + } + + a[FramingException] shouldBe thrownBy { + Await.result(result, 3.seconds) + } + } + + "fail when 2nd object is too large" in { + val input = List( + """{ "name": "john" }""", + """{ "name": "jack" }""", + """{ "name": "very very long name somehow. how did this happen?" }""").map(s ⇒ ByteString(s)) + + val probe = Source(input) + .via(JsonFraming.bracketCounting(48)) + .runWith(TestSink.probe) + + probe.ensureSubscription() + probe + .request(1) + .expectNext(ByteString("""{ "name": "john" }""")) // FIXME we should not impact the given json in Framing + .request(1) + .expectNext(ByteString("""{ "name": "jack" }""")) + .request(1) + .expectError().getMessage should include("exceeded") + } + } +} diff --git a/akka-stream/src/main/scala/akka/stream/impl/JsonBracketCounting.scala b/akka-stream/src/main/scala/akka/stream/impl/JsonBracketCounting.scala new file mode 100644 index 00000000000..7bd9e18ca27 --- /dev/null +++ b/akka-stream/src/main/scala/akka/stream/impl/JsonBracketCounting.scala @@ -0,0 +1,150 @@ +/** + * Copyright (C) 2014-2015 Typesafe Inc. + */ +package akka.stream.impl + +import akka.stream.scaladsl.Framing.FramingException +import akka.util.ByteString + +import scala.annotation.switch + +object JsonBracketCounting { + + final val SquareBraceStart = "[".getBytes.head + final val SquareBraceEnd = "]".getBytes.head + final val CurlyBraceStart = "{".getBytes.head + final val CurlyBraceEnd = "}".getBytes.head + final val DoubleQuote = "\"".getBytes.head + final val Backslash = "\\".getBytes.head + final val Comma = ",".getBytes.head + + final val LineBreak = '\n'.toByte + final val LineBreak2 = '\r'.toByte + final val Tab = '\t'.toByte + final val Space = ' '.toByte + + final val Whitespace = Set(LineBreak, LineBreak2, Tab, Space) + + def isWhitespace(input: Byte): Boolean = + Whitespace.contains(input) + +} + +/** + * **Mutable** framing implementation that given any number of [[ByteString]] chunks, can emit JSON objects contained within them. + * Typically JSON objects are separated by new-lines or comas, however a top-level JSON Array can also be understood and chunked up + * into valid JSON objects by this framing implementation. + * + * Leading whitespace between elements will be trimmed. + */ +class JsonBracketCounting(maximumObjectLength: Int = Int.MaxValue) { + import JsonBracketCounting._ + + private var buffer: ByteString = ByteString.empty + + private var pos = 0 // latest position of pointer while scanning for json object end + private var trimFront = 0 // number of chars to drop from the front of the bytestring before emitting (skip whitespace etc) + private var depth = 0 // counter of object-nesting depth, once hits 0 an object should be emitted + + private var charsInObject = 0 + private var completedObject = false + private var inStringExpression = false + private var isStartOfEscapeSequence = false + + /** + * Appends input ByteString to internal byte string buffer. + * Use [[poll]] to extract contained JSON objects. + */ + def offer(input: ByteString): Unit = + buffer ++= input + + def isEmpty: Boolean = buffer.isEmpty + + /** + * Attempt to locate next complete JSON object in buffered ByteString and returns `Some(it)` if found. + * May throw a [[akka.stream.scaladsl.Framing.FramingException]] if the contained JSON is invalid or max object size is exceeded. + */ + def poll(): Option[ByteString] = { + val foundObject = seekObject() + if (!foundObject) None + else + (pos: @switch) match { + case -1 | 0 ⇒ None + case _ ⇒ + val (emit, buf) = buffer.splitAt(pos) + buffer = buf.compact + pos = 0 + + val tf = trimFront + trimFront = 0 + + if (tf == 0) Some(emit) + else { + val trimmed = emit.drop(tf) + if (trimmed.isEmpty) None + else Some(trimmed) + } + } + } + + /** @return true if an entire valid JSON object was found, false otherwise */ + private def seekObject(): Boolean = { + completedObject = false + val bufSize = buffer.size + while (pos != -1 && (pos < bufSize && pos < maximumObjectLength) && !completedObject) + proceed(buffer(pos)) + + if (pos >= maximumObjectLength) + throw new FramingException(s"""JSON element exceeded maximumObjectLength ($maximumObjectLength bytes)!""") + + completedObject + } + + private def proceed(input: Byte): Unit = + if (input == SquareBraceStart && outsideObject) { + // outer object is an array + pos += 1 + trimFront += 1 + } else if (input == SquareBraceEnd && outsideObject) { + // outer array completed! + pos = -1 + } else if (input == Comma && outsideObject) { + // do nothing + pos += 1 + trimFront += 1 + } else if (input == Backslash) { + isStartOfEscapeSequence = true + pos += 1 + } else if (input == DoubleQuote) { + if (!isStartOfEscapeSequence) inStringExpression = !inStringExpression + isStartOfEscapeSequence = false + pos += 1 + } else if (input == CurlyBraceStart && !inStringExpression) { + isStartOfEscapeSequence = false + depth += 1 + pos += 1 + } else if (input == CurlyBraceEnd && !inStringExpression) { + isStartOfEscapeSequence = false + depth -= 1 + pos += 1 + if (depth == 0) { + charsInObject = 0 + completedObject = true + } + } else if (isWhitespace(input) && !inStringExpression) { + pos += 1 + if (depth == 0) trimFront += 1 + } else if (insideObject) { + isStartOfEscapeSequence = false + pos += 1 + } else { + throw new FramingException(s"Invalid JSON encountered as position [$pos] of [$buffer]") + } + + @inline private final def insideObject: Boolean = + !outsideObject + + @inline private final def outsideObject: Boolean = + depth == 0 + +} diff --git a/akka-stream/src/main/scala/akka/stream/javadsl/JsonFraming.scala b/akka-stream/src/main/scala/akka/stream/javadsl/JsonFraming.scala new file mode 100644 index 00000000000..3fb3c28638e --- /dev/null +++ b/akka-stream/src/main/scala/akka/stream/javadsl/JsonFraming.scala @@ -0,0 +1,40 @@ +/** + * Copyright (C) 2015-2016 Lightbend Inc. + */ +package akka.stream.javadsl + +import akka.NotUsed +import akka.util.ByteString + +/** Provides JSON framing stages that can separate valid JSON objects from incoming [[akka.util.ByteString]] objects. */ +object JsonFraming { + + /** + * Returns a Flow that implements a "brace counting" based framing stage for emitting valid JSON chunks. + * + * Typical examples of data that one may want to frame using this stage include: + * + * **Very large arrays**: + * {{{ + * [{"id": 1}, {"id": 2}, [...], {"id": 999}] + * }}} + * + * **Multiple concatenated JSON objects** (with, or without commas between them): + * + * {{{ + * {"id": 1}, {"id": 2}, [...], {"id": 999} + * }}} + * + * The framing works independently of formatting, i.e. it will still emit valid JSON elements even if two + * elements are separated by multiple newlines or other whitespace characters. And of course is insensitive + * (and does not impact the emitting frame) to the JSON object's internal formatting. + * + * Framing raw JSON values (such as integers or strings) is supported as well. + * + * @param maximumObjectLength The maximum length of allowed frames while decoding. If the maximum length is exceeded + * this Flow will fail the stream. + */ + def bracketCounting(maximumObjectLength: Int): Flow[ByteString, ByteString, NotUsed] = + akka.stream.scaladsl.JsonFraming.bracketCounting(maximumObjectLength).asJava + +} diff --git a/akka-stream/src/main/scala/akka/stream/scaladsl/Framing.scala b/akka-stream/src/main/scala/akka/stream/scaladsl/Framing.scala index 7ff38cae369..d68b19560ea 100644 --- a/akka-stream/src/main/scala/akka/stream/scaladsl/Framing.scala +++ b/akka-stream/src/main/scala/akka/stream/scaladsl/Framing.scala @@ -287,3 +287,11 @@ object Framing { } } + +/** + * Wrapper around a framing Flow (as provided by [[Framing.delimiter]] for example. + * Used for providing a framing implicitly for other components which may need one (such as framed entity streaming in Akka HTTP). + */ +trait Framing { + def flow: Flow[ByteString, ByteString, NotUsed] +} diff --git a/akka-stream/src/main/scala/akka/stream/scaladsl/JsonFraming.scala b/akka-stream/src/main/scala/akka/stream/scaladsl/JsonFraming.scala new file mode 100644 index 00000000000..0a48c0a3c48 --- /dev/null +++ b/akka-stream/src/main/scala/akka/stream/scaladsl/JsonFraming.scala @@ -0,0 +1,72 @@ +/** + * Copyright (C) 2015-2016 Lightbend Inc. + */ +package akka.stream.scaladsl + +import akka.NotUsed +import akka.stream.Attributes +import akka.stream.impl.JsonBracketCounting +import akka.stream.impl.fusing.GraphStages.SimpleLinearGraphStage +import akka.stream.stage.{ InHandler, OutHandler, GraphStageLogic } +import akka.util.ByteString + +import scala.util.control.NonFatal + +/** Provides JSON framing stages that can separate valid JSON objects from incoming [[ByteString]] objects. */ +object JsonFraming { + + /** + * Returns a Flow that implements a "brace counting" based framing stage for emitting valid JSON chunks. + * + * Typical examples of data that one may want to frame using this stage include: + * + * **Very large arrays**: + * {{{ + * [{"id": 1}, {"id": 2}, [...], {"id": 999}] + * }}} + * + * **Multiple concatenated JSON objects** (with, or without commas between them): + * + * {{{ + * {"id": 1}, {"id": 2}, [...], {"id": 999} + * }}} + * + * The framing works independently of formatting, i.e. it will still emit valid JSON elements even if two + * elements are separated by multiple newlines or other whitespace characters. And of course is insensitive + * (and does not impact the emitting frame) to the JSON object's internal formatting. + * + * Framing raw JSON values (such as integers or strings) is supported as well. + * + * @param maximumObjectLength The maximum length of allowed frames while decoding. If the maximum length is exceeded + * this Flow will fail the stream. + */ + def bracketCounting(maximumObjectLength: Int): Flow[ByteString, ByteString, NotUsed] = + Flow[ByteString].via(new SimpleLinearGraphStage[ByteString] { + private[this] val buffer = new JsonBracketCounting(maximumObjectLength) + + override def createLogic(inheritedAttributes: Attributes) = new GraphStageLogic(shape) with InHandler with OutHandler { + setHandlers(in, out, this) + + override def onPush(): Unit = { + buffer.offer(grab(in)) + tryPopBuffer() + } + + override def onPull(): Unit = + tryPopBuffer() + + override def onUpstreamFinish(): Unit = + if (buffer.isEmpty) completeStage() + + def tryPopBuffer() = { + try buffer.poll() match { + case Some(json) ⇒ push(out, json) + case _ ⇒ if (isClosed(in)) completeStage() else pull(in) + } catch { + case NonFatal(ex) ⇒ failStage(ex) + } + } + } + }).named("jsonFraming(BracketCounting)") + +} From db880a3db01c2e5f9fad97387347ca147b1ea7db Mon Sep 17 00:00:00 2001 From: Konrad Malawski Date: Mon, 25 Jul 2016 01:50:55 +0200 Subject: [PATCH 02/10] +htp #18837 implemented JavaDSL for Source streaming --- .../JsonStreamingExamplesSpec.scala | 2 +- .../akka/http/javadsl/model/ContentTypes.java | 3 + .../http/javadsl/server/JavaTestServer.java | 57 +++++- .../server/IntegrationRoutingSpec.scala | 7 +- .../http/scaladsl/server/TestServer.scala | 74 ++++++-- .../common/CsvSourceRenderingMode.scala | 41 +++++ .../common/FramingWithContentType.scala | 24 +++ .../common/JsonSourceRenderingMode.scala | 101 +++++++++++ .../javadsl/common/SourceRenderingMode.scala | 22 +++ .../http/javadsl/marshalling/Marshaller.scala | 22 +-- .../akka/http/javadsl/server/Directives.scala | 4 +- .../javadsl/server/JsonEntityStreaming.scala | 9 + .../javadsl/server/RoutingJavaMapping.scala | 4 + .../FramedEntityStreamingDirectives.scala | 90 +++++++--- .../server/directives/FutureDirectives.scala | 10 ++ .../javadsl/unmarshalling/Unmarshaller.scala | 3 + .../common/FramingWithContentType.scala | 23 +++ .../common/JsonSourceRenderingMode.scala | 126 +++++++++++++ .../scaladsl/common/SourceRenderingMode.scala | 11 ++ .../http/scaladsl/common/StrictForm.scala | 4 +- .../scaladsl/marshalling/Marshaller.scala | 6 +- .../server/EntityStreamingSupport.scala | 69 +++++++ .../scaladsl/server/JsonEntityStreaming.scala | 168 ------------------ .../FramedEntityStreamingDirectives.scala | 82 ++++----- .../scala/akka/stream/javadsl/Framing.scala | 15 ++ .../scala/akka/stream/scaladsl/Framing.scala | 5 +- 26 files changed, 707 insertions(+), 275 deletions(-) create mode 100644 akka-http/src/main/scala/akka/http/javadsl/common/CsvSourceRenderingMode.scala create mode 100644 akka-http/src/main/scala/akka/http/javadsl/common/FramingWithContentType.scala create mode 100644 akka-http/src/main/scala/akka/http/javadsl/common/JsonSourceRenderingMode.scala create mode 100644 akka-http/src/main/scala/akka/http/javadsl/common/SourceRenderingMode.scala create mode 100644 akka-http/src/main/scala/akka/http/javadsl/server/JsonEntityStreaming.scala create mode 100644 akka-http/src/main/scala/akka/http/scaladsl/common/FramingWithContentType.scala create mode 100644 akka-http/src/main/scala/akka/http/scaladsl/common/JsonSourceRenderingMode.scala create mode 100644 akka-http/src/main/scala/akka/http/scaladsl/common/SourceRenderingMode.scala create mode 100644 akka-http/src/main/scala/akka/http/scaladsl/server/EntityStreamingSupport.scala delete mode 100644 akka-http/src/main/scala/akka/http/scaladsl/server/JsonEntityStreaming.scala diff --git a/akka-docs/rst/scala/code/docs/http/scaladsl/server/directives/JsonStreamingExamplesSpec.scala b/akka-docs/rst/scala/code/docs/http/scaladsl/server/directives/JsonStreamingExamplesSpec.scala index 6a6fffe86fe..b3545048299 100644 --- a/akka-docs/rst/scala/code/docs/http/scaladsl/server/directives/JsonStreamingExamplesSpec.scala +++ b/akka-docs/rst/scala/code/docs/http/scaladsl/server/directives/JsonStreamingExamplesSpec.scala @@ -105,7 +105,7 @@ class JsonStreamingExamplesSpec extends RoutingSpec { val route = path("metrics") { // [4] extract Source[Measurement, _] - entity(stream[Measurement]) { measurements => + entity(asSourceOf[Measurement]) { measurements => println("measurements = " + measurements) val measurementsSubmitted: Future[Int] = measurements diff --git a/akka-http-core/src/main/java/akka/http/javadsl/model/ContentTypes.java b/akka-http-core/src/main/java/akka/http/javadsl/model/ContentTypes.java index 61b3a1c728e..25c403ee63f 100644 --- a/akka-http-core/src/main/java/akka/http/javadsl/model/ContentTypes.java +++ b/akka-http-core/src/main/java/akka/http/javadsl/model/ContentTypes.java @@ -25,6 +25,9 @@ private ContentTypes() { } public static final ContentType.WithCharset TEXT_XML_UTF8 = akka.http.scaladsl.model.ContentTypes.text$divxml$u0028UTF$minus8$u0029(); + public static final ContentType.WithCharset TEXT_CSV_UTF8 = + akka.http.scaladsl.model.ContentTypes.text$divcsv$u0028UTF$minus8$u0029(); + public static ContentType.Binary create(MediaType.Binary mediaType) { return ContentType$.MODULE$.apply((akka.http.scaladsl.model.MediaType.Binary) mediaType); } diff --git a/akka-http-tests/src/test/java/akka/http/javadsl/server/JavaTestServer.java b/akka-http-tests/src/test/java/akka/http/javadsl/server/JavaTestServer.java index 33f2e986643..65ca1cec44d 100644 --- a/akka-http-tests/src/test/java/akka/http/javadsl/server/JavaTestServer.java +++ b/akka-http-tests/src/test/java/akka/http/javadsl/server/JavaTestServer.java @@ -3,15 +3,20 @@ */ package akka.http.javadsl.server; +import akka.NotUsed; import akka.actor.ActorSystem; import akka.http.javadsl.ConnectHttp; import akka.http.javadsl.Http; import akka.http.javadsl.ServerBinding; +import akka.http.javadsl.marshallers.jackson.Jackson; +import akka.http.javadsl.model.HttpEntity; import akka.http.javadsl.model.HttpRequest; import akka.http.javadsl.model.HttpResponse; import akka.http.javadsl.model.StatusCodes; +import akka.http.javadsl.common.JsonSourceRenderingModes; import akka.stream.ActorMaterializer; import akka.stream.javadsl.Flow; +import akka.stream.javadsl.Source; import scala.concurrent.duration.Duration; import scala.runtime.BoxedUnit; @@ -55,18 +60,41 @@ public Route createRoute() { ); final Route crash = path("crash", () -> - path("scala", () -> completeOKWithFutureString(akka.dispatch.Futures.failed(new Exception("Boom!")))).orElse( - path("java", () -> completeOKWithFutureString(CompletableFuture.supplyAsync(() -> { throw new RuntimeException("Boom!"); })))) + path("scala", () -> completeOKWithFutureString(akka.dispatch.Futures.failed(new Exception("Boom!")))).orElse( + path("java", () -> completeOKWithFutureString(CompletableFuture.supplyAsync(() -> { throw new RuntimeException("Boom!"); })))) ); + final Unmarshaller JavaTweets = Jackson.unmarshaller(JavaTweet.class); + final Route tweets = path("tweets", () -> + get(() -> + parameter(StringUnmarshallers.INTEGER, "n", n -> { + final Source tws = Source.repeat(new JavaTweet("Hello World!")).take(n); + return completeOKWithSource(tws, Jackson.marshaller(), JsonSourceRenderingModes.arrayCompact()); + }) + ).orElse( + post(() -> + extractMaterializer(mat -> + entityasSourceOf(JavaTweets, null, sourceOfTweets -> { + final CompletionStage tweetsCount = sourceOfTweets.runFold(0, (acc, tweet) -> acc + 1, mat); + return onComplete(tweetsCount, c -> complete("Total number of tweets: " + c)); + }) + ) + )) + ); + final Route inner = path("inner", () -> getFromResourceDirectory("someDir") ); - return get(() -> - index.orElse(secure).orElse(ping).orElse(crash).orElse(inner).orElse(requestTimeout) - ); + return index + .orElse(secure) + .orElse(ping) + .orElse(crash) + .orElse(inner) + .orElse(requestTimeout) + .orElse(tweets) + ; } private void silentSleep(int millis) { @@ -113,7 +141,7 @@ private void run() throws InterruptedException { final Flow flow = createRoute().flow(system, mat); final CompletionStage binding = - Http.get(system).bindAndHandle(flow, ConnectHttp.toHost("127.0.0.1"), mat); + Http.get(system).bindAndHandle(flow, ConnectHttp.toHost("127.0.0.1", 8080), mat); System.console().readLine("Press [ENTER] to quit..."); shutdown(binding); @@ -131,4 +159,21 @@ private CompletionStage shutdown(CompletionStage binding) { } }); } + + private static final class JavaTweet { + private String message; + + public JavaTweet(String message) { + this.message = message; + } + + public void setMessage(String message) { + this.message = message; + } + + public String getMessage() { + return message; + } + + } } diff --git a/akka-http-tests/src/test/scala/akka/http/scaladsl/server/IntegrationRoutingSpec.scala b/akka-http-tests/src/test/scala/akka/http/scaladsl/server/IntegrationRoutingSpec.scala index a931209d7f5..223b6b2d9b3 100644 --- a/akka-http-tests/src/test/scala/akka/http/scaladsl/server/IntegrationRoutingSpec.scala +++ b/akka-http-tests/src/test/scala/akka/http/scaladsl/server/IntegrationRoutingSpec.scala @@ -18,6 +18,7 @@ import scala.concurrent.Await private[akka] trait IntegrationRoutingSpec extends WordSpecLike with Matchers with BeforeAndAfterAll with Directives with RequestBuilding with ScalaFutures with IntegrationPatience { + import IntegrationRoutingSpec._ implicit val system = ActorSystem(AkkaSpec.getCallerName(getClass)) implicit val mat = ActorMaterializer() @@ -31,8 +32,6 @@ private[akka] trait IntegrationRoutingSpec extends WordSpecLike with Matchers wi def ~!>(route: Route) = new Prepped(request, route) } - final case class Prepped(request: HttpRequest, route: Route) - implicit class Checking(p: Prepped) { def ~!>(checking: HttpResponse ⇒ Unit) = { val (_, host, port) = TestUtils.temporaryServerHostnameAndPort() @@ -47,3 +46,7 @@ private[akka] trait IntegrationRoutingSpec extends WordSpecLike with Matchers wi } } + +object IntegrationRoutingSpec { + final case class Prepped(request: HttpRequest, route: Route) +} diff --git a/akka-http-tests/src/test/scala/akka/http/scaladsl/server/TestServer.scala b/akka-http-tests/src/test/scala/akka/http/scaladsl/server/TestServer.scala index ecca3305dc7..b4901750d5b 100644 --- a/akka-http-tests/src/test/scala/akka/http/scaladsl/server/TestServer.scala +++ b/akka-http-tests/src/test/scala/akka/http/scaladsl/server/TestServer.scala @@ -4,14 +4,18 @@ package akka.http.scaladsl.server +import akka.NotUsed import akka.http.scaladsl.marshallers.xml.ScalaXmlSupport -import akka.http.scaladsl.model.{ StatusCodes, HttpResponse } +import akka.http.scaladsl.model.{ HttpResponse, StatusCodes } import akka.http.scaladsl.server.directives.Credentials -import com.typesafe.config.{ ConfigFactory, Config } +import com.typesafe.config.{ Config, ConfigFactory } import akka.actor.ActorSystem import akka.stream._ import akka.stream.scaladsl._ import akka.http.scaladsl.Http +import akka.http.scaladsl.common.{ FramingWithContentType, JsonSourceRenderingModes, SourceRenderingMode } +import akka.http.scaladsl.marshalling.ToResponseMarshallable + import scala.concurrent.duration._ import scala.io.StdIn @@ -21,10 +25,26 @@ object TestServer extends App { akka.log-dead-letters = off akka.stream.materializer.debug.fuzzing-mode = off """) + implicit val system = ActorSystem("ServerTest", testConf) import system.dispatcher implicit val materializer = ActorMaterializer() + // --------- json streaming --------- + import spray.json.DefaultJsonProtocol._ + import akka.http.scaladsl.marshallers.sprayjson.SprayJsonSupport._ + final case class Tweet(message: String) + implicit val tweetFormat = jsonFormat1(Tweet) + + // FIXME: Need to be able to support composive framing with content type (!!!!!!!) + import akka.http.scaladsl.server.EntityStreamingSupport._ + /* override if extending EntityStreamingSupport */ + implicit val incomingEntityStreamFraming: FramingWithContentType = bracketCountingJsonFraming(128) + /* override if extending EntityStreamingSupport */ + implicit val outgoingEntityStreamRendering: SourceRenderingMode = JsonSourceRenderingModes.LineByLine + + // --------- end of json streaming --------- + import ScalaXmlSupport._ import Directives._ @@ -32,7 +52,8 @@ object TestServer extends App { case p @ Credentials.Provided(name) if p.verify(name + "-password") ⇒ name } - val bindingFuture = Http().bindAndHandle({ + // format: OFF + val routes = { get { path("") { withRequestTimeout(1.milli, _ ⇒ HttpResponse( @@ -42,19 +63,44 @@ object TestServer extends App { complete(index) } } ~ - path("secure") { - authenticateBasicPF("My very secure site", auth) { user ⇒ - complete(Hello { user }. Access has been granted!) - } - } ~ - path("ping") { - complete("PONG!") + path("secure") { + authenticateBasicPF("My very secure site", auth) { user ⇒ + complete( + Hello + + {user} + + . Access has been granted! + ) + } + } ~ + path("ping") { + complete("PONG!") + } ~ + path("crash") { + complete(sys.error("BOOM!")) + } ~ + path("tweet") { + complete(Tweet("Hello, world!")) + } ~ + (path("tweets") & parameter('n.as[Int])) { n => + get { + val tweets = Source.repeat(Tweet("Hello, world!")).take(n) + complete(ToResponseMarshallable(tweets)) } ~ - path("crash") { - complete(sys.error("BOOM!")) + post { + entity(asSourceOf[Tweet]) { tweets ⇒ + // entity(asSourceOf[Tweet](bracketCountingJsonFraming(1024))) { tweets: Source[Tweet, NotUsed] ⇒ + complete(s"Total tweets received: " + tweets.runFold(0)({ case (acc, t) => acc + 1 })) + } } - } ~ pathPrefix("inner")(getFromResourceDirectory("someDir")) - }, interface = "localhost", port = 8080) + } + } ~ + pathPrefix("inner")(getFromResourceDirectory("someDir")) + } + // format: ON + + val bindingFuture = Http().bindAndHandle(routes, interface = "0.0.0.0", port = 8080) println(s"Server online at http://localhost:8080/\nPress RETURN to stop...") StdIn.readLine() diff --git a/akka-http/src/main/scala/akka/http/javadsl/common/CsvSourceRenderingMode.scala b/akka-http/src/main/scala/akka/http/javadsl/common/CsvSourceRenderingMode.scala new file mode 100644 index 00000000000..d755dd6bf2d --- /dev/null +++ b/akka-http/src/main/scala/akka/http/javadsl/common/CsvSourceRenderingMode.scala @@ -0,0 +1,41 @@ +/* + * Copyright (C) 2016 Lightbend Inc. + */ + +package akka.http.javadsl.common + +import akka.http.javadsl.model.ContentType.WithCharset +import akka.http.javadsl.model.ContentTypes +import akka.util.ByteString + +/** + * Specialised rendering mode for streaming elements as CSV. + */ +trait CsvSourceRenderingMode extends SourceRenderingMode { + override val contentType: WithCharset = + ContentTypes.TEXT_CSV_UTF8 +} + +object CsvSourceRenderingModes { + + /** + * Render sequence of values as row-by-row ('\n' separated) series of values. + */ + val create: CsvSourceRenderingMode = + new CsvSourceRenderingMode { + override def between: ByteString = ByteString("\n") + override def end: ByteString = ByteString.empty + override def start: ByteString = ByteString.empty + } + + /** + * Render sequence of values as row-by-row (with custom row separator, + * e.g. if you need to use '\r\n' instead of '\n') series of values. + */ + def custom(rowSeparator: String): CsvSourceRenderingMode = + new CsvSourceRenderingMode { + override def between: ByteString = ByteString(rowSeparator) + override def end: ByteString = ByteString.empty + override def start: ByteString = ByteString.empty + } +} diff --git a/akka-http/src/main/scala/akka/http/javadsl/common/FramingWithContentType.scala b/akka-http/src/main/scala/akka/http/javadsl/common/FramingWithContentType.scala new file mode 100644 index 00000000000..127c9bf8902 --- /dev/null +++ b/akka-http/src/main/scala/akka/http/javadsl/common/FramingWithContentType.scala @@ -0,0 +1,24 @@ +/* + * Copyright (C) 2016 Lightbend Inc. + */ + +package akka.http.javadsl.common + +import akka.http.javadsl.model.ContentTypeRange +import akka.stream.javadsl.Framing + +trait FramingWithContentType extends Framing { self ⇒ + import akka.http.impl.util.JavaMapping.Implicits._ + + override def asScala: akka.http.scaladsl.common.FramingWithContentType = + this match { + case f: akka.http.scaladsl.common.FramingWithContentType ⇒ f + case _ ⇒ new akka.http.scaladsl.common.FramingWithContentType { + override def flow = self.getFlow.asScala + override def supported = self.supported.asScala + } + } + + def supported: ContentTypeRange + def matches(ct: akka.http.javadsl.model.ContentType): Boolean = supported.matches(ct) +} diff --git a/akka-http/src/main/scala/akka/http/javadsl/common/JsonSourceRenderingMode.scala b/akka-http/src/main/scala/akka/http/javadsl/common/JsonSourceRenderingMode.scala new file mode 100644 index 00000000000..b5fe32fdf93 --- /dev/null +++ b/akka-http/src/main/scala/akka/http/javadsl/common/JsonSourceRenderingMode.scala @@ -0,0 +1,101 @@ +/* + * Copyright (C) 2016 Lightbend Inc. + */ + +package akka.http.javadsl.common + +import akka.http.javadsl.model.{ ContentType, ContentTypes } + +/** + * Specialised rendering mode for streaming elements as JSON. + * + * See also: JSON Streaming on Wikipedia. + */ +trait JsonSourceRenderingMode extends SourceRenderingMode { + override val contentType: ContentType.WithFixedCharset = + ContentTypes.APPLICATION_JSON +} + +/** + * Provides default JSON rendering modes. + */ +object JsonSourceRenderingModes { + + /** + * Most compact rendering mode. + * It does not intersperse any separator between the signalled elements. + * + * It can be used with [[akka.stream.javadsl.JsonFraming.bracketCounting]]. + * + * {{{ + * {"id":42}{"id":43}{"id":44} + * }}} + */ + val compact = akka.http.scaladsl.common.JsonSourceRenderingModes.Compact + + /** + * Simple rendering mode, similar to [[compact]] however interspersing elements with a `\n` character. + * + * {{{ + * {"id":42},{"id":43},{"id":44} + * }}} + */ + val compactCommaSeparated = akka.http.scaladsl.common.JsonSourceRenderingModes.CompactCommaSeparated + + /** + * Rendering mode useful when the receiving end expects a valid JSON Array. + * It can be useful when the client wants to detect when the stream has been successfully received in-full, + * which it can determine by seeing the terminating `]` character. + * + * The framing's terminal `]` will ONLY be emitted if the stream has completed successfully, + * in other words - the stream has been emitted completely, without errors occuring before the final element has been signaled. + * + * {{{ + * [{"id":42},{"id":43},{"id":44}] + * }}} + */ + val arrayCompact = akka.http.scaladsl.common.JsonSourceRenderingModes.ArrayCompact + + /** + * Rendering mode useful when the receiving end expects a valid JSON Array. + * It can be useful when the client wants to detect when the stream has been successfully received in-full, + * which it can determine by seeing the terminating `]` character. + * + * The framing's terminal `]` will ONLY be emitted if the stream has completed successfully, + * in other words - the stream has been emitted completely, without errors occuring before the final element has been signaled. + * + * {{{ + * [{"id":42}, + * {"id":43}, + * {"id":44}] + * }}} + */ + val arrayLineByLine = akka.http.scaladsl.common.JsonSourceRenderingModes.ArrayLineByLine + + /** + * Recommended rendering mode. + * + * It is a nice balance between valid and human-readable as well as resonably small size overhead (just the `\n` between elements). + * A good example of API's using this syntax is Twitter's Firehose (last verified at 1.1 version of that API). + * + * {{{ + * {"id":42} + * {"id":43} + * {"id":44} + * }}} + */ + val lineByLine = akka.http.scaladsl.common.JsonSourceRenderingModes.LineByLine + + /** + * Simple rendering mode interspersing each pair of elements with both `,\n`. + * Picking the [[lineByLine]] format may be preferable, as it is slightly simpler to parse - each line being a valid json object (no need to trim the comma). + * + * {{{ + * {"id":42}, + * {"id":43}, + * {"id":44} + * }}} + */ + val lineByLineCommaSeparated = akka.http.scaladsl.common.JsonSourceRenderingModes.LineByLineCommaSeparated + +} diff --git a/akka-http/src/main/scala/akka/http/javadsl/common/SourceRenderingMode.scala b/akka-http/src/main/scala/akka/http/javadsl/common/SourceRenderingMode.scala new file mode 100644 index 00000000000..5144f336f6f --- /dev/null +++ b/akka-http/src/main/scala/akka/http/javadsl/common/SourceRenderingMode.scala @@ -0,0 +1,22 @@ +/* + * Copyright (C) 2016 Lightbend Inc. + */ + +package akka.http.javadsl.common + +import akka.http.javadsl.model.ContentType +import akka.util.ByteString + +/** + * Defines how to render a [[akka.stream.javadsl.Source]] into a raw [[ByteString]] + * output. + * + * This can be used to render a source into an [[akka.http.scaladsl.model.HttpEntity]]. + */ +trait SourceRenderingMode { + def contentType: ContentType + + def start: ByteString + def between: ByteString + def end: ByteString +} diff --git a/akka-http/src/main/scala/akka/http/javadsl/marshalling/Marshaller.scala b/akka-http/src/main/scala/akka/http/javadsl/marshalling/Marshaller.scala index c1dc4ab6815..dfd73886527 100644 --- a/akka-http/src/main/scala/akka/http/javadsl/marshalling/Marshaller.scala +++ b/akka-http/src/main/scala/akka/http/javadsl/marshalling/Marshaller.scala @@ -16,6 +16,7 @@ import akka.japi.Util import akka.util.ByteString import scala.concurrent.ExecutionContext +import scala.annotation.unchecked.uncheckedVariance import scala.language.implicitConversions object Marshaller { @@ -56,29 +57,29 @@ object Marshaller { // TODO make sure these are actually usable in a sane way def wrapEntity[A, C](f: function.BiFunction[ExecutionContext, C, A], m: Marshaller[A, RequestEntity], mediaType: MediaType): Marshaller[C, RequestEntity] = { - val scalaMarshaller = m.asScalaToEntityMarshaller + val scalaMarshaller = m.asScalaCastOutput fromScala(scalaMarshaller.wrapWithEC(mediaType.asScala) { ctx ⇒ c: C ⇒ f(ctx, c) }(ContentTypeOverrider.forEntity)) } def wrapEntity[A, C, E <: RequestEntity](f: function.Function[C, A], m: Marshaller[A, E], mediaType: MediaType): Marshaller[C, RequestEntity] = { - val scalaMarshaller = m.asScalaToEntityMarshaller + val scalaMarshaller = m.asScalaCastOutput fromScala(scalaMarshaller.wrap(mediaType.asScala)((in: C) ⇒ f.apply(in))(ContentTypeOverrider.forEntity)) } def entityToOKResponse[A](m: Marshaller[A, _ <: RequestEntity]): Marshaller[A, HttpResponse] = { - fromScala(marshalling.Marshaller.fromToEntityMarshaller[A]()(m.asScalaToEntityMarshaller)) + fromScala(marshalling.Marshaller.fromToEntityMarshaller[A]()(m.asScalaCastOutput)) } def entityToResponse[A, R <: RequestEntity](status: StatusCode, m: Marshaller[A, R]): Marshaller[A, HttpResponse] = { - fromScala(marshalling.Marshaller.fromToEntityMarshaller[A](status.asScala)(m.asScalaToEntityMarshaller)) + fromScala(marshalling.Marshaller.fromToEntityMarshaller[A](status.asScala)(m.asScalaCastOutput)) } def entityToResponse[A](status: StatusCode, headers: java.lang.Iterable[HttpHeader], m: Marshaller[A, _ <: RequestEntity]): Marshaller[A, HttpResponse] = { - fromScala(marshalling.Marshaller.fromToEntityMarshaller[A](status.asScala, Util.immutableSeq(headers).map(_.asScala))(m.asScalaToEntityMarshaller)) // TODO can we avoid the map() ? + fromScala(marshalling.Marshaller.fromToEntityMarshaller[A](status.asScala, Util.immutableSeq(headers).map(_.asScala))(m.asScalaCastOutput)) // TODO can we avoid the map() ? } def entityToOKResponse[A](headers: java.lang.Iterable[HttpHeader], m: Marshaller[A, _ <: RequestEntity]): Marshaller[A, HttpResponse] = { - fromScala(marshalling.Marshaller.fromToEntityMarshaller[A](headers = Util.immutableSeq(headers).map(_.asScala))(m.asScalaToEntityMarshaller)) // TODO avoid the map() + fromScala(marshalling.Marshaller.fromToEntityMarshaller[A](headers = Util.immutableSeq(headers).map(_.asScala))(m.asScalaCastOutput)) // TODO avoid the map() } // these are methods not varargs to avoid call site warning about unchecked type params @@ -140,13 +141,14 @@ object Marshaller { m.asScala.map(_.asScala) } -class Marshaller[A, B] private (implicit val asScala: marshalling.Marshaller[A, B]) { +class Marshaller[-A, +B] private (implicit val asScala: marshalling.Marshaller[A, B]) { import Marshaller.fromScala + /** INTERNAL API: involves unsafe cast (however is very fast) */ // TODO would be nice to not need this special case - def asScalaToEntityMarshaller[C]: marshalling.Marshaller[A, C] = asScala.asInstanceOf[marshalling.Marshaller[A, C]] + private[akka] def asScalaCastOutput[C]: marshalling.Marshaller[A, C] = asScala.asInstanceOf[marshalling.Marshaller[A, C]] - def map[C](f: function.Function[B, C]): Marshaller[A, C] = fromScala(asScala.map(f.apply)) + def map[C](f: function.Function[B @uncheckedVariance, C]): Marshaller[A, C] = fromScala(asScala.map(f.apply)) - def compose[C](f: function.Function[C, A]): Marshaller[C, B] = fromScala(asScala.compose(f.apply)) + def compose[C](f: function.Function[C, A @uncheckedVariance]): Marshaller[C, B] = fromScala(asScala.compose(f.apply)) } diff --git a/akka-http/src/main/scala/akka/http/javadsl/server/Directives.scala b/akka-http/src/main/scala/akka/http/javadsl/server/Directives.scala index 7bc2100a2df..2f966a1a7ec 100644 --- a/akka-http/src/main/scala/akka/http/javadsl/server/Directives.scala +++ b/akka-http/src/main/scala/akka/http/javadsl/server/Directives.scala @@ -5,11 +5,11 @@ package akka.http.javadsl.server import akka.http.impl.util.JavaMapping -import akka.http.javadsl.server.directives.TimeoutDirectives +import akka.http.javadsl.server.directives.{ FramedEntityStreamingDirectives, TimeoutDirectives } import scala.annotation.varargs -abstract class AllDirectives extends TimeoutDirectives +abstract class AllDirectives extends FramedEntityStreamingDirectives /** * INTERNAL API diff --git a/akka-http/src/main/scala/akka/http/javadsl/server/JsonEntityStreaming.scala b/akka-http/src/main/scala/akka/http/javadsl/server/JsonEntityStreaming.scala new file mode 100644 index 00000000000..529ae2dea3a --- /dev/null +++ b/akka-http/src/main/scala/akka/http/javadsl/server/JsonEntityStreaming.scala @@ -0,0 +1,9 @@ +/* + * Copyright (C) 2009-2015 Typesafe Inc. + */ +package akka.http.javadsl.server + +class JsonEntityStreaming { + +} + diff --git a/akka-http/src/main/scala/akka/http/javadsl/server/RoutingJavaMapping.scala b/akka-http/src/main/scala/akka/http/javadsl/server/RoutingJavaMapping.scala index 1cb892cfd81..dda99cbb75d 100644 --- a/akka-http/src/main/scala/akka/http/javadsl/server/RoutingJavaMapping.scala +++ b/akka-http/src/main/scala/akka/http/javadsl/server/RoutingJavaMapping.scala @@ -10,7 +10,9 @@ import akka.http.impl.util.JavaMapping._ import akka.http.impl.util._ import akka.http.{ javadsl, scaladsl } import akka.http.scaladsl.server.{ directives ⇒ sdirectives } +import akka.http.scaladsl.{ common ⇒ scommon } import akka.http.javadsl.server.{ directives ⇒ jdirectives } +import akka.http.javadsl.{ common ⇒ jcommon } import scala.collection.immutable /** @@ -43,6 +45,8 @@ private[http] object RoutingJavaMapping { } implicit object convertRouteResult extends Inherited[javadsl.server.RouteResult, scaladsl.server.RouteResult] + implicit object convertSourceRenderingMode extends Inherited[jcommon.SourceRenderingMode, scommon.SourceRenderingMode] + implicit object convertDirectoryRenderer extends Inherited[jdirectives.DirectoryRenderer, sdirectives.FileAndResourceDirectives.DirectoryRenderer] implicit object convertContentTypeResolver extends Inherited[jdirectives.ContentTypeResolver, sdirectives.ContentTypeResolver] implicit object convertDirectoryListing extends Inherited[jdirectives.DirectoryListing, sdirectives.DirectoryListing] diff --git a/akka-http/src/main/scala/akka/http/javadsl/server/directives/FramedEntityStreamingDirectives.scala b/akka-http/src/main/scala/akka/http/javadsl/server/directives/FramedEntityStreamingDirectives.scala index ec05f046e8c..e39288a8b20 100644 --- a/akka-http/src/main/scala/akka/http/javadsl/server/directives/FramedEntityStreamingDirectives.scala +++ b/akka-http/src/main/scala/akka/http/javadsl/server/directives/FramedEntityStreamingDirectives.scala @@ -3,49 +3,87 @@ */ package akka.http.javadsl.server.directives -import akka.http.javadsl.model.{ContentType, HttpEntity} +import akka.http.javadsl.model.{ ContentType, HttpEntity } import akka.util.ByteString -import java.util.{List => JList, Map => JMap} +import java.util.{ List ⇒ JList, Map ⇒ JMap } import java.util.AbstractMap.SimpleImmutableEntry import java.util.Optional -import java.util.function.{Function => JFunction} +import java.util.function.{ Function ⇒ JFunction } import akka.NotUsed import scala.collection.JavaConverters._ import akka.http.impl.util.JavaMapping.Implicits._ -import akka.http.javadsl.server.directives.FramedEntityStreamingDirectives.SourceRenderingMode -import akka.http.javadsl.server.{Route, Unmarshaller} -import akka.http.scaladsl.marshalling.ToResponseMarshallable -import akka.http.scaladsl.server.{FramingWithContentType, Directives => D} -import akka.http.scaladsl.server.directives.ParameterDirectives._ +import akka.http.javadsl.common.{ FramingWithContentType, SourceRenderingMode } +import akka.http.javadsl.server.{ Marshaller, Route, Unmarshaller } +import akka.http.javadsl.model._ +import akka.http.scaladsl.marshalling.{ ToResponseMarshallable, ToResponseMarshaller } +import akka.http.scaladsl.server.{ Directives ⇒ D } +import akka.http.scaladsl.unmarshalling import akka.stream.javadsl.Source -import scala.compat.java8.OptionConverters - /** EXPERIMENTAL API */ -trait FramedEntityStreamingDirectives { - - def entityAsStream[T](clazz: Class[T], um: Unmarshaller[_ >: HttpEntity, T], framing: FramingWithContentType, - inner: java.util.function.Function[Source[T, NotUsed], Route]): Route = RouteAdapter { - D.entity[T](D.stream[T](um, framing)) { s => - +abstract class FramedEntityStreamingDirectives extends TimeoutDirectives { + // important import, as we implicitly resolve some marshallers inside the below directives + import akka.http.scaladsl.server.directives.FramedEntityStreamingDirectives._ + + @CorrespondsTo("asSourceOf") + def entityasSourceOf[T](um: Unmarshaller[HttpEntity, T], framing: FramingWithContentType, + inner: java.util.function.Function[Source[T, NotUsed], Route]): Route = RouteAdapter { + val sum = um.asScalaCastInput[akka.http.scaladsl.model.HttpEntity] + D.entity(D.asSourceOf[T](framing.asScala)(sum)) { s: akka.stream.scaladsl.Source[T, NotUsed] ⇒ + inner(s.asJava).delegate + } + } + + @CorrespondsTo("asSourceOfAsync") + def entityAsSourceAsyncOf[T]( + parallelism: Int, + um: Unmarshaller[HttpEntity, T], framing: FramingWithContentType, + inner: java.util.function.Function[Source[T, NotUsed], Route]): Route = RouteAdapter { + val sum = um.asScalaCastInput[akka.http.scaladsl.model.HttpEntity] + D.entity(D.asSourceOfAsync[T](parallelism, framing.asScala)(sum)) { s: akka.stream.scaladsl.Source[T, NotUsed] ⇒ + inner(s.asJava).delegate + } + } + + @CorrespondsTo("asSourceOfAsyncUnordered") + def entityAsSourceAsyncUnorderedOf[T]( + parallelism: Int, + um: Unmarshaller[HttpEntity, T], framing: FramingWithContentType, + inner: java.util.function.Function[Source[T, NotUsed], Route]): Route = RouteAdapter { + val sum = um.asScalaCastInput[akka.http.scaladsl.model.HttpEntity] + D.entity(D.asSourceOfAsyncUnordered[T](parallelism, framing.asScala)(sum)) { s: akka.stream.scaladsl.Source[T, NotUsed] ⇒ + inner(s.asJava).delegate } - ??? } - - def completeWithSource[T](source: Source[T, Any], rendering: SourceRenderingMode): Route = RouteAdapter { + + // implicit used internally, Java caller does not benefit or use it + @CorrespondsTo("complete") + def completeWithSource[T, M](implicit source: Source[T, M], m: Marshaller[T, ByteString], rendering: SourceRenderingMode): Route = RouteAdapter { + import akka.http.scaladsl.marshalling.PredefinedToResponseMarshallers._ + implicit val mm = _sourceMarshaller(m.map(ByteStringAsEntityFn), rendering) + val response = ToResponseMarshallable(source) + D.complete(response) + } + + @CorrespondsTo("complete") + def completeOKWithSource[T, M](implicit source: Source[T, M], m: Marshaller[T, RequestEntity], rendering: SourceRenderingMode): Route = RouteAdapter { + implicit val mm = _sourceMarshaller[T, M](m, rendering) val response = ToResponseMarshallable(source) D.complete(response) } -} -object FramedEntityStreamingDirectives extends FramedEntityStreamingDirectives { - trait SourceRenderingMode { - def getContentType: ContentType + implicit private def _sourceMarshaller[T, M](implicit m: Marshaller[T, HttpEntity], rendering: SourceRenderingMode) = { + import akka.http.javadsl.server.RoutingJavaMapping._ + import akka.http.javadsl.server.RoutingJavaMapping.Implicits._ + val mm = m.asScalaCastOutput + D._sourceMarshaller[T, M](mm, rendering.asScala).compose({ h: akka.stream.javadsl.Source[T, M] ⇒ h.asScala }) + } - def start: ByteString - def between: ByteString - def end: ByteString + private[this] val ByteStringAsEntityFn = new java.util.function.Function[ByteString, HttpEntity]() { + override def apply(bs: ByteString): HttpEntity = HttpEntities.create(bs) } } + +object FramedEntityStreamingDirectives extends FramedEntityStreamingDirectives diff --git a/akka-http/src/main/scala/akka/http/javadsl/server/directives/FutureDirectives.scala b/akka-http/src/main/scala/akka/http/javadsl/server/directives/FutureDirectives.scala index 7a73f294ef2..dd48c4c1877 100644 --- a/akka-http/src/main/scala/akka/http/javadsl/server/directives/FutureDirectives.scala +++ b/akka-http/src/main/scala/akka/http/javadsl/server/directives/FutureDirectives.scala @@ -30,6 +30,16 @@ abstract class FutureDirectives extends FormFieldDirectives { } } + /** + * "Unwraps" a `CompletionStage` and runs the inner route after future + * completion with the future's value as an extraction of type `Try`. + */ + def onComplete[T](cs: CompletionStage[T], inner: JFunction[Try[T], Route]) = RouteAdapter { + D.onComplete(cs.toScala.recover(unwrapCompletionException)) { value ⇒ + inner(value).delegate + } + } + /** * "Unwraps" a `CompletionStage[T]` and runs the inner route after future * completion with the future's value as an extraction of type `T` if diff --git a/akka-http/src/main/scala/akka/http/javadsl/unmarshalling/Unmarshaller.scala b/akka-http/src/main/scala/akka/http/javadsl/unmarshalling/Unmarshaller.scala index 8a7a0a76595..368ebe68c30 100644 --- a/akka-http/src/main/scala/akka/http/javadsl/unmarshalling/Unmarshaller.scala +++ b/akka-http/src/main/scala/akka/http/javadsl/unmarshalling/Unmarshaller.scala @@ -101,6 +101,9 @@ abstract class Unmarshaller[-A, B] extends UnmarshallerBase[A, B] { implicit def asScala: akka.http.scaladsl.unmarshalling.Unmarshaller[A, B] + /** INTERNAL API */ + private[akka] def asScalaCastInput[I]: unmarshalling.Unmarshaller[I, B] = asScala.asInstanceOf[unmarshalling.Unmarshaller[I, B]] + def unmarshall(a: A, ec: ExecutionContext, mat: Materializer): CompletionStage[B] = asScala.apply(a)(ec, mat).toJava /** diff --git a/akka-http/src/main/scala/akka/http/scaladsl/common/FramingWithContentType.scala b/akka-http/src/main/scala/akka/http/scaladsl/common/FramingWithContentType.scala new file mode 100644 index 00000000000..75743b674de --- /dev/null +++ b/akka-http/src/main/scala/akka/http/scaladsl/common/FramingWithContentType.scala @@ -0,0 +1,23 @@ +/* + * Copyright (C) 2016 Lightbend Inc. + */ + +package akka.http.scaladsl.common + +import akka.NotUsed +import akka.event.Logging +import akka.http.scaladsl.model.{ ContentType, ContentTypeRange } +import akka.stream.scaladsl.{ Flow, Framing } +import akka.util.ByteString + +/** + * Same as [[akka.stream.scaladsl.Framing]] but additionally can express which [[ContentType]] it supports, + * which can be used to reject routes if content type does not match used framing. + */ +abstract class FramingWithContentType extends akka.http.javadsl.common.FramingWithContentType with Framing { + def flow: Flow[ByteString, ByteString, NotUsed] + override def supported: ContentTypeRange + override def matches(ct: akka.http.javadsl.model.ContentType): Boolean = supported.matches(ct) + + override def toString = s"${Logging.simpleName(getClass)}($supported)" +} diff --git a/akka-http/src/main/scala/akka/http/scaladsl/common/JsonSourceRenderingMode.scala b/akka-http/src/main/scala/akka/http/scaladsl/common/JsonSourceRenderingMode.scala new file mode 100644 index 00000000000..52e3d1b7549 --- /dev/null +++ b/akka-http/src/main/scala/akka/http/scaladsl/common/JsonSourceRenderingMode.scala @@ -0,0 +1,126 @@ +/* + * Copyright (C) 2016 Lightbend Inc. + */ + +package akka.http.scaladsl.common + +import akka.http.scaladsl.model.{ ContentType, ContentTypes } +import akka.util.ByteString + +/** + * Specialised rendering mode for streaming elements as JSON. + * + * See also: JSON Streaming on Wikipedia. + */ +trait JsonSourceRenderingMode extends akka.http.javadsl.common.JsonSourceRenderingMode with SourceRenderingMode { + override val contentType: ContentType.WithFixedCharset = + ContentTypes.`application/json` +} + +/** + * Provides default JSON rendering modes. + */ +object JsonSourceRenderingModes { + + /** + * Most compact rendering mode. + * It does not intersperse any separator between the signalled elements. + * + * It is the most compact form to render JSON and can be framed properly by using [[akka.stream.javadsl.JsonFraming.bracketCounting]]. + * + * {{{ + * {"id":42}{"id":43}{"id":44} + * }}} + */ + object Compact extends JsonSourceRenderingMode { + override val start: ByteString = ByteString.empty + override val between: ByteString = ByteString.empty + override val end: ByteString = ByteString.empty + } + + /** + * Simple rendering mode, similar to [[Compact]] however interspersing elements with a `\n` character. + * + * {{{ + * {"id":42},{"id":43},{"id":44} + * }}} + */ + object CompactCommaSeparated extends JsonSourceRenderingMode { + override val start: ByteString = ByteString.empty + override val between: ByteString = ByteString(",") + override val end: ByteString = ByteString.empty + } + + /** + * Rendering mode useful when the receiving end expects a valid JSON Array. + * It can be useful when the client wants to detect when the stream has been successfully received in-full, + * which it can determine by seeing the terminating `]` character. + * + * The framing's terminal `]` will ONLY be emitted if the stream has completed successfully, + * in other words - the stream has been emitted completely, without errors occuring before the final element has been signaled. + * + * {{{ + * [{"id":42},{"id":43},{"id":44}] + * }}} + */ + object ArrayCompact extends JsonSourceRenderingMode { + override val start: ByteString = ByteString("[") + override val between: ByteString = ByteString(",") + override val end: ByteString = ByteString("]") + } + + /** + * Rendering mode useful when the receiving end expects a valid JSON Array. + * It can be useful when the client wants to detect when the stream has been successfully received in-full, + * which it can determine by seeing the terminating `]` character. + * + * The framing's terminal `]` will ONLY be emitted if the stream has completed successfully, + * in other words - the stream has been emitted completely, without errors occuring before the final element has been signaled. + * + * {{{ + * [{"id":42}, + * {"id":43}, + * {"id":44}] + * }}} + */ + object ArrayLineByLine extends JsonSourceRenderingMode { + override val start: ByteString = ByteString("[") + override val between: ByteString = ByteString(",\n") + override val end: ByteString = ByteString("]") + } + + /** + * Recommended rendering mode. + * + * It is a nice balance between valid and human-readable as well as resonably small size overhead (just the `\n` between elements). + * A good example of API's using this syntax is Twitter's Firehose (last verified at 1.1 version of that API). + * + * {{{ + * {"id":42} + * {"id":43} + * {"id":44} + * }}} + */ + object LineByLine extends JsonSourceRenderingMode { + override val start: ByteString = ByteString.empty + override val between: ByteString = ByteString("\n") + override val end: ByteString = ByteString.empty + } + + /** + * Simple rendering mode interspersing each pair of elements with both `,\n`. + * Picking the [[LineByLine]] format may be preferable, as it is slightly simpler to parse - each line being a valid json object (no need to trim the comma). + * + * {{{ + * {"id":42}, + * {"id":43}, + * {"id":44} + * }}} + */ + object LineByLineCommaSeparated extends JsonSourceRenderingMode { + override val start: ByteString = ByteString.empty + override val between: ByteString = ByteString(",\n") + override val end: ByteString = ByteString.empty + } + +} diff --git a/akka-http/src/main/scala/akka/http/scaladsl/common/SourceRenderingMode.scala b/akka-http/src/main/scala/akka/http/scaladsl/common/SourceRenderingMode.scala new file mode 100644 index 00000000000..61abd851440 --- /dev/null +++ b/akka-http/src/main/scala/akka/http/scaladsl/common/SourceRenderingMode.scala @@ -0,0 +1,11 @@ +/* + * Copyright (C) 2016 Lightbend Inc. + */ + +package akka.http.scaladsl.common + +import akka.http.scaladsl.model.ContentType + +trait SourceRenderingMode extends akka.http.javadsl.common.SourceRenderingMode { + override def contentType: ContentType +} diff --git a/akka-http/src/main/scala/akka/http/scaladsl/common/StrictForm.scala b/akka-http/src/main/scala/akka/http/scaladsl/common/StrictForm.scala index 124746ccd5d..615941d25c7 100644 --- a/akka-http/src/main/scala/akka/http/scaladsl/common/StrictForm.scala +++ b/akka-http/src/main/scala/akka/http/scaladsl/common/StrictForm.scala @@ -61,8 +61,8 @@ object StrictForm { fsu(value.entity.data.decodeString(charsetName)) }) - @implicitNotFound("In order to unmarshal a `StrictForm.Field` to type `${T}` you need to supply a " + - "`FromStringUnmarshaller[${T}]` and/or a `FromEntityUnmarshaller[${T}]`") + @implicitNotFound(s"In order to unmarshal a `StrictForm.Field` to type `$${T}` you need to supply a " + + s"`FromStringUnmarshaller[$${T}]` and/or a `FromEntityUnmarshaller[$${T}]`") sealed trait FieldUnmarshaller[T] { def unmarshalString(value: String)(implicit ec: ExecutionContext, mat: Materializer): Future[T] def unmarshalPart(value: Multipart.FormData.BodyPart.Strict)(implicit ec: ExecutionContext, mat: Materializer): Future[T] diff --git a/akka-http/src/main/scala/akka/http/scaladsl/marshalling/Marshaller.scala b/akka-http/src/main/scala/akka/http/scaladsl/marshalling/Marshaller.scala index 81c0290448b..6606ff9f37b 100644 --- a/akka-http/src/main/scala/akka/http/scaladsl/marshalling/Marshaller.scala +++ b/akka-http/src/main/scala/akka/http/scaladsl/marshalling/Marshaller.scala @@ -4,7 +4,9 @@ package akka.http.scaladsl.marshalling -import scala.concurrent.{ Future, ExecutionContext } +import akka.http.scaladsl.marshalling.Marshalling.Opaque + +import scala.concurrent.{ ExecutionContext, Future } import scala.util.control.NonFatal import akka.http.scaladsl.model._ import akka.http.scaladsl.util.FastFuture @@ -174,4 +176,4 @@ object Marshalling { def map[B](f: A ⇒ B): Opaque[B] = copy(marshal = () ⇒ f(marshal())) } } -//# \ No newline at end of file +//# diff --git a/akka-http/src/main/scala/akka/http/scaladsl/server/EntityStreamingSupport.scala b/akka-http/src/main/scala/akka/http/scaladsl/server/EntityStreamingSupport.scala new file mode 100644 index 00000000000..9b6f9bf9889 --- /dev/null +++ b/akka-http/src/main/scala/akka/http/scaladsl/server/EntityStreamingSupport.scala @@ -0,0 +1,69 @@ +/* + * Copyright (C) 2009-2015 Typesafe Inc. + */ +package akka.http.scaladsl.server + +import akka.NotUsed +import akka.actor.ActorSystem +import akka.event.Logging +import akka.http.scaladsl.common.{ FramingWithContentType, SourceRenderingMode } +import akka.http.scaladsl.model.{ ContentTypeRange, ContentTypes, MediaRange, MediaRanges } +import akka.stream.scaladsl.{ Flow, Framing } +import akka.util.ByteString +import com.typesafe.config.Config + +/** + * Entity streaming support, independent of used Json parsing library etc. + * + * Can be extended by various Support traits (e.g. "SprayJsonSupport"), + * in order to provide users with both `framing` (this trait) and `marshalling` + * (implemented by a library) by using a single trait. + */ +trait EntityStreamingSupport extends EntityStreamingSupportBase { + + /** + * Implement as `implicit val` with required framing implementation, for example in + * the case of streaming JSON uploads it could be `bracketCountingJsonFraming(maximumObjectLength)`. + */ + def incomingEntityStreamFraming: FramingWithContentType + + /** + * Implement as `implicit val` with the rendering mode to be used when redering `Source` instances. + * For example for JSON it could be [[akka.http.scaladsl.common.JsonSourceRenderingMode.CompactArray]] + * or [[akka.http.scaladsl.common.JsonSourceRenderingMode.LineByLine]]. + */ + def outgoingEntityStreamRendering: SourceRenderingMode +} + +trait EntityStreamingSupportBase { + /** `application/json` specific Framing implementation */ + def bracketCountingJsonFraming(maximumObjectLength: Int): FramingWithContentType = + new ApplicationJsonBracketCountingFraming(maximumObjectLength) + + /** + * Frames incoming `text / *` entities on a line-by-line basis. + * Useful for accepting `text/csv` uploads as a stream of rows. + */ + def newLineFraming(maximumObjectLength: Int, supportedContentTypes: ContentTypeRange): FramingWithContentType = + new FramingWithContentType { + override final val flow: Flow[ByteString, ByteString, NotUsed] = + Flow[ByteString].via(Framing.delimiter(ByteString("\n"), maximumObjectLength)) + + override final val supported: ContentTypeRange = + ContentTypeRange(MediaRanges.`text/*`) + } +} + +/** + * Entity streaming support, independent of used Json parsing library etc. + * + * Can be extended by various Support traits (e.g. "SprayJsonSupport"), + * in order to provide users with both `framing` (this trait) and `marshalling` + * (implemented by a library) by using a single trait. + */ +object EntityStreamingSupport extends EntityStreamingSupportBase + +final class ApplicationJsonBracketCountingFraming(maximumObjectLength: Int) extends FramingWithContentType { + override final val flow = Flow[ByteString].via(akka.stream.scaladsl.JsonFraming.bracketCounting(maximumObjectLength)) + override final val supported = ContentTypeRange(ContentTypes.`application/json`) +} diff --git a/akka-http/src/main/scala/akka/http/scaladsl/server/JsonEntityStreaming.scala b/akka-http/src/main/scala/akka/http/scaladsl/server/JsonEntityStreaming.scala deleted file mode 100644 index 228aaf96a82..00000000000 --- a/akka-http/src/main/scala/akka/http/scaladsl/server/JsonEntityStreaming.scala +++ /dev/null @@ -1,168 +0,0 @@ -/* - * Copyright (C) 2009-2015 Typesafe Inc. - */ -package akka.http.scaladsl.server - -import akka.NotUsed -import akka.actor.ActorSystem -import akka.http.impl.util.JavaMapping -import akka.http.scaladsl.model.{ ContentType, ContentTypes } -import akka.http.scaladsl.server.directives.FramedEntityStreamingDirectives.SourceRenderingMode -import akka.stream.scaladsl.{ Flow, Framing } -import akka.util.ByteString -import com.typesafe.config.Config - -import scala.collection.immutable - -/** - * Same as [[akka.stream.scaladsl.Framing]] but additionally can express which [[ContentType]] it supports, - * which can be used to reject routes if content type does not match used framing. - */ -abstract class FramingWithContentType extends Framing { - def flow: Flow[ByteString, ByteString, NotUsed] - def supported: immutable.Set[ContentType] - def isSupported(ct: akka.http.javadsl.model.ContentType): Boolean = supported(JavaMapping.ContentType.toScala(ct)) -} -object FramingWithContentType { - def apply(framing: Flow[ByteString, ByteString, NotUsed], contentType: ContentType, moreContentTypes: ContentType*) = - new FramingWithContentType { - override def flow = framing - - override val supported: immutable.Set[ContentType] = - if (moreContentTypes.isEmpty) Set(contentType) - else Set(contentType) ++ moreContentTypes - } -} - -/** - * Json entity streaming support, independent of used Json parsing library. - * - * Can be extended by various Support traits (e.g. "SprayJsonSupport"), - * in order to provide users with both `framing` (this trait) and `marshalling` - * (implemented by a library) by using a single trait. - */ -trait JsonEntityFramingSupport { - - /** `application/json` specific Framing implementation */ - def bracketCountingJsonFraming(maximumObjectLength: Int) = new FramingWithContentType { - override final val flow = Flow[ByteString].via(akka.stream.scaladsl.JsonFraming.bracketCounting(maximumObjectLength)) - - override val supported: immutable.Set[ContentType] = Set(ContentTypes.`application/json`) - } -} -object JsonEntityFramingSupport extends JsonEntityFramingSupport - -/** - * Specialised rendering mode for streaming elements as JSON. - * - * See also: JSON Streaming on Wikipedia. - */ -trait JsonSourceRenderingMode extends SourceRenderingMode { - override val contentType = ContentTypes.`application/json` -} - -object JsonSourceRenderingMode { - - /** - * Most compact rendering mode - * It does not intersperse any separator between the signalled elements. - * - * {{{ - * {"id":42}{"id":43}{"id":44} - * }}} - */ - object Compact extends JsonSourceRenderingMode { - override val start: ByteString = ByteString.empty - override val between: ByteString = ByteString.empty - override val end: ByteString = ByteString.empty - } - - /** - * Simple rendering mode, similar to [[Compact]] however interspersing elements with a `\n` character. - * - * {{{ - * {"id":42},{"id":43},{"id":44} - * }}} - */ - object CompactCommaSeparated extends JsonSourceRenderingMode { - override val start: ByteString = ByteString.empty - override val between: ByteString = ByteString(",") - override val end: ByteString = ByteString.empty - } - - /** - * Rendering mode useful when the receiving end expects a valid JSON Array. - * It can be useful when the client wants to detect when the stream has been successfully received in-full, - * which it can determine by seeing the terminating `]` character. - * - * The framing's terminal `]` will ONLY be emitted if the stream has completed successfully, - * in other words - the stream has been emitted completely, without errors occuring before the final element has been signaled. - * - * {{{ - * [{"id":42},{"id":43},{"id":44}] - * }}} - */ - object CompactArray extends JsonSourceRenderingMode { - override val start: ByteString = ByteString("[") - override val between: ByteString = ByteString(",") - override val end: ByteString = ByteString("]") - } - - /** - * Recommended rendering mode. - * - * It is a nice balance between valid and human-readable as well as resonably small size overhead (just the `\n` between elements). - * A good example of API's using this syntax is Twitter's Firehose (last verified at 1.1 version of that API). - * - * {{{ - * {"id":42} - * {"id":43} - * {"id":44} - * }}} - */ - object LineByLine extends JsonSourceRenderingMode { - override val start: ByteString = ByteString.empty - override val between: ByteString = ByteString("\n") - override val end: ByteString = ByteString.empty - } - - /** - * Simple rendering mode interspersing each pair of elements with both `,\n`. - * Picking the [[LineByLine]] format may be preferable, as it is slightly simpler to parse - each line being a valid json object (no need to trim the comma). - * - * {{{ - * {"id":42}, - * {"id":43}, - * {"id":44} - * }}} - */ - object LineByLineCommaSeparated extends JsonSourceRenderingMode { - override val start: ByteString = ByteString.empty - override val between: ByteString = ByteString(",\n") - override val end: ByteString = ByteString.empty - } - -} - -object JsonStreamingSettings { - - def apply(sys: ActorSystem): JsonStreamingSettings = - apply(sys.settings.config.getConfig("akka.http.json-streaming")) - - def apply(c: Config): JsonStreamingSettings = { - JsonStreamingSettings( - c.getInt("max-object-size"), - renderingMode(c.getString("rendering-mode"))) - } - - def renderingMode(name: String): SourceRenderingMode = name match { - case "line-by-line" ⇒ JsonSourceRenderingMode.LineByLine // the default - case "line-by-line-comma-separated" ⇒ JsonSourceRenderingMode.LineByLineCommaSeparated - case "compact" ⇒ JsonSourceRenderingMode.Compact - case "compact-comma-separated" ⇒ JsonSourceRenderingMode.CompactCommaSeparated - case "compact-array" ⇒ JsonSourceRenderingMode.CompactArray - } -} -final case class JsonStreamingSettings( - maxObjectSize: Int, - style: SourceRenderingMode) diff --git a/akka-http/src/main/scala/akka/http/scaladsl/server/directives/FramedEntityStreamingDirectives.scala b/akka-http/src/main/scala/akka/http/scaladsl/server/directives/FramedEntityStreamingDirectives.scala index e9a6996a966..c93c6e4020a 100644 --- a/akka-http/src/main/scala/akka/http/scaladsl/server/directives/FramedEntityStreamingDirectives.scala +++ b/akka-http/src/main/scala/akka/http/scaladsl/server/directives/FramedEntityStreamingDirectives.scala @@ -4,9 +4,9 @@ package akka.http.scaladsl.server.directives import akka.NotUsed +import akka.http.scaladsl.common.{ FramingWithContentType, SourceRenderingMode } import akka.http.scaladsl.marshalling._ import akka.http.scaladsl.model._ -import akka.http.scaladsl.server.FramingWithContentType import akka.http.scaladsl.unmarshalling.{ Unmarshal, Unmarshaller, _ } import akka.http.scaladsl.util.FastFuture import akka.stream.Materializer @@ -28,33 +28,31 @@ trait FramedEntityStreamingDirectives extends MarshallingDirectives { // TODO DOCS - final def stream[T](implicit um: Unmarshaller[ByteString, T], framing: FramingWithContentType): RequestToSourceUnmarshaller[T] = - streamAsync(1)(um, framing) - final def stream[T](framing: FramingWithContentType)(implicit um: Unmarshaller[ByteString, T]): RequestToSourceUnmarshaller[T] = - streamAsync(1)(um, framing) + final def asSourceOf[T](implicit um: Unmarshaller[HttpEntity, T], framing: FramingWithContentType): RequestToSourceUnmarshaller[T] = + asSourceOfAsync(1)(um, framing) + final def asSourceOf[T](framing: FramingWithContentType)(implicit um: Unmarshaller[HttpEntity, T]): RequestToSourceUnmarshaller[T] = + asSourceOfAsync(1)(um, framing) - final def streamAsync[T](parallelism: Int)(implicit um: Unmarshaller[ByteString, T], framing: FramingWithContentType): RequestToSourceUnmarshaller[T] = - streamInternal[T](framing, (ec, mat) ⇒ Flow[ByteString].mapAsync(parallelism)(Unmarshal(_).to[T](um, ec, mat))) - final def streamAsync[T](parallelism: Int, framing: FramingWithContentType)(implicit um: Unmarshaller[ByteString, T]): RequestToSourceUnmarshaller[T] = - streamAsync(parallelism)(um, framing) + final def asSourceOfAsync[T](parallelism: Int)(implicit um: Unmarshaller[HttpEntity, T], framing: FramingWithContentType): RequestToSourceUnmarshaller[T] = + asSourceOfInternal[T](framing, (ec, mat) ⇒ Flow[HttpEntity].mapAsync(parallelism)(Unmarshal(_).to[T](um, ec, mat))) + final def asSourceOfAsync[T](parallelism: Int, framing: FramingWithContentType)(implicit um: Unmarshaller[HttpEntity, T]): RequestToSourceUnmarshaller[T] = + asSourceOfAsync(parallelism)(um, framing) - final def streamAsyncUnordered[T](parallelism: Int)(implicit um: Unmarshaller[ByteString, T], framing: FramingWithContentType): RequestToSourceUnmarshaller[T] = - streamInternal[T](framing, (ec, mat) ⇒ Flow[ByteString].mapAsyncUnordered(parallelism)(Unmarshal(_).to[T](um, ec, mat))) - final def streamAsyncUnordered[T](parallelism: Int, framing: FramingWithContentType)(implicit um: Unmarshaller[ByteString, T]): RequestToSourceUnmarshaller[T] = - streamAsyncUnordered(parallelism)(um, framing) - - // TODO materialized value may want to be "drain/cancel" or something like it? - // TODO could expose `streamMat`, for more fine grained picking of Marshaller + final def asSourceOfAsyncUnordered[T](parallelism: Int)(implicit um: Unmarshaller[HttpEntity, T], framing: FramingWithContentType): RequestToSourceUnmarshaller[T] = + asSourceOfInternal[T](framing, (ec, mat) ⇒ Flow[HttpEntity].mapAsyncUnordered(parallelism)(Unmarshal(_).to[T](um, ec, mat))) + final def asSourceOfAsyncUnordered[T](parallelism: Int, framing: FramingWithContentType)(implicit um: Unmarshaller[HttpEntity, T]): RequestToSourceUnmarshaller[T] = + asSourceOfAsyncUnordered(parallelism)(um, framing) // format: OFF - private def streamInternal[T](framing: FramingWithContentType, marshalling: (ExecutionContext, Materializer) => Flow[ByteString, ByteString, NotUsed]#ReprMat[T, NotUsed]): RequestToSourceUnmarshaller[T] = + private def asSourceOfInternal[T](framing: FramingWithContentType, marshalling: (ExecutionContext, Materializer) => Flow[HttpEntity, ByteString, NotUsed]#ReprMat[T, NotUsed]): RequestToSourceUnmarshaller[T] = Unmarshaller.withMaterializer[HttpRequest, Source[T, NotUsed]] { implicit ec ⇒ implicit mat ⇒ req ⇒ val entity = req.entity - if (!framing.supported(entity.contentType)) { - val supportedContentTypes = framing.supported.map(ContentTypeRange(_)) + if (!framing.matches(entity.contentType)) { + val supportedContentTypes = framing.supported FastFuture.failed(Unmarshaller.UnsupportedContentTypeException(supportedContentTypes)) } else { - val stream = entity.dataBytes.via(framing.flow).via(marshalling(ec, mat)).mapMaterializedValue(_ => NotUsed) +// val stream = entity.dataBytes.via(framing.flow).via(marshalling(ec, mat)).mapMaterializedValue(_ => NotUsed) + val stream = Source.single(entity.transformDataBytes(framing.flow)).via(marshalling(ec, mat)).mapMaterializedValue(_ => NotUsed) FastFuture.successful(stream) } } @@ -65,7 +63,7 @@ trait FramedEntityStreamingDirectives extends MarshallingDirectives { implicit def _sourceMarshaller[T, M](implicit m: ToEntityMarshaller[T], mode: SourceRenderingMode): ToResponseMarshaller[Source[T, M]] = Marshaller[Source[T, M], HttpResponse] { implicit ec ⇒ source ⇒ FastFuture successful { - Marshalling.WithFixedContentType(mode.contentType, () ⇒ { // TODO charset? + Marshalling.WithFixedContentType(mode.contentType, () ⇒ { val bytes = source .mapAsync(1)(t ⇒ Marshal(t).to[HttpEntity]) .map(_.dataBytes) @@ -79,7 +77,7 @@ trait FramedEntityStreamingDirectives extends MarshallingDirectives { implicit def _sourceParallelismMarshaller[T](implicit m: ToEntityMarshaller[T], mode: SourceRenderingMode): ToResponseMarshaller[AsyncRenderingOf[T]] = Marshaller[AsyncRenderingOf[T], HttpResponse] { implicit ec ⇒ rendering ⇒ FastFuture successful { - Marshalling.WithFixedContentType(mode.contentType, () ⇒ { // TODO charset? + Marshalling.WithFixedContentType(mode.contentType, () ⇒ { val bytes = rendering.source .mapAsync(rendering.parallelism)(t ⇒ Marshal(t).to[HttpEntity]) .map(_.dataBytes) @@ -93,7 +91,7 @@ trait FramedEntityStreamingDirectives extends MarshallingDirectives { implicit def _sourceUnorderedMarshaller[T](implicit m: ToEntityMarshaller[T], mode: SourceRenderingMode): ToResponseMarshaller[AsyncUnorderedRenderingOf[T]] = Marshaller[AsyncUnorderedRenderingOf[T], HttpResponse] { implicit ec ⇒ rendering ⇒ FastFuture successful { - Marshalling.WithFixedContentType(mode.contentType, () ⇒ { // TODO charset? + Marshalling.WithFixedContentType(mode.contentType, () ⇒ { val bytes = rendering.source .mapAsync(rendering.parallelism)(t ⇒ Marshal(t).to[HttpEntity]) .map(_.dataBytes) @@ -106,32 +104,34 @@ trait FramedEntityStreamingDirectives extends MarshallingDirectives { // special rendering modes - implicit def enableSpecialSourceRenderingModes[T](source: Source[T, Any]): EnableSpecialSourceRenderingModes[T] = + implicit def _enableSpecialSourceRenderingModes[T](source: Source[T, Any]): EnableSpecialSourceRenderingModes[T] = new EnableSpecialSourceRenderingModes(source) } object FramedEntityStreamingDirectives extends FramedEntityStreamingDirectives { - /** - * Defines ByteStrings to be injected before the first, between, and after all elements of a [[Source]], - * when used to complete a request. - * - * A typical example would be rendering a ``Source[T, _]`` as JSON array, - * where start is `[`, between is `,`, and end is `]` - which procudes a valid json array, assuming each element can - * be properly marshalled as JSON object. - * - * The corresponding values will typically be put into an [[Source.intersperse]] call on the to-be-rendered Source. - */ - trait SourceRenderingMode extends akka.http.javadsl.server.directives.FramedEntityStreamingDirectives.SourceRenderingMode { - override final def getContentType = contentType - def contentType: ContentType - } - - final class AsyncRenderingOf[T](val source: Source[T, Any], val parallelism: Int) - final class AsyncUnorderedRenderingOf[T](val source: Source[T, Any], val parallelism: Int) + sealed class AsyncSourceRenderingMode + final class AsyncRenderingOf[T](val source: Source[T, Any], val parallelism: Int) extends AsyncSourceRenderingMode + final class AsyncUnorderedRenderingOf[T](val source: Source[T, Any], val parallelism: Int) extends AsyncSourceRenderingMode } final class EnableSpecialSourceRenderingModes[T](val source: Source[T, Any]) extends AnyVal { + /** + * Causes the response stream to be marshalled asynchronously (up to `parallelism` elements at once), + * while retaining the ordering of incoming elements. + * + * See also [[Source.mapAsync]]. + */ def renderAsync(parallelism: Int) = new FramedEntityStreamingDirectives.AsyncRenderingOf(source, parallelism) + /** + * Causes the response stream to be marshalled asynchronously (up to `parallelism` elements at once), + * emitting the first one that finished marshalling onto the wire. + * + * This sacrifices ordering of the incoming data in regards to data actually rendered onto the wire, + * but may be faster if some elements are smaller than other ones by not stalling the small elements + * from being written while the large one still is being marshalled. + * + * See also [[Source.mapAsyncUnordered]]. + */ def renderAsyncUnordered(parallelism: Int) = new FramedEntityStreamingDirectives.AsyncUnorderedRenderingOf(source, parallelism) } diff --git a/akka-stream/src/main/scala/akka/stream/javadsl/Framing.scala b/akka-stream/src/main/scala/akka/stream/javadsl/Framing.scala index b7b21030b1d..6ae30d85644 100644 --- a/akka-stream/src/main/scala/akka/stream/javadsl/Framing.scala +++ b/akka-stream/src/main/scala/akka/stream/javadsl/Framing.scala @@ -115,3 +115,18 @@ object Framing { scaladsl.Framing.simpleFramingProtocol(maximumMessageLength).asJava } + +/** + * Wrapper around a framing Flow (as provided by [[Framing.delimiter]] for example. + * Used for providing a framing implicitly for other components which may need one (such as framed entity streaming in Akka HTTP). + */ +trait Framing { + def asScala: akka.stream.scaladsl.Framing = + this match { + case f: akka.stream.scaladsl.Framing ⇒ f + case _ ⇒ new akka.stream.scaladsl.Framing { + override def flow = getFlow.asScala + } + } + def getFlow: Flow[ByteString, ByteString, NotUsed] +} diff --git a/akka-stream/src/main/scala/akka/stream/scaladsl/Framing.scala b/akka-stream/src/main/scala/akka/stream/scaladsl/Framing.scala index d68b19560ea..4dadca97566 100644 --- a/akka-stream/src/main/scala/akka/stream/scaladsl/Framing.scala +++ b/akka-stream/src/main/scala/akka/stream/scaladsl/Framing.scala @@ -292,6 +292,9 @@ object Framing { * Wrapper around a framing Flow (as provided by [[Framing.delimiter]] for example. * Used for providing a framing implicitly for other components which may need one (such as framed entity streaming in Akka HTTP). */ -trait Framing { +trait Framing extends akka.stream.javadsl.Framing { + final def asJava: akka.stream.javadsl.Framing = this + override final def getFlow = flow.asJava + def flow: Flow[ByteString, ByteString, NotUsed] } From c76ec2ac15d50ecc3f3251394a8274c12f6980c1 Mon Sep 17 00:00:00 2001 From: Konrad Malawski Date: Mon, 25 Jul 2016 01:50:55 +0200 Subject: [PATCH 03/10] +htp #18837 more docs and final cleanups, complete java docs --- .../server/JsonStreamingExamplesTest.java | 101 +++++++++++++ .../FramedEntityStreamingExamplesTest.java | 33 ----- akka-docs/rst/java/http/routing-dsl/index.rst | 2 +- .../routing-dsl/source-streaming-support.rst | 74 +++++++++ .../JsonStreamingExamplesSpec.scala | 45 +++--- .../MarshallingDirectivesExamplesSpec.scala | 8 +- .../rst/scala/http/routing-dsl/index.rst | 1 + ...pport.rst => source-streaming-support.rst} | 23 ++- .../javadsl/marshallers/jackson/Jackson.java | 9 ++ .../sprayjson/SprayJsonSupport.scala | 4 +- .../http/javadsl/server/JavaTestServer.java | 3 +- .../MarshallingDirectivesSpec.scala | 2 +- .../directives/RouteDirectivesSpec.scala | 5 +- .../common/FramingWithContentType.scala | 18 ++- .../common/JsonSourceRenderingMode.scala | 2 + .../server/EntityStreamingSupport.scala | 41 +++++ .../FramedEntityStreamingDirectives.scala | 37 ++--- .../common/FramingWithContentType.scala | 10 +- .../common/JsonSourceRenderingMode.scala | 2 + .../server/EntityStreamingSupport.scala | 21 ++- .../FramedEntityStreamingDirectives.scala | 140 ++++++++++++++++-- .../stream/scaladsl/JsonFramingSpec.scala | 23 ++- .../stream/impl/JsonBracketCounting.scala | 23 +-- .../scala/akka/stream/javadsl/Framing.scala | 15 -- .../scala/akka/stream/scaladsl/Framing.scala | 11 -- 25 files changed, 475 insertions(+), 178 deletions(-) create mode 100644 akka-docs/rst/java/code/docs/http/javadsl/server/JsonStreamingExamplesTest.java delete mode 100644 akka-docs/rst/java/code/docs/http/javadsl/server/directives/FramedEntityStreamingExamplesTest.java create mode 100644 akka-docs/rst/java/http/routing-dsl/source-streaming-support.rst rename akka-docs/rst/scala/http/routing-dsl/{json-streaming-support.rst => source-streaming-support.rst} (80%) create mode 100644 akka-http/src/main/scala/akka/http/javadsl/server/EntityStreamingSupport.scala diff --git a/akka-docs/rst/java/code/docs/http/javadsl/server/JsonStreamingExamplesTest.java b/akka-docs/rst/java/code/docs/http/javadsl/server/JsonStreamingExamplesTest.java new file mode 100644 index 00000000000..acc6bbfac2e --- /dev/null +++ b/akka-docs/rst/java/code/docs/http/javadsl/server/JsonStreamingExamplesTest.java @@ -0,0 +1,101 @@ +/* + * Copyright (C) 2016 Lightbend Inc. + */ + +package docs.http.javadsl.server; + +import akka.NotUsed; +import akka.http.javadsl.common.FramingWithContentType; +import akka.http.javadsl.common.JsonSourceRenderingModes; +import akka.http.javadsl.marshallers.jackson.Jackson; +import akka.http.javadsl.model.*; +import akka.http.javadsl.model.headers.Accept; +import akka.http.javadsl.server.*; +import akka.http.javadsl.testkit.JUnitRouteTest; +import akka.http.javadsl.testkit.TestRoute; +import akka.stream.javadsl.Source; +import akka.util.ByteString; +import docs.http.javadsl.server.testkit.MyAppService; +import org.junit.Test; + +import java.util.concurrent.CompletionStage; + +public class JsonStreamingExamplesTest extends JUnitRouteTest { + + //#routes + final Route tweets() { + //#formats + final Unmarshaller JavaTweets = Jackson.byteStringUnmarshaller(JavaTweet.class); + //#formats + + //#response-streaming + final Route responseStreaming = path("tweets", () -> + get(() -> + parameter(StringUnmarshallers.INTEGER, "n", n -> { + final Source tws = + Source.repeat(new JavaTweet("Hello World!")).take(n); + return completeOKWithSource(tws, Jackson.marshaller(), JsonSourceRenderingModes.arrayCompact()); + }) + ) + ); + //#response-streaming + + //#incoming-request-streaming + final Route incomingStreaming = path("tweets", () -> + post(() -> + extractMaterializer(mat -> { + final FramingWithContentType jsonFraming = EntityStreamingSupport.bracketCountingJsonFraming(128); + + return entityasSourceOf(JavaTweets, jsonFraming, sourceOfTweets -> { + final CompletionStage tweetsCount = sourceOfTweets.runFold(0, (acc, tweet) -> acc + 1, mat); + return onComplete(tweetsCount, c -> complete("Total number of tweets: " + c)); + }); + } + ) + ) + ); + //#incoming-request-streaming + + return responseStreaming.orElse(incomingStreaming); + } + //#routes + + @Test + public void getTweetsTest() { + //#response-streaming + // tests: + final TestRoute routes = testRoute(tweets()); + + // test happy path + final Accept acceptApplication = Accept.create(MediaRanges.create(MediaTypes.APPLICATION_JSON)); + routes.run(HttpRequest.GET("/tweets?n=2").addHeader(acceptApplication)) + .assertStatusCode(200) + .assertEntity("[{\"message\":\"Hello World!\"},{\"message\":\"Hello World!\"}]"); + + // test responses to potential errors + final Accept acceptText = Accept.create(MediaRanges.ALL_TEXT); + routes.run(HttpRequest.GET("/tweets?n=3").addHeader(acceptText)) + .assertStatusCode(StatusCodes.NOT_ACCEPTABLE) // 406 + .assertEntity("Resource representation is only available with these types:\napplication/json"); + //#response-streaming + } + + //#models + private static final class JavaTweet { + private String message; + + public JavaTweet(String message) { + this.message = message; + } + + public void setMessage(String message) { + this.message = message; + } + + public String getMessage() { + return message; + } + + } + //#models +} diff --git a/akka-docs/rst/java/code/docs/http/javadsl/server/directives/FramedEntityStreamingExamplesTest.java b/akka-docs/rst/java/code/docs/http/javadsl/server/directives/FramedEntityStreamingExamplesTest.java deleted file mode 100644 index aee10b19359..00000000000 --- a/akka-docs/rst/java/code/docs/http/javadsl/server/directives/FramedEntityStreamingExamplesTest.java +++ /dev/null @@ -1,33 +0,0 @@ -/* - * Copyright (C) 2015-2016 Lightbend Inc. - */ - -package docs.http.javadsl.server.directives; - -import akka.http.javadsl.model.HttpRequest; -import akka.http.javadsl.model.HttpResponse; -import akka.http.javadsl.server.Route; -import akka.http.javadsl.server.directives.FramedEntityStreamingDirectives; -import akka.http.javadsl.server.directives.LogEntry; -import akka.http.javadsl.testkit.JUnitRouteTest; -import akka.http.scaladsl.server.Rejection; -import org.junit.Test; - -import java.util.List; -import java.util.Optional; -import java.util.function.BiFunction; -import java.util.function.Function; -import java.util.stream.Collectors; - -import static akka.event.Logging.InfoLevel; - -import static akka.http.javadsl.server.directives.FramedEntityStreamingDirectives.*; - -public class FramedEntityStreamingExamplesTest extends JUnitRouteTest { - - @Test - public void testRenderSource() { - FramedEntityStreamingDirectives. - } - -} diff --git a/akka-docs/rst/java/http/routing-dsl/index.rst b/akka-docs/rst/java/http/routing-dsl/index.rst index f84f20759fe..85fff2bcb51 100644 --- a/akka-docs/rst/java/http/routing-dsl/index.rst +++ b/akka-docs/rst/java/http/routing-dsl/index.rst @@ -18,6 +18,7 @@ To use the high-level API you need to add a dependency to the ``akka-http-experi directives/index marshalling exception-handling + source-streaming-support rejections testkit @@ -51,7 +52,6 @@ in the :ref:`exception-handling-java` section of the documtnation. You can use t File uploads ^^^^^^^^^^^^ -TODO not possible in Java DSL since there For high level directives to handle uploads see the :ref:`FileUploadDirectives-java`. diff --git a/akka-docs/rst/java/http/routing-dsl/source-streaming-support.rst b/akka-docs/rst/java/http/routing-dsl/source-streaming-support.rst new file mode 100644 index 00000000000..b83de64dc35 --- /dev/null +++ b/akka-docs/rst/java/http/routing-dsl/source-streaming-support.rst @@ -0,0 +1,74 @@ +.. _json-streaming-java: + +Source Streaming +================ + +Akka HTTP supports completing a request with an Akka ``Source``, which makes it possible to very easily build +streaming end-to-end APIs which apply back-pressure throughout the entire stack. + +It is possible to complete requests with raw ``Source``, however often it is more convenient to +stream on an element-by-element basis, and allow Akka HTTP to handle the rendering internally - for example as a JSON array, +or CSV stream (where each element is separated by a new-line). + +In the following sections we investigate how to make use of the JSON Streaming infrastructure, +however the general hints apply to any kind of element-by-element streaming you could imagine. + +It is possible to implement your own framing for any content type you might need, including bianary formats +by implementing :class:`FramingWithContentType`. + +JSON Streaming +============== + +`JSON Streaming`_ is a term refering to streaming a (possibly infinite) stream of element as independent JSON +objects as a continuous HTTP request or response. The elements are most often separated using newlines, +however do not have to be. Concatenating elements side-by-side or emitting "very long" JSON array is also another +use case. + +In the below examples, we'll be refering to the ``User`` and ``Measurement`` case classes as our model, which are defined as: + +.. includecode:: ../../code/docs/http/javadsl/server/JsonStreamingExamplesTest.java#models + +.. _Json Streaming: https://en.wikipedia.org/wiki/JSON_Streaming + +Responding with JSON Streams +---------------------------- + +In this example we implement an API representing an infinite stream of tweets, very much like Twitter's `Streaming API`_. + +Firstly, we'll need to get some additional marshalling infrastructure set up, that is able to marshal to and from an +Akka Streams ``Source``. One such trait, containing the needed marshallers is ``SprayJsonSupport``, which uses +spray-json (a high performance json parser library), and is shipped as part of Akka HTTP in the +``akka-http-spray-json-experimental`` module. + +The last bit of setup, before we can render a streaming json response + +.. includecode:: ../../code/docs/http/javadsl/server/JsonStreamingExamplesTest.java#response-streaming + +.. _Streaming API: https://dev.twitter.com/streaming/overview + +Consuming JSON Streaming uploads +-------------------------------- + +Sometimes the client may be sending a streaming request, for example an embedded device initiated a connection with +the server and is feeding it with one line of measurement data. + +In this example, we want to consume this data in a streaming fashion from the request entity, and also apply +back-pressure to the underlying TCP connection, if the server can not cope with the rate of incoming data (back-pressure +will be applied automatically thanks to using Akka HTTP/Streams). + +.. includecode:: ../../code/docs/http/javadsl/server/JsonStreamingExamplesTest.java#formats + +.. includecode:: ../../code/docs/http/javadsl/server/JsonStreamingExamplesTest.java#incoming-request-streaming + +Implementing custom (Un)Marshaller support for JSON streaming +------------------------------------------------------------- + +The following types that may need to be implemented by a custom framed-streaming support library are: + +- ``SourceRenderingMode`` which can customise how to render the begining / between-elements and ending of such + stream (while writing a response, i.e. by calling ``complete(source)``). + Implementations for JSON are available in ``akka.http.scaladsl.common.JsonSourceRenderingMode``. +- ``FramingWithContentType`` which is needed to be able to split incoming ``ByteString`` + chunks into frames of the higher-level data type format that is understood by the provided unmarshallers. + In the case of JSON it means chunking up ByteStrings such that each emitted element corresponds to exactly one JSON object, + this framing is implemented in ``EntityStreamingSupport``. diff --git a/akka-docs/rst/scala/code/docs/http/scaladsl/server/directives/JsonStreamingExamplesSpec.scala b/akka-docs/rst/scala/code/docs/http/scaladsl/server/directives/JsonStreamingExamplesSpec.scala index b3545048299..bb30f99612e 100644 --- a/akka-docs/rst/scala/code/docs/http/scaladsl/server/directives/JsonStreamingExamplesSpec.scala +++ b/akka-docs/rst/scala/code/docs/http/scaladsl/server/directives/JsonStreamingExamplesSpec.scala @@ -5,13 +5,13 @@ package docs.http.scaladsl.server.directives import akka.NotUsed +import akka.http.scaladsl.common.{ FramingWithContentType, JsonSourceRenderingModes } import akka.http.scaladsl.marshalling.ToResponseMarshallable import akka.http.scaladsl.model._ import akka.http.scaladsl.model.headers.Accept -import akka.http.scaladsl.server.{ UnsupportedRequestContentTypeRejection, UnacceptedResponseContentTypeRejection, JsonSourceRenderingMode } +import akka.http.scaladsl.server.{ UnacceptedResponseContentTypeRejection, UnsupportedRequestContentTypeRejection } import akka.stream.scaladsl.{ Flow, Source } import docs.http.scaladsl.server.RoutingSpec -import spray.json.{ JsValue, JsObject, DefaultJsonProtocol } import scala.concurrent.Future @@ -30,7 +30,7 @@ class JsonStreamingExamplesSpec extends RoutingSpec { //#formats object MyJsonProtocol extends spray.json.DefaultJsonProtocol { - implicit val userFormat = jsonFormat2(Tweet.apply) + implicit val tweetFormat = jsonFormat2(Tweet.apply) implicit val measurementFormat = jsonFormat2(Measurement.apply) } //# @@ -43,19 +43,22 @@ class JsonStreamingExamplesSpec extends RoutingSpec { import MyJsonProtocol._ // [3] pick json rendering mode: - implicit val jsonRenderingMode = JsonSourceRenderingMode.LineByLine + // HINT: if you extend `akka.http.scaladsl.server.EntityStreamingSupport` + // it'll guide you to do so via abstract defs + val maximumObjectLength = 128 + implicit val jsonRenderingMode = JsonSourceRenderingModes.LineByLine val route = - path("users") { - val users: Source[Tweet, NotUsed] = getTweets() - complete(ToResponseMarshallable(users)) + path("tweets") { + val tweets: Source[Tweet, NotUsed] = getTweets() + complete(ToResponseMarshallable(tweets)) } // tests: val AcceptJson = Accept(MediaRange(MediaTypes.`application/json`)) val AcceptXml = Accept(MediaRange(MediaTypes.`text/xml`)) - Get("/users").withHeaders(AcceptJson) ~> route ~> check { + Get("/tweets").withHeaders(AcceptJson) ~> route ~> check { responseAs[String] shouldEqual """{"uid":1,"txt":"#Akka rocks!"}""" + "\n" + """{"uid":2,"txt":"Streaming is so hot right now!"}""" + "\n" + @@ -63,7 +66,7 @@ class JsonStreamingExamplesSpec extends RoutingSpec { } // endpoint can only marshal Json, so it will *reject* requests for application/xml: - Get("/users").withHeaders(AcceptXml) ~> route ~> check { + Get("/tweets").withHeaders(AcceptXml) ~> route ~> check { handled should ===(false) rejection should ===(UnacceptedResponseContentTypeRejection(Set(ContentTypes.`application/json`))) } @@ -72,19 +75,19 @@ class JsonStreamingExamplesSpec extends RoutingSpec { "response-streaming-modes" in { import akka.http.scaladsl.marshallers.sprayjson.SprayJsonSupport._ import MyJsonProtocol._ - implicit val jsonRenderingMode = JsonSourceRenderingMode.LineByLine + implicit val jsonRenderingMode = JsonSourceRenderingModes.LineByLine //#async-rendering - path("users") { - val users: Source[Tweet, NotUsed] = getTweets() - complete(users.renderAsync(parallelism = 8)) + path("tweets") { + val tweets: Source[Tweet, NotUsed] = getTweets() + complete(tweets.renderAsync(parallelism = 8)) } //# //#async-unordered-rendering - path("users" / "unordered") { - val users: Source[Tweet, NotUsed] = getTweets() - complete(users.renderAsyncUnordered(parallelism = 8)) + path("tweets" / "unordered") { + val tweets: Source[Tweet, NotUsed] = getTweets() + complete(tweets.renderAsyncUnordered(parallelism = 8)) } //# } @@ -94,7 +97,9 @@ class JsonStreamingExamplesSpec extends RoutingSpec { import akka.http.scaladsl.marshallers.sprayjson.SprayJsonSupport._ // [1.1] import framing mode - implicit val jsonFramingMode = akka.http.scaladsl.server.JsonEntityFramingSupport.bracketCountingJsonFraming(Int.MaxValue) + import akka.http.scaladsl.server.EntityStreamingSupport + implicit val jsonFramingMode: FramingWithContentType = + EntityStreamingSupport.bracketCountingJsonFraming(Int.MaxValue) // [2] import "my protocol", for unmarshalling Measurement objects: import MyJsonProtocol._ @@ -106,14 +111,10 @@ class JsonStreamingExamplesSpec extends RoutingSpec { path("metrics") { // [4] extract Source[Measurement, _] entity(asSourceOf[Measurement]) { measurements => - println("measurements = " + measurements) val measurementsSubmitted: Future[Int] = measurements .via(persistMetrics) - .runFold(0) { (cnt, _) => - println("cnt = " + cnt) - cnt + 1 - } + .runFold(0) { (cnt, _) => cnt + 1 } complete { measurementsSubmitted.map(n => Map("msg" -> s"""Total metrics received: $n""")) diff --git a/akka-docs/rst/scala/code/docs/http/scaladsl/server/directives/MarshallingDirectivesExamplesSpec.scala b/akka-docs/rst/scala/code/docs/http/scaladsl/server/directives/MarshallingDirectivesExamplesSpec.scala index a0720452cd5..8f7c855dc95 100644 --- a/akka-docs/rst/scala/code/docs/http/scaladsl/server/directives/MarshallingDirectivesExamplesSpec.scala +++ b/akka-docs/rst/scala/code/docs/http/scaladsl/server/directives/MarshallingDirectivesExamplesSpec.scala @@ -71,8 +71,8 @@ class MarshallingDirectivesExamplesSpec extends RoutingSpec { // tests: Get("/") ~> route ~> check { mediaType shouldEqual `application/json` - responseAs[String] should include(""""name": "Jane"""") - responseAs[String] should include(""""favoriteNumber": 42""") + responseAs[String] should include(""""name":"Jane"""") + responseAs[String] should include(""""favoriteNumber":42""") } } @@ -95,8 +95,8 @@ class MarshallingDirectivesExamplesSpec extends RoutingSpec { Post("/", HttpEntity(`application/json`, """{ "name": "Jane", "favoriteNumber" : 42 }""")) ~> route ~> check { mediaType shouldEqual `application/json` - responseAs[String] should include(""""name": "Jane"""") - responseAs[String] should include(""""favoriteNumber": 42""") + responseAs[String] should include(""""name":"Jane"""") + responseAs[String] should include(""""favoriteNumber":42""") } } } diff --git a/akka-docs/rst/scala/http/routing-dsl/index.rst b/akka-docs/rst/scala/http/routing-dsl/index.rst index a4e1ee51216..10942ef517b 100644 --- a/akka-docs/rst/scala/http/routing-dsl/index.rst +++ b/akka-docs/rst/scala/http/routing-dsl/index.rst @@ -23,6 +23,7 @@ static content serving. exception-handling path-matchers case-class-extraction + source-streaming-support testkit websocket-support diff --git a/akka-docs/rst/scala/http/routing-dsl/json-streaming-support.rst b/akka-docs/rst/scala/http/routing-dsl/source-streaming-support.rst similarity index 80% rename from akka-docs/rst/scala/http/routing-dsl/json-streaming-support.rst rename to akka-docs/rst/scala/http/routing-dsl/source-streaming-support.rst index 34dda6235b0..d6c7cbdc46d 100644 --- a/akka-docs/rst/scala/http/routing-dsl/json-streaming-support.rst +++ b/akka-docs/rst/scala/http/routing-dsl/source-streaming-support.rst @@ -1,11 +1,27 @@ .. _json-streaming-scala: +Source Streaming +================ + +Akka HTTP supports completing a request with an Akka ``Source[T, _]``, which makes it possible to very easily build +streaming end-to-end APIs which apply back-pressure throughout the entire stack. + +It is possible to complete requests with raw ``Source[ByteString, _]``, however often it is more convenient to +stream on an element-by-element basis, and allow Akka HTTP to handle the rendering internally - for example as a JSON array, +or CSV stream (where each element is separated by a new-line). + +In the following sections we investigate how to make use of the JSON Streaming infrastructure, +however the general hints apply to any kind of element-by-element streaming you could imagine. + +It is possible to implement your own framing for any content type you might need, including bianary formats +by implementing :class:`FramingWithContentType`. + JSON Streaming ============== `JSON Streaming`_ is a term refering to streaming a (possibly infinite) stream of element as independent JSON -objects onto one continious HTTP connection. The elements are most often separated using newlines, -however do not have to be and concatenating elements side-by-side or emitting "very long" JSON array is also another +objects as a continuous HTTP request or response. The elements are most often separated using newlines, +however do not have to be. Concatenating elements side-by-side or emitting "very long" JSON array is also another use case. In the below examples, we'll be refering to the ``User`` and ``Measurement`` case classes as our model, which are defined as: @@ -30,7 +46,6 @@ Firstly, we'll need to get some additional marshalling infrastructure set up, th Akka Streams ``Source[T,_]``. One such trait, containing the needed marshallers is ``SprayJsonSupport``, which uses spray-json (a high performance json parser library), and is shipped as part of Akka HTTP in the ``akka-http-spray-json-experimental`` module. -to and from ``Source[T,_]`` by using spray-json provided Next we import our model's marshallers, generated by spray-json. @@ -68,7 +83,7 @@ in case one element in front of the stream takes a long time to marshall, yet ot Consuming JSON Streaming uploads -------------------------------- -Sometimes the client may be sending in a streaming request, for example an embedded device initiated a connection with +Sometimes the client may be sending a streaming request, for example an embedded device initiated a connection with the server and is feeding it with one line of measurement data. In this example, we want to consume this data in a streaming fashion from the request entity, and also apply diff --git a/akka-http-marshallers-java/akka-http-jackson/src/main/java/akka/http/javadsl/marshallers/jackson/Jackson.java b/akka-http-marshallers-java/akka-http-jackson/src/main/java/akka/http/javadsl/marshallers/jackson/Jackson.java index eb9e8ec78db..92cd6b9bb61 100644 --- a/akka-http-marshallers-java/akka-http-jackson/src/main/java/akka/http/javadsl/marshallers/jackson/Jackson.java +++ b/akka-http-marshallers-java/akka-http-jackson/src/main/java/akka/http/javadsl/marshallers/jackson/Jackson.java @@ -11,6 +11,7 @@ import akka.http.javadsl.marshalling.Marshaller; import akka.http.javadsl.unmarshalling.Unmarshaller; +import akka.util.ByteString; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.MapperFeature; import com.fasterxml.jackson.databind.ObjectMapper; @@ -31,6 +32,10 @@ public static Marshaller marshaller(ObjectMapper mapper) { ); } + public static Unmarshaller byteStringUnmarshaller(Class expectedType) { + return byteStringUnmarshaller(defaultObjectMapper, expectedType); + } + public static Unmarshaller unmarshaller(Class expectedType) { return unmarshaller(defaultObjectMapper, expectedType); } @@ -39,6 +44,10 @@ public static Unmarshaller unmarshaller(ObjectMapper mapper, return Unmarshaller.forMediaType(MediaTypes.APPLICATION_JSON, Unmarshaller.entityToString()) .thenApply(s -> fromJSON(mapper, s, expectedType)); } + + public static Unmarshaller byteStringUnmarshaller(ObjectMapper mapper, Class expectedType) { + return Unmarshaller.sync(s -> fromJSON(mapper, s.utf8String(), expectedType)); + } private static String toJSON(ObjectMapper mapper, Object object) { try { diff --git a/akka-http-marshallers-scala/akka-http-spray-json/src/main/scala/akka/http/scaladsl/marshallers/sprayjson/SprayJsonSupport.scala b/akka-http-marshallers-scala/akka-http-spray-json/src/main/scala/akka/http/scaladsl/marshallers/sprayjson/SprayJsonSupport.scala index c62f51314f6..ea998045020 100644 --- a/akka-http-marshallers-scala/akka-http-spray-json/src/main/scala/akka/http/scaladsl/marshallers/sprayjson/SprayJsonSupport.scala +++ b/akka-http-marshallers-scala/akka-http-spray-json/src/main/scala/akka/http/scaladsl/marshallers/sprayjson/SprayJsonSupport.scala @@ -34,11 +34,11 @@ trait SprayJsonSupport { JsonParser(input) } - implicit def sprayJsonMarshallerConverter[T](writer: RootJsonWriter[T])(implicit printer: JsonPrinter = PrettyPrinter): ToEntityMarshaller[T] = + implicit def sprayJsonMarshallerConverter[T](writer: RootJsonWriter[T])(implicit printer: JsonPrinter = CompactPrinter): ToEntityMarshaller[T] = sprayJsonMarshaller[T](writer, printer) implicit def sprayJsonMarshaller[T](implicit writer: RootJsonWriter[T], printer: JsonPrinter = CompactPrinter): ToEntityMarshaller[T] = sprayJsValueMarshaller compose writer.write - implicit def sprayJsValueMarshaller(implicit printer: JsonPrinter = PrettyPrinter): ToEntityMarshaller[JsValue] = + implicit def sprayJsValueMarshaller(implicit printer: JsonPrinter = CompactPrinter): ToEntityMarshaller[JsValue] = Marshaller.StringMarshaller.wrap(MediaTypes.`application/json`)(printer) implicit def sprayByteStringMarshaller[T](implicit writer: RootJsonFormat[T], printer: JsonPrinter = CompactPrinter): Marshaller[T, ByteString] = sprayJsValueMarshaller.map(s ⇒ ByteString(s.toString)) compose writer.write diff --git a/akka-http-tests/src/test/java/akka/http/javadsl/server/JavaTestServer.java b/akka-http-tests/src/test/java/akka/http/javadsl/server/JavaTestServer.java index 65ca1cec44d..70900a9987c 100644 --- a/akka-http-tests/src/test/java/akka/http/javadsl/server/JavaTestServer.java +++ b/akka-http-tests/src/test/java/akka/http/javadsl/server/JavaTestServer.java @@ -17,6 +17,7 @@ import akka.stream.ActorMaterializer; import akka.stream.javadsl.Flow; import akka.stream.javadsl.Source; +import akka.util.ByteString; import scala.concurrent.duration.Duration; import scala.runtime.BoxedUnit; @@ -64,7 +65,7 @@ public Route createRoute() { path("java", () -> completeOKWithFutureString(CompletableFuture.supplyAsync(() -> { throw new RuntimeException("Boom!"); })))) ); - final Unmarshaller JavaTweets = Jackson.unmarshaller(JavaTweet.class); + final Unmarshaller JavaTweets = Jackson.byteStringUnmarshaller(JavaTweet.class); final Route tweets = path("tweets", () -> get(() -> parameter(StringUnmarshallers.INTEGER, "n", n -> { diff --git a/akka-http-tests/src/test/scala/akka/http/scaladsl/server/directives/MarshallingDirectivesSpec.scala b/akka-http-tests/src/test/scala/akka/http/scaladsl/server/directives/MarshallingDirectivesSpec.scala index 50be2b4c584..fc049363532 100644 --- a/akka-http-tests/src/test/scala/akka/http/scaladsl/server/directives/MarshallingDirectivesSpec.scala +++ b/akka-http-tests/src/test/scala/akka/http/scaladsl/server/directives/MarshallingDirectivesSpec.scala @@ -184,7 +184,7 @@ class MarshallingDirectivesSpec extends RoutingSpec with Inside { "render JSON with UTF-8 encoding if no `Accept-Charset` request header is present" in { Get() ~> complete(foo) ~> check { - responseEntity shouldEqual HttpEntity(`application/json`, foo.toJson.prettyPrint) + responseEntity shouldEqual HttpEntity(`application/json`, foo.toJson.compactPrint) } } "reject JSON rendering if an `Accept-Charset` request header requests a non-UTF-8 encoding" in { diff --git a/akka-http-tests/src/test/scala/akka/http/scaladsl/server/directives/RouteDirectivesSpec.scala b/akka-http-tests/src/test/scala/akka/http/scaladsl/server/directives/RouteDirectivesSpec.scala index 26e9f65b294..df9ee7c2b54 100644 --- a/akka-http-tests/src/test/scala/akka/http/scaladsl/server/directives/RouteDirectivesSpec.scala +++ b/akka-http-tests/src/test/scala/akka/http/scaladsl/server/directives/RouteDirectivesSpec.scala @@ -98,10 +98,7 @@ class RouteDirectivesSpec extends FreeSpec with GenericRoutingSpec { import akka.http.scaladsl.model.headers.Accept Get().withHeaders(Accept(MediaTypes.`application/json`)) ~> route ~> check { responseAs[String] shouldEqual - """{ - | "name": "Ida", - | "age": 83 - |}""".stripMarginWithNewline("\n") + """{"name":"Ida","age":83}""" } Get().withHeaders(Accept(MediaTypes.`text/xml`)) ~> route ~> check { responseAs[xml.NodeSeq] shouldEqual Ida83 diff --git a/akka-http/src/main/scala/akka/http/javadsl/common/FramingWithContentType.scala b/akka-http/src/main/scala/akka/http/javadsl/common/FramingWithContentType.scala index 127c9bf8902..483b3d624fd 100644 --- a/akka-http/src/main/scala/akka/http/javadsl/common/FramingWithContentType.scala +++ b/akka-http/src/main/scala/akka/http/javadsl/common/FramingWithContentType.scala @@ -4,13 +4,23 @@ package akka.http.javadsl.common +import akka.NotUsed +import akka.event.Logging import akka.http.javadsl.model.ContentTypeRange -import akka.stream.javadsl.Framing +import akka.stream.javadsl.{ Flow, Framing } +import akka.util.ByteString -trait FramingWithContentType extends Framing { self ⇒ +/** + * Wraps a framing [[akka.stream.javadsl.Flow]] (as provided by [[Framing]] for example) + * that chunks up incoming [[akka.util.ByteString]] according to some [[akka.http.javadsl.model.ContentType]] + * specific logic. + */ +abstract class FramingWithContentType { self ⇒ import akka.http.impl.util.JavaMapping.Implicits._ - override def asScala: akka.http.scaladsl.common.FramingWithContentType = + def getFlow: Flow[ByteString, ByteString, NotUsed] + + def asScala: akka.http.scaladsl.common.FramingWithContentType = this match { case f: akka.http.scaladsl.common.FramingWithContentType ⇒ f case _ ⇒ new akka.http.scaladsl.common.FramingWithContentType { @@ -21,4 +31,6 @@ trait FramingWithContentType extends Framing { self ⇒ def supported: ContentTypeRange def matches(ct: akka.http.javadsl.model.ContentType): Boolean = supported.matches(ct) + + override def toString = s"${Logging.simpleName(getClass)}($supported)" } diff --git a/akka-http/src/main/scala/akka/http/javadsl/common/JsonSourceRenderingMode.scala b/akka-http/src/main/scala/akka/http/javadsl/common/JsonSourceRenderingMode.scala index b5fe32fdf93..7d4ca7b7275 100644 --- a/akka-http/src/main/scala/akka/http/javadsl/common/JsonSourceRenderingMode.scala +++ b/akka-http/src/main/scala/akka/http/javadsl/common/JsonSourceRenderingMode.scala @@ -10,6 +10,8 @@ import akka.http.javadsl.model.{ ContentType, ContentTypes } * Specialised rendering mode for streaming elements as JSON. * * See also: JSON Streaming on Wikipedia. + * + * See [[JsonSourceRenderingModes]] for commonly used pre-defined rendering modes. */ trait JsonSourceRenderingMode extends SourceRenderingMode { override val contentType: ContentType.WithFixedCharset = diff --git a/akka-http/src/main/scala/akka/http/javadsl/server/EntityStreamingSupport.scala b/akka-http/src/main/scala/akka/http/javadsl/server/EntityStreamingSupport.scala new file mode 100644 index 00000000000..571867ac634 --- /dev/null +++ b/akka-http/src/main/scala/akka/http/javadsl/server/EntityStreamingSupport.scala @@ -0,0 +1,41 @@ +/* + * Copyright (C) 2016 Lightbend Inc. + */ + +package akka.http.javadsl.server + +import akka.NotUsed +import akka.http.javadsl.common.{ FramingWithContentType, SourceRenderingMode } +import akka.http.javadsl.model.{ ContentTypeRange, MediaRanges } +import akka.http.scaladsl.server.ApplicationJsonBracketCountingFraming +import akka.stream.javadsl.{ Flow, Framing } +import akka.util.ByteString + +/** + * Entity streaming support, independent of used Json parsing library etc. + * + * Can be extended by various Support traits (e.g. "SprayJsonSupport"), + * in order to provide users with both `framing` (this trait) and `marshalling` + * (implemented by a library) by using a single trait. + */ +object EntityStreamingSupport { + // in the ScalaDSL version we make users implement abstract methods that are supposed to be + // implicit vals. This helps to guide in implementing the needed values, however in Java that would not really help. + + /** `application/json` specific Framing implementation */ + def bracketCountingJsonFraming(maximumObjectLength: Int): FramingWithContentType = + new ApplicationJsonBracketCountingFraming(maximumObjectLength) + + /** + * Frames incoming `text / *` entities on a line-by-line basis. + * Useful for accepting `text/csv` uploads as a stream of rows. + */ + def newLineFraming(maximumObjectLength: Int, supportedContentTypes: ContentTypeRange): FramingWithContentType = + new FramingWithContentType { + override final val getFlow: Flow[ByteString, ByteString, NotUsed] = + Flow.of(classOf[ByteString]).via(Framing.delimiter(ByteString("\n"), maximumObjectLength)) + + override final val supported: ContentTypeRange = + akka.http.scaladsl.model.ContentTypeRange(akka.http.scaladsl.model.MediaRanges.`text/*`) + } +} diff --git a/akka-http/src/main/scala/akka/http/javadsl/server/directives/FramedEntityStreamingDirectives.scala b/akka-http/src/main/scala/akka/http/javadsl/server/directives/FramedEntityStreamingDirectives.scala index e39288a8b20..153ee0601e8 100644 --- a/akka-http/src/main/scala/akka/http/javadsl/server/directives/FramedEntityStreamingDirectives.scala +++ b/akka-http/src/main/scala/akka/http/javadsl/server/directives/FramedEntityStreamingDirectives.scala @@ -3,35 +3,25 @@ */ package akka.http.javadsl.server.directives -import akka.http.javadsl.model.{ ContentType, HttpEntity } -import akka.util.ByteString -import java.util.{ List ⇒ JList, Map ⇒ JMap } -import java.util.AbstractMap.SimpleImmutableEntry -import java.util.Optional import java.util.function.{ Function ⇒ JFunction } +import java.util.{ List ⇒ JList, Map ⇒ JMap } import akka.NotUsed - -import scala.collection.JavaConverters._ -import akka.http.impl.util.JavaMapping.Implicits._ import akka.http.javadsl.common.{ FramingWithContentType, SourceRenderingMode } +import akka.http.javadsl.model.{ HttpEntity, _ } import akka.http.javadsl.server.{ Marshaller, Route, Unmarshaller } -import akka.http.javadsl.model._ -import akka.http.scaladsl.marshalling.{ ToResponseMarshallable, ToResponseMarshaller } +import akka.http.scaladsl.marshalling.ToResponseMarshallable import akka.http.scaladsl.server.{ Directives ⇒ D } -import akka.http.scaladsl.unmarshalling import akka.stream.javadsl.Source +import akka.util.ByteString /** EXPERIMENTAL API */ abstract class FramedEntityStreamingDirectives extends TimeoutDirectives { - // important import, as we implicitly resolve some marshallers inside the below directives - import akka.http.scaladsl.server.directives.FramedEntityStreamingDirectives._ @CorrespondsTo("asSourceOf") - def entityasSourceOf[T](um: Unmarshaller[HttpEntity, T], framing: FramingWithContentType, + def entityasSourceOf[T](um: Unmarshaller[ByteString, T], framing: FramingWithContentType, inner: java.util.function.Function[Source[T, NotUsed], Route]): Route = RouteAdapter { - val sum = um.asScalaCastInput[akka.http.scaladsl.model.HttpEntity] - D.entity(D.asSourceOf[T](framing.asScala)(sum)) { s: akka.stream.scaladsl.Source[T, NotUsed] ⇒ + D.entity(D.asSourceOf[T](framing.asScala)(um.asScala)) { s: akka.stream.scaladsl.Source[T, NotUsed] ⇒ inner(s.asJava).delegate } } @@ -39,10 +29,9 @@ abstract class FramedEntityStreamingDirectives extends TimeoutDirectives { @CorrespondsTo("asSourceOfAsync") def entityAsSourceAsyncOf[T]( parallelism: Int, - um: Unmarshaller[HttpEntity, T], framing: FramingWithContentType, + um: Unmarshaller[ByteString, T], framing: FramingWithContentType, inner: java.util.function.Function[Source[T, NotUsed], Route]): Route = RouteAdapter { - val sum = um.asScalaCastInput[akka.http.scaladsl.model.HttpEntity] - D.entity(D.asSourceOfAsync[T](parallelism, framing.asScala)(sum)) { s: akka.stream.scaladsl.Source[T, NotUsed] ⇒ + D.entity(D.asSourceOfAsync[T](parallelism, framing.asScala)(um.asScala)) { s: akka.stream.scaladsl.Source[T, NotUsed] ⇒ inner(s.asJava).delegate } } @@ -50,18 +39,16 @@ abstract class FramedEntityStreamingDirectives extends TimeoutDirectives { @CorrespondsTo("asSourceOfAsyncUnordered") def entityAsSourceAsyncUnorderedOf[T]( parallelism: Int, - um: Unmarshaller[HttpEntity, T], framing: FramingWithContentType, + um: Unmarshaller[ByteString, T], framing: FramingWithContentType, inner: java.util.function.Function[Source[T, NotUsed], Route]): Route = RouteAdapter { - val sum = um.asScalaCastInput[akka.http.scaladsl.model.HttpEntity] - D.entity(D.asSourceOfAsyncUnordered[T](parallelism, framing.asScala)(sum)) { s: akka.stream.scaladsl.Source[T, NotUsed] ⇒ + D.entity(D.asSourceOfAsyncUnordered[T](parallelism, framing.asScala)(um.asScala)) { s: akka.stream.scaladsl.Source[T, NotUsed] ⇒ inner(s.asJava).delegate } } - // implicit used internally, Java caller does not benefit or use it + // implicits used internally, Java caller does not benefit or use it @CorrespondsTo("complete") def completeWithSource[T, M](implicit source: Source[T, M], m: Marshaller[T, ByteString], rendering: SourceRenderingMode): Route = RouteAdapter { - import akka.http.scaladsl.marshalling.PredefinedToResponseMarshallers._ implicit val mm = _sourceMarshaller(m.map(ByteStringAsEntityFn), rendering) val response = ToResponseMarshallable(source) D.complete(response) @@ -75,8 +62,8 @@ abstract class FramedEntityStreamingDirectives extends TimeoutDirectives { } implicit private def _sourceMarshaller[T, M](implicit m: Marshaller[T, HttpEntity], rendering: SourceRenderingMode) = { - import akka.http.javadsl.server.RoutingJavaMapping._ import akka.http.javadsl.server.RoutingJavaMapping.Implicits._ + import akka.http.javadsl.server.RoutingJavaMapping._ val mm = m.asScalaCastOutput D._sourceMarshaller[T, M](mm, rendering.asScala).compose({ h: akka.stream.javadsl.Source[T, M] ⇒ h.asScala }) } diff --git a/akka-http/src/main/scala/akka/http/scaladsl/common/FramingWithContentType.scala b/akka-http/src/main/scala/akka/http/scaladsl/common/FramingWithContentType.scala index 75743b674de..13dc72ecbad 100644 --- a/akka-http/src/main/scala/akka/http/scaladsl/common/FramingWithContentType.scala +++ b/akka-http/src/main/scala/akka/http/scaladsl/common/FramingWithContentType.scala @@ -6,16 +6,18 @@ package akka.http.scaladsl.common import akka.NotUsed import akka.event.Logging -import akka.http.scaladsl.model.{ ContentType, ContentTypeRange } +import akka.http.scaladsl.model.ContentTypeRange import akka.stream.scaladsl.{ Flow, Framing } import akka.util.ByteString /** - * Same as [[akka.stream.scaladsl.Framing]] but additionally can express which [[ContentType]] it supports, - * which can be used to reject routes if content type does not match used framing. + * Wraps a framing [[akka.stream.scaladsl.Flow]] (as provided by [[Framing]] for example) + * that chunks up incoming [[akka.util.ByteString]] according to some [[akka.http.javadsl.model.ContentType]] + * specific logic. */ -abstract class FramingWithContentType extends akka.http.javadsl.common.FramingWithContentType with Framing { +abstract class FramingWithContentType extends akka.http.javadsl.common.FramingWithContentType { def flow: Flow[ByteString, ByteString, NotUsed] + override final def getFlow = flow.asJava override def supported: ContentTypeRange override def matches(ct: akka.http.javadsl.model.ContentType): Boolean = supported.matches(ct) diff --git a/akka-http/src/main/scala/akka/http/scaladsl/common/JsonSourceRenderingMode.scala b/akka-http/src/main/scala/akka/http/scaladsl/common/JsonSourceRenderingMode.scala index 52e3d1b7549..824af23b8c3 100644 --- a/akka-http/src/main/scala/akka/http/scaladsl/common/JsonSourceRenderingMode.scala +++ b/akka-http/src/main/scala/akka/http/scaladsl/common/JsonSourceRenderingMode.scala @@ -11,6 +11,8 @@ import akka.util.ByteString * Specialised rendering mode for streaming elements as JSON. * * See also: JSON Streaming on Wikipedia. + * + * See [[JsonSourceRenderingModes]] for commonly used pre-defined rendering modes. */ trait JsonSourceRenderingMode extends akka.http.javadsl.common.JsonSourceRenderingMode with SourceRenderingMode { override val contentType: ContentType.WithFixedCharset = diff --git a/akka-http/src/main/scala/akka/http/scaladsl/server/EntityStreamingSupport.scala b/akka-http/src/main/scala/akka/http/scaladsl/server/EntityStreamingSupport.scala index 9b6f9bf9889..f4656e8612c 100644 --- a/akka-http/src/main/scala/akka/http/scaladsl/server/EntityStreamingSupport.scala +++ b/akka-http/src/main/scala/akka/http/scaladsl/server/EntityStreamingSupport.scala @@ -4,13 +4,10 @@ package akka.http.scaladsl.server import akka.NotUsed -import akka.actor.ActorSystem -import akka.event.Logging import akka.http.scaladsl.common.{ FramingWithContentType, SourceRenderingMode } -import akka.http.scaladsl.model.{ ContentTypeRange, ContentTypes, MediaRange, MediaRanges } +import akka.http.scaladsl.model.{ ContentTypeRange, ContentTypes, MediaRanges } import akka.stream.scaladsl.{ Flow, Framing } import akka.util.ByteString -import com.typesafe.config.Config /** * Entity streaming support, independent of used Json parsing library etc. @@ -45,13 +42,7 @@ trait EntityStreamingSupportBase { * Useful for accepting `text/csv` uploads as a stream of rows. */ def newLineFraming(maximumObjectLength: Int, supportedContentTypes: ContentTypeRange): FramingWithContentType = - new FramingWithContentType { - override final val flow: Flow[ByteString, ByteString, NotUsed] = - Flow[ByteString].via(Framing.delimiter(ByteString("\n"), maximumObjectLength)) - - override final val supported: ContentTypeRange = - ContentTypeRange(MediaRanges.`text/*`) - } + new TextNewLineFraming(maximumObjectLength, supportedContentTypes) } /** @@ -67,3 +58,11 @@ final class ApplicationJsonBracketCountingFraming(maximumObjectLength: Int) exte override final val flow = Flow[ByteString].via(akka.stream.scaladsl.JsonFraming.bracketCounting(maximumObjectLength)) override final val supported = ContentTypeRange(ContentTypes.`application/json`) } + +final class TextNewLineFraming(maximumLineLength: Int, supportedContentTypes: ContentTypeRange) extends FramingWithContentType { + override final val flow: Flow[ByteString, ByteString, NotUsed] = + Flow[ByteString].via(Framing.delimiter(ByteString("\n"), maximumLineLength)) + + override final val supported: ContentTypeRange = + ContentTypeRange(MediaRanges.`text/*`) +} diff --git a/akka-http/src/main/scala/akka/http/scaladsl/server/directives/FramedEntityStreamingDirectives.scala b/akka-http/src/main/scala/akka/http/scaladsl/server/directives/FramedEntityStreamingDirectives.scala index c93c6e4020a..1e97170a11b 100644 --- a/akka-http/src/main/scala/akka/http/scaladsl/server/directives/FramedEntityStreamingDirectives.scala +++ b/akka-http/src/main/scala/akka/http/scaladsl/server/directives/FramedEntityStreamingDirectives.scala @@ -11,48 +11,150 @@ import akka.http.scaladsl.unmarshalling.{ Unmarshal, Unmarshaller, _ } import akka.http.scaladsl.util.FastFuture import akka.stream.Materializer import akka.stream.impl.ConstantFun -import akka.stream.scaladsl.{ Flow, Source } +import akka.stream.scaladsl.{ Flow, Keep, Source } import akka.util.ByteString import scala.concurrent.ExecutionContext import scala.language.implicitConversions /** - * Allows the [[MarshallingDirectives.entity]] directive to extract a `stream[T]` for framed messages. - * See `JsonEntityStreamingSupport` and classes extending it, such as `SprayJsonSupport` to get marshallers. + * Allows the [[MarshallingDirectives.entity]] directive to extract a [[Source]] of elements. + * + * See [[akka.http.scaladsl.server.EntityStreamingSupport]] for useful default [[FramingWithContentType]] instances and + * support traits such as `SprayJsonSupport` (or your other favourite JSON library) to provide the needed [[Marshaller]] s. */ trait FramedEntityStreamingDirectives extends MarshallingDirectives { import FramedEntityStreamingDirectives._ type RequestToSourceUnmarshaller[T] = FromRequestUnmarshaller[Source[T, NotUsed]] - // TODO DOCS - - final def asSourceOf[T](implicit um: Unmarshaller[HttpEntity, T], framing: FramingWithContentType): RequestToSourceUnmarshaller[T] = + /** + * Extracts entity as [[Source]] of elements of type `T`. + * This is achieved by applying the implicitly provided (in the following order): + * + * - 1st: [[FramingWithContentType]] in order to chunk-up the incoming [[ByteString]]s according to the + * `Content-Type` aware framing (for example, [[akka.http.scaladsl.server.EntityStreamingSupport.bracketCountingJsonFraming]]). + * - 2nd: [[Unmarshaller]] (from [[ByteString]] to `T`) for each of the respective "chunks" (e.g. for each JSON element contained within an array). + * + * The request will be rejected with an [[akka.http.scaladsl.server.UnsupportedRequestContentTypeRejection]] if + * its [[ContentType]] is not supported by the used `framing` or `unmarshaller`. + * + * It is recommended to use the [[akka.http.scaladsl.server.EntityStreamingSupport]] trait in conjunction with this + * directive as it helps provide the right [[FramingWithContentType]] and [[SourceRenderingMode]] for the most + * typical usage scenarios (JSON, CSV, ...). + * + * Cancelling extracted [[Source]] closes the connection abruptly (same as cancelling the `entity.dataBytes`). + * + * If looking to improve marshalling performance in face of many elements (possibly of different sizes), + * you may be interested in using [[asSourceOfAsyncUnordered]] instead. + * + * See also [[MiscDirectives.withoutSizeLimit]] as you may want to allow streaming infinite streams of data in this route. + * By default the uploaded data is limited by the `akka.http.parsing.max-content-length`. + */ + final def asSourceOf[T](implicit um: Unmarshaller[ByteString, T], framing: FramingWithContentType): RequestToSourceUnmarshaller[T] = asSourceOfAsync(1)(um, framing) - final def asSourceOf[T](framing: FramingWithContentType)(implicit um: Unmarshaller[HttpEntity, T]): RequestToSourceUnmarshaller[T] = + + /** + * Extracts entity as [[Source]] of elements of type `T`. + * This is achieved by applying the implicitly provided (in the following order): + * + * - 1st: [[FramingWithContentType]] in order to chunk-up the incoming [[ByteString]]s according to the + * `Content-Type` aware framing (for example, [[akka.http.scaladsl.server.EntityStreamingSupport.bracketCountingJsonFraming]]). + * - 2nd: [[Unmarshaller]] (from [[ByteString]] to `T`) for each of the respective "chunks" (e.g. for each JSON element contained within an array). + * + * The request will be rejected with an [[akka.http.scaladsl.server.UnsupportedRequestContentTypeRejection]] if + * its [[ContentType]] is not supported by the used `framing` or `unmarshaller`. + * + * It is recommended to use the [[akka.http.scaladsl.server.EntityStreamingSupport]] trait in conjunction with this + * directive as it helps provide the right [[FramingWithContentType]] and [[SourceRenderingMode]] for the most + * typical usage scenarios (JSON, CSV, ...). + * + * Cancelling extracted [[Source]] closes the connection abruptly (same as cancelling the `entity.dataBytes`). + * + * If looking to improve marshalling performance in face of many elements (possibly of different sizes), + * you may be interested in using [[asSourceOfAsyncUnordered]] instead. + * + * See also [[MiscDirectives.withoutSizeLimit]] as you may want to allow streaming infinite streams of data in this route. + * By default the uploaded data is limited by the `akka.http.parsing.max-content-length`. + */ + final def asSourceOf[T](framing: FramingWithContentType)(implicit um: Unmarshaller[ByteString, T]): RequestToSourceUnmarshaller[T] = asSourceOfAsync(1)(um, framing) - final def asSourceOfAsync[T](parallelism: Int)(implicit um: Unmarshaller[HttpEntity, T], framing: FramingWithContentType): RequestToSourceUnmarshaller[T] = - asSourceOfInternal[T](framing, (ec, mat) ⇒ Flow[HttpEntity].mapAsync(parallelism)(Unmarshal(_).to[T](um, ec, mat))) - final def asSourceOfAsync[T](parallelism: Int, framing: FramingWithContentType)(implicit um: Unmarshaller[HttpEntity, T]): RequestToSourceUnmarshaller[T] = + /** + * Similar to [[asSourceOf]] however will apply at most `parallelism` unmarshallers in parallel. + * + * The source elements emitted preserve the order in which they are sent in the incoming [[HttpRequest]]. + * If you want to sacrivice ordering in favour of (potential) slight performance improvements in reading the input + * you may want to use [[asSourceOfAsyncUnordered]] instead, which lifts the ordering guarantee. + * + * Refer to [[asSourceOf]] for more in depth-documentation and guidelines. + * + * If looking to improve marshalling performance in face of many elements (possibly of different sizes), + * you may be interested in using [[asSourceOfAsyncUnordered]] instead. + * + * See also [[MiscDirectives.withoutSizeLimit]] as you may want to allow streaming infinite streams of data in this route. + * By default the uploaded data is limited by the `akka.http.parsing.max-content-length`. + */ + final def asSourceOfAsync[T](parallelism: Int)(implicit um: Unmarshaller[ByteString, T], framing: FramingWithContentType): RequestToSourceUnmarshaller[T] = + asSourceOfInternal[T](framing, (ec, mat) ⇒ Flow[ByteString].mapAsync(parallelism)(Unmarshal(_).to[T](um, ec, mat))) + + /** + * Similar to [[asSourceOf]] however will apply at most `parallelism` unmarshallers in parallel. + * + * The source elements emitted preserve the order in which they are sent in the incoming [[HttpRequest]]. + * If you want to sacrivice ordering in favour of (potential) slight performance improvements in reading the input + * you may want to use [[asSourceOfAsyncUnordered]] instead, which lifts the ordering guarantee. + * + * Refer to [[asSourceOf]] for more in depth-documentation and guidelines. + * + * See also [[MiscDirectives.withoutSizeLimit]] as you may want to allow streaming infinite streams of data in this route. + * By default the uploaded data is limited by the `akka.http.parsing.max-content-length`. + */ + final def asSourceOfAsync[T](parallelism: Int, framing: FramingWithContentType)(implicit um: Unmarshaller[ByteString, T]): RequestToSourceUnmarshaller[T] = asSourceOfAsync(parallelism)(um, framing) - final def asSourceOfAsyncUnordered[T](parallelism: Int)(implicit um: Unmarshaller[HttpEntity, T], framing: FramingWithContentType): RequestToSourceUnmarshaller[T] = - asSourceOfInternal[T](framing, (ec, mat) ⇒ Flow[HttpEntity].mapAsyncUnordered(parallelism)(Unmarshal(_).to[T](um, ec, mat))) - final def asSourceOfAsyncUnordered[T](parallelism: Int, framing: FramingWithContentType)(implicit um: Unmarshaller[HttpEntity, T]): RequestToSourceUnmarshaller[T] = + /** + * Similar to [[asSourceOfAsync]], as it will apply at most `parallelism` unmarshallers in parallel. + * + * The source elements emitted preserve the order in which they are sent in the incoming [[HttpRequest]]. + * If you want to sacrivice ordering in favour of (potential) slight performance improvements in reading the input + * you may want to use [[asSourceOfAsyncUnordered]] instead, which lifts the ordering guarantee. + * + * Refer to [[asSourceOf]] for more in depth-documentation and guidelines. + * + * See also [[MiscDirectives.withoutSizeLimit]] as you may want to allow streaming infinite streams of data in this route. + * By default the uploaded data is limited by the `akka.http.parsing.max-content-length`. + */ + final def asSourceOfAsyncUnordered[T](parallelism: Int)(implicit um: Unmarshaller[ByteString, T], framing: FramingWithContentType): RequestToSourceUnmarshaller[T] = + asSourceOfInternal[T](framing, (ec, mat) ⇒ Flow[ByteString].mapAsyncUnordered(parallelism)(Unmarshal(_).to[T](um, ec, mat))) + /** + * Similar to [[asSourceOfAsync]], as it will apply at most `parallelism` unmarshallers in parallel. + * + * The source elements emitted preserve the order in which they are sent in the incoming [[HttpRequest]]. + * If you want to sacrivice ordering in favour of (potential) slight performance improvements in reading the input + * you may want to use [[asSourceOfAsyncUnordered]] instead, which lifts the ordering guarantee. + * + * Refer to [[asSourceOf]] for more in depth-documentation and guidelines. + * + * See also [[MiscDirectives.withoutSizeLimit]] as you may want to allow streaming infinite streams of data in this route. + * By default the uploaded data is limited by the `akka.http.parsing.max-content-length`. + */ + final def asSourceOfAsyncUnordered[T](parallelism: Int, framing: FramingWithContentType)(implicit um: Unmarshaller[ByteString, T]): RequestToSourceUnmarshaller[T] = asSourceOfAsyncUnordered(parallelism)(um, framing) // format: OFF - private def asSourceOfInternal[T](framing: FramingWithContentType, marshalling: (ExecutionContext, Materializer) => Flow[HttpEntity, ByteString, NotUsed]#ReprMat[T, NotUsed]): RequestToSourceUnmarshaller[T] = + private final def asSourceOfInternal[T](framing: FramingWithContentType, marshalling: (ExecutionContext, Materializer) => Flow[ByteString, ByteString, NotUsed]#ReprMat[T, NotUsed]): RequestToSourceUnmarshaller[T] = Unmarshaller.withMaterializer[HttpRequest, Source[T, NotUsed]] { implicit ec ⇒ implicit mat ⇒ req ⇒ val entity = req.entity if (!framing.matches(entity.contentType)) { val supportedContentTypes = framing.supported FastFuture.failed(Unmarshaller.UnsupportedContentTypeException(supportedContentTypes)) } else { -// val stream = entity.dataBytes.via(framing.flow).via(marshalling(ec, mat)).mapMaterializedValue(_ => NotUsed) - val stream = Source.single(entity.transformDataBytes(framing.flow)).via(marshalling(ec, mat)).mapMaterializedValue(_ => NotUsed) + val bytes = entity.dataBytes + val frames = bytes.viaMat(framing.flow)(Keep.right) + val elements = frames.viaMat(marshalling(ec, mat))(Keep.right) + val stream = elements.mapMaterializedValue(_ => NotUsed) +// val stream = Source.single(entity.transformDataBytes(framing.flow)).via(marshalling(ec, mat)).mapMaterializedValue(_ => NotUsed) FastFuture.successful(stream) } } @@ -108,6 +210,11 @@ trait FramedEntityStreamingDirectives extends MarshallingDirectives { new EnableSpecialSourceRenderingModes(source) } +/** + * Allows the [[MarshallingDirectives.entity]] directive to extract a [[Source]] of elements. + * + * See [[FramedEntityStreamingDirectives]] for detailed documentation. + */ object FramedEntityStreamingDirectives extends FramedEntityStreamingDirectives { sealed class AsyncSourceRenderingMode final class AsyncRenderingOf[T](val source: Source[T, Any], val parallelism: Int) extends AsyncSourceRenderingMode @@ -115,6 +222,7 @@ object FramedEntityStreamingDirectives extends FramedEntityStreamingDirectives { } +/** Provides DSL for special rendering modes, e.g. `complete(source.renderAsync)` */ final class EnableSpecialSourceRenderingModes[T](val source: Source[T, Any]) extends AnyVal { /** * Causes the response stream to be marshalled asynchronously (up to `parallelism` elements at once), diff --git a/akka-stream-tests/src/test/scala/akka/stream/scaladsl/JsonFramingSpec.scala b/akka-stream-tests/src/test/scala/akka/stream/scaladsl/JsonFramingSpec.scala index ccd58690dde..e6b657fd592 100644 --- a/akka-stream-tests/src/test/scala/akka/stream/scaladsl/JsonFramingSpec.scala +++ b/akka-stream-tests/src/test/scala/akka/stream/scaladsl/JsonFramingSpec.scala @@ -21,7 +21,7 @@ class JsonFramingSpec extends AkkaSpec { implicit val mat = ActorMaterializer() "collecting multiple json" should { - "xoxo parse json array" in { + "parse json array" in { val input = """ |[ @@ -38,9 +38,9 @@ class JsonFramingSpec extends AkkaSpec { } result.futureValue shouldBe Seq( - """{ "name" : "john" }""".stripMargin, - """{ "name" : "jack" }""".stripMargin, - """{ "name" : "katie" }""".stripMargin) + """{ "name" : "john" }""", + """{ "name" : "jack" }""", + """{ "name" : "katie" }""") } "emit single json element from string" in { @@ -56,7 +56,7 @@ class JsonFramingSpec extends AkkaSpec { case (acc, entry) ⇒ acc ++ Seq(entry.utf8String) } - Await.result(result, 3.seconds) shouldBe Seq("""{ "name": "john" }""".stripMargin) + Await.result(result, 3.seconds) shouldBe Seq("""{ "name": "john" }""") } "parse line delimited" in { @@ -73,9 +73,9 @@ class JsonFramingSpec extends AkkaSpec { } Await.result(result, 3.seconds) shouldBe Seq( - """{ "name": "john" }""".stripMargin, - """{ "name": "jack" }""".stripMargin, - """{ "name": "katie" }""".stripMargin) + """{ "name": "john" }""", + """{ "name": "jack" }""", + """{ "name": "katie" }""") } "parse comma delimited" in { @@ -91,7 +91,7 @@ class JsonFramingSpec extends AkkaSpec { } result.futureValue shouldBe Seq( - """{ "name": "john" }""".stripMargin, + """{ "name": "john" }""", """{ "name": "jack" }""", """{ "name": "katie" }""") } @@ -121,7 +121,6 @@ class JsonFramingSpec extends AkkaSpec { } } - // TODO fold these specs into the previous section "collecting json buffer" when { "nothing is supplied" should { "return nothing" in { @@ -378,7 +377,7 @@ class JsonFramingSpec extends AkkaSpec { "returns none until valid json is encountered" in { val buffer = new JsonBracketCounting() - """{ "name": "john"""".stripMargin.foreach { + """{ "name": "john"""".foreach { c ⇒ buffer.offer(ByteString(c)) buffer.poll() should ===(None) @@ -434,7 +433,7 @@ class JsonFramingSpec extends AkkaSpec { probe.ensureSubscription() probe .request(1) - .expectNext(ByteString("""{ "name": "john" }""")) // FIXME we should not impact the given json in Framing + .expectNext(ByteString("""{ "name": "john" }""")) .request(1) .expectNext(ByteString("""{ "name": "jack" }""")) .request(1) diff --git a/akka-stream/src/main/scala/akka/stream/impl/JsonBracketCounting.scala b/akka-stream/src/main/scala/akka/stream/impl/JsonBracketCounting.scala index 7bd9e18ca27..850d3c0f687 100644 --- a/akka-stream/src/main/scala/akka/stream/impl/JsonBracketCounting.scala +++ b/akka-stream/src/main/scala/akka/stream/impl/JsonBracketCounting.scala @@ -8,15 +8,18 @@ import akka.util.ByteString import scala.annotation.switch -object JsonBracketCounting { +/** + * INTERNAL API: Use [[akka.stream.scaladsl.JsonFraming]] instead. + */ +private[akka] object JsonBracketCounting { - final val SquareBraceStart = "[".getBytes.head - final val SquareBraceEnd = "]".getBytes.head - final val CurlyBraceStart = "{".getBytes.head - final val CurlyBraceEnd = "}".getBytes.head - final val DoubleQuote = "\"".getBytes.head - final val Backslash = "\\".getBytes.head - final val Comma = ",".getBytes.head + final val SquareBraceStart = '['.toByte + final val SquareBraceEnd = ']'.toByte + final val CurlyBraceStart = '{'.toByte + final val CurlyBraceEnd = '}'.toByte + final val DoubleQuote = '\''.toByte + final val Backslash = '\\'.toByte + final val Comma = ','.toByte final val LineBreak = '\n'.toByte final val LineBreak2 = '\r'.toByte @@ -31,13 +34,15 @@ object JsonBracketCounting { } /** + * INTERNAL API: Use [[akka.stream.scaladsl.JsonFraming]] instead. + * * **Mutable** framing implementation that given any number of [[ByteString]] chunks, can emit JSON objects contained within them. * Typically JSON objects are separated by new-lines or comas, however a top-level JSON Array can also be understood and chunked up * into valid JSON objects by this framing implementation. * * Leading whitespace between elements will be trimmed. */ -class JsonBracketCounting(maximumObjectLength: Int = Int.MaxValue) { +private[akka] class JsonBracketCounting(maximumObjectLength: Int = Int.MaxValue) { import JsonBracketCounting._ private var buffer: ByteString = ByteString.empty diff --git a/akka-stream/src/main/scala/akka/stream/javadsl/Framing.scala b/akka-stream/src/main/scala/akka/stream/javadsl/Framing.scala index 6ae30d85644..b7b21030b1d 100644 --- a/akka-stream/src/main/scala/akka/stream/javadsl/Framing.scala +++ b/akka-stream/src/main/scala/akka/stream/javadsl/Framing.scala @@ -115,18 +115,3 @@ object Framing { scaladsl.Framing.simpleFramingProtocol(maximumMessageLength).asJava } - -/** - * Wrapper around a framing Flow (as provided by [[Framing.delimiter]] for example. - * Used for providing a framing implicitly for other components which may need one (such as framed entity streaming in Akka HTTP). - */ -trait Framing { - def asScala: akka.stream.scaladsl.Framing = - this match { - case f: akka.stream.scaladsl.Framing ⇒ f - case _ ⇒ new akka.stream.scaladsl.Framing { - override def flow = getFlow.asScala - } - } - def getFlow: Flow[ByteString, ByteString, NotUsed] -} diff --git a/akka-stream/src/main/scala/akka/stream/scaladsl/Framing.scala b/akka-stream/src/main/scala/akka/stream/scaladsl/Framing.scala index 4dadca97566..7ff38cae369 100644 --- a/akka-stream/src/main/scala/akka/stream/scaladsl/Framing.scala +++ b/akka-stream/src/main/scala/akka/stream/scaladsl/Framing.scala @@ -287,14 +287,3 @@ object Framing { } } - -/** - * Wrapper around a framing Flow (as provided by [[Framing.delimiter]] for example. - * Used for providing a framing implicitly for other components which may need one (such as framed entity streaming in Akka HTTP). - */ -trait Framing extends akka.stream.javadsl.Framing { - final def asJava: akka.stream.javadsl.Framing = this - override final def getFlow = flow.asJava - - def flow: Flow[ByteString, ByteString, NotUsed] -} From c3308149be4100aa15c0fefe83211a85153eb322 Mon Sep 17 00:00:00 2001 From: Konrad Malawski Date: Mon, 25 Jul 2016 01:50:55 +0200 Subject: [PATCH 04/10] +htp #18837 allow as[Source[Tweet, NotUsed]] --- .../JsonStreamingExamplesSpec.scala | 2 ++ .../http/scaladsl/server/TestServer.scala | 3 +-- .../FramedEntityStreamingDirectives.scala | 24 ++++++++++++------- 3 files changed, 19 insertions(+), 10 deletions(-) diff --git a/akka-docs/rst/scala/code/docs/http/scaladsl/server/directives/JsonStreamingExamplesSpec.scala b/akka-docs/rst/scala/code/docs/http/scaladsl/server/directives/JsonStreamingExamplesSpec.scala index bb30f99612e..f019b6fef55 100644 --- a/akka-docs/rst/scala/code/docs/http/scaladsl/server/directives/JsonStreamingExamplesSpec.scala +++ b/akka-docs/rst/scala/code/docs/http/scaladsl/server/directives/JsonStreamingExamplesSpec.scala @@ -111,6 +111,8 @@ class JsonStreamingExamplesSpec extends RoutingSpec { path("metrics") { // [4] extract Source[Measurement, _] entity(asSourceOf[Measurement]) { measurements => + // alternative syntax: + // entity(as[Source[Measurement, NotUsed]]) { measurements => val measurementsSubmitted: Future[Int] = measurements .via(persistMetrics) diff --git a/akka-http-tests/src/test/scala/akka/http/scaladsl/server/TestServer.scala b/akka-http-tests/src/test/scala/akka/http/scaladsl/server/TestServer.scala index b4901750d5b..7711dc1b886 100644 --- a/akka-http-tests/src/test/scala/akka/http/scaladsl/server/TestServer.scala +++ b/akka-http-tests/src/test/scala/akka/http/scaladsl/server/TestServer.scala @@ -89,8 +89,7 @@ object TestServer extends App { complete(ToResponseMarshallable(tweets)) } ~ post { - entity(asSourceOf[Tweet]) { tweets ⇒ - // entity(asSourceOf[Tweet](bracketCountingJsonFraming(1024))) { tweets: Source[Tweet, NotUsed] ⇒ + entity(as[Source[Tweet, NotUsed]]) { tweets ⇒ complete(s"Total tweets received: " + tweets.runFold(0)({ case (acc, t) => acc + 1 })) } } diff --git a/akka-http/src/main/scala/akka/http/scaladsl/server/directives/FramedEntityStreamingDirectives.scala b/akka-http/src/main/scala/akka/http/scaladsl/server/directives/FramedEntityStreamingDirectives.scala index 1e97170a11b..61c39c68439 100644 --- a/akka-http/src/main/scala/akka/http/scaladsl/server/directives/FramedEntityStreamingDirectives.scala +++ b/akka-http/src/main/scala/akka/http/scaladsl/server/directives/FramedEntityStreamingDirectives.scala @@ -146,22 +146,30 @@ trait FramedEntityStreamingDirectives extends MarshallingDirectives { private final def asSourceOfInternal[T](framing: FramingWithContentType, marshalling: (ExecutionContext, Materializer) => Flow[ByteString, ByteString, NotUsed]#ReprMat[T, NotUsed]): RequestToSourceUnmarshaller[T] = Unmarshaller.withMaterializer[HttpRequest, Source[T, NotUsed]] { implicit ec ⇒ implicit mat ⇒ req ⇒ val entity = req.entity - if (!framing.matches(entity.contentType)) { - val supportedContentTypes = framing.supported - FastFuture.failed(Unmarshaller.UnsupportedContentTypeException(supportedContentTypes)) - } else { + if (framing.matches(entity.contentType)) { val bytes = entity.dataBytes val frames = bytes.viaMat(framing.flow)(Keep.right) val elements = frames.viaMat(marshalling(ec, mat))(Keep.right) - val stream = elements.mapMaterializedValue(_ => NotUsed) -// val stream = Source.single(entity.transformDataBytes(framing.flow)).via(marshalling(ec, mat)).mapMaterializedValue(_ => NotUsed) - FastFuture.successful(stream) - } + FastFuture.successful(elements) + + } else FastFuture.failed(Unmarshaller.UnsupportedContentTypeException(framing.supported)) } // format: ON // TODO note to self - we need the same of ease of streaming stuff for the client side - i.e. the twitter firehose case. + implicit def _asSourceUnmarshaller[T](implicit fem: FromEntityUnmarshaller[T], framing: FramingWithContentType): FromRequestUnmarshaller[Source[T, NotUsed]] = { + Unmarshaller.withMaterializer[HttpRequest, Source[T, NotUsed]] { implicit ec ⇒ implicit mat ⇒ req ⇒ + val entity = req.entity + if (framing.matches(entity.contentType)) { + val bytes = entity.dataBytes + val frames = bytes.viaMat(framing.flow)(Keep.right) + val elements = frames.viaMat(Flow[ByteString].map(HttpEntity(entity.contentType, _)).mapAsync(1)(Unmarshal(_).to[T](fem, ec, mat)))(Keep.right) + FastFuture.successful(elements) + } else FastFuture.failed(Unmarshaller.UnsupportedContentTypeException(framing.supported)) + } + } + implicit def _sourceMarshaller[T, M](implicit m: ToEntityMarshaller[T], mode: SourceRenderingMode): ToResponseMarshaller[Source[T, M]] = Marshaller[Source[T, M], HttpResponse] { implicit ec ⇒ source ⇒ FastFuture successful { From f2419f5a0835a438edd408879a1f0ffe69c0871c Mon Sep 17 00:00:00 2001 From: Konrad Malawski Date: Fri, 29 Jul 2016 15:19:13 +0200 Subject: [PATCH 05/10] =htp framed entity streaming cleanup, renames --- .../akka/routing/ConsistentHashing.scala | 2 +- .../akka/stream/JsonFramingBenchmark.scala | 4 +- .../javadsl/server/JsonEntityStreaming.scala | 9 ----- .../http/scaladsl/common/StrictForm.scala | 3 +- .../PredefinedFromStringUnmarshallers.scala | 6 +++ .../http/scaladsl/unmarshalling/package.scala | 2 + .../stream/scaladsl/JsonFramingSpec.scala | 38 +++++++++---------- ...tCounting.scala => JsonObjectParser.scala} | 6 +-- .../akka/stream/scaladsl/JsonFraming.scala | 4 +- 9 files changed, 37 insertions(+), 37 deletions(-) delete mode 100644 akka-http/src/main/scala/akka/http/javadsl/server/JsonEntityStreaming.scala rename akka-stream/src/main/scala/akka/stream/impl/{JsonBracketCounting.scala => JsonObjectParser.scala} (96%) diff --git a/akka-actor/src/main/scala/akka/routing/ConsistentHashing.scala b/akka-actor/src/main/scala/akka/routing/ConsistentHashing.scala index 78dd4ff505e..4e3461fb691 100644 --- a/akka-actor/src/main/scala/akka/routing/ConsistentHashing.scala +++ b/akka-actor/src/main/scala/akka/routing/ConsistentHashing.scala @@ -91,7 +91,7 @@ object ConsistentHashingRouter { * INTERNAL API */ private[akka] def hashMappingAdapter(mapper: ConsistentHashMapper): ConsistentHashMapping = { - case message if (mapper.hashKey(message).asInstanceOf[AnyRef] ne null) ⇒ + case message if mapper.hashKey(message).asInstanceOf[AnyRef] ne null ⇒ mapper.hashKey(message) } diff --git a/akka-bench-jmh/src/main/scala/akka/stream/JsonFramingBenchmark.scala b/akka-bench-jmh/src/main/scala/akka/stream/JsonFramingBenchmark.scala index 6f353ed4d9e..cf5d7794152 100644 --- a/akka-bench-jmh/src/main/scala/akka/stream/JsonFramingBenchmark.scala +++ b/akka-bench-jmh/src/main/scala/akka/stream/JsonFramingBenchmark.scala @@ -5,7 +5,7 @@ package akka.stream import java.util.concurrent.TimeUnit -import akka.stream.impl.JsonBracketCounting +import akka.stream.impl.JsonObjectParser import akka.util.ByteString import org.openjdk.jmh.annotations._ @@ -35,7 +35,7 @@ class JsonFramingBenchmark { |{"fname":"Bob","name":"Smith","age":42,"id":1337,"boardMember":false}, |{"fname":"Hank","name":"Smith","age":42,"id":1337,"boardMember":false}""".stripMargin) - val bracket = new JsonBracketCounting + val bracket = new JsonObjectParser @Setup(Level.Invocation) def init(): Unit = { diff --git a/akka-http/src/main/scala/akka/http/javadsl/server/JsonEntityStreaming.scala b/akka-http/src/main/scala/akka/http/javadsl/server/JsonEntityStreaming.scala deleted file mode 100644 index 529ae2dea3a..00000000000 --- a/akka-http/src/main/scala/akka/http/javadsl/server/JsonEntityStreaming.scala +++ /dev/null @@ -1,9 +0,0 @@ -/* - * Copyright (C) 2009-2015 Typesafe Inc. - */ -package akka.http.javadsl.server - -class JsonEntityStreaming { - -} - diff --git a/akka-http/src/main/scala/akka/http/scaladsl/common/StrictForm.scala b/akka-http/src/main/scala/akka/http/scaladsl/common/StrictForm.scala index 615941d25c7..cfea86b2857 100644 --- a/akka-http/src/main/scala/akka/http/scaladsl/common/StrictForm.scala +++ b/akka-http/src/main/scala/akka/http/scaladsl/common/StrictForm.scala @@ -61,7 +61,8 @@ object StrictForm { fsu(value.entity.data.decodeString(charsetName)) }) - @implicitNotFound(s"In order to unmarshal a `StrictForm.Field` to type `$${T}` you need to supply a " + + @implicitNotFound(msg = + s"In order to unmarshal a `StrictForm.Field` to type `$${T}` you need to supply a " + s"`FromStringUnmarshaller[$${T}]` and/or a `FromEntityUnmarshaller[$${T}]`") sealed trait FieldUnmarshaller[T] { def unmarshalString(value: String)(implicit ec: ExecutionContext, mat: Materializer): Future[T] diff --git a/akka-http/src/main/scala/akka/http/scaladsl/unmarshalling/PredefinedFromStringUnmarshallers.scala b/akka-http/src/main/scala/akka/http/scaladsl/unmarshalling/PredefinedFromStringUnmarshallers.scala index 6b6ac6b96ab..9f44f677627 100755 --- a/akka-http/src/main/scala/akka/http/scaladsl/unmarshalling/PredefinedFromStringUnmarshallers.scala +++ b/akka-http/src/main/scala/akka/http/scaladsl/unmarshalling/PredefinedFromStringUnmarshallers.scala @@ -6,9 +6,15 @@ package akka.http.scaladsl.unmarshalling import scala.collection.immutable import akka.http.scaladsl.util.FastFuture +import akka.util.ByteString trait PredefinedFromStringUnmarshallers { + implicit def _fromStringUnmarshallerFromByteStringUnmarshaller[T](implicit bsum: FromByteStringUnmarshaller[T]): Unmarshaller[String, T] = { + val bs = Unmarshaller.strict[String, ByteString](s ⇒ ByteString(s)) + bs.flatMap(implicit ec ⇒ implicit mat ⇒ bsum(_)) + } + implicit val byteFromStringUnmarshaller: Unmarshaller[String, Byte] = numberUnmarshaller(_.toByte, "8-bit signed integer") diff --git a/akka-http/src/main/scala/akka/http/scaladsl/unmarshalling/package.scala b/akka-http/src/main/scala/akka/http/scaladsl/unmarshalling/package.scala index 3870019cbd8..46dbbce95a9 100644 --- a/akka-http/src/main/scala/akka/http/scaladsl/unmarshalling/package.scala +++ b/akka-http/src/main/scala/akka/http/scaladsl/unmarshalling/package.scala @@ -6,6 +6,7 @@ package akka.http.scaladsl import akka.http.scaladsl.common.StrictForm import akka.http.scaladsl.model._ +import akka.util.ByteString package object unmarshalling { //# unmarshaller-aliases @@ -13,6 +14,7 @@ package object unmarshalling { type FromMessageUnmarshaller[T] = Unmarshaller[HttpMessage, T] type FromResponseUnmarshaller[T] = Unmarshaller[HttpResponse, T] type FromRequestUnmarshaller[T] = Unmarshaller[HttpRequest, T] + type FromByteStringUnmarshaller[T] = Unmarshaller[ByteString, T] type FromStringUnmarshaller[T] = Unmarshaller[String, T] type FromStrictFormFieldUnmarshaller[T] = Unmarshaller[StrictForm.Field, T] //# diff --git a/akka-stream-tests/src/test/scala/akka/stream/scaladsl/JsonFramingSpec.scala b/akka-stream-tests/src/test/scala/akka/stream/scaladsl/JsonFramingSpec.scala index e6b657fd592..c69bfc5820f 100644 --- a/akka-stream-tests/src/test/scala/akka/stream/scaladsl/JsonFramingSpec.scala +++ b/akka-stream-tests/src/test/scala/akka/stream/scaladsl/JsonFramingSpec.scala @@ -4,7 +4,7 @@ package akka.stream.scaladsl import akka.stream.ActorMaterializer -import akka.stream.impl.JsonBracketCounting +import akka.stream.impl.JsonObjectParser import akka.stream.scaladsl.Framing.FramingException import akka.stream.scaladsl.{ JsonFraming, Framing, Source } import akka.stream.testkit.scaladsl.TestSink @@ -124,7 +124,7 @@ class JsonFramingSpec extends AkkaSpec { "collecting json buffer" when { "nothing is supplied" should { "return nothing" in { - val buffer = new JsonBracketCounting() + val buffer = new JsonObjectParser() buffer.poll() should ===(None) } } @@ -132,25 +132,25 @@ class JsonFramingSpec extends AkkaSpec { "valid json is supplied" which { "has one object" should { "successfully parse empty object" in { - val buffer = new JsonBracketCounting() + val buffer = new JsonObjectParser() buffer.offer(ByteString("""{}""")) buffer.poll().get.utf8String shouldBe """{}""" } "successfully parse single field having string value" in { - val buffer = new JsonBracketCounting() + val buffer = new JsonObjectParser() buffer.offer(ByteString("""{ "name": "john"}""")) buffer.poll().get.utf8String shouldBe """{ "name": "john"}""" } "successfully parse single field having string value containing space" in { - val buffer = new JsonBracketCounting() + val buffer = new JsonObjectParser() buffer.offer(ByteString("""{ "name": "john doe"}""")) buffer.poll().get.utf8String shouldBe """{ "name": "john doe"}""" } "successfully parse single field having string value containing curly brace" in { - val buffer = new JsonBracketCounting() + val buffer = new JsonObjectParser() buffer.offer(ByteString("""{ "name": "john{""")) buffer.offer(ByteString("}")) @@ -161,7 +161,7 @@ class JsonFramingSpec extends AkkaSpec { } "successfully parse single field having string value containing curly brace and escape character" in { - val buffer = new JsonBracketCounting() + val buffer = new JsonObjectParser() buffer.offer(ByteString("""{ "name": "john""")) buffer.offer(ByteString("\\\"")) @@ -177,19 +177,19 @@ class JsonFramingSpec extends AkkaSpec { } "successfully parse single field having integer value" in { - val buffer = new JsonBracketCounting() + val buffer = new JsonObjectParser() buffer.offer(ByteString("""{ "age": 101}""")) buffer.poll().get.utf8String shouldBe """{ "age": 101}""" } "successfully parse single field having decimal value" in { - val buffer = new JsonBracketCounting() + val buffer = new JsonObjectParser() buffer.offer(ByteString("""{ "age": 101}""")) buffer.poll().get.utf8String shouldBe """{ "age": 101}""" } "successfully parse single field having nested object" in { - val buffer = new JsonBracketCounting() + val buffer = new JsonObjectParser() buffer.offer(ByteString( """ |{ "name": "john", @@ -210,7 +210,7 @@ class JsonFramingSpec extends AkkaSpec { } "successfully parse single field having multiple level of nested object" in { - val buffer = new JsonBracketCounting() + val buffer = new JsonObjectParser() buffer.offer(ByteString( """ |{ "name": "john", @@ -239,7 +239,7 @@ class JsonFramingSpec extends AkkaSpec { "has nested array" should { "successfully parse" in { - val buffer = new JsonBracketCounting() + val buffer = new JsonObjectParser() buffer.offer(ByteString( """ |{ "name": "john", @@ -264,7 +264,7 @@ class JsonFramingSpec extends AkkaSpec { "has complex object graph" should { "successfully parse" in { - val buffer = new JsonBracketCounting() + val buffer = new JsonObjectParser() buffer.offer(ByteString( """ |{ @@ -318,13 +318,13 @@ class JsonFramingSpec extends AkkaSpec { "has multiple fields" should { "parse successfully" in { - val buffer = new JsonBracketCounting() + val buffer = new JsonObjectParser() buffer.offer(ByteString("""{ "name": "john", "age": 101}""")) buffer.poll().get.utf8String shouldBe """{ "name": "john", "age": 101}""" } "parse successfully despite valid whitespaces around json" in { - val buffer = new JsonBracketCounting() + val buffer = new JsonObjectParser() buffer.offer(ByteString( """ | @@ -351,7 +351,7 @@ class JsonFramingSpec extends AkkaSpec { | } """.stripMargin - val buffer = new JsonBracketCounting() + val buffer = new JsonObjectParser() buffer.offer(ByteString(input)) buffer.poll().get.utf8String shouldBe @@ -375,7 +375,7 @@ class JsonFramingSpec extends AkkaSpec { } "returns none until valid json is encountered" in { - val buffer = new JsonBracketCounting() + val buffer = new JsonObjectParser() """{ "name": "john"""".foreach { c ⇒ @@ -389,13 +389,13 @@ class JsonFramingSpec extends AkkaSpec { "invalid json is supplied" should { "fail if it's broken from the start" in { - val buffer = new JsonBracketCounting() + val buffer = new JsonObjectParser() buffer.offer(ByteString("""THIS IS NOT VALID { "name": "john"}""")) a[FramingException] shouldBe thrownBy { buffer.poll() } } "fail if it's broken at the end" in { - val buffer = new JsonBracketCounting() + val buffer = new JsonObjectParser() buffer.offer(ByteString("""{ "name": "john"} THIS IS NOT VALID""")) buffer.poll() // first emitting the valid element a[FramingException] shouldBe thrownBy { buffer.poll() } diff --git a/akka-stream/src/main/scala/akka/stream/impl/JsonBracketCounting.scala b/akka-stream/src/main/scala/akka/stream/impl/JsonObjectParser.scala similarity index 96% rename from akka-stream/src/main/scala/akka/stream/impl/JsonBracketCounting.scala rename to akka-stream/src/main/scala/akka/stream/impl/JsonObjectParser.scala index 850d3c0f687..3d0c6ec253a 100644 --- a/akka-stream/src/main/scala/akka/stream/impl/JsonBracketCounting.scala +++ b/akka-stream/src/main/scala/akka/stream/impl/JsonObjectParser.scala @@ -11,7 +11,7 @@ import scala.annotation.switch /** * INTERNAL API: Use [[akka.stream.scaladsl.JsonFraming]] instead. */ -private[akka] object JsonBracketCounting { +private[akka] object JsonObjectParser { final val SquareBraceStart = '['.toByte final val SquareBraceEnd = ']'.toByte @@ -42,8 +42,8 @@ private[akka] object JsonBracketCounting { * * Leading whitespace between elements will be trimmed. */ -private[akka] class JsonBracketCounting(maximumObjectLength: Int = Int.MaxValue) { - import JsonBracketCounting._ +private[akka] class JsonObjectParser(maximumObjectLength: Int = Int.MaxValue) { + import JsonObjectParser._ private var buffer: ByteString = ByteString.empty diff --git a/akka-stream/src/main/scala/akka/stream/scaladsl/JsonFraming.scala b/akka-stream/src/main/scala/akka/stream/scaladsl/JsonFraming.scala index 0a48c0a3c48..bc5f69d0372 100644 --- a/akka-stream/src/main/scala/akka/stream/scaladsl/JsonFraming.scala +++ b/akka-stream/src/main/scala/akka/stream/scaladsl/JsonFraming.scala @@ -5,7 +5,7 @@ package akka.stream.scaladsl import akka.NotUsed import akka.stream.Attributes -import akka.stream.impl.JsonBracketCounting +import akka.stream.impl.JsonObjectParser import akka.stream.impl.fusing.GraphStages.SimpleLinearGraphStage import akka.stream.stage.{ InHandler, OutHandler, GraphStageLogic } import akka.util.ByteString @@ -42,7 +42,7 @@ object JsonFraming { */ def bracketCounting(maximumObjectLength: Int): Flow[ByteString, ByteString, NotUsed] = Flow[ByteString].via(new SimpleLinearGraphStage[ByteString] { - private[this] val buffer = new JsonBracketCounting(maximumObjectLength) + private[this] val buffer = new JsonObjectParser(maximumObjectLength) override def createLogic(inheritedAttributes: Attributes) = new GraphStageLogic(shape) with InHandler with OutHandler { setHandlers(in, out, this) From bc536be32c4266b22309a6a3db62cd8faa8ea3b2 Mon Sep 17 00:00:00 2001 From: Konrad Malawski Date: Fri, 29 Jul 2016 15:21:19 +0200 Subject: [PATCH 06/10] =htp introduce ToByteStringMarshaller alias and fix fixme --- .../marshallers/sprayjson/SprayJsonSupport.scala | 10 +++++----- .../scala/akka/http/scaladsl/marshalling/package.scala | 2 ++ 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/akka-http-marshallers-scala/akka-http-spray-json/src/main/scala/akka/http/scaladsl/marshallers/sprayjson/SprayJsonSupport.scala b/akka-http-marshallers-scala/akka-http-spray-json/src/main/scala/akka/http/scaladsl/marshallers/sprayjson/SprayJsonSupport.scala index ea998045020..d3f13379bda 100644 --- a/akka-http-marshallers-scala/akka-http-spray-json/src/main/scala/akka/http/scaladsl/marshallers/sprayjson/SprayJsonSupport.scala +++ b/akka-http-marshallers-scala/akka-http-spray-json/src/main/scala/akka/http/scaladsl/marshallers/sprayjson/SprayJsonSupport.scala @@ -8,9 +8,9 @@ import akka.http.scaladsl.util.FastFuture import akka.util.ByteString import scala.language.implicitConversions -import akka.http.scaladsl.marshalling.{ ToEntityMarshaller, Marshaller } -import akka.http.scaladsl.unmarshalling.{ FromEntityUnmarshaller, Unmarshaller } -import akka.http.scaladsl.model.{ ContentTypes, MediaTypes, HttpCharsets } +import akka.http.scaladsl.marshalling.{Marshaller, ToByteStringMarshaller, ToEntityMarshaller} +import akka.http.scaladsl.unmarshalling.{FromEntityUnmarshaller, Unmarshaller} +import akka.http.scaladsl.model.{ContentTypes, HttpCharsets, MediaTypes} import akka.http.scaladsl.model.MediaTypes.`application/json` import spray.json._ @@ -30,7 +30,7 @@ trait SprayJsonSupport { Unmarshaller.byteStringUnmarshaller.forContentTypes(`application/json`).mapWithCharset { (data, charset) ⇒ val input = if (charset == HttpCharsets.`UTF-8`) ParserInput(data.toArray) - else ParserInput(data.decodeString(charset.nioCharset.name)) // FIXME: identify charset by instance, not by name! + else ParserInput(data.decodeString(charset.nioCharset)) JsonParser(input) } @@ -40,7 +40,7 @@ trait SprayJsonSupport { sprayJsValueMarshaller compose writer.write implicit def sprayJsValueMarshaller(implicit printer: JsonPrinter = CompactPrinter): ToEntityMarshaller[JsValue] = Marshaller.StringMarshaller.wrap(MediaTypes.`application/json`)(printer) - implicit def sprayByteStringMarshaller[T](implicit writer: RootJsonFormat[T], printer: JsonPrinter = CompactPrinter): Marshaller[T, ByteString] = + implicit def sprayByteStringMarshaller[T](implicit writer: RootJsonFormat[T], printer: JsonPrinter = CompactPrinter): ToByteStringMarshaller[T] = sprayJsValueMarshaller.map(s ⇒ ByteString(s.toString)) compose writer.write } object SprayJsonSupport extends SprayJsonSupport diff --git a/akka-http/src/main/scala/akka/http/scaladsl/marshalling/package.scala b/akka-http/src/main/scala/akka/http/scaladsl/marshalling/package.scala index a6a8748e975..c1935887e07 100644 --- a/akka-http/src/main/scala/akka/http/scaladsl/marshalling/package.scala +++ b/akka-http/src/main/scala/akka/http/scaladsl/marshalling/package.scala @@ -6,10 +6,12 @@ package akka.http.scaladsl import scala.collection.immutable import akka.http.scaladsl.model._ +import akka.util.ByteString package object marshalling { //# marshaller-aliases type ToEntityMarshaller[T] = Marshaller[T, MessageEntity] + type ToByteStringMarshaller[T] = Marshaller[T, ByteString] type ToHeadersAndEntityMarshaller[T] = Marshaller[T, (immutable.Seq[HttpHeader], MessageEntity)] type ToResponseMarshaller[T] = Marshaller[T, HttpResponse] type ToRequestMarshaller[T] = Marshaller[T, HttpRequest] From 6562ddd2dffc0740fbc9ee0c8d522f23ac64d4cf Mon Sep 17 00:00:00 2001 From: Konrad Malawski Date: Fri, 29 Jul 2016 16:29:50 +0200 Subject: [PATCH 07/10] =htp address review feedback on JSON streaming --- .../routing-dsl/source-streaming-support.rst | 2 +- akka-docs/rst/java/typed-actors.rst | 2 +- .../JsonStreamingExamplesSpec.scala | 5 +- .../http/routing-dsl/directives/index.rst | 2 +- .../routing-dsl/source-streaming-support.rst | 6 +- akka-docs/rst/scala/typed-actors.rst | 2 +- .../SprayJsonByteStringParserInput.scala | 79 +++++++++++++++++++ .../sprayjson/SprayJsonSupport.scala | 16 ++-- .../http/scaladsl/server/TestServer.scala | 6 +- .../server/directives/FutureDirectives.scala | 8 ++ .../http/scaladsl/common/StrictForm.scala | 4 +- .../FramedEntityStreamingDirectives.scala | 2 +- .../akka/stream/impl/JsonObjectParser.scala | 2 +- 13 files changed, 114 insertions(+), 22 deletions(-) create mode 100644 akka-http-marshallers-scala/akka-http-spray-json/src/main/scala/akka/http/scaladsl/marshallers/sprayjson/SprayJsonByteStringParserInput.scala diff --git a/akka-docs/rst/java/http/routing-dsl/source-streaming-support.rst b/akka-docs/rst/java/http/routing-dsl/source-streaming-support.rst index b83de64dc35..f4a0e7c4b61 100644 --- a/akka-docs/rst/java/http/routing-dsl/source-streaming-support.rst +++ b/akka-docs/rst/java/http/routing-dsl/source-streaming-support.rst @@ -3,7 +3,7 @@ Source Streaming ================ -Akka HTTP supports completing a request with an Akka ``Source``, which makes it possible to very easily build +Akka HTTP supports completing a request with an Akka ``Source``, which makes it possible to easily build streaming end-to-end APIs which apply back-pressure throughout the entire stack. It is possible to complete requests with raw ``Source``, however often it is more convenient to diff --git a/akka-docs/rst/java/typed-actors.rst b/akka-docs/rst/java/typed-actors.rst index 11160b69e15..0b6622023c9 100644 --- a/akka-docs/rst/java/typed-actors.rst +++ b/akka-docs/rst/java/typed-actors.rst @@ -25,7 +25,7 @@ lies in interfacing between private sphere and the public, but you don’t want that many doors inside your house, do you? For a longer discussion see `this blog post `_. -A bit more background: TypedActors can very easily be abused as RPC, and that +A bit more background: TypedActors can easily be abused as RPC, and that is an abstraction which is `well-known `_ to be leaky. Hence TypedActors are not what we think of first when we talk diff --git a/akka-docs/rst/scala/code/docs/http/scaladsl/server/directives/JsonStreamingExamplesSpec.scala b/akka-docs/rst/scala/code/docs/http/scaladsl/server/directives/JsonStreamingExamplesSpec.scala index f019b6fef55..4f047ae595f 100644 --- a/akka-docs/rst/scala/code/docs/http/scaladsl/server/directives/JsonStreamingExamplesSpec.scala +++ b/akka-docs/rst/scala/code/docs/http/scaladsl/server/directives/JsonStreamingExamplesSpec.scala @@ -45,13 +45,12 @@ class JsonStreamingExamplesSpec extends RoutingSpec { // [3] pick json rendering mode: // HINT: if you extend `akka.http.scaladsl.server.EntityStreamingSupport` // it'll guide you to do so via abstract defs - val maximumObjectLength = 128 implicit val jsonRenderingMode = JsonSourceRenderingModes.LineByLine val route = path("tweets") { val tweets: Source[Tweet, NotUsed] = getTweets() - complete(ToResponseMarshallable(tweets)) + complete(tweets) } // tests: @@ -104,7 +103,7 @@ class JsonStreamingExamplesSpec extends RoutingSpec { // [2] import "my protocol", for unmarshalling Measurement objects: import MyJsonProtocol._ - // [3] prepareyour persisting logic here + // [3] prepare your persisting logic here val persistMetrics = Flow[Measurement] val route = diff --git a/akka-docs/rst/scala/http/routing-dsl/directives/index.rst b/akka-docs/rst/scala/http/routing-dsl/directives/index.rst index e0082a338a2..4e30d49f503 100644 --- a/akka-docs/rst/scala/http/routing-dsl/directives/index.rst +++ b/akka-docs/rst/scala/http/routing-dsl/directives/index.rst @@ -224,4 +224,4 @@ When you combine directives producing extractions with the ``&`` operator all ex Directives offer a great way of constructing your web service logic from small building blocks in a plug and play fashion while maintaining DRYness and full type-safety. If the large range of :ref:`Predefined Directives` does not -fully satisfy your needs you can also very easily create :ref:`Custom Directives`. +fully satisfy your needs you can also easily create :ref:`Custom Directives`. diff --git a/akka-docs/rst/scala/http/routing-dsl/source-streaming-support.rst b/akka-docs/rst/scala/http/routing-dsl/source-streaming-support.rst index d6c7cbdc46d..8ead39b7660 100644 --- a/akka-docs/rst/scala/http/routing-dsl/source-streaming-support.rst +++ b/akka-docs/rst/scala/http/routing-dsl/source-streaming-support.rst @@ -3,7 +3,7 @@ Source Streaming ================ -Akka HTTP supports completing a request with an Akka ``Source[T, _]``, which makes it possible to very easily build +Akka HTTP supports completing a request with an Akka ``Source[T, _]``, which makes it possible to easily build streaming end-to-end APIs which apply back-pressure throughout the entire stack. It is possible to complete requests with raw ``Source[ByteString, _]``, however often it is more convenient to @@ -99,7 +99,7 @@ Implementing custom (Un)Marshaller support for JSON streaming While not provided by Akka HTTP directly, the infrastructure is extensible and by investigating how ``SprayJsonSupport`` is implemented it is certainly possible to provide the same infrastructure for other marshaller implementations (such as -Play JSON, or Jackson directly for example). Such support traits will want to extend the ``JsonEntityStreamingSupport`` trait. +Play JSON, or Jackson directly for example). Such support traits will want to extend the ``EntityStreamingSupport`` trait. The following types that may need to be implemented by a custom framed-streaming support library are: @@ -108,4 +108,4 @@ The following types that may need to be implemented by a custom framed-streaming - ``FramingWithContentType`` which is needed to be able to split incoming ``ByteString`` chunks into frames of the higher-level data type format that is understood by the provided unmarshallers. In the case of JSON it means chunking up ByteStrings such that each emitted element corresponds to exactly one JSON object, - this framing is implemented in ``JsonEntityStreamingSupport``. + this framing is implemented in ``EntityStreamingSupport``. diff --git a/akka-docs/rst/scala/typed-actors.rst b/akka-docs/rst/scala/typed-actors.rst index f9f5ab8fd5b..75c72637f3a 100644 --- a/akka-docs/rst/scala/typed-actors.rst +++ b/akka-docs/rst/scala/typed-actors.rst @@ -35,7 +35,7 @@ lies in interfacing between private sphere and the public, but you don’t want that many doors inside your house, do you? For a longer discussion see `this blog post `_. -A bit more background: TypedActors can very easily be abused as RPC, and that +A bit more background: TypedActors can easily be abused as RPC, and that is an abstraction which is `well-known `_ to be leaky. Hence TypedActors are not what we think of first when we talk diff --git a/akka-http-marshallers-scala/akka-http-spray-json/src/main/scala/akka/http/scaladsl/marshallers/sprayjson/SprayJsonByteStringParserInput.scala b/akka-http-marshallers-scala/akka-http-spray-json/src/main/scala/akka/http/scaladsl/marshallers/sprayjson/SprayJsonByteStringParserInput.scala new file mode 100644 index 00000000000..71f7fec8cb3 --- /dev/null +++ b/akka-http-marshallers-scala/akka-http-spray-json/src/main/scala/akka/http/scaladsl/marshallers/sprayjson/SprayJsonByteStringParserInput.scala @@ -0,0 +1,79 @@ +/* + * Copyright (C) 2016 Lightbend Inc. + */ + +package akka.http.scaladsl.marshallers.sprayjson + +import java.nio.{ ByteBuffer, CharBuffer } +import java.nio.charset.{ Charset, StandardCharsets } + +import akka.util.ByteString +import spray.json.ParserInput.DefaultParserInput +import scala.annotation.tailrec + +/** + * ParserInput reading directly off a ByteString. (Based on the ByteArrayBasedParserInput) + * This avoids a separate decoding step but assumes that each byte represents exactly one character, + * which is encoded by ISO-8859-1! + * You can therefore use this ParserInput type only if you know that all input will be `ISO-8859-1`-encoded, + * or only contains 7-bit ASCII characters (which is a subset of ISO-8859-1)! + * + * Note that this ParserInput type will NOT work with general `UTF-8`-encoded input as this can contain + * character representations spanning multiple bytes. However, if you know that your input will only ever contain + * 7-bit ASCII characters (0x00-0x7F) then UTF-8 is fine, since the first 127 UTF-8 characters are + * encoded with only one byte that is identical to 7-bit ASCII and ISO-8859-1. + */ +final class SprayJsonByteStringParserInput(bytes: ByteString) extends DefaultParserInput { + + import SprayJsonByteStringParserInput._ + + private[this] val byteBuffer = ByteBuffer.allocate(4) + private[this] val charBuffer = CharBuffer.allocate(1) + + private[this] val decoder = Charset.forName("UTF-8").newDecoder() + + override def nextChar() = { + _cursor += 1 + if (_cursor < bytes.length) (bytes(_cursor) & 0xFF).toChar else EOI + } + + override def nextUtf8Char() = { + @tailrec def decode(byte: Byte, remainingBytes: Int): Char = { + byteBuffer.put(byte) + if (remainingBytes > 0) { + _cursor += 1 + if (_cursor < bytes.length) decode(bytes(_cursor), remainingBytes - 1) else ErrorChar + } else { + byteBuffer.flip() + val coderResult = decoder.decode(byteBuffer, charBuffer, false) + charBuffer.flip() + val result = if (coderResult.isUnderflow & charBuffer.hasRemaining) charBuffer.get() else ErrorChar + byteBuffer.clear() + charBuffer.clear() + result + } + } + + _cursor += 1 + if (_cursor < bytes.length) { + val byte = bytes(_cursor) + if (byte >= 0) byte.toChar // 7-Bit ASCII + else if ((byte & 0xE0) == 0xC0) decode(byte, 1) // 2-byte UTF-8 sequence + else if ((byte & 0xF0) == 0xE0) decode(byte, 2) // 3-byte UTF-8 sequence + else if ((byte & 0xF8) == 0xF0) decode(byte, 3) // 4-byte UTF-8 sequence, will probably produce an (unsupported) surrogate pair + else ErrorChar + } else EOI + } + + override def length: Int = bytes.size + override def sliceString(start: Int, end: Int): String = + bytes.slice(start, end - start).decodeString(StandardCharsets.ISO_8859_1) + override def sliceCharArray(start: Int, end: Int): Array[Char] = + StandardCharsets.ISO_8859_1.decode(bytes.slice(start, end).asByteBuffer).array() +} + +object SprayJsonByteStringParserInput { + private final val EOI = '\uFFFF' + // compile-time constant + private final val ErrorChar = '\uFFFD' // compile-time constant, universal UTF-8 replacement character '�' +} diff --git a/akka-http-marshallers-scala/akka-http-spray-json/src/main/scala/akka/http/scaladsl/marshallers/sprayjson/SprayJsonSupport.scala b/akka-http-marshallers-scala/akka-http-spray-json/src/main/scala/akka/http/scaladsl/marshallers/sprayjson/SprayJsonSupport.scala index d3f13379bda..cc5fe074954 100644 --- a/akka-http-marshallers-scala/akka-http-spray-json/src/main/scala/akka/http/scaladsl/marshallers/sprayjson/SprayJsonSupport.scala +++ b/akka-http-marshallers-scala/akka-http-spray-json/src/main/scala/akka/http/scaladsl/marshallers/sprayjson/SprayJsonSupport.scala @@ -4,15 +4,15 @@ package akka.http.scaladsl.marshallers.sprayjson +import akka.http.scaladsl.marshalling.{ Marshaller, ToByteStringMarshaller, ToEntityMarshaller } +import akka.http.scaladsl.model.MediaTypes.`application/json` +import akka.http.scaladsl.model.{ HttpCharsets, MediaTypes } +import akka.http.scaladsl.unmarshalling.{ FromEntityUnmarshaller, Unmarshaller } import akka.http.scaladsl.util.FastFuture import akka.util.ByteString +import spray.json._ import scala.language.implicitConversions -import akka.http.scaladsl.marshalling.{Marshaller, ToByteStringMarshaller, ToEntityMarshaller} -import akka.http.scaladsl.unmarshalling.{FromEntityUnmarshaller, Unmarshaller} -import akka.http.scaladsl.model.{ContentTypes, HttpCharsets, MediaTypes} -import akka.http.scaladsl.model.MediaTypes.`application/json` -import spray.json._ /** * A trait providing automatic to and from JSON marshalling/unmarshalling using an in-scope *spray-json* protocol. @@ -24,7 +24,11 @@ trait SprayJsonSupport { sprayJsValueUnmarshaller.map(jsonReader[T].read) implicit def sprayJsonByteStringUnmarshaller[T](implicit reader: RootJsonReader[T]): Unmarshaller[ByteString, T] = Unmarshaller.withMaterializer[ByteString, JsValue](_ ⇒ implicit mat ⇒ { bs ⇒ - FastFuture.successful(JsonParser(bs.toArray[Byte])) + // .compact so addressing into any address is very fast (also for large chunks) + // TODO we could optimise ByteStrings to better handle lienear access like this (or provide ByteStrings.linearAccessOptimised) + // TODO IF it's worth it. + val parserInput = new SprayJsonByteStringParserInput(bs.compact) + FastFuture.successful(JsonParser(parserInput)) }).map(jsonReader[T].read) implicit def sprayJsValueUnmarshaller: FromEntityUnmarshaller[JsValue] = Unmarshaller.byteStringUnmarshaller.forContentTypes(`application/json`).mapWithCharset { (data, charset) ⇒ diff --git a/akka-http-tests/src/test/scala/akka/http/scaladsl/server/TestServer.scala b/akka-http-tests/src/test/scala/akka/http/scaladsl/server/TestServer.scala index 7711dc1b886..1512bcb0d98 100644 --- a/akka-http-tests/src/test/scala/akka/http/scaladsl/server/TestServer.scala +++ b/akka-http-tests/src/test/scala/akka/http/scaladsl/server/TestServer.scala @@ -86,11 +86,13 @@ object TestServer extends App { (path("tweets") & parameter('n.as[Int])) { n => get { val tweets = Source.repeat(Tweet("Hello, world!")).take(n) - complete(ToResponseMarshallable(tweets)) + complete(tweets) } ~ post { entity(as[Source[Tweet, NotUsed]]) { tweets ⇒ - complete(s"Total tweets received: " + tweets.runFold(0)({ case (acc, t) => acc + 1 })) + onComplete(tweets.runFold(0)({ case (acc, t) => acc + 1 })) { count => + complete(s"Total tweets received: " + count) + } } } } diff --git a/akka-http/src/main/scala/akka/http/javadsl/server/directives/FutureDirectives.scala b/akka-http/src/main/scala/akka/http/javadsl/server/directives/FutureDirectives.scala index dd48c4c1877..50a0cc75efd 100644 --- a/akka-http/src/main/scala/akka/http/javadsl/server/directives/FutureDirectives.scala +++ b/akka-http/src/main/scala/akka/http/javadsl/server/directives/FutureDirectives.scala @@ -23,6 +23,8 @@ abstract class FutureDirectives extends FormFieldDirectives { /** * "Unwraps" a `CompletionStage` and runs the inner route after future * completion with the future's value as an extraction of type `Try`. + * + * @group future */ def onComplete[T](f: Supplier[CompletionStage[T]], inner: JFunction[Try[T], Route]) = RouteAdapter { D.onComplete(f.get.toScala.recover(unwrapCompletionException)) { value ⇒ @@ -33,6 +35,8 @@ abstract class FutureDirectives extends FormFieldDirectives { /** * "Unwraps" a `CompletionStage` and runs the inner route after future * completion with the future's value as an extraction of type `Try`. + * + * @group future */ def onComplete[T](cs: CompletionStage[T], inner: JFunction[Try[T], Route]) = RouteAdapter { D.onComplete(cs.toScala.recover(unwrapCompletionException)) { value ⇒ @@ -61,6 +65,8 @@ abstract class FutureDirectives extends FormFieldDirectives { * completion with the stage's value as an extraction of type `T`. * If the stage fails its failure Throwable is bubbled up to the nearest * ExceptionHandler. + * + * @group future */ def onSuccess[T](f: Supplier[CompletionStage[T]], inner: JFunction[T, Route]) = RouteAdapter { D.onSuccess(f.get.toScala.recover(unwrapCompletionException)) { value ⇒ @@ -74,6 +80,8 @@ abstract class FutureDirectives extends FormFieldDirectives { * If the completion stage succeeds the request is completed using the values marshaller * (This directive therefore requires a marshaller for the completion stage value type to be * provided.) + * + * @group future */ def completeOrRecoverWith[T](f: Supplier[CompletionStage[T]], marshaller: Marshaller[T, RequestEntity], inner: JFunction[Throwable, Route]): Route = RouteAdapter { val magnet = CompleteOrRecoverWithMagnet(f.get.toScala)(Marshaller.asScalaEntityMarshaller(marshaller)) diff --git a/akka-http/src/main/scala/akka/http/scaladsl/common/StrictForm.scala b/akka-http/src/main/scala/akka/http/scaladsl/common/StrictForm.scala index cfea86b2857..f6a5d35206a 100644 --- a/akka-http/src/main/scala/akka/http/scaladsl/common/StrictForm.scala +++ b/akka-http/src/main/scala/akka/http/scaladsl/common/StrictForm.scala @@ -61,9 +61,9 @@ object StrictForm { fsu(value.entity.data.decodeString(charsetName)) }) - @implicitNotFound(msg = + @implicitNotFound(msg = s"In order to unmarshal a `StrictForm.Field` to type `$${T}` you need to supply a " + - s"`FromStringUnmarshaller[$${T}]` and/or a `FromEntityUnmarshaller[$${T}]`") + s"`FromStringUnmarshaller[$${T}]` and/or a `FromEntityUnmarshaller[$${T}]`") sealed trait FieldUnmarshaller[T] { def unmarshalString(value: String)(implicit ec: ExecutionContext, mat: Materializer): Future[T] def unmarshalPart(value: Multipart.FormData.BodyPart.Strict)(implicit ec: ExecutionContext, mat: Materializer): Future[T] diff --git a/akka-http/src/main/scala/akka/http/scaladsl/server/directives/FramedEntityStreamingDirectives.scala b/akka-http/src/main/scala/akka/http/scaladsl/server/directives/FramedEntityStreamingDirectives.scala index 61c39c68439..9f3dffd6dff 100644 --- a/akka-http/src/main/scala/akka/http/scaladsl/server/directives/FramedEntityStreamingDirectives.scala +++ b/akka-http/src/main/scala/akka/http/scaladsl/server/directives/FramedEntityStreamingDirectives.scala @@ -148,7 +148,7 @@ trait FramedEntityStreamingDirectives extends MarshallingDirectives { val entity = req.entity if (framing.matches(entity.contentType)) { val bytes = entity.dataBytes - val frames = bytes.viaMat(framing.flow)(Keep.right) + val frames = bytes.via(framing.flow) val elements = frames.viaMat(marshalling(ec, mat))(Keep.right) FastFuture.successful(elements) diff --git a/akka-stream/src/main/scala/akka/stream/impl/JsonObjectParser.scala b/akka-stream/src/main/scala/akka/stream/impl/JsonObjectParser.scala index 3d0c6ec253a..ed779646073 100644 --- a/akka-stream/src/main/scala/akka/stream/impl/JsonObjectParser.scala +++ b/akka-stream/src/main/scala/akka/stream/impl/JsonObjectParser.scala @@ -143,7 +143,7 @@ private[akka] class JsonObjectParser(maximumObjectLength: Int = Int.MaxValue) { isStartOfEscapeSequence = false pos += 1 } else { - throw new FramingException(s"Invalid JSON encountered as position [$pos] of [$buffer]") + throw new FramingException(s"Invalid JSON encountered at position [$pos] of [$buffer]") } @inline private final def insideObject: Boolean = From 9cc32c3aba18950bddf1b9a72a079ec2e1a3cfa1 Mon Sep 17 00:00:00 2001 From: Konrad Malawski Date: Mon, 1 Aug 2016 11:47:35 +0200 Subject: [PATCH 08/10] +htp #18837 completely rewrite EntityStreamingSupport added CSV examples updated docs EntityStreamingSupport is now an entry point, to all streaming things both read and write side use it it's easy to extend as well --- .../server/JsonStreamingExamplesTest.java | 96 +++++++- .../routing-dsl/source-streaming-support.rst | 61 +++-- .../JsonStreamingExamplesSpec.scala | 154 +++++++++---- .../routing-dsl/source-streaming-support.rst | 81 ++++--- .../sprayjson/SprayJsonSupport.scala | 23 +- .../http/javadsl/server/JavaTestServer.java | 9 +- .../http/scaladsl/server/TestServer.scala | 31 ++- .../common/CsvSourceRenderingMode.scala | 41 ---- .../common/EntityStreamingSupport.scala | 149 +++++++++++++ .../common/FramingWithContentType.scala | 36 --- .../common/JsonSourceRenderingMode.scala | 103 --------- .../javadsl/common/SourceRenderingMode.scala | 22 -- .../server/EntityStreamingSupport.scala | 41 ---- .../javadsl/server/RoutingJavaMapping.scala | 4 +- .../FramedEntityStreamingDirectives.scala | 63 ++---- .../server/directives/FutureDirectives.scala | 8 +- .../common/CsvEntityStreamingSupport.scala | 48 ++++ .../common/EntityStreamingSupport.scala | 141 ++++++++++++ .../common/FramingWithContentType.scala | 25 --- .../common/JsonEntityStreamingSupport.scala | 49 ++++ .../common/JsonSourceRenderingMode.scala | 128 ----------- .../scaladsl/common/SourceRenderingMode.scala | 11 - .../PredefinedToResponseMarshallers.scala | 72 ++++++ .../server/EntityStreamingSupport.scala | 68 ------ .../FramedEntityStreamingDirectives.scala | 209 ++---------------- .../stream/scaladsl/JsonFramingSpec.scala | 23 +- .../akka/stream/javadsl/JsonFraming.scala | 4 +- .../akka/stream/scaladsl/JsonFraming.scala | 5 +- 28 files changed, 860 insertions(+), 845 deletions(-) delete mode 100644 akka-http/src/main/scala/akka/http/javadsl/common/CsvSourceRenderingMode.scala create mode 100644 akka-http/src/main/scala/akka/http/javadsl/common/EntityStreamingSupport.scala delete mode 100644 akka-http/src/main/scala/akka/http/javadsl/common/FramingWithContentType.scala delete mode 100644 akka-http/src/main/scala/akka/http/javadsl/common/JsonSourceRenderingMode.scala delete mode 100644 akka-http/src/main/scala/akka/http/javadsl/common/SourceRenderingMode.scala delete mode 100644 akka-http/src/main/scala/akka/http/javadsl/server/EntityStreamingSupport.scala create mode 100644 akka-http/src/main/scala/akka/http/scaladsl/common/CsvEntityStreamingSupport.scala create mode 100644 akka-http/src/main/scala/akka/http/scaladsl/common/EntityStreamingSupport.scala delete mode 100644 akka-http/src/main/scala/akka/http/scaladsl/common/FramingWithContentType.scala create mode 100644 akka-http/src/main/scala/akka/http/scaladsl/common/JsonEntityStreamingSupport.scala delete mode 100644 akka-http/src/main/scala/akka/http/scaladsl/common/JsonSourceRenderingMode.scala delete mode 100644 akka-http/src/main/scala/akka/http/scaladsl/common/SourceRenderingMode.scala delete mode 100644 akka-http/src/main/scala/akka/http/scaladsl/server/EntityStreamingSupport.scala diff --git a/akka-docs/rst/java/code/docs/http/javadsl/server/JsonStreamingExamplesTest.java b/akka-docs/rst/java/code/docs/http/javadsl/server/JsonStreamingExamplesTest.java index acc6bbfac2e..c43d6241e53 100644 --- a/akka-docs/rst/java/code/docs/http/javadsl/server/JsonStreamingExamplesTest.java +++ b/akka-docs/rst/java/code/docs/http/javadsl/server/JsonStreamingExamplesTest.java @@ -5,17 +5,21 @@ package docs.http.javadsl.server; import akka.NotUsed; -import akka.http.javadsl.common.FramingWithContentType; -import akka.http.javadsl.common.JsonSourceRenderingModes; +import akka.http.javadsl.common.CsvEntityStreamingSupport; +import akka.http.javadsl.common.JsonEntityStreamingSupport; import akka.http.javadsl.marshallers.jackson.Jackson; +import akka.http.javadsl.marshalling.Marshaller; import akka.http.javadsl.model.*; import akka.http.javadsl.model.headers.Accept; import akka.http.javadsl.server.*; import akka.http.javadsl.testkit.JUnitRouteTest; import akka.http.javadsl.testkit.TestRoute; +import akka.http.javadsl.unmarshalling.StringUnmarshallers; +import akka.http.javadsl.common.EntityStreamingSupport; +import akka.http.javadsl.unmarshalling.Unmarshaller; +import akka.stream.javadsl.Flow; import akka.stream.javadsl.Source; import akka.util.ByteString; -import docs.http.javadsl.server.testkit.MyAppService; import org.junit.Test; import java.util.concurrent.CompletionStage; @@ -29,12 +33,32 @@ final Route tweets() { //#formats //#response-streaming + + // Step 1: Enable JSON streaming + // we're not using this in the example, but it's the simplest way to start: + // The default rendering is a JSON array: `[el, el, el , ...]` + final JsonEntityStreamingSupport jsonStreaming = EntityStreamingSupport.json(); + + // Step 1.1: Enable and customise how we'll render the JSON, as a compact array: + final ByteString start = ByteString.fromString("["); + final ByteString between = ByteString.fromString(","); + final ByteString end = ByteString.fromString("]"); + final Flow compactArrayRendering = + Flow.of(ByteString.class).intersperse(start, between, end); + + final JsonEntityStreamingSupport compactJsonSupport = EntityStreamingSupport.json() + .withFramingRendererFlow(compactArrayRendering); + + + // Step 2: implement the route final Route responseStreaming = path("tweets", () -> get(() -> parameter(StringUnmarshallers.INTEGER, "n", n -> { final Source tws = - Source.repeat(new JavaTweet("Hello World!")).take(n); - return completeOKWithSource(tws, Jackson.marshaller(), JsonSourceRenderingModes.arrayCompact()); + Source.repeat(new JavaTweet(12, "Hello World!")).take(n); + + // Step 3: call complete* with your source, marshaller, and stream rendering mode + return completeOKWithSource(tws, Jackson.marshaller(), compactJsonSupport); }) ) ); @@ -44,9 +68,9 @@ final Route tweets() { final Route incomingStreaming = path("tweets", () -> post(() -> extractMaterializer(mat -> { - final FramingWithContentType jsonFraming = EntityStreamingSupport.bracketCountingJsonFraming(128); + final JsonEntityStreamingSupport jsonSupport = EntityStreamingSupport.json(); - return entityasSourceOf(JavaTweets, jsonFraming, sourceOfTweets -> { + return entityAsSourceOf(JavaTweets, jsonSupport, sourceOfTweets -> { final CompletionStage tweetsCount = sourceOfTweets.runFold(0, (acc, tweet) -> acc + 1, mat); return onComplete(tweetsCount, c -> complete("Total number of tweets: " + c)); }); @@ -58,6 +82,29 @@ final Route tweets() { return responseStreaming.orElse(incomingStreaming); } + + final Route csvTweets() { + //#csv-example + final Marshaller renderAsCsv = + Marshaller.withFixedContentType(ContentTypes.TEXT_CSV_UTF8, t -> + ByteString.fromString(t.getId() + "," + t.getMessage()) + ); + + final CsvEntityStreamingSupport compactJsonSupport = EntityStreamingSupport.csv(); + + final Route responseStreaming = path("tweets", () -> + get(() -> + parameter(StringUnmarshallers.INTEGER, "n", n -> { + final Source tws = + Source.repeat(new JavaTweet(12, "Hello World!")).take(n); + return completeWithSource(tws, renderAsCsv, compactJsonSupport); + }) + ) + ); + //#csv-example + + return responseStreaming; + } //#routes @Test @@ -70,7 +117,7 @@ public void getTweetsTest() { final Accept acceptApplication = Accept.create(MediaRanges.create(MediaTypes.APPLICATION_JSON)); routes.run(HttpRequest.GET("/tweets?n=2").addHeader(acceptApplication)) .assertStatusCode(200) - .assertEntity("[{\"message\":\"Hello World!\"},{\"message\":\"Hello World!\"}]"); + .assertEntity("[{\"id\":12,\"message\":\"Hello World!\"},{\"id\":12,\"message\":\"Hello World!\"}]"); // test responses to potential errors final Accept acceptText = Accept.create(MediaRanges.ALL_TEXT); @@ -79,15 +126,46 @@ public void getTweetsTest() { .assertEntity("Resource representation is only available with these types:\napplication/json"); //#response-streaming } + + @Test + public void csvExampleTweetsTest() { + //#response-streaming + // tests -------------------------------------------- + final TestRoute routes = testRoute(csvTweets()); + + // test happy path + final Accept acceptCsv = Accept.create(MediaRanges.create(MediaTypes.TEXT_CSV)); + routes.run(HttpRequest.GET("/tweets?n=2").addHeader(acceptCsv)) + .assertStatusCode(200) + .assertEntity("12,Hello World!\n" + + "12,Hello World!"); + + // test responses to potential errors + final Accept acceptText = Accept.create(MediaRanges.ALL_APPLICATION); + routes.run(HttpRequest.GET("/tweets?n=3").addHeader(acceptText)) + .assertStatusCode(StatusCodes.NOT_ACCEPTABLE) // 406 + .assertEntity("Resource representation is only available with these types:\ntext/csv; charset=UTF-8"); + //#response-streaming + } //#models private static final class JavaTweet { + private int id; private String message; - public JavaTweet(String message) { + public JavaTweet(int id, String message) { + this.id = id; this.message = message; } + public int getId() { + return id; + } + + public void setId(int id) { + this.id = id; + } + public void setMessage(String message) { this.message = message; } diff --git a/akka-docs/rst/java/http/routing-dsl/source-streaming-support.rst b/akka-docs/rst/java/http/routing-dsl/source-streaming-support.rst index f4a0e7c4b61..acf46421b62 100644 --- a/akka-docs/rst/java/http/routing-dsl/source-streaming-support.rst +++ b/akka-docs/rst/java/http/routing-dsl/source-streaming-support.rst @@ -3,18 +3,15 @@ Source Streaming ================ -Akka HTTP supports completing a request with an Akka ``Source``, which makes it possible to easily build -streaming end-to-end APIs which apply back-pressure throughout the entire stack. +Akka HTTP supports completing a request with an Akka ``Source``, which makes it possible to easily build +and consume streaming end-to-end APIs which apply back-pressure throughout the entire stack. -It is possible to complete requests with raw ``Source``, however often it is more convenient to +It is possible to complete requests with raw ``Source``, however often it is more convenient to stream on an element-by-element basis, and allow Akka HTTP to handle the rendering internally - for example as a JSON array, or CSV stream (where each element is separated by a new-line). In the following sections we investigate how to make use of the JSON Streaming infrastructure, -however the general hints apply to any kind of element-by-element streaming you could imagine. - -It is possible to implement your own framing for any content type you might need, including bianary formats -by implementing :class:`FramingWithContentType`. +however the general hints apply to any kind of element-by-element streaming you could imagine. JSON Streaming ============== @@ -24,7 +21,7 @@ objects as a continuous HTTP request or response. The elements are most often se however do not have to be. Concatenating elements side-by-side or emitting "very long" JSON array is also another use case. -In the below examples, we'll be refering to the ``User`` and ``Measurement`` case classes as our model, which are defined as: +In the below examples, we'll be refering to the ``Tweet`` and ``Measurement`` case classes as our model, which are defined as: .. includecode:: ../../code/docs/http/javadsl/server/JsonStreamingExamplesTest.java#models @@ -36,11 +33,21 @@ Responding with JSON Streams In this example we implement an API representing an infinite stream of tweets, very much like Twitter's `Streaming API`_. Firstly, we'll need to get some additional marshalling infrastructure set up, that is able to marshal to and from an -Akka Streams ``Source``. One such trait, containing the needed marshallers is ``SprayJsonSupport``, which uses -spray-json (a high performance json parser library), and is shipped as part of Akka HTTP in the -``akka-http-spray-json-experimental`` module. +Akka Streams ``Source``. Here we'll use the ``Jackson`` helper class from ``akka-http-jackson`` (a separate library +that you should add as a dependency if you want to use Jackson with Akka HTTP). + +First we enable JSON Streaming by making an implicit ``EntityStreamingSupport`` instance available (Step 1). + +The default mode of rendering a ``Source`` is to represent it as an JSON Array. If you want to change this representation +for example to use Twitter style new-line separated JSON objects, you can do so by configuring the support trait accordingly. -The last bit of setup, before we can render a streaming json response +In Step 1.1. we demonstrate to configure configude the rendering to be new-line separated, and also how parallel marshalling +can be applied. We configure the Support object to render the JSON as series of new-line separated JSON objects, +simply by providing the ``start``, ``sep`` and ``end`` ByteStrings, which will be emitted at the apropriate +places in the rendered stream. Although this format is *not* valid JSON, it is pretty popular since parsing it is relatively +simple - clients need only to find the new-lines and apply JSON unmarshalling for an entire line of JSON. + +The final step is simply completing a request using a Source of tweets, as simple as that: .. includecode:: ../../code/docs/http/javadsl/server/JsonStreamingExamplesTest.java#response-streaming @@ -60,15 +67,25 @@ will be applied automatically thanks to using Akka HTTP/Streams). .. includecode:: ../../code/docs/http/javadsl/server/JsonStreamingExamplesTest.java#incoming-request-streaming -Implementing custom (Un)Marshaller support for JSON streaming -------------------------------------------------------------- -The following types that may need to be implemented by a custom framed-streaming support library are: +Simple CSV streaming example +---------------------------- + +Akka HTTP provides another ``EntityStreamingSupport`` out of the box, namely ``csv`` (comma-separated values). +For completeness, we demonstrate its usage in the below snippet. As you'll notice, switching betweeen streaming +modes is fairly simple, one only has to make sure that an implicit ``Marshaller`` of the requested type is available, +and that the streaming support operates on the same ``Content-Type`` as the rendered values. Otherwise you'll see +an error during runtime that the marshaller did not expose the expected content type and thus we can not render +the streaming response). + +.. includecode:: ../../code/docs/http/javadsl/server/JsonStreamingExamplesTest.java#csv-example + +Implementing custom EntityStreamingSupport traits +------------------------------------------------- + +The ``EntityStreamingSupport`` infrastructure is open for extension and not bound to any single format, content type +or marshalling library. The provided JSON support does not rely on Spray JSON directly, but uses ``Marshaller`` +instances, which can be provided using any JSON marshalling library (such as Circe, Jawn or Play JSON). -- ``SourceRenderingMode`` which can customise how to render the begining / between-elements and ending of such - stream (while writing a response, i.e. by calling ``complete(source)``). - Implementations for JSON are available in ``akka.http.scaladsl.common.JsonSourceRenderingMode``. -- ``FramingWithContentType`` which is needed to be able to split incoming ``ByteString`` - chunks into frames of the higher-level data type format that is understood by the provided unmarshallers. - In the case of JSON it means chunking up ByteStrings such that each emitted element corresponds to exactly one JSON object, - this framing is implemented in ``EntityStreamingSupport``. +When implementing a custom support trait, one should simply extend the ``EntityStreamingSupport`` abstract class, +and implement all of it's methods. It's best to use the existing implementations as a guideline. diff --git a/akka-docs/rst/scala/code/docs/http/scaladsl/server/directives/JsonStreamingExamplesSpec.scala b/akka-docs/rst/scala/code/docs/http/scaladsl/server/directives/JsonStreamingExamplesSpec.scala index 4f047ae595f..c311515c1b5 100644 --- a/akka-docs/rst/scala/code/docs/http/scaladsl/server/directives/JsonStreamingExamplesSpec.scala +++ b/akka-docs/rst/scala/code/docs/http/scaladsl/server/directives/JsonStreamingExamplesSpec.scala @@ -5,13 +5,15 @@ package docs.http.scaladsl.server.directives import akka.NotUsed -import akka.http.scaladsl.common.{ FramingWithContentType, JsonSourceRenderingModes } -import akka.http.scaladsl.marshalling.ToResponseMarshallable +import akka.http.scaladsl.common.{ EntityStreamingSupport, JsonEntityStreamingSupport } +import akka.http.scaladsl.marshalling._ import akka.http.scaladsl.model._ import akka.http.scaladsl.model.headers.Accept import akka.http.scaladsl.server.{ UnacceptedResponseContentTypeRejection, UnsupportedRequestContentTypeRejection } import akka.stream.scaladsl.{ Flow, Source } +import akka.util.ByteString import docs.http.scaladsl.server.RoutingSpec +import spray.json.JsValue import scala.concurrent.Future @@ -29,39 +31,41 @@ class JsonStreamingExamplesSpec extends RoutingSpec { Tweet(3, "You cannot enter the same river twice."))) //#formats - object MyJsonProtocol extends spray.json.DefaultJsonProtocol { + object MyJsonProtocol + extends akka.http.scaladsl.marshallers.sprayjson.SprayJsonSupport + with spray.json.DefaultJsonProtocol { + implicit val tweetFormat = jsonFormat2(Tweet.apply) implicit val measurementFormat = jsonFormat2(Measurement.apply) } //# "spray-json-response-streaming" in { - // [1] import generic spray-json marshallers support: - import akka.http.scaladsl.marshallers.sprayjson.SprayJsonSupport._ - - // [2] import "my protocol", for marshalling Tweet objects: + // [1] import "my protocol", for marshalling Tweet objects: import MyJsonProtocol._ - // [3] pick json rendering mode: - // HINT: if you extend `akka.http.scaladsl.server.EntityStreamingSupport` - // it'll guide you to do so via abstract defs - implicit val jsonRenderingMode = JsonSourceRenderingModes.LineByLine + // [2] pick a Source rendering support trait: + // Note that the default support renders the Source as JSON Array + implicit val jsonStreamingSupport: JsonEntityStreamingSupport = EntityStreamingSupport.json() val route = path("tweets") { + // [3] simply complete a request with a source of tweets: val tweets: Source[Tweet, NotUsed] = getTweets() complete(tweets) } - // tests: + // tests ------------------------------------------------------------ val AcceptJson = Accept(MediaRange(MediaTypes.`application/json`)) val AcceptXml = Accept(MediaRange(MediaTypes.`text/xml`)) Get("/tweets").withHeaders(AcceptJson) ~> route ~> check { responseAs[String] shouldEqual - """{"uid":1,"txt":"#Akka rocks!"}""" + "\n" + - """{"uid":2,"txt":"Streaming is so hot right now!"}""" + "\n" + - """{"uid":3,"txt":"You cannot enter the same river twice."}""" + """[""" + + """{"uid":1,"txt":"#Akka rocks!"},""" + + """{"uid":2,"txt":"Streaming is so hot right now!"},""" + + """{"uid":3,"txt":"You cannot enter the same river twice."}""" + + """]""" } // endpoint can only marshal Json, so it will *reject* requests for application/xml: @@ -71,44 +75,115 @@ class JsonStreamingExamplesSpec extends RoutingSpec { } } - "response-streaming-modes" in { - import akka.http.scaladsl.marshallers.sprayjson.SprayJsonSupport._ + "line-by-line-json-response-streaming" in { import MyJsonProtocol._ - implicit val jsonRenderingMode = JsonSourceRenderingModes.LineByLine - //#async-rendering - path("tweets") { - val tweets: Source[Tweet, NotUsed] = getTweets() - complete(tweets.renderAsync(parallelism = 8)) + // Configure the EntityStreamingSupport to render the elements as: + // {"example":42} + // {"example":43} + // ... + // {"example":1000} + val start = ByteString.empty + val sep = ByteString("\n") + val end = ByteString.empty + + implicit val jsonStreamingSupport = EntityStreamingSupport.json() + .withFramingRenderer(Flow[ByteString].intersperse(start, sep, end)) + + val route = + path("tweets") { + // [3] simply complete a request with a source of tweets: + val tweets: Source[Tweet, NotUsed] = getTweets() + complete(tweets) + } + + // tests ------------------------------------------------------------ + val AcceptJson = Accept(MediaRange(MediaTypes.`application/json`)) + + Get("/tweets").withHeaders(AcceptJson) ~> route ~> check { + responseAs[String] shouldEqual + """{"uid":1,"txt":"#Akka rocks!"}""" + "\n" + + """{"uid":2,"txt":"Streaming is so hot right now!"}""" + "\n" + + """{"uid":3,"txt":"You cannot enter the same river twice."}""" } - //# + } - //#async-unordered-rendering - path("tweets" / "unordered") { - val tweets: Source[Tweet, NotUsed] = getTweets() - complete(tweets.renderAsyncUnordered(parallelism = 8)) + "csv-example" in { + // [1] provide a marshaller to ByteString + implicit val tweetAsCsv = Marshaller.strict[Tweet, ByteString] { t => + Marshalling.WithFixedContentType(ContentTypes.`text/csv(UTF-8)`, () => { + val txt = t.txt.replaceAll(",", ".") + val uid = t.uid + ByteString(List(uid, txt).mkString(",")) + }) + } + + // [2] enable csv streaming: + implicit val csvStreaming = EntityStreamingSupport.csv() + + val route = + path("tweets") { + val tweets: Source[Tweet, NotUsed] = getTweets() + complete(tweets) + } + + // tests ------------------------------------------------------------ + val AcceptCsv = Accept(MediaRange(MediaTypes.`text/csv`)) + + Get("/tweets").withHeaders(AcceptCsv) ~> route ~> check { + responseAs[String] shouldEqual + """|1,#Akka rocks! + |2,Streaming is so hot right now! + |3,You cannot enter the same river twice.""" + .stripMargin } - //# } - "spray-json-request-streaming" in { - // [1] import generic spray-json (un)marshallers support: - import akka.http.scaladsl.marshallers.sprayjson.SprayJsonSupport._ + "response-streaming-modes" in { - // [1.1] import framing mode - import akka.http.scaladsl.server.EntityStreamingSupport - implicit val jsonFramingMode: FramingWithContentType = - EntityStreamingSupport.bracketCountingJsonFraming(Int.MaxValue) + { + //#async-rendering + import MyJsonProtocol._ + implicit val jsonStreamingSupport: JsonEntityStreamingSupport = + EntityStreamingSupport.json() + .withParallelMarshalling(parallelism = 8, unordered = false) - // [2] import "my protocol", for unmarshalling Measurement objects: + path("tweets") { + val tweets: Source[Tweet, NotUsed] = getTweets() + complete(tweets) + } + //# + } + + { + + //#async-unordered-rendering + import MyJsonProtocol._ + implicit val jsonStreamingSupport: JsonEntityStreamingSupport = + EntityStreamingSupport.json() + .withParallelMarshalling(parallelism = 8, unordered = true) + + path("tweets" / "unordered") { + val tweets: Source[Tweet, NotUsed] = getTweets() + complete(tweets) + } + //# + } + } + + "spray-json-request-streaming" in { + // [1] import "my protocol", for unmarshalling Measurement objects: import MyJsonProtocol._ - // [3] prepare your persisting logic here + // [2] enable Json Streaming + implicit val jsonStreamingSupport = EntityStreamingSupport.json() + + // prepare your persisting logic here val persistMetrics = Flow[Measurement] val route = path("metrics") { - // [4] extract Source[Measurement, _] + // [3] extract Source[Measurement, _] entity(asSourceOf[Measurement]) { measurements => // alternative syntax: // entity(as[Source[Measurement, NotUsed]]) { measurements => @@ -123,7 +198,8 @@ class JsonStreamingExamplesSpec extends RoutingSpec { } } - // tests: + // tests ------------------------------------------------------------ + // uploading an array or newline separated values works out of the box val data = HttpEntity( ContentTypes.`application/json`, """ diff --git a/akka-docs/rst/scala/http/routing-dsl/source-streaming-support.rst b/akka-docs/rst/scala/http/routing-dsl/source-streaming-support.rst index 8ead39b7660..4a2d121b801 100644 --- a/akka-docs/rst/scala/http/routing-dsl/source-streaming-support.rst +++ b/akka-docs/rst/scala/http/routing-dsl/source-streaming-support.rst @@ -4,7 +4,7 @@ Source Streaming ================ Akka HTTP supports completing a request with an Akka ``Source[T, _]``, which makes it possible to easily build -streaming end-to-end APIs which apply back-pressure throughout the entire stack. +and consume streaming end-to-end APIs which apply back-pressure throughout the entire stack. It is possible to complete requests with raw ``Source[ByteString, _]``, however often it is more convenient to stream on an element-by-element basis, and allow Akka HTTP to handle the rendering internally - for example as a JSON array, @@ -13,9 +13,6 @@ or CSV stream (where each element is separated by a new-line). In the following sections we investigate how to make use of the JSON Streaming infrastructure, however the general hints apply to any kind of element-by-element streaming you could imagine. -It is possible to implement your own framing for any content type you might need, including bianary formats -by implementing :class:`FramingWithContentType`. - JSON Streaming ============== @@ -24,7 +21,7 @@ objects as a continuous HTTP request or response. The elements are most often se however do not have to be. Concatenating elements side-by-side or emitting "very long" JSON array is also another use case. -In the below examples, we'll be refering to the ``User`` and ``Measurement`` case classes as our model, which are defined as: +In the below examples, we'll be refering to the ``Tweet`` and ``Measurement`` case classes as our model, which are defined as: .. includecode2:: ../../code/docs/http/scaladsl/server/directives/JsonStreamingExamplesSpec.scala :snippet: models @@ -47,32 +44,53 @@ Akka Streams ``Source[T,_]``. One such trait, containing the needed marshallers spray-json (a high performance json parser library), and is shipped as part of Akka HTTP in the ``akka-http-spray-json-experimental`` module. -Next we import our model's marshallers, generated by spray-json. +Once the general infrastructure is prepared we import our model's marshallers, generated by spray-json (Step 1), +and enable JSON Streaming by making an implicit ``EntityStreamingSupport`` instance available (Step 2). +Akka HTTP pre-packages JSON and CSV entity streaming support, however it is simple to add your own, in case you'd +like to stream a different content type (for example plists or protobuf). -The last bit of setup, before we can render a streaming json response +The final step is simply completing a request using a Source of tweets, as simple as that: .. includecode2:: ../../code/docs/http/scaladsl/server/directives/JsonStreamingExamplesSpec.scala :snippet: spray-json-response-streaming +The reason the ``EntityStreamingSupport`` has to be enabled explicitly is that one might want to configure how the +stream should be rendered. We'll dicuss this in depth in the next section though. + .. _Streaming API: https://dev.twitter.com/streaming/overview Customising response rendering mode ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -The mode in which a response is marshalled and then rendered to the HttpResponse from the provided ``Source[T,_]`` -is customisable (thanks to conversions originating from ``Directives`` via ``EntityStreamingDirectives``). +Since it is not always possible to directly and confidently answer the question of how a stream of ``T`` should look on +the wire, the ``EntityStreamingSupport`` traits come into play and allow fine-tuning the streams rendered representation. + +For example, in case of JSON Streaming, there isn't really one standard about rendering the response. Some APIs prefer +to render multiple JSON objects in a line-by-line fashion (Twitter's streaming APIs for example), while others simply return +very large arrays, which could be streamed as well. + +Akka defaults to the second one (streaming a JSON Array), as it is correct JSON and clients not expecting +a streaming API would still be able to consume it in a naive way if they'd want to. + +The line-by-line aproach however is also pretty popular even though it is not valid JSON. It's relatively simplicity for +client-side parsing is a strong point in case to pick this format for your Streaming APIs. +Below we demonstrate how to reconfigure the support trait to render the JSON as + +.. includecode2:: ../../code/docs/http/scaladsl/server/directives/JsonStreamingExamplesSpec.scala + :snippet: line-by-line-json-response-streaming -Since Marshalling is a potentially asynchronous operation in Akka HTTP (because transforming ``T`` to ``JsValue`` may -potentially take a long time (depending on your definition of "long time"), we allow to run marshalling concurrently -(up to ``parallelism`` concurrent marshallings) by using the ``renderAsync(parallelism)`` mode: +Another interesting feature is parallel marshalling. Since marshalling can potentially take much time, +it is possible to marshal multiple elements of the stream in parallel. This is simply a configuration +option on ``EntityStreamingSupport`` and is configurable like this: .. includecode2:: ../../code/docs/http/scaladsl/server/directives/JsonStreamingExamplesSpec.scala :snippet: async-rendering -The ``renderAsync`` mode perserves ordering of the Source's elements, which may sometimes be a required property, +The above shown mode perserves ordering of the Source's elements, which may sometimes be a required property, for example when streaming a strictly ordered dataset. Sometimes the contept of strict-order does not apply to the -data being streamed though, which allows us to explit this property and use ``renderAsyncUnordered(parallelism)``, -which will concurrently marshall up to ``parallelism`` elements and emit the first which is marshalled onto -the HttpResponse: +data being streamed though, which allows us to exploit this property and use an ``unordered`` rendering. + +This also is a configuration option and is used as shown below. Effectively this will allow Akka's marshalling infrastructure +to concurrently marshallup to ``parallelism`` elements and emit the first which is marshalled onto the ``HttpResponse``: .. includecode2:: ../../code/docs/http/scaladsl/server/directives/JsonStreamingExamplesSpec.scala :snippet: async-unordered-rendering @@ -94,18 +112,25 @@ will be applied automatically thanks to using Akka HTTP/Streams). .. includecode2:: ../../code/docs/http/scaladsl/server/directives/JsonStreamingExamplesSpec.scala :snippet: spray-json-request-streaming -Implementing custom (Un)Marshaller support for JSON streaming -------------------------------------------------------------- +Simple CSV streaming example +---------------------------- + +Akka HTTP provides another ``EntityStreamingSupport`` out of the box, namely ``csv`` (comma-separated values). +For completeness, we demonstrate its usage in the below snippet. As you'll notice, switching betweeen streaming +modes is fairly simple, one only has to make sure that an implicit ``Marshaller`` of the requested type is available, +and that the streaming support operates on the same ``Content-Type`` as the rendered values. Otherwise you'll see +an error during runtime that the marshaller did not expose the expected content type and thus we can not render +the streaming response). + +.. includecode2:: ../../code/docs/http/scaladsl/server/directives/JsonStreamingExamplesSpec.scala + :snippet: csv-example -While not provided by Akka HTTP directly, the infrastructure is extensible and by investigating how ``SprayJsonSupport`` -is implemented it is certainly possible to provide the same infrastructure for other marshaller implementations (such as -Play JSON, or Jackson directly for example). Such support traits will want to extend the ``EntityStreamingSupport`` trait. +Implementing custom EntityStreamingSupport traits +------------------------------------------------- -The following types that may need to be implemented by a custom framed-streaming support library are: +The ``EntityStreamingSupport`` infrastructure is open for extension and not bound to any single format, content type +or marshalling library. The provided JSON support does not rely on Spray JSON directly, but uses ``Marshaller[T, ByteString]`` +instances, which can be provided using any JSON marshalling library (such as Circe, Jawn or Play JSON). -- ``SourceRenderingMode`` which can customise how to render the begining / between-elements and ending of such stream (while writing a response, i.e. by calling ``complete(source)``). - Implementations for JSON are available in ``akka.http.scaladsl.server.JsonSourceRenderingMode``. -- ``FramingWithContentType`` which is needed to be able to split incoming ``ByteString`` chunks into frames - of the higher-level data type format that is understood by the provided unmarshallers. - In the case of JSON it means chunking up ByteStrings such that each emitted element corresponds to exactly one JSON object, - this framing is implemented in ``EntityStreamingSupport``. +When implementing a custom support trait, one should simply extend the ``EntityStreamingSupport`` abstract class, +and implement all of it's methods. It's best to use the existing implementations as a guideline. diff --git a/akka-http-marshallers-scala/akka-http-spray-json/src/main/scala/akka/http/scaladsl/marshallers/sprayjson/SprayJsonSupport.scala b/akka-http-marshallers-scala/akka-http-spray-json/src/main/scala/akka/http/scaladsl/marshallers/sprayjson/SprayJsonSupport.scala index cc5fe074954..e399ca77069 100644 --- a/akka-http-marshallers-scala/akka-http-spray-json/src/main/scala/akka/http/scaladsl/marshallers/sprayjson/SprayJsonSupport.scala +++ b/akka-http-marshallers-scala/akka-http-spray-json/src/main/scala/akka/http/scaladsl/marshallers/sprayjson/SprayJsonSupport.scala @@ -4,11 +4,14 @@ package akka.http.scaladsl.marshallers.sprayjson -import akka.http.scaladsl.marshalling.{ Marshaller, ToByteStringMarshaller, ToEntityMarshaller } +import akka.NotUsed +import akka.http.scaladsl.common.EntityStreamingSupport +import akka.http.scaladsl.marshalling._ import akka.http.scaladsl.model.MediaTypes.`application/json` import akka.http.scaladsl.model.{ HttpCharsets, MediaTypes } -import akka.http.scaladsl.unmarshalling.{ FromEntityUnmarshaller, Unmarshaller } +import akka.http.scaladsl.unmarshalling.{ FromEntityUnmarshaller, FromRequestUnmarshaller, Unmarshaller } import akka.http.scaladsl.util.FastFuture +import akka.stream.scaladsl.{ Flow, Keep, Source } import akka.util.ByteString import spray.json._ @@ -44,7 +47,19 @@ trait SprayJsonSupport { sprayJsValueMarshaller compose writer.write implicit def sprayJsValueMarshaller(implicit printer: JsonPrinter = CompactPrinter): ToEntityMarshaller[JsValue] = Marshaller.StringMarshaller.wrap(MediaTypes.`application/json`)(printer) - implicit def sprayByteStringMarshaller[T](implicit writer: RootJsonFormat[T], printer: JsonPrinter = CompactPrinter): ToByteStringMarshaller[T] = - sprayJsValueMarshaller.map(s ⇒ ByteString(s.toString)) compose writer.write + + // support for as[Source[T, NotUsed]] + implicit def sprayJsonSourceReader[T](implicit rootJsonReader: RootJsonReader[T], support: EntityStreamingSupport): FromRequestUnmarshaller[Source[T, NotUsed]] = + Unmarshaller.withMaterializer { implicit ec ⇒ implicit mat ⇒ r ⇒ + if (support.supported.matches(r.entity.contentType)) { + val bytes = r.entity.dataBytes + val frames = bytes.via(support.framingDecoder) + val unmarshalling = + if (support.unordered) Flow[ByteString].mapAsyncUnordered(support.parallelism)(bs ⇒ sprayJsonByteStringUnmarshaller(rootJsonReader)(bs)) + else Flow[ByteString].mapAsync(support.parallelism)(bs ⇒ sprayJsonByteStringUnmarshaller(rootJsonReader)(bs)) + val elements = frames.viaMat(unmarshalling)(Keep.right) + FastFuture.successful(elements) + } else FastFuture.failed(Unmarshaller.UnsupportedContentTypeException(support.supported)) + } } object SprayJsonSupport extends SprayJsonSupport diff --git a/akka-http-tests/src/test/java/akka/http/javadsl/server/JavaTestServer.java b/akka-http-tests/src/test/java/akka/http/javadsl/server/JavaTestServer.java index 70900a9987c..761a8e2ba9d 100644 --- a/akka-http-tests/src/test/java/akka/http/javadsl/server/JavaTestServer.java +++ b/akka-http-tests/src/test/java/akka/http/javadsl/server/JavaTestServer.java @@ -8,12 +8,13 @@ import akka.http.javadsl.ConnectHttp; import akka.http.javadsl.Http; import akka.http.javadsl.ServerBinding; +import akka.http.javadsl.common.EntityStreamingSupport; import akka.http.javadsl.marshallers.jackson.Jackson; -import akka.http.javadsl.model.HttpEntity; import akka.http.javadsl.model.HttpRequest; import akka.http.javadsl.model.HttpResponse; import akka.http.javadsl.model.StatusCodes; -import akka.http.javadsl.common.JsonSourceRenderingModes; +import akka.http.javadsl.unmarshalling.StringUnmarshallers; +import akka.http.javadsl.unmarshalling.Unmarshaller; import akka.stream.ActorMaterializer; import akka.stream.javadsl.Flow; import akka.stream.javadsl.Source; @@ -70,12 +71,12 @@ public Route createRoute() { get(() -> parameter(StringUnmarshallers.INTEGER, "n", n -> { final Source tws = Source.repeat(new JavaTweet("Hello World!")).take(n); - return completeOKWithSource(tws, Jackson.marshaller(), JsonSourceRenderingModes.arrayCompact()); + return completeOKWithSource(tws, Jackson.marshaller(), EntityStreamingSupport.json()); }) ).orElse( post(() -> extractMaterializer(mat -> - entityasSourceOf(JavaTweets, null, sourceOfTweets -> { + entityAsSourceOf(JavaTweets, null, sourceOfTweets -> { final CompletionStage tweetsCount = sourceOfTweets.runFold(0, (acc, tweet) -> acc + 1, mat); return onComplete(tweetsCount, c -> complete("Total number of tweets: " + c)); }) diff --git a/akka-http-tests/src/test/scala/akka/http/scaladsl/server/TestServer.scala b/akka-http-tests/src/test/scala/akka/http/scaladsl/server/TestServer.scala index 1512bcb0d98..ea5cb71894f 100644 --- a/akka-http-tests/src/test/scala/akka/http/scaladsl/server/TestServer.scala +++ b/akka-http-tests/src/test/scala/akka/http/scaladsl/server/TestServer.scala @@ -13,8 +13,9 @@ import akka.actor.ActorSystem import akka.stream._ import akka.stream.scaladsl._ import akka.http.scaladsl.Http -import akka.http.scaladsl.common.{ FramingWithContentType, JsonSourceRenderingModes, SourceRenderingMode } +import akka.http.scaladsl.common.EntityStreamingSupport import akka.http.scaladsl.marshalling.ToResponseMarshallable +import spray.json.RootJsonReader import scala.concurrent.duration._ import scala.io.StdIn @@ -30,20 +31,12 @@ object TestServer extends App { import system.dispatcher implicit val materializer = ActorMaterializer() - // --------- json streaming --------- import spray.json.DefaultJsonProtocol._ import akka.http.scaladsl.marshallers.sprayjson.SprayJsonSupport._ final case class Tweet(message: String) implicit val tweetFormat = jsonFormat1(Tweet) - // FIXME: Need to be able to support composive framing with content type (!!!!!!!) - import akka.http.scaladsl.server.EntityStreamingSupport._ - /* override if extending EntityStreamingSupport */ - implicit val incomingEntityStreamFraming: FramingWithContentType = bracketCountingJsonFraming(128) - /* override if extending EntityStreamingSupport */ - implicit val outgoingEntityStreamRendering: SourceRenderingMode = JsonSourceRenderingModes.LineByLine - - // --------- end of json streaming --------- + implicit val jsonStreaming = EntityStreamingSupport.json() import ScalaXmlSupport._ import Directives._ @@ -65,13 +58,7 @@ object TestServer extends App { } ~ path("secure") { authenticateBasicPF("My very secure site", auth) { user ⇒ - complete( - Hello - - {user} - - . Access has been granted! - ) + complete( Hello {user}. Access has been granted! ) } } ~ path("ping") { @@ -89,6 +76,14 @@ object TestServer extends App { complete(tweets) } ~ post { + entity(asSourceOf[Tweet]) { tweets ⇒ + onComplete(tweets.runFold(0)({ case (acc, t) => acc + 1 })) { count => + complete(s"Total tweets received: " + count) + } + } + } ~ + put { + // checking the alternative syntax also works: entity(as[Source[Tweet, NotUsed]]) { tweets ⇒ onComplete(tweets.runFold(0)({ case (acc, t) => acc + 1 })) { count => complete(s"Total tweets received: " + count) @@ -103,7 +98,7 @@ object TestServer extends App { val bindingFuture = Http().bindAndHandle(routes, interface = "0.0.0.0", port = 8080) - println(s"Server online at http://localhost:8080/\nPress RETURN to stop...") + println(s"Server online at http://0.0.0.0:8080/\nPress RETURN to stop...") StdIn.readLine() bindingFuture.flatMap(_.unbind()).onComplete(_ ⇒ system.terminate()) diff --git a/akka-http/src/main/scala/akka/http/javadsl/common/CsvSourceRenderingMode.scala b/akka-http/src/main/scala/akka/http/javadsl/common/CsvSourceRenderingMode.scala deleted file mode 100644 index d755dd6bf2d..00000000000 --- a/akka-http/src/main/scala/akka/http/javadsl/common/CsvSourceRenderingMode.scala +++ /dev/null @@ -1,41 +0,0 @@ -/* - * Copyright (C) 2016 Lightbend Inc. - */ - -package akka.http.javadsl.common - -import akka.http.javadsl.model.ContentType.WithCharset -import akka.http.javadsl.model.ContentTypes -import akka.util.ByteString - -/** - * Specialised rendering mode for streaming elements as CSV. - */ -trait CsvSourceRenderingMode extends SourceRenderingMode { - override val contentType: WithCharset = - ContentTypes.TEXT_CSV_UTF8 -} - -object CsvSourceRenderingModes { - - /** - * Render sequence of values as row-by-row ('\n' separated) series of values. - */ - val create: CsvSourceRenderingMode = - new CsvSourceRenderingMode { - override def between: ByteString = ByteString("\n") - override def end: ByteString = ByteString.empty - override def start: ByteString = ByteString.empty - } - - /** - * Render sequence of values as row-by-row (with custom row separator, - * e.g. if you need to use '\r\n' instead of '\n') series of values. - */ - def custom(rowSeparator: String): CsvSourceRenderingMode = - new CsvSourceRenderingMode { - override def between: ByteString = ByteString(rowSeparator) - override def end: ByteString = ByteString.empty - override def start: ByteString = ByteString.empty - } -} diff --git a/akka-http/src/main/scala/akka/http/javadsl/common/EntityStreamingSupport.scala b/akka-http/src/main/scala/akka/http/javadsl/common/EntityStreamingSupport.scala new file mode 100644 index 00000000000..b2c461f4c88 --- /dev/null +++ b/akka-http/src/main/scala/akka/http/javadsl/common/EntityStreamingSupport.scala @@ -0,0 +1,149 @@ +/* + * Copyright (C) 2016 Lightbend Inc. + */ + +package akka.http.javadsl.common + +import akka.NotUsed +import akka.http.javadsl.model.{ ContentType, ContentTypeRange } +import akka.http.scaladsl.common +import akka.stream.javadsl.Flow +import akka.util.ByteString + +/** + * Entity streaming support trait allowing rendering and receiving incoming ``Source[T, _]`` from HTTP entities. + * + * See [[JsonEntityStreamingSupport]] or [[CsvEntityStreamingSupport]] for default implementations. + */ +abstract class EntityStreamingSupport { + + /** Read-side, what content types it is able to frame and unmarshall. */ + def supported: ContentTypeRange + /** Write-side, defines what Content-Type the Marshaller should offer and the final Content-Type of the response. */ + def contentType: ContentType + + /** + * Read-side, decode incoming framed entity. + * For example with an incoming JSON array, chunk it up into JSON objects contained within that array. + */ + def getFramingDecoder: Flow[ByteString, ByteString, NotUsed] + /** + * Write-side, apply framing to outgoing entity stream. + * + * Most typical usage will be a variant of `Flow[ByteString].intersperse`. + * + * For example for rendering a JSON array one would return + * `Flow[ByteString].intersperse(ByteString("["), ByteString(","), ByteString("]"))` + * and for rendering a new-line separated CSV simply `Flow[ByteString].intersperse(ByteString("\n"))`. + */ + def getFramingRenderer: Flow[ByteString, ByteString, NotUsed] + + /** + * Read-side, allows changing what content types are accepted by this framing. + * + * EntityStreamingSupport traits MUST support re-configuring the accepted [[ContentTypeRange]]. + * + * This is in order to support a-typical APIs which users still want to communicate with using + * the provided support trait. Typical examples include APIs which return valid `application/json` + * however advertise the content type as being `application/javascript` or vendor specific content types, + * which still parse correctly as JSON, CSV or something else that a provided support trait is built for. + * + * NOTE: Implementations should specialize the return type to their own Type! + */ + def withSupported(range: ContentTypeRange): EntityStreamingSupport + + /** + * Write-side, defines what Content-Type the Marshaller should offer and the final Content-Type of the response. + * + * EntityStreamingSupport traits MUST support re-configuring the offered [[ContentType]]. + * This is due to the need integrating with existing systems which sometimes excpect custom Content-Types, + * however really are just plain JSON or something else internally (perhaps with slight extensions). + * + * NOTE: Implementations should specialize the return type to their own Type! + */ + def withContentType(range: ContentType): EntityStreamingSupport + + /** + * Write-side / read-side, defines if (un)marshalling should be done in parallel. + * + * This may be beneficial marshalling the bottleneck in the pipeline. + * + * See also [[parallelism]] and [[withParallelMarshalling]]. + */ + def parallelism: Int + + /** + * Write-side / read-side, defines if (un)marshalling of incoming stream elements should be perserved or not. + * + * Allowing for parallel and unordered (un)marshalling often yields higher throughput and also allows avoiding + * head-of-line blocking if some elements are much larger than others. + * + * See also [[parallelism]] and [[withParallelMarshalling]]. + */ + def unordered: Boolean + + /** + * Write-side / read-side, defines parallelism and if ordering should be preserved or not of Source element marshalling. + * + * Sometimes marshalling multiple elements at once (esp. when elements are not evenly sized, and ordering is not enforced) + * may yield in higher throughput. + * + * NOTE: Implementations should specialize the return type to their own Type! + */ + def withParallelMarshalling(parallelism: Int, unordered: Boolean): EntityStreamingSupport + +} + +/** + * Entity streaming support, independent of used Json parsing library etc. + */ +object EntityStreamingSupport { + + /** + * Default `application/json` entity streaming support. + * + * Provides framing (based on scanning the incoming dataBytes for valid JSON objects, so for example uploads using arrays or + * new-line separated JSON objects are all parsed correctly) and rendering of Sources as JSON Arrays. + * A different very popular style of returning streaming JSON is to separate JSON objects on a line-by-line basis, + * you can configure the support trait to do so by calling `withFramingRendererFlow`. + * + * Limits the maximum JSON object length to 8KB, if you want to increase this limit provide a value explicitly. + * + * See also https://en.wikipedia.org/wiki/JSON_Streaming + */ + def json(): JsonEntityStreamingSupport = json(8 * 1024) + /** + * Default `application/json` entity streaming support. + * + * Provides framing (based on scanning the incoming dataBytes for valid JSON objects, so for example uploads using arrays or + * new-line separated JSON objects are all parsed correctly) and rendering of Sources as JSON Arrays. + * A different very popular style of returning streaming JSON is to separate JSON objects on a line-by-line basis, + * you can configure the support trait to do so by calling `withFramingRendererFlow`. + * + * See also https://en.wikipedia.org/wiki/JSON_Streaming + */ + def json(maxObjectLength: Int): JsonEntityStreamingSupport = common.EntityStreamingSupport.json(maxObjectLength) + + /** + * Default `text/csv(UTF-8)` entity streaming support. + * Provides framing and rendering of `\n` separated lines and marshalling Sources into such values. + * + * Limits the maximum line-length to 8KB, if you want to increase this limit provide a value explicitly. + */ + def csv(): CsvEntityStreamingSupport = csv(8 * 1024) + /** + * Default `text/csv(UTF-8)` entity streaming support. + * Provides framing and rendering of `\n` separated lines and marshalling Sources into such values. + */ + def csv(maxLineLength: Int): CsvEntityStreamingSupport = common.EntityStreamingSupport.csv(maxLineLength) +} + +// extends Scala base, in order to get linearization right and (as we can't go into traits here, because companion object needed) +abstract class JsonEntityStreamingSupport extends common.EntityStreamingSupport { + def withFramingRendererFlow(flow: Flow[ByteString, ByteString, NotUsed]): JsonEntityStreamingSupport +} + +// extends Scala base, in order to get linearization right and (as we can't go into traits here, because companion object needed) +abstract class CsvEntityStreamingSupport extends common.EntityStreamingSupport { + def withFramingRendererFlow(flow: Flow[ByteString, ByteString, NotUsed]): CsvEntityStreamingSupport +} diff --git a/akka-http/src/main/scala/akka/http/javadsl/common/FramingWithContentType.scala b/akka-http/src/main/scala/akka/http/javadsl/common/FramingWithContentType.scala deleted file mode 100644 index 483b3d624fd..00000000000 --- a/akka-http/src/main/scala/akka/http/javadsl/common/FramingWithContentType.scala +++ /dev/null @@ -1,36 +0,0 @@ -/* - * Copyright (C) 2016 Lightbend Inc. - */ - -package akka.http.javadsl.common - -import akka.NotUsed -import akka.event.Logging -import akka.http.javadsl.model.ContentTypeRange -import akka.stream.javadsl.{ Flow, Framing } -import akka.util.ByteString - -/** - * Wraps a framing [[akka.stream.javadsl.Flow]] (as provided by [[Framing]] for example) - * that chunks up incoming [[akka.util.ByteString]] according to some [[akka.http.javadsl.model.ContentType]] - * specific logic. - */ -abstract class FramingWithContentType { self ⇒ - import akka.http.impl.util.JavaMapping.Implicits._ - - def getFlow: Flow[ByteString, ByteString, NotUsed] - - def asScala: akka.http.scaladsl.common.FramingWithContentType = - this match { - case f: akka.http.scaladsl.common.FramingWithContentType ⇒ f - case _ ⇒ new akka.http.scaladsl.common.FramingWithContentType { - override def flow = self.getFlow.asScala - override def supported = self.supported.asScala - } - } - - def supported: ContentTypeRange - def matches(ct: akka.http.javadsl.model.ContentType): Boolean = supported.matches(ct) - - override def toString = s"${Logging.simpleName(getClass)}($supported)" -} diff --git a/akka-http/src/main/scala/akka/http/javadsl/common/JsonSourceRenderingMode.scala b/akka-http/src/main/scala/akka/http/javadsl/common/JsonSourceRenderingMode.scala deleted file mode 100644 index 7d4ca7b7275..00000000000 --- a/akka-http/src/main/scala/akka/http/javadsl/common/JsonSourceRenderingMode.scala +++ /dev/null @@ -1,103 +0,0 @@ -/* - * Copyright (C) 2016 Lightbend Inc. - */ - -package akka.http.javadsl.common - -import akka.http.javadsl.model.{ ContentType, ContentTypes } - -/** - * Specialised rendering mode for streaming elements as JSON. - * - * See also: JSON Streaming on Wikipedia. - * - * See [[JsonSourceRenderingModes]] for commonly used pre-defined rendering modes. - */ -trait JsonSourceRenderingMode extends SourceRenderingMode { - override val contentType: ContentType.WithFixedCharset = - ContentTypes.APPLICATION_JSON -} - -/** - * Provides default JSON rendering modes. - */ -object JsonSourceRenderingModes { - - /** - * Most compact rendering mode. - * It does not intersperse any separator between the signalled elements. - * - * It can be used with [[akka.stream.javadsl.JsonFraming.bracketCounting]]. - * - * {{{ - * {"id":42}{"id":43}{"id":44} - * }}} - */ - val compact = akka.http.scaladsl.common.JsonSourceRenderingModes.Compact - - /** - * Simple rendering mode, similar to [[compact]] however interspersing elements with a `\n` character. - * - * {{{ - * {"id":42},{"id":43},{"id":44} - * }}} - */ - val compactCommaSeparated = akka.http.scaladsl.common.JsonSourceRenderingModes.CompactCommaSeparated - - /** - * Rendering mode useful when the receiving end expects a valid JSON Array. - * It can be useful when the client wants to detect when the stream has been successfully received in-full, - * which it can determine by seeing the terminating `]` character. - * - * The framing's terminal `]` will ONLY be emitted if the stream has completed successfully, - * in other words - the stream has been emitted completely, without errors occuring before the final element has been signaled. - * - * {{{ - * [{"id":42},{"id":43},{"id":44}] - * }}} - */ - val arrayCompact = akka.http.scaladsl.common.JsonSourceRenderingModes.ArrayCompact - - /** - * Rendering mode useful when the receiving end expects a valid JSON Array. - * It can be useful when the client wants to detect when the stream has been successfully received in-full, - * which it can determine by seeing the terminating `]` character. - * - * The framing's terminal `]` will ONLY be emitted if the stream has completed successfully, - * in other words - the stream has been emitted completely, without errors occuring before the final element has been signaled. - * - * {{{ - * [{"id":42}, - * {"id":43}, - * {"id":44}] - * }}} - */ - val arrayLineByLine = akka.http.scaladsl.common.JsonSourceRenderingModes.ArrayLineByLine - - /** - * Recommended rendering mode. - * - * It is a nice balance between valid and human-readable as well as resonably small size overhead (just the `\n` between elements). - * A good example of API's using this syntax is Twitter's Firehose (last verified at 1.1 version of that API). - * - * {{{ - * {"id":42} - * {"id":43} - * {"id":44} - * }}} - */ - val lineByLine = akka.http.scaladsl.common.JsonSourceRenderingModes.LineByLine - - /** - * Simple rendering mode interspersing each pair of elements with both `,\n`. - * Picking the [[lineByLine]] format may be preferable, as it is slightly simpler to parse - each line being a valid json object (no need to trim the comma). - * - * {{{ - * {"id":42}, - * {"id":43}, - * {"id":44} - * }}} - */ - val lineByLineCommaSeparated = akka.http.scaladsl.common.JsonSourceRenderingModes.LineByLineCommaSeparated - -} diff --git a/akka-http/src/main/scala/akka/http/javadsl/common/SourceRenderingMode.scala b/akka-http/src/main/scala/akka/http/javadsl/common/SourceRenderingMode.scala deleted file mode 100644 index 5144f336f6f..00000000000 --- a/akka-http/src/main/scala/akka/http/javadsl/common/SourceRenderingMode.scala +++ /dev/null @@ -1,22 +0,0 @@ -/* - * Copyright (C) 2016 Lightbend Inc. - */ - -package akka.http.javadsl.common - -import akka.http.javadsl.model.ContentType -import akka.util.ByteString - -/** - * Defines how to render a [[akka.stream.javadsl.Source]] into a raw [[ByteString]] - * output. - * - * This can be used to render a source into an [[akka.http.scaladsl.model.HttpEntity]]. - */ -trait SourceRenderingMode { - def contentType: ContentType - - def start: ByteString - def between: ByteString - def end: ByteString -} diff --git a/akka-http/src/main/scala/akka/http/javadsl/server/EntityStreamingSupport.scala b/akka-http/src/main/scala/akka/http/javadsl/server/EntityStreamingSupport.scala deleted file mode 100644 index 571867ac634..00000000000 --- a/akka-http/src/main/scala/akka/http/javadsl/server/EntityStreamingSupport.scala +++ /dev/null @@ -1,41 +0,0 @@ -/* - * Copyright (C) 2016 Lightbend Inc. - */ - -package akka.http.javadsl.server - -import akka.NotUsed -import akka.http.javadsl.common.{ FramingWithContentType, SourceRenderingMode } -import akka.http.javadsl.model.{ ContentTypeRange, MediaRanges } -import akka.http.scaladsl.server.ApplicationJsonBracketCountingFraming -import akka.stream.javadsl.{ Flow, Framing } -import akka.util.ByteString - -/** - * Entity streaming support, independent of used Json parsing library etc. - * - * Can be extended by various Support traits (e.g. "SprayJsonSupport"), - * in order to provide users with both `framing` (this trait) and `marshalling` - * (implemented by a library) by using a single trait. - */ -object EntityStreamingSupport { - // in the ScalaDSL version we make users implement abstract methods that are supposed to be - // implicit vals. This helps to guide in implementing the needed values, however in Java that would not really help. - - /** `application/json` specific Framing implementation */ - def bracketCountingJsonFraming(maximumObjectLength: Int): FramingWithContentType = - new ApplicationJsonBracketCountingFraming(maximumObjectLength) - - /** - * Frames incoming `text / *` entities on a line-by-line basis. - * Useful for accepting `text/csv` uploads as a stream of rows. - */ - def newLineFraming(maximumObjectLength: Int, supportedContentTypes: ContentTypeRange): FramingWithContentType = - new FramingWithContentType { - override final val getFlow: Flow[ByteString, ByteString, NotUsed] = - Flow.of(classOf[ByteString]).via(Framing.delimiter(ByteString("\n"), maximumObjectLength)) - - override final val supported: ContentTypeRange = - akka.http.scaladsl.model.ContentTypeRange(akka.http.scaladsl.model.MediaRanges.`text/*`) - } -} diff --git a/akka-http/src/main/scala/akka/http/javadsl/server/RoutingJavaMapping.scala b/akka-http/src/main/scala/akka/http/javadsl/server/RoutingJavaMapping.scala index dda99cbb75d..ebb6eeb3137 100644 --- a/akka-http/src/main/scala/akka/http/javadsl/server/RoutingJavaMapping.scala +++ b/akka-http/src/main/scala/akka/http/javadsl/server/RoutingJavaMapping.scala @@ -8,11 +8,13 @@ import java.util.concurrent.CompletionStage import akka.http.impl.util.JavaMapping._ import akka.http.impl.util._ +import akka.http.javadsl.common.EntityStreamingSupport import akka.http.{ javadsl, scaladsl } import akka.http.scaladsl.server.{ directives ⇒ sdirectives } import akka.http.scaladsl.{ common ⇒ scommon } import akka.http.javadsl.server.{ directives ⇒ jdirectives } import akka.http.javadsl.{ common ⇒ jcommon } + import scala.collection.immutable /** @@ -45,7 +47,7 @@ private[http] object RoutingJavaMapping { } implicit object convertRouteResult extends Inherited[javadsl.server.RouteResult, scaladsl.server.RouteResult] - implicit object convertSourceRenderingMode extends Inherited[jcommon.SourceRenderingMode, scommon.SourceRenderingMode] + implicit object convertEntityStreamingSupport extends Inherited[EntityStreamingSupport, scommon.EntityStreamingSupport] implicit object convertDirectoryRenderer extends Inherited[jdirectives.DirectoryRenderer, sdirectives.FileAndResourceDirectives.DirectoryRenderer] implicit object convertContentTypeResolver extends Inherited[jdirectives.ContentTypeResolver, sdirectives.ContentTypeResolver] diff --git a/akka-http/src/main/scala/akka/http/javadsl/server/directives/FramedEntityStreamingDirectives.scala b/akka-http/src/main/scala/akka/http/javadsl/server/directives/FramedEntityStreamingDirectives.scala index 153ee0601e8..eef4a3d1f9c 100644 --- a/akka-http/src/main/scala/akka/http/javadsl/server/directives/FramedEntityStreamingDirectives.scala +++ b/akka-http/src/main/scala/akka/http/javadsl/server/directives/FramedEntityStreamingDirectives.scala @@ -7,10 +7,12 @@ import java.util.function.{ Function ⇒ JFunction } import java.util.{ List ⇒ JList, Map ⇒ JMap } import akka.NotUsed -import akka.http.javadsl.common.{ FramingWithContentType, SourceRenderingMode } +import akka.http.javadsl.common.EntityStreamingSupport +import akka.http.javadsl.marshalling.Marshaller import akka.http.javadsl.model.{ HttpEntity, _ } -import akka.http.javadsl.server.{ Marshaller, Route, Unmarshaller } -import akka.http.scaladsl.marshalling.ToResponseMarshallable +import akka.http.javadsl.server.Route +import akka.http.javadsl.unmarshalling.Unmarshaller +import akka.http.scaladsl.marshalling.{ Marshalling, ToByteStringMarshaller, ToResponseMarshallable } import akka.http.scaladsl.server.{ Directives ⇒ D } import akka.stream.javadsl.Source import akka.util.ByteString @@ -18,56 +20,39 @@ import akka.util.ByteString /** EXPERIMENTAL API */ abstract class FramedEntityStreamingDirectives extends TimeoutDirectives { + import akka.http.javadsl.server.RoutingJavaMapping._ + import akka.http.javadsl.server.RoutingJavaMapping.Implicits._ + @CorrespondsTo("asSourceOf") - def entityasSourceOf[T](um: Unmarshaller[ByteString, T], framing: FramingWithContentType, + def entityAsSourceOf[T](um: Unmarshaller[ByteString, T], support: EntityStreamingSupport, inner: java.util.function.Function[Source[T, NotUsed], Route]): Route = RouteAdapter { - D.entity(D.asSourceOf[T](framing.asScala)(um.asScala)) { s: akka.stream.scaladsl.Source[T, NotUsed] ⇒ - inner(s.asJava).delegate - } - } - - @CorrespondsTo("asSourceOfAsync") - def entityAsSourceAsyncOf[T]( - parallelism: Int, - um: Unmarshaller[ByteString, T], framing: FramingWithContentType, - inner: java.util.function.Function[Source[T, NotUsed], Route]): Route = RouteAdapter { - D.entity(D.asSourceOfAsync[T](parallelism, framing.asScala)(um.asScala)) { s: akka.stream.scaladsl.Source[T, NotUsed] ⇒ + val umm = D.asSourceOf(um.asScala, support.asScala) + D.entity(umm) { s: akka.stream.scaladsl.Source[T, NotUsed] ⇒ inner(s.asJava).delegate } } - @CorrespondsTo("asSourceOfAsyncUnordered") - def entityAsSourceAsyncUnorderedOf[T]( - parallelism: Int, - um: Unmarshaller[ByteString, T], framing: FramingWithContentType, - inner: java.util.function.Function[Source[T, NotUsed], Route]): Route = RouteAdapter { - D.entity(D.asSourceOfAsyncUnordered[T](parallelism, framing.asScala)(um.asScala)) { s: akka.stream.scaladsl.Source[T, NotUsed] ⇒ - inner(s.asJava).delegate - } - } - - // implicits used internally, Java caller does not benefit or use it + // implicits and multiple parameter lists used internally, Java caller does not benefit or use it @CorrespondsTo("complete") - def completeWithSource[T, M](implicit source: Source[T, M], m: Marshaller[T, ByteString], rendering: SourceRenderingMode): Route = RouteAdapter { - implicit val mm = _sourceMarshaller(m.map(ByteStringAsEntityFn), rendering) - val response = ToResponseMarshallable(source) + def completeWithSource[T, M](source: Source[T, M])(implicit m: Marshaller[T, ByteString], support: EntityStreamingSupport): Route = RouteAdapter { + import akka.http.scaladsl.marshalling.PredefinedToResponseMarshallers._ + val mm = m.map(ByteStringAsEntityFn).asScalaCastOutput[akka.http.scaladsl.model.RequestEntity] + val mmm = fromEntityStreamingSupportAndEntityMarshaller[T, M](support.asScala, mm) + val response = ToResponseMarshallable(source.asScala)(mmm) D.complete(response) } + // implicits and multiple parameter lists used internally, Java caller does not benefit or use it @CorrespondsTo("complete") - def completeOKWithSource[T, M](implicit source: Source[T, M], m: Marshaller[T, RequestEntity], rendering: SourceRenderingMode): Route = RouteAdapter { - implicit val mm = _sourceMarshaller[T, M](m, rendering) - val response = ToResponseMarshallable(source) + def completeOKWithSource[T, M](source: Source[T, M])(implicit m: Marshaller[T, RequestEntity], support: EntityStreamingSupport): Route = RouteAdapter { + import akka.http.scaladsl.marshalling.PredefinedToResponseMarshallers._ + // don't try this at home: + val mm = m.asScalaCastOutput[akka.http.scaladsl.model.RequestEntity].map(_.httpEntity.asInstanceOf[akka.http.scaladsl.model.RequestEntity]) + implicit val mmm = fromEntityStreamingSupportAndEntityMarshaller[T, M](support.asScala, mm) + val response = ToResponseMarshallable(source.asScala) D.complete(response) } - implicit private def _sourceMarshaller[T, M](implicit m: Marshaller[T, HttpEntity], rendering: SourceRenderingMode) = { - import akka.http.javadsl.server.RoutingJavaMapping.Implicits._ - import akka.http.javadsl.server.RoutingJavaMapping._ - val mm = m.asScalaCastOutput - D._sourceMarshaller[T, M](mm, rendering.asScala).compose({ h: akka.stream.javadsl.Source[T, M] ⇒ h.asScala }) - } - private[this] val ByteStringAsEntityFn = new java.util.function.Function[ByteString, HttpEntity]() { override def apply(bs: ByteString): HttpEntity = HttpEntities.create(bs) } diff --git a/akka-http/src/main/scala/akka/http/javadsl/server/directives/FutureDirectives.scala b/akka-http/src/main/scala/akka/http/javadsl/server/directives/FutureDirectives.scala index 50a0cc75efd..f89ac51563c 100644 --- a/akka-http/src/main/scala/akka/http/javadsl/server/directives/FutureDirectives.scala +++ b/akka-http/src/main/scala/akka/http/javadsl/server/directives/FutureDirectives.scala @@ -23,7 +23,7 @@ abstract class FutureDirectives extends FormFieldDirectives { /** * "Unwraps" a `CompletionStage` and runs the inner route after future * completion with the future's value as an extraction of type `Try`. - * + * * @group future */ def onComplete[T](f: Supplier[CompletionStage[T]], inner: JFunction[Try[T], Route]) = RouteAdapter { @@ -35,7 +35,7 @@ abstract class FutureDirectives extends FormFieldDirectives { /** * "Unwraps" a `CompletionStage` and runs the inner route after future * completion with the future's value as an extraction of type `Try`. - * + * * @group future */ def onComplete[T](cs: CompletionStage[T], inner: JFunction[Try[T], Route]) = RouteAdapter { @@ -65,7 +65,7 @@ abstract class FutureDirectives extends FormFieldDirectives { * completion with the stage's value as an extraction of type `T`. * If the stage fails its failure Throwable is bubbled up to the nearest * ExceptionHandler. - * + * * @group future */ def onSuccess[T](f: Supplier[CompletionStage[T]], inner: JFunction[T, Route]) = RouteAdapter { @@ -80,7 +80,7 @@ abstract class FutureDirectives extends FormFieldDirectives { * If the completion stage succeeds the request is completed using the values marshaller * (This directive therefore requires a marshaller for the completion stage value type to be * provided.) - * + * * @group future */ def completeOrRecoverWith[T](f: Supplier[CompletionStage[T]], marshaller: Marshaller[T, RequestEntity], inner: JFunction[Throwable, Route]): Route = RouteAdapter { diff --git a/akka-http/src/main/scala/akka/http/scaladsl/common/CsvEntityStreamingSupport.scala b/akka-http/src/main/scala/akka/http/scaladsl/common/CsvEntityStreamingSupport.scala new file mode 100644 index 00000000000..536a86f0039 --- /dev/null +++ b/akka-http/src/main/scala/akka/http/scaladsl/common/CsvEntityStreamingSupport.scala @@ -0,0 +1,48 @@ +/* + * Copyright (C) 2016 Lightbend Inc. + */ + +package akka.http.scaladsl.common + +import akka.NotUsed +import akka.event.Logging +import akka.http.javadsl.{ common, model ⇒ jm } +import akka.http.scaladsl.model.{ ContentType, ContentTypeRange, ContentTypes } +import akka.stream.scaladsl.{ Flow, Framing } +import akka.util.ByteString + +final class CsvEntityStreamingSupport private[akka] ( + maxLineLength: Int, + val supported: ContentTypeRange, + val contentType: ContentType, + val framingRenderer: Flow[ByteString, ByteString, NotUsed], + val parallelism: Int, + val unordered: Boolean +) extends common.CsvEntityStreamingSupport { + import akka.http.impl.util.JavaMapping.Implicits._ + + def this(maxObjectSize: Int) = + this( + maxObjectSize, + ContentTypeRange(ContentTypes.`text/csv(UTF-8)`), + ContentTypes.`text/csv(UTF-8)`, + Flow[ByteString].intersperse(ByteString("\n")), + 1, false) + + override val framingDecoder: Flow[ByteString, ByteString, NotUsed] = + Framing.delimiter(ByteString("\n"), maxLineLength) + + override def withFramingRendererFlow(framingRendererFlow: akka.stream.javadsl.Flow[ByteString, ByteString, NotUsed]): CsvEntityStreamingSupport = + withFramingRenderer(framingRendererFlow.asScala) + def withFramingRenderer(framingRendererFlow: Flow[ByteString, ByteString, NotUsed]): CsvEntityStreamingSupport = + new CsvEntityStreamingSupport(maxLineLength, supported, contentType, framingRendererFlow, parallelism, unordered) + + override def withContentType(ct: jm.ContentType): CsvEntityStreamingSupport = + new CsvEntityStreamingSupport(maxLineLength, supported, ct.asScala, framingRenderer, parallelism, unordered) + override def withSupported(range: jm.ContentTypeRange): CsvEntityStreamingSupport = + new CsvEntityStreamingSupport(maxLineLength, range.asScala, contentType, framingRenderer, parallelism, unordered) + override def withParallelMarshalling(parallelism: Int, unordered: Boolean): CsvEntityStreamingSupport = + new CsvEntityStreamingSupport(maxLineLength, supported, contentType, framingRenderer, parallelism, unordered) + + override def toString = s"""${Logging.simpleName(getClass)}($maxLineLength, $supported, $contentType)""" +} diff --git a/akka-http/src/main/scala/akka/http/scaladsl/common/EntityStreamingSupport.scala b/akka-http/src/main/scala/akka/http/scaladsl/common/EntityStreamingSupport.scala new file mode 100644 index 00000000000..aea219666fa --- /dev/null +++ b/akka-http/src/main/scala/akka/http/scaladsl/common/EntityStreamingSupport.scala @@ -0,0 +1,141 @@ +/* + * Copyright (C) 2016 Lightbend Inc. + */ +package akka.http.scaladsl.common + +import akka.NotUsed +import akka.http.javadsl.{ common, model ⇒ jm } +import akka.http.scaladsl.model._ +import akka.stream.scaladsl.Flow +import akka.util.ByteString + +/** + * Entity streaming support trait allowing rendering and receiving incoming ``Source[T, _]`` from HTTP entities. + * + * See [[JsonEntityStreamingSupport]] or [[CsvEntityStreamingSupport]] for default implementations. + */ +abstract class EntityStreamingSupport extends common.EntityStreamingSupport { + /** Read-side, what content types it is able to frame and unmarshall. */ + def supported: ContentTypeRange + /** Write-side, defines what Content-Type the Marshaller should offer and the final Content-Type of the response. */ + def contentType: ContentType + + /** + * Read-side, decode incoming framed entity. + * For example with an incoming JSON array, chunk it up into JSON objects contained within that array. + */ + def framingDecoder: Flow[ByteString, ByteString, NotUsed] + override final def getFramingDecoder = framingDecoder.asJava + + /** + * Write-side, apply framing to outgoing entity stream. + * + * Most typical usage will be a variant of `Flow[ByteString].intersperse`. + * + * For example for rendering a JSON array one would return + * `Flow[ByteString].intersperse(ByteString("["), ByteString(","), ByteString("]"))` + * and for rendering a new-line separated CSV simply `Flow[ByteString].intersperse(ByteString("\n"))`. + */ + def framingRenderer: Flow[ByteString, ByteString, NotUsed] + override final def getFramingRenderer = framingRenderer.asJava + + /** + * Read-side, allows changing what content types are accepted by this framing. + * + * EntityStreamingSupport traits MUST support re-configuring the accepted [[ContentTypeRange]]. + * + * This is in order to support a-typical APIs which users still want to communicate with using + * the provided support trait. Typical examples include APIs which return valid `application/json` + * however advertise the content type as being `application/javascript` or vendor specific content types, + * which still parse correctly as JSON, CSV or something else that a provided support trait is built for. + * + * NOTE: Implementations should specialize the return type to their own Type! + */ + override def withSupported(range: jm.ContentTypeRange): EntityStreamingSupport + + /** + * Write-side, defines what Content-Type the Marshaller should offer and the final Content-Type of the response. + * + * EntityStreamingSupport traits MUST support re-configuring the offered [[ContentType]]. + * This is due to the need integrating with existing systems which sometimes excpect custom Content-Types, + * however really are just plain JSON or something else internally (perhaps with slight extensions). + * + * NOTE: Implementations should specialize the return type to their own Type! + */ + override def withContentType(range: jm.ContentType): EntityStreamingSupport + + /** + * Write-side / read-side, defines if (un)marshalling should be done in parallel. + * + * This may be beneficial marshalling the bottleneck in the pipeline. + * + * See also [[parallelism]] and [[withParallelMarshalling]]. + */ + def parallelism: Int + + /** + * Write-side / read-side, defines if (un)marshalling of incoming stream elements should be perserved or not. + * + * Allowing for parallel and unordered (un)marshalling often yields higher throughput and also allows avoiding + * head-of-line blocking if some elements are much larger than others. + * + * See also [[parallelism]] and [[withParallelMarshalling]]. + */ + def unordered: Boolean + + /** + * Write-side / read-side, defines parallelism and if ordering should be preserved or not of Source element marshalling. + * + * Sometimes marshalling multiple elements at once (esp. when elements are not evenly sized, and ordering is not enforced) + * may yield in higher throughput. + * + * NOTE: Implementations should specialize the return type to their own Type! + */ + def withParallelMarshalling(parallelism: Int, unordered: Boolean): EntityStreamingSupport + +} + +/** + * Entity streaming support, independent of used Json parsing library etc. + */ +object EntityStreamingSupport { + + /** + * Default `application/json` entity streaming support. + * + * Provides framing (based on scanning the incoming dataBytes for valid JSON objects, so for example uploads using arrays or + * new-line separated JSON objects are all parsed correctly) and rendering of Sources as JSON Arrays. + * A different very popular style of returning streaming JSON is to separate JSON objects on a line-by-line basis, + * you can configure the support trait to do so by calling `withFramingRendererFlow`. + * + * Limits the maximum JSON object length to 8KB, if you want to increase this limit provide a value explicitly. + * + * See also https://en.wikipedia.org/wiki/JSON_Streaming + */ + def json(): JsonEntityStreamingSupport = json(8 * 1024) + /** + * Default `application/json` entity streaming support. + * + * Provides framing (based on scanning the incoming dataBytes for valid JSON objects, so for example uploads using arrays or + * new-line separated JSON objects are all parsed correctly) and rendering of Sources as JSON Arrays. + * A different very popular style of returning streaming JSON is to separate JSON objects on a line-by-line basis, + * you can configure the support trait to do so by calling `withFramingRendererFlow`. + * + * See also https://en.wikipedia.org/wiki/JSON_Streaming + */ + def json(maxObjectLength: Int): JsonEntityStreamingSupport = new JsonEntityStreamingSupport(maxObjectLength) + + /** + * Default `text/csv(UTF-8)` entity streaming support. + * Provides framing and rendering of `\n` separated lines and marshalling Sources into such values. + * + * Limits the maximum line-length to 8KB, if you want to increase this limit provide a value explicitly. + */ + def csv(): CsvEntityStreamingSupport = csv(8 * 1024) + /** + * Default `text/csv(UTF-8)` entity streaming support. + * Provides framing and rendering of `\n` separated lines and marshalling Sources into such values. + */ + def csv(maxLineLength: Int): CsvEntityStreamingSupport = new CsvEntityStreamingSupport(maxLineLength) +} + diff --git a/akka-http/src/main/scala/akka/http/scaladsl/common/FramingWithContentType.scala b/akka-http/src/main/scala/akka/http/scaladsl/common/FramingWithContentType.scala deleted file mode 100644 index 13dc72ecbad..00000000000 --- a/akka-http/src/main/scala/akka/http/scaladsl/common/FramingWithContentType.scala +++ /dev/null @@ -1,25 +0,0 @@ -/* - * Copyright (C) 2016 Lightbend Inc. - */ - -package akka.http.scaladsl.common - -import akka.NotUsed -import akka.event.Logging -import akka.http.scaladsl.model.ContentTypeRange -import akka.stream.scaladsl.{ Flow, Framing } -import akka.util.ByteString - -/** - * Wraps a framing [[akka.stream.scaladsl.Flow]] (as provided by [[Framing]] for example) - * that chunks up incoming [[akka.util.ByteString]] according to some [[akka.http.javadsl.model.ContentType]] - * specific logic. - */ -abstract class FramingWithContentType extends akka.http.javadsl.common.FramingWithContentType { - def flow: Flow[ByteString, ByteString, NotUsed] - override final def getFlow = flow.asJava - override def supported: ContentTypeRange - override def matches(ct: akka.http.javadsl.model.ContentType): Boolean = supported.matches(ct) - - override def toString = s"${Logging.simpleName(getClass)}($supported)" -} diff --git a/akka-http/src/main/scala/akka/http/scaladsl/common/JsonEntityStreamingSupport.scala b/akka-http/src/main/scala/akka/http/scaladsl/common/JsonEntityStreamingSupport.scala new file mode 100644 index 00000000000..743f5fd8aa0 --- /dev/null +++ b/akka-http/src/main/scala/akka/http/scaladsl/common/JsonEntityStreamingSupport.scala @@ -0,0 +1,49 @@ +/* + * Copyright (C) 2016 Lightbend Inc. + */ + +package akka.http.scaladsl.common + +import akka.NotUsed +import akka.event.Logging +import akka.http.javadsl.{ common, model ⇒ jm } +import akka.http.scaladsl.model.{ ContentType, ContentTypeRange, ContentTypes } +import akka.stream.scaladsl.Flow +import akka.util.ByteString + +final class JsonEntityStreamingSupport private[akka] ( + maxObjectSize: Int, + val supported: ContentTypeRange, + val contentType: ContentType, + val framingRenderer: Flow[ByteString, ByteString, NotUsed], + val parallelism: Int, + val unordered: Boolean +) extends common.JsonEntityStreamingSupport { + import akka.http.impl.util.JavaMapping.Implicits._ + + def this(maxObjectSize: Int) = + this( + maxObjectSize, + ContentTypeRange(ContentTypes.`application/json`), + ContentTypes.`application/json`, + Flow[ByteString].intersperse(ByteString("["), ByteString(","), ByteString("]")), + 1, false) + + override val framingDecoder: Flow[ByteString, ByteString, NotUsed] = + akka.stream.scaladsl.JsonFraming.objectScanner(maxObjectSize) + + override def withFramingRendererFlow(framingRendererFlow: akka.stream.javadsl.Flow[ByteString, ByteString, NotUsed]): JsonEntityStreamingSupport = + withFramingRenderer(framingRendererFlow.asScala) + def withFramingRenderer(framingRendererFlow: Flow[ByteString, ByteString, NotUsed]): JsonEntityStreamingSupport = + new JsonEntityStreamingSupport(maxObjectSize, supported, contentType, framingRendererFlow, parallelism, unordered) + + override def withContentType(ct: jm.ContentType): JsonEntityStreamingSupport = + new JsonEntityStreamingSupport(maxObjectSize, supported, ct.asScala, framingRenderer, parallelism, unordered) + override def withSupported(range: jm.ContentTypeRange): JsonEntityStreamingSupport = + new JsonEntityStreamingSupport(maxObjectSize, range.asScala, contentType, framingRenderer, parallelism, unordered) + override def withParallelMarshalling(parallelism: Int, unordered: Boolean): JsonEntityStreamingSupport = + new JsonEntityStreamingSupport(maxObjectSize, supported, contentType, framingRenderer, parallelism, unordered) + + override def toString = s"""${Logging.simpleName(getClass)}($maxObjectSize, $supported, $contentType)""" + +} diff --git a/akka-http/src/main/scala/akka/http/scaladsl/common/JsonSourceRenderingMode.scala b/akka-http/src/main/scala/akka/http/scaladsl/common/JsonSourceRenderingMode.scala deleted file mode 100644 index 824af23b8c3..00000000000 --- a/akka-http/src/main/scala/akka/http/scaladsl/common/JsonSourceRenderingMode.scala +++ /dev/null @@ -1,128 +0,0 @@ -/* - * Copyright (C) 2016 Lightbend Inc. - */ - -package akka.http.scaladsl.common - -import akka.http.scaladsl.model.{ ContentType, ContentTypes } -import akka.util.ByteString - -/** - * Specialised rendering mode for streaming elements as JSON. - * - * See also: JSON Streaming on Wikipedia. - * - * See [[JsonSourceRenderingModes]] for commonly used pre-defined rendering modes. - */ -trait JsonSourceRenderingMode extends akka.http.javadsl.common.JsonSourceRenderingMode with SourceRenderingMode { - override val contentType: ContentType.WithFixedCharset = - ContentTypes.`application/json` -} - -/** - * Provides default JSON rendering modes. - */ -object JsonSourceRenderingModes { - - /** - * Most compact rendering mode. - * It does not intersperse any separator between the signalled elements. - * - * It is the most compact form to render JSON and can be framed properly by using [[akka.stream.javadsl.JsonFraming.bracketCounting]]. - * - * {{{ - * {"id":42}{"id":43}{"id":44} - * }}} - */ - object Compact extends JsonSourceRenderingMode { - override val start: ByteString = ByteString.empty - override val between: ByteString = ByteString.empty - override val end: ByteString = ByteString.empty - } - - /** - * Simple rendering mode, similar to [[Compact]] however interspersing elements with a `\n` character. - * - * {{{ - * {"id":42},{"id":43},{"id":44} - * }}} - */ - object CompactCommaSeparated extends JsonSourceRenderingMode { - override val start: ByteString = ByteString.empty - override val between: ByteString = ByteString(",") - override val end: ByteString = ByteString.empty - } - - /** - * Rendering mode useful when the receiving end expects a valid JSON Array. - * It can be useful when the client wants to detect when the stream has been successfully received in-full, - * which it can determine by seeing the terminating `]` character. - * - * The framing's terminal `]` will ONLY be emitted if the stream has completed successfully, - * in other words - the stream has been emitted completely, without errors occuring before the final element has been signaled. - * - * {{{ - * [{"id":42},{"id":43},{"id":44}] - * }}} - */ - object ArrayCompact extends JsonSourceRenderingMode { - override val start: ByteString = ByteString("[") - override val between: ByteString = ByteString(",") - override val end: ByteString = ByteString("]") - } - - /** - * Rendering mode useful when the receiving end expects a valid JSON Array. - * It can be useful when the client wants to detect when the stream has been successfully received in-full, - * which it can determine by seeing the terminating `]` character. - * - * The framing's terminal `]` will ONLY be emitted if the stream has completed successfully, - * in other words - the stream has been emitted completely, without errors occuring before the final element has been signaled. - * - * {{{ - * [{"id":42}, - * {"id":43}, - * {"id":44}] - * }}} - */ - object ArrayLineByLine extends JsonSourceRenderingMode { - override val start: ByteString = ByteString("[") - override val between: ByteString = ByteString(",\n") - override val end: ByteString = ByteString("]") - } - - /** - * Recommended rendering mode. - * - * It is a nice balance between valid and human-readable as well as resonably small size overhead (just the `\n` between elements). - * A good example of API's using this syntax is Twitter's Firehose (last verified at 1.1 version of that API). - * - * {{{ - * {"id":42} - * {"id":43} - * {"id":44} - * }}} - */ - object LineByLine extends JsonSourceRenderingMode { - override val start: ByteString = ByteString.empty - override val between: ByteString = ByteString("\n") - override val end: ByteString = ByteString.empty - } - - /** - * Simple rendering mode interspersing each pair of elements with both `,\n`. - * Picking the [[LineByLine]] format may be preferable, as it is slightly simpler to parse - each line being a valid json object (no need to trim the comma). - * - * {{{ - * {"id":42}, - * {"id":43}, - * {"id":44} - * }}} - */ - object LineByLineCommaSeparated extends JsonSourceRenderingMode { - override val start: ByteString = ByteString.empty - override val between: ByteString = ByteString(",\n") - override val end: ByteString = ByteString.empty - } - -} diff --git a/akka-http/src/main/scala/akka/http/scaladsl/common/SourceRenderingMode.scala b/akka-http/src/main/scala/akka/http/scaladsl/common/SourceRenderingMode.scala deleted file mode 100644 index 61abd851440..00000000000 --- a/akka-http/src/main/scala/akka/http/scaladsl/common/SourceRenderingMode.scala +++ /dev/null @@ -1,11 +0,0 @@ -/* - * Copyright (C) 2016 Lightbend Inc. - */ - -package akka.http.scaladsl.common - -import akka.http.scaladsl.model.ContentType - -trait SourceRenderingMode extends akka.http.javadsl.common.SourceRenderingMode { - override def contentType: ContentType -} diff --git a/akka-http/src/main/scala/akka/http/scaladsl/marshalling/PredefinedToResponseMarshallers.scala b/akka-http/src/main/scala/akka/http/scaladsl/marshalling/PredefinedToResponseMarshallers.scala index 62777103a22..35bc19f3d0f 100644 --- a/akka-http/src/main/scala/akka/http/scaladsl/marshalling/PredefinedToResponseMarshallers.scala +++ b/akka-http/src/main/scala/akka/http/scaladsl/marshalling/PredefinedToResponseMarshallers.scala @@ -4,12 +4,19 @@ package akka.http.scaladsl.marshalling +import akka.http.scaladsl.common.EntityStreamingSupport import akka.stream.impl.ConstantFun import scala.collection.immutable import akka.http.scaladsl.util.FastFuture._ import akka.http.scaladsl.model.MediaTypes._ import akka.http.scaladsl.model._ +import akka.http.scaladsl.server.ContentNegotiator +import akka.http.scaladsl.util.FastFuture +import akka.stream.scaladsl.Source +import akka.util.ByteString + +import scala.language.higherKinds trait PredefinedToResponseMarshallers extends LowPriorityToResponseMarshallerImplicits { @@ -41,6 +48,37 @@ trait PredefinedToResponseMarshallers extends LowPriorityToResponseMarshallerImp Marshaller(implicit ec ⇒ { case (status, headers, value) ⇒ mt(value).fast map (_ map (_ map (HttpResponse(status, headers, _)))) }) + + implicit def fromEntityStreamingSupportAndByteStringMarshaller[T, M](implicit s: EntityStreamingSupport, m: ToByteStringMarshaller[T]): ToResponseMarshaller[Source[T, M]] = { + Marshaller[Source[T, M], HttpResponse] { implicit ec ⇒ source ⇒ + FastFuture successful { + Marshalling.WithFixedContentType(s.contentType, () ⇒ { + val availableMarshallingsPerElement = source.mapAsync(1) { t ⇒ m(t)(ec) } + + // TODO optimise such that we pick the optimal marshalling only once (headAndTail needed?) + // TODO, NOTE: this is somewhat duplicated from Marshal.scala it could be made DRYer + val bestMarshallingPerElement = availableMarshallingsPerElement mapConcat { marshallings ⇒ + // pick the Marshalling that matches our EntityStreamingSupport + (s.contentType match { + case best @ (_: ContentType.Binary | _: ContentType.WithFixedCharset) ⇒ + marshallings collectFirst { case Marshalling.WithFixedContentType(`best`, marshal) ⇒ marshal } + + case best @ ContentType.WithCharset(bestMT, bestCS) ⇒ + marshallings collectFirst { + case Marshalling.WithFixedContentType(`best`, marshal) ⇒ marshal + case Marshalling.WithOpenCharset(`bestMT`, marshal) ⇒ () ⇒ marshal(bestCS) + } + }).toList + } + val marshalledElements: Source[ByteString, M] = + bestMarshallingPerElement.map(_.apply()) // marshal! + .via(s.framingRenderer) + + HttpResponse(entity = HttpEntity(s.contentType, marshalledElements)) + }) :: Nil + } + } + } } trait LowPriorityToResponseMarshallerImplicits { @@ -48,6 +86,40 @@ trait LowPriorityToResponseMarshallerImplicits { liftMarshaller(m) implicit def liftMarshaller[T](implicit m: ToEntityMarshaller[T]): ToResponseMarshaller[T] = PredefinedToResponseMarshallers.fromToEntityMarshaller() + + // FIXME deduplicate this!!! + implicit def fromEntityStreamingSupportAndEntityMarshaller[T, M](implicit s: EntityStreamingSupport, m: ToEntityMarshaller[T]): ToResponseMarshaller[Source[T, M]] = { + Marshaller[Source[T, M], HttpResponse] { implicit ec ⇒ source ⇒ + FastFuture successful { + Marshalling.WithFixedContentType(s.contentType, () ⇒ { + val availableMarshallingsPerElement = source.mapAsync(1) { t ⇒ m(t)(ec) } + + // TODO optimise such that we pick the optimal marshalling only once (headAndTail needed?) + // TODO, NOTE: this is somewhat duplicated from Marshal.scala it could be made DRYer + val bestMarshallingPerElement = availableMarshallingsPerElement mapConcat { marshallings ⇒ + // pick the Marshalling that matches our EntityStreamingSupport + (s.contentType match { + case best @ (_: ContentType.Binary | _: ContentType.WithFixedCharset) ⇒ + marshallings collectFirst { case Marshalling.WithFixedContentType(`best`, marshal) ⇒ marshal } + + case best @ ContentType.WithCharset(bestMT, bestCS) ⇒ + marshallings collectFirst { + case Marshalling.WithFixedContentType(`best`, marshal) ⇒ marshal + case Marshalling.WithOpenCharset(`bestMT`, marshal) ⇒ () ⇒ marshal(bestCS) + } + }).toList + } + val marshalledElements: Source[ByteString, M] = + bestMarshallingPerElement.map(_.apply()) // marshal! + .flatMapConcat(_.dataBytes) // extract raw dataBytes + .via(s.framingRenderer) + + HttpResponse(entity = HttpEntity(s.contentType, marshalledElements)) + }) :: Nil + } + } + } + } object PredefinedToResponseMarshallers extends PredefinedToResponseMarshallers diff --git a/akka-http/src/main/scala/akka/http/scaladsl/server/EntityStreamingSupport.scala b/akka-http/src/main/scala/akka/http/scaladsl/server/EntityStreamingSupport.scala deleted file mode 100644 index f4656e8612c..00000000000 --- a/akka-http/src/main/scala/akka/http/scaladsl/server/EntityStreamingSupport.scala +++ /dev/null @@ -1,68 +0,0 @@ -/* - * Copyright (C) 2009-2015 Typesafe Inc. - */ -package akka.http.scaladsl.server - -import akka.NotUsed -import akka.http.scaladsl.common.{ FramingWithContentType, SourceRenderingMode } -import akka.http.scaladsl.model.{ ContentTypeRange, ContentTypes, MediaRanges } -import akka.stream.scaladsl.{ Flow, Framing } -import akka.util.ByteString - -/** - * Entity streaming support, independent of used Json parsing library etc. - * - * Can be extended by various Support traits (e.g. "SprayJsonSupport"), - * in order to provide users with both `framing` (this trait) and `marshalling` - * (implemented by a library) by using a single trait. - */ -trait EntityStreamingSupport extends EntityStreamingSupportBase { - - /** - * Implement as `implicit val` with required framing implementation, for example in - * the case of streaming JSON uploads it could be `bracketCountingJsonFraming(maximumObjectLength)`. - */ - def incomingEntityStreamFraming: FramingWithContentType - - /** - * Implement as `implicit val` with the rendering mode to be used when redering `Source` instances. - * For example for JSON it could be [[akka.http.scaladsl.common.JsonSourceRenderingMode.CompactArray]] - * or [[akka.http.scaladsl.common.JsonSourceRenderingMode.LineByLine]]. - */ - def outgoingEntityStreamRendering: SourceRenderingMode -} - -trait EntityStreamingSupportBase { - /** `application/json` specific Framing implementation */ - def bracketCountingJsonFraming(maximumObjectLength: Int): FramingWithContentType = - new ApplicationJsonBracketCountingFraming(maximumObjectLength) - - /** - * Frames incoming `text / *` entities on a line-by-line basis. - * Useful for accepting `text/csv` uploads as a stream of rows. - */ - def newLineFraming(maximumObjectLength: Int, supportedContentTypes: ContentTypeRange): FramingWithContentType = - new TextNewLineFraming(maximumObjectLength, supportedContentTypes) -} - -/** - * Entity streaming support, independent of used Json parsing library etc. - * - * Can be extended by various Support traits (e.g. "SprayJsonSupport"), - * in order to provide users with both `framing` (this trait) and `marshalling` - * (implemented by a library) by using a single trait. - */ -object EntityStreamingSupport extends EntityStreamingSupportBase - -final class ApplicationJsonBracketCountingFraming(maximumObjectLength: Int) extends FramingWithContentType { - override final val flow = Flow[ByteString].via(akka.stream.scaladsl.JsonFraming.bracketCounting(maximumObjectLength)) - override final val supported = ContentTypeRange(ContentTypes.`application/json`) -} - -final class TextNewLineFraming(maximumLineLength: Int, supportedContentTypes: ContentTypeRange) extends FramingWithContentType { - override final val flow: Flow[ByteString, ByteString, NotUsed] = - Flow[ByteString].via(Framing.delimiter(ByteString("\n"), maximumLineLength)) - - override final val supported: ContentTypeRange = - ContentTypeRange(MediaRanges.`text/*`) -} diff --git a/akka-http/src/main/scala/akka/http/scaladsl/server/directives/FramedEntityStreamingDirectives.scala b/akka-http/src/main/scala/akka/http/scaladsl/server/directives/FramedEntityStreamingDirectives.scala index 9f3dffd6dff..fde94d9e4d8 100644 --- a/akka-http/src/main/scala/akka/http/scaladsl/server/directives/FramedEntityStreamingDirectives.scala +++ b/akka-http/src/main/scala/akka/http/scaladsl/server/directives/FramedEntityStreamingDirectives.scala @@ -4,27 +4,24 @@ package akka.http.scaladsl.server.directives import akka.NotUsed -import akka.http.scaladsl.common.{ FramingWithContentType, SourceRenderingMode } +import akka.http.scaladsl.common +import akka.http.scaladsl.common.EntityStreamingSupport import akka.http.scaladsl.marshalling._ import akka.http.scaladsl.model._ -import akka.http.scaladsl.unmarshalling.{ Unmarshal, Unmarshaller, _ } +import akka.http.scaladsl.unmarshalling.{ Unmarshaller, _ } import akka.http.scaladsl.util.FastFuture -import akka.stream.Materializer -import akka.stream.impl.ConstantFun import akka.stream.scaladsl.{ Flow, Keep, Source } import akka.util.ByteString -import scala.concurrent.ExecutionContext import scala.language.implicitConversions /** * Allows the [[MarshallingDirectives.entity]] directive to extract a [[Source]] of elements. * - * See [[akka.http.scaladsl.server.EntityStreamingSupport]] for useful default [[FramingWithContentType]] instances and + * See [[common.EntityStreamingSupport]] for useful default framing `Flow` instances and * support traits such as `SprayJsonSupport` (or your other favourite JSON library) to provide the needed [[Marshaller]] s. */ trait FramedEntityStreamingDirectives extends MarshallingDirectives { - import FramedEntityStreamingDirectives._ type RequestToSourceUnmarshaller[T] = FromRequestUnmarshaller[Source[T, NotUsed]] @@ -32,41 +29,33 @@ trait FramedEntityStreamingDirectives extends MarshallingDirectives { * Extracts entity as [[Source]] of elements of type `T`. * This is achieved by applying the implicitly provided (in the following order): * - * - 1st: [[FramingWithContentType]] in order to chunk-up the incoming [[ByteString]]s according to the - * `Content-Type` aware framing (for example, [[akka.http.scaladsl.server.EntityStreamingSupport.bracketCountingJsonFraming]]). - * - 2nd: [[Unmarshaller]] (from [[ByteString]] to `T`) for each of the respective "chunks" (e.g. for each JSON element contained within an array). + * - 1st: chunk-up the incoming [[ByteString]]s by applying the `Content-Type`-aware framing + * - 2nd: apply the [[Unmarshaller]] (from [[ByteString]] to `T`) for each of the respective "chunks" (e.g. for each JSON element contained within an array). * * The request will be rejected with an [[akka.http.scaladsl.server.UnsupportedRequestContentTypeRejection]] if * its [[ContentType]] is not supported by the used `framing` or `unmarshaller`. * - * It is recommended to use the [[akka.http.scaladsl.server.EntityStreamingSupport]] trait in conjunction with this - * directive as it helps provide the right [[FramingWithContentType]] and [[SourceRenderingMode]] for the most - * typical usage scenarios (JSON, CSV, ...). - * * Cancelling extracted [[Source]] closes the connection abruptly (same as cancelling the `entity.dataBytes`). * - * If looking to improve marshalling performance in face of many elements (possibly of different sizes), - * you may be interested in using [[asSourceOfAsyncUnordered]] instead. - * * See also [[MiscDirectives.withoutSizeLimit]] as you may want to allow streaming infinite streams of data in this route. * By default the uploaded data is limited by the `akka.http.parsing.max-content-length`. */ - final def asSourceOf[T](implicit um: Unmarshaller[ByteString, T], framing: FramingWithContentType): RequestToSourceUnmarshaller[T] = - asSourceOfAsync(1)(um, framing) + final def asSourceOf[T](implicit um: FromByteStringUnmarshaller[T], support: EntityStreamingSupport): RequestToSourceUnmarshaller[T] = + asSourceOfInternal(um, support) /** * Extracts entity as [[Source]] of elements of type `T`. * This is achieved by applying the implicitly provided (in the following order): * - * - 1st: [[FramingWithContentType]] in order to chunk-up the incoming [[ByteString]]s according to the - * `Content-Type` aware framing (for example, [[akka.http.scaladsl.server.EntityStreamingSupport.bracketCountingJsonFraming]]). + * - 1st: [[FramingFlow]] in order to chunk-up the incoming [[ByteString]]s according to the + * `Content-Type` aware framing (for example, [[common.EntityStreamingSupport.bracketCountingJsonFraming]]). * - 2nd: [[Unmarshaller]] (from [[ByteString]] to `T`) for each of the respective "chunks" (e.g. for each JSON element contained within an array). * * The request will be rejected with an [[akka.http.scaladsl.server.UnsupportedRequestContentTypeRejection]] if * its [[ContentType]] is not supported by the used `framing` or `unmarshaller`. * - * It is recommended to use the [[akka.http.scaladsl.server.EntityStreamingSupport]] trait in conjunction with this - * directive as it helps provide the right [[FramingWithContentType]] and [[SourceRenderingMode]] for the most + * It is recommended to use the [[common.EntityStreamingSupport]] trait in conjunction with this + * directive as it helps provide the right [[FramingFlow]] and [[SourceRenderingMode]] for the most * typical usage scenarios (JSON, CSV, ...). * * Cancelling extracted [[Source]] closes the connection abruptly (same as cancelling the `entity.dataBytes`). @@ -77,177 +66,25 @@ trait FramedEntityStreamingDirectives extends MarshallingDirectives { * See also [[MiscDirectives.withoutSizeLimit]] as you may want to allow streaming infinite streams of data in this route. * By default the uploaded data is limited by the `akka.http.parsing.max-content-length`. */ - final def asSourceOf[T](framing: FramingWithContentType)(implicit um: Unmarshaller[ByteString, T]): RequestToSourceUnmarshaller[T] = - asSourceOfAsync(1)(um, framing) - - /** - * Similar to [[asSourceOf]] however will apply at most `parallelism` unmarshallers in parallel. - * - * The source elements emitted preserve the order in which they are sent in the incoming [[HttpRequest]]. - * If you want to sacrivice ordering in favour of (potential) slight performance improvements in reading the input - * you may want to use [[asSourceOfAsyncUnordered]] instead, which lifts the ordering guarantee. - * - * Refer to [[asSourceOf]] for more in depth-documentation and guidelines. - * - * If looking to improve marshalling performance in face of many elements (possibly of different sizes), - * you may be interested in using [[asSourceOfAsyncUnordered]] instead. - * - * See also [[MiscDirectives.withoutSizeLimit]] as you may want to allow streaming infinite streams of data in this route. - * By default the uploaded data is limited by the `akka.http.parsing.max-content-length`. - */ - final def asSourceOfAsync[T](parallelism: Int)(implicit um: Unmarshaller[ByteString, T], framing: FramingWithContentType): RequestToSourceUnmarshaller[T] = - asSourceOfInternal[T](framing, (ec, mat) ⇒ Flow[ByteString].mapAsync(parallelism)(Unmarshal(_).to[T](um, ec, mat))) - - /** - * Similar to [[asSourceOf]] however will apply at most `parallelism` unmarshallers in parallel. - * - * The source elements emitted preserve the order in which they are sent in the incoming [[HttpRequest]]. - * If you want to sacrivice ordering in favour of (potential) slight performance improvements in reading the input - * you may want to use [[asSourceOfAsyncUnordered]] instead, which lifts the ordering guarantee. - * - * Refer to [[asSourceOf]] for more in depth-documentation and guidelines. - * - * See also [[MiscDirectives.withoutSizeLimit]] as you may want to allow streaming infinite streams of data in this route. - * By default the uploaded data is limited by the `akka.http.parsing.max-content-length`. - */ - final def asSourceOfAsync[T](parallelism: Int, framing: FramingWithContentType)(implicit um: Unmarshaller[ByteString, T]): RequestToSourceUnmarshaller[T] = - asSourceOfAsync(parallelism)(um, framing) - - /** - * Similar to [[asSourceOfAsync]], as it will apply at most `parallelism` unmarshallers in parallel. - * - * The source elements emitted preserve the order in which they are sent in the incoming [[HttpRequest]]. - * If you want to sacrivice ordering in favour of (potential) slight performance improvements in reading the input - * you may want to use [[asSourceOfAsyncUnordered]] instead, which lifts the ordering guarantee. - * - * Refer to [[asSourceOf]] for more in depth-documentation and guidelines. - * - * See also [[MiscDirectives.withoutSizeLimit]] as you may want to allow streaming infinite streams of data in this route. - * By default the uploaded data is limited by the `akka.http.parsing.max-content-length`. - */ - final def asSourceOfAsyncUnordered[T](parallelism: Int)(implicit um: Unmarshaller[ByteString, T], framing: FramingWithContentType): RequestToSourceUnmarshaller[T] = - asSourceOfInternal[T](framing, (ec, mat) ⇒ Flow[ByteString].mapAsyncUnordered(parallelism)(Unmarshal(_).to[T](um, ec, mat))) - /** - * Similar to [[asSourceOfAsync]], as it will apply at most `parallelism` unmarshallers in parallel. - * - * The source elements emitted preserve the order in which they are sent in the incoming [[HttpRequest]]. - * If you want to sacrivice ordering in favour of (potential) slight performance improvements in reading the input - * you may want to use [[asSourceOfAsyncUnordered]] instead, which lifts the ordering guarantee. - * - * Refer to [[asSourceOf]] for more in depth-documentation and guidelines. - * - * See also [[MiscDirectives.withoutSizeLimit]] as you may want to allow streaming infinite streams of data in this route. - * By default the uploaded data is limited by the `akka.http.parsing.max-content-length`. - */ - final def asSourceOfAsyncUnordered[T](parallelism: Int, framing: FramingWithContentType)(implicit um: Unmarshaller[ByteString, T]): RequestToSourceUnmarshaller[T] = - asSourceOfAsyncUnordered(parallelism)(um, framing) + final def asSourceOf[T](support: EntityStreamingSupport)(implicit um: FromByteStringUnmarshaller[T]): RequestToSourceUnmarshaller[T] = + asSourceOfInternal(um, support) // format: OFF - private final def asSourceOfInternal[T](framing: FramingWithContentType, marshalling: (ExecutionContext, Materializer) => Flow[ByteString, ByteString, NotUsed]#ReprMat[T, NotUsed]): RequestToSourceUnmarshaller[T] = + private final def asSourceOfInternal[T](um: Unmarshaller[ByteString, T], support: EntityStreamingSupport): RequestToSourceUnmarshaller[T] = Unmarshaller.withMaterializer[HttpRequest, Source[T, NotUsed]] { implicit ec ⇒ implicit mat ⇒ req ⇒ val entity = req.entity - if (framing.matches(entity.contentType)) { + if (support.supported.matches(entity.contentType)) { val bytes = entity.dataBytes - val frames = bytes.via(framing.flow) - val elements = frames.viaMat(marshalling(ec, mat))(Keep.right) + val frames = bytes.via(support.framingDecoder) + val marshalling = + if (support.unordered) Flow[ByteString].mapAsyncUnordered(support.parallelism)(bs => um(bs)(ec, mat)) + else Flow[ByteString].mapAsync(support.parallelism)(bs => um(bs)(ec, mat)) + + val elements = frames.viaMat(marshalling)(Keep.right) FastFuture.successful(elements) - } else FastFuture.failed(Unmarshaller.UnsupportedContentTypeException(framing.supported)) + } else FastFuture.failed(Unmarshaller.UnsupportedContentTypeException(support.supported)) } // format: ON - // TODO note to self - we need the same of ease of streaming stuff for the client side - i.e. the twitter firehose case. - - implicit def _asSourceUnmarshaller[T](implicit fem: FromEntityUnmarshaller[T], framing: FramingWithContentType): FromRequestUnmarshaller[Source[T, NotUsed]] = { - Unmarshaller.withMaterializer[HttpRequest, Source[T, NotUsed]] { implicit ec ⇒ implicit mat ⇒ req ⇒ - val entity = req.entity - if (framing.matches(entity.contentType)) { - val bytes = entity.dataBytes - val frames = bytes.viaMat(framing.flow)(Keep.right) - val elements = frames.viaMat(Flow[ByteString].map(HttpEntity(entity.contentType, _)).mapAsync(1)(Unmarshal(_).to[T](fem, ec, mat)))(Keep.right) - FastFuture.successful(elements) - } else FastFuture.failed(Unmarshaller.UnsupportedContentTypeException(framing.supported)) - } - } - - implicit def _sourceMarshaller[T, M](implicit m: ToEntityMarshaller[T], mode: SourceRenderingMode): ToResponseMarshaller[Source[T, M]] = - Marshaller[Source[T, M], HttpResponse] { implicit ec ⇒ source ⇒ - FastFuture successful { - Marshalling.WithFixedContentType(mode.contentType, () ⇒ { - val bytes = source - .mapAsync(1)(t ⇒ Marshal(t).to[HttpEntity]) - .map(_.dataBytes) - .flatMapConcat(ConstantFun.scalaIdentityFunction) - .intersperse(mode.start, mode.between, mode.end) - HttpResponse(entity = HttpEntity(mode.contentType, bytes)) - }) :: Nil - } - } - - implicit def _sourceParallelismMarshaller[T](implicit m: ToEntityMarshaller[T], mode: SourceRenderingMode): ToResponseMarshaller[AsyncRenderingOf[T]] = - Marshaller[AsyncRenderingOf[T], HttpResponse] { implicit ec ⇒ rendering ⇒ - FastFuture successful { - Marshalling.WithFixedContentType(mode.contentType, () ⇒ { - val bytes = rendering.source - .mapAsync(rendering.parallelism)(t ⇒ Marshal(t).to[HttpEntity]) - .map(_.dataBytes) - .flatMapConcat(ConstantFun.scalaIdentityFunction) - .intersperse(mode.start, mode.between, mode.end) - HttpResponse(entity = HttpEntity(mode.contentType, bytes)) - }) :: Nil - } - } - - implicit def _sourceUnorderedMarshaller[T](implicit m: ToEntityMarshaller[T], mode: SourceRenderingMode): ToResponseMarshaller[AsyncUnorderedRenderingOf[T]] = - Marshaller[AsyncUnorderedRenderingOf[T], HttpResponse] { implicit ec ⇒ rendering ⇒ - FastFuture successful { - Marshalling.WithFixedContentType(mode.contentType, () ⇒ { - val bytes = rendering.source - .mapAsync(rendering.parallelism)(t ⇒ Marshal(t).to[HttpEntity]) - .map(_.dataBytes) - .flatMapConcat(ConstantFun.scalaIdentityFunction) - .intersperse(mode.start, mode.between, mode.end) - HttpResponse(entity = HttpEntity(mode.contentType, bytes)) - }) :: Nil - } - } - - // special rendering modes - - implicit def _enableSpecialSourceRenderingModes[T](source: Source[T, Any]): EnableSpecialSourceRenderingModes[T] = - new EnableSpecialSourceRenderingModes(source) - -} -/** - * Allows the [[MarshallingDirectives.entity]] directive to extract a [[Source]] of elements. - * - * See [[FramedEntityStreamingDirectives]] for detailed documentation. - */ -object FramedEntityStreamingDirectives extends FramedEntityStreamingDirectives { - sealed class AsyncSourceRenderingMode - final class AsyncRenderingOf[T](val source: Source[T, Any], val parallelism: Int) extends AsyncSourceRenderingMode - final class AsyncUnorderedRenderingOf[T](val source: Source[T, Any], val parallelism: Int) extends AsyncSourceRenderingMode - -} - -/** Provides DSL for special rendering modes, e.g. `complete(source.renderAsync)` */ -final class EnableSpecialSourceRenderingModes[T](val source: Source[T, Any]) extends AnyVal { - /** - * Causes the response stream to be marshalled asynchronously (up to `parallelism` elements at once), - * while retaining the ordering of incoming elements. - * - * See also [[Source.mapAsync]]. - */ - def renderAsync(parallelism: Int) = new FramedEntityStreamingDirectives.AsyncRenderingOf(source, parallelism) - /** - * Causes the response stream to be marshalled asynchronously (up to `parallelism` elements at once), - * emitting the first one that finished marshalling onto the wire. - * - * This sacrifices ordering of the incoming data in regards to data actually rendered onto the wire, - * but may be faster if some elements are smaller than other ones by not stalling the small elements - * from being written while the large one still is being marshalled. - * - * See also [[Source.mapAsyncUnordered]]. - */ - def renderAsyncUnordered(parallelism: Int) = new FramedEntityStreamingDirectives.AsyncUnorderedRenderingOf(source, parallelism) } diff --git a/akka-stream-tests/src/test/scala/akka/stream/scaladsl/JsonFramingSpec.scala b/akka-stream-tests/src/test/scala/akka/stream/scaladsl/JsonFramingSpec.scala index c69bfc5820f..6f34dc2fd79 100644 --- a/akka-stream-tests/src/test/scala/akka/stream/scaladsl/JsonFramingSpec.scala +++ b/akka-stream-tests/src/test/scala/akka/stream/scaladsl/JsonFramingSpec.scala @@ -6,11 +6,9 @@ package akka.stream.scaladsl import akka.stream.ActorMaterializer import akka.stream.impl.JsonObjectParser import akka.stream.scaladsl.Framing.FramingException -import akka.stream.scaladsl.{ JsonFraming, Framing, Source } import akka.stream.testkit.scaladsl.TestSink import akka.testkit.AkkaSpec import akka.util.ByteString -import org.scalatest.concurrent.ScalaFutures import scala.collection.immutable.Seq import scala.concurrent.Await @@ -26,21 +24,22 @@ class JsonFramingSpec extends AkkaSpec { """ |[ | { "name" : "john" }, + | { "name" : "Ég get etið gler án þess að meiða mig" }, | { "name" : "jack" }, - | { "name" : "katie" } |] |""".stripMargin // also should complete once notices end of array val result = Source.single(ByteString(input)) - .via(JsonFraming.bracketCounting(Int.MaxValue)) + .via(JsonFraming.objectScanner(Int.MaxValue)) .runFold(Seq.empty[String]) { case (acc, entry) ⇒ acc ++ Seq(entry.utf8String) } result.futureValue shouldBe Seq( """{ "name" : "john" }""", - """{ "name" : "jack" }""", - """{ "name" : "katie" }""") + """{ "name" : "Ég get etið gler án þess að meiða mig" }""", + """{ "name" : "jack" }""" + ) } "emit single json element from string" in { @@ -50,7 +49,7 @@ class JsonFramingSpec extends AkkaSpec { """.stripMargin val result = Source.single(ByteString(input)) - .via(JsonFraming.bracketCounting(Int.MaxValue)) + .via(JsonFraming.objectScanner(Int.MaxValue)) .take(1) .runFold(Seq.empty[String]) { case (acc, entry) ⇒ acc ++ Seq(entry.utf8String) @@ -67,7 +66,7 @@ class JsonFramingSpec extends AkkaSpec { """.stripMargin val result = Source.single(ByteString(input)) - .via(JsonFraming.bracketCounting(Int.MaxValue)) + .via(JsonFraming.objectScanner(Int.MaxValue)) .runFold(Seq.empty[String]) { case (acc, entry) ⇒ acc ++ Seq(entry.utf8String) } @@ -85,7 +84,7 @@ class JsonFramingSpec extends AkkaSpec { """.stripMargin val result = Source.single(ByteString(input)) - .via(JsonFraming.bracketCounting(Int.MaxValue)) + .via(JsonFraming.objectScanner(Int.MaxValue)) .runFold(Seq.empty[String]) { case (acc, entry) ⇒ acc ++ Seq(entry.utf8String) } @@ -109,7 +108,7 @@ class JsonFramingSpec extends AkkaSpec { """"}]"""").map(ByteString(_)) val result = Source.apply(input) - .via(JsonFraming.bracketCounting(Int.MaxValue)) + .via(JsonFraming.objectScanner(Int.MaxValue)) .runFold(Seq.empty[String]) { case (acc, entry) ⇒ acc ++ Seq(entry.utf8String) } @@ -410,7 +409,7 @@ class JsonFramingSpec extends AkkaSpec { """.stripMargin val result = Source.single(ByteString(input)) - .via(JsonFraming.bracketCounting(5)).map(_.utf8String) + .via(JsonFraming.objectScanner(5)).map(_.utf8String) .runFold(Seq.empty[String]) { case (acc, entry) ⇒ acc ++ Seq(entry) } @@ -427,7 +426,7 @@ class JsonFramingSpec extends AkkaSpec { """{ "name": "very very long name somehow. how did this happen?" }""").map(s ⇒ ByteString(s)) val probe = Source(input) - .via(JsonFraming.bracketCounting(48)) + .via(JsonFraming.objectScanner(48)) .runWith(TestSink.probe) probe.ensureSubscription() diff --git a/akka-stream/src/main/scala/akka/stream/javadsl/JsonFraming.scala b/akka-stream/src/main/scala/akka/stream/javadsl/JsonFraming.scala index 3fb3c28638e..4bad96f7906 100644 --- a/akka-stream/src/main/scala/akka/stream/javadsl/JsonFraming.scala +++ b/akka-stream/src/main/scala/akka/stream/javadsl/JsonFraming.scala @@ -34,7 +34,7 @@ object JsonFraming { * @param maximumObjectLength The maximum length of allowed frames while decoding. If the maximum length is exceeded * this Flow will fail the stream. */ - def bracketCounting(maximumObjectLength: Int): Flow[ByteString, ByteString, NotUsed] = - akka.stream.scaladsl.JsonFraming.bracketCounting(maximumObjectLength).asJava + def objectScanner(maximumObjectLength: Int): Flow[ByteString, ByteString, NotUsed] = + akka.stream.scaladsl.JsonFraming.objectScanner(maximumObjectLength).asJava } diff --git a/akka-stream/src/main/scala/akka/stream/scaladsl/JsonFraming.scala b/akka-stream/src/main/scala/akka/stream/scaladsl/JsonFraming.scala index bc5f69d0372..79e35909ccd 100644 --- a/akka-stream/src/main/scala/akka/stream/scaladsl/JsonFraming.scala +++ b/akka-stream/src/main/scala/akka/stream/scaladsl/JsonFraming.scala @@ -17,6 +17,7 @@ object JsonFraming { /** * Returns a Flow that implements a "brace counting" based framing stage for emitting valid JSON chunks. + * It scans the incoming data stream for valid JSON objects and returns chunks of ByteStrings containing only those valid chunks. * * Typical examples of data that one may want to frame using this stage include: * @@ -40,7 +41,7 @@ object JsonFraming { * @param maximumObjectLength The maximum length of allowed frames while decoding. If the maximum length is exceeded * this Flow will fail the stream. */ - def bracketCounting(maximumObjectLength: Int): Flow[ByteString, ByteString, NotUsed] = + def objectScanner(maximumObjectLength: Int): Flow[ByteString, ByteString, NotUsed] = Flow[ByteString].via(new SimpleLinearGraphStage[ByteString] { private[this] val buffer = new JsonObjectParser(maximumObjectLength) @@ -67,6 +68,6 @@ object JsonFraming { } } } - }).named("jsonFraming(BracketCounting)") + }).named("JsonFraming.objectScanner") } From 3c2d021742ee7f642640d105f2c8601babf2f15f Mon Sep 17 00:00:00 2001 From: Konrad Malawski Date: Tue, 2 Aug 2016 13:10:33 +0200 Subject: [PATCH 09/10] =htc improve patience on FramingSpec because Jenkins --- .../test/scala/akka/stream/scaladsl/JsonFramingSpec.scala | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/akka-stream-tests/src/test/scala/akka/stream/scaladsl/JsonFramingSpec.scala b/akka-stream-tests/src/test/scala/akka/stream/scaladsl/JsonFramingSpec.scala index 6f34dc2fd79..37b2a5aad81 100644 --- a/akka-stream-tests/src/test/scala/akka/stream/scaladsl/JsonFramingSpec.scala +++ b/akka-stream-tests/src/test/scala/akka/stream/scaladsl/JsonFramingSpec.scala @@ -16,6 +16,8 @@ import scala.concurrent.duration._ class JsonFramingSpec extends AkkaSpec { + override implicit val patience = PatienceConfig(timeout = 10.seconds) + implicit val mat = ActorMaterializer() "collecting multiple json" should { @@ -79,9 +81,7 @@ class JsonFramingSpec extends AkkaSpec { "parse comma delimited" in { val input = - """ - | { "name": "john" }, { "name": "jack" }, { "name": "katie" } - """.stripMargin + """ { "name": "john" }, { "name": "jack" }, { "name": "katie" } """ val result = Source.single(ByteString(input)) .via(JsonFraming.objectScanner(Int.MaxValue)) From 8a1f8f27dca7cfe1cb50cba0c141ac16a50ecd09 Mon Sep 17 00:00:00 2001 From: Konrad Malawski Date: Tue, 2 Aug 2016 15:56:39 +0200 Subject: [PATCH 10/10] =htc fix JsonFraming completion bug, exposed by new Interpreter --- .../scala/akka/stream/JsonFramingBenchmark.scala | 3 ++- .../directives/FramedEntityStreamingDirectives.scala | 12 ++---------- .../scala/akka/stream/scaladsl/JsonFramingSpec.scala | 2 -- .../scala/akka/stream/scaladsl/JsonFraming.scala | 8 ++++++-- 4 files changed, 10 insertions(+), 15 deletions(-) diff --git a/akka-bench-jmh/src/main/scala/akka/stream/JsonFramingBenchmark.scala b/akka-bench-jmh/src/main/scala/akka/stream/JsonFramingBenchmark.scala index cf5d7794152..8e7cfe884e8 100644 --- a/akka-bench-jmh/src/main/scala/akka/stream/JsonFramingBenchmark.scala +++ b/akka-bench-jmh/src/main/scala/akka/stream/JsonFramingBenchmark.scala @@ -33,7 +33,8 @@ class JsonFramingBenchmark { |{"fname":"Bob","name":"Smith","age":42,"id":1337,"boardMember":false}, |{"fname":"Bob","name":"Smith","age":42,"id":1337,"boardMember":false}, |{"fname":"Bob","name":"Smith","age":42,"id":1337,"boardMember":false}, - |{"fname":"Hank","name":"Smith","age":42,"id":1337,"boardMember":false}""".stripMargin) + |{"fname":"Hank","name":"Smith","age":42,"id":1337,"boardMember":false}""".stripMargin + ) val bracket = new JsonObjectParser diff --git a/akka-http/src/main/scala/akka/http/scaladsl/server/directives/FramedEntityStreamingDirectives.scala b/akka-http/src/main/scala/akka/http/scaladsl/server/directives/FramedEntityStreamingDirectives.scala index fde94d9e4d8..dc4a5ab0015 100644 --- a/akka-http/src/main/scala/akka/http/scaladsl/server/directives/FramedEntityStreamingDirectives.scala +++ b/akka-http/src/main/scala/akka/http/scaladsl/server/directives/FramedEntityStreamingDirectives.scala @@ -47,22 +47,14 @@ trait FramedEntityStreamingDirectives extends MarshallingDirectives { * Extracts entity as [[Source]] of elements of type `T`. * This is achieved by applying the implicitly provided (in the following order): * - * - 1st: [[FramingFlow]] in order to chunk-up the incoming [[ByteString]]s according to the - * `Content-Type` aware framing (for example, [[common.EntityStreamingSupport.bracketCountingJsonFraming]]). - * - 2nd: [[Unmarshaller]] (from [[ByteString]] to `T`) for each of the respective "chunks" (e.g. for each JSON element contained within an array). + * - 1st: chunk-up the incoming [[ByteString]]s by applying the `Content-Type`-aware framing + * - 2nd: apply the [[Unmarshaller]] (from [[ByteString]] to `T`) for each of the respective "chunks" (e.g. for each JSON element contained within an array). * * The request will be rejected with an [[akka.http.scaladsl.server.UnsupportedRequestContentTypeRejection]] if * its [[ContentType]] is not supported by the used `framing` or `unmarshaller`. * - * It is recommended to use the [[common.EntityStreamingSupport]] trait in conjunction with this - * directive as it helps provide the right [[FramingFlow]] and [[SourceRenderingMode]] for the most - * typical usage scenarios (JSON, CSV, ...). - * * Cancelling extracted [[Source]] closes the connection abruptly (same as cancelling the `entity.dataBytes`). * - * If looking to improve marshalling performance in face of many elements (possibly of different sizes), - * you may be interested in using [[asSourceOfAsyncUnordered]] instead. - * * See also [[MiscDirectives.withoutSizeLimit]] as you may want to allow streaming infinite streams of data in this route. * By default the uploaded data is limited by the `akka.http.parsing.max-content-length`. */ diff --git a/akka-stream-tests/src/test/scala/akka/stream/scaladsl/JsonFramingSpec.scala b/akka-stream-tests/src/test/scala/akka/stream/scaladsl/JsonFramingSpec.scala index 37b2a5aad81..57bfa79349b 100644 --- a/akka-stream-tests/src/test/scala/akka/stream/scaladsl/JsonFramingSpec.scala +++ b/akka-stream-tests/src/test/scala/akka/stream/scaladsl/JsonFramingSpec.scala @@ -16,8 +16,6 @@ import scala.concurrent.duration._ class JsonFramingSpec extends AkkaSpec { - override implicit val patience = PatienceConfig(timeout = 10.seconds) - implicit val mat = ActorMaterializer() "collecting multiple json" should { diff --git a/akka-stream/src/main/scala/akka/stream/scaladsl/JsonFraming.scala b/akka-stream/src/main/scala/akka/stream/scaladsl/JsonFraming.scala index 79e35909ccd..f7c7aeb50e9 100644 --- a/akka-stream/src/main/scala/akka/stream/scaladsl/JsonFraming.scala +++ b/akka-stream/src/main/scala/akka/stream/scaladsl/JsonFraming.scala @@ -56,8 +56,12 @@ object JsonFraming { override def onPull(): Unit = tryPopBuffer() - override def onUpstreamFinish(): Unit = - if (buffer.isEmpty) completeStage() + override def onUpstreamFinish(): Unit = { + try buffer.poll() match { + case Some(json) ⇒ emit(out, json, () ⇒ completeStage()) + case _ ⇒ completeStage() + } + } def tryPopBuffer() = { try buffer.poll() match {