Skip to content

Commit

Permalink
updated json iterators
Browse files Browse the repository at this point in the history
  • Loading branch information
agolovenko committed Oct 25, 2022
1 parent d47cfe4 commit 25d7fe0
Show file tree
Hide file tree
Showing 7 changed files with 94 additions and 67 deletions.
6 changes: 3 additions & 3 deletions json/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -36,10 +36,10 @@ and `2.13`
* `timestamp-millis`
* `timestamp-micros`

### [JsonArrayIterator](src/main/scala/io/github/agolovenko/avro/json/JsonArrayIterator.scala)
* A utility class that implements `Iterator[JsObject]` and parses array of objects in stream-like fashion
### [JsonIterator](src/main/scala/io/github/agolovenko/avro/json/JsonIterator.scala)
* A utility class that implements `Iterator[JsObject]` and parses an array of objects or a single object in stream-like fashion

### [JsonLIterator](src/main/scala/io/github/agolovenko/avro/json/JsonLIterator.scala)
### [JsonLinesIterator](src/main/scala/io/github/agolovenko/avro/json/JsonLinesIterator.scala)
* A utility class that implements `Iterator[JsObject]` and parses `JsonL` format in stream-like fashion

## Usage
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
package io.github.agolovenko.avro.json

import scala.util.control.NoStackTrace

class JsonException(message: String, cause: Throwable = null) extends RuntimeException(message, cause) with NoStackTrace
Original file line number Diff line number Diff line change
@@ -1,34 +1,34 @@
package io.github.agolovenko.avro.json

import com.fasterxml.jackson.core.{JsonParser => JJsonParser, JsonToken}
import com.fasterxml.jackson.core.{JsonToken, JsonParser => JJsonParser}
import com.fasterxml.jackson.databind.ObjectMapper
import com.fasterxml.jackson.databind.node.ObjectNode
import play.api.libs.json.jackson.PlayJsonModule
import play.api.libs.json.{JsObject, JsonParserSettings}

import java.io.Reader

class JsonArrayIterator(reader: => Reader) extends Iterator[JsObject] {
class JsonIterator(reader: => Reader) extends Iterator[JsObject] {
private val mapper = new ObjectMapper()
.registerModule(new PlayJsonModule(JsonParserSettings()))
.enable(JJsonParser.Feature.AUTO_CLOSE_SOURCE)

private val parser = mapper.createParser(reader)
private val jsObjectReader = mapper.readerFor(classOf[JsObject])

private var nextToken = {
val next = parser.nextToken
if (next != JsonToken.START_ARRAY) throw new IllegalStateException(s"Expected start of array, got instead: $next")

parser.nextToken
private var nextToken = parser.nextToken match {
case JsonToken.START_ARRAY => parser.nextToken
case JsonToken.START_OBJECT => JsonToken.START_OBJECT
case next => throw new JsonException(s"Expected start of an array or an object, got instead: $next")
}

override def hasNext: Boolean =
if (nextToken == JsonToken.START_OBJECT) true
else if (nextToken == JsonToken.END_ARRAY) {
override def hasNext: Boolean = nextToken match {
case JsonToken.START_OBJECT => true
case JsonToken.END_ARRAY | null =>
parser.close()
false
} else throw new IllegalStateException(s"Expected end of array, got instead: $nextToken")
case next => throw new JsonException(s"Expected end of an array or an object, got instead: $next")
}

override def next(): JsObject = {
val objectNode = mapper.readTree[ObjectNode](parser)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@ import play.api.libs.json.{JsObject, Json}

import java.io.{BufferedReader, Reader}

class JsonLIterator(reader: => Reader) extends Iterator[JsObject] {
class JsonLinesIterator(reader: => Reader) extends Iterator[JsObject] {
private val bufferedReader = reader match {
case buffered: BufferedReader => buffered
case unBuffered => new BufferedReader(unBuffered)
Expand Down

This file was deleted.

Original file line number Diff line number Diff line change
@@ -0,0 +1,68 @@
package io.github.agolovenko.avro.json

import org.scalatest.matchers.should.Matchers
import org.scalatest.wordspec.AnyWordSpec

import java.io.StringReader

class JsonIteratorSpec extends AnyWordSpec with Matchers {
"parses json array" in {
val json =
"""
|[
| {
| "lastUpdated": "2020-04-17T19:38:47.04+00:00",
| "id": "16289",
| "clientId": 123,
| "description": {
| "type": "normal",
| "text": "good one"
| }
| },
| {
| "lastUpdated": "2020-04-17T19:38:40.04+00:00",
| "id": "16288",
| "clientId": 124,
| "description": {
| "type": "normal",
| "text": "very good one"
| }
| }
|]
|""".stripMargin

val it = new JsonIterator(new StringReader(json))
val jsObjects = it.toList

jsObjects should have size 2
(jsObjects(1) \ "clientId").as[Int] shouldBe 124
}

"parses json object" in {
val json =
"""
| {
| "lastUpdated": "2020-04-17T19:38:47.04+00:00",
| "id": "16289",
| "clientId": 123,
| "description": {
| "type": "normal",
| "text": "good one"
| }
| }
|""".stripMargin

val it = new JsonIterator(new StringReader(json))
val jsObjects = it.toList

jsObjects should have size 1
(jsObjects.head \ "clientId").as[Int] shouldBe 123
}

"expects array of objects as input" in {
an[JsonException] should be thrownBy new JsonIterator(new StringReader(" [{}, 1234]")).foreach(_ => ())

noException should be thrownBy new JsonIterator(new StringReader(" {}")).foreach(_ => ())
noException should be thrownBy new JsonIterator(new StringReader(""" [{}, {"number": 1234} ]""")).foreach(_ => ())
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@ import org.scalatest.wordspec.AnyWordSpec

import java.io.StringReader

class JsonLIteratorSpec extends AnyWordSpec with Matchers {
class JsonLinesIteratorSpec extends AnyWordSpec with Matchers {
"parses jsonL" in {
val json =
"""
Expand All @@ -16,19 +16,19 @@ class JsonLIteratorSpec extends AnyWordSpec with Matchers {
|
|""".stripMargin

val it = new JsonLIterator(new StringReader(json))
val jsObjects = it.toList
val it = new JsonLinesIterator(new StringReader(json))
val jsObjects = it.toVector

jsObjects should have size 2
(jsObjects(1) \ "clientId").as[Int] shouldBe 124
}

"accepts empty input" in {
noException should be thrownBy new JsonLIterator(new StringReader(" ")).foreach(_ => ())
noException should be thrownBy new JsonLinesIterator(new StringReader(" ")).foreach(_ => ())
}

"expects jsonL as input" in {
an[Exception] should be thrownBy new JsonLIterator(new StringReader("[{}]")).foreach(_ => ())
noException should be thrownBy new JsonLIterator(new StringReader("{}\n\n{}")).foreach(_ => ())
an[Exception] should be thrownBy new JsonLinesIterator(new StringReader("[{}]")).foreach(_ => ())
noException should be thrownBy new JsonLinesIterator(new StringReader("{}\n\n{}")).foreach(_ => ())
}
}

0 comments on commit 25d7fe0

Please sign in to comment.