From 9349f711e8d92765ae4914b3797b85ba2a8bfc1d Mon Sep 17 00:00:00 2001 From: Leonid Startsev Date: Tue, 11 Apr 2023 20:45:01 +0200 Subject: [PATCH 1/2] Fix incorrect json decoding iterator's .hasNext() behavior on array-wrapped inputs: hasNext() shouldn't throw exception if it was called more than once after the stream has ended. Fixes #2267 --- .../serialization/features/JsonLazySequenceTest.kt | 8 ++++++++ .../commonMain/src/kotlinx/serialization/json/Json.kt | 2 +- .../kotlinx/serialization/json/internal/JsonIterator.kt | 6 ++++-- .../serialization/json/internal/lexer/JsonLexer.kt | 3 +++ 4 files changed, 16 insertions(+), 3 deletions(-) diff --git a/formats/json-tests/jvmTest/src/kotlinx/serialization/features/JsonLazySequenceTest.kt b/formats/json-tests/jvmTest/src/kotlinx/serialization/features/JsonLazySequenceTest.kt index f6d13f702..23887660d 100644 --- a/formats/json-tests/jvmTest/src/kotlinx/serialization/features/JsonLazySequenceTest.kt +++ b/formats/json-tests/jvmTest/src/kotlinx/serialization/features/JsonLazySequenceTest.kt @@ -83,6 +83,7 @@ class JsonLazySequenceTest { iter.assertNext(StringData("b")) iter.assertNext(StringData("c")) assertFalse(iter.hasNext()) + assertFalse(iter.hasNext()) // Subsequent calls to .hasNext() should not throw EOF or anything assertFailsWithMessage("EOF") { iter.next() } @@ -186,4 +187,11 @@ class JsonLazySequenceTest { assertEquals(inputList, json.decodeToSequence(paddedWs.asInputStream(), StringData.serializer(), DecodeSequenceMode.ARRAY_WRAPPED).toList()) } + @Test + fun testToIteratorAndBack() = withInputs { ins -> + val iterator = Json.decodeToSequence(ins, StringData.serializer()).iterator() + val values = iterator.asSequence().take(3).toList() + assertEquals(inputList, values) + assertFalse(iterator.hasNext()) + } } diff --git a/formats/json/commonMain/src/kotlinx/serialization/json/Json.kt b/formats/json/commonMain/src/kotlinx/serialization/json/Json.kt index d73bb39d1..6f5ba9fb2 100644 --- a/formats/json/commonMain/src/kotlinx/serialization/json/Json.kt +++ b/formats/json/commonMain/src/kotlinx/serialization/json/Json.kt @@ -169,7 +169,7 @@ public enum class DecodeSequenceMode { /** * Declares that parser itself should select between [WHITESPACE_SEPARATED] and [ARRAY_WRAPPED] modes. - * The selection is performed by looking on the first meaningful character of the stream. + * The selection is performed by looking at the first meaningful character of the stream. * * In most cases, auto-detection is sufficient to correctly parse an input. * If the input is _whitespace-separated stream of the arrays_, parser could select an incorrect mode, diff --git a/formats/json/commonMain/src/kotlinx/serialization/json/internal/JsonIterator.kt b/formats/json/commonMain/src/kotlinx/serialization/json/internal/JsonIterator.kt index 3929c840a..338142c09 100644 --- a/formats/json/commonMain/src/kotlinx/serialization/json/internal/JsonIterator.kt +++ b/formats/json/commonMain/src/kotlinx/serialization/json/internal/JsonIterator.kt @@ -68,6 +68,7 @@ private class JsonIteratorArrayWrapped( private val deserializer: DeserializationStrategy ) : Iterator { private var first = true + private var sawClosingBrace = false override fun next(): T { if (first) { @@ -84,6 +85,7 @@ private class JsonIteratorArrayWrapped( */ override fun hasNext(): Boolean { if (lexer.peekNextToken() == TC_END_LIST) { + sawClosingBrace = true lexer.consumeNextToken(TC_END_LIST) if (lexer.isNotEof()) { if (lexer.peekNextToken() == TC_BEGIN_LIST) lexer.fail("There is a start of the new array after the one parsed to sequence. " + @@ -93,7 +95,7 @@ private class JsonIteratorArrayWrapped( } return false } - if (!lexer.isNotEof()) lexer.fail(TC_END_LIST) - return true + if (!lexer.isNotEof() && !sawClosingBrace) lexer.fail(TC_END_LIST) + return !sawClosingBrace } } diff --git a/formats/json/commonMain/src/kotlinx/serialization/json/internal/lexer/JsonLexer.kt b/formats/json/commonMain/src/kotlinx/serialization/json/internal/lexer/JsonLexer.kt index fa4772c7d..2881fd79b 100644 --- a/formats/json/commonMain/src/kotlinx/serialization/json/internal/lexer/JsonLexer.kt +++ b/formats/json/commonMain/src/kotlinx/serialization/json/internal/lexer/JsonLexer.kt @@ -30,6 +30,9 @@ internal class ArrayAsSequence(private val buffer: CharArray) : CharSequence { fun trim(newSize: Int) { length = minOf(buffer.size, newSize) } + + // source.toString() is used in JsonDecodingException + override fun toString(): String = substring(0, length) } internal class ReaderJsonLexer( From a68914484af2452e8b433389d5b6b371834bcd8f Mon Sep 17 00:00:00 2001 From: Leonid Startsev Date: Wed, 19 Apr 2023 17:29:55 +0200 Subject: [PATCH 2/2] fixup! Fix incorrect json decoding iterator's .hasNext() behavior on array-wrapped inputs: hasNext() shouldn't throw exception if it was called more than once after the stream has ended. --- .../kotlinx/serialization/json/internal/JsonIterator.kt | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/formats/json/commonMain/src/kotlinx/serialization/json/internal/JsonIterator.kt b/formats/json/commonMain/src/kotlinx/serialization/json/internal/JsonIterator.kt index 338142c09..00f36b2b0 100644 --- a/formats/json/commonMain/src/kotlinx/serialization/json/internal/JsonIterator.kt +++ b/formats/json/commonMain/src/kotlinx/serialization/json/internal/JsonIterator.kt @@ -68,7 +68,7 @@ private class JsonIteratorArrayWrapped( private val deserializer: DeserializationStrategy ) : Iterator { private var first = true - private var sawClosingBrace = false + private var finished = false override fun next(): T { if (first) { @@ -84,8 +84,9 @@ private class JsonIteratorArrayWrapped( * Note: if array separator (comma) is missing, hasNext() returns true, but next() throws an exception. */ override fun hasNext(): Boolean { + if (finished) return false if (lexer.peekNextToken() == TC_END_LIST) { - sawClosingBrace = true + finished = true lexer.consumeNextToken(TC_END_LIST) if (lexer.isNotEof()) { if (lexer.peekNextToken() == TC_BEGIN_LIST) lexer.fail("There is a start of the new array after the one parsed to sequence. " + @@ -95,7 +96,7 @@ private class JsonIteratorArrayWrapped( } return false } - if (!lexer.isNotEof() && !sawClosingBrace) lexer.fail(TC_END_LIST) - return !sawClosingBrace + if (!lexer.isNotEof() && !finished) lexer.fail(TC_END_LIST) + return true } }