diff --git a/ext/json/ext/parser/parser.c b/ext/json/ext/parser/parser.c index a60f6a3c..b1cab6f0 100644 --- a/ext/json/ext/parser/parser.c +++ b/ext/json/ext/parser/parser.c @@ -2737,6 +2737,41 @@ static VALUE cResumableParser_eos_p(VALUE self) return eos(&parser->state) ? Qtrue : Qfalse; } +/* + * call-seq: partial_value? -> true or false + * + * Returns whether a document is currently under construction: an unclosed + * container, a key awaiting its value, etc. + * + * It answers the same question as !partial_value.nil?, but as a + * cheap predicate on the parser's internal state, without materializing the + * partially parsed Ruby objects: + * parser << '{"a":1,' + * parser.parse # => false + * parser.partial_value? # => true + * + * A fully parsed document whose value hasn't been retrieved yet is not under + * construction: #value? returns true and #partial_value? returns false. + */ +static VALUE cResumableParser_partial_value_p(VALUE self) +{ + JSON_ResumableParser *parser = cResumableParser_get(self); + + // Mirror of #value?: values on the stack while the document isn't DONE + // belong to a partially built document. A container whose first key or + // element hasn't been parsed yet has no frame nor value registered (the + // tokenizer rewinds to the container start on EOS), so that state is + // observable through the buffer (#eos?/#rest) instead, keeping this + // predicate consistent with #partial_value returning nil. + if (parser->value_stack.head > 0) { + json_frame *frame = json_frame_stack_peek(&parser->frames); + if (frame->phase != JSON_PHASE_DONE) { + return Qtrue; + } + } + return Qfalse; +} + /* * call-seq: parsed_bytes -> integer * @@ -2793,6 +2828,7 @@ void Init_parser(void) rb_define_method(cResumableParser, "value", cResumableParser_value, 0); rb_define_method(cResumableParser, "value?", cResumableParser_value_p, 0); rb_define_method(cResumableParser, "partial_value", cResumableParser_partial_value, 0); + rb_define_method(cResumableParser, "partial_value?", cResumableParser_partial_value_p, 0); rb_define_method(cResumableParser, "clear", cResumableParser_clear, 0); rb_define_method(cResumableParser, "rest", cResumableParser_rest, 0); rb_define_method(cResumableParser, "eos?", cResumableParser_eos_p, 0); diff --git a/lib/json/ext.rb b/lib/json/ext.rb index 5bacc5e3..28500f14 100644 --- a/lib/json/ext.rb +++ b/lib/json/ext.rb @@ -41,5 +41,31 @@ def parse end end + if defined?(ResumableParser) # Not yet available on JRuby + class ResumableParser + # Returns whether the parser is entirely done: no unconsumed bytes in + # the buffer, no document under construction and no parsed value + # awaiting retrieval. + # + # The main use case is detecting a truncated stream once the input is + # exhausted: + # + # loop do + # begin + # parser << socket.readpartial(4096) + # rescue EOFError + # break + # end + # while parser.parse + # process(parser.value) + # end + # end + # warn "stream was truncated" unless parser.empty? + def empty? + eos? && !partial_value? && !value? + end + end + end + JSON_LOADED = true unless defined?(JSON::JSON_LOADED) end diff --git a/test/json/resumable_parser_test.rb b/test/json/resumable_parser_test.rb index 734d6e22..d634e277 100644 --- a/test/json/resumable_parser_test.rb +++ b/test/json/resumable_parser_test.rb @@ -239,6 +239,89 @@ def test_eos assert_predicate @parser, :eos? end + def test_empty_predicate + # empty? is defined on the state left after parsing everything that + # could be parsed from the fed bytes, so drain with parse/value first. + { + '' => true, # nothing fed: vacuously empty + '{"a":1}' => true, + '{"a":1}{"b":2}' => true, + '{"a":1} ' => true, # trailing whitespace + '{"a":1}{"b":2' => false, # inside a number token + '{"a":1}{"b":' => false, # right after a colon (token boundary) + '{"a":1}{' => false, # right after an object open + '{"a":1,' => false, # right after a comma (token boundary) + '"abc' => false, # inside a string token + '[1,2' => false, # unclosed array + }.each do |json, expected| + parser = new_parser + parser << json + parser.value while parser.parse + assert_equal expected, parser.empty?, "expected #{json.inspect} to be empty? == #{expected}" + end + end + + def test_empty_predicate_with_undrained_buffer + @parser << '{"a":1}{"b":2}' + assert @parser.parse + refute_predicate @parser, :empty? # second document still in the buffer + assert_equal({ "a" => 1 }, @parser.value) + assert @parser.parse + assert_equal({ "b" => 2 }, @parser.value) + assert_predicate @parser, :empty? + end + + def test_empty_predicate_with_pending_value + # A fully parsed document awaiting retrieval with #value is not empty. + @parser << '{"a":1}' + assert @parser.parse + refute_predicate @parser, :empty? + assert_equal({ "a" => 1 }, @parser.value) + assert_predicate @parser, :empty? + end + + def test_empty_predicate_across_feeds + @parser << '{"a' # chunk boundary inside a string literal + refute @parser.parse + refute_predicate @parser, :empty? + + @parser << '":1' + refute @parser.parse + refute_predicate @parser, :empty? + + @parser << '}' + assert @parser.parse + refute_predicate @parser, :empty? # value not retrieved yet + assert_equal({ "a" => 1 }, @parser.value) + assert_predicate @parser, :empty? + end + + def test_partial_value_predicate + { + '' => false, + '{"a":1}' => false, + '{"a":1}{"b":2}' => false, + '{"a":1} ' => false, + '{"a":1}{"b":2' => true, # inside a number token + '{"a":1}{"b":' => true, # right after a colon (token boundary) + # The tokenizer rewinds to the token start on EOS, so nothing is + # registered yet for a lone '{' or an unterminated top-level string: + # partial_value returns nil and partial_value? agrees. The truncation + # is still observable through the buffer: eos? is false, rest isn't + # empty. + '{"a":1}{' => false, # right after an object open + '"abc' => false, # inside a string token + '{"a":1,' => true, # right after a comma (token boundary) + '[1,2' => true, # unclosed array + }.each do |json, expected| + parser = new_parser + parser << json + parser.value while parser.parse + assert_equal expected, parser.partial_value?, "expected #{json.inspect} to be partial_value? == #{expected}" + assert_equal !parser.partial_value.nil?, parser.partial_value?, "partial_value?/partial_value mismatch for #{json.inspect}" + end + end + def test_partial_value assert_nil @parser.partial_value assert_partial_value [1, 2, 3], '[1, 2, 3, "unterminated string'