Add JSON::ResumableParser#partial_value? and #empty?

ext/json/ext/parser/parser.c

@@ -2737,6 +2737,41 @@ static VALUE cResumableParser_eos_p(VALUE self)
     return eos(&parser->state) ? Qtrue : Qfalse;
 }
 
+/*
+ * call-seq: partial_value? -> true or false
+ *
+ * Returns whether a document is currently under construction: an unclosed
+ * container, a key awaiting its value, etc.
+ *
+ * It answers the same question as <tt>!partial_value.nil?</tt>, but as a
+ * cheap predicate on the parser's internal state, without materializing the
+ * partially parsed Ruby objects:
+ *   parser << '{"a":1,'
+ *   parser.parse # => false
+ *   parser.partial_value? # => true
+ *
+ * A fully parsed document whose value hasn't been retrieved yet is not under
+ * construction: #value? returns true and #partial_value? returns false.
+ */
+static VALUE cResumableParser_partial_value_p(VALUE self)
+{
+    JSON_ResumableParser *parser = cResumableParser_get(self);
+
+    // Mirror of #value?: values on the stack while the document isn't DONE
+    // belong to a partially built document. A container whose first key or
+    // element hasn't been parsed yet has no frame nor value registered (the
+    // tokenizer rewinds to the container start on EOS), so that state is
+    // observable through the buffer (#eos?/#rest) instead, keeping this
+    // predicate consistent with #partial_value returning nil.
+    if (parser->value_stack.head > 0) {
+        json_frame *frame = json_frame_stack_peek(&parser->frames);
+        if (frame->phase != JSON_PHASE_DONE) {
+            return Qtrue;
+        }
+    }
+    return Qfalse;
+}
+
 /*
  * call-seq: parsed_bytes -> integer
  *
@@ -2793,6 +2828,7 @@ void Init_parser(void)
     rb_define_method(cResumableParser, "value", cResumableParser_value, 0);
     rb_define_method(cResumableParser, "value?", cResumableParser_value_p, 0);
     rb_define_method(cResumableParser, "partial_value", cResumableParser_partial_value, 0);
+    rb_define_method(cResumableParser, "partial_value?", cResumableParser_partial_value_p, 0);
     rb_define_method(cResumableParser, "clear", cResumableParser_clear, 0);
     rb_define_method(cResumableParser, "rest", cResumableParser_rest, 0);
     rb_define_method(cResumableParser, "eos?", cResumableParser_eos_p, 0);

lib/json/ext.rb

@@ -41,5 +41,31 @@ def parse
     end
   end
 
+  if defined?(ResumableParser) # Not yet available on JRuby
+    class ResumableParser
+      # Returns whether the parser is entirely done: no unconsumed bytes in
+      # the buffer, no document under construction and no parsed value
+      # awaiting retrieval.
+      #
+      # The main use case is detecting a truncated stream once the input is
+      # exhausted:
+      #
+      #   loop do
+      #     begin
+      #       parser << socket.readpartial(4096)
+      #     rescue EOFError
+      #       break
+      #     end
+      #     while parser.parse
+      #       process(parser.value)
+      #     end
+      #   end
+      #   warn "stream was truncated" unless parser.empty?
+      def empty?
+        eos? && !partial_value? && !value?
+      end
+    end
+  end
+
   JSON_LOADED = true unless defined?(JSON::JSON_LOADED)
 end

test/json/resumable_parser_test.rb

@@ -239,6 +239,89 @@ def test_eos
     assert_predicate @parser, :eos?
   end
 
+  def test_empty_predicate
+    # empty? is defined on the state left after parsing everything that
+    # could be parsed from the fed bytes, so drain with parse/value first.
+    {
+      ''               => true,  # nothing fed: vacuously empty
+      '{"a":1}'        => true,
+      '{"a":1}{"b":2}' => true,
+      '{"a":1} '       => true,  # trailing whitespace
+      '{"a":1}{"b":2'  => false, # inside a number token
+      '{"a":1}{"b":'   => false, # right after a colon (token boundary)
+      '{"a":1}{'       => false, # right after an object open
+      '{"a":1,'        => false, # right after a comma (token boundary)
+      '"abc'           => false, # inside a string token
+      '[1,2'           => false, # unclosed array
+    }.each do |json, expected|
+      parser = new_parser
+      parser << json
+      parser.value while parser.parse
+      assert_equal expected, parser.empty?, "expected #{json.inspect} to be empty? == #{expected}"
+    end
+  end
+
+  def test_empty_predicate_with_undrained_buffer
+    @parser << '{"a":1}{"b":2}'
+    assert @parser.parse
+    refute_predicate @parser, :empty? # second document still in the buffer
+    assert_equal({ "a" => 1 }, @parser.value)
+    assert @parser.parse
+    assert_equal({ "b" => 2 }, @parser.value)
+    assert_predicate @parser, :empty?
+  end
+
+  def test_empty_predicate_with_pending_value
+    # A fully parsed document awaiting retrieval with #value is not empty.
+    @parser << '{"a":1}'
+    assert @parser.parse
+    refute_predicate @parser, :empty?
+    assert_equal({ "a" => 1 }, @parser.value)
+    assert_predicate @parser, :empty?
+  end
+
+  def test_empty_predicate_across_feeds
+    @parser << '{"a' # chunk boundary inside a string literal
+    refute @parser.parse
+    refute_predicate @parser, :empty?
+
+    @parser << '":1'
+    refute @parser.parse
+    refute_predicate @parser, :empty?
+
+    @parser << '}'
+    assert @parser.parse
+    refute_predicate @parser, :empty? # value not retrieved yet
+    assert_equal({ "a" => 1 }, @parser.value)
+    assert_predicate @parser, :empty?
+  end
+
+  def test_partial_value_predicate
+    {
+      ''               => false,
+      '{"a":1}'        => false,
+      '{"a":1}{"b":2}' => false,
+      '{"a":1} '       => false,
+      '{"a":1}{"b":2'  => true,  # inside a number token
+      '{"a":1}{"b":'   => true,  # right after a colon (token boundary)
+      # The tokenizer rewinds to the token start on EOS, so nothing is
+      # registered yet for a lone '{' or an unterminated top-level string:
+      # partial_value returns nil and partial_value? agrees. The truncation
+      # is still observable through the buffer: eos? is false, rest isn't
+      # empty.
+      '{"a":1}{'       => false, # right after an object open
+      '"abc'           => false, # inside a string token
+      '{"a":1,'        => true,  # right after a comma (token boundary)
+      '[1,2'           => true,  # unclosed array
+    }.each do |json, expected|
+      parser = new_parser
+      parser << json
+      parser.value while parser.parse
+      assert_equal expected, parser.partial_value?, "expected #{json.inspect} to be partial_value? == #{expected}"
+      assert_equal !parser.partial_value.nil?, parser.partial_value?, "partial_value?/partial_value mismatch for #{json.inspect}"
+    end
+  end
+
   def test_partial_value
     assert_nil @parser.partial_value
     assert_partial_value [1, 2, 3], '[1, 2, 3, "unterminated string'