Refinement of upb_bytesrc interface.

Added a upb_byteregion that tracks a region of
the input buffer; decoders use this instead of
using a upb_bytesrc directly.  upb_byteregion
is also used as the way of passing a string to
a upb_handlers callback.  This symmetry makes
decoders compose better; if you want to take
a parsed string and decode it as something else,
you can take the string directly from the callback
and feed it as input to another parser.

A commented-out version of a pinning interface
is present; I decline to actually implement it
(and accept its extra complexity) until/unless
it is clear that it is actually a win.  But it
is included as a proof-of-concept, to show that
it fits well with the existing interface.

benchmarks/parsestream.upb.c

@@ -76,8 +76,7 @@ static size_t run(int i)
   (void)i;
   upb_status status = UPB_STATUS_INIT;
   upb_stringsrc_reset(&stringsrc, input_str, input_len);
-  upb_decoder_reset(&decoder, upb_stringsrc_bytesrc(&stringsrc),
-                    0, UPB_NONDELIMITED, NULL);
+  upb_decoder_reset(&decoder, upb_stringsrc_allbytes(&stringsrc), NULL);
   upb_decoder_decode(&decoder, &status);
   if(!upb_ok(&status)) goto err;
   return input_len;

benchmarks/parsetoproto2.upb.cc

@@ -24,6 +24,7 @@
 #include <google/protobuf/descriptor.h>
 #undef private
 
+char *str;
 static size_t len;
 MESSAGE_CIDENT msg[NUM_MESSAGES];
 MESSAGE_CIDENT msg2;
@@ -53,9 +54,13 @@ upb_flow_t proto2_setstr(void *m, upb_value fval, upb_value val) {
   const upb_fielddef *f = upb_value_getfielddef(fval);
   std::string **str = (std::string**)UPB_INDEX(m, f->offset, 1);
   if (*str == f->default_ptr) *str = new std::string;
-  const upb_strref *ref = upb_value_getstrref(val);
+  const upb_byteregion *ref = upb_value_getbyteregion(val);
+  uint32_t len;
+  (*str)->assign(
+      upb_byteregion_getptr(ref, upb_byteregion_startofs(ref), &len),
+      upb_byteregion_len(ref));
+  assert(len == upb_byteregion_len(ref));
   // XXX: only supports contiguous strings atm.
-  (*str)->assign(ref->ptr, ref->len);
   return UPB_CONTINUE;
 }
 
@@ -64,9 +69,13 @@ upb_flow_t proto2_append_str(void *_r, upb_value fval, upb_value val) {
   typedef google::protobuf::RepeatedPtrField<std::string> R;
   (void)fval;
   R *r = (R*)_r;
-  const upb_strref *ref = upb_value_getstrref(val);
+  const upb_byteregion *ref = upb_value_getbyteregion(val);
   // XXX: only supports contiguous strings atm.
-  r->Add()->assign(ref->ptr, ref->len);
+  uint32_t len;
+  r->Add()->assign(
+      upb_byteregion_getptr(ref, upb_byteregion_startofs(ref), &len),
+      upb_byteregion_len(ref));
+  assert(len == upb_byteregion_len(ref));
   return UPB_CONTINUE;
 }
 
@@ -265,7 +274,7 @@ static bool initialize()
   upb_symtab_unref(s);
 
   // Read the message data itself.
-  char *str = upb_readfile(MESSAGE_FILE, &len);
+  str = upb_readfile(MESSAGE_FILE, &len);
   if(str == NULL) {
     fprintf(stderr, "Error reading " MESSAGE_FILE "\n");
     return false;
@@ -275,7 +284,6 @@ static bool initialize()
   msg2.ParseFromArray(str, len);
 
   upb_stringsrc_init(&strsrc);
-  upb_stringsrc_reset(&strsrc, str, len);
   upb_handlers *h = upb_handlers_new();
   upb_accessors_reghandlers(h, def);
   if (!JIT) h->should_jit = false;
@@ -296,8 +304,8 @@ static size_t run(int i)
   (void)i;
   upb_status status = UPB_STATUS_INIT;
   msg[i % NUM_MESSAGES].Clear();
-  upb_decoder_reset(&d, upb_stringsrc_bytesrc(&strsrc),
-                    0, UPB_NONDELIMITED, &msg[i % NUM_MESSAGES]);
+  upb_stringsrc_reset(&strsrc, str, len);
+  upb_decoder_reset(&d, upb_stringsrc_allbytes(&strsrc), &msg[i % NUM_MESSAGES]);
   upb_decoder_decode(&d, &status);
   if(!upb_ok(&status)) goto err;
   return len;

benchmarks/parsetostruct.upb.c

@@ -8,6 +8,7 @@
 #include "upb/pb/glue.h"
 
 static const upb_msgdef *def;
+char *str;
 static size_t len;
 static void *msg[NUM_MESSAGES];
 static upb_stringsrc strsrc;
@@ -33,7 +34,7 @@ static bool initialize()
   upb_symtab_unref(s);
 
   // Read the message data itself.
-  char *str = upb_readfile(MESSAGE_FILE, &len);
+  str = upb_readfile(MESSAGE_FILE, &len);
   if(str == NULL) {
     fprintf(stderr, "Error reading " MESSAGE_FILE "\n");
     return false;
@@ -43,7 +44,6 @@ static bool initialize()
     msg[i] = upb_stdmsg_new(def);
 
   upb_stringsrc_init(&strsrc);
-  upb_stringsrc_reset(&strsrc, str, len);
   upb_handlers *h = upb_handlers_new();
   upb_accessors_reghandlers(h, def);
   if (!JIT) h->should_jit = false;
@@ -70,8 +70,8 @@ static size_t run(int i)
   upb_status status = UPB_STATUS_INIT;
   i %= NUM_MESSAGES;
   upb_msg_clear(msg[i], def);
-  upb_decoder_reset(&d, upb_stringsrc_bytesrc(&strsrc),
-                    0, UPB_NONDELIMITED, msg[i]);
+  upb_stringsrc_reset(&strsrc, str, len);
+  upb_decoder_reset(&d, upb_stringsrc_allbytes(&strsrc), msg[i]);
   upb_decoder_decode(&d, &status);
   if(!upb_ok(&status)) goto err;
   return len;

tests/test_cpp.cc

@@ -7,6 +7,7 @@
  * Tests for C++ wrappers.
  */
 
+#include <stdio.h>
 #include <iostream>
 #include "upb/def.hpp"
 #include "upb/pb/glue.hpp"

tests/test_decoder.c

@@ -52,7 +52,7 @@ int main(int argc, char *argv[]) {
 
   upb_decoder d;
   upb_decoder_init(&d, handlers);
-  upb_decoder_reset(&d, upb_stdio_bytesrc(&in), 0, UPB_NONDELIMITED, p);
+  upb_decoder_reset(&d, upb_stdio_allbytes(&in), p);
 
   upb_status_clear(&status);
   upb_decoder_decode(&d, &status);

tests/test_varint.c

@@ -4,6 +4,7 @@
  * Copyright (c) 2011 Google Inc.  See LICENSE for details.
  */
 
+#include <stdio.h>
 #include "upb/pb/varint.h"
 #include "upb_test.h"
 

tests/tests.c

@@ -16,15 +16,16 @@ static upb_symtab *load_test_proto() {
   ASSERT(s);
   upb_status status = UPB_STATUS_INIT;
   if (!upb_load_descriptor_file_into_symtab(s, descriptor_file, &status)) {
-    fprintf(stderr, "Error loading descriptor file: %s\n", upb_status_getstr(&status));
+    fprintf(stderr, "Error loading descriptor file: %s\n",
+            upb_status_getstr(&status));
     exit(1);
   }
   upb_status_uninit(&status);
   return s;
 }
 
-static upb_flow_t upb_test_onvalue(void *closure, upb_value fval, upb_value val) {
-  (void)closure;
+static upb_flow_t upb_test_onvalue(void *c, upb_value fval, upb_value val) {
+  (void)c;
   (void)fval;
   (void)val;
   return UPB_CONTINUE;
@@ -56,6 +57,7 @@ static void test_upb_symtab() {
   upb_symtab_unref(s);
   const upb_msgdef *m = upb_downcast_msgdef_const(def);
   upb_msg_iter i = upb_msg_begin(m);
+  ASSERT(!upb_msg_done(i));
   upb_fielddef *f = upb_msg_iter_field(i);
   ASSERT(upb_hassubdef(f));
   upb_def *def2 = f->def;