encoding/json: add JSON streaming parse API

This change adds new methods to Decoder.

 * Decoder.Token steps through a JSON document, returning a value for each token.
 * Decoder.Decode unmarshals the entire value at the token stream's current
   position (in addition to its existing function in a stream of JSON values)

Fixes #6050.
Fixes #6499.

Change-Id: Iff283e0e7b537221ae256392aca6529f06ebe211
Reviewed-on: https://go-review.googlesource.com/9073
Reviewed-by: Russ Cox <rsc@golang.org>

src/encoding/json/bench_test.go

@@ -15,6 +15,7 @@ import (
 	"compress/gzip"
 	"io/ioutil"
 	"os"
+	"strings"
 	"testing"
 )
 
@@ -126,6 +127,28 @@ func BenchmarkCodeDecoder(b *testing.B) {
 	b.SetBytes(int64(len(codeJSON)))
 }
 
+func BenchmarkDecoderStream(b *testing.B) {
+	b.StopTimer()
+	var buf bytes.Buffer
+	dec := NewDecoder(&buf)
+	buf.WriteString(`"` + strings.Repeat("x", 1000000) + `"` + "\n\n\n")
+	var x interface{}
+	if err := dec.Decode(&x); err != nil {
+		b.Fatal("Decode:", err)
+	}
+	ones := strings.Repeat(" 1\n", 300000) + "\n\n\n"
+	b.StartTimer()
+	for i := 0; i < b.N; i++ {
+		if i%300000 == 0 {
+			buf.WriteString(ones)
+		}
+		x = nil
+		if err := dec.Decode(&x); err != nil || x != 1.0 {
+			b.Fatalf("Decode: %v after %d", err, i)
+		}
+	}
+}
+
 func BenchmarkCodeUnmarshal(b *testing.B) {
 	if codeJSON == nil {
 		b.StopTimer()

src/encoding/json/example_test.go

@@ -83,6 +83,97 @@ func ExampleDecoder() {
 	// Ed: Go fmt yourself!
 }
 
+// This example uses a Decoder to decode a stream of distinct JSON values.
+func ExampleDecoder_Token() {
+	const jsonStream = `
+		{"Message": "Hello", "Array": [1, 2, 3], "Null": null, "Number": 1.234}
+	`
+	dec := json.NewDecoder(strings.NewReader(jsonStream))
+	for {
+		t, err := dec.Token()
+		if err == io.EOF {
+			break
+		}
+		if err != nil {
+			log.Fatal(err)
+		}
+		fmt.Printf("%T: %v", t, t)
+		if dec.More() {
+			fmt.Printf(" (more)")
+		}
+		fmt.Printf("\n")
+	}
+	// Output:
+	// json.Delim: { (more)
+	// string: Message (more)
+	// string: Hello (more)
+	// string: Array (more)
+	// json.Delim: [ (more)
+	// float64: 1 (more)
+	// float64: 2 (more)
+	// float64: 3
+	// json.Delim: ] (more)
+	// string: Null (more)
+	// <nil>: <nil> (more)
+	// string: Number (more)
+	// float64: 1.234
+	// json.Delim: }
+}
+
+// This example uses a Decoder to decode a streaming array of JSON objects.
+func ExampleDecoder_Decode_stream() {
+	const jsonStream = `
+		[
+			{"Name": "Ed", "Text": "Knock knock."},
+			{"Name": "Sam", "Text": "Who's there?"},
+			{"Name": "Ed", "Text": "Go fmt."},
+			{"Name": "Sam", "Text": "Go fmt who?"},
+			{"Name": "Ed", "Text": "Go fmt yourself!"}
+		]
+	`
+	type Message struct {
+		Name, Text string
+	}
+	dec := json.NewDecoder(strings.NewReader(jsonStream))
+
+	// read open bracket
+	t, err := dec.Token()
+	if err != nil {
+		log.Fatal(err)
+	}
+	fmt.Printf("%T: %v\n", t, t)
+
+	var m Message
+	// while the array contains values
+	for dec.More() {
+
+		// decode an array value (Message)
+		err := dec.Decode(&m)
+		if err != nil {
+			log.Fatal(err)
+		}
+
+		fmt.Printf("%v: %v\n", m.Name, m.Text)
+	}
+
+	// read closing bracket
+	t, err = dec.Token()
+	if err != nil {
+		log.Fatal(err)
+	}
+	fmt.Printf("%T: %v\n", t, t)
+
+	// Output:
+	// json.Delim: [
+	// Ed: Knock knock.
+	// Sam: Who's there?
+	// Ed: Go fmt.
+	// Sam: Go fmt who?
+	// Ed: Go fmt yourself!
+	// json.Delim: ]
+
+}
+
 // This example uses RawMessage to delay parsing part of a JSON message.
 func ExampleRawMessage() {
 	type Color struct {