Skip to content
Permalink
Browse files

GODRIVER-33 Implement Extended JSON

  • Loading branch information...
StevenConnors authored and saghm committed Oct 21, 2017
1 parent fa5e1f1 commit 0ce096f1097f133278c2cdf1e034bc026b1a3c08
@@ -0,0 +1,3 @@
[submodule "specifications"]
path = specifications
url = git@github.com:mongodb/specifications.git
@@ -21,7 +21,6 @@ import (
"crypto/rand"
"encoding/binary"
"encoding/hex"
"encoding/json"
"errors"
"fmt"
"io"
@@ -32,6 +31,8 @@ import (
"sync"
"sync/atomic"
"time"

"github.com/10gen/mongo-go-driver/bson/internal/json"
)

//go:generate go run bson_corpus_spec_test_generator.go
@@ -1,30 +1,266 @@
// Code generated by "bson_corpus_spec_test_generator.go"; DO NOT EDIT

package bson_test

import (
"encoding/hex"
"io/ioutil"
"path"
"strings"
"testing"

"github.com/10gen/mongo-go-driver/bson"
"github.com/10gen/mongo-go-driver/bson/extjson"
"github.com/10gen/mongo-go-driver/bson/internal/json"
"github.com/10gen/mongo-go-driver/bson/internal/testutil"
"github.com/stretchr/testify/require"
)

func testValid(t *testing.T, in []byte, expected []byte, result interface{}) {
err := bson.Unmarshal(in, result)
type parseError struct {
Description string
String string
}

type decodeError struct {
Description string
Bson string
}

type valid struct {
Description string
CanonicalBson string `json:"canonical_bson"`
CanonicalExtjson string `json:"canonical_extjson"`
RelaxedExtjson string `json:"relaxed_extjson", omitempty`
DegenerateBson string `json:"degenerate_bson", omitempty`
DegenerateExtjson string `json:"degenerate_extjson", omitempty`
ConvertedBson string `json:"converted_bson", omitempty`
ConvertedExtjson string `json:"converted_extjson", omitempty`
Lossy bool `json:omitempty`
}

type testCase struct {
Description string
BsonType string `json:"bson_type"`
TestKey string `json:"test_key, omitempty"`
Valid []valid `json:omitempty`
DecodeErrors []decodeError `json:omitempty`
ParseErrors []parseError `json:omitempty`
Deprecated bool `json:omitempty`
}

const testsDir string = "../data/bson-corpus/"

func TestBSONSpec(t *testing.T) {
for _, file := range testutil.FindJSONFilesInDir(t, testsDir) {
runTest(t, file)
}
}

func runTest(t *testing.T, filename string) {
filepath := path.Join(testsDir, filename)
content, err := ioutil.ReadFile(filepath)
require.NoError(t, err)

// remove .json extention
filename = filename[:len(filename)-5]
testName := filename + ";"

t.Run(testName, func(t *testing.T) {
var test testCase
require.NoError(t, json.Unmarshal(content, &test))
if test.Deprecated {
return
}

bsonType := test.BsonType
for _, validCase := range test.Valid {
lossy := validCase.Lossy
cEJ := validCase.CanonicalExtjson
cB := validCase.CanonicalBson

t.Run(testName+"validateCanonicalBSON:"+validCase.Description, func(t *testing.T) {
validateCanonicalBSON(t, cB, cEJ)
})
t.Run(testName+"validateCanonicalExtendedJSON:"+validCase.Description, func(t *testing.T) {
validateCanonicalExtendedJSON(t, cB, cEJ, lossy)
})

rEJ := validCase.RelaxedExtjson
if rEJ != "" {
t.Run(testName+"validateBsonToRelaxedJSON:"+validCase.Description, func(t *testing.T) {
validateBsonToRelaxedJSON(t, cB, rEJ)
})
t.Run(testName+"validateRelaxedExtendedJSON:"+validCase.Description, func(t *testing.T) {
validateRelaxedExtendedJSON(t, rEJ, bsonType)
})
}

dB := validCase.DegenerateBson
if dB != "" {
t.Run(testName+"validateDegenerateBSON:"+validCase.Description, func(t *testing.T) {
validateDegenerateBSON(t, dB, cB)
})
}

dEJ := validCase.DegenerateExtjson
if dEJ != "" {
t.Run(testName+"validateDegenerateExtendedJSON:"+validCase.Description, func(t *testing.T) {
validateDegenerateExtendedJSON(t, dEJ, cEJ, cB, lossy)
})
}
}
for _, decodeTest := range test.DecodeErrors {
t.Run(testName+"testDecodeError:"+decodeTest.Description, func(t *testing.T) {
testDecodeError(t, decodeTest.Bson)
})
}
for _, parseTest := range test.ParseErrors {
t.Run(testName+"testParseError:"+parseTest.Description, func(t *testing.T) {
testParseError(t, parseTest.String)
})
}
})
}

// This method validates round trip accuracy for canonical BSON and conversion from canonical BSON to canonical
// Extended JSON.
func validateCanonicalBSON(t *testing.T, cB string, cEJ string) {
// 1. native_to_bson( bson_to_native(cB) ) = cB
decoded, err := hex.DecodeString(cB)
require.NoError(t, err)

out, err := bson.Marshal(result)
nativeReprD := bson.D{}
err = bson.Unmarshal([]byte(decoded), &nativeReprD)
require.NoError(t, err)

require.Equal(t, string(expected), string(out))
roundTripCBByteRepr, err := bson.Marshal(nativeReprD)
roundTripCB := hex.EncodeToString(roundTripCBByteRepr)
require.Equal(t, cB, strings.ToUpper(roundTripCB))

// 2. native_to_canonical_extended_json( bson_to_native(cB) ) = cEJ
var nativeReprBsonD bson.D
err = bson.Unmarshal([]byte(decoded), &nativeReprBsonD)
require.NoError(t, err)

roundTripCEJ, err := extjson.EncodeBSONDtoJSON(nativeReprBsonD)
require.NoError(t, err)
validateExtendedJSONWithCondition(t, cEJ, string(roundTripCEJ), nativeReprBsonD)
}

func testDecodeSkip(t *testing.T, in []byte) {
err := bson.Unmarshal(in, &struct{}{})
// This method validates round trip accuracy for canonical Extended JSON and conversion from canonical Extended JSON
// to canonical BSON.
func validateCanonicalExtendedJSON(t *testing.T, cB string, cEJ string, lossy bool) {
marshalDDoc := extjson.MarshalD{}
err := json.Unmarshal([]byte(cEJ), &marshalDDoc)
require.NoError(t, err)
bsonDDoc := bson.D(marshalDDoc)

// 1. native_to_canonical_extended_json( json_to_native(cEJ) ) = cEJ
roundTripCEJByteRepr, err := extjson.EncodeBSONDtoJSON(bsonDDoc)
require.NoError(t, err)
validateExtendedJSONWithCondition(t, cEJ, string(roundTripCEJByteRepr), bsonDDoc)

// 2. native_to_bson( json_to_native(cEJ) ) = cB (unless lossy)
if !lossy {
bsonHexDecoded, err := bson.Marshal(bsonDDoc)
require.NoError(t, err)

roundTripCB := hex.EncodeToString(bsonHexDecoded)
require.Equal(t, cB, strings.ToUpper(roundTripCB))
}
}

func testDecodeError(t *testing.T, in []byte, result interface{}) {
err := bson.Unmarshal(in, result)
// This method validates conversion from canonical BSON into relaxed Extended JSON
func validateBsonToRelaxedJSON(t *testing.T, cB string, rEJ string) {
// 1. native_to_relaxed_extended_json( bson_to_native(cB) ) = rEJ (if rEJ exists)
decoded, err := hex.DecodeString(cB)
require.NoError(t, err)

nativeRepr := bson.M{}
error := bson.Unmarshal([]byte(decoded), nativeRepr)
require.NoError(t, error)

roundTripREJ, err := json.Marshal(nativeRepr)
require.NoError(t, err)

bsonDDoc := bson.D{}
bsonDDoc.AppendMap(nativeRepr)
validateExtendedJSONWithCondition(t, rEJ, string(roundTripREJ), bsonDDoc)
}

// This method validates round trip accuracy for relaxed Extended JSON.
func validateRelaxedExtendedJSON(t *testing.T, rEJ string, bsonType string) {
// 1. native_to_relaxed_extended_json( json_to_native(rEJ) ) = rEJ
nativeRepr := bson.M{}
require.NoError(t, json.Unmarshal([]byte(rEJ), &nativeRepr))

roundTripREJ, err := json.Marshal(nativeRepr)
require.NoError(t, err)
nativeReprBsonD := bson.D{}
nativeReprBsonD.AppendMap(nativeRepr)
validateExtendedJSONWithCondition(t, rEJ, string(roundTripREJ), nativeReprBsonD)
}

// This method validates conversion from degenerate BSON into canonical BSON.
func validateDegenerateBSON(t *testing.T, dB string, cB string) {
// 1. native_to_bson( bson_to_native(dB) ) = cB
decoded, err := hex.DecodeString(dB)
require.NoError(t, err)

nativeRepr := bson.M{}
err = bson.Unmarshal([]byte(decoded), nativeRepr)
require.NoError(t, err)

dBByteRepr, err := bson.Marshal(nativeRepr)
require.NoError(t, err)
roundTripDB := hex.EncodeToString(dBByteRepr)
require.Equal(t, cB, strings.ToUpper(roundTripDB))
}

// This method validates conversion from degenerate Extended JSON into canonical Extended JSON, as well as conversion
// from degenerate Extended JSON into canonical Extended JSON.
func validateDegenerateExtendedJSON(t *testing.T, dEJ string, cEJ string, cB string, lossy bool) {
// 1. native_to_canonical_extended_json( json_to_native(dEJ) ) = cEJ
marshalDDoc := extjson.MarshalD{}
err := json.Unmarshal([]byte(dEJ), &marshalDDoc)
require.NoError(t, err)
nativeD := bson.D(marshalDDoc)

roundTripCEJ, err := extjson.EncodeBSONDtoJSON(nativeD)
require.NoError(t, err)
require.Equal(t, testutil.CompressJSON(cEJ), string(roundTripCEJ))

// 1. native_to_bson( json_to_native(dEJ) ) = cB (unless lossy)
if !lossy {
dBByteRepr, err := bson.Marshal(nativeD)
require.NoError(t, err)
roundTripDB := hex.EncodeToString(dBByteRepr)
require.Equal(t, cB, strings.ToUpper(roundTripDB))
}
}

func testDecodeError(t *testing.T, b string) {
decoded, err := hex.DecodeString(b)
require.NoError(t, err)
nativeReprD := bson.D{}
err = bson.Unmarshal([]byte(decoded), &nativeReprD)
require.Error(t, err)
}

func testParseError(t *testing.T, s string) {
var nativeReprBsonD bson.D
d := bson.Unmarshal([]byte(s), &nativeReprBsonD)
require.Error(t, d)
}

// This method is required as due to the double values in double.json cannot be represented in 64bit floats in Go.
// Therefore we will check for the condition and if the condition holds, we'll convert the expected string into Go's
// native representation and compare those values.
func validateExtendedJSONWithCondition(t *testing.T, expected string, returned string, nativeRepr bson.D) {
if testutil.CompressJSON(expected) != string(returned) {
marshalDDoc := extjson.MarshalD{}
err := json.Unmarshal([]byte(expected), &marshalDDoc)
require.NoError(t, err)
require.Equal(t, marshalDDoc[0].Value, nativeRepr[0].Value)
} else {
require.Equal(t, testutil.CompressJSON(expected), string(returned))
}
}
@@ -536,11 +536,11 @@ var unmarshalItems = []testItemType{
"\x03rawd\x00" + wrapInDoc("\x0Aa\x00\x0Ac\x00\x08b\x00\x01")},

// Decode old binary.
{bson.M{"_": []byte("old")},
{bson.M{"_": bson.Binary{0x02, []byte("old")}},
"\x05_\x00\x07\x00\x00\x00\x02\x03\x00\x00\x00old"},

// Decode old binary without length. According to the spec, this shouldn't happen.
{bson.M{"_": []byte("old")},
{bson.M{"_": bson.Binary{0x02, []byte("old")}},
"\x05_\x00\x03\x00\x00\x00\x02old"},

// Decode a doc within a doc in to a slice within a doc; shouldn't error
@@ -17,6 +17,7 @@ import (
"strconv"
"sync"
"time"
"unicode/utf8"
)

type decoder struct {
@@ -482,7 +483,7 @@ func (d *decoder) readElemTo(out reflect.Value, kind byte) (good bool) {

start := d.i

if kind == 0x03 {
if kind == 0x03 { // Document
// Delegate unmarshaling of documents.
outt := out.Type()
outk := out.Kind()
@@ -540,7 +541,7 @@ func (d *decoder) readElemTo(out reflect.Value, kind byte) (good bool) {
}
case 0x05: // Binary
b := d.readBinary()
if b.Kind == 0x00 || b.Kind == 0x02 {
if b.Kind == 0x00 {
in = b.Data
} else {
in = b
@@ -828,6 +829,9 @@ func (d *decoder) readBinary() Binary {
func (d *decoder) readStr() string {
l := d.readInt32()
b := d.readBytes(l - 1)
if !utf8.Valid(b) {
corrupted()
}
if d.readByte() != '\x00' {
corrupted()
}
@@ -384,7 +384,6 @@ func (e *encoder) addElem(name string, v reflect.Value, minSize bool) {
case Binary:
e.addElemName(0x05, name)
e.addBinary(s.Kind, s.Data)

case Decimal128:
e.addElemName(0x13, name)
e.addInt64(int64(s.l))
Oops, something went wrong.

0 comments on commit 0ce096f

Please sign in to comment.
You can’t perform that action at this time.