Proposal: support for embedded / raw json #18

maj-o · 2018-05-05T08:03:52Z

Some kind of embedded json saved handled as raw []byte is usefull.
If the type of value does not matter or is unknown at this point of computation. Taking a modified json rpc request as example for test, shows the benefits. A router is interested in id and method. The method itself is interested in params. So params should not be routers problem. I made the changes, so it works for me. It would be great, if the code could be reviewed and added in some way.

Proposed code for decoder.go:

// Decode reads the next JSON-encoded value from its input and stores it in the value pointed to by v.
//
// See the documentation for Unmarshal for details about the conversion of JSON into a Go value.
func (dec *Decoder) Decode(v interface{}) error {
	if dec.isPooled == 1 {
		panic(InvalidUsagePooledDecoderError("Invalid usage of pooled decoder"))
	}
	switch vt := v.(type) {
	case *string:
		return dec.decodeString(vt)
	case *int:
		return dec.decodeInt(vt)
	case *int32:
		return dec.decodeInt32(vt)
	case *uint32:
		return dec.decodeUint32(vt)
	case *int64:
		return dec.decodeInt64(vt)
	case *uint64:
		return dec.decodeUint64(vt)
	case *float64:
		return dec.decodeFloat64(vt)
	case *bool:
		return dec.decodeBool(vt)
	case UnmarshalerObject:
		_, err := dec.decodeObject(vt)
		return err
	case UnmarshalerArray:
		_, err := dec.decodeArray(vt)
		return err
	case *EmbeddedJson:    // <-----------------------------------------  NEW
		return dec.decodeEmbeddedJson(vt)     // <------------  NEW
	default:
		return InvalidUnmarshalError(fmt.Sprintf(invalidUnmarshalErrorMsg, reflect.TypeOf(vt).String()))
	}
}


// >> NEW >>

type EmbeddedJson []byte

func (dec *Decoder) decodeEmbeddedJson(ej *EmbeddedJson) error {
	var err error
	if ej == nil {
		ej = &EmbeddedJson{}
	}
	beginOfEmbeddedJson := dec.cursor
	for ; dec.cursor < dec.length || dec.read(); dec.cursor++ {
		switch dec.data[dec.cursor] {
		case ' ', '\n', '\t', '\r', ',':
			continue
		// is null
		case 'n', 't':
			dec.cursor = dec.cursor + 4
		// is false
		case 'f':
			dec.cursor = dec.cursor + 5
		// is an object
		case '{':
			dec.cursor = dec.cursor + 1
			dec.cursor, err = dec.skipObject()
		// is string
		case '"':
			dec.cursor = dec.cursor + 1
			err = dec.skipString()                    // why no new dec.cursor in result?
		// is array
		case '[':
			dec.cursor = dec.cursor + 1
			dec.cursor, err = dec.skipArray()
		case '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', '-':
			dec.cursor, err = dec.skipNumber()
		}
		break
	}
	if err == nil {
		if dec.cursor-1 > beginOfEmbeddedJson {
			*ej = append(*ej, dec.data[beginOfEmbeddedJson:dec.cursor]...)
		}
	}
	return err
}

func (dec *Decoder) AddEmbeddedJson(v *EmbeddedJson) error {
	err := dec.decodeEmbeddedJson(v)
	if err != nil {
		return err
	}
	dec.called |= 1
	return nil
}

// << NEW <<

QUICK TEST:

type Request struct {
	id     string
	method string
	params EmbeddedJson
	more   int
}

func (r *Request) UnmarshalObject(dec *Decoder, key string) error {
	switch key {
	case "id":
		return dec.AddString(&r.id)
	case "method":
		return dec.AddString(&r.method)
	case "params":
		return dec.AddEmbeddedJson(&r.params)
	case "more":
		dec.AddInt(&r.more)
	}
	return nil
}

func (r *Request) NKeys() int {
	return 4
}

func TestEmbeddedJson(t *testing.T) {
	json := []byte(`{"id":"someid","method":"getmydata","params":{"example":"of raw data"}, "more":123}`)
	request := &Request{}
	Unmarshal(json, request)
	t.Log(request)
	t.Log(string(request.params))
}

I know test is not complete - but shows, what the benefits are.
For json rpc the params are important inside of the method using them.
This change would make things simpler (simpler ist better ;-)

ENCODER:

func Marshal(v interface{}) ([]byte, error) {
	switch vt := v.(type) {
	case MarshalerObject:
		enc := BorrowEncoder(nil)
		defer enc.Release()
		return enc.encodeObject(vt)
	case MarshalerArray:
		enc := BorrowEncoder(nil)
		defer enc.Release()
		return enc.encodeArray(vt)
	case string:
		enc := BorrowEncoder(nil)
		defer enc.Release()
		return enc.encodeString(vt)
	case bool:
		enc := BorrowEncoder(nil)
		defer enc.Release()
		return enc.encodeBool(vt)
	case int:
		enc := BorrowEncoder(nil)
		defer enc.Release()
		return enc.encodeInt(vt)
	case int64:
		enc := BorrowEncoder(nil)
		defer enc.Release()
		return enc.encodeInt64(vt)
	case int32:
		enc := BorrowEncoder(nil)
		defer enc.Release()
		return enc.encodeInt(int(vt))
	case int16:
		enc := BorrowEncoder(nil)
		defer enc.Release()
		return enc.encodeInt(int(vt))
	case int8:
		enc := BorrowEncoder(nil)
		defer enc.Release()
		return enc.encodeInt(int(vt))
	case uint64:
		enc := BorrowEncoder(nil)
		defer enc.Release()
		return enc.encodeInt(int(vt))
	case uint32:
		enc := BorrowEncoder(nil)
		defer enc.Release()
		return enc.encodeInt(int(vt))
	case uint16:
		enc := BorrowEncoder(nil)
		defer enc.Release()
		return enc.encodeInt(int(vt))
	case uint8:
		enc := BorrowEncoder(nil)
		defer enc.Release()
		return enc.encodeInt(int(vt))
	case float64:
		enc := BorrowEncoder(nil)
		defer enc.Release()
		return enc.encodeFloat(vt)
	case float32:
		enc := BorrowEncoder(nil)
		defer enc.Release()
		return enc.encodeFloat32(vt)
	case *EmbeddedJson:                   // << ------------ NEW
		enc := BorrowEncoder(nil)         // << ------------ NEW
		defer enc.Release()               // << ------------ NEW
		return enc.encodeEmbeddedJson(vt) // << ------------ NEW
	default:
		return nil, InvalidMarshalError(fmt.Sprintf(invalidMarshalErrorMsg, reflect.TypeOf(vt).String()))
	}
}

// >> NEW >>

func (enc *Encoder) encodeEmbeddedJson(v *EmbeddedJson) ([]byte, error) {
	enc.writeBytes(*v)
	return enc.buf, nil
}

func (enc *Encoder) AddEmbeddedJson(v *EmbeddedJson) {
	enc.grow(len(*v) + 4)
	r, ok := enc.getPreviousRune()
	if ok && r != '[' {
		enc.writeByte(',')
	}
	enc.writeBytes(*v)
}

func (enc *Encoder) AddEmbeddedJsonOmitEmpty(v *EmbeddedJson) {
	if v == nil || len(*v) == 0 {
		return
	}
	r, ok := enc.getPreviousRune()
	if ok && r != '[' {
		enc.writeByte(',')
	}
	enc.writeBytes(*v)
}

func (enc *Encoder) AddEmbeddedJsonKey(key string, v *EmbeddedJson) {
	enc.grow(len(key) + len(*v) + 5)
	r, ok := enc.getPreviousRune()
	if ok && r != '{' {
		enc.writeByte(',')
	}
	enc.writeByte('"')
	enc.writeStringEscape(key)
	enc.writeBytes(objKey)
	enc.writeBytes(*v)
}

func (enc *Encoder) AddEmbeddedJsonKeyOmitEmpty(key string, v *EmbeddedJson) {
	if v == nil || len(*v) == 0 {
		return
	}
	enc.grow(len(key) + len(*v) + 5)
	r, ok := enc.getPreviousRune()
	if ok && r != '{' {
		enc.writeByte(',')
	}
	enc.writeByte('"')
	enc.writeStringEscape(key)
	enc.writeBytes(objKey)
	enc.writeBytes(*v)
}


// << NEW <<

Encoder, Decoder Test:

type Request struct {
	id     string
	method string
	params EmbeddedJson
	more   int
}

func (r *Request) UnmarshalObject(dec *Decoder, key string) error {
	switch key {
	case "id":
		return dec.AddString(&r.id)
	case "method":
		return dec.AddString(&r.method)
	case "params":
		return dec.AddEmbeddedJson(&r.params)

	case "more":
		dec.AddInt(&r.more)
	}
	return nil
}

func (r *Request) NKeys() int {
	return 4
}

func (r *Request) MarshalObject(enc *Encoder) {
	enc.AddStringKeyOmitEmpty("id", r.id)
	enc.AddStringKey("method", r.method)
	enc.AddEmbeddedJsonKey("params", &r.params)
	enc.AddIntKeyOmitEmpty("more", r.more)
}
func (r *Request) IsNil() bool {
	return r == nil
}

var request = &Request{}

func TestDecodeEmbeddedJson(t *testing.T) {
	json := []byte(`{"id":"someid","method":"getmydata","params":{"example":"of raw data"},"more":123}`)
	Unmarshal(json, request)
	t.Log(request)
	t.Log(string(request.params))
}

func TestEncodeEmbeddedJson(t *testing.T) {
	json := `{"id":"someid","method":"getmydata","params":{"example":"of raw data"},"more":123}`
	marshalled, err := MarshalObject(request)
	if err != nil {
		t.Error(err)
	}
	t.Log(string(marshalled))
	if string(marshalled) != json {
		t.Error("bad", string(marshalled), " != ", json)
	}
}

The text was updated successfully, but these errors were encountered:

francoispqt · 2018-05-05T16:10:40Z

Hi, thank you for your proposal, as discussed before, it is definitely an interesting feature.

I'm going to review your implementation and will let you know my feedback as soon as possible.

maj-o · 2018-05-05T18:42:40Z

Great, maj-o is my private nick. Regards Andreas LG Andreas

…

-----Original Message----- From: Francois Parquet <notifications@github.com> To: francoispqt/gojay <gojay@noreply.github.com> Cc: Andreas <ajmatuschek@gmail.com>, Author <author@noreply.github.com> Sent: Sa., 05 Mai 2018 18:10 Subject: Re: [francoispqt/gojay] Proposal: support for embedded / raw json (#18) Hi, thank you for your proposal, has discussed before, it is definitely an interesting feature. I'm going to review your implementation and will let you know my feedback as soon as possible.

-- You are receiving this because you authored the thread. Reply to this email directly or view it on GitHub: #18 (comment)

francoispqt · 2018-05-06T03:48:34Z

Hi Andreas,

Sorry was a bit late yesterday when I started reviewing your proposal (I live in Hong Kong). I am creating a branch, where I will add the feature.

branch name:
feature/support-for-embedded-json

If you want to contribute you can submit pull request to that branch.

When ready will merge the branch along with another branch where I added some optimisations, this will be a new release 0.10.5.

francoispqt · 2018-05-06T04:38:11Z

Pull request is open here #19
You can see evolution of implementation. Should be completed soon.

francoispqt · 2018-05-06T04:46:40Z

I renamed EmbeddedJson to EmbeddedJSON as it conventional to go (golint was actually complaining about EmbeddedJson.

Also on the decoding part, I am returning an error if the EmbeddedJSON is a nil pointer.

francoispqt · 2018-05-06T16:15:26Z

Version 0.10.5 has been released. Will add documentation when I have time. Closing the issue.

maj-o · 2018-05-06T19:03:50Z

C H A P E A U !

Had no time to take a look earlier.
No errors or wishes so far (passes all tests) - code is readable and performant - does what it should. :-)
GREAT ! Thank You very much!

francoispqt added the enhancement New feature or request label May 6, 2018

francoispqt self-assigned this May 6, 2018

francoispqt closed this as completed May 6, 2018

Provide feedback

Saved searches

Use saved searches to filter your results more quickly

Proposal: support for embedded / raw json #18

Proposal: support for embedded / raw json #18

maj-o commented May 5, 2018 •

edited

Loading

francoispqt commented May 5, 2018 •

edited

Loading

maj-o commented May 5, 2018 via email

francoispqt commented May 6, 2018 •

edited

Loading

francoispqt commented May 6, 2018

francoispqt commented May 6, 2018

francoispqt commented May 6, 2018 •

edited

Loading

maj-o commented May 6, 2018

Proposal: support for embedded / raw json #18

Proposal: support for embedded / raw json #18

Comments

maj-o commented May 5, 2018 • edited Loading

francoispqt commented May 5, 2018 • edited Loading

maj-o commented May 5, 2018 via email

francoispqt commented May 6, 2018 • edited Loading

francoispqt commented May 6, 2018

francoispqt commented May 6, 2018

francoispqt commented May 6, 2018 • edited Loading

maj-o commented May 6, 2018

maj-o commented May 5, 2018 •

edited

Loading

francoispqt commented May 5, 2018 •

edited

Loading

francoispqt commented May 6, 2018 •

edited

Loading

francoispqt commented May 6, 2018 •

edited

Loading