diff --git a/.changelog/v0.37.5/improvements/2846-speedup-json-encoding.md b/.changelog/v0.37.5/improvements/2846-speedup-json-encoding.md new file mode 100644 index 0000000000..026b5a4398 --- /dev/null +++ b/.changelog/v0.37.5/improvements/2846-speedup-json-encoding.md @@ -0,0 +1,2 @@ +- `[libs/json]` Lower the memory overhead of JSON encoding by using JSON encoders internally + ([\#2846](https://github.com/cometbft/cometbft/pull/2846)). diff --git a/libs/json/encoder.go b/libs/json/encoder.go index 11990e2af6..67112384fd 100644 --- a/libs/json/encoder.go +++ b/libs/json/encoder.go @@ -42,7 +42,7 @@ func MarshalIndent(v interface{}, prefix, indent string) ([]byte, error) { return buf.Bytes(), nil } -func encode(w io.Writer, v interface{}) error { +func encode(w *bytes.Buffer, v any) error { // Bare nil values can't be reflected, so we must handle them here. if v == nil { return writeStr(w, "null") @@ -60,7 +60,7 @@ func encode(w io.Writer, v interface{}) error { return encodeReflect(w, rv) } -func encodeReflect(w io.Writer, rv reflect.Value) error { +func encodeReflect(w *bytes.Buffer, rv reflect.Value) error { if !rv.IsValid() { return errors.New("invalid reflect value") } @@ -115,7 +115,7 @@ func encodeReflect(w io.Writer, rv reflect.Value) error { } } -func encodeReflectList(w io.Writer, rv reflect.Value) error { +func encodeReflectList(w *bytes.Buffer, rv reflect.Value) error { // Emit nil slices as null. if rv.Kind() == reflect.Slice && rv.IsNil() { return writeStr(w, "null") @@ -150,7 +150,7 @@ func encodeReflectList(w io.Writer, rv reflect.Value) error { return writeStr(w, "]") } -func encodeReflectMap(w io.Writer, rv reflect.Value) error { +func encodeReflectMap(w *bytes.Buffer, rv reflect.Value) error { if rv.Type().Key().Kind() != reflect.String { return errors.New("map key must be string") } @@ -181,7 +181,7 @@ func encodeReflectMap(w io.Writer, rv reflect.Value) error { return writeStr(w, "}") } -func encodeReflectStruct(w io.Writer, rv reflect.Value) error { +func encodeReflectStruct(w *bytes.Buffer, rv reflect.Value) error { sInfo := makeStructInfo(rv.Type()) if err := writeStr(w, "{"); err != nil { return err @@ -212,7 +212,7 @@ func encodeReflectStruct(w io.Writer, rv reflect.Value) error { return writeStr(w, "}") } -func encodeReflectInterface(w io.Writer, rv reflect.Value) error { +func encodeReflectInterface(w *bytes.Buffer, rv reflect.Value) error { // Get concrete value and dereference pointers. for rv.Kind() == reflect.Ptr || rv.Kind() == reflect.Interface { if rv.IsNil() { @@ -237,14 +237,17 @@ func encodeReflectInterface(w io.Writer, rv reflect.Value) error { return writeStr(w, "}") } -func encodeStdlib(w io.Writer, v interface{}) error { - // Doesn't stream the output because that adds a newline, as per: - // https://golang.org/pkg/encoding/json/#Encoder.Encode - blob, err := json.Marshal(v) +func encodeStdlib(w *bytes.Buffer, v any) error { + // Stream the output of the JSON marshaling directly into the buffer. + // The stdlib encoder will write a newline, so we must truncate it, + // which is why we pass in a bytes.Buffer throughout, not io.Writer. + enc := json.NewEncoder(w) + err := enc.Encode(v) if err != nil { return err } - _, err = w.Write(blob) + // Remove the last byte from the buffer + w.Truncate(w.Len() - 1) return err } diff --git a/libs/json/encoder_test.go b/libs/json/encoder_test.go index e6eb18a122..8cf536b26b 100644 --- a/libs/json/encoder_test.go +++ b/libs/json/encoder_test.go @@ -102,3 +102,20 @@ func TestMarshal(t *testing.T) { }) } } + +func BenchmarkJsonMarshalStruct(b *testing.B) { + s := "string" + sPtr := &s + i64 := int64(64) + ti := time.Date(2020, 6, 2, 18, 5, 13, 4346374, time.FixedZone("UTC+2", 2*60*60)) + car := &Car{Wheels: 4} + boat := Boat{Sail: true} + for i := 0; i < b.N; i++ { + _, _ = json.Marshal(Struct{ + Bool: true, Float64: 3.14, Int32: 32, Int64: 64, Int64Ptr: &i64, + String: "foo", StringPtrPtr: &sPtr, Bytes: []byte{1, 2, 3}, + Time: ti, Car: car, Boat: boat, Vehicles: []Vehicle{car, boat}, + Child: &Struct{Bool: false, String: "child"}, private: "private", + }) + } +}