Performance issue with StreamDecoder when combined with zstd stream decoder

[satvik007]

I noticed performance issues with sonic StreamDecoder while trying to use it with zstd decoder. Version - v1.8.3

$ go test -bench . -benchtime 1000000x
goos: linux
goarch: amd64
pkg: helloWorld
cpu: AMD Ryzen 5 5600X 6-Core Processor             
BenchmarkJsonDecoder-12                  1000000               521.2 ns/op
BenchmarkJsonUnmarshal-12                1000000               591.5 ns/op
BenchmarkSonicDecoder-12                 1000000              3507 ns/op
BenchmarkSonicDecoderReadAll-12          1000000               380.5 ns/op
BenchmarkSonicUnmarshall-12              1000000               206.1 ns/op
PASS

package main

import (
  "bytes"
  "encoding/json"
  "io"
  "os"
  "testing"

  "github.com/bytedance/sonic"
  "github.com/bytedance/sonic/decoder"
  "github.com/google/uuid"
  "github.com/klauspost/compress/zstd"
)

// generate "temp.json.zst" file used for benchmarking
func TestCreateFile(t *testing.T) {
  var urls []string

  for i := 0; i < 1500000; i++ {
    urls = append(urls, "https://example.com"+uuid.NewString())
  }

  file, _ := os.Create("temp.json.zst")
  defer file.Close()

  zstdWriter, _ := zstd.NewWriter(file)
  defer zstdWriter.Close()

  json.NewEncoder(zstdWriter).Encode(urls)
}

// combining zstd with "encoding/json" decoder
func BenchmarkJsonDecoder(b *testing.B) {
  file, _ := os.Open("temp.json.zst")
  defer file.Close()

  zstdReader, _ := zstd.NewReader(file)
  defer zstdReader.Close()

  var urls []string

  json.NewDecoder(zstdReader).Decode(&urls)
}

// read zstd decoded data into a buffer and unmarshall with encoding/json
func BenchmarkJsonUnmarshal(b *testing.B) {
  file, _ := os.Open("temp.json.zst")
  defer file.Close()

  zstdReader, _ := zstd.NewReader(file)
  defer zstdReader.Close()

  var urls []string

  bytes, _ := io.ReadAll(zstdReader)
  json.Unmarshal(bytes, &urls)
}

// combining zstd decoder with sonic stream decoder
func BenchmarkSonicDecoder(b *testing.B) {
  file, _ := os.Open("temp.json.zst")
  defer file.Close()

  zstdReader, _ := zstd.NewReader(file)
  defer zstdReader.Close()

  var urls []string

  decoder.NewStreamDecoder(zstdReader).Decode(&urls)
}

// read zstd decoded file into a buffer and then decode with sonic stream decoder
func BenchmarkSonicDecoderReadAll(b *testing.B) {
  file, _ := os.Open("temp.json.zst")
  defer file.Close()

  zstdReader, _ := zstd.NewReader(file)
  defer zstdReader.Close()

  var urls []string

  by, _ := io.ReadAll(zstdReader)
  decoder.NewStreamDecoder(bytes.NewReader(by)).Decode(&urls)
}

// read zstd decoded file into a buffer and then Unmarshal with sonic
func BenchmarkSonicUnmarshall(b *testing.B) {
  file, _ := os.Open("temp.json.zst")
  defer file.Close()

  zstdReader, _ := zstd.NewReader(file)
  defer zstdReader.Close()

  var urls []string

  by, _ := io.ReadAll(zstdReader)
  sonic.Unmarshal(by, &urls)
}

[AsterDY]

StreamDecoder is designed to reduce memory consumption not suitable for incontinuous IO buffers. As you found, U can use io.ReadAll() by yourself

[AsterDY]

fixed by #444