decode, interp: More buffer reuse

wader · Sep 16, 2021 · c7416e6 · c7416e6
1 parent 6207fcc
commit c7416e6
Show file tree

Hide file tree

Showing 7 changed files with 24 additions and 27 deletions.
diff --git a/format/gzip/gzip.go b/format/gzip/gzip.go
@@ -112,7 +112,7 @@ func gzDecode(d *decode.D, in interface{}) interface{} {
 		deflateR := flate.NewReader(compressedBB)
 		uncompressed := &bytes.Buffer{}
 		crc32W := crc32.NewIEEE()
-		if _, err := io.Copy(io.MultiWriter(uncompressed, crc32W), deflateR); err != nil { //nolint:gosec
+		if _, err := decode.Copy(d, io.MultiWriter(uncompressed, crc32W), deflateR); err != nil {
 			d.Invalid(err.Error())
 		}
 		calculatedCRC32 = crc32W.Sum(nil)

diff --git a/pkg/bitio/bitio.go b/pkg/bitio/bitio.go
@@ -69,8 +69,11 @@ func (a *AlignBitReader) ReadBitsAt(p []byte, nBits int, bitOff int64) (n int, e
 	return n, err
 }
 
-func Copy(dst BitWriter, src BitReader) (n int64, err error) {
-	buf := make([]byte, 32*1024)
+func CopyBuffer(dst BitWriter, src BitReader, buf []byte) (n int64, err error) {
+	// same default size as io.Copy
+	if buf == nil {
+		buf = make([]byte, 32*1024)
+	}
 	var written int64
 
 	for {
@@ -98,6 +101,10 @@ func Copy(dst BitWriter, src BitReader) (n int64, err error) {
 	return written, err
 }
 
+func Copy(dst BitWriter, src BitReader) (n int64, err error) {
+	return CopyBuffer(dst, src, nil)
+}
+
 func BitsByteCount(nBits int64) int64 {
 	n := nBits / 8
 	if nBits%8 != 0 {

diff --git a/pkg/decode/decode.go b/pkg/decode/decode.go
@@ -241,11 +241,11 @@ func NewDecoder(ctx context.Context, name string, description string, format *Fo
 	}
 }
 
-func (d *D) AllocReadBuf(n int) []byte {
+func (d *D) SharedReadBuf(n int) []byte {
 	if d.readBuf == nil {
 		d.readBuf = new([]byte)
 	}
-	if cap(*d.readBuf) < n {
+	if len(*d.readBuf) < n {
 		*d.readBuf = make([]byte, n)
 	}
 	return *d.readBuf

diff --git a/pkg/decode/decode_readers.go b/pkg/decode/decode_readers.go
@@ -71,7 +71,7 @@ func (d *D) TryPeekBits(nBits int) (uint64, error) {
 // Bits reads nBits bits from buffer
 func (d *D) bits(nBits int) (uint64, error) {
 	// 64 bits max, 9 byte worse case if not byte aligned
-	buf := d.AllocReadBuf(9)
+	buf := d.SharedReadBuf(9)
 	_, err := bitio.ReadFull(d.bitBuf, buf, nBits)
 	if err != nil {
 		return 0, err

diff --git a/pkg/decode/io.go b/pkg/decode/io.go
@@ -8,8 +8,8 @@ import (
 )
 
 func Copy(d *D, r io.Writer, w io.Reader) (int64, error) {
-	// TODO: what size?
-	buf := d.AllocReadBuf(64 * 1024)
+	// TODO: what size? now same as io.Copy
+	buf := d.SharedReadBuf(32 * 1024)
 	return io.CopyBuffer(r, w, buf)
 }
 

diff --git a/pkg/interp/dump.go b/pkg/interp/dump.go
@@ -33,7 +33,7 @@ func isCompound(v *decode.Value) bool {
 	}
 }
 
-func dumpEx(v *decode.Value, cw *columnwriter.Writer, depth int, rootV *decode.Value, rootDepth int, addrWidth int, opts Options) error {
+func dumpEx(v *decode.Value, buf []byte, cw *columnwriter.Writer, depth int, rootV *decode.Value, rootDepth int, addrWidth int, opts Options) error {
 	deco := opts.Decorator
 	// no error check as we write into buffering column
 	// we check for err later for Flush()
@@ -265,14 +265,16 @@ func dumpEx(v *decode.Value, cw *columnwriter.Writer, depth int, rootV *decode.V
 		asciiFn := func(b byte) string { return deco.ByteColor(b).Wrap(asciiwriter.SafeASCII(b)) }
 
 		if vBitBuf != nil {
-			if _, err := io.Copy(
+			if _, err := io.CopyBuffer(
 				hexpairwriter.New(cw.Columns[colHex], opts.LineBytes, int(startLineByteOffset), hexpairFn),
-				io.LimitReader(vBitBuf.Copy(), displaySizeBytes)); err != nil {
+				io.LimitReader(vBitBuf.Copy(), displaySizeBytes),
+				buf); err != nil {
 				return err
 			}
-			if _, err := io.Copy(
+			if _, err := io.CopyBuffer(
 				asciiwriter.New(cw.Columns[colAscii], opts.LineBytes, int(startLineByteOffset), asciiFn),
-				io.LimitReader(vBitBuf.Copy(), displaySizeBytes)); err != nil {
+				io.LimitReader(vBitBuf.Copy(), displaySizeBytes),
+				buf); err != nil {
 				return err
 			}
 		}
@@ -340,8 +342,9 @@ func dump(v *decode.Value, w io.Writer, opts Options) error {
 	}))
 
 	cw := columnwriter.New(w, []int{maxAddrIndentWidth, 1, opts.LineBytes*3 - 1, 1, opts.LineBytes, 1, -1})
+	buf := make([]byte, 32*1024)
 
 	return v.WalkPreOrder(makeWalkFn(func(v *decode.Value, rootV *decode.Value, depth int, rootDepth int) error {
-		return dumpEx(v, cw, depth, rootV, rootDepth, maxAddrIndentWidth-rootDepth, opts)
+		return dumpEx(v, buf, cw, depth, rootV, rootDepth, maxAddrIndentWidth-rootDepth, opts)
 	}))
 }
diff --git a/pkg/interp/funcs.go b/pkg/interp/funcs.go
@@ -717,19 +717,6 @@ func (i *Interp) tovalue(c interface{}, a []interface{}) interface{} {
 	return v
 }
 
-// func (i *Interp) md5(c interface{}, a []interface{}) interface{} {
-// 	bb, _, err := toBuffer(c)
-// 	if err != nil {
-// 		return err
-// 	}
-
-// 	if _, err := io.Copy(md5, bb); err != nil {
-// 		return err
-// 	}
-
-// 	return md5.Sum(nil)
-// }
-
 func (i *Interp) queryEscape(c interface{}, a []interface{}) interface{} {
 	s, err := toString(c)
 	if err != nil {