I didn't expect the bottle neck of my program to came from the standard lib, but this seems to be an exception.
The DotReader/DotWriter from the net/textproto library uses bufio.ReadByte/WriteByte on each input/output byte, which is over 300 times slower than bufio.Read/Write functions for the same amount of data. This is causing significant impact on data throughput for applications using these interfaces.
package my_test
import (
"bufio"
"io"
"net/textproto"
"testing"
)
type FillReader byte
func (r FillReader) Read(b []byte) (n int, err error) {
n = len(b)
if n > 0 {
b[0] = byte(r)
for i := 1; i < n; i *= 2 {
copy(b[i:], b[:i])
}
}
return n, nil
}
func BenchmarkIONull(b *testing.B) {
size := int64(1024 * 1024 * 4)
b.SetBytes(size)
b.ResetTimer()
for i := 0; i < b.N; i++ {
io.Copy(io.Discard, io.LimitReader(FillReader('.'), size))
}
}
func BenchmarkIOBufioWrite(b *testing.B) {
size := int64(1024 * 1024 * 4)
b.SetBytes(size)
b.ResetTimer()
for i := 0; i < b.N; i++ {
r := io.LimitReader(FillReader('.'), size)
io.Copy(io.Discard, r)
}
}
func BenchmarkIOBufioWriteByte(b *testing.B) {
size := int64(1024 * 1024 * 4)
b.SetBytes(size)
b.ResetTimer()
for i := 0; i < b.N; i++ {
w := bufio.NewWriter(io.Discard)
for j := int64(0); j < size; j++ {
w.WriteByte(0)
}
}
}
func BenchmarkIODotWriter(b *testing.B) {
size := int64(1024 * 1024 * 4)
b.SetBytes(size)
b.ResetTimer()
for i := 0; i < b.N; i++ {
r := io.LimitReader(FillReader('.'), size)
w := textproto.NewWriter(bufio.NewWriter(io.Discard)).DotWriter()
io.Copy(w, r)
}
}
func BenchmarkIOBufioRead(b *testing.B) {
size := int64(1024 * 1024 * 4)
b.SetBytes(size)
b.ResetTimer()
for i := 0; i < b.N; i++ {
r := bufio.NewReader(io.LimitReader(FillReader('.'), size))
io.Copy(io.Discard, r)
}
}
func BenchmarkIOBufioReadByte(b *testing.B) {
size := int64(1024 * 1024 * 4)
b.SetBytes(size)
b.ResetTimer()
for i := 0; i < b.N; i++ {
r := bufio.NewReader(io.LimitReader(FillReader('.'), size))
for j := int64(0); j < size; j++ {
r.ReadByte()
}
}
}
func BenchmarkIODotReader(b *testing.B) {
size := int64(1024 * 1024 * 4)
b.SetBytes(size)
b.ResetTimer()
for i := 0; i < b.N; i++ {
r := textproto.NewReader(bufio.NewReader(io.LimitReader(FillReader('.'), size))).DotReader()
io.Copy(io.Discard, r)
}
}❯ go test -cpuprofile cpu.prof -memprofile mem.prof -benchmem -bench IO
goos: windows
goarch: amd64
pkg: mytest
cpu: Intel(R) Core(TM) i7-8086K CPU @ 4.00GHz
BenchmarkIONull-12 13660 91626 ns/op 45776.23 MB/s 24 B/op 1 allocs/op
BenchmarkIOBufioWrite-12 19095 55199 ns/op 75984.94 MB/s 24 B/op 1 allocs/op
BenchmarkIOBufioWriteByte-12 99 10311620 ns/op 406.76 MB/s 4178 B/op 1 allocs/op
BenchmarkIODotWriter-12 67 16815748 ns/op 249.43 MB/s 36984 B/op 6 allocs/op
BenchmarkIOBufioRead-12 20808 58743 ns/op 71401.44 MB/s 4223 B/op 3 allocs/op
BenchmarkIOBufioReadByte-12 99 10507806 ns/op 399.16 MB/s 4202 B/op 2 allocs/op
BenchmarkIODotReader-12 60 19456187 ns/op 215.58 MB/s 4305 B/op 5 allocs/op
PASS
ok mytest 11.788s
I didn't expect the bottle neck of my program to came from the standard lib, but this seems to be an exception.
The DotReader/DotWriter from the net/textproto library uses bufio.ReadByte/WriteByte on each input/output byte, which is over 300 times slower than bufio.Read/Write functions for the same amount of data. This is causing significant impact on data throughput for applications using these interfaces.