Closed
Description
What version of Go are you using (go version)?
$ go version go version go1.16 linux/amd64
Does this issue reproduce with the latest release?
Yes
What operating system and processor architecture are you using (go env)?
go env Output
$ go env go env ... GOARCH="amd64" GOOS="linux" ...
What did you do?
Sending a HTTP PUT request with bytes.Reader
url := "http://IP:Port/xxxx/yyy"
data := make([]byte, 1<<20)
// write something into data
r, err := http.NewRequest(http.MethodPut, url, bytes.NewReader(data))
if err != nil {
return err
}
res, err := http.DefaultClient.Do(r)
if err != nil {
return err
}
// do something moreThere is a significant performance degradation due to some problems with the net/http.
Simple analysis:
The http.DefaultClient.Do call will eventually go to io.Copy.
func Copy(dst Writer, src Reader) (written int64, err error) {
return copyBuffer(dst, src, nil)
}In this case, dst is a *bufio.Writer, and src is an ioutil.nopCloser. And, in the copyBuffer, bufio.Writer.ReadFrom will be called. In bufio.Writer.ReadFrom, becasue some headers have been written to b.buf and not been Flush, the for loop will be executed. In this loop, it will only copy 4KiB to b.buf each time, resulting in useless memory copies and a large number of system calls, which in turn leads to performance degradation.
func (b *Writer) ReadFrom(r io.Reader) (n int64, err error) {
if b.err != nil {
return 0, b.err
}
if b.Buffered() == 0 { // b.Buffered() > 0, becasue some headers have been written here and not been `Flush`
if w, ok := b.wr.(io.ReaderFrom); ok {
n, err = w.ReadFrom(r)
b.err = err
return n, err
}
}
var m int
for {
if b.Available() == 0 {
if err1 := b.Flush(); err1 != nil {
return n, err1
}
}
nr := 0
for nr < maxConsecutiveEmptyReads {
m, err = r.Read(b.buf[b.n:])
if m != 0 || err != nil {
break
}
nr++
}
// ...
}
// ...
}Suggested fix
use
// src/net/http/transfer.go
func (t *transferWriter) unwrapBody() io.Reader {
if reflect.TypeOf(t.Body) == nopCloserType {
return reflect.ValueOf(t.Body).Field(0).Interface().(io.Reader)
}
if r, ok := t.Body.(*readTrackingBody); ok {
r.didRead = true
if reflect.TypeOf(r.ReadCloser) == nopCloserType {
return reflect.ValueOf(r.ReadCloser).Field(0).Interface().(io.Reader)
}
return r.ReadCloser
}
return t.Body
}instead of
func (t *transferWriter) unwrapBody() io.Reader {
if reflect.TypeOf(t.Body) == nopCloserType {
return reflect.ValueOf(t.Body).Field(0).Interface().(io.Reader)
}
if r, ok := t.Body.(*readTrackingBody); ok {
r.didRead = true
return r.ReadCloser
}
return t.Body
}.