crypto/tls: Conn.rawInput buffer has no chance to shrink on large number of idle conns

[cch123]

What version of Go are you using (go version)?

go version go1.14.12 linux/amd64

Does this issue reproduce with the latest release?

yes

What did you do?

Open TLS on HTTP server.

TLS HTTP server:

package main

import (
	"bufio"
	"crypto/tls"
	"fmt"
	"io/ioutil"
	"net"
	"net/http"
	"time"

	_ "net/http/pprof"
)

func init() {
	go http.ListenAndServe(":9999", nil)
}

func main() {
	l, err := net.Listen("tcp4", ":1234")
	if err != nil {
		fmt.Println(err)
		return
	}

	cert, err := tls.LoadX509KeyPair("server.crt", "server.key")
	if err != nil {
		fmt.Println(err)
		return
	}

	for {
		c, err := l.Accept()
		if err != nil {
			return
		}

		go func() {
			c = tls.Server(c, &tls.Config{
				Certificates: []tls.Certificate{cert}, InsecureSkipVerify: true,
			})

			if err != nil {
				fmt.Println(err)
				return
			}

			r := bufio.NewReader(c)
			for {
				c.SetReadDeadline(time.Now().Add(time.Second * 5))
				req, err := http.ReadRequest(r)
				if err != nil {
					if e, ok := err.(net.Error); ok && e.Timeout() {
						continue
					}
					c.Close()
					return
				}

				_, err = ioutil.ReadAll(req.Body)
				if err != nil {
					fmt.Println(err)
					return
				}

				// write respose
				resp := &http.Response{ProtoMajor: 1, ProtoMinor: 1, StatusCode: http.StatusOK, Header: http.Header{}, Body: http.NoBody}
				err = resp.Write(c)
				if err != nil {
					fmt.Println(err)
					c.Close()
					return
				}
			}
		}()
	}
}

TLS HTTP client.

package main

import (
	"crypto/tls"
	"fmt"
	"bytes"
	"io/ioutil"
	"net/http"
	"os"
	"strconv"
	"sync"

	_ "net/http/pprof"

	"go.uber.org/ratelimit"
)

func init() {
	go http.ListenAndServe(":19999", nil)
}

func main() {
	url := os.Args[3]
	connNum, err := strconv.ParseInt(os.Args[1], 10, 64)
	if err != nil {
		fmt.Println(err)
		return
	}

	qps, err := strconv.ParseInt(os.Args[2], 10, 64)
	if err != nil {
		fmt.Println(err)
		return
	}

	bucket := ratelimit.New(int(qps))

	var l sync.Mutex
	connList := make([]*http.Client, connNum)

	for i := 0; ; i++ {
		bucket.Take()
		i := i
		go func() {
			l.Lock()
			if connList[i%len(connList)] == nil {
				connList[i%len(connList)] = &http.Client{
					Transport: &http.Transport{
						TLSClientConfig:     &tls.Config{InsecureSkipVerify: true},
						IdleConnTimeout:     0,
						MaxIdleConns:        1,
						MaxIdleConnsPerHost: 1,
					},
				}
			}
			conn := connList[i%len(connList)]
			l.Unlock()
			if resp, e := conn.Post(url, "application/json", bytes.NewReader(make([]byte, 16000))); e != nil {
				fmt.Println(e)
			} else {
				defer resp.Body.Close()
				ioutil.ReadAll(resp.Body)
			}
		}()
	}

}

./client 40000 1000 https://ip:1234/

This TLS HTTP server costs about 2.2GB RSS.

To reduce the memory cost, we shrink the TLS read buffer when read timeout:

TLS Read function:		
....
if e, ok := err.(net.Error); ok && e.Timeout() {
    if Conn.rawInput.Len() == 0 && Conn.input.Len() == 0 && Conn.hand.Len() == 0 {
        c.rawInput = *bytes.NewBuffer(make([]byte, 0, bytes.MinRead))
    }
}
....

And the memory usage decreased from 2.2GB to around 560 MB

But I don't know whether this is a proper fix.

If it is. I'm happy to open a PR for this

[toothrot]

/cc @FiloSottile

[FiloSottile]

I assume this is after the #42035 fix?

[cch123]

I assume this is after the #42035 fix?

Yes

[jvilhuber]

I'm very interested in this topic as well. Any progress on this issue and the linked PR?

[cch123]

I'm very interested in this topic as well. Any progress on this issue and the linked PR?

I don't have a good enough server to test this PR, maybe the bench data is not convincing

[jkralik]

@FiloSottile Is there any news? Because I have the same issue with golang 1.17.3 .

[gopherbot]

Change https://golang.org/cl/370581 mentions this issue: crypto/tls: shrink tls Conn's rawInput with a small buffer

[jvilhuber]

@FiloSottile Any update?

[JellyZero]

@FiloSottile ？

[mitar]

It seemsbytes.Buffer do not have some way to shrink itself?