goroutine leak in parseHTTP2

[cowsay1]

Hi
I found that the memory consumption of the app is growing with each request. My knowledge of Go is very basic, so I couldn't solve it.

	for {
		fmt.Println("NumGoroutine:", runtime.NumGoroutine())
		conn, err := listener.Accept()

I added a counter in the main loop and observed that the NumGoroutine count increases with each request without decreasing. This issue occurs only with HTTP/2 requests, so I suspect the problem lies with the "go parseHTTP2" frame-reader in the infinite "for" loop.

To address this, I tried sending a signal to close the loop, which seemed to resolve the issue initially (code below). The https://localhost/api/all endpoint now opens in curl and Chrome without increasing the goroutine count, but it doesn't open in Firefox. I think the high number of PRIORITY frames might be causing some issue in Firefox. I have tried a few other similar methods to terminate this function with goroutine, such as using channels and timeouts, but unfortunately, I'm stuck on it.

connection_handler.go

import (
	"bytes"
	"fmt"
	"log"
	"net"
	"strconv"
	"strings"
	"time"

	tls "github.com/wwhtrbbtt/utls"
	"golang.org/x/net/http2"
	"golang.org/x/net/http2/hpack"
)

const HTTP2_PREAMBLE = "PRI * HTTP/2.0\r\n\r\nSM\r\n\r\n"

func parseHTTP1(request []byte) Response {
	// Split the request into lines
	lines := strings.Split(string(request), "\r\n")

	// Split the first line into the method, path and http version
	firstLine := strings.Split(lines[0], " ")

	// Split the headers into an array
	var headers []string
	for _, line := range lines {
		if strings.Contains(line, ":") {
			headers = append(headers, line)
		}
	}

	if len(firstLine) != 3 {
		return Response{
			HTTPVersion: "--",
			Method:      "--",
			path:        "--",
		}
	}
	return Response{
		HTTPVersion: firstLine[2],
		path:        firstLine[1],
		Method:      firstLine[0],
		Http1: &Http1Details{
			Headers: headers,
		},
	}
}

func parseHTTP2(f *http2.Framer, c chan ParsedFrame, quit chan struct{}) {
	for {
		frame, err := f.ReadFrame()
		if err != nil {
			r := "ERROR_CLOSE"
			if strings.HasSuffix(err.Error(), "unknown certificate") {
				r = "ERROR"
			}
			// log.Println("Error reading frame", err, r)
			c <- ParsedFrame{Type: r}
			return
		}

		select {
		case <-quit:
			fmt.Println("parseHTTP2 quit")
			return
		default:
			p := ParsedFrame{}
			p.Type = frame.Header().Type.String()
			p.Stream = frame.Header().StreamID
			p.Length = frame.Header().Length
			p.Flags = GetAllFlags(frame)

			switch frame := frame.(type) {
			case *http2.SettingsFrame:
				p.Settings = []string{}
				frame.ForeachSetting(func(s http2.Setting) error {
					setting := fmt.Sprintf("%q", s)
					setting = strings.Replace(setting, "\"", "", -1)
					setting = strings.Replace(setting, "[", "", -1)
					setting = strings.Replace(setting, "]", "", -1)

					p.Settings = append(p.Settings, setting)
					return nil
				})
			case *http2.HeadersFrame:
				d := hpack.NewDecoder(4096, func(hf hpack.HeaderField) {})
				d.SetEmitEnabled(true)
				h2Headers, err := d.DecodeFull(frame.HeaderBlockFragment())
				if err != nil {
					//log.Println("Error decoding headers", err)
					return
				}

				for _, h := range h2Headers {
					h := fmt.Sprintf("%q: %q", h.Name, h.Value)
					h = strings.Trim(h, "\"")
					h = strings.Replace(h, "\": \"", ": ", -1)
					p.Headers = append(p.Headers, h)
				}
				if frame.HasPriority() {
					prio := Priority{}
					p.Priority = &prio
					// 6.2: Weight: An 8-bit weight for the stream; Add one to the value to obtain a weight between 1 and 256
					p.Priority.Weight = int(frame.Priority.Weight) + 1
					p.Priority.DependsOn = int(frame.Priority.StreamDep)
					if frame.Priority.Exclusive {
						p.Priority.Exclusive = 1
					}
				}
			case *http2.DataFrame:
				p.Payload = frame.Data()
			case *http2.WindowUpdateFrame:
				p.Increment = frame.Increment
			case *http2.PriorityFrame:

				prio := Priority{}
				p.Priority = &prio
				// 6.3: Weight: An 8-bit weight for the stream; Add one to the value to obtain a weight between 1 and 256
				p.Priority.Weight = int(frame.PriorityParam.Weight) + 1
				p.Priority.DependsOn = int(frame.PriorityParam.StreamDep)
				if frame.PriorityParam.Exclusive {
					p.Priority.Exclusive = 1
				}
			case *http2.GoAwayFrame:
				p.GoAway = &GoAway{}
				p.GoAway.LastStreamID = frame.LastStreamID
				p.GoAway.ErrCode = uint32(frame.ErrCode)
				p.GoAway.DebugData = frame.DebugData()
			}

			c <- p
		}
	}
}

func HandleTLSConnection(conn net.Conn) bool {
	// Read the first line of the request
	// We only read the first line to determine if the connection is HTTP1 or HTTP2
	// If we know that it isnt HTTP2, we can read the rest of the request and then start processing it
	// If we know that it is HTTP2, we start the HTTP2 handler

	l := len([]byte(HTTP2_PREAMBLE))
	request := make([]byte, l)

	_, err := conn.Read(request)
	if err != nil {
		//log.Println("Error reading request", err)
		if strings.HasSuffix(err.Error(), "unknown certificate") && local {
			log.Println("Local error (probably developement) - not closing conn")
			return true
		}
		return false
	}

	hs := conn.(*tls.Conn).ClientHello

	parsedClientHello := ParseClientHello(hs)
	JA3Data := CalculateJA3(parsedClientHello)
	peetfp, peetprintHash := CalculatePeetPrint(parsedClientHello, JA3Data)
	tlsDetails := TLSDetails{
		Ciphers:          JA3Data.ReadableCiphers,
		Extensions:       parsedClientHello.Extensions,
		RecordVersion:    JA3Data.Version,
		NegotiatedVesion: fmt.Sprintf("%v", conn.(*tls.Conn).ConnectionState().Version),
		JA3:              JA3Data.JA3,
		JA3Hash:          JA3Data.JA3Hash,
		PeetPrint:        peetfp,
		PeetPrintHash:    peetprintHash,
		SessionID:        parsedClientHello.SessionID,
		ClientRandom:     parsedClientHello.ClientRandom,
	}

	// Check if the first line is HTTP/2
	if string(request) == HTTP2_PREAMBLE {
		handleHTTP2(conn, tlsDetails)
	} else {
		// Read the rest of the request
		r2 := make([]byte, 1024-l)
		_, err := conn.Read(r2)
		if err != nil {
			log.Println(err)
			return true
		}
		// Append it to the first line
		request = append(request, r2...)

		// Parse and handle the request
		details := parseHTTP1(request)
		details.IP = conn.RemoteAddr().String()
		details.TLS = tlsDetails
		respondToHTTP1(conn, details)
	}
	return true
}

func respondToHTTP1(conn net.Conn, resp Response) {
	// log.Println("Request:", resp.ToJson())
	// log.Println(len(resp.ToJson()))

	res1, ctype := Router(resp.path, resp)

	res := "HTTP/1.1 200 OK\r\n"
	res += "Content-Length: " + fmt.Sprintf("%v\r\n", len(res1))
	res += "Content-Type: " + ctype + "; charset=utf-8\r\n"
	res += "Server: TrackMe\r\n"
	res += "\r\n"
	res += string(res1)
	res += "\r\n\r\n"

	_, err := conn.Write([]byte(res))
	if err != nil {
		log.Println("Error writing HTTP/1 data", err)
		return
	}
	err = conn.Close()
	if err != nil {
		log.Println("Error closing HTTP/1 connection", err)
		return
	}
}

// https://stackoverflow.com/questions/52002623/golang-tcp-server-how-to-write-http2-data
func handleHTTP2(conn net.Conn, tlsFingerprint TLSDetails) {
	// make a new framer to encode/decode frames
	fr := http2.NewFramer(conn, conn)
	c := make(chan ParsedFrame)
	var frames []ParsedFrame

	// Same settings that google uses
	err := fr.WriteSettings(
		http2.Setting{
			ID: http2.SettingInitialWindowSize, Val: 1048576,
		},
		http2.Setting{
			ID: http2.SettingMaxConcurrentStreams, Val: 100,
		},
		http2.Setting{
			ID: http2.SettingMaxHeaderListSize, Val: 65536,
		},
	)
	if err != nil {
		log.Println(err)
		return
	}

	var frame ParsedFrame
	var headerFrame ParsedFrame

	quit := make(chan struct{})
	go parseHTTP2(fr, c, quit)

	for {
		frame = <-c
		if frame.Type == "ERROR_CLOSE" {
			err = conn.Close()
			if err != nil {
				log.Println("Cant close connection", err)
			}
			return
		} else if frame.Type == "ERROR" {
			return
		}
		// log.Println(frame)
		frames = append(frames, frame)
		if frame.Type == "HEADERS" {
			headerFrame = frame
		}
		if len(frame.Flags) > 0 && frame.Flags[0] == "EndStream (0x1)" {
			quit <- struct{}{}
			break
		}
	}

	// get method, path and user-agent from the header frame
	var path string
	var method string
	var userAgent string

	for _, h := range headerFrame.Headers {
		if strings.HasPrefix(h, ":method") {
			method = strings.Split(h, ": ")[1]
		}
		if strings.HasPrefix(h, ":path") {
			path = strings.Split(h, ": ")[1]
		}
		if strings.HasPrefix(h, "user-agent") {
			userAgent = strings.Split(h, ": ")[1]
		}
	}

	resp := Response{
		IP:          conn.RemoteAddr().String(),
		HTTPVersion: "h2",
		path:        path,
		Method:      method,
		UserAgent:   userAgent,
		Http2: &Http2Details{
			SendFrames:            frames,
			AkamaiFingerprint:     GetAkamaiFingerprint(frames),
			AkamaiFingerprintHash: GetMD5Hash(GetAkamaiFingerprint(frames)),
		},
		TLS: tlsFingerprint,
	}

	res, ctype := Router(path, resp)

	// Prepare HEADERS
	hbuf := bytes.NewBuffer([]byte{})
	encoder := hpack.NewEncoder(hbuf)
	encoder.WriteField(hpack.HeaderField{Name: ":status", Value: "200"})
	encoder.WriteField(hpack.HeaderField{Name: "server", Value: "TrackMe.peet.ws"})
	encoder.WriteField(hpack.HeaderField{Name: "content-length", Value: strconv.Itoa(len(res))})
	encoder.WriteField(hpack.HeaderField{Name: "content-type", Value: ctype})

	// Write HEADERS frame
	err = fr.WriteHeaders(http2.HeadersFrameParam{StreamID: headerFrame.Stream, BlockFragment: hbuf.Bytes(), EndHeaders: true})
	if err != nil {
		log.Println("could not write headers: ", err)
		return
	}

	chunks := splitBytesIntoChunks(res, 1024)
	for _, c := range chunks {
		fr.WriteData(headerFrame.Stream, false, c)
	}
	fr.WriteData(headerFrame.Stream, true, []byte{})
	fr.WriteGoAway(headerFrame.Stream, http2.ErrCodeNo, []byte{})

	time.Sleep(time.Millisecond * 500)
	conn.Close()
}

[pagpeter]

Good find! Thanks a lot.
I will investigate the issue in the following days