[release-1.7] Cache the http client in order to reuse the connection (#…

…27983)

* manually cherry pick #27864

* Add comments

pilot/cmd/pilot-agent/status/server.go

@@ -99,6 +99,7 @@ type Server struct {
 	prometheus          *PrometheusScrapeConfiguration
 	mutex               sync.RWMutex
 	appKubeProbers      KubeAppProbers
+	appProbeClient      map[string]*http.Client
 	statusPort          uint16
 	lastProbeSuccessful bool
 	envoyStatsPort      int
@@ -136,6 +137,8 @@ func NewServer(config Config) (*Server, error) {
 	if err := json.Unmarshal([]byte(config.KubeAppProbers), &s.appKubeProbers); err != nil {
 		return nil, fmt.Errorf("failed to decode app prober err = %v, json string = %v", err, config.KubeAppProbers)
 	}
+
+	s.appProbeClient = make(map[string]*http.Client, len(s.appKubeProbers))
 	// Validate the map key matching the regex pattern.
 	for path, prober := range s.appKubeProbers {
 		if !appProberPattern.Match([]byte(path)) {
@@ -147,6 +150,18 @@ func NewServer(config Config) (*Server, error) {
 		if prober.HTTPGet.Port.Type != intstr.Int {
 			return nil, fmt.Errorf("invalid prober config for %v, the port must be int type", path)
 		}
+		// Construct a http client and cache it in order to reuse the connection.
+		// The Client's Transport typically has internal state (cached TCP
+		// connections), so Clients should be reused instead of created as
+		// needed. Clients are safe for concurrent use by multiple goroutines.
+		s.appProbeClient[path] = &http.Client{
+			Timeout: time.Duration(prober.TimeoutSeconds) * time.Second,
+			// We skip the verification since kubelet skips the verification for HTTPS prober as well
+			// https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-probes/#configure-probes
+			Transport: &http.Transport{
+				TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
+			},
+		}
 	}
 
 	// Enable prometheus server if its configured and a sidecar
@@ -414,16 +429,7 @@ func (s *Server) handleAppProbe(w http.ResponseWriter, req *http.Request) {
 		_, _ = w.Write([]byte(fmt.Sprintf("app prober config does not exists for %v", path)))
 		return
 	}
-
-	// Construct a request sent to the application.
-	httpClient := &http.Client{
-		Timeout: time.Duration(prober.TimeoutSeconds) * time.Second,
-		// We skip the verification since kubelet skips the verification for HTTPS prober as well
-		// https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-probes/#configure-probes
-		Transport: &http.Transport{
-			TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
-		},
-	}
+	httpClient := s.appProbeClient[path]
 	proberPath := prober.HTTPGet.Path
 	if !strings.HasPrefix(proberPath, "/") {
 		proberPath = "/" + proberPath

pilot/cmd/pilot-agent/status/server_test.go

@@ -16,6 +16,7 @@ package status
 
 import (
 	"context"
+	"encoding/json"
 	"fmt"
 	"net"
 	"net/http"
@@ -342,7 +343,6 @@ func TestAppProbe(t *testing.T) {
 			},
 		},
 	}
-	_ = simpleConfig
 	testCases := []struct {
 		probePath  string
 		config     KubeAppProbers
@@ -406,10 +406,16 @@ func TestAppProbe(t *testing.T) {
 	}
 	for _, tc := range testCases {
 		t.Run(tc.probePath, func(t *testing.T) {
+			appProber, err := json.Marshal(tc.config)
+			if err != nil {
+				t.Fatalf("invalid app probers")
+			}
+			config := Config{
+				StatusPort:     0,
+				KubeAppProbers: string(appProber),
+			}
 			// Starts the pilot agent status server.
-			server, err := NewServer(Config{
-				StatusPort: 0,
-			})
+			server, err := NewServer(config)
 			if err != nil {
 				t.Fatalf("failed to create status server %v", err)
 			}

releasenotes/notes/27726.yaml

@@ -0,0 +1,8 @@
+apiVersion: release-notes/v2
+kind: bug-fix
+area: networking
+issue:
+  - 27726
+releaseNotes:
+  - |
+    **Fixed** pilot agent app probe connection leak.