From 0ab393fa6f59099cde0fd15a4eab8510ee9b3b37 Mon Sep 17 00:00:00 2001 From: Anthony Seure Date: Fri, 3 Jul 2020 15:14:15 +0200 Subject: [PATCH] fix(transport): prevent memory leak when many clients are instantiated by multiple goroutines After the report from @asurak about a potential memory leak within the Go API client, we've been able to reproduce the issue by starting new `search.Client` instances in the same program from many goroutines (~50 or higher) and performing simple search queries to the engine over the network. Under these conditions, the program was seeing its system memory increased linearly, without any memory being released to the OS by the GC. When performing the search queries with hardcoded requests using the Go standard library's `http.DefaultClient`, the issue disappeared. After profiling the memory usage (using `pprof`), we've noticed that our top memory usage was due to many invocations of `bytes.MakeSlice` made by the `*http.Client.Transport` instance that we create for each new instance of the Go API client. After some research, those `bytes.MakeSlice` calls are the consequence of `tls.Conn.readHandshake` calls being performed when the `http.Transport` needs to establish a new connection or renegociate one with a server. The allocated memory of the `bytes.MakeSlice` invocations not being released when used by multiple goroutines is a known issue, as we can see from other related discussions: - https://forum.golangbridge.org/t/bytes-makeslice-in-goroutines-creates-abnormal-memory-usage/2038 - https://topic.alibabacloud.com/a/golang-memory-growing-bytesmakeslice_8_8_31497889.html - https://github.com/gorilla/websocket/issues/134 After comparing how we instantiate our `http.Transport` instance vs. how the `http.DefaultClient` instantiates its own `http.Transport`, we've noticed that the latter uses a singleton instance whereas the Go API client was creating a new instance of its default `http.Transport` even though it was the same for all API client instances. This commit reproduces the behavior of the Go standard library's `http.DefaultClient` by only instantiating a single default `http.Transport` instance for all API client instances. Finally, here are some before/after comparison benchmarks as reported by runtime.MemStats: ``` // Memory consumption using `http.DefaultClient` Alloc = 3 MiB TotalAlloc = 865 MiB Sys = 71 MiB NumGC = 470 Alloc = 2 MiB TotalAlloc = 1807 MiB Sys = 71 MiB NumGC = 977 Alloc = 3 MiB TotalAlloc = 2787 MiB Sys = 71 MiB NumGC = 1514 Alloc = 3 MiB TotalAlloc = 3722 MiB Sys = 71 MiB NumGC = 2008 Alloc = 4 MiB TotalAlloc = 4611 MiB Sys = 71 MiB NumGC = 2453 Alloc = 4 MiB TotalAlloc = 5548 MiB Sys = 71 MiB NumGC = 2959 Alloc = 3 MiB TotalAlloc = 6395 MiB Sys = 71 MiB NumGC = 3387 Alloc = 2 MiB TotalAlloc = 7269 MiB Sys = 71 MiB NumGC = 3867 Alloc = 3 MiB TotalAlloc = 8198 MiB Sys = 71 MiB NumGC = 4356 Alloc = 2 MiB TotalAlloc = 9087 MiB Sys = 71 MiB NumGC = 4806 Alloc = 6 MiB TotalAlloc = 10006 MiB Sys = 71 MiB NumGC = 5271 Alloc = 3 MiB TotalAlloc = 10887 MiB Sys = 71 MiB NumGC = 5743 Alloc = 4 MiB TotalAlloc = 11740 MiB Sys = 71 MiB NumGC = 6182 Alloc = 3 MiB TotalAlloc = 12601 MiB Sys = 71 MiB NumGC = 6644 Alloc = 3 MiB TotalAlloc = 13414 MiB Sys = 71 MiB NumGC = 7061 Alloc = 4 MiB TotalAlloc = 14343 MiB Sys = 71 MiB NumGC = 7528 Alloc = 3 MiB TotalAlloc = 15245 MiB Sys = 71 MiB NumGC = 8002 Alloc = 3 MiB TotalAlloc = 16144 MiB Sys = 71 MiB NumGC = 8481 Alloc = 3 MiB TotalAlloc = 17046 MiB Sys = 71 MiB NumGC = 8941 Alloc = 4 MiB TotalAlloc = 17980 MiB Sys = 71 MiB NumGC = 9441 // Memory consumption using Go API client before the fix Alloc = 27 MiB TotalAlloc = 1077 MiB Sys = 71 MiB NumGC = 136 Alloc = 59 MiB TotalAlloc = 2171 MiB Sys = 137 MiB NumGC = 173 Alloc = 95 MiB TotalAlloc = 3265 MiB Sys = 205 MiB NumGC = 195 Alloc = 113 MiB TotalAlloc = 4363 MiB Sys = 206 MiB NumGC = 211 Alloc = 181 MiB TotalAlloc = 5476 MiB Sys = 272 MiB NumGC = 223 Alloc = 216 MiB TotalAlloc = 6578 MiB Sys = 340 MiB NumGC = 233 Alloc = 162 MiB TotalAlloc = 7667 MiB Sys = 340 MiB NumGC = 242 Alloc = 231 MiB TotalAlloc = 8753 MiB Sys = 408 MiB NumGC = 249 Alloc = 340 MiB TotalAlloc = 9849 MiB Sys = 474 MiB NumGC = 255 Alloc = 350 MiB TotalAlloc = 10960 MiB Sys = 475 MiB NumGC = 261 Alloc = 433 MiB TotalAlloc = 12051 MiB Sys = 542 MiB NumGC = 266 Alloc = 417 MiB TotalAlloc = 13148 MiB Sys = 610 MiB NumGC = 271 Alloc = 289 MiB TotalAlloc = 14230 MiB Sys = 610 MiB NumGC = 276 Alloc = 324 MiB TotalAlloc = 15312 MiB Sys = 677 MiB NumGC = 280 Alloc = 570 MiB TotalAlloc = 16400 MiB Sys = 744 MiB NumGC = 283 Alloc = 465 MiB TotalAlloc = 17499 MiB Sys = 811 MiB NumGC = 287 Alloc = 613 MiB TotalAlloc = 18601 MiB Sys = 812 MiB NumGC = 290 Alloc = 707 MiB TotalAlloc = 19710 MiB Sys = 879 MiB NumGC = 293 Alloc = 713 MiB TotalAlloc = 20789 MiB Sys = 880 MiB NumGC = 296 Alloc = 708 MiB TotalAlloc = 21872 MiB Sys = 880 MiB NumGC = 299 // Memory consumption using Go API client after the fix Alloc = 5 MiB TotalAlloc = 143 MiB Sys = 71 MiB NumGC = 40 Alloc = 5 MiB TotalAlloc = 275 MiB Sys = 71 MiB NumGC = 75 Alloc = 5 MiB TotalAlloc = 408 MiB Sys = 71 MiB NumGC = 109 Alloc = 4 MiB TotalAlloc = 541 MiB Sys = 71 MiB NumGC = 146 Alloc = 5 MiB TotalAlloc = 660 MiB Sys = 71 MiB NumGC = 179 Alloc = 6 MiB TotalAlloc = 793 MiB Sys = 71 MiB NumGC = 213 Alloc = 7 MiB TotalAlloc = 925 MiB Sys = 71 MiB NumGC = 249 Alloc = 4 MiB TotalAlloc = 1058 MiB Sys = 71 MiB NumGC = 285 Alloc = 6 MiB TotalAlloc = 1191 MiB Sys = 71 MiB NumGC = 320 Alloc = 5 MiB TotalAlloc = 1323 MiB Sys = 71 MiB NumGC = 355 Alloc = 6 MiB TotalAlloc = 1456 MiB Sys = 71 MiB NumGC = 390 Alloc = 6 MiB TotalAlloc = 1576 MiB Sys = 71 MiB NumGC = 422 Alloc = 5 MiB TotalAlloc = 1708 MiB Sys = 71 MiB NumGC = 459 Alloc = 5 MiB TotalAlloc = 1841 MiB Sys = 71 MiB NumGC = 495 Alloc = 6 MiB TotalAlloc = 1973 MiB Sys = 71 MiB NumGC = 531 Alloc = 6 MiB TotalAlloc = 2106 MiB Sys = 71 MiB NumGC = 567 Alloc = 7 MiB TotalAlloc = 2239 MiB Sys = 71 MiB NumGC = 602 Alloc = 4 MiB TotalAlloc = 2358 MiB Sys = 71 MiB NumGC = 636 Alloc = 4 MiB TotalAlloc = 2491 MiB Sys = 71 MiB NumGC = 673 Alloc = 5 MiB TotalAlloc = 2624 MiB Sys = 71 MiB NumGC = 708 ``` --- algolia/transport/requester.go | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/algolia/transport/requester.go b/algolia/transport/requester.go index 95c904d62..0060df23e 100644 --- a/algolia/transport/requester.go +++ b/algolia/transport/requester.go @@ -24,19 +24,21 @@ const ( // pass it to an HTTP interceptor. func DefaultHTTPClient() *http.Client { return &http.Client{ - Transport: &http.Transport{ - Dial: (&net.Dialer{ - KeepAlive: DefaultKeepAliveDuration, - Timeout: DefaultConnectTimeout, - }).Dial, - DisableKeepAlives: false, - MaxIdleConnsPerHost: DefaultMaxIdleConnsPerHost, - Proxy: http.ProxyFromEnvironment, - TLSHandshakeTimeout: DefaultTLSHandshakeTimeout, - }, + Transport: defaultTransport, } } +var defaultTransport http.RoundTripper = &http.Transport{ + Dial: (&net.Dialer{ + KeepAlive: DefaultKeepAliveDuration, + Timeout: DefaultConnectTimeout, + }).Dial, + DisableKeepAlives: false, + MaxIdleConnsPerHost: DefaultMaxIdleConnsPerHost, + Proxy: http.ProxyFromEnvironment, + TLSHandshakeTimeout: DefaultTLSHandshakeTimeout, +} + type Requester interface { Request(req *http.Request) (*http.Response, error) }