GODRIVER-2050 - Add settable http.Clients and gracefully close provid…

…ed defaults (#1093)

Co-authored-by: Benjamin Rewis <32186188+benjirewis@users.noreply.github.com>
Co-authored-by: Matt Dale <9760375+matthewdale@users.noreply.github.com>

internal/http.go

@@ -0,0 +1,38 @@
+// Copyright (C) MongoDB, Inc. 2022-present.
+//
+// Licensed under the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License. You may obtain
+// a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+
+package internal // import "go.mongodb.org/mongo-driver/internal"
+
+import (
+	"net/http"
+	"time"
+)
+
+// DefaultHTTPClient is the default HTTP client used across the driver.
+var DefaultHTTPClient = &http.Client{
+	// TODO(GODRIVER-2623): Use "http.DefaultTransport.Clone" once we change the minimum supported Go version to 1.13.
+	Transport: &http.Transport{
+		Proxy:                 http.ProxyFromEnvironment,
+		MaxIdleConns:          100,
+		IdleConnTimeout:       90 * time.Second,
+		TLSHandshakeTimeout:   10 * time.Second,
+		ExpectContinueTimeout: 1 * time.Second,
+	},
+}
+
+// CloseIdleHTTPConnections closes any connections which were previously
+// connected from previous requests but are now sitting idle in
+// a "keep-alive" state. It does not interrupt any connections currently
+// in use.
+// Borrowed from go standard library.
+func CloseIdleHTTPConnections(client *http.Client) {
+	type closeIdler interface {
+		CloseIdleConnections()
+	}
+	if tr, ok := client.Transport.(closeIdler); ok {
+		tr.CloseIdleConnections()
+	}
+}

mongo/client.go

@@ -10,11 +10,13 @@ import (
 	"context"
 	"errors"
 	"fmt"
+	"net/http"
 	"time"
 
 	"go.mongodb.org/mongo-driver/bson"
 	"go.mongodb.org/mongo-driver/bson/bsoncodec"
 	"go.mongodb.org/mongo-driver/event"
+	"go.mongodb.org/mongo-driver/internal"
 	"go.mongodb.org/mongo-driver/internal/uuid"
 	"go.mongodb.org/mongo-driver/mongo/description"
 	"go.mongodb.org/mongo-driver/mongo/options"
@@ -64,6 +66,7 @@ type Client struct {
 	serverMonitor  *event.ServerMonitor
 	sessionPool    *session.Pool
 	timeout        *time.Duration
+	httpClient     *http.Client
 
 	// client-side encryption fields
 	keyVaultClientFLE  *Client
@@ -173,6 +176,7 @@ func NewClient(opts ...*options.ClientOptions) (*Client, error) {
 	}
 	// Timeout
 	client.timeout = clientOpt.Timeout
+	client.httpClient = clientOpt.HTTPClient
 	// WriteConcern
 	if clientOpt.WriteConcern != nil {
 		client.writeConcern = clientOpt.WriteConcern
@@ -307,6 +311,11 @@ func (c *Client) Disconnect(ctx context.Context) error {
 	if disconnector, ok := c.deployment.(driver.Disconnector); ok {
 		return replaceErrors(disconnector.Disconnect(ctx))
 	}
+
+	if c.httpClient == internal.DefaultHTTPClient {
+		internal.CloseIdleHTTPConnections(c.httpClient)
+	}
+
 	return nil
 }
 
@@ -600,6 +609,7 @@ func (c *Client) configureCryptFLE(mc *mongocrypt.MongoCrypt, opts *options.Auto
 		KeyFn:                kr.cryptKeys,
 		MarkFn:               c.mongocryptdFLE.markCommand,
 		TLSConfig:            opts.TLSConfig,
+		HTTPClient:           opts.HTTPClient,
 		BypassAutoEncryption: bypass,
 	})
 }

mongo/client_encryption.go

@@ -66,6 +66,7 @@ func NewClientEncryption(keyVaultClient *Client, opts ...*options.ClientEncrypti
 		KeyFn:      kr.cryptKeys,
 		CollInfoFn: cir.cryptCollInfo,
 		TLSConfig:  ceo.TLSConfig,
+		HTTPClient: ceo.HTTPClient,
 	})
 
 	return ce, nil

mongo/options/autoencryptionoptions.go

@@ -8,6 +8,9 @@ package options
 
 import (
 	"crypto/tls"
+	"net/http"
+
+	"go.mongodb.org/mongo-driver/internal"
 )
 
 // AutoEncryptionOptions represents options used to configure auto encryption/decryption behavior for a mongo.Client
@@ -32,13 +35,16 @@ type AutoEncryptionOptions struct {
 	BypassAutoEncryption  *bool
 	ExtraOptions          map[string]interface{}
 	TLSConfig             map[string]*tls.Config
+	HTTPClient            *http.Client
 	EncryptedFieldsMap    map[string]interface{}
 	BypassQueryAnalysis   *bool
 }
 
 // AutoEncryption creates a new AutoEncryptionOptions configured with default values.
 func AutoEncryption() *AutoEncryptionOptions {
-	return &AutoEncryptionOptions{}
+	return &AutoEncryptionOptions{
+		HTTPClient: internal.DefaultHTTPClient,
+	}
 }
 
 // SetKeyVaultClientOptions specifies options for the client used to communicate with the key vault collection.
@@ -194,6 +200,9 @@ func MergeAutoEncryptionOptions(opts ...*AutoEncryptionOptions) *AutoEncryptionO
 		if opt.BypassQueryAnalysis != nil {
 			aeo.BypassQueryAnalysis = opt.BypassQueryAnalysis
 		}
+		if opt.HTTPClient != nil {
+			aeo.HTTPClient = opt.HTTPClient
+		}
 	}
 
 	return aeo

mongo/options/clientencryptionoptions.go

@@ -9,18 +9,24 @@ package options
 import (
 	"crypto/tls"
 	"fmt"
+	"net/http"
+
+	"go.mongodb.org/mongo-driver/internal"
 )
 
 // ClientEncryptionOptions represents all possible options used to configure a ClientEncryption instance.
 type ClientEncryptionOptions struct {
 	KeyVaultNamespace string
 	KmsProviders      map[string]map[string]interface{}
 	TLSConfig         map[string]*tls.Config
+	HTTPClient        *http.Client
 }
 
 // ClientEncryption creates a new ClientEncryptionOptions instance.
 func ClientEncryption() *ClientEncryptionOptions {
-	return &ClientEncryptionOptions{}
+	return &ClientEncryptionOptions{
+		HTTPClient: internal.DefaultHTTPClient,
+	}
 }
 
 // SetKeyVaultNamespace specifies the namespace of the key vault collection. This is required.
@@ -132,6 +138,9 @@ func MergeClientEncryptionOptions(opts ...*ClientEncryptionOptions) *ClientEncry
 		if opt.TLSConfig != nil {
 			ceo.TLSConfig = opt.TLSConfig
 		}
+		if opt.HTTPClient != nil {
+			ceo.HTTPClient = opt.HTTPClient
+		}
 	}
 
 	return ceo

mongo/options/clientoptions.go

@@ -16,6 +16,7 @@ import (
 	"fmt"
 	"io/ioutil"
 	"net"
+	"net/http"
 	"strings"
 	"time"
 
@@ -104,6 +105,7 @@ type ClientOptions struct {
 	DisableOCSPEndpointCheck *bool
 	HeartbeatInterval        *time.Duration
 	Hosts                    []string
+	HTTPClient               *http.Client
 	LoadBalanced             *bool
 	LocalThreshold           *time.Duration
 	MaxConnIdleTime          *time.Duration
@@ -162,7 +164,9 @@ type ClientOptions struct {
 
 // Client creates a new ClientOptions instance.
 func Client() *ClientOptions {
-	return new(ClientOptions)
+	return &ClientOptions{
+		HTTPClient: internal.DefaultHTTPClient,
+	}
 }
 
 // Validate validates the client options. This method will return the first error found.
@@ -767,6 +771,14 @@ func (c *ClientOptions) SetTLSConfig(cfg *tls.Config) *ClientOptions {
 	return c
 }
 
+// SetHTTPClient specifies the http.Client to be used for any HTTP requests.
+//
+// This should only be used to set custom HTTP client configurations. By default, the connection will use an internal.DefaultHTTPClient.
+func (c *ClientOptions) SetHTTPClient(client *http.Client) *ClientOptions {
+	c.HTTPClient = client
+	return c
+}
+
 // SetWriteConcern specifies the write concern to use to for write operations. This can also be set through the following
 // URI options:
 //
@@ -889,6 +901,9 @@ func MergeClientOptions(opts ...*ClientOptions) *ClientOptions {
 		if len(opt.Hosts) > 0 {
 			c.Hosts = opt.Hosts
 		}
+		if opt.HTTPClient != nil {
+			c.HTTPClient = opt.HTTPClient
+		}
 		if opt.LoadBalanced != nil {
 			c.LoadBalanced = opt.LoadBalanced
 		}