CA file

.github/workflows/go.yml

@@ -65,12 +65,19 @@ jobs:
         with:
           go-version: "${{env.GO_VERSION}}"
 
+      - name: Generate certificates
+        run: make -C e2e/certs certs
+
       - name: Build Docker image
         run: make update-devel-image
 
       - name: Run e2e test
-        run: cd e2e && make run-e2e
+        run: make -C e2e run-e2e
+
+      - name: Docker Compose file
+        if: failure()
+        run: cat e2e/docker-compose.yaml
 
-      - name: Dump Logs
+      - name: Docker Logs
         if: failure()
-        run: cd e2e && make dump-logs
+        run: make -C e2e dump-logs

.golangci.yml

@@ -88,6 +88,7 @@ issues:
       linters:
         - bodyclose
         - funlen
+        - gochecknoglobals
         - gocognit
         - gomnd
         - gosec

bind/flag.go

@@ -46,7 +46,8 @@ func PAC(fs *pflag.FlagSet, pac **url.URL) {
 	fs.VarP(anyflag.NewValue[*url.URL](*pac, pac, fileurl.ParseFilePathOrURL),
 		"pac", "p", "<path or URL>"+
 			"Proxy Auto-Configuration file to use for upstream proxy selection. "+
-			"It can be a local file or a URL, you can also use '-' to read from stdin. ")
+			"It can be a local file or a URL, you can also use '-' to read from stdin. "+
+			"The data URI scheme is supported, the format is data:base64,<encoded data>. ")
 }
 
 func RequestHeaders(fs *pflag.FlagSet, headers *[]header.Header) {
@@ -109,9 +110,7 @@ func HTTPTransportConfig(fs *pflag.FlagSet, cfg *forwarder.HTTPTransportConfig)
 		"The maximum amount of time a dial will wait for a connect to complete. "+
 			"With or without a timeout, the operating system may impose its own earlier timeout. For instance, TCP timeouts are often around 3 minutes. ")
 
-	fs.DurationVar(&cfg.TLSHandshakeTimeout,
-		"http-tls-handshake-timeout", cfg.TLSHandshakeTimeout,
-		"The maximum amount of time waiting to wait for a TLS handshake. Zero means no limit.")
+	TLSClientConfig(fs, &cfg.TLSClientConfig)
 
 	fs.DurationVar(&cfg.IdleConnTimeout,
 		"http-idle-conn-timeout", cfg.IdleConnTimeout,
@@ -123,10 +122,23 @@ func HTTPTransportConfig(fs *pflag.FlagSet, cfg *forwarder.HTTPTransportConfig)
 		"The amount of time to wait for a server's response headers after fully writing the request (including its body, if any)."+
 			"This time does not include the time to read the response body. "+
 			"Zero means no limit. ")
+}
 
-	fs.BoolVar(&cfg.TLSConfig.InsecureSkipVerify, "insecure", cfg.TLSConfig.InsecureSkipVerify,
+func TLSClientConfig(fs *pflag.FlagSet, cfg *forwarder.TLSClientConfig) {
+	fs.DurationVar(&cfg.HandshakeTimeout,
+		"http-tls-handshake-timeout", cfg.HandshakeTimeout,
+		"The maximum amount of time waiting to wait for a TLS handshake. Zero means no limit.")
+
+	fs.BoolVar(&cfg.InsecureSkipVerify, "insecure", cfg.InsecureSkipVerify,
 		"Don't verify the server's certificate chain and host name. "+
 			"Enable to work with self-signed certificates. ")
+
+	fs.StringSliceVar(&cfg.CAFiles,
+		"cacert-file", cfg.CAFiles, "<path or base64>"+
+			"Add your own CA certificates to verify against. "+
+			"The system root certificates will be used in addition to any certificates in this list. "+
+			"Can be a path to a file or \"data:\" followed by base64 encoded certificate. "+
+			"Use this flag multiple times to specify multiple CA certificate files. ")
 }
 
 func HTTPServerConfig(fs *pflag.FlagSet, cfg *forwarder.HTTPServerConfig, prefix string, schemes ...forwarder.Scheme) {
@@ -167,13 +179,7 @@ func HTTPServerConfig(fs *pflag.FlagSet, cfg *forwarder.HTTPServerConfig, prefix
 				"For https and h2 protocols, if TLS certificate is not specified, "+
 				"the server will use a self-signed certificate. ")
 
-		fs.StringVar(&cfg.CertFile,
-			namePrefix+"tls-cert-file", cfg.CertFile, "<path>"+
-				"TLS certificate to use if the server protocol is https or h2. ")
-
-		fs.StringVar(&cfg.KeyFile,
-			namePrefix+"tls-key-file", cfg.KeyFile, "<path>"+
-				"TLS private key to use if the server protocol is https or h2. ")
+		TLSServerConfig(fs, &cfg.TLSServerConfig, namePrefix)
 	}
 
 	fs.DurationVar(&cfg.ReadHeaderTimeout,
@@ -200,6 +206,18 @@ func HTTPServerConfig(fs *pflag.FlagSet, cfg *forwarder.HTTPServerConfig, prefix
 			"Setting this to none disables logging. ")
 }
 
+func TLSServerConfig(fs *pflag.FlagSet, cfg *forwarder.TLSServerConfig, namePrefix string) {
+	fs.StringVar(&cfg.CertFile,
+		namePrefix+"cert-file", cfg.CertFile, "<path or base64>"+
+			"TLS certificate to use if the server protocol is https or h2. "+
+			"Can be a path to a file or \"data:\" followed by base64 encoded certificate. ")
+
+	fs.StringVar(&cfg.KeyFile,
+		namePrefix+"key-file", cfg.KeyFile, "<path or base64>"+
+			"TLS private key to use if the server protocol is https or h2. "+
+			"Can be a path to a file or \"data:\" followed by base64 encoded key. ")
+}
+
 func LogConfig(fs *pflag.FlagSet, cfg *log.Config) {
 	fs.VarP(anyflag.NewValue[*os.File](nil, &cfg.File,
 		forwarder.OpenFileParser(os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0o600, 0o700)),

cmd/forwarder/pac/eval/eval.go

@@ -34,9 +34,12 @@ func (c *command) RunE(cmd *cobra.Command, args []string) error {
 		}
 		resolver = r
 	}
-	t := forwarder.NewHTTPTransport(c.httpTransportConfig, resolver)
+	t, err := forwarder.NewHTTPTransport(c.httpTransportConfig, resolver)
+	if err != nil {
+		return err
+	}
 
-	script, err := forwarder.ReadURL(c.pac, t)
+	script, err := forwarder.ReadURLString(c.pac, t)
 	if err != nil {
 		return fmt.Errorf("read PAC file: %w", err)
 	}

cmd/forwarder/pac/server/server.go

@@ -36,9 +36,12 @@ func (c *command) RunE(cmd *cobra.Command, args []string) error {
 	logger := stdlog.New(c.logConfig)
 	logger.Debugf("configuration\n%s", config)
 
-	t := forwarder.NewHTTPTransport(c.httpTransportConfig, nil)
+	t, err := forwarder.NewHTTPTransport(c.httpTransportConfig, nil)
+	if err != nil {
+		return err
+	}
 
-	script, err := forwarder.ReadURL(c.pac, t)
+	script, err := forwarder.ReadURLString(c.pac, t)
 	if err != nil {
 		return fmt.Errorf("read PAC file: %w", err)
 	}

cmd/forwarder/root.go

@@ -56,8 +56,12 @@ func rootCommand() *cobra.Command {
 			Prefix: []string{"dns"},
 		},
 		{
-			Name:   "HTTP client options",
-			Prefix: []string{"http", "insecure"},
+			Name: "HTTP client options",
+			Prefix: []string{
+				"http",
+				"cacert-file",
+				"insecure",
+			},
 		},
 		{
 			Name:   "Logging options",

cmd/forwarder/run/run.go

@@ -71,12 +71,16 @@ func (c *command) RunE(cmd *cobra.Command, args []string) error {
 			}
 			resolver = r
 		}
-		rt = forwarder.NewHTTPTransport(c.httpTransportConfig, resolver)
+		var err error
+		rt, err = forwarder.NewHTTPTransport(c.httpTransportConfig, resolver)
+		if err != nil {
+			return err
+		}
 	}
 
 	if c.pac != nil {
 		var err error
-		script, err = forwarder.ReadURL(c.pac, rt)
+		script, err = forwarder.ReadURLString(c.pac, rt)
 		if err != nil {
 			return fmt.Errorf("read PAC file: %w", err)
 		}

e2e/README.md

@@ -2,8 +2,9 @@
 
 ## Running the e2e tests with the test runner
 
-1. Build the forwarder image `make -C ../ update-devel-image`
-1. Start the test runner `make run-e2e`
+1. Generate certificates `make -C certs certs`, this can be done once
+1. Build the forwarder image `make -C ../ update-devel-image`, this needs to be done after each forwarder code change
+2. Start the test runner `make run-e2e`
 1. The test runner will run all the tests sequentially and output the results to the console
 1. If one of the test fails, the procedure will stop, test output will be printed
 1. Environment will not be pruned once the error occurred, remember to manually clean it up with `make down`

e2e/certs/.gitignore

@@ -0,0 +1,3 @@
+ca.srl
+*.crt
+*.key

e2e/certs/Makefile

@@ -0,0 +1,7 @@
+.PHONY: certs
+certs:
+	@./gen.sh
+
+.PHONY: test
+test:
+	@go test -v -tags manual .

e2e/certs/cert_test.go

@@ -0,0 +1,52 @@
+//go:build manual
+
+package certs_test
+
+import (
+	"crypto/tls"
+	"net/http"
+	"testing"
+
+	"github.com/saucelabs/forwarder"
+)
+
+func TestCertificate(t *testing.T) {
+	server := http.Server{
+		Addr: "127.0.0.1:8443",
+		Handler: http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+			w.Write([]byte("Hello, world!"))
+		}),
+	}
+	defer server.Close()
+
+	go server.ListenAndServeTLS("httpbin.crt", "httpbin.key")
+
+	tlsCfg := &tls.Config{
+		ServerName: "httpbin",
+	}
+	cfg := forwarder.TLSClientConfig{
+		CAFiles: []string{
+			"./ca.crt",
+		},
+	}
+	cfg.ConfigureTLSConfig(tlsCfg)
+
+	tr := http.DefaultTransport.(*http.Transport).Clone()
+	tr.TLSClientConfig = tlsCfg
+
+	req, err := http.NewRequest("GET", "https://"+server.Addr, http.NoBody)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	res, err := tr.RoundTrip(req)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if res.StatusCode != http.StatusOK {
+		t.Fatal("unexpected status code:", res.StatusCode)
+	}
+
+	tr.CloseIdleConnections()
+}

e2e/certs/gen.sh

@@ -0,0 +1,60 @@
+#!/usr/bin/env bash
+
+set -eu -o pipefail
+
+# Common variables
+CA_ORGANIZATION="Sauce Labs Inc."
+CA_KEY="ca.key"
+CA_CERT="ca.crt"
+CA_SUBJECT="/C=US/O=${CA_ORGANIZATION}"
+
+EC_CURVE="prime256v1"
+
+# Generate CA key and self-signed certificate with SHA-256
+openssl ecparam -genkey -name ${EC_CURVE} -out ${CA_KEY}
+openssl req -new -x509 -sha256 -days 365 -nodes -key ${CA_KEY} -subj "${CA_SUBJECT}" -out ${CA_CERT} \
+-extensions v3_ca -config <(cat /etc/ssl/openssl.cnf - << EOF
+[v3_ca]
+subjectKeyIdentifier=hash
+authorityKeyIdentifier=keyid:always,issuer
+basicConstraints=critical,CA:true
+keyUsage=critical,keyCertSign,cRLSign
+EOF
+)
+
+# Function to generate certificates for each host name
+generate_certificate() {
+    local HOST_NAME="$1"
+    local KEY="${HOST_NAME}.key"
+    local CSR="${HOST_NAME}.csr"
+    local CERT="${HOST_NAME}.crt"
+    local SUBJECT="/C=US/O=${CA_ORGANIZATION}/CN=${HOST_NAME}"
+
+    # Generate host key and certificate signing request (CSR)
+    openssl ecparam -genkey -name ${EC_CURVE} -out ${KEY}
+    openssl req -new -key ${KEY} -subj "${SUBJECT}" -out ${CSR}
+
+    # Sign the CSR with the CA to generate the host certificate
+    openssl x509 -req -sha256 -days 365 -in ${CSR} -CA ${CA_CERT} -CAkey ${CA_KEY} -CAcreateserial -out ${CERT}\
+    -extensions v3_req -extfile <(cat /etc/ssl/openssl.cnf - << EOF
+[v3_req]
+basicConstraints=critical,CA:FALSE
+authorityKeyIdentifier=keyid,issuer
+subjectAltName=@alt_names
+keyUsage=digitalSignature,keyEncipherment
+[ alt_names ]
+DNS.1 = ${HOST_NAME}
+DNS.2 = localhost
+EOF
+    )
+
+    # Remove the CSR (not needed anymore)
+    rm ${CSR}
+}
+
+# Generate certificates for each host name
+generate_certificate "proxy"
+generate_certificate "upstream-proxy"
+generate_certificate "httpbin"
+
+chmod 644 *.key *.crt

e2e/forwarder/service.go

@@ -68,13 +68,24 @@ func HttpbinService() *Service {
 
 func (s *Service) WithProtocol(protocol string) *Service {
 	s.Environment["FORWARDER_PROTOCOL"] = protocol
+
+	if protocol == "https" || protocol == "h2" {
+		s.Environment["FORWARDER_CERT_FILE"] = "/etc/forwarder/certs/" + s.Name + ".crt"
+		s.Environment["FORWARDER_KEY_FILE"] = "/etc/forwarder/private/" + s.Name + ".key"
+		s.Volumes = append(s.Volumes,
+			"./certs/"+s.Name+".crt:/etc/forwarder/certs/"+s.Name+".crt:ro",
+			"./certs/"+s.Name+".key:/etc/forwarder/private/"+s.Name+".key:ro",
+		)
+	}
+
 	return s
 }
 
 func (s *Service) WithUpstream(name, protocol string) *Service {
 	s.Environment["FORWARDER_PROXY"] = protocol + "://" + name + ":3128"
 	if protocol == "https" {
-		s.Environment["FORWARDER_INSECURE"] = "true"
+		s.Environment["FORWARDER_CACERT_FILE"] = "/etc/forwarder/certs/ca-certificates.crt"
+		s.Volumes = append(s.Volumes, "./certs/ca.crt:/etc/forwarder/certs/ca-certificates.crt:ro")
 	}
 	return s
 }

fileurl/fileurl_test.go

@@ -97,6 +97,13 @@ func TestParseFilePathOrURL(t *testing.T) {
 				Path:   "/path/to/file",
 			},
 		},
+		{
+			input: "data:text/plain;base64,U2F1Y2VMYWJzCg==",
+			want: url.URL{
+				Scheme: "data",
+				Opaque: "text/plain;base64,U2F1Y2VMYWJzCg==",
+			},
+		},
 	}
 
 	for i := range tests {

http_proxy.go

@@ -132,8 +132,10 @@ func NewHTTPProxy(cfg *HTTPProxyConfig, pr PACResolver, cm *CredentialsMatcher,
 	if rt == nil {
 		log.Infof("HTTP transport not configured, using standard library default")
 		rt = http.DefaultTransport.(*http.Transport).Clone() //nolint:forcetypeassert // we know it's a *http.Transport
-	} else if _, ok := rt.(*http.Transport); !ok {
+	} else if tr, ok := rt.(*http.Transport); !ok {
 		log.Debugf("using custom HTTP transport %T", rt)
+	} else if tr.TLSClientConfig != nil && tr.TLSClientConfig.RootCAs != nil {
+		log.Infof("using custom root CA certificates")
 	}
 	hp := &HTTPProxy{
 		config:    *cfg,
@@ -160,11 +162,9 @@ func (hp *HTTPProxy) configureHTTPS() error {
 		hp.log.Debugf("loading TLS certificate from %s and %s", hp.config.CertFile, hp.config.KeyFile)
 	}
 
-	tlsCfg := httpsTLSConfigTemplate()
-	err := LoadCertificateFromTLSConfig(tlsCfg, &hp.config.TLSConfig)
-	hp.TLSConfig = tlsCfg
+	hp.TLSConfig = httpsTLSConfigTemplate()
 
-	return err
+	return hp.config.ConfigureTLSConfig(hp.TLSConfig)
 }
 
 func (hp *HTTPProxy) configureProxy() {