Allow http/https scheme in otlpexporter endpoint configuration (#3575)

otlpexporter endpoint configuration allows for an `http` or `https` scheme. The scheme is stripped off when instantiating the gRPC channel. A scheme of `https` indicates a secure channel. This aligns the collector with this hopefully soon-to-be-merged specification PR open-telemetry/opentelemetry-specification#1729

**Link to tracking Issue:** #2539

**Documentation:** Updated readme

CHANGELOG.md

@@ -39,6 +39,7 @@
 
 - `scraperhelper`: Include the scraper name in log messages (#3487)
 - `scraperhelper`: fix case when returned pdata is empty (#3520) 
+- `otlpexporter`: Allow endpoint to be configured with a scheme of `http` or `https` (#3575)
 - Record the correct number of points not metrics in Kafka receiver (#3553) 
 - Validate the Prometheus configuration (#3589) 
 

config/configgrpc/configgrpc.go

@@ -15,6 +15,7 @@
 package configgrpc
 
 import (
+	"crypto/tls"
 	"fmt"
 	"net"
 	"strings"
@@ -159,6 +160,26 @@ type GRPCServerSettings struct {
 	Auth *configauth.Authentication `mapstructure:"auth,omitempty"`
 }
 
+// SanitizedEndpoint strips the prefix of either http:// or https:// from configgrpc.GRPCClientSettings.Endpoint.
+func (gcs *GRPCClientSettings) SanitizedEndpoint() string {
+	switch {
+	case gcs.isSchemeHTTP():
+		return strings.TrimPrefix(gcs.Endpoint, "http://")
+	case gcs.isSchemeHTTPS():
+		return strings.TrimPrefix(gcs.Endpoint, "https://")
+	default:
+		return gcs.Endpoint
+	}
+}
+
+func (gcs *GRPCClientSettings) isSchemeHTTP() bool {
+	return strings.HasPrefix(gcs.Endpoint, "http://")
+}
+
+func (gcs *GRPCClientSettings) isSchemeHTTPS() bool {
+	return strings.HasPrefix(gcs.Endpoint, "https://")
+}
+
 // ToDialOptions maps configgrpc.GRPCClientSettings to a slice of dial options for gRPC.
 func (gcs *GRPCClientSettings) ToDialOptions(ext map[config.ComponentID]component.Extension) ([]grpc.DialOption, error) {
 	var opts []grpc.DialOption
@@ -177,6 +198,8 @@ func (gcs *GRPCClientSettings) ToDialOptions(ext map[config.ComponentID]componen
 	tlsDialOption := grpc.WithInsecure()
 	if tlsCfg != nil {
 		tlsDialOption = grpc.WithTransportCredentials(credentials.NewTLS(tlsCfg))
+	} else if gcs.isSchemeHTTPS() {
+		tlsDialOption = grpc.WithTransportCredentials(credentials.NewTLS(&tls.Config{}))
 	}
 	opts = append(opts, tlsDialOption)
 

exporter/otlpexporter/README.md

@@ -15,7 +15,8 @@ The following settings are required:
 
 - `endpoint` (no default): host:port to which the exporter is going to send OTLP trace data,
 using the gRPC protocol. The valid syntax is described
-[here](https://github.com/grpc/grpc/blob/master/doc/naming.md)
+[here](https://github.com/grpc/grpc/blob/master/doc/naming.md).
+If a scheme of `https` is used then client transport security is enabled and overrides the `insecure` setting.
 
 By default, TLS is enabled:
 

exporter/otlpexporter/otlp.go

@@ -101,7 +101,7 @@ func newGrpcSender(config *Config, ext map[config.ComponentID]component.Extensio
 	}
 
 	var clientConn *grpc.ClientConn
-	if clientConn, err = grpc.Dial(config.GRPCClientSettings.Endpoint, dialOpts...); err != nil {
+	if clientConn, err = grpc.Dial(config.GRPCClientSettings.SanitizedEndpoint(), dialOpts...); err != nil {
 		return nil, err
 	}
 

exporter/otlpexporter/otlp_test.go

@@ -17,6 +17,8 @@ package otlpexporter
 import (
 	"context"
 	"net"
+	"path/filepath"
+	"runtime"
 	"sync"
 	"sync/atomic"
 	"testing"
@@ -25,6 +27,7 @@ import (
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 	"google.golang.org/grpc"
+	"google.golang.org/grpc/credentials"
 	"google.golang.org/grpc/metadata"
 
 	"go.opentelemetry.io/collector/component"
@@ -71,10 +74,25 @@ func (r *mockTracesReceiver) GetLastRequest() pdata.Traces {
 	return r.lastRequest
 }
 
-func otlpTracesReceiverOnGRPCServer(ln net.Listener) *mockTracesReceiver {
+func otlpTracesReceiverOnGRPCServer(ln net.Listener, useTLS bool) (*mockTracesReceiver, error) {
+	sopts := []grpc.ServerOption{}
+
+	if useTLS {
+		_, currentFile, _, _ := runtime.Caller(0)
+		basepath := filepath.Dir(currentFile)
+		certpath := filepath.Join(basepath, "testdata/test_cert.pem")
+		keypath := filepath.Join(basepath, "testdata/test_key.pem")
+
+		creds, err := credentials.NewServerTLSFromFile(certpath, keypath)
+		if err != nil {
+			return nil, err
+		}
+		sopts = append(sopts, grpc.Creds(creds))
+	}
+
 	rcv := &mockTracesReceiver{
 		mockReceiver: mockReceiver{
-			srv: grpc.NewServer(),
+			srv: grpc.NewServer(sopts...),
 		},
 	}
 
@@ -84,7 +102,7 @@ func otlpTracesReceiverOnGRPCServer(ln net.Listener) *mockTracesReceiver {
 		_ = rcv.srv.Serve(ln)
 	}()
 
-	return rcv
+	return rcv, nil
 }
 
 type mockLogsReceiver struct {
@@ -165,7 +183,7 @@ func TestSendTraces(t *testing.T) {
 	// Start an OTLP-compatible receiver.
 	ln, err := net.Listen("tcp", "localhost:")
 	require.NoError(t, err, "Failed to find an available address to run the gRPC server: %v", err)
-	rcv := otlpTracesReceiverOnGRPCServer(ln)
+	rcv, _ := otlpTracesReceiverOnGRPCServer(ln, false)
 	// Also closes the connection.
 	defer rcv.srv.GracefulStop()
 
@@ -229,6 +247,79 @@ func TestSendTraces(t *testing.T) {
 	require.EqualValues(t, rcv.GetMetadata().Get("header"), expectedHeader)
 }
 
+func TestSendTracesWhenEndpointHasHttpScheme(t *testing.T) {
+	tests := []struct {
+		name               string
+		useTLS             bool
+		scheme             string
+		gRPCClientSettings configgrpc.GRPCClientSettings
+	}{
+		{
+			name:               "Use https scheme",
+			useTLS:             true,
+			scheme:             "https://",
+			gRPCClientSettings: configgrpc.GRPCClientSettings{},
+		},
+		{
+			name:   "Use http scheme",
+			useTLS: false,
+			scheme: "http://",
+			gRPCClientSettings: configgrpc.GRPCClientSettings{
+				TLSSetting: configtls.TLSClientSetting{
+					Insecure: true,
+				},
+			},
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			// Start an OTLP-compatible receiver.
+			ln, err := net.Listen("tcp", "localhost:")
+			require.NoError(t, err, "Failed to find an available address to run the gRPC server: %v", err)
+			rcv, err := otlpTracesReceiverOnGRPCServer(ln, test.useTLS)
+			require.NoError(t, err, "Failed to start mock OTLP receiver")
+			// Also closes the connection.
+			defer rcv.srv.GracefulStop()
+
+			// Start an OTLP exporter and point to the receiver.
+			factory := NewFactory()
+			cfg := factory.CreateDefaultConfig().(*Config)
+			cfg.GRPCClientSettings = test.gRPCClientSettings
+			cfg.GRPCClientSettings.Endpoint = test.scheme + ln.Addr().String()
+			if test.useTLS {
+				cfg.GRPCClientSettings.TLSSetting.InsecureSkipVerify = true
+			}
+			set := componenttest.NewNopExporterCreateSettings()
+			exp, err := factory.CreateTracesExporter(context.Background(), set, cfg)
+			require.NoError(t, err)
+			require.NotNil(t, exp)
+
+			defer func() {
+				assert.NoError(t, exp.Shutdown(context.Background()))
+			}()
+
+			host := componenttest.NewNopHost()
+			assert.NoError(t, exp.Start(context.Background(), host))
+
+			// Ensure that initially there is no data in the receiver.
+			assert.EqualValues(t, 0, atomic.LoadInt32(&rcv.requestCount))
+
+			// Send empty trace.
+			td := pdata.NewTraces()
+			assert.NoError(t, exp.ConsumeTraces(context.Background(), td))
+
+			// Wait until it is received.
+			assert.Eventually(t, func() bool {
+				return atomic.LoadInt32(&rcv.requestCount) > 0
+			}, 10*time.Second, 5*time.Millisecond)
+
+			// Ensure it was received empty.
+			assert.EqualValues(t, 0, atomic.LoadInt32(&rcv.totalItems))
+		})
+	}
+}
+
 func TestSendMetrics(t *testing.T) {
 	// Start an OTLP-compatible receiver.
 	ln, err := net.Listen("tcp", "localhost:")
@@ -397,7 +488,7 @@ func TestSendTraceDataServerStartWhileRequest(t *testing.T) {
 	}()
 
 	time.Sleep(2 * time.Second)
-	rcv := otlpTracesReceiverOnGRPCServer(ln)
+	rcv, _ := otlpTracesReceiverOnGRPCServer(ln, false)
 	defer rcv.srv.GracefulStop()
 	// Wait until one of the conditions below triggers.
 	select {
@@ -410,7 +501,7 @@ func TestSendTraceDataServerStartWhileRequest(t *testing.T) {
 }
 
 func startServerAndMakeRequest(t *testing.T, exp component.TracesExporter, td pdata.Traces, ln net.Listener) {
-	rcv := otlpTracesReceiverOnGRPCServer(ln)
+	rcv, _ := otlpTracesReceiverOnGRPCServer(ln, false)
 	defer rcv.srv.GracefulStop()
 	// Ensure that initially there is no data in the receiver.
 	assert.EqualValues(t, 0, atomic.LoadInt32(&rcv.requestCount))

exporter/otlpexporter/testdata/test_cert.pem

@@ -1,29 +1,17 @@
 -----BEGIN CERTIFICATE-----
-MIIE6jCCAtICCQDVU4PtqpqADTANBgkqhkiG9w0BAQsFADA3MQswCQYDVQQGEwJV
-UzETMBEGA1UECAwKY2FsaWZvcm5pYTETMBEGA1UECgwKb3BlbmNlbnN1czAeFw0x
-OTAzMDQxODA3MjZaFw0yMDAzMDMxODA3MjZaMDcxCzAJBgNVBAYTAlVTMRMwEQYD
-VQQIDApjYWxpZm9ybmlhMRMwEQYDVQQKDApvcGVuY2Vuc3VzMIICIjANBgkqhkiG
-9w0BAQEFAAOCAg8AMIICCgKCAgEAy9JQiAOMzArcdiS4szbTuzg5yYijSSY6SvGj
-XMs4/LEFLxgGmFfyHXxoVQzV26lTu/AiUFlZi4JY2qlkZyPwmmmSg4fmzikpVPiC
-Vv9pvSIojs8gs0sHaOt40Q8ym43bNt3Mh8rYrs+XMERi6Ol9//j4LnfePkNU5uEo
-qC8KQamckaMR6UEHFNunyOwvNBsipgTPldQUPGVnCsNKk8olYGAXS7DR25bgbPli
-4T9VCSElsSPAODmyo+2MEDagVXa1vVYxKyO2k6oeBS0lsvdRqRTmGggcg0B/dk+a
-H1CL9ful0cu9P3dQif+hfGay8udPkwDLPEq1+WnjJFut3Pmbk3SqUCas5iWt76kK
-eKFh4k8fCy4yiaZxzvSbm9+bEBHAl0ZXd8pjvAsBfCKe6G9SBzE1DK4FjWiiEGCb
-5dGsyTKr33q3DekLvT3LF8ZeON/13d9toucX9PqG2HDwMP/Fb4WjQIzOc/H9wIak
-pf7u6QBDGUiCMmoDrp1d8RsI1RPbEhoywH0YlLmwgf+cr1dU7vlISf576EsGxFz4
-+/sZjIBvZBHn/x0MH+bs4J8V3vMujfDoRdhL07bK7q/AkEALUxljKEfoWeqiuVzK
-F9BVv3xNhiua2kgPVbMNWPrQ5uotkNp8IykJ3QOuQ3p5pzxdGfpLd6f8gmJDmcbi
-AI9dWTcCAwEAATANBgkqhkiG9w0BAQsFAAOCAgEAVVi4t/Sumre+AGTaU7np9dl2
-tpllbES5ixe6m2uezt5wAzYNNyuQ2mMG2XrSkMy5gvBZRT9nRNSmLV8VEcxZihG0
-YHS5soXnLL3Jdlwxp98WTDPvM1ntxcHyEyqrrg9YDfKn4sOrr5vo2yZzoKwtxtc7
-lue9JormVx7GxMi7NwaUtCbnwAIcqJJpFjt1EhmJOxGqTJPgUvTBdeGvRj30c6fk
-pqpUdPbZ7RKPEtbLoMoCBujKnErv+H0G6Vp9WyCHN+Mi9uTMsGwH14cmJjmfwGDC
-8/WF4LdlawFnf/arIp9YcVwcP91d4ywyvbuuo2M7qdosQ7k4uRZ3tyggLYShS3RW
-BMEhMRDz9dM0oKGF+HnaS824BIh6O6Hn82Vt8uCKS7IbEX99/kkN1KcqqQe6Lwjq
-tG/lm4K5yf+FJVDivpZ9mYTvqTBjhTaOp6m3HYSNJfS0hLQVvEuBNXd8bHiXkcLp
-rmFOYUWsjxV1Qku3U5Rner0UpB2Fuw9nJcXuDgWG0gjwzAZ83y3du1VIZp0Ad8Vv
-IYpaucbImGJszMtNXn3l72K1wvQVIhm9eRwYc3QteJzweHaDsbytZEoS/GhTrZIT
-wRe5ZGrjJBJngRANRSm1BH8j6PjLem9mzPb2eytwJJA0lLhUk4vYproVvXcx0vow
-5F+5VB1YB8/tbWePmpo=
+MIICpDCCAYwCCQC5oaFsqLW3GTANBgkqhkiG9w0BAQsFADAUMRIwEAYDVQQDDAls
+b2NhbGhvc3QwHhcNMjEwNzE0MDAxMzU2WhcNMzEwNzEyMDAxMzU2WjAUMRIwEAYD
+VQQDDAlsb2NhbGhvc3QwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQDO
+mKaE1qg5VLMwaUnSzufT23rRJFbuy/HDXwsH63yZVSsISQkGjkBYBgrqAMtVnsI/
+l4gXtBWkZtJFs68Sbo9ps3W0PdB5+d12R5NUNA1rkZtx3jtEN33dpGhifug/TIZe
+7Zr0G1z6gNoaEezk0Jpg4KsH7QpIeHPRhIZMyWeqddgD/qL4/ukaU4NOORuF3WoT
+oo2LpI3jUq66mz2N2Inq0V/OX7BYB4Ur6EtjWh2baiUuw9fq+oLUlgZd6ypnugC/
++YfgYqvWtRntmEr0Z+O4Kz81P2IpH/0h1RFhWyK6thVGa9cx6aseCp3V2cMXfGfc
+z4n3Uvz87v+bZvGbcse/AgMBAAEwDQYJKoZIhvcNAQELBQADggEBAAlvNBNoqXUQ
+ohR0eozIHGeJ94U7WK5zXf2NSvmRlwHzHXvUq6GKd+8Bv1foMjI6OpSOZmjtRGsc
+rWET1WjSyQddRfqYazhWp1IyYu5LfATwPS+RXJAkWixKVfG+Ta2x6u+aT/bSZwEg
+NwRerc6pyqv5UG8Z7Pe1kAxbgOwZv5KXAewIgTSbEkmIp1Dg8GhGeWD5pjYNCkJV
+Na2KMAUWP3PeQzdSBKmBNpsRUALuSTxb5u7pl+PA7FLInTtDeyZn8xpO1GPBhbJE
+trDbmTbj5YexOXEaQtGtZ6fwRw2jnUm8nqtXozxIomnVTBO8vLmZAUgyJ71trRw0
+gE9tH5Ndlug=
 -----END CERTIFICATE-----

exporter/otlpexporter/testdata/test_key.pem

@@ -0,0 +1,28 @@
+-----BEGIN PRIVATE KEY-----
+MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQDOmKaE1qg5VLMw
+aUnSzufT23rRJFbuy/HDXwsH63yZVSsISQkGjkBYBgrqAMtVnsI/l4gXtBWkZtJF
+s68Sbo9ps3W0PdB5+d12R5NUNA1rkZtx3jtEN33dpGhifug/TIZe7Zr0G1z6gNoa
+Eezk0Jpg4KsH7QpIeHPRhIZMyWeqddgD/qL4/ukaU4NOORuF3WoToo2LpI3jUq66
+mz2N2Inq0V/OX7BYB4Ur6EtjWh2baiUuw9fq+oLUlgZd6ypnugC/+YfgYqvWtRnt
+mEr0Z+O4Kz81P2IpH/0h1RFhWyK6thVGa9cx6aseCp3V2cMXfGfcz4n3Uvz87v+b
+ZvGbcse/AgMBAAECggEADeR39iDVKR3H+u5pl3JwZm+w35V4/w/ZzxB6FmtAcrMm
+dKUspTM1onWtkDTDd5t4ZnxTG3zxo5+Cbkt571xd6na16Ivrk/g4aza+8n+Zk200
+LcEK7ThqD1h56H2uMmt78bA6pkWcx/+YKv6flndsmi0hcyP+eAcZirJFsa4teWna
+P6rhI9zThc9OcecqGZIlmzJQ4cLbIO86QqkWW6yjKYg6riOb2g+i3e97ZngMCTcV
+lni+sksLlXBNKPqh1AkiUFe4pInRBh4LGQ5rNSYswEqlQY0iW0u4Hs3HNou0On+8
+1T8m5wzKQ+23AN+vVRJ/MHssQiB/TPK92jXVgEz6eQKBgQD2GEb7NzDIxsAQZBQo
+tt3jYitNcAEqMWeT7wxCMMue4wIrT6Fp6NuG5NMVqLglzx72m6TXg7YzZxPrAnlH
+jblWI4sxwVC8BjjYyGud7qMuhUIZmI8aS9HuYW0ODSxkcpVVXd4HDUYKg7PafAkl
+cj745E5KGD+qW44KASTTQ1SwRQKBgQDW6WLp/nPVPO5YEK4nzS7b1RRC8ypHiKd6
+LzhA2izgcsmO3F3Y5ZZ5rzeFbjgZiGFTUB/r1mgomI8kZyIGP1AN6o8oY9I89gHY
+/DEEagIsFK5jAEoMeN0qbgqasOXpi+uUHCNidWa7OWOL9Rsh7dyVT54xcqMC2Qak
+Vpoy5miiMwKBgQDuOHH9nF9M+5fQRhB9mQcRpWXlgBagkVKCkVR8fl+dXoIrCtpl
+e1OGMNtki/42G1kNv3zCYm1tNMrDI5HjAf32tFF5yHguipdcwiXqq6aq0bQ6ssNT
+4TFGYGkAwR/H3GNST5stmFvEsdjYFlmENiNfKyHd97spXZcReCn9l5/TQQKBgDRG
+PpYWG4zBrmPjYskxonU8ZhpG1YDi34Hb3H4B06qgoSBLv9QTPD/K++FLxv+G6c1/
+DtSpqVo+iYrcPy1v1wQbisjTRv8nA5oI9c9SDcc1HJneJyTTfVBlxdSMtM/TBfFX
+ys+XKO7fbbRMYVYmamIzJJJ4hOgba/8rRYSeANN7AoGBAMDdrT+ig3aDMratbAvY
+lqsfN3AtxoZ+ZVQYyUbzTSZPZ/to9eNuBzhRKcQ3QfG95nrHb7OnWHa7+1kc4p/Q
+jMgzJgRpajlES+F3CCMPgJIJg7Ev+yiSCJLP9ZOsC+E96bK265hUcDyCXwb3Wzmg
+4L9sc1QsQW80QO/RnaEzGO51
+-----END PRIVATE KEY-----