fix: Fix tls not working correctly for es-https, imporve tls config m…

…anagement, fix verify peers, make cert/key required for carver receiver

docs/log-carver/Configuration.md

@@ -753,7 +753,7 @@ Sets the minimum TLS version allowed for connections on this transport. The TLS
 
 ### `ssl certificate` (receiver)
 
-Filepath. Optional
+Filepath. Required
 Available when `transport` is `tls`
 
 Path to a PEM encoded certificate file to use as the server certificate.
@@ -769,7 +769,7 @@ A list of paths to PEM encoded client certificate authorities that can be used t
 
 ### `ssl key` (receiver)
 
-Filepath. Optional
+Filepath. Required with `ssl certificate`  
 Available when `transport` is `tls`
 
 Path to a PEM encoded private key to use with the server certificate.

docs/log-courier/Configuration.md

@@ -722,7 +722,7 @@ This will become default in a future major version change as a breaking change,
 
 Enable Elastic Common Schema (ECS) fields in events. By default, events are generated in a similar style to Filebeat and the original Logstash Forwarder. This will enable ECS compatible fields instead **which are not backwards compatible**. This will need a change to the template used within Elasticsearch to make the fields usable. Additionally, you will need to ensure you are using fresh indexes as the ECS field types differ in such a way Elasticsearch will refuse to store them if it had previously stored non-ECS fields. Specifically, the `host` field changes from a string to an object containing `name` and `hostname`.
 
-See [Event Format](#event-format) for more information.
+See [Event Format](../Events.md#event-format) for more information.
 
 ### `fields`
 

lc-lib/transports/clienttlsconfiguration.go

@@ -0,0 +1,59 @@
+/*
+ * Copyright 2012-2023 Jason Woods and contributors
+ *
+ * This file is a modification of code from Logstash Forwarder.
+ * Copyright 2012-2013 Jordan Sissel and contributors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package transports
+
+import (
+	"fmt"
+
+	"github.com/driskell/log-courier/lc-lib/config"
+)
+
+type ClientTlsConfiguration struct {
+	SSLCA string `config:"ssl ca"`
+
+	*TlsConfiguration `config:",embed"`
+}
+
+func (f *ClientTlsConfiguration) TlsValidate(enableTls bool, p *config.Parser, configPath string) (err error) {
+	if enableTls {
+		if len(f.SSLCA) == 0 {
+			return fmt.Errorf("%sssl ca is required for secure transports", configPath)
+		}
+		if f.CaList, err = AddCertificates(f.CaList, f.SSLCA); err != nil {
+			return fmt.Errorf("failure loading %sssl ca: %s", configPath, err)
+		}
+	} else {
+		if len(f.SSLCA) > 0 {
+			return fmt.Errorf("%[1]sssl ca is not supported for non-secure transports", configPath)
+		}
+	}
+
+	return f.TlsConfiguration.TlsValidate(enableTls, p, configPath)
+}
+
+func (f *ClientTlsConfiguration) Defaults() {
+}
+
+func (f *ClientTlsConfiguration) HasChanged(newConfig *ClientTlsConfiguration) bool {
+	if newConfig.SSLCA != f.SSLCA {
+		return true
+	}
+	return f.TlsConfiguration.HasChanged(newConfig.TlsConfiguration)
+}

lc-lib/transports/es/transportesfactory.go

@@ -65,11 +65,12 @@ type TransportESFactory struct {
 	Username         string        `config:"username"`
 	TemplateFile     string        `config:"template file"`
 	TemplatePatterns []string      `config:"template patterns"`
+	SSLCA            string        `config:"ssl ca"`
 
 	// Internal
 	template []byte
 
-	*transports.TlsConfiguration
+	*transports.ClientTlsConfiguration `config:",embed"`
 }
 
 // NewTransportESFactory create a new TransportESFactory from the provided
@@ -130,7 +131,7 @@ func (f *TransportESFactory) Validate(p *config.Parser, configPath string) (err
 		f.templatePatternSingleJSON = string(result)
 	}
 
-	return f.TlsConfiguration.TlsValidate(f.transport == TransportESHTTPS, p, configPath)
+	return f.ClientTlsConfiguration.TlsValidate(f.transport == TransportESHTTPS, p, configPath)
 }
 
 // Defaults sets the default configuration values
@@ -191,7 +192,7 @@ func (t *TransportESFactory) ShouldRestart(newConfig transports.TransportFactory
 		return true
 	}
 
-	return t.TlsConfiguration.HasChanged(newConfigImpl.TlsConfiguration)
+	return t.ClientTlsConfiguration.HasChanged(newConfigImpl.ClientTlsConfiguration)
 }
 
 // Register the transports

lc-lib/transports/servertlsconfiguration.go

@@ -0,0 +1,69 @@
+/*
+ * Copyright 2012-2023 Jason Woods and contributors
+ *
+ * This file is a modification of code from Logstash Forwarder.
+ * Copyright 2012-2013 Jordan Sissel and contributors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package transports
+
+import (
+	"fmt"
+	"reflect"
+
+	"github.com/driskell/log-courier/lc-lib/config"
+)
+
+const (
+	defaultSSLVerifyPeers = true
+)
+
+type ServerTlsConfiguration struct {
+	SSLClientCA    []string `config:"ssl client ca"`
+	SSLVerifyPeers bool     `config:"verify peers"`
+
+	*TlsConfiguration `config:",embed"`
+}
+
+func (f *ServerTlsConfiguration) TlsValidate(enableTls bool, p *config.Parser, configPath string) (err error) {
+	if enableTls {
+		if len(f.SSLCertificate) == 0 || len(f.SSLKey) == 0 {
+			return fmt.Errorf("%sssl certificate and ssl key must be specified for secure receivers", configPath)
+		}
+
+		for idx, clientCA := range f.SSLClientCA {
+			if f.CaList, err = AddCertificates(f.CaList, clientCA); err != nil {
+				return fmt.Errorf("failure loading %sssl client ca[%d]: %s", configPath, idx, err)
+			}
+		}
+	} else {
+		if len(f.SSLClientCA) > 0 {
+			return fmt.Errorf("%sssl client ca is not supported for non-secure receivers", configPath)
+		}
+	}
+
+	return f.TlsConfiguration.TlsValidate(enableTls, p, configPath)
+}
+
+func (f *ServerTlsConfiguration) Defaults() {
+	f.SSLVerifyPeers = defaultSSLVerifyPeers
+}
+
+func (f *ServerTlsConfiguration) HasChanged(newConfig *ServerTlsConfiguration) bool {
+	if !reflect.DeepEqual(newConfig.SSLClientCA, f.SSLClientCA) {
+		return true
+	}
+	return f.TlsConfiguration.HasChanged(newConfig.TlsConfiguration)
+}

lc-lib/transports/tcp/courier/receiverfactory.go

@@ -29,7 +29,7 @@ import (
 // ReceiverFactory holds the configuration from the configuration file
 // It allows creation of ReceiverTCP instances that use this configuration
 type ReceiverFactory struct {
-	*tcp.ReceiverFactory
+	*tcp.ReceiverFactory `,config:"embed"`
 
 	// Constructor
 	config         *config.Config
@@ -57,15 +57,6 @@ func NewReceiverFactory(p *config.Parser, configPath string, unUsed map[string]i
 	return ret, nil
 }
 
-// Validate the configuration
-func (f *ReceiverFactory) Validate(p *config.Parser, configPath string) (err error) {
-	if err = f.ReceiverFactory.Validate(p, configPath); err != nil {
-		return err
-	}
-
-	return nil
-}
-
 // NewReceiver returns a new Receiver interface using the settings from the ReceiverFactory
 func (f *ReceiverFactory) NewReceiver(ctx context.Context, pool *addresspool.Pool, eventChan chan<- transports.Event) transports.Receiver {
 	return f.ReceiverFactory.NewReceiverWithProtocol(ctx, pool, eventChan, &protocolFactory{isClient: false})

lc-lib/transports/tcp/courier/transportfactory.go

@@ -21,7 +21,6 @@ package courier
 
 import (
 	"context"
-	"fmt"
 	"regexp"
 
 	"github.com/driskell/log-courier/lc-lib/addresspool"
@@ -33,7 +32,7 @@ import (
 // TransportFactory holds the configuration from the configuration file
 // It allows creation of TransportTCP instances that use this configuration
 type TransportFactory struct {
-	*tcp.TransportFactory
+	*tcp.TransportFactory `,config:"embed"`
 
 	// Constructor
 	config         *config.Config
@@ -61,24 +60,6 @@ func NewTransportFactory(p *config.Parser, configPath string, unUsed map[string]
 	return ret, nil
 }
 
-// Validate the configuration
-func (f *TransportFactory) Validate(p *config.Parser, configPath string) (err error) {
-	if f.transport == TransportCourierTLS {
-		if len(f.SSLCA) == 0 {
-			return fmt.Errorf("%sssl ca is required when the transport is tls", configPath)
-		}
-		if f.CaList, err = transports.AddCertificates(f.CaList, f.SSLCA); err != nil {
-			return fmt.Errorf("failure loading %sssl ca: %s", configPath, err)
-		}
-	} else {
-		if len(f.SSLCA) > 0 {
-			return fmt.Errorf("%[1]sssl ca is not supported when the transport is tcp", configPath)
-		}
-	}
-
-	return f.TransportFactory.Validate(p, configPath)
-}
-
 // NewTransport returns a new Transport interface using the settings from the
 // TransportFactory.
 func (f *TransportFactory) NewTransport(ctx context.Context, pool *addresspool.Pool, eventChan chan<- transports.Event) transports.Transport {

lc-lib/transports/tcp/factory.go

@@ -0,0 +1,36 @@
+/*
+ * Copyright 2012-2023 Jason Woods and contributors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package tcp
+
+import (
+	"regexp"
+
+	"github.com/driskell/log-courier/lc-lib/config"
+)
+
+// Factory holds common TCP factory settings
+type Factory struct {
+	config         *config.Config
+	hostportRegexp *regexp.Regexp
+}
+
+func newFactory(config *config.Config) *Factory {
+	return &Factory{
+		config:         config,
+		hostportRegexp: regexp.MustCompile(`^\[?([^]]+)\]?:([0-9]+)$`),
+	}
+}

lc-lib/transports/tcp/receiverfactory.go

@@ -21,9 +21,6 @@ package tcp
 
 import (
 	"context"
-	"fmt"
-	"reflect"
-	"regexp"
 	"time"
 
 	"github.com/driskell/log-courier/lc-lib/addresspool"
@@ -32,32 +29,12 @@ import (
 	"github.com/driskell/log-courier/lc-lib/transports"
 )
 
-const (
-	defaultSSLVerifyPeers = true
-)
-
-// Factory holds common TCP factory settings
-type Factory struct {
-	config         *config.Config
-	hostportRegexp *regexp.Regexp
-}
-
-func newFactory(config *config.Config) *Factory {
-	return &Factory{
-		config:         config,
-		hostportRegexp: regexp.MustCompile(`^\[?([^]]+)\]?:([0-9]+)$`),
-	}
-}
-
 type ReceiverFactory struct {
-	*Factory
+	*Factory `config:",embed"`
 
 	EnableTls bool
 
-	SSLClientCA    []string `config:"ssl client ca"`
-	SSLVerifyPeers bool     `config:"verify_peers"`
-
-	*transports.TlsConfiguration `config:",embed"`
+	*transports.ServerTlsConfiguration `config:",embed"`
 }
 
 func NewReceiverFactory(p *config.Parser, configPath string, unUsed map[string]interface{}, enableTls bool) (*ReceiverFactory, error) {
@@ -95,40 +72,14 @@ func (f *ReceiverFactory) NewReceiverWithProtocol(ctx context.Context, pool *add
 	return ret
 }
 
-// Defaults sets the default configuration values
 func (f *ReceiverFactory) Defaults() {
-	f.SSLVerifyPeers = defaultSSLVerifyPeers
 }
 
-// Validate the configuration
 func (f *ReceiverFactory) Validate(p *config.Parser, configPath string) (err error) {
-	if f.EnableTls {
-		for idx, clientCA := range f.SSLClientCA {
-			if f.CaList, err = transports.AddCertificates(f.CaList, clientCA); err != nil {
-				return fmt.Errorf("failure loading %sssl client ca[%d]: %s", configPath, idx, err)
-			}
-		}
-	} else {
-		if len(f.SSLClientCA) > 0 {
-			return fmt.Errorf("%sssl client ca is not supported when the transport is tcp", configPath)
-		}
-	}
-
-	if err = f.TlsValidate(f.EnableTls, p, configPath); err != nil {
-		return err
-	}
-
-	return nil
+	return f.ServerTlsConfiguration.TlsValidate(f.EnableTls, p, configPath)
 }
 
 func (f *ReceiverFactory) ShouldRestart(newFactory transports.ReceiverFactory) bool {
 	newFactoryImpl := newFactory.(*ReceiverFactory)
-
-	if !reflect.DeepEqual(newFactoryImpl.SSLClientCA, f.SSLClientCA) {
-		return true
-	}
-	if f.TlsConfiguration.HasChanged(newFactoryImpl.TlsConfiguration) {
-		return true
-	}
-	return false
+	return f.ServerTlsConfiguration.HasChanged(newFactoryImpl.ServerTlsConfiguration)
 }