Merge pull request #2021 from vparfonov/release-5.6-log3445-2

[release-5.6]LOG-4146: Proceede tls.insecureSkipVerify=true configuration even if certificate not added to the secret
openshift · May 30, 2023 · 64bdbfb · 64bdbfb
2 parents e5c60be + f95870a
commit 64bdbfb
Show file tree

Hide file tree

Showing 11 changed files with 1,002 additions and 559 deletions.
diff --git a/internal/generator/vector/output/elasticsearch/elasticsearch.go b/internal/generator/vector/output/elasticsearch/elasticsearch.go
@@ -256,30 +256,32 @@ func Output(o logging.OutputSpec, inputs []string, secret *corev1.Secret, op Opt
 
 func TLSConf(o logging.OutputSpec, secret *corev1.Secret) []Element {
 	conf := []Element{}
-	if o.Secret != nil {
-		hasTLS := false
-		conf = append(conf, security.TLSConf{
-			ComponentID:        helpers.FormatComponentID(o.Name),
-			InsecureSkipVerify: o.TLS != nil && o.TLS.InsecureSkipVerify,
-		})
-		if o.Name == logging.OutputNameDefault || security.HasTLSCertAndKey(secret) {
-			hasTLS = true
-			kc := TLSKeyCert{
-				CertPath: security.SecretPath(o.Secret.Name, constants.ClientCertKey),
-				KeyPath:  security.SecretPath(o.Secret.Name, constants.ClientPrivateKey),
-			}
-			conf = append(conf, kc)
-		}
-		if o.Name == logging.OutputNameDefault || security.HasCABundle(secret) {
-			hasTLS = true
-			ca := CAFile{
-				CAFilePath: security.SecretPath(o.Secret.Name, constants.TrustedCABundleKey),
-			}
-			conf = append(conf, ca)
+
+	hasTLS := false
+	conf = append(conf, security.TLSConf{
+		ComponentID:        helpers.FormatComponentID(o.Name),
+		InsecureSkipVerify: o.TLS != nil && o.TLS.InsecureSkipVerify,
+	})
+	if o.Name == logging.OutputNameDefault || security.HasTLSCertAndKey(secret) {
+		hasTLS = true
+		kc := TLSKeyCert{
+			CertPath: security.SecretPath(o.Secret.Name, constants.ClientCertKey),
+			KeyPath:  security.SecretPath(o.Secret.Name, constants.ClientPrivateKey),
 		}
-		if !hasTLS {
-			return []Element{}
+		conf = append(conf, kc)
+	}
+	if o.Name == logging.OutputNameDefault || security.HasCABundle(secret) {
+		hasTLS = true
+		ca := CAFile{
+			CAFilePath: security.SecretPath(o.Secret.Name, constants.TrustedCABundleKey),
 		}
+		conf = append(conf, ca)
+	}
+	if o.TLS != nil && o.TLS.InsecureSkipVerify {
+		hasTLS = true
+	}
+	if !hasTLS {
+		return []Element{}
 	}
 	return conf
 }

diff --git a/internal/generator/vector/output/elasticsearch/elasticsearch_test.go b/internal/generator/vector/output/elasticsearch/elasticsearch_test.go
@@ -294,6 +294,134 @@ enabled = true
 key_file = "/var/run/ocp-collector/secrets/es-1/tls.key"
 crt_file = "/var/run/ocp-collector/secrets/es-1/tls.crt"
 ca_file = "/var/run/ocp-collector/secrets/es-1/ca-bundle.crt"
+`,
+		}),
+		Entry("with tls without secret", helpers.ConfGenerateTest{
+			CLFSpec: logging.ClusterLogForwarderSpec{
+				Outputs: []logging.OutputSpec{
+					{
+						Type:   logging.OutputTypeElasticsearch,
+						Name:   "es-1",
+						URL:    "http://es.svc.infra.cluster:9200",
+						Secret: nil,
+						TLS: &logging.OutputTLSSpec{
+							InsecureSkipVerify: true,
+						},
+					},
+				},
+			},
+			Secrets: security.NoSecrets,
+			ExpectedConf: `
+# Set Elasticsearch index
+[transforms.es_1_add_es_index]
+type = "remap"
+inputs = ["application"]
+source = '''
+  index = "default"
+  if (.log_type == "application"){
+    index = "app"
+  }
+  if (.log_type == "infrastructure"){
+    index = "infra"
+  }
+  if (.log_type == "audit"){
+    index = "audit"
+  }
+  .write_index = index + "-write"
+  ._id = encode_base64(uuid_v4())
+  del(.file)
+  del(.tag)
+  del(.source_type)
+'''
+
+[transforms.es_1_dedot_and_flatten]
+type = "lua"
+inputs = ["es_1_add_es_index"]
+version = "2"
+hooks.init = "init"
+hooks.process = "process"
+source = '''
+    function init()
+        count = 0
+    end
+    function process(event, emit)
+        count = count + 1
+        event.log.openshift.sequence = count
+        if event.log.kubernetes == nil then
+            emit(event)
+            return
+        end
+        if event.log.kubernetes.labels == nil then
+            emit(event)
+            return
+        end
+        dedot(event.log.kubernetes.namespace_labels)
+        dedot(event.log.kubernetes.labels)
+        flatten_labels(event)
+        prune_labels(event)
+        emit(event)
+    end
+	
+    function dedot(map)
+        if map == nil then
+            return
+        end
+        local new_map = {}
+        local changed_keys = {}
+        for k, v in pairs(map) do
+            local dedotted = string.gsub(k, "[./]", "_")
+            if dedotted ~= k then
+                new_map[dedotted] = v
+                changed_keys[k] = true
+            end
+        end
+        for k in pairs(changed_keys) do
+            map[k] = nil
+        end
+        for k, v in pairs(new_map) do
+            map[k] = v
+        end
+    end
+
+    function flatten_labels(event)
+        -- create "flat_labels" key
+        event.log.kubernetes.flat_labels = {}
+        i = 1
+        -- flatten the labels
+        for k,v in pairs(event.log.kubernetes.labels) do
+          event.log.kubernetes.flat_labels[i] = k.."="..v
+          i=i+1
+        end
+    end 
+
+	function prune_labels(event)
+		local exclusions = {"app_kubernetes_io_name", "app_kubernetes_io_instance", "app_kubernetes_io_version", "app_kubernetes_io_component", "app_kubernetes_io_part-of", "app_kubernetes_io_managed-by", "app_kubernetes_io_created-by"}
+		local keys = {}
+		for k,v in pairs(event.log.kubernetes.labels) do
+			for index, e in pairs(exclusions) do
+				if k == e then
+					keys[k] = v
+				end
+			end
+		end
+		event.log.kubernetes.labels = keys
+	end
+'''
+
+[sinks.es_1]
+type = "elasticsearch"
+inputs = ["es_1_dedot_and_flatten"]
+endpoint = "http://es.svc.infra.cluster:9200"
+bulk.index = "{{ write_index }}"
+bulk.action = "create"
+encoding.except_fields = ["write_index"]
+request.timeout_secs = 2147483648
+id_key = "_id"
+
+[sinks.es_1.tls]
+enabled = true
+verify_certificate = false
+verify_hostname = false
 `,
 		}),
 		Entry("without security", helpers.ConfGenerateTest{

diff --git a/internal/generator/vector/output/loki/loki.go b/internal/generator/vector/output/loki/loki.go
@@ -191,32 +191,35 @@ func Tenant(l *logging.Loki) Element {
 
 func TLSConf(o logging.OutputSpec, secret *corev1.Secret) []Element {
 	conf := []Element{}
-	if o.Secret != nil {
-		hasTLS := false
-		conf = append(conf, security.TLSConf{
-			ComponentID:        strings.ToLower(vectorhelpers.Replacer.Replace(o.Name)),
-			InsecureSkipVerify: o.TLS != nil && o.TLS.InsecureSkipVerify,
-		})
 
-		if o.Name == logging.OutputNameDefault || security.HasTLSCertAndKey(secret) {
-			hasTLS = true
-			kc := TLSKeyCert{
-				CertPath: security.SecretPath(o.Secret.Name, constants.ClientCertKey),
-				KeyPath:  security.SecretPath(o.Secret.Name, constants.ClientPrivateKey),
-			}
-			conf = append(conf, kc)
-		}
-		if o.Name == logging.OutputNameDefault || security.HasCABundle(secret) {
-			hasTLS = true
-			ca := CAFile{
-				CAFilePath: security.SecretPath(o.Secret.Name, constants.TrustedCABundleKey),
-			}
-			conf = append(conf, ca)
+	hasTLS := false
+	conf = append(conf, security.TLSConf{
+		ComponentID:        strings.ToLower(vectorhelpers.Replacer.Replace(o.Name)),
+		InsecureSkipVerify: o.TLS != nil && o.TLS.InsecureSkipVerify,
+	})
+
+	if o.Name == logging.OutputNameDefault || security.HasTLSCertAndKey(secret) {
+		hasTLS = true
+		kc := TLSKeyCert{
+			CertPath: security.SecretPath(o.Secret.Name, constants.ClientCertKey),
+			KeyPath:  security.SecretPath(o.Secret.Name, constants.ClientPrivateKey),
 		}
-		if !hasTLS {
-			return []Element{}
+		conf = append(conf, kc)
+	}
+	if o.Name == logging.OutputNameDefault || security.HasCABundle(secret) {
+		hasTLS = true
+		ca := CAFile{
+			CAFilePath: security.SecretPath(o.Secret.Name, constants.TrustedCABundleKey),
 		}
-	} else if secret != nil {
+		conf = append(conf, ca)
+	}
+	if o.TLS != nil && o.TLS.InsecureSkipVerify {
+		hasTLS = true
+	}
+	if !hasTLS {
+		return []Element{}
+	}
+	if o.Secret == nil && secret != nil {
 		// Set CA from logcollector ServiceAccount for internal Loki
 		return []Element{
 			security.TLSConf{