diff --git a/api/v1/common_types.go b/api/v1/common_types.go index 3321dce..adf8b89 100644 --- a/api/v1/common_types.go +++ b/api/v1/common_types.go @@ -67,9 +67,11 @@ type LogCollection struct { // +kubebuilder:default:={"requests":{"cpu":"100m","memory":"200Mi"},"limits":{"cpu":"200m","memory":"500Mi"}} Resources *corev1.ResourceRequirements `json:"resources,omitempty"` // +kubebuilder:default:={errorLogs: true, accessLogs: true, requestLogs: true, crashLogs: true, auditLogs: true} - Files LogFilesConfig `json:"files,omitempty"` - // +kubebuilder:default:="[OUTPUT]\n name loki\n match *\n host loki.default.svc.cluster.local\n port 3100\n labels job=fluent-bit\n http_user admin\n http_passwd admin" - Outputs string `json:"outputs,omitempty"` + Files LogFilesConfig `json:"files,omitempty"` + Outputs string `json:"outputs,omitempty"` + Filters string `json:"filters,omitempty"` + Inputs string `json:"inputs,omitempty"` + Parsers string `json:"parsers,omitempty"` } type LogFilesConfig struct { diff --git a/charts/marklogic-operator-kubernetes/templates/marklogiccluster-crd.yaml b/charts/marklogic-operator-kubernetes/templates/marklogiccluster-crd.yaml index 4a71d0e..0718272 100644 --- a/charts/marklogic-operator-kubernetes/templates/marklogiccluster-crd.yaml +++ b/charts/marklogic-operator-kubernetes/templates/marklogiccluster-crd.yaml @@ -4735,6 +4735,8 @@ spec: requestLogs: type: boolean type: object + filters: + type: string image: default: fluent/fluent-bit:4.1.1 type: string @@ -4756,16 +4758,11 @@ spec: type: object x-kubernetes-map-type: atomic type: array + inputs: + type: string outputs: - default: |- - [OUTPUT] - name loki - match * - host loki.default.svc.cluster.local - port 3100 - labels job=fluent-bit - http_user admin - http_passwd admin + type: string + parsers: type: string resources: default: @@ -8284,6 +8281,8 @@ spec: requestLogs: type: boolean type: object + filters: + type: string image: default: fluent/fluent-bit:4.1.1 type: string @@ -8305,16 +8304,11 @@ spec: type: object x-kubernetes-map-type: atomic type: array + inputs: + type: string outputs: - default: |- - [OUTPUT] - name loki - match * - host loki.default.svc.cluster.local - port 3100 - labels job=fluent-bit - http_user admin - http_passwd admin + type: string + parsers: type: string resources: default: diff --git a/charts/marklogic-operator-kubernetes/templates/marklogicgroup-crd.yaml b/charts/marklogic-operator-kubernetes/templates/marklogicgroup-crd.yaml index 4eb6f1f..7ffdfde 100644 --- a/charts/marklogic-operator-kubernetes/templates/marklogicgroup-crd.yaml +++ b/charts/marklogic-operator-kubernetes/templates/marklogicgroup-crd.yaml @@ -3487,6 +3487,8 @@ spec: requestLogs: type: boolean type: object + filters: + type: string image: default: fluent/fluent-bit:4.1.1 type: string @@ -3508,16 +3510,11 @@ spec: type: object x-kubernetes-map-type: atomic type: array + inputs: + type: string outputs: - default: |- - [OUTPUT] - name loki - match * - host loki.default.svc.cluster.local - port 3100 - labels job=fluent-bit - http_user admin - http_passwd admin + type: string + parsers: type: string resources: default: diff --git a/config/crd/bases/marklogic.progress.com_marklogicclusters.yaml b/config/crd/bases/marklogic.progress.com_marklogicclusters.yaml index 97da7ac..25d96b2 100644 --- a/config/crd/bases/marklogic.progress.com_marklogicclusters.yaml +++ b/config/crd/bases/marklogic.progress.com_marklogicclusters.yaml @@ -4744,6 +4744,8 @@ spec: requestLogs: type: boolean type: object + filters: + type: string image: default: fluent/fluent-bit:4.1.1 type: string @@ -4765,16 +4767,11 @@ spec: type: object x-kubernetes-map-type: atomic type: array + inputs: + type: string outputs: - default: |- - [OUTPUT] - name loki - match * - host loki.default.svc.cluster.local - port 3100 - labels job=fluent-bit - http_user admin - http_passwd admin + type: string + parsers: type: string resources: default: @@ -8306,6 +8303,8 @@ spec: requestLogs: type: boolean type: object + filters: + type: string image: default: fluent/fluent-bit:4.1.1 type: string @@ -8327,16 +8326,11 @@ spec: type: object x-kubernetes-map-type: atomic type: array + inputs: + type: string outputs: - default: |- - [OUTPUT] - name loki - match * - host loki.default.svc.cluster.local - port 3100 - labels job=fluent-bit - http_user admin - http_passwd admin + type: string + parsers: type: string resources: default: diff --git a/config/crd/bases/marklogic.progress.com_marklogicgroups.yaml b/config/crd/bases/marklogic.progress.com_marklogicgroups.yaml index eb55ac0..9b1df84 100644 --- a/config/crd/bases/marklogic.progress.com_marklogicgroups.yaml +++ b/config/crd/bases/marklogic.progress.com_marklogicgroups.yaml @@ -3494,6 +3494,8 @@ spec: requestLogs: type: boolean type: object + filters: + type: string image: default: fluent/fluent-bit:4.1.1 type: string @@ -3515,16 +3517,11 @@ spec: type: object x-kubernetes-map-type: atomic type: array + inputs: + type: string outputs: - default: |- - [OUTPUT] - name loki - match * - host loki.default.svc.cluster.local - port 3100 - labels job=fluent-bit - http_user admin - http_passwd admin + type: string + parsers: type: string resources: default: diff --git a/pkg/k8sutil/configmap.go b/pkg/k8sutil/configmap.go index 532d8a8..79fd3b0 100644 --- a/pkg/k8sutil/configmap.go +++ b/pkg/k8sutil/configmap.go @@ -168,10 +168,12 @@ func (oc *OperatorContext) getFluentBitData() map[string]string { pipeline: inputs:` - // Add INPUT sections based on enabled log types - if oc.MarklogicGroup.Spec.LogCollection.Files.ErrorLogs { - fluentBitData["fluent-bit.yaml"] += ` + if strings.TrimSpace(oc.MarklogicGroup.Spec.LogCollection.Inputs) != "" { + fluentBitData["fluent-bit.yaml"] += "\n" + normalizeYAMLIndentation(oc.MarklogicGroup.Spec.LogCollection.Inputs, 4, 6) + } else { + if oc.MarklogicGroup.Spec.LogCollection.Files.ErrorLogs { + fluentBitData["fluent-bit.yaml"] += ` - name: tail path: /var/opt/MarkLogic/Logs/*ErrorLog.txt read_from_head: true @@ -179,10 +181,10 @@ pipeline: path_key: path parser: error_parser mem_buf_limit: 4MB` - } + } - if oc.MarklogicGroup.Spec.LogCollection.Files.AccessLogs { - fluentBitData["fluent-bit.yaml"] += ` + if oc.MarklogicGroup.Spec.LogCollection.Files.AccessLogs { + fluentBitData["fluent-bit.yaml"] += ` - name: tail path: /var/opt/MarkLogic/Logs/*AccessLog.txt read_from_head: true @@ -190,10 +192,10 @@ pipeline: path_key: path parser: access_parser mem_buf_limit: 4MB` - } + } - if oc.MarklogicGroup.Spec.LogCollection.Files.RequestLogs { - fluentBitData["fluent-bit.yaml"] += ` + if oc.MarklogicGroup.Spec.LogCollection.Files.RequestLogs { + fluentBitData["fluent-bit.yaml"] += ` - name: tail path: /var/opt/MarkLogic/Logs/*RequestLog.txt read_from_head: true @@ -201,88 +203,72 @@ pipeline: path_key: path parser: json_parser mem_buf_limit: 4MB` - } + } - if oc.MarklogicGroup.Spec.LogCollection.Files.CrashLogs { - fluentBitData["fluent-bit.yaml"] += ` + if oc.MarklogicGroup.Spec.LogCollection.Files.CrashLogs { + fluentBitData["fluent-bit.yaml"] += ` - name: tail path: /var/opt/MarkLogic/Logs/CrashLog.txt read_from_head: true tag: kube.marklogic.logs.crash path_key: path mem_buf_limit: 4MB` - } + } - if oc.MarklogicGroup.Spec.LogCollection.Files.AuditLogs { - fluentBitData["fluent-bit.yaml"] += ` + if oc.MarklogicGroup.Spec.LogCollection.Files.AuditLogs { + fluentBitData["fluent-bit.yaml"] += ` - name: tail path: /var/opt/MarkLogic/Logs/AuditLog.txt read_from_head: true tag: kube.marklogic.logs.audit path_key: path mem_buf_limit: 4MB` + } } // Add FILTER sections fluentBitData["fluent-bit.yaml"] += ` - filters: - - name: modify - match: "*" - add: pod ${POD_NAME} - add: namespace ${NAMESPACE} - - name: modify - match: kube.marklogic.logs.error - add: tag kube.marklogic.logs.error - - name: modify - match: kube.marklogic.logs.access - add: tag kube.marklogic.logs.access - - name: modify - match: kube.marklogic.logs.request - add: tag kube.marklogic.logs.request - - name: modify - match: kube.marklogic.logs.audit - add: tag kube.marklogic.logs.audit - - name: modify - match: kube.marklogic.logs.crash - add: tag kube.marklogic.logs.crash + filters:` + if strings.TrimSpace(oc.MarklogicGroup.Spec.LogCollection.Filters) != "" { + fluentBitData["fluent-bit.yaml"] += "\n" + normalizeYAMLIndentation(oc.MarklogicGroup.Spec.LogCollection.Filters, 4, 6) + } else { + fluentBitData["fluent-bit.yaml"] += ` + - name: modify + match: "*" + add: + - pod ${POD_NAME} + - namespace ${NAMESPACE} + - name: modify + match: kube.marklogic.logs.error + add: + - tag kube.marklogic.logs.error + - name: modify + match: kube.marklogic.logs.access + add: + - tag kube.marklogic.logs.access + - name: modify + match: kube.marklogic.logs.request + add: + - tag kube.marklogic.logs.request + - name: modify + match: kube.marklogic.logs.audit + add: + - tag kube.marklogic.logs.audit + - name: modify + match: kube.marklogic.logs.crash + add: + - tag kube.marklogic.logs.crash + ` + } - outputs:` + // Add OUTPUT sections + fluentBitData["fluent-bit.yaml"] += ` + outputs:` // Handle user-defined outputs from LogCollection.Outputs - if oc.MarklogicGroup.Spec.LogCollection.Outputs != "" { - // Process user-defined outputs, adjusting indentation to match pipeline level - outputs := oc.MarklogicGroup.Spec.LogCollection.Outputs - // Split into lines and process indentation - lines := strings.Split(outputs, "\n") - processedLines := make([]string, 0, len(lines)) - for _, line := range lines { - if strings.TrimSpace(line) == "" { - continue // Skip empty lines - } - // Count leading spaces in original line - leadingSpaces := len(line) - len(strings.TrimLeft(line, " ")) - content := strings.TrimLeft(line, " ") - if content != "" { - // For YAML list items starting with "-", use 4 spaces base indentation - // For properties under list items, use 6 spaces base indentation - var newIndent int - const yamlListItemIndent = 4 - const yamlPropertyIndent = 6 - if strings.HasPrefix(content, "- ") { - newIndent = yamlListItemIndent // List items at pipeline outputs level - } else { - newIndent = yamlPropertyIndent // Properties under list items - } - // Add any additional relative indentation beyond the first level - const baseIndentOffset = 2 - if leadingSpaces > baseIndentOffset { - newIndent += leadingSpaces - baseIndentOffset - } - processedLines = append(processedLines, strings.Repeat(" ", newIndent)+content) - } - } - fluentBitData["fluent-bit.yaml"] += "\n" + strings.Join(processedLines, "\n") + if strings.TrimSpace(oc.MarklogicGroup.Spec.LogCollection.Outputs) != "" { + fluentBitData["fluent-bit.yaml"] += "\n" + normalizeYAMLIndentation(oc.MarklogicGroup.Spec.LogCollection.Outputs, 4, 6) } else { // Default stdout output if none specified fluentBitData["fluent-bit.yaml"] += ` @@ -292,7 +278,11 @@ pipeline: } // Parsers in YAML format - fluentBitData["parsers.yaml"] = `parsers: + fluentBitData["parsers.yaml"] = `parsers:` + if strings.TrimSpace(oc.MarklogicGroup.Spec.LogCollection.Parsers) != "" { + fluentBitData["parsers.yaml"] += "\n" + normalizeYAMLIndentation(oc.MarklogicGroup.Spec.LogCollection.Parsers, 2, 4) + } else { + fluentBitData["parsers.yaml"] += ` - name: error_parser format: regex regex: ^(?