From 4166b7f48d74ac218e61d89137f5bc36198cf98c Mon Sep 17 00:00:00 2001 From: craig Date: Wed, 22 Feb 2023 19:19:11 +0800 Subject: [PATCH] feat: add efk for log collection --- helm/templates/_helpers.tpl | 28 ++++++++ helm/templates/elasticsearch.yaml | 97 ++++++++++++++++++++++++++++ helm/templates/fluentd.yaml | 102 ++++++++++++++++++++++++++++++ helm/templates/kibana.yaml | 47 ++++++++++++++ helm/values.yaml | 48 +++++++++++++- 5 files changed, 321 insertions(+), 1 deletion(-) create mode 100644 helm/templates/elasticsearch.yaml create mode 100644 helm/templates/fluentd.yaml create mode 100644 helm/templates/kibana.yaml diff --git a/helm/templates/_helpers.tpl b/helm/templates/_helpers.tpl index a00d782c..4f8e2c60 100644 --- a/helm/templates/_helpers.tpl +++ b/helm/templates/_helpers.tpl @@ -73,3 +73,31 @@ Create a default fully qualified app name for Prometheus. {{- $consoleFullName := include "higress-console.fullname" . }} {{- printf "%s-prometheus" ($consoleFullName | trunc 52) }} {{- end }} + +{{- define "kubernetes.conf" }} +{{- range $target := .Values.fluentd.targets }} + + @type tail + @id in_tail_container_logs + path /var/log/containers/*_{{ $target.namespace }}_{{ $target.name }}-*.log + pos_file /var/log/fluentd-containers.log.pos + tag kubernetes.* + read_from_head true + + @type regexp + expression /^(? + +{{- end }} + + @type parser + + @type json + json_parser json + + replace_invalid_sequence true + emit_invalid_record_to_error false + key_name log + reserve_data true + +{{- end }} diff --git a/helm/templates/elasticsearch.yaml b/helm/templates/elasticsearch.yaml new file mode 100644 index 00000000..5f1b665a --- /dev/null +++ b/helm/templates/elasticsearch.yaml @@ -0,0 +1,97 @@ +apiVersion: v1 +kind: Service +metadata: + name: elasticsearch + namespace: {{ .Values.elasticsearch.namespace }} + labels: + app: elasticsearch +spec: + selector: + app: elasticsearch + clusterIP: None + ports: + - port: 9200 + name: rest + - port: 9300 + name: inter-node + +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: es + namespace: {{ .Values.elasticsearch.namespace }} +spec: + serviceName: elasticsearch + replicas: {{ .Values.elasticsearch.replicas }} + selector: + matchLabels: + app: elasticsearch + template: + metadata: + labels: + app: elasticsearch + spec: + containers: + - name: elasticsearch + image: docker.elastic.co/elasticsearch/elasticsearch:7.17.0 + resources: + limits: + cpu: {{ .Values.elasticsearch.resources.limits.cpu }} + memory: {{ .Values.elasticsearch.resources.limits.memory }} + requests: + cpu: {{ .Values.elasticsearch.resources.requests.cpu }} + memory: {{ .Values.elasticsearch.resources.requests.memory }} + ports: + - containerPort: 9200 + name: rest + protocol: TCP + - containerPort: 9300 + name: inter-node + protocol: TCP + volumeMounts: + - name: data + mountPath: /usr/share/elasticsearch/data + env: + - name: cluster.name + value: k8s-logs + - name: node.name + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: discovery.seed_hosts + value: "es-0.elasticsearch,es-1.elasticsearch,es-2.elasticsearch" + - name: cluster.initial_master_nodes + value: "es-0,es-1,es-2" + - name: ES_JAVA_OPTS + value: {{ .Values.elasticsearch.env.javaOpts }} + initContainers: + - name: fix-permissions + image: busybox + command: ["sh", "-c", "chown -R 1000:1000 /usr/share/elasticsearch/data"] + securityContext: + privileged: true + volumeMounts: + - name: data + mountPath: /usr/share/elasticsearch/data + - name: increase-vm-max-map + image: busybox + command: ["sysctl", "-w", "vm.max_map_count={{ .Values.elasticsearch.env.mapMapCount }}"] + securityContext: + privileged: true + - name: increase-fd-ulimit + image: busybox + command: ["sh", "-c", "ulimit -n {{ .Values.elasticsearch.env.ulimit }}"] + securityContext: + privileged: true + volumeClaimTemplates: + - metadata: + name: data + labels: + app: elasticsearch + spec: + accessModes: [ {{ .Values.elasticsearch.volumeClaim.accessMode }} ] + storageClassName: {{ .Values.elasticsearch.volumeClaim.storageClassName }} + resources: + requests: + storage: {{ .Values.elasticsearch.volumeClaim.storage }} \ No newline at end of file diff --git a/helm/templates/fluentd.yaml b/helm/templates/fluentd.yaml new file mode 100644 index 00000000..6851a875 --- /dev/null +++ b/helm/templates/fluentd.yaml @@ -0,0 +1,102 @@ +kind: ConfigMap +apiVersion: v1 +metadata: + name: fluentd-config + namespace: {{ .Values.fluentd.namespace }} +data: + kubernetes.conf: |- + {{- include "kubernetes.conf" . | nindent 4 }} + +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: fluentd + labels: + app: fluentd +rules: + - apiGroups: + - "" + resources: + - pods + - namespaces + verbs: + - get + - list + - watch + +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: fluentd + namespace: {{ .Values.fluentd.namespace }} + labels: + app: fluentd + +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: fluentd +roleRef: + kind: ClusterRole + name: fluentd + apiGroup: rbac.authorization.k8s.io +subjects: + - kind: ServiceAccount + name: fluentd + namespace: {{ .Values.fluentd.namespace }} + +--- +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: fluentd + namespace: {{ .Values.fluentd.namespace }} + labels: + app: fluentd +spec: + selector: + matchLabels: + app: fluentd + template: + metadata: + labels: + app: fluentd + spec: + serviceAccount: fluentd + serviceAccountName: fluentd + containers: + - name: fluentd + image: fluent/fluentd-kubernetes-daemonset:v1.15.3-debian-elasticsearch7-1.0 + env: + - name: FLUENT_ELASTICSEARCH_HOST + value: "elasticsearch.{{ .Values.fluentd.namespace }}.svc.cluster.local" + - name: FLUENT_ELASTICSEARCH_PORT + value: "9200" + - name: FLUENT_ELASTICSEARCH_SCHEME + value: "http" + - name: FLUENTD_SYSTEMD_CONF + value: disable + resources: + limits: + cpu: {{ .Values.fluentd.resources.limits.cpu }} + memory: {{ .Values.fluentd.resources.limits.memory }} + requests: + cpu: {{ .Values.fluentd.resources.requests.cpu }} + memory: {{ .Values.fluentd.resources.requests.memory }} + volumeMounts: + - name: varlog + mountPath: /var/log + - name: config + mountPath: /fluentd/etc/kubernetes.conf + subPath: kubernetes.conf + terminationGracePeriodSeconds: 30 + volumes: + - name: varlog + hostPath: + path: /var/log + - name: config + configMap: + name: fluentd-config \ No newline at end of file diff --git a/helm/templates/kibana.yaml b/helm/templates/kibana.yaml new file mode 100644 index 00000000..e2a3dfea --- /dev/null +++ b/helm/templates/kibana.yaml @@ -0,0 +1,47 @@ +apiVersion: v1 +kind: Service +metadata: + name: kibana-np + namespace: {{ .Values.kibana.namespace }} +spec: + selector: + app: kibana + type: NodePort + ports: + - port: 8080 + targetPort: 5601 + nodePort: 30000 + +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: kibana + namespace: {{ .Values.kibana.namespace }} + labels: + app: kibana +spec: + replicas: 1 + selector: + matchLabels: + app: kibana + template: + metadata: + labels: + app: kibana + spec: + containers: + - name: kibana + image: docker.elastic.co/kibana/kibana:7.17.0 + resources: + limits: + cpu: {{ .Values.kibana.resources.limits.cpu }} + memory: {{ .Values.kibana.resources.limits.memory }} + requests: + cpu: {{ .Values.kibana.resources.requests.cpu }} + memory: {{ .Values.kibana.resources.requests.memory }} + env: + - name: ELASTICSEARCH_URL + value: http://elasticsearch:9200 + ports: + - containerPort: 5601 \ No newline at end of file diff --git a/helm/values.yaml b/helm/values.yaml index 193cce6c..82b79dba 100644 --- a/helm/values.yaml +++ b/helm/values.yaml @@ -83,4 +83,50 @@ prometheus: memory: 100Mi limits: cpu: 500m - memory: 2Gi \ No newline at end of file + memory: 2Gi + +elasticsearch: + namespace: efk + replicas: 3 + env: + javaOpts: "-Xms512m -Xmx512m" + maxMapCount: 262144 + ulimit: 65536 + resources: + limits: + cpu: 1000m + memory: 4Gi + requests: + cpu: 100m + memory: 256Mi + volumeClaim: + accessMode: ReadWriteOnce + storageClassName: standard + storage: 3Gi +kibana: + namespace: efk + resources: + limits: + cpu: 1000m + memory: 1Gi + requests: + cpu: 100m + memory: 256Mi +fluentd: + namespace: efk + resources: + limits: + cpu: 1000m + memory: 512Mi + requests: + cpu: 100m + memory: 200Mi + targets: + - namespace: higress-system + name: higress-gateway + es: + host: elasticsearch.{{ .Values.elasticsearch.namespace }} + port: 9200 + flushThreadCount: 2 + flushInterval: 5s + retryMaxInterval: 30