From 698f465cd7da7e9c7c89d4d00b33e3938fb2bf5a Mon Sep 17 00:00:00 2001 From: David Grove Date: Sun, 27 Mar 2022 17:21:31 -0400 Subject: [PATCH] support for logging stack Co-authored-by: Jaroslaw Cwiklik cwiklik@Jaroslaws-MacBook-Pro.local --- scripts/logging/README | 46 ++++++ scripts/logging/add-efk-node.sh | 18 +++ .../elasticsearch/elasticsearch_svc.yaml | 32 ++++ .../elasticsearch_statefulset.template.yaml | 115 ++++++++++++++ scripts/logging/fluentd/aggregator-cm.yaml | 62 ++++++++ scripts/logging/fluentd/aggregator.yaml | 140 ++++++++++++++++++ scripts/logging/fluentd/collector-cm.yaml | 80 ++++++++++ scripts/logging/fluentd/collector.yaml | 125 ++++++++++++++++ scripts/logging/fluentd/fluentd-agg-svc.yaml | 27 ++++ scripts/logging/kibana/kibana.yaml | 67 +++++++++ scripts/start-k3d-logstack.sh | 85 +++++++++++ scripts/stop-k3d-logstack.sh | 38 +++++ 12 files changed, 835 insertions(+) create mode 100644 scripts/logging/README create mode 100755 scripts/logging/add-efk-node.sh create mode 100644 scripts/logging/elasticsearch/elasticsearch_svc.yaml create mode 100644 scripts/logging/elasticsearch/template/elasticsearch_statefulset.template.yaml create mode 100644 scripts/logging/fluentd/aggregator-cm.yaml create mode 100644 scripts/logging/fluentd/aggregator.yaml create mode 100644 scripts/logging/fluentd/collector-cm.yaml create mode 100644 scripts/logging/fluentd/collector.yaml create mode 100644 scripts/logging/fluentd/fluentd-agg-svc.yaml create mode 100644 scripts/logging/kibana/kibana.yaml create mode 100755 scripts/start-k3d-logstack.sh create mode 100755 scripts/stop-k3d-logstack.sh diff --git a/scripts/logging/README b/scripts/logging/README new file mode 100644 index 00000000..d7deeec9 --- /dev/null +++ b/scripts/logging/README @@ -0,0 +1,46 @@ +# +# Copyright IBM Corporation 2020,2021 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +EFK log stack install +======================= + +cd kar/scripts/efk + +With k3d cluster running, add a new cluster node to host EFK specific pods by running: +./add-efk-node.sh + +To deploy EFK log stack, run the following from kar/scripts: +./start-k3d-logstack.sh + +Once EFK log stack deploys succefully, access Kibana GUI by forwarding local port to the Kubernetes node hosting Kibana pod. + +kubectl port-forward svc/kibana 5602:5601 --namespace=logging + +Deploy reefer application. + +Next, in your web browser, visit the following URL: http://localhost:5602 +When the GUI loads, there is a vertical menu on the very left side of the app. The first active icon (from the top) in this menu will +show "Discover" in its tooltip., click it to show "Create Index Pattern" screen with Index Pattern value= index-patttern-*. Replace the +value with logstash-*, and press Next Step button. You will then see "Configure Settings" screen. Click on a drop-down +selector and choose "@timestamp" field, and hit Create Index Pattern button. Now, hit Discover again on the left hand navigation menu. +You should see log entries from the reefer app, sorted by time (newest on top). + + +To remove EFK log stack, run the following from kar/scripts: +./stop-k3d-logstack.sh + + + diff --git a/scripts/logging/add-efk-node.sh b/scripts/logging/add-efk-node.sh new file mode 100755 index 00000000..db2f3179 --- /dev/null +++ b/scripts/logging/add-efk-node.sh @@ -0,0 +1,18 @@ +# +# Copyright IBM Corporation 2020,2021 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +k3d node create efk-master --wait +kubectl label nodes k3d-efk-master-0 efk-type=master diff --git a/scripts/logging/elasticsearch/elasticsearch_svc.yaml b/scripts/logging/elasticsearch/elasticsearch_svc.yaml new file mode 100644 index 00000000..3c7d2c6e --- /dev/null +++ b/scripts/logging/elasticsearch/elasticsearch_svc.yaml @@ -0,0 +1,32 @@ +# +# Copyright IBM Corporation 2020,2021 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +kind: Service +apiVersion: v1 +metadata: + name: elasticsearch + namespace: logging + labels: + app: elasticsearch +spec: + selector: + app: elasticsearch + clusterIP: None + ports: + - port: 9200 + name: rest + - port: 9300 + name: inter-node \ No newline at end of file diff --git a/scripts/logging/elasticsearch/template/elasticsearch_statefulset.template.yaml b/scripts/logging/elasticsearch/template/elasticsearch_statefulset.template.yaml new file mode 100644 index 00000000..8f4e3a95 --- /dev/null +++ b/scripts/logging/elasticsearch/template/elasticsearch_statefulset.template.yaml @@ -0,0 +1,115 @@ +# +# Copyright IBM Corporation 2020,2021 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: elasticsearch + namespace: logging +spec: + serviceName: elasticsearch + replicas: 1 + selector: + matchLabels: + app: elasticsearch + template: + metadata: + labels: + app: elasticsearch + spec: + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: efk-type + operator: In + values: + - master + hostname: elasticsearch-0 + containers: + - name: elasticsearch + image: docker.elastic.co/elasticsearch/elasticsearch:7.2.0 + resources: + limits: + cpu: 1000m + requests: + cpu: 100m + ports: + - containerPort: 9200 + name: rest + protocol: TCP + - containerPort: 9300 + name: inter-node + protocol: TCP + env: + - name: cluster.name + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.name + - name: discovery.seed_hosts + value: elasticsearch-0 + - name: cluster.initial_master_nodes + value: elasticsearch-0 + - name: network.host + value: 0.0.0.0 + - name: node.data + value: "true" + - name: node.ingest + value: "true" + - name: node.master + value: "true" + - name: ES_JAVA_OPTS + value: "-Xms512m -Xmx512m" + - name: "xpack.security.enabled" + value: "false" + - name: discovery.zen.minimum_master_nodes + value: "1" + volumeMounts: + - name: data + mountPath: {{data-dir}} + + initContainers: + - name: fix-permissions + image: busybox + command: ["sh", "-c", "chown -R 1000:1000 {{data-dir}}"] + securityContext: + privileged: true + volumeMounts: + - name: data + mountPath: {{data-dir}} + - name: increase-vm-max-map + image: busybox + command: ["sysctl", "-w", "vm.max_map_count=262144"] + securityContext: + privileged: true + - name: increase-fd-ulimit + image: busybox + command: ["sh", "-c", "ulimit -n 65536"] + securityContext: + privileged: true + volumeClaimTemplates: + - metadata: + name: data + labels: + app: elasticsearch + spec: + accessModes: [ "ReadWriteOnce" ] +# storageClassName: "standard" + resources: + requests: + storage: {{es-storage-size}} diff --git a/scripts/logging/fluentd/aggregator-cm.yaml b/scripts/logging/fluentd/aggregator-cm.yaml new file mode 100644 index 00000000..f47533f5 --- /dev/null +++ b/scripts/logging/fluentd/aggregator-cm.yaml @@ -0,0 +1,62 @@ +# +# Copyright IBM Corporation 2020,2021 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +apiVersion: v1 +data: + fluent.conf: | + + @type forward + bind 0.0.0.0 + port 24224 + + + @type http + bind 0.0.0.0 + port 9880 + + + @type stdout + + + @type null + + + @type null + + + @id filter_kubernetes_metadata + @type kubernetes_metadata + + + + @type elasticsearch + host elasticsearch.logging + logstash_format true + port 9200 + path "" + + flush_at_shutdown true + flush_thread_count 8 + flush_thread_interval 20 + flush_thread_burst_interval 1 + retry_forever true + retry_type exponential_backoff + + +kind: ConfigMap +metadata: + name: fluentd-aggregator-conf + namespace: logging diff --git a/scripts/logging/fluentd/aggregator.yaml b/scripts/logging/fluentd/aggregator.yaml new file mode 100644 index 00000000..047565f6 --- /dev/null +++ b/scripts/logging/fluentd/aggregator.yaml @@ -0,0 +1,140 @@ +# +# Copyright IBM Corporation 2020,2021 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +apiVersion: v1 +kind: ServiceAccount +metadata: + name: fluentd-agg +# namespace: logging + labels: + app: fluentd-agg +--- + +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: fluentd-agg +# namespace: logging + labels: + app: fluentd-agg +rules: +- apiGroups: + - "" + resources: + - pods + - namespaces + verbs: + - get + - list + - watch +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: fluentd-agg +# namespace: logging +roleRef: + kind: ClusterRole + name: fluentd-agg + apiGroup: rbac.authorization.k8s.io +subjects: +- kind: ServiceAccount + name: fluentd-agg + namespace: logging +--- +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: fluentd-agg + namespace: logging + labels: + app: fluentd-agg +spec: + selector: + matchLabels: + app: fluentd-agg + template: + metadata: + labels: + app: fluentd-agg + spec: + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: efk-type + operator: In + values: + - master + + serviceAccount: fluentd-agg + serviceAccountName: fluentd-agg + tolerations: + - key: node-role.kubernetes.io/master + effect: NoSchedule + + containers: + - name: fluentd-agg + image: fluent/fluentd-kubernetes-daemonset:v1.4.2-debian-elasticsearch-1.1 + env: + - name: FLUENT_ELASTICSEARCH_HOST + value: "elasticsearch.logging.svc.cluster.local" + - name: FLUENT_ELASTICSEARCH_PORT + value: "9200" + - name: FLUENT_ELASTICSEARCH_SCHEME + value: "http" + - name: FLUENT_ELASTICSEARCH_USER + value: "user" + - name: FLUENT_ELASTICSEARCH_PASSWORD + value: "changeme" +# - name: FLUENT_CONTAINER_TAIL_EXCLUDE_PATH +# value: > +# [ +# "/var/log/containers/fluentd-agg-*", +# "/var/log/containers/svclb*", +# "/var/log/containers/kibana*", +# "/var/log/containers/elasticsearch*" +# ] + - name: FLUENTD_SYSTEMD_CONF + value: disable +# - name: FLUENT_CONTAINER_TAIL_PARSER_TYPE +# value: /^(?