support for logging stack

scripts/logging/README

@@ -0,0 +1,46 @@
+#
+# Copyright IBM Corporation 2020,2021
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+EFK log stack install
+=======================
+
+cd kar/scripts/efk
+
+With k3d cluster running, add a new cluster node to host EFK specific pods by running:
+./add-efk-node.sh
+
+To deploy EFK log stack, run the following from kar/scripts:
+./start-k3d-logstack.sh
+
+Once EFK log stack deploys succefully, access Kibana GUI by forwarding local port to the Kubernetes node hosting Kibana pod.
+
+kubectl port-forward svc/kibana 5602:5601 --namespace=logging
+
+Deploy reefer application.
+
+Next, in your web browser, visit the following URL: http://localhost:5602
+When the GUI loads, there is a vertical menu on the very left side of the app. The first active icon (from the top) in this menu will
+show "Discover" in its tooltip., click it to show "Create Index Pattern" screen with Index Pattern value= index-patttern-*. Replace the
+value with logstash-*, and press Next Step button. You will then see "Configure Settings" screen. Click on a drop-down
+selector and choose "@timestamp" field, and hit Create Index Pattern button. Now, hit Discover again on the left hand navigation menu.
+You should see log entries from the reefer app, sorted by time (newest on top).
+
+
+To remove EFK log stack, run the following from kar/scripts:
+./stop-k3d-logstack.sh
+
+
+

scripts/logging/add-efk-node.sh

@@ -0,0 +1,18 @@
+#
+# Copyright IBM Corporation 2020,2021
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+k3d node create efk-master --wait
+kubectl label nodes k3d-efk-master-0 efk-type=master

scripts/logging/elasticsearch/elasticsearch_svc.yaml

@@ -0,0 +1,32 @@
+#
+# Copyright IBM Corporation 2020,2021
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+kind: Service
+apiVersion: v1
+metadata:
+  name: elasticsearch
+  namespace: logging
+  labels:
+    app: elasticsearch
+spec:
+  selector:
+    app: elasticsearch
+  clusterIP: None
+  ports:
+    - port: 9200
+      name: rest
+    - port: 9300
+      name: inter-node

scripts/logging/elasticsearch/template/elasticsearch_statefulset.template.yaml

@@ -0,0 +1,115 @@
+#
+# Copyright IBM Corporation 2020,2021
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+  name: elasticsearch
+  namespace: logging
+spec:
+  serviceName: elasticsearch
+  replicas: 1
+  selector:
+    matchLabels:
+      app: elasticsearch
+  template:
+    metadata:
+      labels:
+        app: elasticsearch
+    spec:
+      affinity:
+        nodeAffinity:
+          requiredDuringSchedulingIgnoredDuringExecution:
+            nodeSelectorTerms:
+            - matchExpressions:
+              - key: efk-type
+                operator: In
+                values:
+                - master
+      hostname: elasticsearch-0
+      containers:
+      - name: elasticsearch
+        image: docker.elastic.co/elasticsearch/elasticsearch:7.2.0
+        resources:
+            limits:
+              cpu: 1000m
+            requests:
+              cpu: 100m
+        ports:
+        - containerPort: 9200
+          name: rest
+          protocol: TCP
+        - containerPort: 9300
+          name: inter-node
+          protocol: TCP
+        env:
+          - name: cluster.name
+            valueFrom:
+              fieldRef:
+                 apiVersion: v1
+                 fieldPath: metadata.name
+          - name: discovery.seed_hosts
+            value: elasticsearch-0
+          - name: cluster.initial_master_nodes
+            value: elasticsearch-0
+          - name: network.host
+            value: 0.0.0.0
+          - name: node.data
+            value: "true"
+          - name: node.ingest
+            value: "true"
+          - name: node.master
+            value: "true"    
+          - name: ES_JAVA_OPTS
+            value: "-Xms512m -Xmx512m"
+          - name: "xpack.security.enabled"
+            value: "false"
+          - name: discovery.zen.minimum_master_nodes
+            value: "1"
+        volumeMounts:
+        - name: data
+          mountPath: {{data-dir}}
+
+      initContainers:
+      - name: fix-permissions
+        image: busybox
+        command: ["sh", "-c", "chown -R 1000:1000 {{data-dir}}"]
+        securityContext:
+          privileged: true
+        volumeMounts:
+        - name: data
+          mountPath: {{data-dir}}
+      - name: increase-vm-max-map
+        image: busybox
+        command: ["sysctl", "-w", "vm.max_map_count=262144"]
+        securityContext:
+          privileged: true
+      - name: increase-fd-ulimit
+        image: busybox
+        command: ["sh", "-c", "ulimit -n 65536"]
+        securityContext:
+          privileged: true
+  volumeClaimTemplates:
+  - metadata:
+      name: data
+      labels:
+        app: elasticsearch
+    spec:
+      accessModes: [ "ReadWriteOnce" ]
+#      storageClassName: "standard"
+      resources:
+        requests:
+          storage: {{es-storage-size}}

scripts/logging/fluentd/aggregator-cm.yaml

@@ -0,0 +1,62 @@
+#
+# Copyright IBM Corporation 2020,2021
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+apiVersion: v1
+data: 
+  fluent.conf: |
+    <source>
+      @type forward
+      bind 0.0.0.0
+      port 24224
+    </source>
+    <source>
+      @type http
+      bind 0.0.0.0
+      port 9880
+    </source>
+    <match fluentd.healthcheck>
+      @type stdout
+    </match>
+    <match fluentd-**>
+      @type null
+    </match>
+    <match kubernetes.var.log.containers.fluentd-**>
+      @type null
+    </match>
+    <filter kubernetes.**>
+      @id filter_kubernetes_metadata
+      @type kubernetes_metadata
+    </filter>
+
+    <match **>
+      @type elasticsearch
+      host elasticsearch.logging
+      logstash_format true
+      port 9200
+      path ""
+      <buffer>
+        flush_at_shutdown true
+        flush_thread_count 8
+        flush_thread_interval 20
+        flush_thread_burst_interval 1
+        retry_forever true
+        retry_type exponential_backoff
+      </buffer>
+    </match>
+kind: ConfigMap
+metadata: 
+  name: fluentd-aggregator-conf
+  namespace: logging