Separate configuration into folders, add Fluentd and Fluent-bit confi…

…gurations.

kubernetes/README.md

@@ -14,52 +14,40 @@ Minimum Requirements
 - 8 GB of free RAM
 - 10 GB of free disk space (actual disk occupation will mainly depend of the number of rules and the amount of traffic on the network). 200GB+ SSD grade is recommended.
 - ``git``, ``curl``
-- ``docker`` > 17.06.0 (will be installed during SELKS initial setup)
-- ``docker-compose`` > 1.27.0 (will be installed during SELKS initial setup)
+- ``Kubernetes`` >= 1.21 (tested on k3s 1.22)
+- ``docker or containerd``
 
 Install process
 ---------------
 ### Basic installation
+Clone the Git repository from SELKS
 
 ```bash
 git clone https://github.com/StamusNetworks/SELKS.git
 cd SELKS/kubernetes/
+```
+
+Update the PV's and storage class according to your own needs. Replace username and password in the secret definitions.
 
-# Alter YAML's to your need. Update the PV's and storage class according to your own needs. Replace username and password in the secret definitions.
+Choose between Logstash with Filebeat, or Fluentd with Fluent-bit. Fluentd uses rather significantly less memory (Logstash uses 1G to 1,5G by default, Fluentd uses about 100M), but you need to build your own container image with certain plugins and push to a (self-hosted) Private Docker Registry in order to use all of the features available by default via Logstash. We've included some basic Kubernetes logging in the Fluentd/Fluent-bit configuration.
 
+```bash
 # Setup storage
 mkdir -p /data/arkime/{pcap,logs} /data/suricata/{logrotate,rules,run,logs/fpc} /data/scirius/{data,logs,static} /data/elasticsearch
-
 chown -R 998:996 /data/suricata
 chown -R 1000:996 /data/scirius
 chown -R 1000:1000 /data/elasticsearch
 chown -R 1000:1000 /data/arkime
 
 # Create NGINX TLS keys and create secret template
 openssl req -new -nodes -x509 -subj "/C=FR/ST=IDF/L=Paris/O=Stamus/CN=SELKS" -days 3650 -keyout ./tls.key -out tls.crt -extensions v3_ca
-kubectl create secret tls nginx-tls --cert=tls.crt --key=tls.key --dry-run -o yaml > nginx-secret.yaml
-
-
-# Apply configuration
-kubectl create --save-config -f *-ns.yaml
-
-kubectl create --save-config -f storageclass.yaml
-kubectl create --save-config -f *-pv.yaml
-kubectl create --save-config -f *-pvc.yaml
-
-kubectl create --save-config -f *-configmap.yaml
-kubectl create --save-config -f *-secret.yaml
-
-kubectl create --save-config -f *-statefulset.yaml
-kubectl create --save-config -f *-deployment.yaml
-kubectl create --save-config -f *-daemonset.yaml
-
-kubectl create --save-config -f *-service.yaml
+kubectl create secret tls nginx-tls --cert=tls.crt --key=tls.key --dry-run -o yaml > nginx/nginx-secret.yaml
 
-kubectl create --save-config -f *-cronjob.yaml
+chmod +x install.sh
+./install.sh
 
 # To load the Kibana dashboards, once Kibana is up and running
-kubectl create --save-config -f alpine.yaml
+kubectl create --save-config -f kibana/alpine.yaml
 ```
 Once the services have been applied, you can get the NodePort using the following command:
 ```

kubernetes/arkime-configmap.yaml → kubernetes/arkime/arkime-configmap.yaml

@@ -41,7 +41,7 @@ data:
     # Comma seperated list of elasticsearch host:port combinations.  If not using a
     # elasticsearch VIP, a different elasticsearch node in the cluster can be specified
     # for each Moloch node to help spread load on high volume clusters
-    elasticsearch=http://elasticsearch.elasticsearch:9200
+    elasticsearch=http://elasticsearch.logging:9200
 
     # How often to create a new elasticsearch index. hourly,hourly6,daily,weekly,monthly
     # Changing the value will cause previous sessions to be unreachable
@@ -328,15 +328,15 @@ data:
     [node1]
     nodeClass = class1
     # Might use a different elasticsearch node
-    elasticsearch=elasticsearchhost1
+    elasticsearch.logginghost1
 
     # Uncomment if this node should process the cron queries, only ONE node should process cron queries
     # cronQueries = true
 
     [node2]
     nodeClass = class2
     # Might use a different elasticsearch node
-    elasticsearch=elasticsearchhost2
+    elasticsearch.logginghost2
     # Uses a different interface
     interface = eth4
 

kubernetes/arkime-deployment.yaml → kubernetes/arkime/arkime-deployment.yaml

@@ -24,7 +24,7 @@ spec:
           image: ghcr.io/stamusnetworks/arkimeviewer:master
           env:
             - name: ES_HOST
-              value: elasticsearch.elasticsearch
+              value: elasticsearch.logging
             - name: ES_PORT
               value: "9200"
             - name: ARKIME_ADMIN_USERNAME

kubernetes/arkime-pv.yaml → kubernetes/arkime/arkime-pv.yaml

@@ -15,12 +15,12 @@ spec:
     name: arkime-log
     namespace: suricata
   local:
-    path: /data/suricata/arkime/logs/
+    path: /data/arkime/logs/
   nodeAffinity:
     required:
       nodeSelectorTerms:
       - matchExpressions:
         - key: kubernetes.io/hostname
           operator: In
           values:
-          - localhost.localdomain
+          - HOSTNAME

kubernetes/common/logging-ns.yaml

@@ -0,0 +1,8 @@
+---
+
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: logging
+  labels:
+    kubernetes.io/metadata.name: logging

kubernetes/elasticsearch-pv.yaml → kubernetes/elasticsearch/elasticsearch-pv.yaml

@@ -13,14 +13,14 @@ spec:
   persistentVolumeReclaimPolicy: Retain
   claimRef:
     name: elasticsearch-data
-    namespace: elasticsearch
+    namespace: logging
   local:
-    path: /data/suricata/elasticsearch/
+    path: /data/elasticsearch/
   nodeAffinity:
     required:
       nodeSelectorTerms:
       - matchExpressions:
         - key: kubernetes.io/hostname
           operator: In
           values:
-          - localhost.localdomain
+          - HOSTNAME

kubernetes/elasticsearch-pvc.yaml → kubernetes/elasticsearch/elasticsearch-pvc.yaml

@@ -6,7 +6,7 @@ metadata:
   labels:
     app: elasticsearch-data
   name: elasticsearch-data
-  namespace: elasticsearch
+  namespace: logging
 spec:
   storageClassName: "local-storage"
   volumeName: elasticsearch-data

kubernetes/elasticsearch-service.yaml → kubernetes/elasticsearch/elasticsearch-service.yaml

@@ -5,7 +5,7 @@ metadata:
   name: es
   labels:
     service: elasticsearch
-  namespace: elasticsearch
+  namespace: logging
 spec:
   # Setting clusterIP to none makes sure communication is directly between pods
   clusterIP: None
@@ -22,7 +22,7 @@ metadata:
   labels:
     service: elasticsearch
   name: elasticsearch
-  namespace: elasticsearch
+  namespace: logging
 spec:
   ports:
   - port: 9200

kubernetes/elasticsearch-statefulset.yaml → kubernetes/elasticsearch/elasticsearch-statefulset.yaml

@@ -6,7 +6,7 @@ metadata:
   name: elasticsearch-initcontainer
   labels:
     app.kubernetes.io/name: elasticsearch
-  namespace: elasticsearch
+  namespace: logging
 data:
   folder-setup.sh: |-
     #!/bin/bash
@@ -25,7 +25,7 @@ metadata:
   name: elasticsearch
   labels:
     service: elasticsearch
-  namespace: elasticsearch
+  namespace: logging
 spec:
   updateStrategy:
     type: RollingUpdate
@@ -192,14 +192,6 @@ spec:
           periodSeconds: 10
           successThreshold: 3
           timeoutSeconds: 5
-        livenessProbe:
-          exec:
-            command:
-              - curl --silent --fail localhost:9200/_cluster/health || exit 1
-          failureThreshold: 3
-          initialDelaySeconds: 30
-          periodSeconds: 10
-          timeoutSeconds: 5
 
         resources:
           limits:

kubernetes/evebox-deployment.yaml → kubernetes/evebox/evebox-deployment.yaml

@@ -5,7 +5,7 @@ metadata:
   labels:
     app: evebox
   name: evebox
-  namespace: suricata
+  namespace: logging
 spec:
   replicas: 1
   selector:
@@ -20,7 +20,7 @@ spec:
       containers:
         - args:
             - -e
-            - http://elasticsearch.elasticsearch:9200
+            - http://elasticsearch.logging:9200
           image: jasonish/evebox:master
           name: evebox
 

kubernetes/evebox-service.yaml → kubernetes/evebox/evebox-service.yaml

@@ -5,7 +5,7 @@ metadata:
   labels:
     app: evebox
   name: evebox
-  namespace: suricata
+  namespace: logging
 spec:
   ports:
     - name: "web"

kubernetes/fluentd_fluent-bit/Dockerfile

@@ -0,0 +1,8 @@
+FROM fluent/fluentd-kubernetes-daemonset:v1.14.5-debian-elasticsearch7-amd64-1.1
+USER root
+RUN ["gem", "install", "fluent-plugin-record-modifier", "--no-document"]
+RUN ["gem", "install", "fluent-plugin-ua-parser", "--no-document"]
+
+# GeoIP requires some more build options, rather than just a simple plugin.
+# RUN ["gem", "install", "fluent-plugin-geoip", "--no-document"]
+USER fluent