From b2eca282bbe5c06e862fe093bbf04cf9036f5cfb Mon Sep 17 00:00:00 2001
From: Qixiang Cheng <Qixiang.Cheng@microsoft.com>
Date: Wed, 18 Mar 2020 13:04:24 +0000
Subject: [PATCH 1/8] Add fluent bit

---
 .../services/logging/fluent-bit-config.yaml   | 106 ++++++++++++++++++
 .../services/logging/fluent-bit.yaml          |  68 +++++++++++
 .../services/logging/launch_order             |   4 +-
 3 files changed, 176 insertions(+), 2 deletions(-)
 create mode 100644 src/ClusterBootstrap/services/logging/fluent-bit-config.yaml
 create mode 100644 src/ClusterBootstrap/services/logging/fluent-bit.yaml

diff --git a/src/ClusterBootstrap/services/logging/fluent-bit-config.yaml b/src/ClusterBootstrap/services/logging/fluent-bit-config.yaml
new file mode 100644
index 000000000..1748a0742
--- /dev/null
+++ b/src/ClusterBootstrap/services/logging/fluent-bit-config.yaml
@@ -0,0 +1,106 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: fluent-bit-config
+  namespace: kube-system
+  labels:
+    k8s-app: fluent-bit
+data:
+  # Configuration files: server, input, filters and output
+  # ======================================================
+  fluent-bit.conf: |
+    [SERVICE]
+        Flush         1
+        Log_Level     info
+        Daemon        off
+        Parsers_File  parsers.conf
+        HTTP_Server   On
+        HTTP_Listen   0.0.0.0
+        HTTP_Port     2020
+
+    @INCLUDE input-kubernetes.conf
+    @INCLUDE filter-kubernetes.conf
+    @INCLUDE output-elasticsearch.conf
+
+  input-kubernetes.conf: |
+    [INPUT]
+        Name              tail
+        Tag               kube.*
+        Path              /var/log/containers/*.log
+        Parser            docker
+        DB                /var/log/flb_kube.db
+        Mem_Buf_Limit     5MB
+        Skip_Long_Lines   On
+        Refresh_Interval  10
+
+  filter-kubernetes.conf: |
+    [FILTER]
+        Name                kubernetes
+        Match               kube.*
+        Kube_URL            https://kubernetes.default.svc:443
+        Kube_CA_File        /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
+        Kube_Token_File     /var/run/secrets/kubernetes.io/serviceaccount/token
+        Kube_Tag_Prefix     kube.var.log.containers.
+        Merge_Log           On
+        Merge_Log_Key       log_processed
+        K8S-Logging.Parser  On
+        K8S-Logging.Exclude Off
+
+  output-elasticsearch.conf: |
+    [OUTPUT]
+        Name            es
+        Match           *
+        Host            ${FLUENT_ELASTICSEARCH_HOST}
+        Port            ${FLUENT_ELASTICSEARCH_PORT}
+        Buffer_Size     False
+        Type            fluentd
+        Logstash_Format On
+        Replace_Dots    On
+        Retry_Limit     False
+
+  parsers.conf: |
+    [PARSER]
+        Name   apache
+        Format regex
+        Regex  ^(?<host>[^ ]*) [^ ]* (?<user>[^ ]*) \[(?<time>[^\]]*)\] "(?<method>\S+)(?: +(?<path>[^\"]*?)(?: +\S*)?)?" (?<code>[^ ]*) (?<size>[^ ]*)(?: "(?<referer>[^\"]*)" "(?<agent>[^\"]*)")?$
+        Time_Key time
+        Time_Format %d/%b/%Y:%H:%M:%S %z
+
+    [PARSER]
+        Name   apache2
+        Format regex
+        Regex  ^(?<host>[^ ]*) [^ ]* (?<user>[^ ]*) \[(?<time>[^\]]*)\] "(?<method>\S+)(?: +(?<path>[^ ]*) +\S*)?" (?<code>[^ ]*) (?<size>[^ ]*)(?: "(?<referer>[^\"]*)" "(?<agent>[^\"]*)")?$
+        Time_Key time
+        Time_Format %d/%b/%Y:%H:%M:%S %z
+
+    [PARSER]
+        Name   apache_error
+        Format regex
+        Regex  ^\[[^ ]* (?<time>[^\]]*)\] \[(?<level>[^\]]*)\](?: \[pid (?<pid>[^\]]*)\])?( \[client (?<client>[^\]]*)\])? (?<message>.*)$
+
+    [PARSER]
+        Name   nginx
+        Format regex
+        Regex ^(?<remote>[^ ]*) (?<host>[^ ]*) (?<user>[^ ]*) \[(?<time>[^\]]*)\] "(?<method>\S+)(?: +(?<path>[^\"]*?)(?: +\S*)?)?" (?<code>[^ ]*) (?<size>[^ ]*)(?: "(?<referer>[^\"]*)" "(?<agent>[^\"]*)")?$
+        Time_Key time
+        Time_Format %d/%b/%Y:%H:%M:%S %z
+
+    [PARSER]
+        Name   json
+        Format json
+        Time_Key time
+        Time_Format %d/%b/%Y:%H:%M:%S %z
+
+    [PARSER]
+        Name        docker
+        Format      json
+        Time_Key    time
+        Time_Format %Y-%m-%dT%H:%M:%S.%L
+        Time_Keep   On
+
+    [PARSER]
+        Name        syslog
+        Format      regex
+        Regex       ^\<(?<pri>[0-9]+)\>(?<time>[^ ]* {1,2}[^ ]* [^ ]*) (?<host>[^ ]*) (?<ident>[a-zA-Z0-9_\/\.\-]*)(?:\[(?<pid>[0-9]+)\])?(?:[^\:]*\:)? *(?<message>.*)$
+        Time_Key    time
+        Time_Format %b %d %H:%M:%S
\ No newline at end of file
diff --git a/src/ClusterBootstrap/services/logging/fluent-bit.yaml b/src/ClusterBootstrap/services/logging/fluent-bit.yaml
new file mode 100644
index 000000000..8d27bf684
--- /dev/null
+++ b/src/ClusterBootstrap/services/logging/fluent-bit.yaml
@@ -0,0 +1,68 @@
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  name: fluent-bit
+  namespace: kube-system
+  labels:
+    k8s-app: fluent-bit
+    version: v1
+    kubernetes.io/cluster-service: "true"
+spec:
+  selector:
+    matchLabels:
+      k8s-app: fluent-bit
+  template:
+    metadata:
+      labels:
+        k8s-app: fluent-bit
+        version: v1
+        kubernetes.io/cluster-service: "true"
+      annotations:
+        prometheus.io/scrape: "true"
+        prometheus.io/port: "2020"
+        prometheus.io/path: /api/v1/metrics/prometheus
+    spec:
+      containers:
+      - name: fluent-bit
+        image: fluent/fluent-bit:1.3.9
+        imagePullPolicy: Always
+        ports:
+        - containerPort: 2020
+        env:
+        - name: FLUENT_ELASTICSEARCH_HOST
+          value: "$(ELASTICSEARCH_SERVICE_HOST)"
+        - name: FLUENT_ELASTICSEARCH_PORT
+          value: "$(ELASTICSEARCH_SERVICE_PORT)"
+        volumeMounts:
+        - name: varlog
+          mountPath: /var/log
+        - name: varlibdockercontainers
+          mountPath: /var/lib/docker/containers
+          readOnly: true
+        - name: datalibdockercontainers
+          mountPath: /data/lib/docker/containers
+          readOnly: true
+        - name: fluent-bit-config
+          mountPath: /fluent-bit/etc/
+      terminationGracePeriodSeconds: 10
+      volumes:
+      - name: varlog
+        hostPath:
+          path: /var/log
+      - name: varlibdockercontainers
+        hostPath:
+          path: /var/lib/docker/containers
+      - name: datalibdockercontainers
+        hostPath:
+          path: /data/lib/docker/containers
+      - name: fluent-bit-config
+        configMap:
+          name: fluent-bit-config
+      tolerations:
+      - key: node-role.kubernetes.io/master
+        operator: Exists
+        effect: NoSchedule
+      - operator: "Exists"
+        effect: "NoExecute"
+      - operator: "Exists"
+        effect: "NoSchedule"
\ No newline at end of file
diff --git a/src/ClusterBootstrap/services/logging/launch_order b/src/ClusterBootstrap/services/logging/launch_order
index 5e039b38a..611a537c1 100755
--- a/src/ClusterBootstrap/services/logging/launch_order
+++ b/src/ClusterBootstrap/services/logging/launch_order
@@ -1,4 +1,4 @@
 elasticsearch.yaml
-fluentd-config.yaml
-fluentd.yaml
+fluent-bit-config.yaml
+fluent-bit.yaml
 kibana.yaml

From 9b62dba9456aa50dbfe1f2e3e51474dab025c4c0 Mon Sep 17 00:00:00 2001
From: Qixiang Cheng <Qixiang.Cheng@microsoft.com>
Date: Thu, 19 Mar 2020 08:20:08 +0000
Subject: [PATCH 2/8] Bring time_nsec back

---
 .../services/logging/fluent-bit-config.yaml    | 18 +++++++++++++++++-
 1 file changed, 17 insertions(+), 1 deletion(-)

diff --git a/src/ClusterBootstrap/services/logging/fluent-bit-config.yaml b/src/ClusterBootstrap/services/logging/fluent-bit-config.yaml
index 1748a0742..b3a4948f4 100644
--- a/src/ClusterBootstrap/services/logging/fluent-bit-config.yaml
+++ b/src/ClusterBootstrap/services/logging/fluent-bit-config.yaml
@@ -20,6 +20,7 @@ data:
 
     @INCLUDE input-kubernetes.conf
     @INCLUDE filter-kubernetes.conf
+    @INCLUDE filter-time_nsec.conf
     @INCLUDE output-elasticsearch.conf
 
   input-kubernetes.conf: |
@@ -46,6 +47,21 @@ data:
         K8S-Logging.Parser  On
         K8S-Logging.Exclude Off
 
+  filter-time_nsec.conf: |
+    [FILTER]
+        NAME         lua
+        Match        kube.*
+        Script       time_nsec.lua
+        Call         time_nsec
+        Type_int_key time_nsec
+
+  time_nsec.lua: |
+    function time_nsec(tag, timestamp, record)
+      sec, nsec = math.modf(timestamp)
+      record['time_nsec'] = nsec * 1e9
+      return 1, timestamp, record
+    end
+
   output-elasticsearch.conf: |
     [OUTPUT]
         Name            es
@@ -103,4 +119,4 @@ data:
         Format      regex
         Regex       ^\<(?<pri>[0-9]+)\>(?<time>[^ ]* {1,2}[^ ]* [^ ]*) (?<host>[^ ]*) (?<ident>[a-zA-Z0-9_\/\.\-]*)(?:\[(?<pid>[0-9]+)\])?(?:[^\:]*\:)? *(?<message>.*)$
         Time_Key    time
-        Time_Format %b %d %H:%M:%S
\ No newline at end of file
+        Time_Format %b %d %H:%M:%S

From 7fa11d5f1cfcb11545ea3b1e8dd5b42274d37667 Mon Sep 17 00:00:00 2001
From: Qixiang Cheng <Qixiang.Cheng@microsoft.com>
Date: Thu, 19 Mar 2020 08:22:57 +0000
Subject: [PATCH 3/8] Refresh by 1s

---
 src/ClusterBootstrap/services/logging/fluent-bit-config.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/ClusterBootstrap/services/logging/fluent-bit-config.yaml b/src/ClusterBootstrap/services/logging/fluent-bit-config.yaml
index b3a4948f4..1c377eb76 100644
--- a/src/ClusterBootstrap/services/logging/fluent-bit-config.yaml
+++ b/src/ClusterBootstrap/services/logging/fluent-bit-config.yaml
@@ -32,7 +32,7 @@ data:
         DB                /var/log/flb_kube.db
         Mem_Buf_Limit     5MB
         Skip_Long_Lines   On
-        Refresh_Interval  10
+        Refresh_Interval  1
 
   filter-kubernetes.conf: |
     [FILTER]

From 9d527ece0e78fc7625aa77cdbbb6e058511a0c41 Mon Sep 17 00:00:00 2001
From: Qixiang Cheng <Qixiang.Cheng@microsoft.com>
Date: Thu, 19 Mar 2020 08:34:10 +0000
Subject: [PATCH 4/8] Drop fluentd

---
 .../services/logging/fluentd-config.yaml      | 301 ------------------
 .../services/logging/fluentd.yaml             |  74 -----
 2 files changed, 375 deletions(-)
 delete mode 100644 src/ClusterBootstrap/services/logging/fluentd-config.yaml
 delete mode 100755 src/ClusterBootstrap/services/logging/fluentd.yaml

diff --git a/src/ClusterBootstrap/services/logging/fluentd-config.yaml b/src/ClusterBootstrap/services/logging/fluentd-config.yaml
deleted file mode 100644
index 154029985..000000000
--- a/src/ClusterBootstrap/services/logging/fluentd-config.yaml
+++ /dev/null
@@ -1,301 +0,0 @@
-apiVersion: v1
-kind: ConfigMap
-metadata:
-  name: fluentd-config
-  namespace: kube-system
-data:
-  fluent.conf: |
-    @include "#{ENV['FLUENTD_SYSTEMD_CONF'] || 'systemd'}.conf"
-    @include "#{ENV['FLUENTD_PROMETHEUS_CONF'] || 'prometheus'}.conf"
-    @include kubernetes.conf
-    @include conf.d/*.conf
-
-    <match **>
-       @type elasticsearch
-       @id out_es
-       @log_level info
-       include_tag_key true
-       host "#{ENV['FLUENT_ELASTICSEARCH_HOST']}"
-       port "#{ENV['FLUENT_ELASTICSEARCH_PORT']}"
-       path "#{ENV['FLUENT_ELASTICSEARCH_PATH']}"
-       scheme "#{ENV['FLUENT_ELASTICSEARCH_SCHEME'] || 'http'}"
-       ssl_verify "#{ENV['FLUENT_ELASTICSEARCH_SSL_VERIFY'] || 'true'}"
-       ssl_version "#{ENV['FLUENT_ELASTICSEARCH_SSL_VERSION'] || 'TLSv1'}"
-       reload_connections "#{ENV['FLUENT_ELASTICSEARCH_RELOAD_CONNECTIONS'] || 'false'}"
-       reconnect_on_error "#{ENV['FLUENT_ELASTICSEARCH_RECONNECT_ON_ERROR'] || 'true'}"
-       reload_on_failure "#{ENV['FLUENT_ELASTICSEARCH_RELOAD_ON_FAILURE'] || 'true'}"
-       log_es_400_reason "#{ENV['FLUENT_ELASTICSEARCH_LOG_ES_400_REASON'] || 'false'}"
-       logstash_prefix "#{ENV['FLUENT_ELASTICSEARCH_LOGSTASH_PREFIX'] || 'logstash'}"
-       logstash_format "#{ENV['FLUENT_ELASTICSEARCH_LOGSTASH_FORMAT'] || 'true'}"
-       index_name "#{ENV['FLUENT_ELASTICSEARCH_LOGSTASH_INDEX_NAME'] || 'logstash'}"
-       type_name "#{ENV['FLUENT_ELASTICSEARCH_LOGSTASH_TYPE_NAME'] || 'fluentd'}"
-       <buffer>
-         flush_thread_count "#{ENV['FLUENT_ELASTICSEARCH_BUFFER_FLUSH_THREAD_COUNT'] || '8'}"
-         flush_interval "#{ENV['FLUENT_ELASTICSEARCH_BUFFER_FLUSH_INTERVAL'] || '5s'}"
-         chunk_limit_size "#{ENV['FLUENT_ELASTICSEARCH_BUFFER_CHUNK_LIMIT_SIZE'] || '2M'}"
-         queue_limit_length "#{ENV['FLUENT_ELASTICSEARCH_BUFFER_QUEUE_LIMIT_LENGTH'] || '32'}"
-         retry_max_interval "#{ENV['FLUENT_ELASTICSEARCH_BUFFER_RETRY_MAX_INTERVAL'] || '30'}"
-         retry_forever true
-       </buffer>
-    </match>
-  kubernetes.conf: |
-    <match fluent.**>
-      @type null
-    </match>
-
-    <source>
-      @type tail
-      @id in_tail_container_logs
-      refresh_interval 1s
-      path /var/log/containers/*.log
-      pos_file /var/log/fluentd-containers.log.pos
-      tag "#{ENV['FLUENT_CONTAINER_TAIL_TAG'] || 'kubernetes.*'}"
-      read_from_head true
-      <parse>
-        @type "#{ENV['FLUENT_CONTAINER_TAIL_PARSER_TYPE'] || 'json'}"
-        time_format %Y-%m-%dT%H:%M:%S.%NZ
-      </parse>
-    </source>
-
-    <source>
-      @type tail
-      @id in_tail_minion
-      path /var/log/salt/minion
-      pos_file /var/log/fluentd-salt.pos
-      tag salt
-      <parse>
-        @type regexp
-        expression /^(?<time>[^ ]* [^ ,]*)[^\[]*\[[^\]]*\]\[(?<severity>[^ \]]*) *\] (?<message>.*)$/
-        time_format %Y-%m-%d %H:%M:%S
-      </parse>
-    </source>
-
-    <source>
-      @type tail
-      @id in_tail_startupscript
-      path /var/log/startupscript.log
-      pos_file /var/log/fluentd-startupscript.log.pos
-      tag startupscript
-      <parse>
-        @type syslog
-      </parse>
-    </source>
-
-    <source>
-      @type tail
-      @id in_tail_docker
-      path /var/log/docker.log
-      pos_file /var/log/fluentd-docker.log.pos
-      tag docker
-      <parse>
-        @type regexp
-        expression /^time="(?<time>[^)]*)" level=(?<severity>[^ ]*) msg="(?<message>[^"]*)"( err="(?<error>[^"]*)")?( statusCode=($<status_code>\d+))?/
-      </parse>
-    </source>
-
-    <source>
-      @type tail
-      @id in_tail_etcd
-      path /var/log/etcd.log
-      pos_file /var/log/fluentd-etcd.log.pos
-      tag etcd
-      <parse>
-        @type none
-      </parse>
-    </source>
-
-    <source>
-      @type tail
-      @id in_tail_kubelet
-      multiline_flush_interval 5s
-      path /var/log/kubelet.log
-      pos_file /var/log/fluentd-kubelet.log.pos
-      tag kubelet
-      <parse>
-        @type kubernetes
-      </parse>
-    </source>
-
-    <source>
-      @type tail
-      @id in_tail_kube_proxy
-      multiline_flush_interval 5s
-      path /var/log/kube-proxy.log
-      pos_file /var/log/fluentd-kube-proxy.log.pos
-      tag kube-proxy
-      <parse>
-        @type kubernetes
-      </parse>
-    </source>
-
-    <source>
-      @type tail
-      @id in_tail_kube_apiserver
-      multiline_flush_interval 5s
-      path /var/log/kube-apiserver.log
-      pos_file /var/log/fluentd-kube-apiserver.log.pos
-      tag kube-apiserver
-      <parse>
-        @type kubernetes
-      </parse>
-    </source>
-
-    <source>
-      @type tail
-      @id in_tail_kube_controller_manager
-      multiline_flush_interval 5s
-      path /var/log/kube-controller-manager.log
-      pos_file /var/log/fluentd-kube-controller-manager.log.pos
-      tag kube-controller-manager
-      <parse>
-        @type kubernetes
-      </parse>
-    </source>
-
-    <source>
-      @type tail
-      @id in_tail_kube_scheduler
-      multiline_flush_interval 5s
-      path /var/log/kube-scheduler.log
-      pos_file /var/log/fluentd-kube-scheduler.log.pos
-      tag kube-scheduler
-      <parse>
-        @type kubernetes
-      </parse>
-    </source>
-
-    <source>
-      @type tail
-      @id in_tail_rescheduler
-      multiline_flush_interval 5s
-      path /var/log/rescheduler.log
-      pos_file /var/log/fluentd-rescheduler.log.pos
-      tag rescheduler
-      <parse>
-        @type kubernetes
-      </parse>
-    </source>
-
-    <source>
-      @type tail
-      @id in_tail_glbc
-      multiline_flush_interval 5s
-      path /var/log/glbc.log
-      pos_file /var/log/fluentd-glbc.log.pos
-      tag glbc
-      <parse>
-        @type kubernetes
-      </parse>
-    </source>
-
-    <source>
-      @type tail
-      @id in_tail_cluster_autoscaler
-      multiline_flush_interval 5s
-      path /var/log/cluster-autoscaler.log
-      pos_file /var/log/fluentd-cluster-autoscaler.log.pos
-      tag cluster-autoscaler
-      <parse>
-        @type kubernetes
-      </parse>
-    </source>
-
-    # Example:
-    # 2017-02-09T00:15:57.992775796Z AUDIT: id="90c73c7c-97d6-4b65-9461-f94606ff825f" ip="104.132.1.72" method="GET" user="kubecfg" as="<self>" asgroups="<lookup>" namespace="default" uri="/api/v1/namespaces/default/pods"
-    # 2017-02-09T00:15:57.993528822Z AUDIT: id="90c73c7c-97d6-4b65-9461-f94606ff825f" response="200"
-    <source>
-      @type tail
-      @id in_tail_kube_apiserver_audit
-      multiline_flush_interval 5s
-      path /var/log/kubernetes/kube-apiserver-audit.log
-      pos_file /var/log/kube-apiserver-audit.log.pos
-      tag kube-apiserver-audit
-      <parse>
-        @type multiline
-        format_firstline /^\S+\s+AUDIT:/
-        # Fields must be explicitly captured by name to be parsed into the record.
-        # Fields may not always be present, and order may change, so this just looks
-        # for a list of key="\"quoted\" value" pairs separated by spaces.
-        # Unknown fields are ignored.
-        # Note: We can't separate query/response lines as format1/format2 because
-        #       they don't always come one after the other for a given query.
-        format1 /^(?<time>\S+) AUDIT:(?: (?:id="(?<id>(?:[^"\\]|\\.)*)"|ip="(?<ip>(?:[^"\\]|\\.)*)"|method="(?<method>(?:[^"\\]|\\.)*)"|user="(?<user>(?:[^"\\]|\\.)*)"|groups="(?<groups>(?:[^"\\]|\\.)*)"|as="(?<as>(?:[^"\\]|\\.)*)"|asgroups="(?<asgroups>(?:[^"\\]|\\.)*)"|namespace="(?<namespace>(?:[^"\\]|\\.)*)"|uri="(?<uri>(?:[^"\\]|\\.)*)"|response="(?<response>(?:[^"\\]|\\.)*)"|\w+="(?:[^"\\]|\\.)*"))*/
-        time_format %Y-%m-%dT%T.%L%Z
-      </parse>
-    </source>
-
-    <filter kubernetes.**>
-      @type kubernetes_metadata
-      @id filter_kube_metadata
-    </filter>
-
-    <filter kubernetes.**>
-      @type record_transformer
-      enable_ruby
-      <record>
-        time_nsec ${time.nsec}
-      </record>
-    </filter>
-  prometheus.conf: |
-    # Prometheus metric exposed on 0.0.0.0:24231/metrics
-    <source>
-      @type prometheus
-      bind "#{ENV['FLUENTD_PROMETHEUS_BIND'] || '0.0.0.0'}"
-      port "#{ENV['FLUENTD_PROMETHEUS_PORT'] || '24231'}"
-      metrics_path "#{ENV['FLUENTD_PROMETHEUS_PATH'] || '/metrics'}"
-    </source>
-
-    <source>
-      @type prometheus_output_monitor
-    </source>
-  systemd.conf: |
-    <source>
-      @type systemd
-      @id in_systemd_kubelet
-      matches [{ "_SYSTEMD_UNIT": "kubelet.service" }]
-      <storage>
-        @type local
-        persistent true
-        path /var/log/fluentd-journald-kubelet-cursor.json
-      </storage>
-      <entry>
-        fields_strip_underscores true
-      </entry>
-      read_from_head true
-      tag kubelet
-    </source>
-
-    # Logs from docker-systemd
-    <source>
-      @type systemd
-      @id in_systemd_docker
-      matches [{ "_SYSTEMD_UNIT": "docker.service" }]
-      <storage>
-        @type local
-        persistent true
-        path /var/log/fluentd-journald-docker-cursor.json
-      </storage>
-      <entry>
-        fields_strip_underscores true
-      </entry>
-      read_from_head true
-      tag docker.systemd
-    </source>
-
-    # Logs from systemd-journal for interesting services.
-    <source>
-      @type systemd
-      @id in_systemd_bootkube
-      matches [{ "_SYSTEMD_UNIT": "bootkube.service" }]
-      <storage>
-        @type local
-        persistent true
-        path /var/log/fluentd-journald-bootkube-cursor.json
-      </storage>
-      <entry>
-        fields_strip_underscores true
-      </entry>
-      read_from_head true
-      tag bootkube
-    </source>
-  disable.conf: ""
diff --git a/src/ClusterBootstrap/services/logging/fluentd.yaml b/src/ClusterBootstrap/services/logging/fluentd.yaml
deleted file mode 100755
index 6fe701aa5..000000000
--- a/src/ClusterBootstrap/services/logging/fluentd.yaml
+++ /dev/null
@@ -1,74 +0,0 @@
-apiVersion: apps/v1
-kind: DaemonSet
-metadata:
-  name: fluentd
-  namespace: kube-system
-  labels:
-    app: fluentd
-    kubernetes.io/cluster-service: "true"
-    addonmanager.kubernetes.io/mode: Reconcile
-spec:
-  selector:
-    matchLabels:
-      app: fluentd
-  template:
-    metadata:
-      labels:
-        app: fluentd
-        kubernetes.io/cluster-service: "true"
-      annotations:
-        prometheus.io/scrape: 'true'
-        prometheus.io/path: '/metrics'
-        prometheus.io/port: '{{cnf["fluentd"]["port"]}}'
-    spec:
-      hostNetwork: true
-      containers:
-      - name: fluentd
-        image: '{{cnf["dockers"]["container"]["fluentd-kubernetes-daemonset"]["fullname"]}}'
-        env:
-        - name: FLUENT_ELASTICSEARCH_SED_DISABLE
-          value: "true"
-        - name: FLUENT_ELASTICSEARCH_SCHEME
-          value: "http"
-        - name: FLUENT_ELASTICSEARCH_HOST
-          value: '$(ELASTICSEARCH_SERVICE_HOST)'
-        - name: FLUENT_ELASTICSEARCH_PORT
-          value: '$(ELASTICSEARCH_SERVICE_PORT)'
-        - name: FLUENTD_PROMETHEUS_PORT
-          value: '{{cnf["fluentd"]["port"]}}'
-        resources:
-          limits:
-            memory: 200Mi
-          requests:
-            cpu: 100m
-            memory: 200Mi
-        ports:
-        - containerPort: {{cnf["fluentd"]["port"]}}
-          name: http
-          protocol: TCP
-        volumeMounts:
-        - name: config
-          mountPath: /fluentd/etc
-          readOnly: true
-        - name: varlog
-          mountPath: /var/log
-        - name: varlibdockercontainers
-          mountPath: /var/lib/docker/containers
-          readOnly: true
-        - name: datalibdockercontainers
-          mountPath: /data/lib/docker/containers
-          readOnly: true
-      terminationGracePeriodSeconds: 30
-      volumes:
-      - name: config
-        configMap:
-          name: fluentd-config
-      - name: varlog
-        hostPath:
-          path: /var/log
-      - name: varlibdockercontainers
-        hostPath:
-          path: /var/lib/docker/containers
-      - name: datalibdockercontainers
-        hostPath:
-          path: /data/lib/docker/containers

From f70cf29b1e309ccb47212124fdb596dde7a8cb73 Mon Sep 17 00:00:00 2001
From: Qixiang Cheng <Qixiang.Cheng@microsoft.com>
Date: Fri, 20 Mar 2020 11:16:30 +0000
Subject: [PATCH 5/8] Remove docker image

---
 src/ClusterBootstrap/params.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/src/ClusterBootstrap/params.py b/src/ClusterBootstrap/params.py
index a9e4e1b0f..6f6c06d06 100755
--- a/src/ClusterBootstrap/params.py
+++ b/src/ClusterBootstrap/params.py
@@ -661,7 +661,6 @@
             "phpmyadmin":{"fullname":"dlws/phpmyadmin:4.7.6"},
             "elasticsearch":{"fullname":"dlws/elasticsearch:6.8.5"},
             "elasticsearch-exporter":{"fullname":"dlws/elasticsearch-exporter:1.1.0"},
-            "fluentd-kubernetes-daemonset":{"fullname":"dlws/fluentd-kubernetes-daemonset:v1.7.4-debian-elasticsearch6-1.1"},
             "kibana":{"fullname":"dlws/kibana:6.8.5"},
             "fluentd-elasticsearch":{"fullname":"dlws/fluentd-elasticsearch:v2.0.2"},
             "binstore":{"fullname":"dlws/binstore:v1.0"},

From 6a2ccf68ea8262889c75eeffdf26a15864058c7c Mon Sep 17 00:00:00 2001
From: Qixiang Cheng <Qixiang.Cheng@microsoft.com>
Date: Mon, 23 Mar 2020 08:58:35 +0000
Subject: [PATCH 6/8] Deploy in hostNetwork

---
 src/ClusterBootstrap/services/logging/fluent-bit.yaml | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/src/ClusterBootstrap/services/logging/fluent-bit.yaml b/src/ClusterBootstrap/services/logging/fluent-bit.yaml
index 8d27bf684..b6c922382 100644
--- a/src/ClusterBootstrap/services/logging/fluent-bit.yaml
+++ b/src/ClusterBootstrap/services/logging/fluent-bit.yaml
@@ -44,6 +44,8 @@ spec:
           readOnly: true
         - name: fluent-bit-config
           mountPath: /fluent-bit/etc/
+      hostNetwork: true
+      dnsPolicy: ClusterFirstWithHostNet
       terminationGracePeriodSeconds: 10
       volumes:
       - name: varlog

From 0e9da0534450373de9e4fbbb8d95bcfefa767b99 Mon Sep 17 00:00:00 2001
From: Qixiang Cheng <Qixiang.Cheng@microsoft.com>
Date: Mon, 23 Mar 2020 09:13:57 +0000
Subject: [PATCH 7/8] Workaround DNS issue

---
 src/ClusterBootstrap/services/logging/fluent-bit-config.yaml | 3 ++-
 src/ClusterBootstrap/services/logging/fluent-bit.yaml        | 1 -
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/ClusterBootstrap/services/logging/fluent-bit-config.yaml b/src/ClusterBootstrap/services/logging/fluent-bit-config.yaml
index 1c377eb76..fb7a42cde 100644
--- a/src/ClusterBootstrap/services/logging/fluent-bit-config.yaml
+++ b/src/ClusterBootstrap/services/logging/fluent-bit-config.yaml
@@ -38,7 +38,7 @@ data:
     [FILTER]
         Name                kubernetes
         Match               kube.*
-        Kube_URL            https://kubernetes.default.svc:443
+        Kube_URL            https://${KUBERNETES_SERVICE_HOST}:${KUBERNETES_SERVICE_PORT}
         Kube_CA_File        /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
         Kube_Token_File     /var/run/secrets/kubernetes.io/serviceaccount/token
         Kube_Tag_Prefix     kube.var.log.containers.
@@ -46,6 +46,7 @@ data:
         Merge_Log_Key       log_processed
         K8S-Logging.Parser  On
         K8S-Logging.Exclude Off
+        tls.verify          Off
 
   filter-time_nsec.conf: |
     [FILTER]
diff --git a/src/ClusterBootstrap/services/logging/fluent-bit.yaml b/src/ClusterBootstrap/services/logging/fluent-bit.yaml
index b6c922382..9e4f21806 100644
--- a/src/ClusterBootstrap/services/logging/fluent-bit.yaml
+++ b/src/ClusterBootstrap/services/logging/fluent-bit.yaml
@@ -45,7 +45,6 @@ spec:
         - name: fluent-bit-config
           mountPath: /fluent-bit/etc/
       hostNetwork: true
-      dnsPolicy: ClusterFirstWithHostNet
       terminationGracePeriodSeconds: 10
       volumes:
       - name: varlog

From dbbdb4c299e72555b9c2c09f03e51ab80fcf2651 Mon Sep 17 00:00:00 2001
From: Qixiang Cheng <Qixiang.Cheng@microsoft.com>
Date: Mon, 23 Mar 2020 14:58:51 +0000
Subject: [PATCH 8/8] Revert cursor test

---
 src/ClusterManager/test/utils.py | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/src/ClusterManager/test/utils.py b/src/ClusterManager/test/utils.py
index 53b9272c3..b13f71691 100644
--- a/src/ClusterManager/test/utils.py
+++ b/src/ClusterManager/test/utils.py
@@ -310,14 +310,13 @@ def get_job_log(rest_url, email, jid):
             break
         resp_json = resp.json()
         log = resp_json["log"]
-        new_cursor = resp_json["cursor"]
+        cursor = resp_json["cursor"]
         if isinstance(log, dict):
             job_logs.extend(log.values())
         else:
             job_logs.append(log)
-        if new_cursor == cursor:
+        if cursor is None:
             break
-        cursor = new_cursor
     return '\n'.join(job_logs)