Merge pull request #74 from danielhartnell/cloudwatch-log-exporter

adding first draft of CloudWatch log exporter

cluster-conf/logging/fluentd-cloudwatch.yaml

@@ -0,0 +1,286 @@
+---
+# Source: fluentd-cloudwatch/templates/configmap.yaml
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: fluentd-cloudwatch
+  labels:
+    app: fluentd-cloudwatch
+    chart: "fluentd-cloudwatch-0.5.1"
+    heritage: "Tiller"
+    release: "fluentd-cloudwatch-0.5.1"
+data:
+  fluent.conf:   |
+    <match fluent.**>
+      type null
+    </match>
+  
+    <source>
+      type tail
+      enable_stat_watcher false
+      path /var/log/containers/*.log
+      pos_file /var/log/fluentd-containers.log.pos
+      time_format %Y-%m-%dT%H:%M:%S.%NZ
+      tag kubernetes.*
+      format json
+      read_from_head true
+    </source>
+  
+    <source>
+      type tail
+      enable_stat_watcher false
+      format /^(?<time>[^ ]* [^ ,]*)[^\[]*\[[^\]]*\]\[(?<severity>[^ \]]*) *\] (?<message>.*)$/
+      time_format %Y-%m-%d %H:%M:%S
+      path /var/log/salt/minion
+      pos_file /var/log/fluentd-salt.pos
+      tag salt
+    </source>
+  
+    <source>
+      type tail
+      enable_stat_watcher false
+      format syslog
+      path /var/log/startupscript.log
+      pos_file /var/log/fluentd-startupscript.log.pos
+      tag startupscript
+    </source>
+  
+    <source>
+      type tail
+      enable_stat_watcher false
+      format /^time="(?<time>[^)]*)" level=(?<severity>[^ ]*) msg="(?<message>[^"]*)"( err="(?<error>[^"]*)")?( statusCode=($<status_code>\d+))?/
+      path /var/log/docker.log
+      pos_file /var/log/fluentd-docker.log.pos
+      tag docker
+    </source>
+  
+    <source>
+      type tail
+      enable_stat_watcher false
+      format none
+      path /var/log/etcd.log
+      pos_file /var/log/fluentd-etcd.log.pos
+      tag etcd
+    </source>
+  
+    <source>
+      type tail
+      enable_stat_watcher false
+      format kubernetes
+      multiline_flush_interval 5s
+      path /var/log/kubelet.log
+      pos_file /var/log/fluentd-kubelet.log.pos
+      tag kubelet
+    </source>
+  
+    <source>
+      type tail
+      enable_stat_watcher false
+      format kubernetes
+      multiline_flush_interval 5s
+      path /var/log/kube-proxy.log
+      pos_file /var/log/fluentd-kube-proxy.log.pos
+      tag kube-proxy
+    </source>
+  
+    <source>
+      type tail
+      enable_stat_watcher false
+      format kubernetes
+      multiline_flush_interval 5s
+      path /var/log/kube-apiserver.log
+      pos_file /var/log/fluentd-kube-apiserver.log.pos
+      tag kube-apiserver
+    </source>
+  
+    <source>
+      type tail
+      enable_stat_watcher false
+      format kubernetes
+      multiline_flush_interval 5s
+      path /var/log/kube-controller-manager.log
+      pos_file /var/log/fluentd-kube-controller-manager.log.pos
+      tag kube-controller-manager
+    </source>
+  
+    <source>
+      type tail
+      enable_stat_watcher false
+      format kubernetes
+      multiline_flush_interval 5s
+      path /var/log/kube-scheduler.log
+      pos_file /var/log/fluentd-kube-scheduler.log.pos
+      tag kube-scheduler
+    </source>
+  
+    <source>
+      type tail
+      enable_stat_watcher false
+      format kubernetes
+      multiline_flush_interval 5s
+      path /var/log/rescheduler.log
+      pos_file /var/log/fluentd-rescheduler.log.pos
+      tag rescheduler
+    </source>
+  
+    <source>
+      type tail
+      enable_stat_watcher false
+      format kubernetes
+      multiline_flush_interval 5s
+      path /var/log/glbc.log
+      pos_file /var/log/fluentd-glbc.log.pos
+      tag glbc
+    </source>
+  
+    <source>
+      type tail
+      enable_stat_watcher false
+      format kubernetes
+      multiline_flush_interval 5s
+      path /var/log/cluster-autoscaler.log
+      pos_file /var/log/fluentd-cluster-autoscaler.log.pos
+      tag cluster-autoscaler
+    </source>
+  
+    <filter kubernetes.**>
+      type kubernetes_metadata
+    </filter>
+  
+    <match **>
+      type cloudwatch_logs
+      log_group_name "#{ENV['LOG_GROUP_NAME']}"
+      auto_create_stream true
+      use_tag_as_stream true
+    </match>
+  
+
+---
+# Source: fluentd-cloudwatch/templates/serviceaccount.yaml
+
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: fluentd-cloudwatch
+  labels:
+    app: fluentd-cloudwatch
+    chart: "fluentd-cloudwatch-0.5.1"
+    release: "fluentd-cloudwatch-0.5.1"
+    heritage: "Tiller"
+
+---
+# Source: fluentd-cloudwatch/templates/clusterrole.yaml
+
+apiVersion: rbac.authorization.k8s.io/v1beta1
+kind: ClusterRole
+metadata:
+  name: fluentd-cloudwatch
+  labels:
+    app: fluentd-cloudwatch
+    chart: "fluentd-cloudwatch-0.5.1"
+    release: "fluentd-cloudwatch-0.5.1"
+    heritage: "Tiller"
+rules:
+- apiGroups: [""]
+  resources: ["namespaces", "pods"]
+  verbs: ["get", "list", "watch"]
+
+---
+# Source: fluentd-cloudwatch/templates/clusterrolebinding.yaml
+
+apiVersion: rbac.authorization.k8s.io/v1beta1
+kind: ClusterRoleBinding
+metadata:
+  name: fluentd-cloudwatch
+  labels:
+    app: fluentd-cloudwatch
+    chart: "fluentd-cloudwatch-0.5.1"
+    release: "fluentd-cloudwatch-0.5.1"
+    heritage: "Tiller"
+subjects:
+- kind: ServiceAccount
+  name: fluentd-cloudwatch
+  namespace: default
+roleRef:
+  kind: ClusterRole
+  name: fluentd-cloudwatch
+  apiGroup: rbac.authorization.k8s.io
+
+
+---
+# Source: fluentd-cloudwatch/templates/daemonset.yaml
+apiVersion: extensions/v1beta1
+kind: DaemonSet
+metadata:
+  name: fluentd-cloudwatch
+  labels:
+    app: fluentd-cloudwatch
+    chart: "fluentd-cloudwatch-0.5.1"
+    heritage: "Tiller"
+    release: "fluentd-cloudwatch-0.5.1"
+spec:
+  template:
+    metadata:
+      labels:
+        app: fluentd-cloudwatch
+        release: "fluentd-cloudwatch-0.5.1"
+      annotations:
+        iam.amazonaws.com/role: "arn:aws:iam::320464205386:role/kubernetesSendCloudWatchLogs"
+
+    spec:
+      serviceAccountName: fluentd-cloudwatch
+      initContainers:
+        - name: copy-fluentd-config
+          image: busybox
+          command: ['sh', '-c', 'cp /config-volume/* /etc/fluentd']
+          volumeMounts:
+            - name: config-volume
+              mountPath: /config-volume
+            - name: config
+              mountPath: /etc/fluentd
+      containers:
+      - name: fluentd-cloudwatch
+        image: "fluent/fluentd-kubernetes-daemonset@sha256:fe67be752f17dd4a66b0c88a46b1d937bff1fa9bd653c5be18880a6b744744cb"
+        imagePullPolicy: "IfNotPresent"
+        #hostNetwork: false
+        env:
+        - name: AWS_REGION
+          value: us-west-2
+        - name: LOG_GROUP_NAME
+          value: kubernetes
+        - name: AWS_ACCESS_KEY_ID
+          value:
+        - name: AWS_SECRET_ACCESS_KEY
+          value:
+        - name: FLUENT_UID
+          value: "0"
+        resources:
+          limits:
+            cpu: 100m
+            memory: 200Mi
+          requests:
+            cpu: 100m
+            memory: 200Mi
+
+        volumeMounts:
+        - name: varlog
+          mountPath: /var/log
+        - name: varlibdockercontainers
+          mountPath: /var/lib/docker/containers
+          readOnly: true
+        - name: config
+          mountPath: /fluentd/etc
+      terminationGracePeriodSeconds: 30
+      volumes:
+      - name: varlog
+        hostPath:
+          path: /var/log
+      - name: varlibdockercontainers
+        hostPath:
+          path: /var/lib/docker/containers
+      - name: config
+        emptyDir: {}
+      - name: config-volume
+        configMap:
+          name: fluentd-cloudwatch
+