Byi override yaml update

deploy/helm/overrides.yaml

@@ -1,56 +1,137 @@
 prometheus:
   prometheusSpec:
+    externalLabels:
+      # Set this to a value to distinguish between different k8s clusters
+      cluster: kubernetes
     remoteWrite:
+    # kube state metrics
     - url: http://fluentd:9888/prometheus.metrics.state.daemonset
       writeRelabelConfigs:
       - action: keep
-        regex: kube-state-metrics;(kube_daemonset_status_current_number_scheduled|kube_daemonset_status_desired_number_scheduled|kube_daemonset_status_number_misscheduled|kube_daemonset_status_number_unavailable|kube_daemonset_metadata_generation)
+        regex: kube-state-metrics;kube_daemonset_status_(?:current_number_scheduled|desired_number_scheduled|number_misscheduled|number_unavailable)
+        sourceLabels: [job, __name__]
+    - url: http://fluentd:9888/prometheus.metrics.state.daemonset
+      writeRelabelConfigs:
+      - action: keep
+        regex: kube-state-metrics;kube_daemonset_metadata_generation
+        sourceLabels: [job, __name__]
+    - url: http://fluentd:9888/prometheus.metrics.state.deployment
+      writeRelabelConfigs:
+      - action: keep
+        regex: kube-state-metrics;kube_deployment_(?:metadata_generation|spec_paused|spec_replicas|spec_strategy_rollingupdate_max_unavailable)
         sourceLabels: [job, __name__]
     - url: http://fluentd:9888/prometheus.metrics.state.deployment
       writeRelabelConfigs:
       - action: keep
-        regex: kube-state-metrics;(kube_deployment_metadata_generation|kube_deployment_spec_paused|kube_deployment_spec_replicas|kube_deployment_status_observed_generation|kube_deployment_spec_strategy_rollingupdate_max_unavailable|kube_deployment_status_replicas_available|kube_deployment_status_replicas_unavailable)
+        regex: kube-state-metrics;kube_deployment_status_(?:replicas_available|observed_generation|replicas_unavailable)
         sourceLabels: [job, __name__]
     - url: http://fluentd:9888/prometheus.metrics.state.node
       writeRelabelConfigs:
       - action: keep
-        regex: kube-state-metrics;(kube_node_info|kube_node_spec_unschedulable|kube_node_status_allocatable|kube_node_status_capacity|kube_node_status_condition)
+        regex: kube-state-metrics;kube_node_(?:info|spec_unschedulable|status_allocatable|status_capacity|status_condition)
         sourceLabels: [job, __name__]
     - url: http://fluentd:9888/prometheus.metrics.state.pod
       writeRelabelConfigs:
       - action: keep
-        regex: kube-state-metrics;(kube_pod_container_info|kube_pod_container_resource_requests|kube_pod_container_resource_limits|kube_pod_container_status_ready|kube_pod_container_status_terminated_reason|kube_pod_container_status_waiting_reason|kube_pod_status_phase)
+        regex: kube-state-metrics;kube_pod_container_(?:info|resource_requests|resource_limits|status_ready|status_terminated_reason|status_waiting_reason)
         sourceLabels: [job, __name__]
+    - url: http://fluentd:9888/prometheus.metrics.state.pod
+      writeRelabelConfigs:
+      - action: keep
+        regex: kube-state-metrics;kube_pod_status_phase
+        sourceLabels: [job, __name__]
+    # controller manager metrics
     - url: http://fluentd:9888/prometheus.metrics.controller-manager
       writeRelabelConfigs:
       - action: keep
-        regex: cloudprovider_.*_api_request_duration_seconds.*
-        sourceLabels: [__name__]
+        regex: kube-controller-manager;cloudprovider_.*_api_request_duration_seconds.*
+        sourceLabels: [job, __name__]
+    # scheduler metrics
     - url: http://fluentd:9888/prometheus.metrics.scheduler
       writeRelabelConfigs:
       - action: keep
-        regex: scheduler_e2e_scheduling_latency_microseconds.*|scheduler_binding_latency_microseconds.*|scheduler_scheduling_algorithm_latency_microseconds.*
-        sourceLabels: [__name__]
+        regex: kube-scheduler;scheduler_(?:e2e_scheduling|binding|scheduling_algorithm)_latency_microseconds.*
+        sourceLabels: [job, __name__]
+    # api server metrics
     - url: http://fluentd:9888/prometheus.metrics.apiserver
       writeRelabelConfigs:
       - action: keep
-        regex: apiserver;(apiserver_request_count|apiserver_request_latencies.*|etcd_request_cache_get_latencies_summary.*|etcd_request_cache_add_latencies_summary.*)
+        regex: apiserver;apiserver_request_(?:count|latencies.*)
         sourceLabels: [job, __name__]
+    - url: http://fluentd:9888/prometheus.metrics.apiserver
+      writeRelabelConfigs:
+      - action: keep
+        regex: apiserver;etcd_request_cache_(?:get|add)_latencies_summary.*
+        sourceLabels: [job, __name__]
+    # kubelet metrics
     - url: http://fluentd:9888/prometheus.metrics.kubelet
       writeRelabelConfigs:
       - action: keep
-        regex: kubelet_docker_operations_errors|kubelet_docker_operations_latency_microseconds.*|kubelet_running_container_count|kubelet_running_pod_count|kubelet_runtime_operations_latency_microseconds.*
-        sourceLabels: [__name__]
+        regex: kubelet;kubelet_docker_operations_(?:errors|latency_microseconds.*)
+        sourceLabels: [job, __name__]
+    - url: http://fluentd:9888/prometheus.metrics.kubelet
+      writeRelabelConfigs:
+      - action: keep
+        regex: kubelet;kubelet_(running_container_count|running_pod_count|runtime_operations_latency_microseconds.*)
+        sourceLabels: [job, __name__]
+    # cadvisor container metrics
     - url: http://fluentd:9888/prometheus.metrics.container
       writeRelabelConfigs:
       - action: keep
-        regex: container_cpu_load_average_10s|container_cpu_system_seconds_total|container_cpu_usage_seconds_total|container_cpu_cfs_throttled_seconds_total|container_memory_usage_bytes|container_spec_memory_limit_bytes|container_memory_swap|container_spec_memory_swap_limit_bytes|container_spec_memory_reservation_limit_bytes|container_fs_usage_bytes|container_fs_limit_bytes|container_fs_writes_bytes_total|container_fs_reads_bytes_total|container_network_receive_bytes_total|container_network_transmit_bytes_total|container_network_receive_errors_total|container_network_transmit_errors_total
-        sourceLabels: [__name__]
+        regex: kubelet;container_cpu_(?:load_average_10s|(?:system|usage)_seconds_total)
+        sourceLabels: [job, __name__]
+    - url: http://fluentd:9888/prometheus.metrics.container
+      writeRelabelConfigs:
+      - action: keep
+        regex: kubelet;container_memory_(?:usage_bytes|memory_swap)
+        sourceLabels: [job, __name__]
+    - url: http://fluentd:9888/prometheus.metrics.container
+      writeRelabelConfigs:
+      - action: keep
+        regex: kubelet;container_spec_memory_(?:|swap|reservation)_limit_bytes
+        sourceLabels: [job, __name__]
+    - url: http://fluentd:9888/prometheus.metrics.container
+      writeRelabelConfigs:
+      - action: keep
+        regex: kubelet;container_fs_(?:(?:usage|limit)_bytes|(?:reads|writes)_bytes_total)
+        sourceLabels: [job, __name__]
+    - url: http://fluentd:9888/prometheus.metrics.container
+      writeRelabelConfigs:
+      - action: keep
+        regex: kubelet;container_network_(?:receive|transmit)_(?:bytes|errors)_total
+        sourceLabels: [job, __name__]
+    # node exporter metrics
+    - url: http://fluentd:9888/prometheus.metrics.node
+      writeRelabelConfigs:
+      - action: keep
+        regex: node-exporter;node_load(?:1|5|15)
+        sourceLabels: [job, __name__]
+    - url: http://fluentd:9888/prometheus.metrics.node
+      writeRelabelConfigs:
+      - action: keep
+        regex: node-exporter;node_cpu_seconds_total
+        sourceLabels: [job, __name__]
+    - url: http://fluentd:9888/prometheus.metrics.node
+      writeRelabelConfigs:
+      - action: keep
+        regex: node-exporter;node_memory_(?:MemAvailable|MemTotal|Buffers|SwapCached)_bytes
+        sourceLabels: [job, __name__]
+    - url: http://fluentd:9888/prometheus.metrics.node
+      writeRelabelConfigs:
+      - action: keep
+        regex: node-exporter;node_ipvs_(?:incoming|outgoing)_(?:bytes|packets)_total
+        sourceLabels: [job, __name__]
+    - url: http://fluentd:9888/prometheus.metrics.node
+      writeRelabelConfigs:
+      - action: keep
+        regex: node-exporter;node_disk_(?:(?:reads|writes)_completed|(?:read|written)_bytes)_total
+        sourceLabels: [job, __name__]
     - url: http://fluentd:9888/prometheus.metrics.node
       writeRelabelConfigs:
       - action: keep
-        regex: node-exporter;(node_load15|node_load5|node_cpu|node_memory_MemFree|node_memory_MemTotal|node_memory_Buffers|node_memory_Cached|node_ipvs_incoming_packets_total|node_ipvs_outgoing_packets_total|node_ipvs_outgoing_bytes_total|node_ipvs_incoming_bytes_total|node_disk_reads_completed|node_disk_bytes_read|node_disk_writes_completed|node_disk_bytes_written|node_filesystem_avail|node_filesystem_free|node_filesystem_size)
+        regex: node-exporter;node_filesystem_(?:avail|free|size)_bytes
         sourceLabels: [job, __name__]
+    # golang related metrics
     - url: http://fluentd:9888/prometheus.metrics.go
       writeRelabelConfigs:
       - action: keep

deploy/kubernetes/fluentd-sumologic.yaml

@@ -15,11 +15,10 @@ type: Opaque
 stringData:
   endpoint-metrics: XXXX
   endpoint-metrics-apiserver: XXXX
-  endpoint-metrics-cadvisor: XXXX
-  endpoint-metrics-kubelet: XXXX
   endpoint-metrics-kube-controller-manager: XXXX
   endpoint-metrics-kube-scheduler: XXXX
   endpoint-metrics-kube-state: XXXX
+  endpoint-metrics-kubelet: XXXX
   endpoint-metrics-node-exporter: XXXX
   endpoint-metrics-prometheus-operator: XXXX
 ---
@@ -102,12 +101,6 @@ data:
       data_type metrics
       metric_data_format carbon2
     </match>
-    <match prometheus.datapoint.cadvisor**>
-      @type sumologic
-      endpoint "#{ENV['SUMO_ENDPOINT_METRICS_CADVISOR']}"
-      data_type metrics
-      metric_data_format carbon2
-    </match>
     <match prometheus.datapoint.kubelet**>
       @type sumologic
       endpoint "#{ENV['SUMO_ENDPOINT_METRICS_KUBELET']}"
@@ -242,11 +235,6 @@ spec:
             secretKeyRef:
               name: sumologic
               key: endpoint-metrics-apiserver
-        - name: SUMO_ENDPOINT_METRICS_CADVISOR
-          valueFrom:
-            secretKeyRef:
-              name: sumologic
-              key: endpoint-metrics-cadvisor
         - name: SUMO_ENDPOINT_METRICS_KUBELET
           valueFrom:
             secretKeyRef: