Support custom release and namespace

ci/build.sh

@@ -132,7 +132,7 @@ if [ -n "$GITHUB_TOKEN" ] && [ "$TRAVIS_EVENT_TYPE" == "pull_request" ]; then
   prometheus_remote_write="${prometheus_remote_write//&/\\&}"
   prometheus_remote_write="${prometheus_remote_write//$'\n'/\\n}"
   echo "// This file is autogenerated" > deploy/kubernetes/kube-prometheus-sumo-logic-mixin.libsonnet
-  sed "s#\[\/\*REMOTE_WRITE\*\/\]#$prometheus_remote_write#" ci/jsonnet-mixin.tmpl | sed 's#"http://collection-sumologic.sumologic.svc.cluster.local:9888\/#$._config.sumologicCollectorSvc + "#g' | sed 's/+:     /+: /' | sed -r 's/"(\w*)":/\1:/g' > deploy/kubernetes/kube-prometheus-sumo-logic-mixin.libsonnet
+  sed "s#\[\/\*REMOTE_WRITE\*\/\]#$prometheus_remote_write#" ci/jsonnet-mixin.tmpl | sed 's#"http://$(CHART).$(NAMESPACE).svc.cluster.local:9888\/#$._config.sumologicCollectorSvc + "#g' | sed 's/+:     /+: /' | sed -r 's/"(\w*)":/\1:/g' > deploy/kubernetes/kube-prometheus-sumo-logic-mixin.libsonnet
 
   echo "Copy falco section from 'values.yaml' to 'falco-overrides.yaml'"
   echo "# This file is auto-generated." > deploy/helm/falco-overrides.yaml

deploy/helm/fluent-bit-overrides.yaml

@@ -3,11 +3,21 @@ service:
   flush: 5
 metrics:
   enabled: true
+env:
+- name: CHART
+  valueFrom:
+    configMapKeyRef:
+      name: sumologic-configmap
+      key: release
+- name: NAMESPACE
+  valueFrom:
+    fieldRef:
+      fieldPath: metadata.namespace
 backend:
   type: forward
   forward:
     # NOTE: Requires trailing "." for fully-qualified name resolution
-    host: collection-sumologic.sumologic.svc.cluster.local.
+    host: ${CHART}.${NAMESPACE}.svc.cluster.local.
     port: 24321
     tls: "off"
     tls_verify: "on"

deploy/helm/prometheus-overrides.yaml

@@ -49,39 +49,53 @@ prometheus:
       matchLabels:
         app: fluent-bit
   prometheusSpec:
+    thanos:
+      version: v0.10.0
+    containers:
+    - name: "prometheus-config-reloader"
+      env:
+      - name: CHART
+        valueFrom:
+          configMapKeyRef:
+            name: sumologic-configmap
+            key: release
+      - name: NAMESPACE
+        valueFrom:
+          fieldRef:
+            fieldPath: metadata.namespace
     remoteWrite:
     - # kube state metrics
-      url: http://collection-sumologic.sumologic.svc.cluster.local:9888/prometheus.metrics.state
+      url: http://$(CHART).$(NAMESPACE).svc.cluster.local:9888/prometheus.metrics.state
       writeRelabelConfigs:
       - action: keep
         regex: kube-state-metrics;(?:kube_statefulset_status_observed_generation|kube_statefulset_status_replicas|kube_statefulset_replicas|kube_statefulset_metadata_generation|kube_daemonset_status_current_number_scheduled|kube_daemonset_status_desired_number_scheduled|kube_daemonset_status_number_misscheduled|kube_daemonset_status_number_unavailable|kube_daemonset_metadata_generation|kube_deployment_metadata_generation|kube_deployment_spec_paused|kube_deployment_spec_replicas|kube_deployment_spec_strategy_rollingupdate_max_unavailable|kube_deployment_status_replicas_available|kube_deployment_status_observed_generation|kube_deployment_status_replicas_unavailable|kube_node_info|kube_node_spec_unschedulable|kube_node_status_allocatable|kube_node_status_capacity|kube_node_status_condition|kube_pod_container_info|kube_pod_container_resource_requests|kube_pod_container_resource_limits|kube_pod_container_status_ready|kube_pod_container_status_terminated_reason|kube_pod_container_status_waiting_reason|kube_pod_container_status_restarts_total|kube_pod_status_phase)
         sourceLabels: [job, __name__]
     - # controller manager metrics
-      url: http://collection-sumologic.sumologic.svc.cluster.local:9888/prometheus.metrics.controller-manager
+      url: http://$(CHART).$(NAMESPACE).svc.cluster.local:9888/prometheus.metrics.controller-manager
       writeRelabelConfigs:
       - action: keep
         regex: kubelet;cloudprovider_.*_api_request_duration_seconds.*
         sourceLabels: [job, __name__]
     - # scheduler metrics
-      url: http://collection-sumologic.sumologic.svc.cluster.local:9888/prometheus.metrics.scheduler
+      url: http://$(CHART).$(NAMESPACE).svc.cluster.local:9888/prometheus.metrics.scheduler
       writeRelabelConfigs:
       - action: keep
         regex: kube-scheduler;scheduler_(?:e2e_scheduling|binding|scheduling_algorithm)_latency_microseconds.*
         sourceLabels: [job, __name__]
     - # api server metrics
-      url: http://collection-sumologic.sumologic.svc.cluster.local:9888/prometheus.metrics.apiserver
+      url: http://$(CHART).$(NAMESPACE).svc.cluster.local:9888/prometheus.metrics.apiserver
       writeRelabelConfigs:
       - action: keep
         regex: apiserver;(?:apiserver_request_(?:count|total)|apiserver_request_(?:latencies|duration_seconds).*|etcd_request_cache_get_(?:latencies_summary|duration_seconds).*|etcd_request_cache_add_(?:latencies_summary|duration_seconds).*|etcd_helper_cache_hit_(?:count|total)|etcd_helper_cache_miss_(?:count|total))
         sourceLabels: [job, __name__]
     - # kubelet metrics
-      url: http://collection-sumologic.sumologic.svc.cluster.local:9888/prometheus.metrics.kubelet
+      url: http://$(CHART).$(NAMESPACE).svc.cluster.local:9888/prometheus.metrics.kubelet
       writeRelabelConfigs:
       - action: keep
         regex: kubelet;(?:kubelet_docker_operations_errors.*|kubelet_docker_operations_(?:latency_micro|duration_)seconds.*|kubelet_running_container_count|kubelet_running_pod_count|kubelet_runtime_operations_(?:latency_micro|duration_)seconds.*)
         sourceLabels: [job, __name__]
     - # cadvisor container metrics
-      url: http://collection-sumologic.sumologic.svc.cluster.local:9888/prometheus.metrics.container
+      url: http://$(CHART).$(NAMESPACE).svc.cluster.local:9888/prometheus.metrics.container
       writeRelabelConfigs:
       - action: labelmap
         regex: container_name
@@ -93,25 +107,25 @@ prometheus:
         regex: kubelet;.+;(?:container_cpu_load_average_10s|container_cpu_system_seconds_total|container_cpu_usage_seconds_total|container_cpu_cfs_throttled_seconds_total|container_memory_usage_bytes|container_memory_swap|container_memory_working_set_bytes|container_spec_memory_limit_bytes|container_spec_memory_swap_limit_bytes|container_spec_memory_reservation_limit_bytes|container_spec_cpu_quota|container_spec_cpu_period|container_fs_usage_bytes|container_fs_limit_bytes|container_fs_reads_bytes_total|container_fs_writes_bytes_total|)
         sourceLabels: [job, container, __name__]
     - # cadvisor aggregate container metrics
-      url: http://collection-sumologic.sumologic.svc.cluster.local:9888/prometheus.metrics.container
+      url: http://$(CHART).$(NAMESPACE).svc.cluster.local:9888/prometheus.metrics.container
       writeRelabelConfigs:
       - action: keep
         regex: kubelet;(?:container_network_receive_bytes_total|container_network_transmit_bytes_total|container_network_receive_errors_total|container_network_transmit_errors_total|container_network_receive_packets_dropped_total|container_network_transmit_packets_dropped_total)
         sourceLabels: [job, __name__]
     - # node exporter metrics
-      url: http://collection-sumologic.sumologic.svc.cluster.local:9888/prometheus.metrics.node
+      url: http://$(CHART).$(NAMESPACE).svc.cluster.local:9888/prometheus.metrics.node
       writeRelabelConfigs:
       - action: keep
         regex: node-exporter;(?:node_load1|node_load5|node_load15|node_cpu_seconds_total|node_memory_MemAvailable_bytes|node_memory_MemTotal_bytes|node_memory_Buffers_bytes|node_memory_SwapCached_bytes|node_memory_Cached_bytes|node_memory_MemFree_bytes|node_memory_SwapFree_bytes|node_ipvs_incoming_bytes_total|node_ipvs_outgoing_bytes_total|node_ipvs_incoming_packets_total|node_ipvs_outgoing_packets_total|node_disk_reads_completed_total|node_disk_writes_completed_total|node_disk_read_bytes_total|node_disk_written_bytes_total|node_filesystem_avail_bytes|node_filesystem_free_bytes|node_filesystem_size_bytes|node_filesystem_files)
         sourceLabels: [job, __name__]
     - # prometheus operator rules
-      url: http://collection-sumologic.sumologic.svc.cluster.local:9888/prometheus.metrics.operator.rule
+      url: http://$(CHART).$(NAMESPACE).svc.cluster.local:9888/prometheus.metrics.operator.rule
       writeRelabelConfigs:
       - action: keep
         regex: 'cluster_quantile:apiserver_request_latencies:histogram_quantile|instance:node_cpu:rate:sum|instance:node_filesystem_usage:sum|instance:node_network_receive_bytes:rate:sum|instance:node_network_transmit_bytes:rate:sum|instance:node_cpu:ratio|cluster:node_cpu:sum_rate5m|cluster:node_cpu:ratio|cluster_quantile:scheduler_e2e_scheduling_latency:histogram_quantile|cluster_quantile:scheduler_scheduling_algorithm_latency:histogram_quantile|cluster_quantile:scheduler_binding_latency:histogram_quantile|node_namespace_pod:kube_pod_info:|:kube_pod_info_node_count:|node:node_num_cpu:sum|:node_cpu_utilisation:avg1m|node:node_cpu_utilisation:avg1m|node:cluster_cpu_utilisation:ratio|:node_cpu_saturation_load1:|node:node_cpu_saturation_load1:|:node_memory_utilisation:|:node_memory_MemFreeCachedBuffers_bytes:sum|:node_memory_MemTotal_bytes:sum|node:node_memory_bytes_available:sum|node:node_memory_bytes_total:sum|node:node_memory_utilisation:ratio|node:cluster_memory_utilisation:ratio|:node_memory_swap_io_bytes:sum_rate|node:node_memory_utilisation:|node:node_memory_utilisation_2:|node:node_memory_swap_io_bytes:sum_rate|:node_disk_utilisation:avg_irate|node:node_disk_utilisation:avg_irate|:node_disk_saturation:avg_irate|node:node_disk_saturation:avg_irate|node:node_filesystem_usage:|node:node_filesystem_avail:|:node_net_utilisation:sum_irate|node:node_net_utilisation:sum_irate|:node_net_saturation:sum_irate|node:node_net_saturation:sum_irate|node:node_inodes_total:|node:node_inodes_free:'
         sourceLabels: [__name__]
     - # health
-      url: http://collection-sumologic.sumologic.svc.cluster.local:9888/prometheus.metrics
+      url: http://$(CHART).$(NAMESPACE).svc.cluster.local:9888/prometheus.metrics
       writeRelabelConfigs:
       - action: keep
         regex: (?:up|prometheus_remote_storage_.*|fluentd_.*|fluentbit.*)

deploy/helm/sumologic/templates/chart-configmap.yaml

@@ -0,0 +1,6 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: sumologic-configmap
+data:
+  release: {{ template "sumologic.fullname" . }}

deploy/helm/sumologic/values.yaml

@@ -215,11 +215,22 @@ fluent-bit:
     flush: 5
   metrics:
     enabled: true
+  env:
+    - name: CHART
+      valueFrom:
+        configMapKeyRef:
+          name: sumologic-configmap
+          key: release
+    - name: NAMESPACE
+      valueFrom:
+        fieldRef:
+          fieldPath: metadata.namespace
+
   backend:
     type: forward
     forward:
       # NOTE: Requires trailing "." for fully-qualified name resolution
-      host: collection-sumologic.sumologic.svc.cluster.local.
+      host: ${CHART}.${NAMESPACE}.svc.cluster.local.
       port: 24321
       tls: "off"
       tls_verify: "on"
@@ -369,39 +380,54 @@ prometheus-operator:
           matchLabels:
             app: fluent-bit
     prometheusSpec:
+      thanos:
+        version: v0.10.0
+      containers:
+      - name: "prometheus-config-reloader"
+        env:
+        - name: CHART
+          valueFrom:
+            configMapKeyRef:
+              name: sumologic-configmap
+              key: release
+        - name: NAMESPACE
+          valueFrom:
+            fieldRef:
+              fieldPath: metadata.namespace
+
       remoteWrite:
         # kube state metrics
-        - url: http://collection-sumologic.sumologic.svc.cluster.local:9888/prometheus.metrics.state
+        - url: http://$(CHART).$(NAMESPACE).svc.cluster.local:9888/prometheus.metrics.state
           writeRelabelConfigs:
             - action: keep
               regex: kube-state-metrics;(?:kube_statefulset_status_observed_generation|kube_statefulset_status_replicas|kube_statefulset_replicas|kube_statefulset_metadata_generation|kube_daemonset_status_current_number_scheduled|kube_daemonset_status_desired_number_scheduled|kube_daemonset_status_number_misscheduled|kube_daemonset_status_number_unavailable|kube_daemonset_metadata_generation|kube_deployment_metadata_generation|kube_deployment_spec_paused|kube_deployment_spec_replicas|kube_deployment_spec_strategy_rollingupdate_max_unavailable|kube_deployment_status_replicas_available|kube_deployment_status_observed_generation|kube_deployment_status_replicas_unavailable|kube_node_info|kube_node_spec_unschedulable|kube_node_status_allocatable|kube_node_status_capacity|kube_node_status_condition|kube_pod_container_info|kube_pod_container_resource_requests|kube_pod_container_resource_limits|kube_pod_container_status_ready|kube_pod_container_status_terminated_reason|kube_pod_container_status_waiting_reason|kube_pod_container_status_restarts_total|kube_pod_status_phase)
               sourceLabels: [job, __name__]
         # controller manager metrics
-        - url: http://collection-sumologic.sumologic.svc.cluster.local:9888/prometheus.metrics.controller-manager
+        - url: http://$(CHART).$(NAMESPACE).svc.cluster.local:9888/prometheus.metrics.controller-manager
           writeRelabelConfigs:
             - action: keep
               regex: kubelet;cloudprovider_.*_api_request_duration_seconds.*
               sourceLabels: [job, __name__]
         # scheduler metrics
-        - url: http://collection-sumologic.sumologic.svc.cluster.local:9888/prometheus.metrics.scheduler
+        - url: http://$(CHART).$(NAMESPACE).svc.cluster.local:9888/prometheus.metrics.scheduler
           writeRelabelConfigs:
             - action: keep
               regex: kube-scheduler;scheduler_(?:e2e_scheduling|binding|scheduling_algorithm)_latency_microseconds.*
               sourceLabels: [job, __name__]
         # api server metrics
-        - url: http://collection-sumologic.sumologic.svc.cluster.local:9888/prometheus.metrics.apiserver
+        - url: http://$(CHART).$(NAMESPACE).svc.cluster.local:9888/prometheus.metrics.apiserver
           writeRelabelConfigs:
             - action: keep
               regex: apiserver;(?:apiserver_request_(?:count|total)|apiserver_request_(?:latencies|duration_seconds).*|etcd_request_cache_get_(?:latencies_summary|duration_seconds).*|etcd_request_cache_add_(?:latencies_summary|duration_seconds).*|etcd_helper_cache_hit_(?:count|total)|etcd_helper_cache_miss_(?:count|total))
               sourceLabels: [job, __name__]
         # kubelet metrics
-        - url: http://collection-sumologic.sumologic.svc.cluster.local:9888/prometheus.metrics.kubelet
+        - url: http://$(CHART).$(NAMESPACE).svc.cluster.local:9888/prometheus.metrics.kubelet
           writeRelabelConfigs:
             - action: keep
               regex: kubelet;(?:kubelet_docker_operations_errors.*|kubelet_docker_operations_(?:latency_micro|duration_)seconds.*|kubelet_running_container_count|kubelet_running_pod_count|kubelet_runtime_operations_(?:latency_micro|duration_)seconds.*)
               sourceLabels: [job, __name__]
         # cadvisor container metrics
-        - url: http://collection-sumologic.sumologic.svc.cluster.local:9888/prometheus.metrics.container
+        - url: http://$(CHART).$(NAMESPACE).svc.cluster.local:9888/prometheus.metrics.container
           writeRelabelConfigs:
             - action: labelmap
               regex: container_name
@@ -413,25 +439,25 @@ prometheus-operator:
               regex: kubelet;.+;(?:container_cpu_load_average_10s|container_cpu_system_seconds_total|container_cpu_usage_seconds_total|container_cpu_cfs_throttled_seconds_total|container_memory_usage_bytes|container_memory_swap|container_memory_working_set_bytes|container_spec_memory_limit_bytes|container_spec_memory_swap_limit_bytes|container_spec_memory_reservation_limit_bytes|container_spec_cpu_quota|container_spec_cpu_period|container_fs_usage_bytes|container_fs_limit_bytes|container_fs_reads_bytes_total|container_fs_writes_bytes_total|)
               sourceLabels: [job,container,__name__]
         # cadvisor aggregate container metrics
-        - url: http://collection-sumologic.sumologic.svc.cluster.local:9888/prometheus.metrics.container
+        - url: http://$(CHART).$(NAMESPACE).svc.cluster.local:9888/prometheus.metrics.container
           writeRelabelConfigs:
             - action: keep
               regex: kubelet;(?:container_network_receive_bytes_total|container_network_transmit_bytes_total|container_network_receive_errors_total|container_network_transmit_errors_total|container_network_receive_packets_dropped_total|container_network_transmit_packets_dropped_total)
               sourceLabels: [job,__name__]
         # node exporter metrics
-        - url: http://collection-sumologic.sumologic.svc.cluster.local:9888/prometheus.metrics.node
+        - url: http://$(CHART).$(NAMESPACE).svc.cluster.local:9888/prometheus.metrics.node
           writeRelabelConfigs:
             - action: keep
               regex: node-exporter;(?:node_load1|node_load5|node_load15|node_cpu_seconds_total|node_memory_MemAvailable_bytes|node_memory_MemTotal_bytes|node_memory_Buffers_bytes|node_memory_SwapCached_bytes|node_memory_Cached_bytes|node_memory_MemFree_bytes|node_memory_SwapFree_bytes|node_ipvs_incoming_bytes_total|node_ipvs_outgoing_bytes_total|node_ipvs_incoming_packets_total|node_ipvs_outgoing_packets_total|node_disk_reads_completed_total|node_disk_writes_completed_total|node_disk_read_bytes_total|node_disk_written_bytes_total|node_filesystem_avail_bytes|node_filesystem_free_bytes|node_filesystem_size_bytes|node_filesystem_files)
               sourceLabels: [job, __name__]
         # prometheus operator rules
-        - url: http://collection-sumologic.sumologic.svc.cluster.local:9888/prometheus.metrics.operator.rule
+        - url: http://$(CHART).$(NAMESPACE).svc.cluster.local:9888/prometheus.metrics.operator.rule
           writeRelabelConfigs:
             - action: keep
               regex: 'cluster_quantile:apiserver_request_latencies:histogram_quantile|instance:node_cpu:rate:sum|instance:node_filesystem_usage:sum|instance:node_network_receive_bytes:rate:sum|instance:node_network_transmit_bytes:rate:sum|instance:node_cpu:ratio|cluster:node_cpu:sum_rate5m|cluster:node_cpu:ratio|cluster_quantile:scheduler_e2e_scheduling_latency:histogram_quantile|cluster_quantile:scheduler_scheduling_algorithm_latency:histogram_quantile|cluster_quantile:scheduler_binding_latency:histogram_quantile|node_namespace_pod:kube_pod_info:|:kube_pod_info_node_count:|node:node_num_cpu:sum|:node_cpu_utilisation:avg1m|node:node_cpu_utilisation:avg1m|node:cluster_cpu_utilisation:ratio|:node_cpu_saturation_load1:|node:node_cpu_saturation_load1:|:node_memory_utilisation:|:node_memory_MemFreeCachedBuffers_bytes:sum|:node_memory_MemTotal_bytes:sum|node:node_memory_bytes_available:sum|node:node_memory_bytes_total:sum|node:node_memory_utilisation:ratio|node:cluster_memory_utilisation:ratio|:node_memory_swap_io_bytes:sum_rate|node:node_memory_utilisation:|node:node_memory_utilisation_2:|node:node_memory_swap_io_bytes:sum_rate|:node_disk_utilisation:avg_irate|node:node_disk_utilisation:avg_irate|:node_disk_saturation:avg_irate|node:node_disk_saturation:avg_irate|node:node_filesystem_usage:|node:node_filesystem_avail:|:node_net_utilisation:sum_irate|node:node_net_utilisation:sum_irate|:node_net_saturation:sum_irate|node:node_net_saturation:sum_irate|node:node_inodes_total:|node:node_inodes_free:'
               sourceLabels: [__name__]
         # health
-        - url: http://collection-sumologic.sumologic.svc.cluster.local:9888/prometheus.metrics
+        - url: http://$(CHART).$(NAMESPACE).svc.cluster.local:9888/prometheus.metrics
           writeRelabelConfigs:
             - action: keep
               regex: (?:up|prometheus_remote_storage_.*|fluentd_.*|fluentbit.*)

deploy/kubernetes/fluentd-sumologic.yaml.tmpl

@@ -1,5 +1,13 @@
 # This file is auto-generated.
 ---
+# Source: sumologic/templates/chart-configmap.yaml
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: sumologic-configmap
+data:
+  release: collection-sumologic
+---
 # Source: sumologic/templates/configmap.yaml
 apiVersion: v1
 kind: ConfigMap