[stable/prometheus-operator] Add sync scripts (#10488)

* [stable/prometheus-operator] add dashboards and rules sync scripts

Signed-off-by: Dmitry Verkhoturov <paskal.07@gmail.com>

* [stable/prometheus-operator] fix script to create dirs in case of their absence

Signed-off-by: Dmitry Verkhoturov <paskal.07@gmail.com>

stable/prometheus-operator/Chart.yaml

@@ -9,7 +9,7 @@ name: prometheus-operator
 sources:
   - https://github.com/coreos/prometheus-operator
   - https://coreos.com/operators/prometheus
-version: 1.5.2
+version: 1.6.0
 appVersion: 0.26.0
 home: https://github.com/coreos/prometheus-operator
 keywords:

stable/prometheus-operator/README.md

@@ -286,6 +286,11 @@ $ helm install --name my-release stable/prometheus-operator -f values1.yaml,valu
 
 > **Tip**: You can use the default [values.yaml](values.yaml)
 
+
+## Developing Prometheus Rules and Grafana Dashboards
+
+This chart Grafana Dashboards and Prometheus Rules are just a copy from coreos/prometheus-operator and other sources, synced (with alterations) by scripts in [hack](hack) folder. In order to introduce any changes you need to first [add them to original repo](https://github.com/coreos/prometheus-operator/blob/master/contrib/kube-prometheus/docs/developing-prometheus-rules-and-grafana-dashboards.md) and then sync there by scripts.
+
 ## Further Information
 
 For more in-depth documentation of configuration options meanings, please see

stable/prometheus-operator/hack/README.md

@@ -0,0 +1,18 @@
+# prometheus-operator hacks
+
+## [sync_prometheus_rules.py](sync_prometheus_rules.py)
+
+This script generates prometheus rules set for alertmanager from any properly formatted kubernetes yaml based on defined input, splitting rules to separate files based on group name.
+
+Currently following imported:
+ - [coreos/prometheus-operator rules set](https://github.com/coreos/prometheus-operator/blob/master/contrib/kube-prometheus/manifests/prometheus-rules.yaml)
+ - [etcd-io/etc rules set](https://github.com/etcd-io/etcd/blob/master/Documentation/op-guide/etcd3_alert.rules.yml) (temporary disabled)
+
+## [sync_grafana_dashboards.py](sync_grafana_dashboards.py)
+
+This script generates grafana dashboards from json files, splitting them to separate files based on group name.
+
+Currently following imported:
+ - [coreos/prometheus-operator dashboards](https://github.com/coreos/prometheus-operator/blob/master/contrib/kube-prometheus/manifests/grafana-deployment.yaml)
+ - [etcd-io/etc dashboard](https://github.com/etcd-io/etcd/blob/master/Documentation/op-guide/grafana.json)
+ - [coreos/prometheus-operator CoreDNS dashboard](https://github.com/helm/charts/blob/master/stable/prometheus-operator/dashboards/grafana-coredns-k8s.json) (not maintained in this location)

stable/prometheus-operator/hack/sync_grafana_dashboards.py

@@ -0,0 +1,143 @@
+#!/usr/bin/env python3
+"""Fetch dashboards from provided urls into this chart."""
+import json
+import textwrap
+from os import makedirs, path
+
+import requests
+import yaml
+from yaml.representer import SafeRepresenter
+
+
+# https://stackoverflow.com/a/20863889/961092
+class LiteralStr(str):
+    pass
+
+
+def change_style(style, representer):
+    def new_representer(dumper, data):
+        scalar = representer(dumper, data)
+        scalar.style = style
+        return scalar
+
+    return new_representer
+
+
+# Source files list
+charts = [
+    {
+        'source': 'https://raw.githubusercontent.com/coreos/prometheus-operator/master/contrib/kube-prometheus/manifests/grafana-dashboardDefinitions.yaml',
+        'destination': '../templates/grafana/dashboards',
+        'type': 'yaml',
+    },
+    {
+        'source': 'https://raw.githubusercontent.com/etcd-io/etcd/master/Documentation/op-guide/grafana.json',
+        'destination': '../templates/grafana/dashboards',
+        'type': 'json',
+    },
+    {
+        'source': 'https://raw.githubusercontent.com/helm/charts/master/stable/prometheus-operator/dashboards/grafana-coredns-k8s.json',
+        'destination': '../templates/grafana/dashboards',
+        'type': 'json',
+    },
+]
+
+# Additional conditions map
+condition_map = {
+    'grafana-coredns-k8s': ' .Values.coreDns.enabled',
+    'etcd': ' .Values.kubeEtcd.enabled',
+}
+
+# standard header
+header = '''# Generated from '%(name)s' from %(url)s
+{{- if and .Values.grafana.enabled .Values.grafana.defaultDashboardsEnabled%(condition)s }}
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: {{ printf "%%s-%%s" (include "prometheus-operator.fullname" $) "%(name)s" | trunc 63 | trimSuffix "-" }}
+  labels:
+    {{- if $.Values.grafana.sidecar.dashboards.label }}
+    {{ $.Values.grafana.sidecar.dashboards.label }}: "1"
+    {{- end }}
+    app: {{ template "prometheus-operator.name" $ }}-grafana
+{{ include "prometheus-operator.labels" $ | indent 4 }}
+data:
+'''
+
+
+def init_yaml_styles():
+    represent_literal_str = change_style('|', SafeRepresenter.represent_str)
+    yaml.add_representer(LiteralStr, represent_literal_str)
+
+
+def escape(s):
+    return s.replace("{{", "{{`{{").replace("}}", "}}`}}")
+
+
+def yaml_str_repr(struct, indent=2):
+    """represent yaml as a string"""
+    text = yaml.dump(
+        struct,
+        width=1000,  # to disable line wrapping
+        default_flow_style=False  # to disable multiple items on single line
+    )
+    text = escape(text)  # escape {{ and }} for helm
+    text = textwrap.indent(text, ' ' * indent)
+    return text
+
+
+def write_group_to_file(resource_name, content, url, destination):
+    # initialize header
+    lines = header % {
+        'name': resource_name,
+        'url': url,
+        'condition': condition_map.get(resource_name, ''),
+    }
+
+    filename_struct = {resource_name + '.json': (LiteralStr(content))}
+    # rules themselves
+    lines += yaml_str_repr(filename_struct)
+
+    # footer
+    lines += '{{- end }}'
+
+    filename = resource_name + '.yaml'
+    new_filename = "%s/%s" % (destination, filename)
+
+    # make sure directories to store the file exist
+    makedirs(destination, exist_ok=True)
+
+    # recreate the file
+    with open(new_filename, 'w') as f:
+        f.write(lines)
+
+    print("Generated %s" % new_filename)
+
+
+def main():
+    init_yaml_styles()
+    # read the rules, create a new template file per group
+    for chart in charts:
+        print("Generating rules from %s" % chart['source'])
+        raw_text = requests.get(chart['source']).text
+        if chart['type'] == 'yaml':
+            yaml_text = yaml.load(raw_text)
+            groups = yaml_text['items']
+            for group in groups:
+                for resource, content in group['data'].items():
+                    write_group_to_file(resource.replace('.json', ''), content, chart['source'], chart['destination'])
+        elif chart['type'] == 'json':
+            json_text = json.loads(raw_text)
+            # is it already a dashboard structure or is it nested (etcd case)?
+            flat_structure = bool(json_text.get('annotations'))
+            if flat_structure:
+                resource = path.basename(chart['source']).replace('.json', '')
+                write_group_to_file(resource, json.dumps(json_text, indent=4), chart['source'], chart['destination'])
+            else:
+                for resource, content in json_text.items():
+                    write_group_to_file(resource.replace('.json', ''), json.dumps(content, indent=4), chart['source'], chart['destination'])
+    print("Finished")
+
+
+if __name__ == '__main__':
+    main()

stable/prometheus-operator/hack/sync_prometheus_rules.py

@@ -0,0 +1,198 @@
+#!/usr/bin/env python3
+"""Fetch alerting and aggregation rules from provided urls into this chart."""
+import textwrap
+from os import makedirs
+
+import requests
+import yaml
+from yaml.representer import SafeRepresenter
+
+
+# https://stackoverflow.com/a/20863889/961092
+class LiteralStr(str):
+    pass
+
+
+def change_style(style, representer):
+    def new_representer(dumper, data):
+        scalar = representer(dumper, data)
+        scalar.style = style
+        return scalar
+
+    return new_representer
+
+
+# Source files list
+charts = [
+    {
+        'source': 'https://raw.githubusercontent.com/coreos/prometheus-operator/master/contrib/kube-prometheus/manifests/prometheus-rules.yaml',
+        'destination': '../templates/alertmanager/rules'
+    },
+    # don't uncomment until https://github.com/etcd-io/etcd/pull/10244 is merged
+    # {
+    #     'source': 'https://raw.githubusercontent.com/etcd-io/etcd/master/Documentation/op-guide/etcd3_alert.rules.yml',
+    #     'destination': '../templates/alertmanager/rules'
+    # },
+]
+
+# Additional conditions map
+condition_map = {
+    'kube-apiserver.rules': ' .Values.kubeApiServer.enabled',
+    'kube-scheduler.rules': ' .Values.kubeScheduler.enabled',
+    'node.rules': ' .Values.nodeExporter.enabled',
+    'kubernetes-apps': ' .Values.kubeStateMetrics.enabled',
+    'etcd': ' .Values.kubeEtcd.enabled',
+}
+
+alert_condition_map = {
+    'KubeAPIDown': '.Values.kubeApiServer.enabled',  # there are more alerts which are left enabled, because they'll never fire without metrics
+    'KubeControllerManagerDown': '.Values.kubeControllerManager.enabled',
+    'KubeSchedulerDown': '.Values.kubeScheduler.enabled',
+    'KubeStateMetricsDown': '.Values.kubeStateMetrics.enabled',  # there are more alerts which are left enabled, because they'll never fire without metrics
+    'KubeletDown': '.Values.prometheusOperator.kubeletService.enabled',  # there are more alerts which are left enabled, because they'll never fire without metrics
+    'PrometheusOperatorDown': '.Values.prometheusOperator.enabled',
+    'NodeExporterDown': '.Values.nodeExporter.enabled',
+    'CoreDNSDown': '.Values.kubeDns.enabled',
+}
+
+replacement_map = {
+    'job="prometheus-operator"': {
+        'replacement': 'job="{{ $operatorJob }}"',
+        'init': '{{- $operatorJob := printf "%s-%s" (include "prometheus-operator.fullname" .) "operator" }}'},
+    'job="prometheus-k8s"': {
+        'replacement': 'job="{{ $prometheusJob }}"',
+        'init': '{{- $prometheusJob := printf "%s-%s" (include "prometheus-operator.fullname" .) "prometheus" }}'},
+    'job="alertmanager-main"': {
+        'replacement': 'job="{{ $alertmanagerJob }}"',
+        'init': '{{- $alertmanagerJob := printf "%s-%s" (include "prometheus-operator.fullname" .) "alertmanager" }}'},
+}
+
+# standard header
+header = '''# Generated from '%(name)s' group from %(url)s
+{{- if and .Values.defaultRules.create%(condition)s }}%(init_line)s
+apiVersion: {{ printf "%%s/v1" (.Values.prometheusOperator.crdApiGroup | default "monitoring.coreos.com") }}
+kind: PrometheusRule
+metadata:
+  name: {{ printf "%%s-%%s" (include "prometheus-operator.fullname" .) "%(name)s" | trunc 63 | trimSuffix "-" }}
+  labels:
+    app: {{ template "prometheus-operator.name" . }}
+{{ include "prometheus-operator.labels" . | indent 4 }}
+{{- if .Values.defaultRules.labels }}
+{{ toYaml .Values.defaultRules.labels | indent 4 }}
+{{- end }}
+{{- if .Values.defaultRules.annotations }}
+  annotations:
+{{ toYaml .Values.defaultRules.annotations | indent 4 }}
+{{- end }}
+spec:
+  groups:
+  -'''
+
+
+def init_yaml_styles():
+    represent_literal_str = change_style('|', SafeRepresenter.represent_str)
+    yaml.add_representer(LiteralStr, represent_literal_str)
+
+
+def escape(s):
+    return s.replace("{{", "{{`{{").replace("}}", "}}`}}")
+
+
+def fix_expr(rules):
+    """Remove trailing whitespaces and line breaks, which happen to creep in
+     due to yaml import specifics;
+     convert multiline expressions to literal style, |-"""
+    for rule in rules:
+        rule['expr'] = rule['expr'].rstrip()
+        if '\n' in rule['expr']:
+            rule['expr'] = LiteralStr(rule['expr'])
+
+
+def yaml_str_repr(struct, indent=4):
+    """represent yaml as a string"""
+    text = yaml.dump(
+        struct,
+        width=1000,  # to disable line wrapping
+        default_flow_style=False  # to disable multiple items on single line
+    )
+    text = escape(text)  # escape {{ and }} for helm
+    text = textwrap.indent(text, ' ' * indent)[indent - 1:]  # indent everything, and remove very first line extra indentation
+    return text
+
+
+def add_rules_conditions(rules, indent=4):
+    """Add if wrapper for rules, listed in alert_condition_map"""
+    rule_condition = '{{- if %s }}\n'
+    for alert_name in alert_condition_map:
+        line_start = ' ' * indent + '- alert: '
+        if line_start + alert_name in rules:
+            rule_text = rule_condition % alert_condition_map[alert_name]
+            # add if condition
+            index = rules.index(line_start + alert_name)
+            rules = rules[:index] + rule_text + rules[index:]
+            # add end of if
+            try:
+                next_index = rules.index(line_start, index + len(rule_text) + 1)
+            except ValueError:
+                # we found the last alert in file if there are no alerts after it
+                next_index = len(rules)
+            rules = rules[:next_index] + '{{- end }}\n' + rules[next_index:]
+    return rules
+
+
+def write_group_to_file(group, url, destination):
+    fix_expr(group['rules'])
+
+    # prepare rules string representation
+    rules = yaml_str_repr(group)
+    # add replacements of custom variables and include their initialisation in case it's needed
+    init_line = ''
+    for line in replacement_map:
+        if line in rules:
+            rules = rules.replace(line, replacement_map[line]['replacement'])
+            init_line += '\n' + replacement_map[line]['init']
+    # append per-alert rules
+    rules = add_rules_conditions(rules)
+    # initialize header
+    lines = header % {
+        'name': group['name'],
+        'url': url,
+        'condition': condition_map.get(group['name'], ''),
+        'init_line': init_line,
+    }
+
+    # rules themselves
+    lines += rules
+
+    # footer
+    lines += '{{- end }}'
+
+    filename = group['name'] + '.yaml'
+    new_filename = "%s/%s" % (destination, filename)
+
+    # make sure directories to store the file exist
+    makedirs(destination, exist_ok=True)
+
+    # recreate the file
+    with open(new_filename, 'w') as f:
+        f.write(lines)
+
+    print("Generated %s" % new_filename)
+
+
+def main():
+    init_yaml_styles()
+    # read the rules, create a new template file per group
+    for chart in charts:
+        print("Generating rules from %s" % chart['source'])
+        raw_text = requests.get(chart['source']).text
+        yaml_text = yaml.load(raw_text)
+        # etcd workaround, their file don't have spec level
+        groups = yaml_text['spec']['groups'] if yaml_text.get('spec') else yaml_text['groups']
+        for group in groups:
+            write_group_to_file(group, chart['source'], chart['destination'])
+    print("Finished")
+
+
+if __name__ == '__main__':
+    main()