diff --git a/stable/prometheus-operator/Chart.yaml b/stable/prometheus-operator/Chart.yaml index 8febecaedece..06593bd6e67d 100644 --- a/stable/prometheus-operator/Chart.yaml +++ b/stable/prometheus-operator/Chart.yaml @@ -9,7 +9,7 @@ name: prometheus-operator sources: - https://github.com/coreos/prometheus-operator - https://coreos.com/operators/prometheus -version: 1.5.2 +version: 1.6.0 appVersion: 0.26.0 home: https://github.com/coreos/prometheus-operator keywords: diff --git a/stable/prometheus-operator/README.md b/stable/prometheus-operator/README.md index ad73bc4c69d5..f16dccc8e0fe 100644 --- a/stable/prometheus-operator/README.md +++ b/stable/prometheus-operator/README.md @@ -286,6 +286,11 @@ $ helm install --name my-release stable/prometheus-operator -f values1.yaml,valu > **Tip**: You can use the default [values.yaml](values.yaml) + +## Developing Prometheus Rules and Grafana Dashboards + +This chart Grafana Dashboards and Prometheus Rules are just a copy from coreos/prometheus-operator and other sources, synced (with alterations) by scripts in [hack](hack) folder. In order to introduce any changes you need to first [add them to original repo](https://github.com/coreos/prometheus-operator/blob/master/contrib/kube-prometheus/docs/developing-prometheus-rules-and-grafana-dashboards.md) and then sync there by scripts. + ## Further Information For more in-depth documentation of configuration options meanings, please see diff --git a/stable/prometheus-operator/hack/README.md b/stable/prometheus-operator/hack/README.md new file mode 100644 index 000000000000..4fc7e6a90fb7 --- /dev/null +++ b/stable/prometheus-operator/hack/README.md @@ -0,0 +1,18 @@ +# prometheus-operator hacks + +## [sync_prometheus_rules.py](sync_prometheus_rules.py) + +This script generates prometheus rules set for alertmanager from any properly formatted kubernetes yaml based on defined input, splitting rules to separate files based on group name. + +Currently following imported: + - [coreos/prometheus-operator rules set](https://github.com/coreos/prometheus-operator/blob/master/contrib/kube-prometheus/manifests/prometheus-rules.yaml) + - [etcd-io/etc rules set](https://github.com/etcd-io/etcd/blob/master/Documentation/op-guide/etcd3_alert.rules.yml) (temporary disabled) + +## [sync_grafana_dashboards.py](sync_grafana_dashboards.py) + +This script generates grafana dashboards from json files, splitting them to separate files based on group name. + +Currently following imported: + - [coreos/prometheus-operator dashboards](https://github.com/coreos/prometheus-operator/blob/master/contrib/kube-prometheus/manifests/grafana-deployment.yaml) + - [etcd-io/etc dashboard](https://github.com/etcd-io/etcd/blob/master/Documentation/op-guide/grafana.json) + - [coreos/prometheus-operator CoreDNS dashboard](https://github.com/helm/charts/blob/master/stable/prometheus-operator/dashboards/grafana-coredns-k8s.json) (not maintained in this location) \ No newline at end of file diff --git a/stable/prometheus-operator/hack/sync_grafana_dashboards.py b/stable/prometheus-operator/hack/sync_grafana_dashboards.py new file mode 100755 index 000000000000..f548d0b7a364 --- /dev/null +++ b/stable/prometheus-operator/hack/sync_grafana_dashboards.py @@ -0,0 +1,143 @@ +#!/usr/bin/env python3 +"""Fetch dashboards from provided urls into this chart.""" +import json +import textwrap +from os import makedirs, path + +import requests +import yaml +from yaml.representer import SafeRepresenter + + +# https://stackoverflow.com/a/20863889/961092 +class LiteralStr(str): + pass + + +def change_style(style, representer): + def new_representer(dumper, data): + scalar = representer(dumper, data) + scalar.style = style + return scalar + + return new_representer + + +# Source files list +charts = [ + { + 'source': 'https://raw.githubusercontent.com/coreos/prometheus-operator/master/contrib/kube-prometheus/manifests/grafana-dashboardDefinitions.yaml', + 'destination': '../templates/grafana/dashboards', + 'type': 'yaml', + }, + { + 'source': 'https://raw.githubusercontent.com/etcd-io/etcd/master/Documentation/op-guide/grafana.json', + 'destination': '../templates/grafana/dashboards', + 'type': 'json', + }, + { + 'source': 'https://raw.githubusercontent.com/helm/charts/master/stable/prometheus-operator/dashboards/grafana-coredns-k8s.json', + 'destination': '../templates/grafana/dashboards', + 'type': 'json', + }, +] + +# Additional conditions map +condition_map = { + 'grafana-coredns-k8s': ' .Values.coreDns.enabled', + 'etcd': ' .Values.kubeEtcd.enabled', +} + +# standard header +header = '''# Generated from '%(name)s' from %(url)s +{{- if and .Values.grafana.enabled .Values.grafana.defaultDashboardsEnabled%(condition)s }} +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ printf "%%s-%%s" (include "prometheus-operator.fullname" $) "%(name)s" | trunc 63 | trimSuffix "-" }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: "1" + {{- end }} + app: {{ template "prometheus-operator.name" $ }}-grafana +{{ include "prometheus-operator.labels" $ | indent 4 }} +data: +''' + + +def init_yaml_styles(): + represent_literal_str = change_style('|', SafeRepresenter.represent_str) + yaml.add_representer(LiteralStr, represent_literal_str) + + +def escape(s): + return s.replace("{{", "{{`{{").replace("}}", "}}`}}") + + +def yaml_str_repr(struct, indent=2): + """represent yaml as a string""" + text = yaml.dump( + struct, + width=1000, # to disable line wrapping + default_flow_style=False # to disable multiple items on single line + ) + text = escape(text) # escape {{ and }} for helm + text = textwrap.indent(text, ' ' * indent) + return text + + +def write_group_to_file(resource_name, content, url, destination): + # initialize header + lines = header % { + 'name': resource_name, + 'url': url, + 'condition': condition_map.get(resource_name, ''), + } + + filename_struct = {resource_name + '.json': (LiteralStr(content))} + # rules themselves + lines += yaml_str_repr(filename_struct) + + # footer + lines += '{{- end }}' + + filename = resource_name + '.yaml' + new_filename = "%s/%s" % (destination, filename) + + # make sure directories to store the file exist + makedirs(destination, exist_ok=True) + + # recreate the file + with open(new_filename, 'w') as f: + f.write(lines) + + print("Generated %s" % new_filename) + + +def main(): + init_yaml_styles() + # read the rules, create a new template file per group + for chart in charts: + print("Generating rules from %s" % chart['source']) + raw_text = requests.get(chart['source']).text + if chart['type'] == 'yaml': + yaml_text = yaml.load(raw_text) + groups = yaml_text['items'] + for group in groups: + for resource, content in group['data'].items(): + write_group_to_file(resource.replace('.json', ''), content, chart['source'], chart['destination']) + elif chart['type'] == 'json': + json_text = json.loads(raw_text) + # is it already a dashboard structure or is it nested (etcd case)? + flat_structure = bool(json_text.get('annotations')) + if flat_structure: + resource = path.basename(chart['source']).replace('.json', '') + write_group_to_file(resource, json.dumps(json_text, indent=4), chart['source'], chart['destination']) + else: + for resource, content in json_text.items(): + write_group_to_file(resource.replace('.json', ''), json.dumps(content, indent=4), chart['source'], chart['destination']) + print("Finished") + + +if __name__ == '__main__': + main() diff --git a/stable/prometheus-operator/hack/sync_prometheus_rules.py b/stable/prometheus-operator/hack/sync_prometheus_rules.py new file mode 100755 index 000000000000..76242923fab9 --- /dev/null +++ b/stable/prometheus-operator/hack/sync_prometheus_rules.py @@ -0,0 +1,198 @@ +#!/usr/bin/env python3 +"""Fetch alerting and aggregation rules from provided urls into this chart.""" +import textwrap +from os import makedirs + +import requests +import yaml +from yaml.representer import SafeRepresenter + + +# https://stackoverflow.com/a/20863889/961092 +class LiteralStr(str): + pass + + +def change_style(style, representer): + def new_representer(dumper, data): + scalar = representer(dumper, data) + scalar.style = style + return scalar + + return new_representer + + +# Source files list +charts = [ + { + 'source': 'https://raw.githubusercontent.com/coreos/prometheus-operator/master/contrib/kube-prometheus/manifests/prometheus-rules.yaml', + 'destination': '../templates/alertmanager/rules' + }, + # don't uncomment until https://github.com/etcd-io/etcd/pull/10244 is merged + # { + # 'source': 'https://raw.githubusercontent.com/etcd-io/etcd/master/Documentation/op-guide/etcd3_alert.rules.yml', + # 'destination': '../templates/alertmanager/rules' + # }, +] + +# Additional conditions map +condition_map = { + 'kube-apiserver.rules': ' .Values.kubeApiServer.enabled', + 'kube-scheduler.rules': ' .Values.kubeScheduler.enabled', + 'node.rules': ' .Values.nodeExporter.enabled', + 'kubernetes-apps': ' .Values.kubeStateMetrics.enabled', + 'etcd': ' .Values.kubeEtcd.enabled', +} + +alert_condition_map = { + 'KubeAPIDown': '.Values.kubeApiServer.enabled', # there are more alerts which are left enabled, because they'll never fire without metrics + 'KubeControllerManagerDown': '.Values.kubeControllerManager.enabled', + 'KubeSchedulerDown': '.Values.kubeScheduler.enabled', + 'KubeStateMetricsDown': '.Values.kubeStateMetrics.enabled', # there are more alerts which are left enabled, because they'll never fire without metrics + 'KubeletDown': '.Values.prometheusOperator.kubeletService.enabled', # there are more alerts which are left enabled, because they'll never fire without metrics + 'PrometheusOperatorDown': '.Values.prometheusOperator.enabled', + 'NodeExporterDown': '.Values.nodeExporter.enabled', + 'CoreDNSDown': '.Values.kubeDns.enabled', +} + +replacement_map = { + 'job="prometheus-operator"': { + 'replacement': 'job="{{ $operatorJob }}"', + 'init': '{{- $operatorJob := printf "%s-%s" (include "prometheus-operator.fullname" .) "operator" }}'}, + 'job="prometheus-k8s"': { + 'replacement': 'job="{{ $prometheusJob }}"', + 'init': '{{- $prometheusJob := printf "%s-%s" (include "prometheus-operator.fullname" .) "prometheus" }}'}, + 'job="alertmanager-main"': { + 'replacement': 'job="{{ $alertmanagerJob }}"', + 'init': '{{- $alertmanagerJob := printf "%s-%s" (include "prometheus-operator.fullname" .) "alertmanager" }}'}, +} + +# standard header +header = '''# Generated from '%(name)s' group from %(url)s +{{- if and .Values.defaultRules.create%(condition)s }}%(init_line)s +apiVersion: {{ printf "%%s/v1" (.Values.prometheusOperator.crdApiGroup | default "monitoring.coreos.com") }} +kind: PrometheusRule +metadata: + name: {{ printf "%%s-%%s" (include "prometheus-operator.fullname" .) "%(name)s" | trunc 63 | trimSuffix "-" }} + labels: + app: {{ template "prometheus-operator.name" . }} +{{ include "prometheus-operator.labels" . | indent 4 }} +{{- if .Values.defaultRules.labels }} +{{ toYaml .Values.defaultRules.labels | indent 4 }} +{{- end }} +{{- if .Values.defaultRules.annotations }} + annotations: +{{ toYaml .Values.defaultRules.annotations | indent 4 }} +{{- end }} +spec: + groups: + -''' + + +def init_yaml_styles(): + represent_literal_str = change_style('|', SafeRepresenter.represent_str) + yaml.add_representer(LiteralStr, represent_literal_str) + + +def escape(s): + return s.replace("{{", "{{`{{").replace("}}", "}}`}}") + + +def fix_expr(rules): + """Remove trailing whitespaces and line breaks, which happen to creep in + due to yaml import specifics; + convert multiline expressions to literal style, |-""" + for rule in rules: + rule['expr'] = rule['expr'].rstrip() + if '\n' in rule['expr']: + rule['expr'] = LiteralStr(rule['expr']) + + +def yaml_str_repr(struct, indent=4): + """represent yaml as a string""" + text = yaml.dump( + struct, + width=1000, # to disable line wrapping + default_flow_style=False # to disable multiple items on single line + ) + text = escape(text) # escape {{ and }} for helm + text = textwrap.indent(text, ' ' * indent)[indent - 1:] # indent everything, and remove very first line extra indentation + return text + + +def add_rules_conditions(rules, indent=4): + """Add if wrapper for rules, listed in alert_condition_map""" + rule_condition = '{{- if %s }}\n' + for alert_name in alert_condition_map: + line_start = ' ' * indent + '- alert: ' + if line_start + alert_name in rules: + rule_text = rule_condition % alert_condition_map[alert_name] + # add if condition + index = rules.index(line_start + alert_name) + rules = rules[:index] + rule_text + rules[index:] + # add end of if + try: + next_index = rules.index(line_start, index + len(rule_text) + 1) + except ValueError: + # we found the last alert in file if there are no alerts after it + next_index = len(rules) + rules = rules[:next_index] + '{{- end }}\n' + rules[next_index:] + return rules + + +def write_group_to_file(group, url, destination): + fix_expr(group['rules']) + + # prepare rules string representation + rules = yaml_str_repr(group) + # add replacements of custom variables and include their initialisation in case it's needed + init_line = '' + for line in replacement_map: + if line in rules: + rules = rules.replace(line, replacement_map[line]['replacement']) + init_line += '\n' + replacement_map[line]['init'] + # append per-alert rules + rules = add_rules_conditions(rules) + # initialize header + lines = header % { + 'name': group['name'], + 'url': url, + 'condition': condition_map.get(group['name'], ''), + 'init_line': init_line, + } + + # rules themselves + lines += rules + + # footer + lines += '{{- end }}' + + filename = group['name'] + '.yaml' + new_filename = "%s/%s" % (destination, filename) + + # make sure directories to store the file exist + makedirs(destination, exist_ok=True) + + # recreate the file + with open(new_filename, 'w') as f: + f.write(lines) + + print("Generated %s" % new_filename) + + +def main(): + init_yaml_styles() + # read the rules, create a new template file per group + for chart in charts: + print("Generating rules from %s" % chart['source']) + raw_text = requests.get(chart['source']).text + yaml_text = yaml.load(raw_text) + # etcd workaround, their file don't have spec level + groups = yaml_text['spec']['groups'] if yaml_text.get('spec') else yaml_text['groups'] + for group in groups: + write_group_to_file(group, chart['source'], chart['destination']) + print("Finished") + + +if __name__ == '__main__': + main()