From e4e11f802d15d306edfd86fb922a38dd79676d76 Mon Sep 17 00:00:00 2001 From: kalinux Date: Wed, 10 Jan 2018 10:10:29 +0100 Subject: [PATCH 01/11] adding remote_write support --- manifests/config.pp | 4 ++++ manifests/init.pp | 17 +++++++++++------ manifests/params.pp | 1 + spec/fixtures/files/prometheus2.yaml | 1 + templates/prometheus.yaml.erb | 2 ++ 5 files changed, 19 insertions(+), 6 deletions(-) diff --git a/manifests/config.pp b/manifests/config.pp index 3c3de2a0a..69f4c1167 100644 --- a/manifests/config.pp +++ b/manifests/config.pp @@ -5,6 +5,7 @@ $rule_files, $scrape_configs, $remote_read_configs, + $remote_write_configs, $config_template = $::prometheus::params::config_template, $storage_retention = $::prometheus::params::storage_retention, ) { @@ -16,6 +17,9 @@ if $remote_read_configs != [] { fail('remote_read_configs requires prometheus 2.X') } + if $remote_write_configs != [] { + fail('remote_write_configs requires prometheus 2.X') + } $daemon_flags = [ "-config.file=${::prometheus::config_dir}/prometheus.yaml", "-storage.local.path=${::prometheus::localstorage}", diff --git a/manifests/init.pp b/manifests/init.pp index daff9fa08..adfb553c6 100644 --- a/manifests/init.pp +++ b/manifests/init.pp @@ -102,6 +102,9 @@ # [*remote_read_configs*] # Prometheus remote_read config to scrape prometheus 1.8+ instances # +# [*remote_write_configs*] +# Prometheus remote_write config to scrape prometheus 1.8+ instances +# # [*alert_relabel_config*] # Prometheus alert relabel config under alerting # @@ -153,6 +156,7 @@ Array $rule_files = $::prometheus::params::rule_files, Array $scrape_configs = $::prometheus::params::scrape_configs, Array $remote_read_configs = $::prometheus::params::remote_read_configs, + Array $remote_write_configs = $::prometheus::params::remote_write_configs, $alerts = $::prometheus::params::alerts, Array $alert_relabel_config = $::prometheus::params::alert_relabel_config, Array $alertmanagers_config = $::prometheus::params::alertmanagers_config, @@ -183,12 +187,13 @@ alerts => $alerts, } -> class { '::prometheus::config': - global_config => $global_config, - rule_files => $rule_files, - scrape_configs => $scrape_configs, - remote_read_configs => $remote_read_configs, - config_template => $config_template, - storage_retention => $storage_retention, + global_config => $global_config, + rule_files => $rule_files, + scrape_configs => $scrape_configs, + remote_read_configs => $remote_read_configs, + remote_write_configs => $remote_write_configs, + config_template => $config_template, + storage_retention => $storage_retention, } -> class { '::prometheus::run_service': } -> class { '::prometheus::service_reload': } diff --git a/manifests/params.pp b/manifests/params.pp index d751a50fe..6d031260e 100644 --- a/manifests/params.pp +++ b/manifests/params.pp @@ -134,6 +134,7 @@ $rule_files = [ "${config_dir}/${alertfile_name}" ] $scrape_configs = [ { 'job_name'=> 'prometheus', 'scrape_interval'=> '10s', 'scrape_timeout'=> '10s', 'static_configs'=> [ { 'targets'=> [ 'localhost:9090' ], 'labels'=> { 'alias'=> 'Prometheus'} } ] } ] $remote_read_configs = [] + $remote_write_configs = [] $shared_dir = '/usr/local/share/prometheus' $statsd_exporter_download_extension = 'tar.gz' $statsd_exporter_download_url_base = 'https://github.com/prometheus/statsd_exporter/releases' diff --git a/spec/fixtures/files/prometheus2.yaml b/spec/fixtures/files/prometheus2.yaml index a6a7741a0..cad77d377 100644 --- a/spec/fixtures/files/prometheus2.yaml +++ b/spec/fixtures/files/prometheus2.yaml @@ -19,3 +19,4 @@ alerting: alert_relabel_configs: [] alertmanagers: [] remote_read: [] +remote_write: [] diff --git a/templates/prometheus.yaml.erb b/templates/prometheus.yaml.erb index cb855848c..84d5aeeba 100644 --- a/templates/prometheus.yaml.erb +++ b/templates/prometheus.yaml.erb @@ -3,6 +3,7 @@ <% rule_files = scope.lookupvar('::prometheus::rule_files') -%> <% scrape_configs = scope.lookupvar('::prometheus::scrape_configs') -%> <% remote_read_configs = scope.lookupvar('::prometheus::remote_read_configs') -%> +<% remote_write_configs = scope.lookupvar('::prometheus::remote_write_configs') -%> <% full_config = { 'global'=>global_config, 'rule_files'=>rule_files, @@ -15,6 +16,7 @@ if @prometheus_v2 # this variable has to be absent on 1.X, so we filter using the helper variable full_config['remote_read'] = remote_read_configs + full_config['remote_write'] = remote_write_configs end -%> <%= full_config.to_yaml(options = {:line_width => -1}).gsub(/source_labels: ".+?"/) { |x| x.gsub('"', '') } -%> From fd97f7df8a09a50c935fa69f7cfd48c62ebfcfaf Mon Sep 17 00:00:00 2001 From: kalinux Date: Tue, 16 Jan 2018 16:48:08 +0100 Subject: [PATCH 02/11] adding Prometheus 2.0.0 to readme --- README.md | 26 ++++++++++++++++++++++++-- 1 file changed, 24 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 2aaa212ea..3417cf754 100644 --- a/README.md +++ b/README.md @@ -35,7 +35,7 @@ On the server (for prometheus version < 1.0.0): class { '::prometheus': global_config => { 'scrape_interval'=> '15s', 'evaluation_interval'=> '15s', 'external_labels'=> { 'monitor'=>'master'}}, rule_files => [ "/etc/prometheus/alert.rules" ], - scrape_configs => [ + scrape_configs => [ { 'job_name'=> 'prometheus', 'scrape_interval'=> '10s', 'scrape_timeout' => '10s', @@ -59,6 +59,25 @@ class { 'prometheus': localstorage => '/prometheus/prometheus', } ``` +On the server (for prometheus version >= 2.0.0): + +```puppet +class { '::prometheus': + version => '2.0.0', + alerts => { 'groups' => [{ 'name' => 'alert.rules', 'rules' => [{ 'alert' => 'InstanceDown', 'expr' => 'up == 0', 'for' => '5m', 'labels' => { 'severity' => 'page', }, 'annotations' => { 'summary' => 'Instance {{ $labels.instance }} down', 'description' => '{{ $labels.instance }} of job {{ $labels.job }} has been down for more than 5 minutes.' }}]}]}, + scrape_configs => [ + { 'job_name' => 'prometheus', + 'scrape_interval' => '10s', + 'scrape_timeout' => '10s', + 'static_configs' => [ + { 'targets' => [ 'localhost:9090' ], + 'labels' => { 'alias' => 'Prometheus'} + } + ] + } + ] +} +``` or simply: ```puppet @@ -93,6 +112,10 @@ alertrules: When using prometheus >= 2.0, we use the new yaml format (https://prometheus.io/docs/prometheus/2.0/migration/#recording-rules-and-alerts) configuration +```puppet + alerts => { 'groups' => [{ 'name' => 'alert.rules', 'rules' => [{ 'alert' => 'InstanceDown', 'expr' => 'up == 0', 'for' => '5m', 'labels' => { 'severity' => 'page', }, 'annotations' => { 'summary' => 'Instance {{ $labels.instance }} down', 'description' => '{{ $labels.instance }} of job {{ $labels.job }} has been down for more than 5 minutes.' } }]}]}, +``` + ```yaml alerts: groups: @@ -133,4 +156,3 @@ In version 0.1.14 of this module the alertmanager was configured to run as the s Do not use version 1.0.0 of Prometheus: https://groups.google.com/forum/#!topic/prometheus-developers/vuSIxxUDff8 ; it does break the compatibility with thus module! Even if the module has templates for several linux distributions, only RH family distributions were tested. - From cefc8a7f26de75d7c816a1c7db3094ba65cc7c56 Mon Sep 17 00:00:00 2001 From: Karen Date: Wed, 17 Jan 2018 10:24:01 +0100 Subject: [PATCH 03/11] Syntax fix Fix syntax at the end of remote_read and remote_write --- templates/prometheus.yaml.erb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/templates/prometheus.yaml.erb b/templates/prometheus.yaml.erb index 84d5aeeba..ba380715c 100644 --- a/templates/prometheus.yaml.erb +++ b/templates/prometheus.yaml.erb @@ -15,8 +15,8 @@ } if @prometheus_v2 # this variable has to be absent on 1.X, so we filter using the helper variable - full_config['remote_read'] = remote_read_configs - full_config['remote_write'] = remote_write_configs + full_config['remote_read'] = remote_read_configs, + full_config['remote_write'] = remote_write_configs, end -%> <%= full_config.to_yaml(options = {:line_width => -1}).gsub(/source_labels: ".+?"/) { |x| x.gsub('"', '') } -%> From 10d9ce1d72cbcd569728ef5334b7c0be6f636c8f Mon Sep 17 00:00:00 2001 From: Karen Date: Wed, 17 Jan 2018 10:39:44 +0100 Subject: [PATCH 04/11] fix last one isn't required . --- templates/prometheus.yaml.erb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates/prometheus.yaml.erb b/templates/prometheus.yaml.erb index ba380715c..6f1d59165 100644 --- a/templates/prometheus.yaml.erb +++ b/templates/prometheus.yaml.erb @@ -16,7 +16,7 @@ if @prometheus_v2 # this variable has to be absent on 1.X, so we filter using the helper variable full_config['remote_read'] = remote_read_configs, - full_config['remote_write'] = remote_write_configs, + full_config['remote_write'] = remote_write_configs end -%> <%= full_config.to_yaml(options = {:line_width => -1}).gsub(/source_labels: ".+?"/) { |x| x.gsub('"', '') } -%> From 8ead0b0bdd913d765eeb800240e0e3688efaacb7 Mon Sep 17 00:00:00 2001 From: kalinux Date: Wed, 17 Jan 2018 13:23:53 +0100 Subject: [PATCH 05/11] adding alertmanager 0.13.0 support --- manifests/alertmanager.pp | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/manifests/alertmanager.pp b/manifests/alertmanager.pp index 13bd9eb34..3a33f22c4 100644 --- a/manifests/alertmanager.pp +++ b/manifests/alertmanager.pp @@ -208,9 +208,17 @@ mode => '0755', } - $options = "-config.file=${prometheus::alertmanager::config_file} -storage.path=${prometheus::alertmanager::storage_path} ${prometheus::alertmanager::extra_options}" + if( versioncmp($::prometheus::alertmanager::version, '0.12.0') == 1 ){ + $options = "--config.file=${prometheus::alertmanager::config_file} --storage.path=${prometheus::alertmanager::storage_path} ${prometheus::alertmanager::extra_options}" + } else { + $options = "-config.file=${prometheus::alertmanager::config_file} -storage.path=${prometheus::alertmanager::storage_path} ${prometheus::alertmanager::extra_options}" + } } else { - $options = "-config.file=${prometheus::alertmanager::config_file} ${prometheus::alertmanager::extra_options}" + if( versioncmp($::prometheus::alertmanager::version, '0.12.0') == 1 ){ + $options = "--config.file=${prometheus::alertmanager::config_file} ${prometheus::alertmanager::extra_options}" + } else { + $options = "-config.file=${prometheus::alertmanager::config_file} ${prometheus::alertmanager::extra_options}" + } } prometheus::daemon { $service_name: From 631f89d914573f4d61be0bdd12546c77088f7120 Mon Sep 17 00:00:00 2001 From: kalinux Date: Wed, 17 Jan 2018 14:28:00 +0100 Subject: [PATCH 06/11] adding real examples and vagrant url --- README.md | 96 +++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 96 insertions(+) diff --git a/README.md b/README.md index 3417cf754..2415acf2d 100644 --- a/README.md +++ b/README.md @@ -149,6 +149,102 @@ class { 'prometheus::node_exporter': For more information regarding class parameters please take a look at class docstring. +## Example + +Real Prometheus >=2.0.0 setup example including alertmanager and slack_configs. + +```puppet +include profiles::prometheus + +class { '::prometheus': + version => '2.0.0', + alerts => { 'groups' => [{ 'name' => 'alert.rules', 'rules' => [{ 'alert' => 'InstanceDown', 'expr' => 'up == 0', 'for' => '5m', 'labels' => { 'severity' => 'page', }, 'annotations' => { 'summary' => 'Instance {{ $labels.instance }} down', 'description' => '{{ $labels.instance }} of job {{ $labels.job }} has been down for more than 5 minutes.' } }]}]}, + scrape_configs => [ + { 'job_name' => 'prometheus', + 'scrape_interval' => '10s', + 'scrape_timeout' => '10s', + 'static_configs' => [ + { 'targets' => [ 'localhost:9090' ], + 'labels' => { 'alias'=> 'Prometheus'} + } + ] + }, + { 'job_name' => 'node', + 'scrape_interval' => '5s', + 'scrape_timeout' => '5s', + 'static_configs' => [ + { 'targets' => [ 'nodexporter.domain.com:9100' ], + 'labels' => { 'alias'=> 'Node'} + } + ] + } + ], + alertmanagers_config => [{ 'static_configs' => [{'targets' => [ 'localhost:9093' ]}]}], +} +class { '::prometheus::alertmanager': + version => '0.13.0', + route => { 'group_by' => [ 'alertname', 'cluster', 'service' ], 'group_wait'=> '30s', 'group_interval'=> '5m', 'repeat_interval'=> '3h', 'receiver'=> 'slack' }, + receivers => [ { 'name' => 'slack', 'slack_configs'=> [ { 'api_url'=> 'https://hooks.slack.com/services/ABCDEFG123456', 'channel' => '#channel', 'send_resolved' => true, 'username' => 'username'}] }] +} +``` + +the same in hiera + +```puppet +rometheus::version: '2.0.0' +prometheus::scrape_configs: + - job_name: 'nodexporter' + scrape_interval: '10s' + scrape_timeout: '10s' + static_configs: + - targets: + - nodexporter.domain.com:9100 + labels: + alias: 'nodexporter' + - job_name: prometheus + scrape_interval: 10s + scrape_timeout: 10s + static_configs: + - targets: + - localhost:9090 + labels: + alias: Prometheus +prometheus::alerts: + groups: + - name: alert.rules + rules: + - alert: 'InstanceDown' + expr: 'up == 0' + for: '5m' + labels: + 'severity': 'page' + annotations: + 'summary': 'Instance {{ $labels.instance }} down' + 'description': '{{ $labels.instance }} of job {{ $labels.job }} has been down for more than 5 minutes.' +prometheus::alertmanagers_config: + - static_configs: + - targets: + - localhost:9093 + +prometheus::alertmanager::version: '0.13.0' +prometheus::alertmanager::route: + group_by: + - alertname + - cluster + - service + group_wait: 30s + group_interval: 5m + repeat_interval: 3h + receiver: slack +prometheus::alertmanager::receivers: + - name: slack + slack_configs: + - api_url: https://hooks.slack.com/services/ABCDEFG123456 + channel: "#channel" + send_resolved: true + username: username +``` + ## Limitations/Known issues In version 0.1.14 of this module the alertmanager was configured to run as the service `alert_manager`. This has been changed in version 0.2.00 to be `alertmanager`. From 293cc2a146656d445086aa47c9f1feec14c77070 Mon Sep 17 00:00:00 2001 From: kalinux Date: Wed, 17 Jan 2018 14:29:35 +0100 Subject: [PATCH 07/11] fix vagrant url --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 2415acf2d..64161ad1c 100644 --- a/README.md +++ b/README.md @@ -244,6 +244,7 @@ prometheus::alertmanager::receivers: send_resolved: true username: username ``` +Test you commit with vagrant https://github.com/kalinux/vagrant-puppet-prometheus.git ## Limitations/Known issues From f50bd90d5e9cc09e2045f43f66ac156c11dae095 Mon Sep 17 00:00:00 2001 From: Kalinux Date: Wed, 17 Jan 2018 17:50:12 +0100 Subject: [PATCH 08/11] adding convert to yaml for prometheus 2 --- templates/prometheus.yaml.erb | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/templates/prometheus.yaml.erb b/templates/prometheus.yaml.erb index 6f1d59165..7ace15b8e 100644 --- a/templates/prometheus.yaml.erb +++ b/templates/prometheus.yaml.erb @@ -15,8 +15,15 @@ } if @prometheus_v2 # this variable has to be absent on 1.X, so we filter using the helper variable - full_config['remote_read'] = remote_read_configs, + full_config['remote_read'] = remote_read_configs +end +if @prometheus_v2 full_config['remote_write'] = remote_write_configs end -%> -<%= full_config.to_yaml(options = {:line_width => -1}).gsub(/source_labels: ".+?"/) { |x| x.gsub('"', '') } -%> +<%= if @prometheus_v2 +full_config.to_yaml +else +full_config.to_yaml(options = {:line_width => -1}).gsub(/source_labels: ".+?"/) { |x| x.gsub('"', '') } +end + -%> From 1f5cdab81174d72afc40268556a187af914c01ad Mon Sep 17 00:00:00 2001 From: kalinux Date: Mon, 22 Jan 2018 11:02:08 +0100 Subject: [PATCH 09/11] add spec tests for remote write --- spec/classes/prometheus_spec.rb | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/spec/classes/prometheus_spec.rb b/spec/classes/prometheus_spec.rb index 263d0e855..627ec81e8 100644 --- a/spec/classes/prometheus_spec.rb +++ b/spec/classes/prometheus_spec.rb @@ -253,6 +253,39 @@ end end end + + context 'with remote write configured' do + [ + { + version: '2.0.0-rc.1', + remote_write_configs: [ + 'url' => 'http://domain.tld/path', + ] + } + ].each do |parameters| + context "with prometheus version #{parameters[:version]}" do + let(:params) do + parameters + end + + prom_version = parameters[:version] || '1.5.2' + prom_major = prom_version[0] + + it { + is_expected.to compile + } + it { + is_expected.to contain_file('prometheus.yaml').with( + 'ensure' => 'present', + 'path' => '/etc/prometheus/prometheus.yaml', + 'owner' => 'prometheus', + 'group' => 'prometheus', + 'content' => /http:\/\/domain.tld\/path/ + ) + } + end + end + end end end end From 0e16bbf4e0336a593d208df2e7f2da2487bf4695 Mon Sep 17 00:00:00 2001 From: kalinux Date: Tue, 23 Jan 2018 13:43:38 +0100 Subject: [PATCH 10/11] fix spec tests --- spec/classes/prometheus_spec.rb | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/spec/classes/prometheus_spec.rb b/spec/classes/prometheus_spec.rb index 627ec81e8..7dec38c6e 100644 --- a/spec/classes/prometheus_spec.rb +++ b/spec/classes/prometheus_spec.rb @@ -259,7 +259,7 @@ { version: '2.0.0-rc.1', remote_write_configs: [ - 'url' => 'http://domain.tld/path', + 'url' => 'http://domain.tld/path' ] } ].each do |parameters| @@ -269,7 +269,6 @@ end prom_version = parameters[:version] || '1.5.2' - prom_major = prom_version[0] it { is_expected.to compile @@ -280,7 +279,7 @@ 'path' => '/etc/prometheus/prometheus.yaml', 'owner' => 'prometheus', 'group' => 'prometheus', - 'content' => /http:\/\/domain.tld\/path/ + 'content' => %r{http://domain.tld/path} ) } end From 6cc23a7ae6900cd43e7f10ee7810819dca4f98cf Mon Sep 17 00:00:00 2001 From: kalinux Date: Tue, 23 Jan 2018 13:49:40 +0100 Subject: [PATCH 11/11] fix useless variable in spec file --- spec/classes/prometheus_spec.rb | 2 -- 1 file changed, 2 deletions(-) diff --git a/spec/classes/prometheus_spec.rb b/spec/classes/prometheus_spec.rb index 7dec38c6e..2a7ec99cd 100644 --- a/spec/classes/prometheus_spec.rb +++ b/spec/classes/prometheus_spec.rb @@ -268,8 +268,6 @@ parameters end - prom_version = parameters[:version] || '1.5.2' - it { is_expected.to compile }