From 3f94405fbcf367a351b57d2237e672d1225a9525 Mon Sep 17 00:00:00 2001 From: Pawel Fiuto Date: Thu, 11 May 2017 17:43:52 +0200 Subject: [PATCH 1/9] Adding consul_exporter --- manifests/consul_exporter.pp | 165 +++++++++++++++++++++++++++++++++++ manifests/params.pp | 13 +++ tests/init.pp | 1 + 3 files changed, 179 insertions(+) create mode 100644 manifests/consul_exporter.pp diff --git a/manifests/consul_exporter.pp b/manifests/consul_exporter.pp new file mode 100644 index 000000000..fa6f3b7ff --- /dev/null +++ b/manifests/consul_exporter.pp @@ -0,0 +1,165 @@ +# Class: prometheus::consul_exporter +# +# This module manages prometheus node consul_exporter +# +# Parameters: +# [*arch*] +# Architecture (amd64 or i386) +# +# [*bin_dir*] +# Directory where binaries are located +# +# [*consul_server*] +# HTTP API address of a Consul server or agent. (prefix with https:// to connect over HTTPS) (default "http://localhost:8500") +# +# [*consul_health_summary*] +# Generate a health summary for each service instance. Needs n+1 queries to collect all information. (default true) +# +# [*download_extension*] +# Extension for the release binary archive +# +# [*download_url*] +# Complete URL corresponding to the where the release binary archive can be downloaded +# +# [*download_url_base*] +# Base URL for the binary archive +# +# [*extra_groups*] +# Extra groups to add the binary user to +# +# [*extra_options*] +# Extra options added to the startup command +# +# [*group*] +# Group under which the binary is running +# +# [*init_style*] +# Service startup scripts style (e.g. rc, upstart or systemd) +# +# [*install_method*] +# Installation method: url or package (only url is supported currently) +# +# [*log_level*] +# Only log messages with the given severity or above. Valid levels: [debug, info, warn, error, fatal] (default "info") +# +# [*manage_group*] +# Whether to create a group for or rely on external code for that +# +# [*manage_service*] +# Should puppet manage the service? (default true) +# +# [*manage_user*] +# Whether to create user or rely on external code for that +# +# [*os*] +# Operating system (linux is the only one supported) +# +# [*package_ensure*] +# If package, then use this for package ensure default 'latest' +# +# [*package_name*] +# The binary package name - not available yet +# +# [*purge_config_dir*] +# Purge config files no longer generated by Puppet +# +# [*restart_on_change*] +# Should puppet restart the service on configuration change? (default true) +# +# [*service_enable*] +# Whether to enable the service from puppet (default true) +# +# [*service_ensure*] +# State ensured for the service (default 'running') +# +# [*user*] +# User which runs the service +# +# [*version*] +# The binary release version +# +# [*web_listen_address*] +# Address to listen on for web interface and telemetry. (default ":9107") +# +# [*web_telemetry_path*] +# Path under which to expose metrics. (default "/metrics") +class prometheus::consul_exporter ( + $arch = $::prometheus::params::arch, + $bin_dir = $::prometheus::params::bin_dir, + $consul_health_summary = $::prometheus::params::consul_exporter_consul_health_summary, + $consul_server = $::prometheus::params::consul_exporter_consul_server, + $download_extension = $::prometheus::params::consul_exporter_download_extension, + $download_url = undef, + $download_url_base = $::prometheus::params::consul_exporter_download_url_base, + $extra_groups = $::prometheus::params::consul_exporter_extra_groups, + $extra_options = '', + $group = $::prometheus::params::consul_exporter_group, + $init_style = $::prometheus::params::init_style, + $install_method = $::prometheus::params::install_method, + $log_level = $::prometheus::params::consul_exporter_log_level, + $manage_group = true, + $manage_service = true, + $manage_user = true, + $os = $::prometheus::params::os, + $package_ensure = $::prometheus::params::consul_exporter_package_ensure, + $package_name = $::prometheus::params::consul_exporter_package_name, + $purge_config_dir = true, + $restart_on_change = true, + $service_enable = true, + $service_ensure = 'running', + $user = $::prometheus::params::consul_exporter_user, + $version = $::prometheus::params::consul_exporter_version, + $web_listen_address = $::prometheus::params::consul_exporter_web_listen_address, + $web_telemetry_path = $::prometheus::params::consul_exporter_web_telemetry_path, +) inherits prometheus::params { + # Prometheus added a 'v' on the realease name at 0.3.0 + if versioncmp ($version, '0.3.0') >= 0 { + $release = "v${version}" + } + else { + $release = $version + } + $real_download_url = pick($download_url,"${download_url_base}/download/${release}/${package_name}-${version}.${os}-${arch}.${download_extension}") + validate_bool($purge_config_dir) + validate_bool($manage_user) + validate_bool($manage_service) + validate_bool($restart_on_change) + validate_bool($consul_health_summary) + + if $consul_health_summary { + $real_consul_health_summary = '-consul.health-summary' + } else { + $real_consul_health_summary = '' + } + + $notify_service = $restart_on_change ? { + true => Service['consul_exporter'], + default => undef, + } + + $options = "-consul.server=${consul_server} ${real_consul_health_summary} -web.listen-address=${web_listen_address} -web.telemetry-path=${web_telemetry_path} -log.level=${log_level} ${extra_options}" + + prometheus::daemon { 'consul_exporter': + install_method => $install_method, + version => $version, + download_extension => $download_extension, + os => $os, + arch => $arch, + real_download_url => $real_download_url, + bin_dir => $bin_dir, + notify_service => $notify_service, + package_name => $package_name, + package_ensure => $package_ensure, + manage_user => $manage_user, + user => $user, + extra_groups => $extra_groups, + group => $group, + manage_group => $manage_group, + purge => $purge_config_dir, + options => $options, + init_style => $init_style, + service_ensure => $service_ensure, + service_enable => $service_enable, + manage_service => $manage_service, + } +} diff --git a/manifests/params.pp b/manifests/params.pp index a991a2764..4d2acd3b4 100644 --- a/manifests/params.pp +++ b/manifests/params.pp @@ -22,6 +22,19 @@ $config_dir = '/etc/prometheus' $config_mode = '0660' $config_template = 'prometheus/prometheus.yaml.erb' + $consul_exporter_consul_health_summary = true + $consul_exporter_consul_server = 'localhost:8500' + $consul_exporter_download_extension = 'tar.gz' + $consul_exporter_download_url_base = 'https://github.com/prometheus/consul_exporter/releases' + $consul_exporter_extra_groups = [] + $consul_exporter_group = 'consul-exporter' + $consul_exporter_log_level = 'info' + $consul_exporter_package_ensure = 'latest' + $consul_exporter_package_name = 'consul_exporter' + $consul_exporter_user = 'consul-exporter' + $consul_exporter_version = '0.3.0' + $consul_exporter_web_listen_address = ':9107' + $consul_exporter_web_telemetry_path = '/metrics' $download_extension = 'tar.gz' $download_url_base = 'https://github.com/prometheus/prometheus/releases' $extra_groups = [] diff --git a/tests/init.pp b/tests/init.pp index 22e0869b8..c23b6970c 100644 --- a/tests/init.pp +++ b/tests/init.pp @@ -3,3 +3,4 @@ include prometheus::alertmanager include prometheus::alerts include prometheus::statsd_exporter +include prometheus::consul_exporter From 35bf3931d11570de056f3935912ef46954975dd3 Mon Sep 17 00:00:00 2001 From: Pawel Fiuto Date: Thu, 11 May 2017 18:24:48 +0200 Subject: [PATCH 2/9] Cater for archives with different binary file location --- manifests/consul_exporter.pp | 2 ++ manifests/daemon.pp | 7 ++++++- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/manifests/consul_exporter.pp b/manifests/consul_exporter.pp index fa6f3b7ff..0ad37c95c 100644 --- a/manifests/consul_exporter.pp +++ b/manifests/consul_exporter.pp @@ -117,6 +117,7 @@ $release = "v${version}" } else { + $custom_binary_location = 'consul_exporter' $release = $version } $real_download_url = pick($download_url,"${download_url_base}/download/${release}/${package_name}-${version}.${os}-${arch}.${download_extension}") @@ -140,6 +141,7 @@ $options = "-consul.server=${consul_server} ${real_consul_health_summary} -web.listen-address=${web_listen_address} -web.telemetry-path=${web_telemetry_path} -log.level=${log_level} ${extra_options}" prometheus::daemon { 'consul_exporter': + custom_binary_location => $custom_binary_location, install_method => $install_method, version => $version, download_extension => $download_extension, diff --git a/manifests/daemon.pp b/manifests/daemon.pp index 108a0dbb8..cc5b4a059 100644 --- a/manifests/daemon.pp +++ b/manifests/daemon.pp @@ -20,12 +20,17 @@ $service_ensure, $service_enable, $manage_service, + $custom_binary_location = '', ) { case $install_method { 'url': { include staging $staging_file = "${name}-${version}.${download_extension}" - $binary = "${::staging::path}/${name}-${version}.${os}-${arch}/${name}" + if $custom_binary_location == '' { + $binary = "${::staging::path}/${name}-${version}.${os}-${arch}/${name}" + } else { + $binary = "${::staging::path}/${custom_binary_location}" + } staging::file { $staging_file: source => $real_download_url, } -> From 5ff45d1e3f4655adbc82faf36366627e2d909e3f Mon Sep 17 00:00:00 2001 From: Pawel Fiuto Date: Thu, 11 May 2017 18:26:11 +0200 Subject: [PATCH 3/9] Linting file --- manifests/consul_exporter.pp | 46 ++++++++++++++++++------------------ 1 file changed, 23 insertions(+), 23 deletions(-) diff --git a/manifests/consul_exporter.pp b/manifests/consul_exporter.pp index 0ad37c95c..8c8cf354e 100644 --- a/manifests/consul_exporter.pp +++ b/manifests/consul_exporter.pp @@ -126,13 +126,13 @@ validate_bool($manage_service) validate_bool($restart_on_change) validate_bool($consul_health_summary) - + if $consul_health_summary { $real_consul_health_summary = '-consul.health-summary' } else { $real_consul_health_summary = '' } - + $notify_service = $restart_on_change ? { true => Service['consul_exporter'], default => undef, @@ -142,26 +142,26 @@ prometheus::daemon { 'consul_exporter': custom_binary_location => $custom_binary_location, - install_method => $install_method, - version => $version, - download_extension => $download_extension, - os => $os, - arch => $arch, - real_download_url => $real_download_url, - bin_dir => $bin_dir, - notify_service => $notify_service, - package_name => $package_name, - package_ensure => $package_ensure, - manage_user => $manage_user, - user => $user, - extra_groups => $extra_groups, - group => $group, - manage_group => $manage_group, - purge => $purge_config_dir, - options => $options, - init_style => $init_style, - service_ensure => $service_ensure, - service_enable => $service_enable, - manage_service => $manage_service, + install_method => $install_method, + version => $version, + download_extension => $download_extension, + os => $os, + arch => $arch, + real_download_url => $real_download_url, + bin_dir => $bin_dir, + notify_service => $notify_service, + package_name => $package_name, + package_ensure => $package_ensure, + manage_user => $manage_user, + user => $user, + extra_groups => $extra_groups, + group => $group, + manage_group => $manage_group, + purge => $purge_config_dir, + options => $options, + init_style => $init_style, + service_ensure => $service_ensure, + service_enable => $service_enable, + manage_service => $manage_service, } } From 40fa05de56aa587ab71cb4d79c3ebb77dabedfda Mon Sep 17 00:00:00 2001 From: Pawel Fiuto Date: Mon, 7 Aug 2017 13:34:16 +0200 Subject: [PATCH 4/9] Drop support for custom_binary_location --- manifests/consul_exporter.pp | 43 ++++++++++++++++++------------------ 1 file changed, 21 insertions(+), 22 deletions(-) diff --git a/manifests/consul_exporter.pp b/manifests/consul_exporter.pp index 8c8cf354e..8e81007c2 100644 --- a/manifests/consul_exporter.pp +++ b/manifests/consul_exporter.pp @@ -141,27 +141,26 @@ $options = "-consul.server=${consul_server} ${real_consul_health_summary} -web.listen-address=${web_listen_address} -web.telemetry-path=${web_telemetry_path} -log.level=${log_level} ${extra_options}" prometheus::daemon { 'consul_exporter': - custom_binary_location => $custom_binary_location, - install_method => $install_method, - version => $version, - download_extension => $download_extension, - os => $os, - arch => $arch, - real_download_url => $real_download_url, - bin_dir => $bin_dir, - notify_service => $notify_service, - package_name => $package_name, - package_ensure => $package_ensure, - manage_user => $manage_user, - user => $user, - extra_groups => $extra_groups, - group => $group, - manage_group => $manage_group, - purge => $purge_config_dir, - options => $options, - init_style => $init_style, - service_ensure => $service_ensure, - service_enable => $service_enable, - manage_service => $manage_service, + install_method => $install_method, + version => $version, + download_extension => $download_extension, + os => $os, + arch => $arch, + real_download_url => $real_download_url, + bin_dir => $bin_dir, + notify_service => $notify_service, + package_name => $package_name, + package_ensure => $package_ensure, + manage_user => $manage_user, + user => $user, + extra_groups => $extra_groups, + group => $group, + manage_group => $manage_group, + purge => $purge_config_dir, + options => $options, + init_style => $init_style, + service_ensure => $service_ensure, + service_enable => $service_enable, + manage_service => $manage_service, } } From 0fb2e2e644cadb097e5ea63930e7383991685be4 Mon Sep 17 00:00:00 2001 From: Pawel Fiuto Date: Tue, 8 Aug 2017 12:26:38 +0200 Subject: [PATCH 5/9] Support version 0.3.0 and higher --- manifests/consul_exporter.pp | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/manifests/consul_exporter.pp b/manifests/consul_exporter.pp index 8e81007c2..d9a34fb99 100644 --- a/manifests/consul_exporter.pp +++ b/manifests/consul_exporter.pp @@ -113,14 +113,11 @@ $web_telemetry_path = $::prometheus::params::consul_exporter_web_telemetry_path, ) inherits prometheus::params { # Prometheus added a 'v' on the realease name at 0.3.0 - if versioncmp ($version, '0.3.0') >= 0 { - $release = "v${version}" + if versioncmp ($version, '0.3.0') == -1 { + fail("I only support consul_exporter version '0.3.0' or higher") } - else { - $custom_binary_location = 'consul_exporter' - $release = $version - } - $real_download_url = pick($download_url,"${download_url_base}/download/${release}/${package_name}-${version}.${os}-${arch}.${download_extension}") + + $real_download_url = pick($download_url,"${download_url_base}/download/v${version}/${package_name}-${version}.${os}-${arch}.${download_extension}") validate_bool($purge_config_dir) validate_bool($manage_user) validate_bool($manage_service) From fe4efecbd28e6ca1cbc0d0a04b58bbd8b5e85592 Mon Sep 17 00:00:00 2001 From: Pawel Fiuto Date: Thu, 16 Nov 2017 09:08:13 +0100 Subject: [PATCH 6/9] Replaced legacy functions with native puppet datatypes --- manifests/consul_exporter.pp | 59 +++++++++++++++++------------------- 1 file changed, 27 insertions(+), 32 deletions(-) diff --git a/manifests/consul_exporter.pp b/manifests/consul_exporter.pp index d9a34fb99..409a96633 100644 --- a/manifests/consul_exporter.pp +++ b/manifests/consul_exporter.pp @@ -84,33 +84,33 @@ # [*web_telemetry_path*] # Path under which to expose metrics. (default "/metrics") class prometheus::consul_exporter ( - $arch = $::prometheus::params::arch, - $bin_dir = $::prometheus::params::bin_dir, - $consul_health_summary = $::prometheus::params::consul_exporter_consul_health_summary, - $consul_server = $::prometheus::params::consul_exporter_consul_server, - $download_extension = $::prometheus::params::consul_exporter_download_extension, - $download_url = undef, - $download_url_base = $::prometheus::params::consul_exporter_download_url_base, - $extra_groups = $::prometheus::params::consul_exporter_extra_groups, - $extra_options = '', - $group = $::prometheus::params::consul_exporter_group, - $init_style = $::prometheus::params::init_style, - $install_method = $::prometheus::params::install_method, - $log_level = $::prometheus::params::consul_exporter_log_level, - $manage_group = true, - $manage_service = true, - $manage_user = true, - $os = $::prometheus::params::os, - $package_ensure = $::prometheus::params::consul_exporter_package_ensure, - $package_name = $::prometheus::params::consul_exporter_package_name, - $purge_config_dir = true, - $restart_on_change = true, - $service_enable = true, - $service_ensure = 'running', - $user = $::prometheus::params::consul_exporter_user, - $version = $::prometheus::params::consul_exporter_version, - $web_listen_address = $::prometheus::params::consul_exporter_web_listen_address, - $web_telemetry_path = $::prometheus::params::consul_exporter_web_telemetry_path, + String $arch = $::prometheus::params::arch, + String $bin_dir = $::prometheus::params::bin_dir, + Boolean $consul_health_summary = $::prometheus::params::consul_exporter_consul_health_summary, + String $consul_server = $::prometheus::params::consul_exporter_consul_server, + String $download_extension = $::prometheus::params::consul_exporter_download_extension, + Optional[String] $download_url = undef, + String $download_url_base = $::prometheus::params::consul_exporter_download_url_base, + Array $extra_groups = $::prometheus::params::consul_exporter_extra_groups, + String $extra_options = '', + String $group = $::prometheus::params::consul_exporter_group, + String $init_style = $::prometheus::params::init_style, + String $install_method = $::prometheus::params::install_method, + String $log_level = $::prometheus::params::consul_exporter_log_level, + Boolean $manage_group = true, + Boolean $manage_service = true, + Boolean $manage_user = true, + String $os = $::prometheus::params::os, + String $package_ensure = $::prometheus::params::consul_exporter_package_ensure, + String $package_name = $::prometheus::params::consul_exporter_package_name, + Boolean $purge_config_dir = true, + Boolean $restart_on_change = true, + Boolean $service_enable = true, + String $service_ensure = 'running', + String $user = $::prometheus::params::consul_exporter_user, + String $version = $::prometheus::params::consul_exporter_version, + String $web_listen_address = $::prometheus::params::consul_exporter_web_listen_address, + String $web_telemetry_path = $::prometheus::params::consul_exporter_web_telemetry_path, ) inherits prometheus::params { # Prometheus added a 'v' on the realease name at 0.3.0 if versioncmp ($version, '0.3.0') == -1 { @@ -118,11 +118,6 @@ } $real_download_url = pick($download_url,"${download_url_base}/download/v${version}/${package_name}-${version}.${os}-${arch}.${download_extension}") - validate_bool($purge_config_dir) - validate_bool($manage_user) - validate_bool($manage_service) - validate_bool($restart_on_change) - validate_bool($consul_health_summary) if $consul_health_summary { $real_consul_health_summary = '-consul.health-summary' From 3ed8dee1b5be8ea6ebea427d969aaa71f70361c2 Mon Sep 17 00:00:00 2001 From: Pawel Fiuto Date: Thu, 16 Nov 2017 11:39:32 +0100 Subject: [PATCH 7/9] Add unit test for consul_exporter --- spec/classes/consul_exporter_spec.rb | 30 ++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) create mode 100644 spec/classes/consul_exporter_spec.rb diff --git a/spec/classes/consul_exporter_spec.rb b/spec/classes/consul_exporter_spec.rb new file mode 100644 index 000000000..1626dbd56 --- /dev/null +++ b/spec/classes/consul_exporter_spec.rb @@ -0,0 +1,30 @@ +require 'spec_helper' + +describe 'prometheus::consul_exporter' do + on_supported_os.each do |os, facts| + context "on #{os}" do + let(:facts) do + facts + end + + context 'with version specified' do + let(:params) do + { + version: '0.3.0', + arch: 'amd64', + os: 'linux', + } + end + + describe 'with all defaults' do + it { is_expected.to compile.with_all_deps } + it { is_expected.to contain_file('/usr/local/bin/consul_exporter').with('target' => '/opt/consul_exporter-0.3.0.linux-amd64/consul_exporter') } + it { is_expected.to contain_prometheus__daemon('consul_exporter') } + it { is_expected.to contain_user('consul-exporter') } + it { is_expected.to contain_group('consul-exporter') } + it { is_expected.to contain_service('consul_exporter') } + end + end + end + end +end From bf73030024e79a08f649aedc28f6bb91ce70c06e Mon Sep 17 00:00:00 2001 From: Pawel Fiuto Date: Thu, 16 Nov 2017 11:50:15 +0100 Subject: [PATCH 8/9] Remove trailing comma --- spec/classes/consul_exporter_spec.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spec/classes/consul_exporter_spec.rb b/spec/classes/consul_exporter_spec.rb index 1626dbd56..3c4a88e0c 100644 --- a/spec/classes/consul_exporter_spec.rb +++ b/spec/classes/consul_exporter_spec.rb @@ -12,7 +12,7 @@ { version: '0.3.0', arch: 'amd64', - os: 'linux', + os: 'linux' } end From 22a1f61cec70daaa8bc7ecd00190a8fbe176c9d8 Mon Sep 17 00:00:00 2001 From: Pawel Fiuto Date: Thu, 16 Nov 2017 12:59:35 +0100 Subject: [PATCH 9/9] Update README --- README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index c0657522b..804d98f5b 100644 --- a/README.md +++ b/README.md @@ -10,6 +10,7 @@ | >= 0.16.2 | latest | node_exporter >= 0.15.0 +consul_exporter >= 0.3.0 ## Background @@ -24,7 +25,7 @@ This module automates the install and configuration of Prometheus monitoring too * Installs a configuration file for prometheus daemon (/etc/prometheus/prometheus.yaml) or for alertmanager (/etc/prometheus/alert.rules) * Manages the services via upstart, sysv, or systemd * Optionally creates alert rules -* The following exporters are currently implemented: node_exporter, statsd_exporter, process_exporter, haproxy_exporter, mysqld_exporter, blackbox_exporter +* The following exporters are currently implemented: node_exporter, statsd_exporter, process_exporter, haproxy_exporter, mysqld_exporter, blackbox_exporter, consul_exporter ## Usage @@ -35,7 +36,7 @@ On the server (for prometheus version < 1.0.0): class { '::prometheus': global_config => { 'scrape_interval'=> '15s', 'evaluation_interval'=> '15s', 'external_labels'=> { 'monitor'=>'master'}}, rule_files => [ "/etc/prometheus/alert.rules" ], - scrape_configs => [ + scrape_configs => [ { 'job_name'=> 'prometheus', 'scrape_interval'=> '10s', 'scrape_timeout' => '10s', @@ -116,4 +117,3 @@ In version 0.1.14 of this module the alertmanager was configured to run as the s Do not use version 1.0.0 of Prometheus: https://groups.google.com/forum/#!topic/prometheus-developers/vuSIxxUDff8 ; it does break the compatibility with thus module! Even if the module has templates for several linux distributions, only RH family distributions were tested. -