Showing with 263 additions and 73 deletions.
  1. +23 −0 CHANGELOG.md
  2. +21 −0 manifests/activemq.pp
  3. +7 −30 manifests/init.pp
  4. +20 −0 manifests/orchestrator.pp
  5. +2 −0 manifests/pe_metric.pp
  6. +155 −0 manifests/puppetdb.pp
  7. +28 −0 manifests/puppetserver.pp
  8. +1 −1 metadata.json
  9. +1 −0 templates/activemq_metrics.epp
  10. +5 −42 templates/tk_metrics.epp
23 changes: 23 additions & 0 deletions CHANGELOG.md
View file
Original file line number Diff line number Diff line change
@@ -1,3 +1,26 @@
# Major Release 4.0.0

This is a major release because some of the PuppetDB metrics are renamed.
For most users this update is only additive, however, if you are post processing
the output of the module then you may need to update to the new names of the metrics.

## Changes
- Rename some PuppetDB metrics
- command_processing_time is now global_processing_time
- command_processed is now global_processed
- replace_catalog_time is now storage_replace-catalog-time
- replace_facts_time is now storage_replace-facts-time
- store_report_time is now storage_store-report-time
- *\_retry and *\_retry-counts metrics are renamed to include mq\_ at the front

## Improvements
- We now collect the output of the status endpoint for orchestrator
- We now collect HakariCP connection pooling metrics for PuppetDB
- We now collect the global metrics for PuppetDB
- We now collect the storage metrics for PuppetDB
- Each component now has its own class to allow customizing parameters per
component

# Z Release 3.0.1

## Bug Fixes:
Expand Down
21 changes: 21 additions & 0 deletions manifests/activemq.pp
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,21 @@
class pe_metric_curl_cron_jobs::activemq (
Integer $collection_frequency = $::pe_metric_curl_cron_jobs::collection_frequency,
Integer $retention_days = $::pe_metric_curl_cron_jobs::retention_days,
String $metrics_ensure = $::pe_metric_curl_cron_jobs::activemq_metrics_ensure,
Array[String] $hosts = $::pe_metric_curl_cron_jobs::activemq_hosts,
Integer $port = $::pe_metric_curl_cron_jobs::activemq_port,
) {
Pe_metric_curl_cron_jobs::Pe_metric {
output_dir => $::pe_metric_curl_cron_jobs::output_dir,
scripts_dir => $::pe_metric_curl_cron_jobs::scripts_dir,
cron_minute => "*/${collection_frequency}",
retention_days => $retention_days,
}

pe_metric_curl_cron_jobs::pe_metric { 'activemq' :
metric_ensure => $metrics_ensure,
hosts => $hosts,
metrics_port => $port,
metric_script_template => 'activemq_metrics.epp',
}
}
37 changes: 7 additions & 30 deletions manifests/init.pp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,9 @@
String $puppetdb_metrics_ensure = 'present',
Array[String] $puppetdb_hosts = [ '127.0.0.1' ],
Integer $puppetdb_port = 8081,
String $orchestrator_metrics_ensure = 'present',
Array[String] $orchestrator_hosts = [ '127.0.0.1' ],
Integer $orchestrator_port = 8143,
String $activemq_metrics_ensure = 'absent',
Array[String] $activemq_hosts = [ '127.0.0.1' ],
Integer $activemq_port = 8161,
Expand All @@ -22,39 +25,13 @@
ensure => directory,
}

Pe_metric_curl_cron_jobs::Pe_metric {
output_dir => $output_dir,
scripts_dir => $scripts_dir,
cron_minute => "*/${collection_frequency}",
retention_days => $retention_days,
}

pe_metric_curl_cron_jobs::pe_metric { 'puppetserver' :
metric_ensure => $puppetserver_metrics_ensure,
hosts => $puppetserver_hosts,
metrics_port => $puppetserver_port,
}
include pe_metric_curl_cron_jobs::puppetserver

pe_metric_curl_cron_jobs::pe_metric { 'puppetdb' :
metric_ensure => $puppetdb_metrics_ensure,
hosts => $puppetdb_hosts,
metrics_port => $puppetdb_port,
}
include pe_metric_curl_cron_jobs::puppetdb

pe_metric_curl_cron_jobs::pe_metric { 'activemq' :
metric_ensure => $activemq_metrics_ensure,
hosts => $activemq_hosts,
metrics_port => $activemq_port,
metric_script_template => 'activemq_metrics.epp',
}
include pe_metric_curl_cron_jobs::orchestrator

# DEPRECATION MECHANISMS
# Ensure remanants of cron jobs and files from older versions of this module
# are cleaned up.
pe_metric_curl_cron_jobs::pe_metric { 'puppet_server' :
metric_ensure => 'absent',
metrics_port => 8140,
}
include pe_metric_curl_cron_jobs::activemq

# Emit deprecation warnings if necessary
if ($puppet_server_metrics_ensure != undef) {
Expand Down
20 changes: 20 additions & 0 deletions manifests/orchestrator.pp
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,20 @@
class pe_metric_curl_cron_jobs::orchestrator (
Integer $collection_frequency = $::pe_metric_curl_cron_jobs::collection_frequency,
Integer $retention_days = $::pe_metric_curl_cron_jobs::retention_days,
String $metrics_ensure = $::pe_metric_curl_cron_jobs::orchestrator_metrics_ensure,
Array[String] $hosts = $::pe_metric_curl_cron_jobs::orchestrator_hosts,
Integer $port = $::pe_metric_curl_cron_jobs::orchestrator_port,
) {
Pe_metric_curl_cron_jobs::Pe_metric {
output_dir => $::pe_metric_curl_cron_jobs::output_dir,
scripts_dir => $::pe_metric_curl_cron_jobs::scripts_dir,
cron_minute => "*/${collection_frequency}",
retention_days => $retention_days,
}

pe_metric_curl_cron_jobs::pe_metric { 'orchestrator' :
metric_ensure => $metrics_ensure,
hosts => $hosts,
metrics_port => $port,
}
}
2 changes: 2 additions & 0 deletions manifests/pe_metric.pp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@
String $cron_minute = '*/5',
Integer $retention_days = 90,
String $metric_script_template = 'tk_metrics.epp',
Array[Hash] $additional_metrics = [],
) {

$metrics_output_dir = "${output_dir}/${metrics_type}"
Expand All @@ -29,6 +30,7 @@
'hosts' => $hosts,
'metrics_type' => $metrics_type,
'metrics_port' => $metrics_port,
'additional_metrics' => $additional_metrics,
}),
}

Expand Down
155 changes: 155 additions & 0 deletions manifests/puppetdb.pp
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,155 @@
class pe_metric_curl_cron_jobs::puppetdb (
Integer $collection_frequency = $::pe_metric_curl_cron_jobs::collection_frequency,
Integer $retention_days = $::pe_metric_curl_cron_jobs::retention_days,
String $metrics_ensure = $::pe_metric_curl_cron_jobs::puppetdb_metrics_ensure,
Array[String] $hosts = $::pe_metric_curl_cron_jobs::puppetdb_hosts,
Integer $port = $::pe_metric_curl_cron_jobs::puppetdb_port,
) {
Pe_metric_curl_cron_jobs::Pe_metric {
output_dir => $::pe_metric_curl_cron_jobs::output_dir,
scripts_dir => $::pe_metric_curl_cron_jobs::scripts_dir,
cron_minute => "*/${collection_frequency}",
retention_days => $retention_days,
}

$activemq_metrics = [
{ 'name' => 'amq_metrics',
'url' => 'org.apache.activemq:type=Broker,brokerName=localhost,destinationType=Queue,destinationName=puppetlabs.puppetdb.commands' }
]

$base_metrics = [
{ 'name' => 'global_command-parse-time',
'url' => 'puppetlabs.puppetdb.mq%3Aname%3Dglobal.command-parse-time' },
{ 'name' => 'global_discarded',
'url' => 'puppetlabs.puppetdb.mq%3Aname%3Dglobal.discarded' },
{ 'name' => 'global_fatal',
'url' => 'puppetlabs.puppetdb.mq%3Aname%3Dglobal.fatal' },
{ 'name' => 'global_generate-retry-message-time',
'url' => 'puppetlabs.puppetdb.mq%3Aname%3Dglobal.generate-retry-message-time' },
{ 'name' => 'global_message-persistence-time',
'url' => 'puppetlabs.puppetdb.mq%3Aname%3Dglobal.message-persistence-time' },
{ 'name' => 'global_retried',
'url' => 'puppetlabs.puppetdb.mq%3Aname%3Dglobal.retried' },
{ 'name' => 'global_retry-counts',
'url' => 'puppetlabs.puppetdb.mq%3Aname%3Dglobal.retry-counts' },
{ 'name' => 'global_retry-persistence-time',
'url' => 'puppetlabs.puppetdb.mq%3Aname%3Dglobal.retry-persistence-time' },
{ 'name' => 'global_seen',
'url' => 'puppetlabs.puppetdb.mq%3Aname%3Dglobal.seen' },
{ 'name' => 'global_processed',
'url' => 'puppetlabs.puppetdb.mq%3Aname%3Dglobal.processed' },
{ 'name' => 'global_processing-time',
'url' => 'puppetlabs.puppetdb.mq%3Aname%3Dglobal.processing-time' },
]

$storage_metrics = [
{ 'name' => 'storage_add-edges',
'url' => 'puppetlabs.puppetdb.storage%3Aname%3Dadd-edges' },
{ 'name' => 'storage_add-resources',
'url' => 'puppetlabs.puppetdb.storage%3Aname%3Dadd-resources' },
{ 'name' => 'storage_catalog-hash',
'url' => 'puppetlabs.puppetdb.storage%3Aname%3Dcatalog-hash' },
{ 'name' => 'storage_catalog-hash-match-time',
'url' => 'puppetlabs.puppetdb.storage%3Aname%3Dcatalog-hash-match-time' },
{ 'name' => 'storage_catalog-hash-miss-time',
'url' => 'puppetlabs.puppetdb.storage%3Aname%3Dcatalog-hash-miss-time' },
{ 'name' => 'storage_gc-catalogs-time',
'url' => 'puppetlabs.puppetdb.storage%3Aname%3Dgc-catalogs-time' },
{ 'name' => 'storage_gc-environments-time',
'url' => 'puppetlabs.puppetdb.storage%3Aname%3Dgc-environments-time' },
{ 'name' => 'storage_gc-fact-paths',
'url' => 'puppetlabs.puppetdb.storage%3Aname%3Dgc-fact-paths' },
{ 'name' => 'storage_gc-params-time',
'url' => 'puppetlabs.puppetdb.storage%3Aname%3Dgc-params-time' },
{ 'name' => 'storage_gc-report-statuses',
'url' => 'puppetlabs.puppetdb.storage%3Aname%3Dgc-report-statuses' },
{ 'name' => 'storage_gc-time',
'url' => 'puppetlabs.puppetdb.storage%3Aname%3Dgc-time' },
{ 'name' => 'storage_new-catalog-time',
'url' => 'puppetlabs.puppetdb.storage%3Aname%3Dnew-catalog-time' },
{ 'name' => 'storage_new-catalogs',
'url' => 'puppetlabs.puppetdb.storage%3Aname%3Dnew-catalogs' },
{ 'name' => 'storage_replace-catalog-time',
'url' => 'puppetlabs.puppetdb.storage%3Aname%3Dreplace-catalog-time' },
{ 'name' => 'storage_replace-facts-time',
'url' => 'puppetlabs.puppetdb.storage%3Aname%3Dreplace-facts-time' },
{ 'name' => 'storage_resource-hashes',
'url' => 'puppetlabs.puppetdb.storage%3Aname%3Dresource-hashes' },
{ 'name' => 'storage_store-report-time',
'url' => 'puppetlabs.puppetdb.storage%3Aname%3Dstore-report-time' },
]

#TODO: Track these on a less frequent cadence because they are slow to run
$storage_metrics_db_queries = [
{ 'name' => 'storage_catalog-volitilty',
'url' => 'puppetlabs.puppetdb.storage%3Aname%3Dcatalog-volitilty' },
{ 'name' => 'storage_duplicate-catalogs',
'url' => 'puppetlabs.puppetdb.storage%3Aname%3Dduplicate-catalogs' },
{ 'name' => 'storage_duplicate-pct',
'url' => 'puppetlabs.puppetdb.storage%3Aname%3Dduplicate-pct' },
]

$numbers = $::pe_server_version ? {
/^2015.2/ => {'catalogs' => 6, 'facts' => 4, 'reports' => 6},
/^2015.3/ => {'catalogs' => 7, 'facts' => 4, 'reports' => 6},
/^2016.(1|2)/ => {'catalogs' => 8, 'facts' => 4, 'reports' => 7},
/^2016.(4|5)/ => {'catalogs' => 9, 'facts' => 5, 'reports' => 8},
/^2017.(1|2)/ => {'catalogs' => 9, 'facts' => 5, 'reports' => 8},
default => {'catalogs' => 9, 'facts' => 5, 'reports' => 8},
}

$version_specific_metrics = [
{ 'name' => 'mq_replace_catalog_retried',
'url' => "puppetlabs.puppetdb.mq%3Aname%3Dreplace+catalog.${numbers['catalogs']}.retried" },
{ 'name' => 'mq_replace_catalog_retry-counts',
'url' => "puppetlabs.puppetdb.mq%3Aname%3Dreplace+catalog.${numbers['catalogs']}.retry-counts" },
{ 'name' => 'mq_replace_facts_retried',
'url' => "puppetlabs.puppetdb.mq%3Aname%3Dreplace+facts.${numbers['facts']}.retried" },
{ 'name' => 'mq_replace_facts_retry-counts',
'url' => "puppetlabs.puppetdb.mq%3Aname%3Dreplace+facts.${numbers['facts']}.retry-counts" },
{ 'name' => 'mq_store_report_retried',
'url' => "puppetlabs.puppetdb.mq%3Aname%3Dstore+report.${numbers['reports']}.retried" },
{ 'name' => 'mq_store_reports_retry-counts',
'url' => "puppetlabs.puppetdb.mq%3Aname%3Dstore+report.${numbers['reports']}.retry-counts" },
]

$connection_pool_metrics = [
{ 'name' => 'PDBReadPool_pool_ActiveConnections',
'url' => 'puppetlabs.puppetdb.database%3Aname%3DPDBReadPool.pool.ActiveConnections' },
{ 'name' => 'PDBReadPool_pool_IdleConnections',
'url' => 'puppetlabs.puppetdb.database%3Aname%3DPDBReadPool.pool.IdleConnections' },
{ 'name' => 'PDBReadPool_pool_PendingConnections',
'url' => 'puppetlabs.puppetdb.database%3Aname%3DPDBReadPool.pool.PendingConnections' },
{ 'name' => 'PDBReadPool_pool_TotalConnections',
'url' => 'puppetlabs.puppetdb.database%3Aname%3DPDBReadPool.pool.TotalConnections' },
{ 'name' => 'PDBReadPool_pool_Usage',
'url' => 'puppetlabs.puppetdb.database%3Aname%3DPDBReadPool.pool.Usage' },
{ 'name' => 'PDBReadPool_pool_Wait',
'url' => 'puppetlabs.puppetdb.database%3Aname%3DPDBReadPool.pool.Wait' },
{ 'name' => 'PDBWritePool_pool_ActiveConnections',
'url' => 'puppetlabs.puppetdb.database%3Aname%3DPDBWritePool.pool.ActiveConnections' },
{ 'name' => 'PDBWritePool_pool_IdleConnections',
'url' => 'puppetlabs.puppetdb.database%3Aname%3DPDBWritePool.pool.IdleConnections' },
{ 'name' => 'PDBWritePool_pool_PendingConnections',
'url' => 'puppetlabs.puppetdb.database%3Aname%3DPDBWritePool.pool.PendingConnections' },
{ 'name' => 'PDBWritePool_pool_TotalConnections',
'url' => 'puppetlabs.puppetdb.database%3Aname%3DPDBWritePool.pool.TotalConnections' },
{ 'name' => 'PDBWritePool_pool_Usage',
'url' => 'puppetlabs.puppetdb.database%3Aname%3DPDBWritePool.pool.Usage' },
{ 'name' => 'PDBWritePool_pool_Wait',
'url' => 'puppetlabs.puppetdb.database%3Aname%3DPDBWritePool.pool.Wait' },
]

$additional_metrics = $::pe_server_version ? {
/^2015./ => $activemq_metrics,
/^2016./ => $activemq_metrics + $base_metrics + $storage_metrics + $connection_pool_metrics + $version_specific_metrics,
default => $base_metrics + $storage_metrics + $connection_pool_metrics + $version_specific_metrics,
}

pe_metric_curl_cron_jobs::pe_metric { 'puppetdb' :
metric_ensure => $metrics_ensure,
hosts => $hosts,
metrics_port => $port,
additional_metrics => $additional_metrics,
}
}
28 changes: 28 additions & 0 deletions manifests/puppetserver.pp
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,28 @@
class pe_metric_curl_cron_jobs::puppetserver (
Integer $collection_frequency = $::pe_metric_curl_cron_jobs::collection_frequency,
Integer $retention_days = $::pe_metric_curl_cron_jobs::retention_days,
String $metrics_ensure = $::pe_metric_curl_cron_jobs::puppetserver_metrics_ensure,
Array[String] $hosts = $::pe_metric_curl_cron_jobs::puppetserver_hosts,
Integer $port = $::pe_metric_curl_cron_jobs::puppetserver_port,
) {
Pe_metric_curl_cron_jobs::Pe_metric {
output_dir => $::pe_metric_curl_cron_jobs::output_dir,
scripts_dir => $::pe_metric_curl_cron_jobs::scripts_dir,
cron_minute => "*/${collection_frequency}",
retention_days => $retention_days,
}

pe_metric_curl_cron_jobs::pe_metric { 'puppetserver' :
metric_ensure => $metrics_ensure,
hosts => $hosts,
metrics_port => $port,
}

# DEPRECATION MECHANISMS
# Ensure remanants of cron jobs and files from older versions of this module
# are cleaned up.
pe_metric_curl_cron_jobs::pe_metric { 'puppet_server' :
metric_ensure => 'absent',
metrics_port => 8140,
}
}
2 changes: 1 addition & 1 deletion metadata.json
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
{
"name": "npwalker/pe_metric_curl_cron_jobs",
"version": "3.0.1",
"version": "4.0.0",
"author": "npwalker",
"summary": "A Puppet module for gathering metrics from PE components",
"license": "Apache-2.0",
Expand Down
1 change: 1 addition & 0 deletions templates/activemq_metrics.epp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@
Array[String] $hosts,
String $metrics_type,
Integer $metrics_port,
Array[Hash] $additional_metrics,
| -%>
#!/opt/puppetlabs/puppet/bin/ruby

Expand Down
Loading