feature: Added Prometheus PushGateway metric support (librenms#8437)

* Added Prometheus PushGateway support

* fixed formatting

* fix set_curl_proxy to correct variable

* add documentation

* Update Metric-Storage.md

* Update mkdocs.yml

check-services.php

@@ -36,6 +36,10 @@
     $config['noinfluxdb'] = true;
 }
 
+if (isset($options['p'])) {
+    $prometheus = false;
+}
+
 if ($config['noinfluxdb'] !== true && $config['influxdb']['enable'] === true) {
     $influxdb = influxdb_connect();
 } else {

doc/Extensions/Metric-Storage.md

@@ -3,12 +3,13 @@ hide_toc: true
 
 ### Metric storage
 By default we ship all metrics to RRD files, either directly or via [RRDCached](RRDCached.md). On top of this 
-you can ship metrics to InfluxDB, Graphite and / or OpenTSDB. At present you can't use these backends to display 
-graphs within LibreNMS and will need to use something like [Grafana](https://grafana.com/).
+you can ship metrics to Graphite, InfluxDB, OpenTSDB or Prometheus. At present you can't use these 
+backends to display graphs within LibreNMS and will need to use something like [Grafana](https://grafana.com/).
 
 For further information on configuring LibreNMS to ship data to one of the other backends then please see 
 the documentation below.
 
->  - [InfluxDB](metrics/InfluxDB.md)
 >  - [Graphite](metrics/Graphite.md)
+>  - [InfluxDB](metrics/InfluxDB.md)
 >  - [OpenTSDB](metrics/OpenTSDB.md)
+>  - [Prometheus](metrics/Prometheus.md)

doc/Extensions/Prometheus.md

@@ -0,0 +1 @@
+<meta http-equiv="refresh" content="0; url=/Extensions/metrics/Prometheus/" />

doc/Extensions/metrics/Prometheus.md

@@ -0,0 +1,41 @@
+source: Extensions/metrics/Prometheus.md
+# Enabling support for Prometheus.
+
+Please be aware Prometheus support is alpha at best, It hasn't been extensively tested and is still in development
+All it provides is the sending of data to a a Prometheus PushGateway. Please be careful when enabling this support 
+you use it at your own risk!
+
+### Requirements (Older versions may work but haven't been tested
+ - Prometheus >= 2.0
+ - PushGateway >= 0.4.0
+ - Grafana
+ - PHP-CURL
+
+The setup of the above is completely out of scope here and we aren't really able to provide any help with this side
+of things.
+
+### What you don't get
+ - Pretty graphs, this is why at present you need Grafana. You need to build your own graphs within Grafana.
+ - Support for Prometheus or Grafana, we would highly recommend that you have some level of experience with these.
+
+RRD will continue to function as normal so LibreNMS itself should continue to function as normal.
+
+### Configuration
+```php
+$config['prometheus']['enable'] = true;
+$config['prometheus']['url'] = 'http://127.0.0.1:9091';
+$config['prometheus']['job'] = 'librenms'; # Optional
+```
+
+
+### Sample Prometheus Scrape Config (for scraping the Push Gateway)
+```yml
+- job_name: pushgateway
+  scrape_interval: 300s
+  honor_labels: true
+  static_configs:
+    - targets: ['127.0.0.1:9091']
+```
+
+The same data then stored within rrd will be sent to Prometheus and recorded. You can then create graphs within Grafana
+to display the information you need.

includes/datastore.inc.php

@@ -66,6 +66,7 @@ function data_update($device, $measurement, $tags, $fields)
     // However, influxdb saves all tags, so we filter out the ones beginning with 'rrd_'.
 
     rrdtool_data_update($device, $measurement, $tags, $fields);
+    prometheus_push($device, $measurement, rrd_array_filter($tags), $fields);
     influx_update($device, $measurement, rrd_array_filter($tags), $fields);
     opentsdb_update($device, $measurement, rrd_array_filter($tags), $fields);
     graphite_update($device, $measurement, $tags, $fields);

includes/defaults.inc.php

@@ -933,6 +933,9 @@
 $config['influxdb']['timeout']      = 0;
 $config['influxdb']['verifySSL']    = false;
 
+// Prometheus Push Default configuration
+$config['prometheus']['job']        = 'librenms';
+
 // Xirrus - Disable station/client polling if true as it may take a long time on larger/heavily used APs.
 $config['xirrus_disable_stations']  = false;
 

includes/init.php

@@ -56,6 +56,7 @@ function module_selected($module, $modules)
 require_once $install_dir . '/includes/dbFacile.php';
 require_once $install_dir . '/includes/rrdtool.inc.php';
 require_once $install_dir . '/includes/influxdb.inc.php';
+require_once $install_dir . '/includes/prometheus.inc.php';
 require_once $install_dir . '/includes/opentsdb.inc.php';
 require_once $install_dir . '/includes/graphite.inc.php';
 require_once $install_dir . '/includes/datastore.inc.php';

includes/polling/ports.inc.php

@@ -771,6 +771,7 @@
         $fields['ifInOctets_rate'] = $port['ifInOctets_rate'];
         $fields['ifOutOctets_rate'] = $port['ifOutOctets_rate'];
 
+        prometheus_push($device, 'ports', rrd_array_filter($tags), $fields);
         influx_update($device, 'ports', rrd_array_filter($tags), $fields);
         graphite_update($device, 'ports|' . $ifName, $tags, $fields);
         opentsdb_update($device, 'port', array('ifName' => $this_port['ifName'], 'ifIndex' => getPortRrdName($port_id)), $fields);

includes/prometheus.inc.php

@@ -0,0 +1,73 @@
+<?php
+
+/*
+ * LibreNMS
+ *
+ * Copyright (c) 2014 Neil Lathwood <https://github.com/laf/ http://www.lathwood.co.uk/fa>
+ *
+ * This program is free software: you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License as published by the
+ * Free Software Foundation, either version 3 of the License, or (at your
+ * option) any later version.  Please see LICENSE.txt at the top level of
+ * the source code distribution for details.
+ */
+
+function prometheus_push($device, $measurement, $tags, $fields)
+{
+    global $prometheus, $config;
+    if ($config['prometheus']['enable'] === true) {
+        if ($prometheus !== false) {
+            try {
+                $ch = curl_init();
+
+                set_curl_proxy($ch);
+                $vals = "";
+                $promtags = "/measurement/".$measurement;
+
+                foreach ($fields as $k => $v) {
+                    if ($v !== null) {
+                        $vals = $vals . "$k $v\n";
+                    }
+                }
+
+                foreach ($tags as $t => $v) {
+                    if ($v !== null) {
+                        $promtags = $promtags . "/$t/$v";
+                    }
+                }
+
+                $promurl = $config['prometheus']['url'].'/metrics/job/'.$config['prometheus']['job'].'/instance/'.$device['hostname'].$promtags;
+                $promurl = str_replace(" ", "-", $promurl); // Prometheus doesn't handle tags with spaces in url
+
+                d_echo("\nPrometheus data:\n");
+                d_echo($measurement);
+                d_echo($tags);
+                d_echo($fields);
+                d_echo($vals);
+                d_echo($promurl);
+                d_echo("\nEND\n");
+
+                curl_setopt($ch, CURLOPT_URL, $promurl);
+                curl_setopt($ch, CURLOPT_POST, 1);
+                curl_setopt($ch, CURLOPT_POSTFIELDS, $vals);
+                curl_setopt($ch, CURLOPT_RETURNTRANSFER, 1);
+
+                $headers = array();
+                $headers[] = "Content-Type: test/plain";
+                curl_setopt($ch, CURLOPT_HTTPHEADER, $headers);
+
+
+                curl_exec($ch);
+
+                if (curl_errno($ch)) {
+                    d_echo('Error:' . curl_error($ch));
+                }
+            } catch (Exception $e) {
+                d_echo("Caught exception: " . $e->getMessage() . PHP_EOL);
+                d_echo($e->getTrace());
+            }
+        } else {
+            c_echo("[%gPrometheus Push Disabled%n]\n");
+        }//end if
+    }//end if
+}// end prometheus_push

mkdocs.yml

@@ -78,9 +78,10 @@ pages:
       - Extensions/Proxmox.md
       - Storing Metrics:
           - Intro: Extensions/Metric-Storage.md
-          - Extensions/metrics/InfluxDB.md
           - Extensions/metrics/Graphite.md
+          - Extensions/metrics/InfluxDB.md
           - Extensions/metrics/OpenTSDB.md
+          - Extensions/metrics/Prometheus.md
       - Extensions/Smokeping.md
       - Extensions/Weathermap.md
   - 7. API:

poller.php

@@ -66,6 +66,7 @@
     echo "Debugging and testing options:\n";
     echo "-r                                           Do not create or update RRDs\n";
     echo "-f                                           Do not insert data into InfluxDB\n";
+    echo "-p                                           Do not insert data into Prometheus\n";
     echo "-d                                           Enable debugging output\n";
     echo "-v                                           Enable verbose debugging output\n";
     echo "-m                                           Specify module(s) to be run\n";
@@ -115,6 +116,10 @@
     $config['noinfluxdb'] = true;
 }
 
+if (isset($options['p'])) {
+    $prometheus = false;
+}
+
 if (isset($options['g'])) {
     $config['nographite'] = true;
 }