add dir python plugin for prometheus metrics

contrib/dir-plugins/prometheus/BareosDirPluginPrometheusExporter.py

@@ -0,0 +1,215 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+#
+# This program is Free Software; you can redistribute it and/or
+# modify it under the terms of version three of the GNU Affero General Public
+# License as published by the Free Software Foundation, which is
+# listed in the file LICENSE.
+#
+# This program is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+# Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+# 02110-1301, USA.
+#
+# Author: Benedikt Braunger
+#
+# Prometheus Exporter as Bareos python plugin
+# Functions taken and adapted from BareosDirPluginGraphiteSender.py
+
+from bareosdir import *
+import bareosdir
+import BareosDirPluginBaseclass
+from prometheus_client import CollectorRegistry, Gauge, Enum, Histogram, Info, push_to_gateway
+from prometheus_client.exposition import basic_auth_handler
+import time
+
+
+class BareosDirPluginPrometheusExporter(BareosDirPluginBaseclass.BareosDirPluginBaseclass):
+    ''' Bareos Python plugin to send data about finished backups to a Prometheus Pushgateway '''
+
+    # maybe this can go to consts
+    job_levels = {
+            'F': 'Full',
+            'I': 'Incremental',
+            'D': 'Differential',
+            'S': 'Unknown',
+            'C': 'Verify catalog',
+            'V': 'Verify init',
+            'O': 'Verfiy volume to catalog',
+            'd': 'Verify disk to catalog',
+            'A': 'Verify Data',
+            'B': 'Base',
+            ' ': 'None',
+            'f': 'Virtual full'}
+    job_status = {
+            'A': 'Canceled',
+            'B': 'Blocked',
+            'C': 'Created',
+            'D': 'Differences',
+            'E': 'Error',
+            'F': 'Wait FD',
+            'I': 'Incomplete',
+            'L': 'Commiting data',
+            'M': 'Wait mount',
+            'R': 'Running',
+            'S': 'Wait SD',
+            'T': 'Terminated',
+            'W': 'Warning',
+            'a': 'SD despooling',
+            'c': 'Wait FD resource',
+            'd': 'Wait max jobs',
+            'e': 'Non-fatal error',
+            'f': 'Fatal error',
+            'i': 'Attr insertion',
+            'j': 'Wait job resource',
+            'l': 'Despooling',
+            'm': 'Wait media',
+            'p': 'Wait priority',
+            'q': 'Wait device',
+            's': 'Wait SD resource',
+            't': 'Wait start time'}
+
+    job_types = {
+            'B': 'Backup',
+            'M': 'Backup migrated',
+            'V': 'Verify',
+            'R': 'Restore',
+            'U': 'Console',
+            'I': 'Internal',
+            'D': 'Admin',
+            'A': 'Archive',
+            'C': 'Copy of a job',
+            'c': 'Copy job',
+            'g': 'Migration',
+            'S': 'Scan',
+            'O': 'Consolidate'}
+
+    def authentication_handler(self, url, method, timeout, headers, data):
+        '''This function handles the basic auth credtials for prometheus-client'''
+        return basic_auth_handler(url, method, timeout, headers, data, self.username, self.password)
+
+    def parse_plugin_definition(self, plugindef):
+        '''
+        Check, if mandatory gateway is set and set default for other unset parameters
+        '''
+        super(BareosDirPluginPrometheusExporter, self).parse_plugin_definition(
+            plugindef)
+        # monitoring Host is mandatory
+        if 'gateway_host' not in self.options:
+            self.gateway_host = "localhost"
+        else:
+            self.gateway_host = self.options['gateway_host']
+
+        if 'gateway_port' not in self.options:
+            self.gateway_port = 9091
+        else:
+            self.gateway_port = int(self.options['gateway_port'])
+
+        if 'username' in self.options and 'password' in self.options:
+            self.username = self.options['username']
+            self.password = self.options['password']
+            self.use_basic_auth = True
+            DebugMessage(100, "Using Basic auth with username={}\n".format(self.username))
+        else:
+            self.use_basic_auth = False
+            DebugMessage(100, "Username/password missing, disabling basic auth\n")
+
+        if 'use_tls' not in self.options:
+            self.use_tls = False
+        else:
+            self.use_tls = self.options['use_tls']
+
+        if 'report_failed' not in self.options:
+            self.report_failed = False
+        else:
+            self.report_failed = bool(self.options['report_failed'])
+
+        # we return OK in anyway, we do not want to produce Bareos errors just because of 
+        # a failing metric exporter
+        return bRCs['bRC_OK']
+
+    def handle_plugin_event(self, event):
+        '''
+        This method is called for every registered event
+        '''
+
+        # We first call the method from our superclass to get job attributes read
+        super(BareosDirPluginPrometheusExporter, self).handle_plugin_event(event)
+
+        if event == bDirEventJobEnd:
+            # This event is called at the end of a job, here evaluate the results
+            self.push_job_information()
+
+        return bRCs['bRC_OK']
+
+    def push_job_information(self):
+        '''
+        Process Bareos job data and send it to the prometheus pushgateway
+        '''
+        registry = CollectorRegistry()
+
+        TIME_BUCKETS=(6, 60, 600, 1800, 3600, 10800, 18000, 28800, 86400)
+
+        bareos_job_status = Enum('bareos_job_status', 'Backup Status',
+                                 states=self.job_status.values(),
+                                 labelnames=['instance', 'jobid'], registry=registry)
+        # see https://github.com/bareos/bareos/blob/master/core/src/include/job_level.h
+        bareos_job_level = Enum('bareos_job_level', 'Backup Level',
+                                states=self.job_levels.values(),
+                                labelnames=['instance', 'jobid'], registry=registry)
+        bareos_job_running_time = Histogram('bareos_job_running_time', 'Job running time',
+                                            labelnames=['instance', 'jobid'], registry=registry,
+                                            buckets=TIME_BUCKETS)
+        bareos_job_files = Gauge('bareos_job_files', 'Backed up files', 
+                                 labelnames=['instance', 'jobid'], registry=registry)
+        bareos_job_bytes = Gauge('bareos_job_bytes', 'Backed up bytes',
+                                 labelnames=['instance', 'jobid'], registry=registry)
+        bareos_job_throughput = Gauge('bareos_job_throughtput', 'Backup throughtput',
+                                      registry=registry, labelnames=['instance', 'jobid'])
+        # see https://github.com/bareos/bareos/blob/master/core/src/include/job_types.h
+        bareos_job_type = Enum('bareos_job_type', 'Job Type',
+                               states=self.job_types.values(),
+                               registry=registry, labelnames=['instance', 'jobid'])
+        bareos_job_client = Info('bareos_job_client', 'Client',
+                               registry=registry, labelnames=['instance', 'jobid'])
+        bareos_job_priority = Gauge('bareos_job_priority', 'Job Priority',
+                               registry=registry, labelnames=['instance', 'jobid'])
+
+        bareos_job_name = '_'.join(self.jobName.split('.')[:-3])
+        bareos_job_id = self.jobId
+
+        if (self.jobStatus == 'E' or self.jobStatus == 'f' or self.jobStatus == 'A') and self.report_failed == False:
+            return
+
+        bareos_job_status.labels(instance=bareos_job_name, jobid=bareos_job_id).state(self.job_status[self.jobStatus])
+        bareos_job_running_time.labels(instance=bareos_job_name, jobid=bareos_job_id).observe(self.jobRunningTime)
+        bareos_job_files.labels(instance=bareos_job_name, jobid=bareos_job_id).set(self.jobFiles)
+        bareos_job_bytes.labels(instance=bareos_job_name, jobid=bareos_job_id).set(self.jobBytes)
+        bareos_job_throughput.labels(instance=bareos_job_name, jobid=bareos_job_id).set(self.throughput)
+        bareos_job_priority.labels(instance=bareos_job_name, jobid=bareos_job_id).set(self.Priority)
+        bareos_job_level.labels(instance=bareos_job_name, jobid=bareos_job_id).state(self.job_levels[self.jobLevel])
+        bareos_job_type.labels(instance=bareos_job_name, jobid=bareos_job_id).state(self.job_types[chr(self.jobType)])
+        bareos_job_client.labels(instance=bareos_job_name, jobid=bareos_job_id).info({'client': self.jobClient})
+
+        if self.use_tls == True or self.use_tls == 'yes':
+            gateway = "https://{}:{}".format(self.gateway_host,self.gateway_port)
+        else:
+            gateway = "{}:{}".format(self.gateway_host,self.gateway_port)
+
+        DebugMessage(100, "Submitting metrics to {}\n".format(gateway))
+        try:
+          if self.use_basic_auth:
+            push_to_gateway('{}'.format(gateway), job='bareos', registry=registry, handler=self.authentication_handler)
+          else:
+              push_to_gateway('{}'.format(gateway), job='bareos', registry=registry)
+        except Exception as excp:
+          DebugMessage(100, "Error: Submitting metrics to pushgateway '{}' failed.\n".format(gateway))
+          DebugMessage(100, "python error was: {}\n".format(excp))
+          JobMessage(M_INFO, "Failed to submit metrics to pushgateway\n")
+
+# vim: ts=4 tabstop=4 expandtab shiftwidth=4 softtabstop=4

contrib/dir-plugins/prometheus/README.md

@@ -0,0 +1,58 @@
+This plugin implements a exporter for Bareos jobs and pushs it to a Prometheus Pushgateway
+
+## Design decision and Concept
+The idea behind this plugin is to submit metrics about every finished job to a prometheus server to have the data in a timeseries database.
+For big setups it is good practice not to crunch big SQL statements on the production catalog but get these information from an independent  
+time series database.
+
+Since the backup metrics are available for the plugin after after a job is finished it will then be sent to a Prometheus Pushgateway instantly
+from which Prometheus can scrap it anytime. Therefore no additional interaction with the Bareos director or catalog database is nessesary.
+
+An alternative design would be to write these values to a textfile and use the textfile collector of the [node_exporter](https://github.com/prometheus/node_exporter).
+
+## Dependencies
+
+* You need a prometheus push gateway
+  https://github.com/prometheus/pushgateway
+* and the Python library to talk to it
+  https://github.com/prometheus/client_python
+* Of course the Bareos Python library is needed
+  https://github.com/bareos/bareos/tree/master/python-bareos/
+
+See requirements.txt for python requirements.
+
+## Usage
+
+In bareos-dir.conf enable director plugins and load the Python plugin:
+
+    Director {
+      Plugin Directory = /usr/lib/bareos/plugins
+      Plugin Names = "python3"
+    }
+
+In your JobDefs or Job Definition configure the plugin itself:
+
+    Job {
+      Name = "BackupClient1"
+      DIR Plugin Options ="python3:module_path=/usr/lib/bareos/plugins:module_name=bareos-dir-prometheus-exporter:gateway_host=pushgateway.domain.tld:gateway_port=443:use_tls=yes:username=monitoring:password=foobar"
+      JobDefs = "DefaultJob"
+    }
+
+## Available parameters
+* `gateway_host` (default=localhost): Where the Prometheus pushgateway is reachable
+* `gateway_port` (default=9091): TCP port on which the Prometheus pushgateway is reachable
+* `username` & `password`: For HTTP Basic Authentication. Both or none must be set to enable/disable the usage of BasicAuth
+* `use_tls` (default=false): Whether TLS encryption should be used to communication with the pushgateway
+* `report_failed` (default=false): Whether failed jobs should be reported or not
+
+## exported Prometheus metrics
+
+* bareos_job_status
+* bareos_job_running_time
+* bareos_job_files
+* bareos_job_bytes
+* bareos_job_throughput
+* bareos_job_priority
+* bareos_job_level
+* bareos_job_type
+* bareos_job_client

contrib/dir-plugins/prometheus/bareos-dir-prometheus-exporter.py

@@ -0,0 +1,47 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+#
+# This program is Free Software; you can redistribute it and/or
+# modify it under the terms of version three of the GNU Affero General Public
+# License as published by the Free Software Foundation, which is
+# listed in the file LICENSE.
+#
+# This program is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+# Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+# 02110-1301, USA.
+#
+# Author: Benedikt Braunger
+#
+
+# Provided by the Bareos Dir Python plugin interface
+import bareosdir
+from bareosdir import *
+
+# This module contains the wrapper functions called by the Bareos-Dir, the
+# functions call the corresponding methods from your plugin class
+import BareosDirWrapper
+from BareosDirWrapper import *
+
+# This module contains the used plugin class
+import BareosDirPluginPrometheusExporter
+
+
+def load_bareos_plugin(plugindef):
+    '''
+    This function is called by the Bareos-Dir to load the plugin
+    We use it to instantiate the plugin class
+    '''
+    # BareosDirWrapper.bareos_dir_plugin_object is the module attribute that
+    # holds the plugin class object
+    BareosDirWrapper.bareos_dir_plugin_object = \
+        BareosDirPluginPrometheusExporter.BareosDirPluginPrometheusExporter(
+            plugindef)
+    return bRC_OK
+
+# the rest is done in the Plugin module

contrib/dir-plugins/prometheus/requirements.txt

@@ -0,0 +1,2 @@
+prometheus_client
+python-bareos