Add metrics json

packages/debian/control

@@ -12,6 +12,7 @@ Package: sanoid
 Architecture: all
 Depends: libcapture-tiny-perl,
          libconfig-inifiles-perl,
+         libjson-perl,
          zfsutils-linux | zfs,
          ${misc:Depends},
          ${perl:Depends}

sanoid

@@ -13,6 +13,7 @@ use Config::IniFiles; # read samba-style conf file
 use Data::Dumper;     # debugging - print contents of hash
 use File::Path 'make_path';
 use Getopt::Long qw(:config auto_version auto_help);
+use JSON;			  # for monitor-metrics-json
 use Pod::Usage;       # pod2usage
 use Time::Local;      # to parse dates in reverse
 use Capture::Tiny ':all';
@@ -25,8 +26,8 @@ my %args = (
 GetOptions(\%args, "verbose", "debug", "cron", "readonly", "quiet",
                    "configdir=s", "cache-dir=s", "run-dir=s",
                    "monitor-health", "force-update",
-                   "monitor-snapshots", "take-snapshots", "prune-snapshots", "force-prune",
-                   "monitor-capacity"
+                   "monitor-snapshots", "monitor-metrics-json", "take-snapshots", "prune-snapshots", 
+				   "force-prune", "monitor-capacity"
           ) or pod2usage(2);
 
 # If only config directory (or nothing) has been specified, default to --cron --verbose
@@ -74,6 +75,7 @@ if ($args{'debug'})		{ $args{'verbose'}=1; blabber (@params); }
 if ($args{'monitor-snapshots'}) { monitor_snapshots(@params); }
 if ($args{'monitor-health'}) 	{ monitor_health(@params); }
 if ($args{'monitor-capacity'})  { monitor_capacity(@params); }
+if ($args{'monitor-metrics-json'})  { monitor_metrics_json(@params); }
 if ($args{'force-update'}) 	{ my $snaps = getsnaps( \%config, $cacheTTL, 1 ); }
 
 if ($args{'cron'}) {
@@ -128,12 +130,20 @@ sub monitor_snapshots {
 	# check_snapshot_date - test ZFS fs creation timestamp for recentness
 	# accepts arguments: $filesystem, $warn (in seconds elapsed), $crit (in seconds elapsed)
 
-	my ($config, $snaps, $snapsbytype, $snapsbypath) = @_;
+	my ($config, $snaps, $snapsbytype, $snapsbypath, $return_info) = @_;
 	my %datestamp = get_date();
 	my $errorlevel = 0;
 	my $msg;
 	my @msgs;
 	my @paths;
+	my %snapshot_info;
+
+	# use Data::Dumper;
+	# print Dumper $config;
+	# print Dumper $snaps;
+	# print Dumper $snapsbytype;
+	# print Dumper $snapsbypath;
+	# print encode_json $snapsbytype;
 
 	foreach my $section (keys %config) {
 		if ($section =~ /^template/) { next; }
@@ -161,41 +171,81 @@ sub monitor_snapshots {
 			my $warn = convertTimePeriod($config{$section}{$typewarn}, $smallerperiod);
 			my $crit = convertTimePeriod($config{$section}{$typecrit}, $smallerperiod);
 			my $elapsed = -1;
+
+			# $errorlevel tracks the overall error level for all snapshots, 
+			# snapshot_health_issues is specific to this path/type combination
+			# 0 = no issues, 1 = warn, 2 = critical
+			my $snapshot_health_issues = 0; 
+
 			if (defined $snapsbytype{$path}{$type}{'newest'}) {
 				$elapsed = $snapsbytype{$path}{$type}{'newest'};
 			}
+
 			my $dispelapsed = displaytime($elapsed);
 			my $dispwarn = displaytime($warn);
 			my $dispcrit = displaytime($crit);
 			if ( $elapsed > $crit || $elapsed == -1) {
 				if ($crit > 0) {
-					if (! $config{$section}{'monitor_dont_crit'}) { $errorlevel = 2; }
+					if (! $config{$section}{'monitor_dont_crit'}) { $snapshot_health_issues = 2; }
 					if ($elapsed == -1) {
 						push @msgs, "CRIT: $path has no $type snapshots at all!";
 					} else {
 						push @msgs, "CRIT: $path newest $type snapshot is $dispelapsed old (should be < $dispcrit)";
 					}
 				}
-			 } elsif ($elapsed > $warn) {
+			} elsif ($elapsed > $warn) {
 				if ($warn > 0) {
-					if (! $config{$section}{'monitor_dont_warn'} && ($errorlevel < 2) ) { $errorlevel = 1; }
+					if (! $config{$section}{'monitor_dont_warn'} && ($snapshot_health_issues < 2) ) { $snapshot_health_issues = 1; }
 					push @msgs, "WARN: $path newest $type snapshot is $dispelapsed old (should be < $dispwarn)";
 				}
 			} else {
 				# push @msgs .= "OK: $path newest $type snapshot is $dispelapsed old \n";
 			}
 
+			if (defined $return_info) {
+				# $return_info has been defined, so export JSON instead of printing
+				$snapshot_info{$path}{$type}{"crit_age_seconds"} = $crit;
+				$snapshot_info{$path}{$type}{"warn_age_seconds"} = $warn;
+				$snapshot_info{$path}{$type}{"monitor_dont_crit"} = int($config{$section}{'monitor_dont_crit'});
+				$snapshot_info{$path}{$type}{"monitor_dont_warn"} = int($config{$section}{'monitor_dont_warn'});
+				$snapshot_info{$path}{$type}{"snapshot_health_issues"} = $snapshot_health_issues;
+
+				if ($elapsed == -1) {
+					# The $path has no $type snapshots
+					$snapshot_info{$path}{$type}{"has_snapshots"} = 0;
+				} else {
+					$snapshot_info{$path}{$type}{"has_snapshots"} = 1;
+					$snapshot_info{$path}{$type}{"newest_age_seconds"} = $elapsed;
+					my $most_recent_snap_of_type_name = $snapsbytype{$path}{$type}{"newestname"};
+					$snapshot_info{$path}{$type}{"newest_snapshot_ctime_seconds"} = int($snaps{$path}{$most_recent_snap_of_type_name}{"ctime"});
+				}
+			}
+
+			if ($snapshot_health_issues > $errorlevel){
+				# This path/type combination a warning or crit level higher than any we have seen so far, 
+				# so adjust the overall error level to match
+				$errorlevel = $snapshot_health_issues;
+			}
+
 		}
 	}
 
-	my @sorted_msgs = sort { lc($a) cmp lc($b) } @msgs;
-	my @sorted_paths = sort { lc($a) cmp lc($b) } @paths;
-	$msg = join (", ", @sorted_msgs);
-	my $paths = join (", ", @sorted_paths);
+	if (defined ($return_info)) {
+		# Called by monitor_metrics_json, so return the JSON
 
-	if ($msg eq '') { $msg = "OK: all monitored datasets \($paths\) have fresh snapshots"; }
+		return \%snapshot_info, $errorlevel;
 
-	print "$msg\n";
+	} else {
+		# A normal run of the subroutine, so we print the information
+		my @sorted_msgs = sort { lc($a) cmp lc($b) } @msgs;
+		my @sorted_paths = sort { lc($a) cmp lc($b) } @paths;
+		$msg = join (", ", @sorted_msgs);
+		my $paths = join (", ", @sorted_paths);
+
+		if ($msg eq '') { $msg = "OK: all monitored datasets \($paths\) have fresh snapshots"; }
+
+		print "$msg\n";
+	}
 	exit $errorlevel;
 }
 
@@ -258,6 +308,30 @@ sub monitor_capacity {
 ####################################################################################
 ####################################################################################
 
+sub monitor_metrics_json {
+	my ($config, $snaps, $snapsbytype, $snapsbypath) = @_;
+	my %metrics;
+
+	# Set a schema version for the JSON each time it changes in case any changes are
+	# not backwards compatible. Today's date backawards followed by an incrementing 
+	# digit (YYYYMMDDX)
+	$metrics{"schema_version"} = 202204041;
+
+	# Add all the information about snapshots we need that would be returned from
+	# --monitor-snapshots (or that let people derive the same information)
+	my ($snapshot_info, $overall_snapshot_health) = monitor_snapshots($config, $snaps, $snapsbytype, $snapsbypath, 1);
+	$metrics{"snapshot_info"} = $snapshot_info;
+	$metrics{"overall_snapshot_health_issues"} = $overall_snapshot_health;
+
+	# my $snapshot_info_json = encode_json $snapshot_info;
+	my $metrics_json = encode_json \%metrics;
+
+	print "$metrics_json\n";
+}
+
+####################################################################################
+####################################################################################
+####################################################################################
 
 sub prune_snapshots {
 
@@ -1701,23 +1775,24 @@ Assumes --cron --verbose if no other arguments (other than configdir) are specif
 
 Options:
 
-  --configdir=DIR       Specify a directory to find config file sanoid.conf
-  --cache-dir=DIR       Specify a directory to store the zfs snapshot cache
-  --run-dir=DIR         Specify a directory for temporary files such as lock files
-
-  --cron                Creates snapshots and purges expired snapshots
-  --verbose             Prints out additional information during a sanoid run
-  --readonly            Simulates creation/deletion of snapshots
-  --quiet               Suppresses non-error output
-  --force-update        Clears out sanoid's zfs snapshot cache
-
-  --monitor-health      Reports on zpool "health", in a Nagios compatible format
-  --monitor-capacity    Reports on zpool capacity, in a Nagios compatible format
-  --monitor-snapshots   Reports on snapshot "health", in a Nagios compatible format
-  --take-snapshots      Creates snapshots as specified in sanoid.conf
-  --prune-snapshots     Purges expired snapshots as specified in sanoid.conf
-  --force-prune         Purges expired snapshots even if a send/recv is in progress
-
-  --help                Prints this helptext
-  --version             Prints the version number
-  --debug               Prints out a lot of additional information during a sanoid run
+  --configdir=DIR        Specify a directory to find config file sanoid.conf
+  --cache-dir=DIR        Specify a directory to store the zfs snapshot cache
+  --run-dir=DIR          Specify a directory for temporary files such as lock files
+
+  --cron                 Creates snapshots and purges expired snapshots
+  --verbose              Prints out additional information during a sanoid run
+  --readonly             Simulates creation/deletion of snapshots
+  --quiet                Suppresses non-error output
+  --force-update         Clears out sanoid's zfs snapshot cache
+
+  --monitor-health       Reports on zpool "health", in a Nagios compatible format
+  --monitor-capacity     Reports on zpool capacity, in a Nagios compatible format
+  --monitor-metrics-json Reports zpool and snapshot metrics in JSON format
+  --monitor-snapshots    Reports on snapshot "health", in a Nagios compatible format
+  --take-snapshots       Creates snapshots as specified in sanoid.conf
+  --prune-snapshots      Purges expired snapshots as specified in sanoid.conf
+  --force-prune          Purges expired snapshots even if a send/recv is in progress
+
+  --help                 Prints this helptext
+  --version              Prints the version number
+  --debug                Prints out a lot of additional information during a sanoid run

tests/3_monitor_snapshots/run.sh

@@ -0,0 +1,20 @@
+#!/bin/bash
+set -x
+
+# this test will create pools in a number of states
+# and check the output text and return code of 
+# sanoid --monitor-snapshots
+# and the JSON data created by
+# sanoid --monitor-metrics-json
+
+. ../common/lib.sh
+
+# prepare
+setup
+checkEnvironment
+disableTimeSync
+
+# set timezone
+ln -sf /usr/share/zoneinfo/Europe/Vienna /etc/localtime
+
+python3 test_monitoring.py

tests/3_monitor_snapshots/sanoid.conf

@@ -0,0 +1,37 @@
+[sanoid-test-1]
+	use_template = production
+
+[sanoid-test-2]
+	use_template = demo
+
+[template_production]
+	hourly = 36
+	daily = 30
+	monthly = 3
+	yearly = 0
+	autosnap = yes
+	autoprune = no
+	hourly_warn = 90m
+	hourly_crit = 360m
+	daily_warn = 28h
+	daily_crit = 32h
+	weekly_warn = 0
+	weekly_crit = 0
+	monthly_warn = 32d
+	monthly_crit = 40d
+	yearly_warn = 0
+	yearly_crit = 0
+
+
+[template_demo]
+	daily = 60
+	hourly_warn = 290m
+	hourly_crit = 360m
+	daily_warn = 28h
+	daily_crit = 48h
+	weekly_warn = 0
+	weekly_crit = 0
+	monthly_warn = 32d
+	monthly_crit = 40d
+	yearly_warn = 0
+	yearly_crit = 0