Begin work on a munin plugin

bin/php/muninplugin.php

@@ -0,0 +1,194 @@
+<?php
+/**
+ * Munin plugin script
+ *
+ * This script implements a munin wildcard plugin.
+ * Designed to be invoked via the shell script ezmuninperflogger_
+ *
+ * @author G. Giunta
+ * @copyright (C) G. Giunta 2012
+ * @license Licensed under GNU General Public License v2.0. See file license.txt
+ *
+ */
+
+require 'autoload.php';
+
+$cli = eZCLI::instance();
+$script = eZScript::instance( array( 'description' => 'Makes available perf. variables for munin graphs',
+                                     'use-session' => false,
+                                     'use-modules' => true,
+                                     'use-extensions' => true ) );
+$script->startup();
+$options = $script->getOptions(
+    '[variable:][range:]', // options
+    '[command]', // params
+    array() );
+$script->initialize();
+
+if ( count( $options['arguments'] ) )
+{
+    $command = $options['arguments'][0];
+}
+else
+{
+    $command = 'fetch';
+}
+
+// the way that munin wildcard plugins work is that many symlinks are created to the
+// plugin, appending graph name to the original plugin file name. The shell script
+// will pass us its own filename in the 'variable' option
+$variable = isset( $options['variable'] ) ? $options['variable'] : '';
+$variable = preg_replace( '/$ezmuninperflogger_/', '', $variable );
+
+// default munin range: 5 minutes
+$range = $options['range'] ? $options['range'] : 60 * 5;
+
+$ini = eZINI::instance( 'ezperformancelogger.ini' );
+
+switch ( $command )
+{
+    case 'autoconf':
+        /// @todo
+        $siteIni = eZINI::instance();
+        if ( !in_array( 'ezperformancelogger', $siteIni->variable( 'ExtensionSettings', 'ActiveExtensions' ) ) )
+        {
+            $cli->output( "no (extension ezperformancelogger not enabled)" );
+            $script->shutdown();
+        }
+        if ( !in_array( 'csv', $ini->variable( 'GeneralSettings', 'LogMethods' ) ) )
+        {
+            $cli->output( "no (extension ezperformancelogger is not logging data to csv log files)" );
+            $script->shutdown();
+        }
+        $cli->output( "yes" );
+        $script->shutdown();
+        break;
+
+    case 'suggest':
+        // this command is called by munin to get a list of graphs that this plugin supports
+        // see http://munin-monitoring.org/wiki/ConcisePlugins
+        foreach( $ini->variable( 'GeneralSettings', 'TrackVariables' ) as $var )
+        {
+            echo "$var\n";
+        }
+        break;
+
+    case 'config':
+        echo "graph_category eZ Performance Logger\n";
+        $title = false;
+        foreach( array( '', '_avg', '_min', '_max' ) as $suffix )
+        {
+            if ( $ini->hasVariable( 'MuninSettings', 'VariableDescription_' . $variable . $suffix ) )
+            {
+                foreach( $ini->variable( 'MuninSettings', 'VariableDescription_' . $variable . $suffix ) as $item => $value )
+                {
+                    echo $suffix == '' ? "$item $value\n" : "{$variable}{$suffix}.$item $value\n";
+                    if ( $item == 'graph_title' )
+                    {
+                       $title = true;
+                    }
+                }
+            }
+        }
+        if ( !$title )
+        {
+            echo "graph_title $variable\n";
+
+        }
+        break;
+
+    case 'fetch':
+    default:
+        if ( $variable == '' || !in_array( $variable, $ini->variable( 'GeneralSettings', 'TrackVariables' ) ) )
+        {
+            $cli->output( "Error: '$variable' is not a tracked perf. variable" );
+            $script->shutdown( -1 );
+        }
+
+        $logMethods = $ini->variable( 'GeneralSettings', 'LogMethods' );
+        if ( in_array( 'csv', $logMethods ) )
+        {
+            $samples = 0;
+            $values = array( 'total' => 0 );
+
+            if ( file_exists( $logfile = $ini->variable( 'csvSettings', 'FileName' ) ) )
+            {
+                $fp = fopen( $logfile, 'r' );
+                if ( $fp )
+                {
+                    $now = time();
+                    $then = $now - $range;
+                    $separator = $ini->variable( 'csvSettings', 'Separator' );
+                    // we count from 1, as 1st element in csv is timestamp
+                    $i = 1;
+                    foreach( $ini->variable( 'GeneralSettings', 'TrackVariables' ) as $varname )
+                    {
+                        if ( $varname == $variable )
+                        {
+                            $pos = $i;
+                            break;
+                        }
+                        ++$i;
+                    }
+
+                    while ( ( $buffer = fgets( $fp, 8192 ) ) !== false )
+                    {
+                        $ts = substr( $buffer, 0, strpos( $buffer, $separator ) );
+                        /// @todo !important optimization: do not check ts for lower bound after 1st valid one found
+                        if ( $ts >= $then )
+                        {
+                            if ( $ts > $now )
+                            {
+                                break;
+                            }
+
+                            ++$samples;
+                            $data = explode( $separator, $buffer );
+                            $val = $data[$pos];
+                            if ( !isset( $values['min'] ) || $val < $values['min'] )
+                            {
+                                $values['min'] = $val;
+                            }
+                            if ( !isset( $values['max'] ) || $val > $values['max'] )
+                            {
+                                $values['max'] = $val;
+                            }
+                            $values['total'] = $values['total'] + $val;
+                        }
+                    }
+                    fclose( $fp );
+                }
+            }
+
+            if ( $samples )
+            {
+                $values['avg'] = $values['total'] / $samples;
+                echo "{$variable}_avg.value {$values['avg']}\n";
+                echo "{$variable}_min.value {$values['min']}\n";
+                echo "{$variable}_max.value {$values['max']}\n";
+            }
+            else
+            {
+                // no logfile or no samples in range
+                echo "{$variable}_avg.value U\n";
+                echo "{$variable}_min.value U\n";
+                echo "{$variable}_max.value U\n";
+            }
+
+        }
+        else if ( in_array( 'logfile', $logMethods ) || in_array( 'apache', 'logmethods' ) )
+        {
+            /// @todo
+            $cli->output( "Error: can not report variables because ezperflogger set to write data to Apache-formatted log files. Only csv files supported so far" );
+            $script->shutdown( -1 );
+        }
+        else
+        {
+            $cli->output( "Error: can not report variables because ezperflogger is not set to write data to log files" );
+            $script->shutdown( -1 );
+        }
+}
+
+$script->shutdown();
+
+?>

bin/scripts/ezmuninperflogger_

@@ -0,0 +1,61 @@
+#!/bin/sh
+
+########################################
+#
+# This is a Munin plugin, that allows display via Munin graphs of all performance
+# variables tracked by ezperformancelogger.
+# NB: for this to work, the logging method 'csv' must be enabled
+# (see ezperformancelogger.ini for details)
+#
+# This file has to be symlinked (or copied) into the Munin plugins directory.
+# It needs 2 config variables, which can be set in file /etc/munin/plugin-conf.d/ezmuninperflogger
+#
+# [ezmuninperflogger_*]
+# env.php /path/to/php
+# env.ezpublishroot /path/to/the/ez/publish/directory
+#
+########################################
+
+# Magic markers:
+#%# family=manual
+#%# capabilities=autoconf suggest
+
+# Config variables setup
+
+# Directory where eZ Publish is installed.
+# @todo if it is not set in config file, use the 'find' command to look for extension/ezperformancelogger/bin/php/muninplugin.php,
+#       and if only one is found, we can set up $EZPUBLISHROOT automagically
+EZPUBLISHROOT=${ezpublishroot:-/path/to/the/ez/publish/directory}
+
+# Location of the PHP Command Line Interface binary.
+# We init it to 'which php' if it is not set in config file
+PHP=${php}
+if [ -z "$PHP" ]; then
+    PHP=`which php`
+    if [ -z "$PHP" ]; then
+       PHP=/usr/local/bin/php
+    fi
+fi
+
+# Support automatic configuration of the plugin:
+# if $1 == "autoconf", test 1st for proper php and ez variables, and if they are,
+# call the php script with autoconf argument
+if [ "$1" = "autoconf" ]; then
+    if [ ! -x "$PHP" ]; then
+      echo "No ('$PHP' is not an executable, need to configure /etc/munin/plugin-conf.d/ezmuninperflogger)"
+      exit 0
+    fi;
+    # @todo test if $PHP is THE php executable by running eg. php -v ...
+    if [ ! -f "$EZPUBLISHROOT/index.php" ]; then
+      echo "No ('$EZPUBLISHROOT' is not an eZ Publish install, need to configure /etc/munin/plugin-conf.d/ezmuninperflogger)"
+      exit 0
+    fi;
+    if [ ! -f "$EZPUBLISHROOT/extension/ezperformancelogger/bin/php/muninplugin.php" ]; then
+      echo "No ('$EZPUBLISHROOT' does not contain ezperformancelogger extension, probably need to configure /etc/munin/plugin-conf.d/ezmuninperflogge\)"
+      exit 0
+    fi;
+fi
+
+# @todo if this is invoked via ". scriptname", $0 will be "-bash" ...
+VARIABLE=`basename $0`
+cd $EZPUBLISHROOT && $PHP extension/ezperformancelogger/bin/php/muninplugin.php --variable=$VARIABLE $*

classes/ezperfloggercsvstorage.php

@@ -33,12 +33,13 @@ public static function insertStats( array $data )
         }
         if ( $addheader )
         {
+            fwrite( $fp, "Timestamp{$separator}" );
             fwrite( $fp, implode( $separator, $ini->variable( 'GeneralSettings', 'TrackVariables' ) ) );
             fwrite( $fp, "{$separator}Date{$separator}IP Address{$separator}Response Status{$separator}Response size{$separator}URL\n" );
         }
         foreach( $data as $line )
         {
-            $data = $line['counters'];
+            $data = array_merge( array( $line['time'] ), $line['counters'] );
             $data[] = date( 'd/M/Y:H:i:s O', $line['time'] );
             $data[] = $line['ip'];
             $data[] = $line['response_status'];

doc/todo.txt

@@ -1,3 +1,7 @@
+1. test custom db data
+2. check: can easily log cluster queries from debug output?
+3. add logging of any existing timing point
+
 perf logging:
 . self-contained db logging
   . direct-to-db logging
@@ -16,7 +20,7 @@ perf logging:
 
 . improve existing logging targets
   . allow csv output to use comma instead of dot for float numbers
-  . create a custom plugin for piwik to better visaulize our data, and possibly store it in dedicated tables
+  . create a custom plugin for piwik to better visualize our data, and possibly store it in dedicated tables
     (see http://openelibrary.org/development/piwik-custom-data-tracking/)
 . improve php api
   . add new perf. measurement functions: increment/decrement value, measure time?

settings/ezperformancelogger.ini

@@ -39,6 +39,8 @@ TrackVariables[3]=db_queries
 # At the end of every page execution, the static method measure() will be invoked on these
 # classes, to allow them to set proper measured data for the custom variables defined above.
 # Every class registered here should implement the eZPerfLoggerProvider interface.
+# Note: for munin compatibility, variable names should contain only a-z A-Z 0-9 _
+#       see http://munin-monitoring.org/wiki/notes_on_datasource_names
 VariableProviders[]
 VariableProviders[]=eZPerfLogger
 
@@ -134,7 +136,32 @@ ExcludeUrls[]=/\.(css|gif|ico|js|jpe?g|png)$/
 # statistics
 StorageClass=eZPerfLoggerCSVStorage
 
-
+[MuninSettings]
+# When using Munin to graph variables, extra info can be used to make the
+# output nicer.
+# See for details http://munin-monitoring.org/wiki/protocol-config
+#
+# For every variable tracked by eZPerformanceLogger, a dedicated Munin graph is
+# shown, showing av, min and max values for it over the last 5 minutes.
+#
+# Use VariableDescription_<varname> for graph-generic params
+# Use VariableDescription_<varname>_[avg|min|Max] for line-generic params
+
+VariableDescription_mem_usage[graph_title]=Memory per page (bytes)
+VariableDescription_mem_usage_avg[min]=0
+VariableDescription_mem_usage_min[min]=0
+VariableDescription_mem_usage_max[min]=0
+VariableDescription_mem_usage_max[warning]=100000000
+
+VariableDescription_execution_time[graph_title]=Execution time per page (seconds)
+VariableDescription_execution_time_avg[min]=0
+VariableDescription_execution_time_min[min]=0
+VariableDescription_execution_time_max[min]=0
+
+VariableDescription_db_queries[graph_title]=DB queries per page
+VariableDescription_db_queries_avg[min]=0
+VariableDescription_db_queries_min[min]=0
+VariableDescription_db_queries_max[min]=0
 
 # Besides logging data, we can enable xhprof to get complete profiling info
 # NB: needs the xhprof PECL extension to work