Adding gzipped-XML-tree support to gmond, gmetad, and gmetad-python

configure.ac

@@ -595,6 +595,24 @@ else
 fi
 AM_CONDITIONAL(HAVE_SYSTEMD, [test -n "$with_systemdsystemunitdir" -a "x$with_systemdsystemunitdir" != xno ])
 
+echo
+echo Checking for zlib
+AC_ARG_WITH([zlib],
+  AS_HELP_STRING([--with-zlib=DIR], [Specify location for zlib]),
+  [if test x"$withval" != xno; then libzlib="yes"; libzlibpath="$withval"; fi])
+if test x"$libzlibpath" != x && test x"$libzlibpath" != xyes; then
+  CFLAGS="$CFLAGS -I$libzlibpath/include"
+  CPPFLAGS="$CPPFLAGS -I$libzlibpath/include"
+  LDFLAGS="$LDFLAGS -L$libzlibpath/${LIB_SUFFIX}"
+  echo "Added -I$libzlibpath/include to CFLAGS and CPPFLAGS"
+  echo "Added -L$libzlibpath/${LIB_SUFFIX} to LDFLAGS"
+fi
+AC_CHECK_HEADERS([zlib.h])
+AC_CHECK_LIB(z, deflate)
+if test x"$ac_cv_lib_z_deflate" != xyes; then
+  echo "zlib library not configured properly"; exit 1;
+fi
+echo "Found a suitable zlib"
 
 echo
 

gmetad-python/Gmetad/gmetad_gmondReader.py

@@ -36,6 +36,7 @@
 import socket
 import time
 import logging
+import zlib
 
 from gmetad_config import GmetadConfig, getConfig
 from gmetad_random import getRandomInterval
@@ -135,6 +136,12 @@ def run(self):
                         break
                     xmlbuf += buf
                 sock.close()
+
+                # These are the gzip header magic numbers, per RFC 1952 section 2.3.1
+                if xmlbuf[0:2] == '\x1f\x8b':
+                    # 32 is a magic number in zlib.h for autodetecting the zlib or gzip header
+                    xmlbuf = zlib.decompress(xmlbuf, zlib.MAX_WBITS + 32)
+
                 # Create an XML parser and parse the buffer
                 gch = GmondContentHandler()
                 xml.sax.parseString(xmlbuf, gch)

gmetad/data_thread.c

@@ -6,6 +6,7 @@
 #include <sys/time.h>
 #include <gmetad.h>
 #include <string.h>
+#include <zlib.h>
 
 #include <apr_time.h>
 
@@ -190,6 +191,97 @@ data_thread ( void *arg )
                   }
             }
 
+         /* These are the gzip header magic numbers, per RFC 1952 section 2.3.1 */
+         if(read_index > 2 && (unsigned char)buf[0] == 0x1f && (unsigned char)buf[1] == 0x8b)
+	   {
+	     /* Uncompress the buffer */
+	     int ret;
+	     z_stream strm;
+	     char * uncompressed;
+	     unsigned int write_index = 0;
+
+	     if( get_debug_msg_level() > 1 )
+	       {
+		 err_msg("GZIP compressed data for [%s] data source, %d bytes", d->name, read_index);
+	       }
+
+	     uncompressed = malloc(buf_size);
+	     if( !uncompressed )
+	       {
+		 err_quit("data_thread() unable to malloc enough room for [%s] GZIP", d->name);
+	       }
+
+	     strm.zalloc = NULL;
+	     strm.zfree = NULL;
+	     strm.opaque = NULL;
+	     strm.next_in  = (Bytef *)buf;
+	     strm.avail_in = read_index;
+
+	     /* Initialize the stream, 15 and 16 are magic numbers (gzip and max window size) */
+	     ret = inflateInit2(&strm, 15 + 16);
+	     if( ret != Z_OK )
+	       {
+		 err_msg("InflateInitError! for [%s] data source, failed to call inflateInit", d->name);
+		 d->dead = 1;
+
+		 free(buf);
+		 buf = uncompressed;
+		 goto take_a_break;
+	       }
+
+	     while (1)
+	       {
+		 /* Create more buffer space if needed */
+		 if ( (write_index + 2048) > buf_size)
+		   {
+		     buf_size += 2048;
+		     uncompressed = realloc(uncompressed, buf_size);
+		     if(!uncompressed)
+		       {
+			 err_quit("data_thread() unable to realloc enough room for [%s] GZIP", d->name) ;
+		       }
+		   }
+
+		 /* Do the inflate */
+		 strm.next_out  = (Bytef *)(uncompressed + write_index);
+		 strm.avail_out = buf_size - write_index - 1;
+
+		 ret = inflate(&strm, Z_FINISH);
+		 write_index = strm.total_out;
+
+		 if (ret == Z_OK || ret == Z_BUF_ERROR)
+		   {
+		     /* These are normal - just continue on */
+		     continue;
+		   }
+		 else if( ret == Z_STREAM_END )
+		   {
+		     /* We have finished, set things up for the XML parser */
+		     free (buf);
+		     buf = uncompressed;
+		     read_index = write_index;
+		     if(get_debug_msg_level() > 1)
+		       {
+			 err_msg("Uncompressed to %d bytes", read_index);
+		       }
+		     break;
+		   }
+		 else
+		   {
+		     /* Oh dear, something bad */
+		     inflateEnd(&strm);
+
+		     err_msg("InflateError! for [%s] data source, failed to call inflate (%s)", d->name, zError(ret));
+		     d->dead = 1;
+
+		     free(buf);
+		     buf = uncompressed;
+		     goto take_a_break;
+		   }
+	       }
+	     inflateEnd(&strm);
+	   }
+
          buf[read_index] = '\0';
 
          /* Parse the buffer */

gmond/cmdline.c.in

@@ -43,6 +43,7 @@ const char *gengetopt_args_info_help[] = {
   "  -b, --bandwidth        Calculate minimum bandwidth use for configuration  \n                           (default=off)",
   "  -r, --convert=STRING   Convert a 2.5.x configuration file to the new 3.x \n                           format",
   "  -p, --pid-file=STRING  Write process-id to file",
+  "  -z, --gzip-output      Compress output with gzip before sending  \n                           (default=off)",
     0
 };
 
@@ -79,6 +80,7 @@ void clear_given (struct gengetopt_args_info *args_info)
   args_info->bandwidth_given = 0 ;
   args_info->convert_given = 0 ;
   args_info->pid_file_given = 0 ;
+  args_info->gzip_output_given = 0 ;
 }
 
 static
@@ -99,6 +101,7 @@ void clear_args (struct gengetopt_args_info *args_info)
   args_info->convert_orig = NULL;
   args_info->pid_file_arg = NULL;
   args_info->pid_file_orig = NULL;
+  args_info->gzip_output_flag = 0;
 
 }
 
@@ -118,6 +121,7 @@ void init_args_info(struct gengetopt_args_info *args_info)
   args_info->bandwidth_help = gengetopt_args_info_help[8] ;
   args_info->convert_help = gengetopt_args_info_help[9] ;
   args_info->pid_file_help = gengetopt_args_info_help[10] ;
+  args_info->gzip_output_help = gengetopt_args_info_help[11] ;
 
 }
 
@@ -259,6 +263,8 @@ cmdline_parser_dump(FILE *outfile, struct gengetopt_args_info *args_info)
     write_into_file(outfile, "convert", args_info->convert_orig, 0);
   if (args_info->pid_file_given)
     write_into_file(outfile, "pid-file", args_info->pid_file_orig, 0);
+  if (args_info->gzip_output_given)
+    write_into_file(outfile, "gzip-output", 0, 0 );
 
 
   i = EXIT_SUCCESS;
@@ -524,10 +530,11 @@ cmdline_parser_internal (
         { "bandwidth",	0, NULL, 'b' },
         { "convert",	1, NULL, 'r' },
         { "pid-file",	1, NULL, 'p' },
+        { "gzip-output",	0, NULL, 'z' },
         { 0,  0, 0, 0 }
       };
 
-      c = getopt_long (argc, argv, "hVc:l:d:ftmbr:p:", long_options, &option_index);
+      c = getopt_long (argc, argv, "hVc:l:d:ftmbr:p:z", long_options, &option_index);
 
       if (c == -1) break;	/* Exit from `while (1)' loop.  */
 
@@ -643,6 +650,16 @@ cmdline_parser_internal (
             goto failure;
 
           break;
+        case 'z':	/* Compress output with gzip before sending.  */
+
+
+          if (update_arg((void *)&(args_info->gzip_output_flag), 0, &(args_info->gzip_output_given),
+              &(local_args_info.gzip_output_given), optarg, 0, 0, ARG_FLAG,
+              check_ambiguity, override, 1, 0, "gzip-output", 'z',
+              additional_error))
+            goto failure;
+
+          break;
 
         case 0:	/* Long option with no short option */
         case '?':	/* Invalid option.  */

gmond/cmdline.h

@@ -62,6 +62,8 @@ struct gengetopt_args_info
   char * pid_file_arg;	/**< @brief Write process-id to file.  */
   char * pid_file_orig;	/**< @brief Write process-id to file original value given at command line.  */
   const char *pid_file_help; /**< @brief Write process-id to file help description.  */
+  unsigned int gzip_output_flag;	/**< @brief Compress output with gzip before sending (default=off).  */
+  const char *gzip_output_help; /**< @brief Compress output with gzip before sending help description.  */
 
   unsigned int help_given ;	/**< @brief Whether help was given.  */
   unsigned int version_given ;	/**< @brief Whether version was given.  */
@@ -74,6 +76,7 @@ struct gengetopt_args_info
   unsigned int bandwidth_given ;	/**< @brief Whether bandwidth was given.  */
   unsigned int convert_given ;	/**< @brief Whether convert was given.  */
   unsigned int pid_file_given ;	/**< @brief Whether pid-file was given.  */
+  unsigned int gzip_output_given ;	/**< @brief Whether gzip-output was given.  */
 
 } ;
 

gmond/cmdline.sh

@@ -15,6 +15,7 @@ option "metrics" m "Print the list of metrics this gmond supports" flag off
 option "bandwidth" b "Calculate minimum bandwidth use for configuration" flag off
 option "convert" r "Convert a 2.5.x configuration file to the new 3.x format" string no
 option "pid-file" p "Write process-id to file" string no
+option "gzip-output" z "Compress output with gzip before sending" flag off
 
 #Usage (a little tutorial)
 #