Merge pull request #306 from JonasKunze/immediate_histo

Immediate histo

src/modules/histogram.c

@@ -29,7 +29,6 @@
  */
 
 #include <inttypes.h>
-#include <mtev_defines.h>
 
 #include <mtev_log.h>
 #include <mtev_b64.h>
@@ -40,6 +39,7 @@
 #include "noit_module.h"
 #include "noit_check.h"
 #include "noit_check_tools.h"
+#include "histogram.h"
 
 static mtev_log_stream_t metrics_log = NULL;
 static int histogram_module_id = -1;
@@ -123,23 +123,18 @@ debug_print_hist(histogram_t *ht) {
     }
   }
 }
-static void
-log_histo(struct histogram_config *conf,
-          noit_check_t *check, u_int64_t whence_s,
-          const char *metric_name, histogram_t *h,
+
+void
+noit_log_histo_encoded_function(noit_check_t *check, u_int64_t whence_s,
+          const char *metric_name, const char *hist_encode, ssize_t hist_encode_len,
           mtev_boolean live_feed) {
   mtev_boolean extended_id = mtev_false;
   char uuid_str[256*3+37];
   const char *v;
-  char *hist_serial = NULL;
-  char *hist_encode = NULL;
   struct timeval whence;
-  ssize_t est, enc_est;
   whence.tv_sec = whence_s;
   whence.tv_usec = 0;
 
-  if(!conf->histogram) return;
-
   SETUP_LOG(metrics, );
   if(metrics_log) {
     v = mtev_log_stream_get_property(metrics_log, "extended_id");
@@ -159,6 +154,40 @@ log_histo(struct histogram_config *conf,
 #define SECPART(a) ((unsigned long)(a)->tv_sec)
 #define MSECPART(a) ((unsigned long)((a)->tv_usec / 1000))
 
+  if(live_feed && check->feeds) {
+    mtev_skiplist_node *curr, *next;
+    curr = next = mtev_skiplist_getlist(check->feeds);
+    while(curr) {
+      const char *feed_name = (char *)curr->data;
+      mtev_log_stream_t ls = mtev_log_stream_find(feed_name);
+      mtev_skiplist_next(check->feeds, &next);
+      if(!ls ||
+         mtev_log(ls, &whence, __FILE__, __LINE__,
+           "H1\t%lu.%03lu\t%s\t%s\t%.*s\n",
+           SECPART(&whence), MSECPART(&whence),
+           uuid_str, metric_name, (int)hist_encode_len, hist_encode))
+        noit_check_transient_remove_feed(check, feed_name);
+      curr = next;
+    }
+  }
+
+  if(!live_feed) {
+    SETUP_LOG(metrics, return);
+    mtev_log(metrics_log, &whence, __FILE__, __LINE__,
+             "H1\t%lu.%03lu\t%s\t%s\t%.*s\n",
+             SECPART(&whence), MSECPART(&whence),
+             uuid_str, metric_name, (int)hist_encode_len, hist_encode);
+  }
+}
+
+static void
+log_histo(noit_check_t *check, u_int64_t whence_s,
+          const char *metric_name, histogram_t *h,
+          mtev_boolean live_feed) {
+  char *hist_serial = NULL;
+  char *hist_encode = NULL;
+  ssize_t est, enc_est;
+
   est = hist_serialize_estimate(h);
   hist_serial = malloc(est);
   if(!hist_serial) {
@@ -182,30 +211,8 @@ log_histo(struct histogram_config *conf,
     goto cleanup;
   }
 
-  if(live_feed && check->feeds) {
-    mtev_skiplist_node *curr, *next;
-    curr = next = mtev_skiplist_getlist(check->feeds);
-    while(curr) {
-      const char *feed_name = (char *)curr->data;
-      mtev_log_stream_t ls = mtev_log_stream_find(feed_name);
-      mtev_skiplist_next(check->feeds, &next);
-      if(!ls ||
-         mtev_log(ls, &whence, __FILE__, __LINE__,
-           "H1\t%lu.%03lu\t%s\t%s\t%.*s\n",
-           SECPART(&whence), MSECPART(&whence),
-           uuid_str, metric_name, (int)enc_est, hist_encode))
-        noit_check_transient_remove_feed(check, feed_name);
-      curr = next;
-    }
-  }
+  noit_log_histo_encoded_function(check, whence_s, metric_name, hist_encode, enc_est, live_feed);
 
-  if(!live_feed) {
-    SETUP_LOG(metrics, goto cleanup);
-    mtev_log(metrics_log, &whence, __FILE__, __LINE__,
-             "H1\t%lu.%03lu\t%s\t%s\t%.*s\n",
-             SECPART(&whence), MSECPART(&whence),
-             uuid_str, metric_name, (int)enc_est, hist_encode);
-  }
  cleanup:
   if(hist_serial) free(hist_serial);
   if(hist_encode) free(hist_encode);
@@ -247,7 +254,8 @@ sweep_roll_n_log(struct histogram_config *conf, noit_check_t *check, histotier *
   }
 
   /* push this out to the log streams */
-  log_histo(conf, check, aligned_seconds, name, tgt, mtev_false);
+  if(conf->histogram)
+    log_histo(check, aligned_seconds, name, tgt, mtev_false);
   debug_print_hist(tgt);
 
   /* drop the tgt, it's ours */
@@ -303,9 +311,10 @@ update_histotier(histotier *ht, u_int64_t s,
       }
     }
     last_bucket = last_second % 10;
-    if(ht->secs[last_bucket] && hist_num_buckets(ht->secs[last_bucket]))
-      log_histo(conf, check, last_minute * 60 + last_second,
-                name, ht->secs[last_bucket], mtev_true);
+    if(ht->secs[last_bucket] && hist_num_buckets(ht->secs[last_bucket])
+        && conf->histogram)
+      log_histo(check, last_minute * 60 + last_second, name,
+          ht->secs[last_bucket], mtev_true);
   }
   if(minute > ht->last_minute) {
     sweep_roll_n_log(conf, check, ht, name);

src/modules/histogram.h

@@ -0,0 +1,47 @@
+/*
+ * Copyright (c) 2012-2015, Circonus, Inc. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above
+ *       copyright notice, this list of conditions and the following
+ *       disclaimer in the documentation and/or other materials provided
+ *       with the distribution.
+ *     * Neither the name Circonus, Inc. nor the names of its contributors
+ *       may be used to endorse or promote products derived from this
+ *       software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef MODULES_HISTOGRAM_H
+#define MODULES_HISTOGRAM_H
+
+#include <mtev_defines.h>
+#include <mtev_hooks.h>
+
+#include <sys/types.h>
+
+#include "noit_check.h"
+
+MTEV_RUNTIME_RESOLVE(noit_log_histo_encoded, noit_log_histo_encoded_function, void,
+                     (noit_check_t *check, u_int64_t whence_s,
+                         const char *metric_name, const char *hist_encode, ssize_t hist_encode_len,
+                         mtev_boolean live_feed), (check, whence_s, metric_name, hist_encode, hist_encode_len, live_feed))
+MTEV_RUNTIME_AVAIL(noit_log_histo_encoded, noit_log_histo_encoded_function)
+
+#endif

src/modules/lua_check.c

@@ -39,6 +39,8 @@
 #include <mtev_dso.h>
 #include <mtev_log.h>
 
+#include <circllhist.h>
+
 #include "noit_config.h"
 #include "noit_module.h"
 #include "noit_check.h"
@@ -464,6 +466,38 @@ noit_lua_set_metric_f(lua_State *L,
   return 1;
 }
 
+static int
+noit_lua_set_histo_metric(lua_State *L) {
+  noit_check_t *check;
+  const char *metric_name;
+  size_t hist_encoded_len;
+  const char *hist_encoded;
+  u_int64_t whence_s;
+
+  if(lua_gettop(L) != 3) luaL_error(L, "need 3 arguments: <metric_name> <encoded_histo> <whence_s>");
+  check = lua_touserdata(L, lua_upvalueindex(1));
+  if(!lua_isstring(L, 1)) luaL_error(L, "argument #1 must be a string");
+  if(!lua_isstring(L, 2)) luaL_error(L, "argument #2 must be a string");
+  if(!lua_isnumber(L, 3)) luaL_error(L, "argument #3 must be a number");
+
+  metric_name = lua_tostring(L, 1);
+  if(lua_isnil(L, 2)) {
+    return luaL_error(L, "argument #2 must not be nil");
+  }
+
+  hist_encoded = lua_tolstring(L, 2, &hist_encoded_len);
+  whence_s = lua_tointeger(L, 3);
+
+  if(noit_stats_log_immediate_histo(check, metric_name, hist_encoded,
+      hist_encoded_len, whence_s) == mtev_false) {
+    return luaL_error(L,
+        "Unable to invoke noit_log_histo_encoded! Did you load the histogram module?!");
+  }
+
+  lua_pushboolean(L, 1);
+  return 1;
+}
+
 static int
 noit_lua_set_metric(lua_State *L) {
   return noit_lua_set_metric_f(L, noit_stats_set_metric);
@@ -606,6 +640,10 @@ noit_check_index_func(lua_State *L) {
       else IF_METRIC_IMMEDIATE_BLOCK("metric_int64", METRIC_INT64)
       else IF_METRIC_IMMEDIATE_BLOCK("metric_uint64", METRIC_UINT64)
       else IF_METRIC_IMMEDIATE_BLOCK("metric_double", METRIC_DOUBLE)
+      else if(!strcmp(k, "immediate_histogram")) {
+        lua_pushlightuserdata(L, check);
+        lua_pushcclosure(L, noit_lua_set_histo_metric, 1);
+      }
       else break;
 
       return 1;

src/noit_check.c

@@ -59,6 +59,7 @@
 #include "noit_module.h"
 #include "noit_check_tools.h"
 #include "noit_check_resolver.h"
+#include "modules/histogram.h"
 
 #define DEFAULT_TEXT_METRIC_SIZE_LIMIT  512
 #define RECYCLE_INTERVAL 60
@@ -2039,10 +2040,10 @@ noit_stats_set_metric_coerce(noit_check_t *check,
   }
   check_stats_set_metric_coerce_hook_invoke(check, c, name, t, v, mtev_true);
 }
-void
-noit_stats_log_immediate_metric(noit_check_t *check,
+static void
+record_immediate_metric(noit_check_t *check,
                                 const char *name, metric_type_t type,
-                                const void *value) {
+                                const void *value, mtev_boolean do_log) {
   struct timeval now;
   stats_t *c;
   metric_t *m = mtev_memory_safe_malloc_cleanup(sizeof(*m), noit_check_safe_free_metric);
@@ -2052,12 +2053,32 @@ noit_stats_log_immediate_metric(noit_check_t *check,
     return;
   }
   gettimeofday(&now, NULL);
-  noit_check_log_metric(check, &now, m);
+  if(do_log == mtev_true) {
+    noit_check_log_metric(check, &now, m);
+  }
   c = noit_check_get_stats_inprogress(check);
   if(__mark_metric_logged(c, m) == mtev_false) {
     mtev_memory_safe_free(m);
   }
 }
+void
+noit_stats_log_immediate_metric(noit_check_t *check,
+                                const char *name, metric_type_t type,
+                                const void *value) {
+  record_immediate_metric(check, name, type, value, mtev_true);
+}
+mtev_boolean
+noit_stats_log_immediate_histo(noit_check_t *check,
+                                const char *name, const char *hist_encoded, size_t hist_encoded_len,
+                                u_int64_t whence_s) {
+  if (noit_log_histo_encoded_available()) {
+    noit_log_histo_encoded(check, whence_s, name, hist_encoded, hist_encoded_len, mtev_false);
+  } else {
+    return mtev_false;
+  }
+  record_immediate_metric(check, name, METRIC_INT32, NULL, mtev_false);
+  return mtev_true;
+}
 
 void
 noit_check_passive_set_stats(noit_check_t *check) {

src/noit_check.h

@@ -346,6 +346,8 @@ API_EXPORT(void) noit_check_log_bundle(noit_check_t *check);
 API_EXPORT(void) noit_check_log_metrics(noit_check_t *check);
 API_EXPORT(void) noit_check_log_metric(noit_check_t *check,
                                        struct timeval *whence, metric_t *m);
+API_EXPORT(void) noit_check_log_histo(noit_check_t *check, u_int64_t whence_s,
+          const char *metric_name, const char *b64_histo, ssize_t b64_histo_len);
 API_EXPORT(void) noit_check_extended_id_split(const char *in, int len,
                                               char *target, int target_len,
                                               char *module, int module_len,

src/noit_check_log.c

@@ -608,4 +608,3 @@ noit_stats_snprint_metric(char *b, int l, metric_t *m) {
     rv = snprintf(b+nl, l-nl, "[[unknown type]]");
   return rv + nl;
 }
-