Skip to content
Browse files

histogram API

Allows for chained histograms, to support changing bin widths.

Change-Id: Ie7bbd4a0697eb78160eb0ad96822700ba194311f
Reviewed-on: http://review.northscale.com/2102
Tested-by: Steve Yen <steve.yen@gmail.com>
Reviewed-by: Steve Yen <steve.yen@gmail.com>
  • Loading branch information...
1 parent 161b8cd commit b0c66f1fcd53d44d29ef0b5658d3b0877d877ed7 @steveyen steveyen committed Aug 21, 2010
Showing with 473 additions and 1 deletion.
  1. +1 −0 .gitignore
  2. +4 −1 Makefile.am
  3. +172 −0 htgram.c
  4. +159 −0 htgram.h
  5. +137 −0 htgram_test.c
View
1 .gitignore
@@ -47,6 +47,7 @@ cscope.out
depcomp
doc/protocol-binary-range.txt
doc/protocol-binary.txt
+htgram_test
install-sh
libmemcached-*/libmemcached/*.la
libmemcached-*/libmemcached/*.lo
View
5 Makefile.am
@@ -3,8 +3,9 @@ ACLOCAL_AMFLAGS = -I m4 --force
bin_PROGRAMS = moxi
noinst_PROGRAMS =
+
if BUILD_TESTAPPS
-noinst_PROGRAMS += sizes testapp timedrun
+noinst_PROGRAMS += sizes testapp timedrun htgram_test
endif
BUILT_SOURCES =
@@ -47,6 +48,8 @@ endif
timedrun_SOURCES = timedrun.c
+htgram_test_SOURCES = htgram_test.c htgram.c htgram.h
+
TESTS = check_util check_moxi check_work
if HAVE_LIBCONFLATE
TESTS += check_moxi_agent
View
172 htgram.c
@@ -0,0 +1,172 @@
+/* -*- Mode: C; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */
+/*
+ * Copyright 2010 NorthScale, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <assert.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <strings.h>
+
+#include <htgram.h>
+
+struct htgram_bin_st {
+ int64_t start;
+ int64_t width;
+ uint64_t count;
+};
+
+struct htgram_st {
+ int64_t bin_start;
+ int64_t bin_start_width;
+ double bin_width_growth;
+ size_t num_bins;
+
+ struct htgram_bin_st *bins;
+
+ uint64_t lt_count; // For data points < the bins.
+ uint64_t gt_count; // For data points > the bins.
+
+ // For data points > the bins, there may be another
+ // histogram instead of using gt_count.
+ //
+ HTGRAM_HANDLE next;
+};
+
+HTGRAM_HANDLE htgram_mk(int64_t bin_start,
+ int64_t bin_start_width,
+ double bin_width_growth,
+ size_t num_bins,
+ HTGRAM_HANDLE next) {
+ struct htgram_st *h = calloc(sizeof(struct htgram_st), 1);
+ if (h == NULL) {
+ return NULL;
+ }
+
+ h->bin_start = bin_start;
+ h->bin_start_width = bin_start_width;
+ h->bin_width_growth = bin_width_growth;
+ h->num_bins = num_bins;
+ h->next = next;
+
+ if (num_bins > 0) {
+ h->bins = calloc(sizeof(struct htgram_bin_st), num_bins);
+ if (h->bins == NULL) {
+ free(h);
+ return NULL;
+ }
+
+ int64_t r = bin_start;
+ int64_t w = bin_start_width;
+ for (size_t i = 0; i < num_bins; i++) {
+ h->bins[i].start = r;
+ h->bins[i].width = w;
+ r = r + w;
+ w = w * bin_width_growth;
+ }
+ }
+
+ return h;
+}
+
+void htgram_destroy(HTGRAM_HANDLE h) {
+ if (h->bins != NULL) {
+ free(h->bins);
+ }
+ if (h->next != NULL) {
+ htgram_destroy(h->next);
+ }
+ free(h);
+}
+
+int64_t htgram_get_bin_start(HTGRAM_HANDLE h) {
+ return h->bin_start;
+}
+
+int64_t htgram_get_bin_start_width(HTGRAM_HANDLE h) {
+ return h->bin_start_width;
+}
+
+double htgram_get_bin_width_growth(HTGRAM_HANDLE h) {
+ return h->bin_width_growth;
+}
+
+size_t htgram_get_num_bins(HTGRAM_HANDLE h) {
+ return h->num_bins;
+}
+
+void htgram_incr(HTGRAM_HANDLE h, int64_t data_point, uint64_t count) {
+ if (data_point < h->bin_start) {
+ h->lt_count += count;
+ return;
+ }
+
+ for (size_t i = 0; i < h->num_bins; i++) {
+ if (data_point < (h->bins[i].start +
+ h->bins[i].width)) {
+ h->bins[i].count += count;
+ return;
+ }
+ }
+
+ if (h->next != NULL) {
+ htgram_incr(h->next, data_point, count);
+ return;
+ }
+
+ h->gt_count += count;
+}
+
+bool htgram_get_bin_data(HTGRAM_HANDLE h, int bin_index,
+ int64_t *out_bin_start,
+ int64_t *out_bin_width,
+ uint64_t *out_bin_count) {
+ if (bin_index < 0) {
+ *out_bin_count = h->lt_count;
+ return false;
+ }
+
+ if (bin_index >= (int) h->num_bins) {
+ if (h->next != NULL) {
+ return htgram_get_bin_data(h->next, bin_index - h->num_bins,
+ out_bin_start,
+ out_bin_width,
+ out_bin_count);
+ }
+
+ *out_bin_count = h->gt_count;
+ return false;
+ }
+
+ *out_bin_start = h->bins[bin_index].start;
+ *out_bin_width = h->bins[bin_index].width;
+ *out_bin_count = h->bins[bin_index].count;
+
+ return true;
+}
+
+void htgram_reset(HTGRAM_HANDLE h) {
+ for (size_t i = 0; i < h->num_bins; i++) {
+ h->bins[i].count = 0;
+ }
+
+ h->lt_count = 0;
+ h->gt_count = 0;
+
+ if (h->next != NULL) {
+ htgram_reset(h->next);
+ }
+}
View
159 htgram.h
@@ -0,0 +1,159 @@
+/* -*- Mode: C; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */
+/*
+ * Copyright 2010 NorthScale, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/*! \mainpage htgram
+ *
+ * \section intro_sec Introduction
+ *
+ * htgram is a short histogram implementation
+ *
+ * \section docs_sec API Documentation
+ *
+ * Jump right into <a href="modules.html">the modules docs</a> to get started.
+ */
+
+/**
+ * Histogram Utility Library.
+ *
+ * \defgroup CD Creation and Destruction
+ * \defgroup Data Collecting and retrieving stats
+ */
+
+#ifndef HTGRAM_H
+#define HTGRAM_H 1
+
+#include <stddef.h>
+#include <stdint.h>
+#include <stdbool.h>
+#include <math.h>
+
+#define HTGRAM_PUBLIC_API
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+ struct htgram_st;
+
+ /**
+ * Opaque histogram representation.
+ */
+ typedef struct htgram_st *HTGRAM_HANDLE;
+
+ /**
+ * \addtogroup CD
+ * @{
+ */
+
+ /**
+ * Create an instance of htgram.
+ *
+ * htgram_mk(0, 5, 1.0, 3, NULL) means have 3 bins, each of width
+ * 5, such as [0, 5), [5, 10), [10, 15).
+ *
+ * htgram_mk(0, 5, 2.0, 3, NULL) means have 3 bins, but the bin
+ * widths should double, such as [0, 5), [5, 15), [15, 35).
+ *
+ * @param bin_start the first bin starts at this range
+ * value, inclusive.
+ * @param bin_start_width width of the first bin.
+ * @param bin_width_growth grow each bin width by this factor.
+ * @param num_bins number of bins.
+ * @param next optional chained HTGRAM_HANDLE, which is useful
+ * to change bin_width_growth factors; may be NULL.
+ */
+ HTGRAM_PUBLIC_API
+ HTGRAM_HANDLE htgram_mk(int64_t bin_start,
+ int64_t bin_start_width,
+ double bin_width_growth,
+ size_t num_bins,
+ HTGRAM_HANDLE next);
+
+ /**
+ * Destroy a htgram.
+ *
+ * @param h the htgram handle
+ */
+ HTGRAM_PUBLIC_API
+ void htgram_destroy(HTGRAM_HANDLE h);
+
+ /**
+ * @}
+ */
+
+ /**
+ * \addtogroup Data
+ * @{
+ */
+
+ /**
+ * Get the range start value for the first bin.
+ */
+ HTGRAM_PUBLIC_API
+ int64_t htgram_get_bin_start(HTGRAM_HANDLE h);
+
+ /**
+ * Get the width of the first bin.
+ */
+ HTGRAM_PUBLIC_API
+ int64_t htgram_get_bin_start_width(HTGRAM_HANDLE h);
+
+ /**
+ * Get the growth factor for bin widths.
+ */
+ HTGRAM_PUBLIC_API
+ double htgram_get_bin_width_growth(HTGRAM_HANDLE h);
+
+ /**
+ * Get the total number of bins.
+ */
+ HTGRAM_PUBLIC_API
+ size_t htgram_get_num_bins(HTGRAM_HANDLE h);
+
+ /**
+ * Add a data_point to the histogram, where the correct bin will
+ * be incremented by count. For example, htgram_incr(h, request_latency, 1);
+ */
+ HTGRAM_PUBLIC_API
+ void htgram_incr(HTGRAM_HANDLE h, int64_t data_point, uint64_t count);
+
+ /**
+ * Retrieves collected data (out_bin_count) for a bin, given a
+ * bin_index. The first width bin has bin_index of 0.
+ * Returns false if there's no bin at the given bin_index.
+ */
+ HTGRAM_PUBLIC_API
+ bool htgram_get_bin_data(HTGRAM_HANDLE h, int bin_index,
+ int64_t *out_bin_start,
+ int64_t *out_bin_width,
+ uint64_t *out_bin_count);
+
+ /**
+ * Reset all bin counts to zero.
+ */
+ HTGRAM_PUBLIC_API
+ void htgram_reset(HTGRAM_HANDLE h);
+
+ /**
+ * @}
+ */
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif
View
137 htgram_test.c
@@ -0,0 +1,137 @@
+/* -*- Mode: C; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */
+
+#include "config.h"
+#include <assert.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+
+#include <htgram.h>
+
+static void testSimple(void) {
+ HTGRAM_HANDLE h0;
+
+ int64_t start;
+ int64_t width;
+ uint64_t count;
+
+ h0 = htgram_mk(0, 5, 1.0, 3, NULL);
+ assert(h0 != NULL);
+ assert(htgram_get_bin_start(h0) == 0);
+ assert(htgram_get_bin_start_width(h0) == 5);
+ assert(htgram_get_bin_width_growth(h0) == 1.0);
+ assert(htgram_get_num_bins(h0) == 3);
+
+ start = width = count = 123;
+ assert(htgram_get_bin_data(h0, -1, &start, &width, &count) == false);
+ assert(start == 123);
+ assert(width == 123);
+ assert(count == 0);
+
+ int i;
+ for (i = 0; i < (int) htgram_get_num_bins(h0); i++) {
+ start = width = count = 123;
+ assert(htgram_get_bin_data(h0, i, &start, &width, &count) == true);
+ assert(start == i * 5);
+ assert(width == 5);
+ assert(count == 0);
+
+ htgram_incr(h0, (i * 5) + 0, 1);
+ htgram_incr(h0, (i * 5) + 1, 2);
+ htgram_incr(h0, (i * 5) + 2, 3);
+ htgram_incr(h0, (i * 5) + 5, 0);
+
+ start = width = count = 123;
+ assert(htgram_get_bin_data(h0, i, &start, &width, &count) == true);
+ assert(start == i * 5);
+ assert(width == 5);
+ assert(count == 6);
+ }
+
+ start = width = count = 123;
+ assert(htgram_get_bin_data(h0, i, &start, &width, &count) == false);
+ assert(start == 123);
+ assert(width == 123);
+ assert(count == 0);
+
+ htgram_reset(h0);
+
+ start = width = count = 123;
+ assert(htgram_get_bin_data(h0, -1, &start, &width, &count) == false);
+ assert(start == 123);
+ assert(width == 123);
+ assert(count == 0);
+
+ for (i = 0; i < (int) htgram_get_num_bins(h0); i++) {
+ start = width = count = 123;
+ assert(htgram_get_bin_data(h0, i, &start, &width, &count) == true);
+ assert(start == i * 5);
+ assert(width == 5);
+ assert(count == 0);
+ }
+
+ start = width = count = 123;
+ assert(htgram_get_bin_data(h0, i, &start, &width, &count) == false);
+ assert(start == 123);
+ assert(width == 123);
+ assert(count == 0);
+
+ htgram_incr(h0, -10000, 111);
+ htgram_incr(h0, 10000, 222);
+
+ start = width = count = 123;
+ assert(htgram_get_bin_data(h0, -1, &start, &width, &count) == false);
+ assert(start == 123);
+ assert(width == 123);
+ assert(count == 111);
+
+ start = width = count = 123;
+ assert(htgram_get_bin_data(h0, i, &start, &width, &count) == false);
+ assert(start == 123);
+ assert(width == 123);
+ assert(count == 222);
+
+ htgram_destroy(h0);
+}
+
+static void testChained(void) {
+ HTGRAM_HANDLE h0, h1;
+
+ int64_t start;
+ int64_t width;
+ uint64_t count;
+
+ // Have 200 bins from [0 to 2000), with bin widths of 10.
+ // Have 36 bins from 2000 onwards, with bin width growing at 1.5, chained.
+ h1 = htgram_mk(2000, 10, 1.5, 36, NULL);
+ h0 = htgram_mk(0, 10, 1.0, 200, h1);
+
+ int i;
+ for (i = 0; i < (int) (htgram_get_num_bins(h0) + htgram_get_num_bins(h1)); i++) {
+ assert(htgram_get_bin_data(h0, i, &start, &width, &count) == true);
+ // printf("%d %d %d %d\n", i, start, width, count);
+ }
+
+ htgram_incr(h0, 28000000, 111);
+
+ assert(htgram_get_bin_data(h0, 200 + 36 - 1, &start, &width, &count) == true);
+ assert(start == 27692301);
+ assert(width == 13845150);
+ assert(count == 111);
+
+ htgram_reset(h0);
+
+ assert(htgram_get_bin_data(h0, 200 + 36 - 1, &start, &width, &count) == true);
+ assert(start == 27692301);
+ assert(width == 13845150);
+ assert(count == 0);
+}
+
+int main(void) {
+ testSimple();
+ testChained();
+
+ return 0;
+}
+
+

0 comments on commit b0c66f1

Please sign in to comment.
Something went wrong with that request. Please try again.