Skip to content
This repository

HTTPS clone URL

Subversion checkout URL

You can clone with HTTPS or Subversion.

Download ZIP
Browse code

histogram API

Allows for chained histograms, to support changing bin widths.

Change-Id: Ie7bbd4a0697eb78160eb0ad96822700ba194311f
Reviewed-on: http://review.northscale.com/2102
Tested-by: Steve Yen <steve.yen@gmail.com>
Reviewed-by: Steve Yen <steve.yen@gmail.com>
  • Loading branch information...
commit b0c66f1fcd53d44d29ef0b5658d3b0877d877ed7 1 parent 161b8cd
Steve Yen steveyen authored
1  .gitignore
@@ -47,6 +47,7 @@ cscope.out
47 47 depcomp
48 48 doc/protocol-binary-range.txt
49 49 doc/protocol-binary.txt
  50 +htgram_test
50 51 install-sh
51 52 libmemcached-*/libmemcached/*.la
52 53 libmemcached-*/libmemcached/*.lo
5 Makefile.am
@@ -3,8 +3,9 @@ ACLOCAL_AMFLAGS = -I m4 --force
3 3
4 4 bin_PROGRAMS = moxi
5 5 noinst_PROGRAMS =
  6 +
6 7 if BUILD_TESTAPPS
7   -noinst_PROGRAMS += sizes testapp timedrun
  8 +noinst_PROGRAMS += sizes testapp timedrun htgram_test
8 9 endif
9 10
10 11 BUILT_SOURCES =
@@ -47,6 +48,8 @@ endif
47 48
48 49 timedrun_SOURCES = timedrun.c
49 50
  51 +htgram_test_SOURCES = htgram_test.c htgram.c htgram.h
  52 +
50 53 TESTS = check_util check_moxi check_work
51 54 if HAVE_LIBCONFLATE
52 55 TESTS += check_moxi_agent
172 htgram.c
... ... @@ -0,0 +1,172 @@
  1 +/* -*- Mode: C; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */
  2 +/*
  3 + * Copyright 2010 NorthScale, Inc.
  4 + *
  5 + * Licensed under the Apache License, Version 2.0 (the "License");
  6 + * you may not use this file except in compliance with the License.
  7 + * You may obtain a copy of the License at
  8 + *
  9 + * http://www.apache.org/licenses/LICENSE-2.0
  10 + *
  11 + * Unless required by applicable law or agreed to in writing, software
  12 + * distributed under the License is distributed on an "AS IS" BASIS,
  13 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  14 + * See the License for the specific language governing permissions and
  15 + * limitations under the License.
  16 + */
  17 +
  18 +#include <assert.h>
  19 +#include <stdio.h>
  20 +#include <stdlib.h>
  21 +#include <string.h>
  22 +#include <strings.h>
  23 +
  24 +#include <htgram.h>
  25 +
  26 +struct htgram_bin_st {
  27 + int64_t start;
  28 + int64_t width;
  29 + uint64_t count;
  30 +};
  31 +
  32 +struct htgram_st {
  33 + int64_t bin_start;
  34 + int64_t bin_start_width;
  35 + double bin_width_growth;
  36 + size_t num_bins;
  37 +
  38 + struct htgram_bin_st *bins;
  39 +
  40 + uint64_t lt_count; // For data points < the bins.
  41 + uint64_t gt_count; // For data points > the bins.
  42 +
  43 + // For data points > the bins, there may be another
  44 + // histogram instead of using gt_count.
  45 + //
  46 + HTGRAM_HANDLE next;
  47 +};
  48 +
  49 +HTGRAM_HANDLE htgram_mk(int64_t bin_start,
  50 + int64_t bin_start_width,
  51 + double bin_width_growth,
  52 + size_t num_bins,
  53 + HTGRAM_HANDLE next) {
  54 + struct htgram_st *h = calloc(sizeof(struct htgram_st), 1);
  55 + if (h == NULL) {
  56 + return NULL;
  57 + }
  58 +
  59 + h->bin_start = bin_start;
  60 + h->bin_start_width = bin_start_width;
  61 + h->bin_width_growth = bin_width_growth;
  62 + h->num_bins = num_bins;
  63 + h->next = next;
  64 +
  65 + if (num_bins > 0) {
  66 + h->bins = calloc(sizeof(struct htgram_bin_st), num_bins);
  67 + if (h->bins == NULL) {
  68 + free(h);
  69 + return NULL;
  70 + }
  71 +
  72 + int64_t r = bin_start;
  73 + int64_t w = bin_start_width;
  74 + for (size_t i = 0; i < num_bins; i++) {
  75 + h->bins[i].start = r;
  76 + h->bins[i].width = w;
  77 + r = r + w;
  78 + w = w * bin_width_growth;
  79 + }
  80 + }
  81 +
  82 + return h;
  83 +}
  84 +
  85 +void htgram_destroy(HTGRAM_HANDLE h) {
  86 + if (h->bins != NULL) {
  87 + free(h->bins);
  88 + }
  89 + if (h->next != NULL) {
  90 + htgram_destroy(h->next);
  91 + }
  92 + free(h);
  93 +}
  94 +
  95 +int64_t htgram_get_bin_start(HTGRAM_HANDLE h) {
  96 + return h->bin_start;
  97 +}
  98 +
  99 +int64_t htgram_get_bin_start_width(HTGRAM_HANDLE h) {
  100 + return h->bin_start_width;
  101 +}
  102 +
  103 +double htgram_get_bin_width_growth(HTGRAM_HANDLE h) {
  104 + return h->bin_width_growth;
  105 +}
  106 +
  107 +size_t htgram_get_num_bins(HTGRAM_HANDLE h) {
  108 + return h->num_bins;
  109 +}
  110 +
  111 +void htgram_incr(HTGRAM_HANDLE h, int64_t data_point, uint64_t count) {
  112 + if (data_point < h->bin_start) {
  113 + h->lt_count += count;
  114 + return;
  115 + }
  116 +
  117 + for (size_t i = 0; i < h->num_bins; i++) {
  118 + if (data_point < (h->bins[i].start +
  119 + h->bins[i].width)) {
  120 + h->bins[i].count += count;
  121 + return;
  122 + }
  123 + }
  124 +
  125 + if (h->next != NULL) {
  126 + htgram_incr(h->next, data_point, count);
  127 + return;
  128 + }
  129 +
  130 + h->gt_count += count;
  131 +}
  132 +
  133 +bool htgram_get_bin_data(HTGRAM_HANDLE h, int bin_index,
  134 + int64_t *out_bin_start,
  135 + int64_t *out_bin_width,
  136 + uint64_t *out_bin_count) {
  137 + if (bin_index < 0) {
  138 + *out_bin_count = h->lt_count;
  139 + return false;
  140 + }
  141 +
  142 + if (bin_index >= (int) h->num_bins) {
  143 + if (h->next != NULL) {
  144 + return htgram_get_bin_data(h->next, bin_index - h->num_bins,
  145 + out_bin_start,
  146 + out_bin_width,
  147 + out_bin_count);
  148 + }
  149 +
  150 + *out_bin_count = h->gt_count;
  151 + return false;
  152 + }
  153 +
  154 + *out_bin_start = h->bins[bin_index].start;
  155 + *out_bin_width = h->bins[bin_index].width;
  156 + *out_bin_count = h->bins[bin_index].count;
  157 +
  158 + return true;
  159 +}
  160 +
  161 +void htgram_reset(HTGRAM_HANDLE h) {
  162 + for (size_t i = 0; i < h->num_bins; i++) {
  163 + h->bins[i].count = 0;
  164 + }
  165 +
  166 + h->lt_count = 0;
  167 + h->gt_count = 0;
  168 +
  169 + if (h->next != NULL) {
  170 + htgram_reset(h->next);
  171 + }
  172 +}
159 htgram.h
... ... @@ -0,0 +1,159 @@
  1 +/* -*- Mode: C; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */
  2 +/*
  3 + * Copyright 2010 NorthScale, Inc.
  4 + *
  5 + * Licensed under the Apache License, Version 2.0 (the "License");
  6 + * you may not use this file except in compliance with the License.
  7 + * You may obtain a copy of the License at
  8 + *
  9 + * http://www.apache.org/licenses/LICENSE-2.0
  10 + *
  11 + * Unless required by applicable law or agreed to in writing, software
  12 + * distributed under the License is distributed on an "AS IS" BASIS,
  13 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  14 + * See the License for the specific language governing permissions and
  15 + * limitations under the License.
  16 + */
  17 +
  18 +/*! \mainpage htgram
  19 + *
  20 + * \section intro_sec Introduction
  21 + *
  22 + * htgram is a short histogram implementation
  23 + *
  24 + * \section docs_sec API Documentation
  25 + *
  26 + * Jump right into <a href="modules.html">the modules docs</a> to get started.
  27 + */
  28 +
  29 +/**
  30 + * Histogram Utility Library.
  31 + *
  32 + * \defgroup CD Creation and Destruction
  33 + * \defgroup Data Collecting and retrieving stats
  34 + */
  35 +
  36 +#ifndef HTGRAM_H
  37 +#define HTGRAM_H 1
  38 +
  39 +#include <stddef.h>
  40 +#include <stdint.h>
  41 +#include <stdbool.h>
  42 +#include <math.h>
  43 +
  44 +#define HTGRAM_PUBLIC_API
  45 +
  46 +#ifdef __cplusplus
  47 +extern "C" {
  48 +#endif
  49 +
  50 + struct htgram_st;
  51 +
  52 + /**
  53 + * Opaque histogram representation.
  54 + */
  55 + typedef struct htgram_st *HTGRAM_HANDLE;
  56 +
  57 + /**
  58 + * \addtogroup CD
  59 + * @{
  60 + */
  61 +
  62 + /**
  63 + * Create an instance of htgram.
  64 + *
  65 + * htgram_mk(0, 5, 1.0, 3, NULL) means have 3 bins, each of width
  66 + * 5, such as [0, 5), [5, 10), [10, 15).
  67 + *
  68 + * htgram_mk(0, 5, 2.0, 3, NULL) means have 3 bins, but the bin
  69 + * widths should double, such as [0, 5), [5, 15), [15, 35).
  70 + *
  71 + * @param bin_start the first bin starts at this range
  72 + * value, inclusive.
  73 + * @param bin_start_width width of the first bin.
  74 + * @param bin_width_growth grow each bin width by this factor.
  75 + * @param num_bins number of bins.
  76 + * @param next optional chained HTGRAM_HANDLE, which is useful
  77 + * to change bin_width_growth factors; may be NULL.
  78 + */
  79 + HTGRAM_PUBLIC_API
  80 + HTGRAM_HANDLE htgram_mk(int64_t bin_start,
  81 + int64_t bin_start_width,
  82 + double bin_width_growth,
  83 + size_t num_bins,
  84 + HTGRAM_HANDLE next);
  85 +
  86 + /**
  87 + * Destroy a htgram.
  88 + *
  89 + * @param h the htgram handle
  90 + */
  91 + HTGRAM_PUBLIC_API
  92 + void htgram_destroy(HTGRAM_HANDLE h);
  93 +
  94 + /**
  95 + * @}
  96 + */
  97 +
  98 + /**
  99 + * \addtogroup Data
  100 + * @{
  101 + */
  102 +
  103 + /**
  104 + * Get the range start value for the first bin.
  105 + */
  106 + HTGRAM_PUBLIC_API
  107 + int64_t htgram_get_bin_start(HTGRAM_HANDLE h);
  108 +
  109 + /**
  110 + * Get the width of the first bin.
  111 + */
  112 + HTGRAM_PUBLIC_API
  113 + int64_t htgram_get_bin_start_width(HTGRAM_HANDLE h);
  114 +
  115 + /**
  116 + * Get the growth factor for bin widths.
  117 + */
  118 + HTGRAM_PUBLIC_API
  119 + double htgram_get_bin_width_growth(HTGRAM_HANDLE h);
  120 +
  121 + /**
  122 + * Get the total number of bins.
  123 + */
  124 + HTGRAM_PUBLIC_API
  125 + size_t htgram_get_num_bins(HTGRAM_HANDLE h);
  126 +
  127 + /**
  128 + * Add a data_point to the histogram, where the correct bin will
  129 + * be incremented by count. For example, htgram_incr(h, request_latency, 1);
  130 + */
  131 + HTGRAM_PUBLIC_API
  132 + void htgram_incr(HTGRAM_HANDLE h, int64_t data_point, uint64_t count);
  133 +
  134 + /**
  135 + * Retrieves collected data (out_bin_count) for a bin, given a
  136 + * bin_index. The first width bin has bin_index of 0.
  137 + * Returns false if there's no bin at the given bin_index.
  138 + */
  139 + HTGRAM_PUBLIC_API
  140 + bool htgram_get_bin_data(HTGRAM_HANDLE h, int bin_index,
  141 + int64_t *out_bin_start,
  142 + int64_t *out_bin_width,
  143 + uint64_t *out_bin_count);
  144 +
  145 + /**
  146 + * Reset all bin counts to zero.
  147 + */
  148 + HTGRAM_PUBLIC_API
  149 + void htgram_reset(HTGRAM_HANDLE h);
  150 +
  151 + /**
  152 + * @}
  153 + */
  154 +
  155 +#ifdef __cplusplus
  156 +}
  157 +#endif
  158 +
  159 +#endif
137 htgram_test.c
... ... @@ -0,0 +1,137 @@
  1 +/* -*- Mode: C; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */
  2 +
  3 +#include "config.h"
  4 +#include <assert.h>
  5 +#include <stdio.h>
  6 +#include <stdlib.h>
  7 +#include <string.h>
  8 +
  9 +#include <htgram.h>
  10 +
  11 +static void testSimple(void) {
  12 + HTGRAM_HANDLE h0;
  13 +
  14 + int64_t start;
  15 + int64_t width;
  16 + uint64_t count;
  17 +
  18 + h0 = htgram_mk(0, 5, 1.0, 3, NULL);
  19 + assert(h0 != NULL);
  20 + assert(htgram_get_bin_start(h0) == 0);
  21 + assert(htgram_get_bin_start_width(h0) == 5);
  22 + assert(htgram_get_bin_width_growth(h0) == 1.0);
  23 + assert(htgram_get_num_bins(h0) == 3);
  24 +
  25 + start = width = count = 123;
  26 + assert(htgram_get_bin_data(h0, -1, &start, &width, &count) == false);
  27 + assert(start == 123);
  28 + assert(width == 123);
  29 + assert(count == 0);
  30 +
  31 + int i;
  32 + for (i = 0; i < (int) htgram_get_num_bins(h0); i++) {
  33 + start = width = count = 123;
  34 + assert(htgram_get_bin_data(h0, i, &start, &width, &count) == true);
  35 + assert(start == i * 5);
  36 + assert(width == 5);
  37 + assert(count == 0);
  38 +
  39 + htgram_incr(h0, (i * 5) + 0, 1);
  40 + htgram_incr(h0, (i * 5) + 1, 2);
  41 + htgram_incr(h0, (i * 5) + 2, 3);
  42 + htgram_incr(h0, (i * 5) + 5, 0);
  43 +
  44 + start = width = count = 123;
  45 + assert(htgram_get_bin_data(h0, i, &start, &width, &count) == true);
  46 + assert(start == i * 5);
  47 + assert(width == 5);
  48 + assert(count == 6);
  49 + }
  50 +
  51 + start = width = count = 123;
  52 + assert(htgram_get_bin_data(h0, i, &start, &width, &count) == false);
  53 + assert(start == 123);
  54 + assert(width == 123);
  55 + assert(count == 0);
  56 +
  57 + htgram_reset(h0);
  58 +
  59 + start = width = count = 123;
  60 + assert(htgram_get_bin_data(h0, -1, &start, &width, &count) == false);
  61 + assert(start == 123);
  62 + assert(width == 123);
  63 + assert(count == 0);
  64 +
  65 + for (i = 0; i < (int) htgram_get_num_bins(h0); i++) {
  66 + start = width = count = 123;
  67 + assert(htgram_get_bin_data(h0, i, &start, &width, &count) == true);
  68 + assert(start == i * 5);
  69 + assert(width == 5);
  70 + assert(count == 0);
  71 + }
  72 +
  73 + start = width = count = 123;
  74 + assert(htgram_get_bin_data(h0, i, &start, &width, &count) == false);
  75 + assert(start == 123);
  76 + assert(width == 123);
  77 + assert(count == 0);
  78 +
  79 + htgram_incr(h0, -10000, 111);
  80 + htgram_incr(h0, 10000, 222);
  81 +
  82 + start = width = count = 123;
  83 + assert(htgram_get_bin_data(h0, -1, &start, &width, &count) == false);
  84 + assert(start == 123);
  85 + assert(width == 123);
  86 + assert(count == 111);
  87 +
  88 + start = width = count = 123;
  89 + assert(htgram_get_bin_data(h0, i, &start, &width, &count) == false);
  90 + assert(start == 123);
  91 + assert(width == 123);
  92 + assert(count == 222);
  93 +
  94 + htgram_destroy(h0);
  95 +}
  96 +
  97 +static void testChained(void) {
  98 + HTGRAM_HANDLE h0, h1;
  99 +
  100 + int64_t start;
  101 + int64_t width;
  102 + uint64_t count;
  103 +
  104 + // Have 200 bins from [0 to 2000), with bin widths of 10.
  105 + // Have 36 bins from 2000 onwards, with bin width growing at 1.5, chained.
  106 + h1 = htgram_mk(2000, 10, 1.5, 36, NULL);
  107 + h0 = htgram_mk(0, 10, 1.0, 200, h1);
  108 +
  109 + int i;
  110 + for (i = 0; i < (int) (htgram_get_num_bins(h0) + htgram_get_num_bins(h1)); i++) {
  111 + assert(htgram_get_bin_data(h0, i, &start, &width, &count) == true);
  112 + // printf("%d %d %d %d\n", i, start, width, count);
  113 + }
  114 +
  115 + htgram_incr(h0, 28000000, 111);
  116 +
  117 + assert(htgram_get_bin_data(h0, 200 + 36 - 1, &start, &width, &count) == true);
  118 + assert(start == 27692301);
  119 + assert(width == 13845150);
  120 + assert(count == 111);
  121 +
  122 + htgram_reset(h0);
  123 +
  124 + assert(htgram_get_bin_data(h0, 200 + 36 - 1, &start, &width, &count) == true);
  125 + assert(start == 27692301);
  126 + assert(width == 13845150);
  127 + assert(count == 0);
  128 +}
  129 +
  130 +int main(void) {
  131 + testSimple();
  132 + testChained();
  133 +
  134 + return 0;
  135 +}
  136 +
  137 +

0 comments on commit b0c66f1

Please sign in to comment.
Something went wrong with that request. Please try again.