Merge pull request #100 from filipecosta90/hdr.hist

Enable full percentile spectrum analysis on latency.

.gitignore

@@ -2,9 +2,11 @@ aclocal.m4
 autom4te.cache
 config.guess
 config.h.in
+config.h.in~
 config.h
 config.sub
 configure
+compile
 depcomp
 install-sh
 ltmain.sh
@@ -19,6 +21,11 @@ config.log
 config.status
 libtool
 stamp-h1
-.idea
-.vscode
+*.DS_Store
+.vscode/*
+.idea/*
+
+# memtier outputs
+*.hgrm
+*.txt
 __pycache__

Makefile.am

@@ -40,7 +40,11 @@ memtier_benchmark_SOURCES = \
 	obj_gen.cpp obj_gen.h \
 	item.cpp item.h \
 	file_io.cpp file_io.h \
-	config_types.cpp config_types.h
+	config_types.cpp config_types.h \
+	deps/hdr_histogram/hdr_histogram_log.c deps/hdr_histogram/hdr_histogram_log.h deps/hdr_histogram/byteorder.h \
+	deps/hdr_histogram/hdr_histogram.c deps/hdr_histogram/hdr_histogram.h \
+	deps/hdr_histogram/hdr_time.c deps/hdr_histogram/hdr_time.h deps/hdr_histogram/hdr_encoding.c deps/hdr_histogram/hdr_encoding.h
+
 memtier_benchmark_LDADD = \
 	$(LIBEVENT_LIBS) \
 	$(LIBEVENT_OPENSSL_LIBS) \

README.md

@@ -156,3 +156,24 @@ Also, the ratio and the key generator is per client (and not connection).
 In this case, setting the ratio to 1:1 does not guarantee 100% hits because
 the keys spread to different connections/nodes.
 
+### Full latency spectrum analysis
+
+For distributions that are non-normal, such as the latency, many “basic rules” of normally distributed statistics are violated.  Instead of computing just the mean, which tries to express the whole distribution in a single result, we can use a sampling of the distribution at intervals -- percentiles, which tell you how many requests actually would experience that delay. 
+
+
+When used for normally distributed data, the samples are usually taken at regular intervals. However, since the data does not obey to a normal distribution it would be very expensive to keep equally spaced intervals of latency records while enabling large value ranges. We can apply algorithms that can calculate a good approximation of percentiles at minimal CPU and memory cost, such as [t-digest](https://github.com/tdunning/t-digest) or [HdrHistogram](https://github.com/HdrHistogram/HdrHistogram_c). On memtier_benchmark we’ve decided to use the  HdrHistogram due to its low memory footprint, high precision, zero allocation during the benchmark and constant access time.
+
+
+By default Memtier will output the 50th, 99th, and 99.9th percentiles. They are the latency thresholds at which 50%, 99%, and 99.9% of commands are faster than that particular presented value. 
+To output different percentiles you should use the --print-percentiles option followed by the comma separated list of values ( example: `--print-percentiles 90,99,99.9,99.99` ).
+
+#### Saving the full latency spectrum
+To save the full latencies you should use the --hdr-file-prefix option followed by the prefix name you wish the filenames to have. 
+Each distinct command will be saved into two different files - one in .txt (textual format) and another in .hgrm (HistogramLogProcessor format).
+The textual format can be hard to analyze solely, but you can use an [online formatter](http://hdrhistogram.github.io/HdrHistogram/plotFiles.html) to generate visual histograms from it. The .hgrm format will be later added as input to Redislabs [mbdirector](https://github.com/redislabs/mbdirector) to enable visualization of time-domain results.
+
+Sample Visual Feel of the full latency spectrum using an [online formatter](http://hdrhistogram.github.io/HdrHistogram/plotFiles.html):
+![alt text][sample_visual_histogram]
+
+
+[sample_visual_histogram]: ./docs/sample_visual_histogram.png "Sample Full Latency Spectrum Histogram"

client.cpp

@@ -56,6 +56,7 @@ bool client::setup_client(benchmark_config *config, abstract_protocol *protocol,
 {
     m_config = config;
     assert(m_config != NULL);
+    unsigned long long total_num_of_clients = config->clients*config->threads;
 
     // create main connection
     shard_connection* conn = new shard_connection(m_connections.size(), this, m_config, m_event_base, protocol);
@@ -74,7 +75,6 @@ bool client::setup_client(benchmark_config *config, abstract_protocol *protocol,
     // Parallel key-pattern determined according to the first command
     if ((config->arbitrary_commands->is_defined() && config->arbitrary_commands->at(0).key_pattern == 'P') ||
         (config->key_pattern[key_pattern_set]=='P')) {
-        unsigned long long total_num_of_clients = config->clients*config->threads;
         unsigned long long client_index = config->next_client_idx % total_num_of_clients;
 
         unsigned long long range = (config->key_maximum - config->key_minimum)/total_num_of_clients + 1;

client.h

@@ -61,17 +61,17 @@ class client : public connections_manager {
     object_generator* m_obj_gen;
     run_stats m_stats;
 
-    unsigned long long m_reqs_processed;      // requests processed (responses received)
-    unsigned long long m_reqs_generated;      // requests generated (wait for responses)
-    unsigned int m_set_ratio_count;     // number of sets counter (overlaps on ratio)
-    unsigned int m_get_ratio_count;     // number of gets counter (overlaps on ratio)
+    unsigned long long m_reqs_processed;          // requests processed (responses received)
+    unsigned long long m_reqs_generated;          // requests generated (wait for responses)
+    unsigned int m_set_ratio_count;               // number of sets counter (overlaps on ratio)
+    unsigned int m_get_ratio_count;               // number of gets counter (overlaps on ratio)
     unsigned int m_arbitrary_command_ratio_count; // number of arbitrary commands counter (overlaps on ratio)
-    unsigned int m_executed_command_index; // current arbitrary command executed
+    unsigned int m_executed_command_index;        // current arbitrary command executed
 
-    unsigned long long m_tot_set_ops;        // Total number of SET ops
-    unsigned long long m_tot_wait_ops;       // Total number of WAIT ops
+    unsigned long long m_tot_set_ops;             // Total number of SET ops
+    unsigned long long m_tot_wait_ops;            // Total number of WAIT ops
 
-    keylist *m_keylist;                 // used to construct multi commands
+    keylist *m_keylist;                           // used to construct multi commands
 
 public:
     client(client_group* group);

config_types.cpp

@@ -35,6 +35,7 @@
 #include <netdb.h>
 
 #include <string>
+#include <iostream>
 #include <stdexcept>
 #include <climits>
 #include <algorithm>
@@ -87,6 +88,34 @@ config_ratio::config_ratio(const char *ratio_str) :
     }
 }
 
+config_quantiles::config_quantiles(){
+
+}
+
+config_quantiles::config_quantiles(const char *str)
+{
+    assert(str != NULL);
+
+    do {
+        float quantile;
+        char *p = NULL;
+        quantile = strtof(str, &p);
+        if (!p || (*p != ',' && *p != '\0')) {
+            quantile_list.clear();
+            return;
+        }
+        str = p;
+        if (*str) str++;
+        quantile_list.push_back(quantile);
+    } while (*str);
+}
+
+bool config_quantiles::is_defined(void)
+{
+    return quantile_list.size() > 0;
+}
+
+
 config_weight_list::config_weight_list() :
     next_size_weight(0)
 {

config_types.h

@@ -46,6 +46,15 @@ struct config_ratio {
     bool is_defined(void) { return (a > 0 || b > 0); }
 };
 
+struct config_quantiles {
+    std::vector<float> quantile_list;
+    config_quantiles();
+    config_quantiles(const char *ratio_str);
+    bool is_defined(void);
+    inline std::vector<float>::iterator begin()  { return quantile_list.begin(); }
+    inline std::vector<float>::iterator end()  { return quantile_list.end(); }
+};
+
 struct config_weight_list {
     struct weight_item {
         unsigned int size;

deps/hdr_histogram/COPYING.txt

@@ -0,0 +1,121 @@
+Creative Commons Legal Code
+
+CC0 1.0 Universal
+
+    CREATIVE COMMONS CORPORATION IS NOT A LAW FIRM AND DOES NOT PROVIDE
+    LEGAL SERVICES. DISTRIBUTION OF THIS DOCUMENT DOES NOT CREATE AN
+    ATTORNEY-CLIENT RELATIONSHIP. CREATIVE COMMONS PROVIDES THIS
+    INFORMATION ON AN "AS-IS" BASIS. CREATIVE COMMONS MAKES NO WARRANTIES
+    REGARDING THE USE OF THIS DOCUMENT OR THE INFORMATION OR WORKS
+    PROVIDED HEREUNDER, AND DISCLAIMS LIABILITY FOR DAMAGES RESULTING FROM
+    THE USE OF THIS DOCUMENT OR THE INFORMATION OR WORKS PROVIDED
+    HEREUNDER.
+
+Statement of Purpose
+
+The laws of most jurisdictions throughout the world automatically confer
+exclusive Copyright and Related Rights (defined below) upon the creator
+and subsequent owner(s) (each and all, an "owner") of an original work of
+authorship and/or a database (each, a "Work").
+
+Certain owners wish to permanently relinquish those rights to a Work for
+the purpose of contributing to a commons of creative, cultural and
+scientific works ("Commons") that the public can reliably and without fear
+of later claims of infringement build upon, modify, incorporate in other
+works, reuse and redistribute as freely as possible in any form whatsoever
+and for any purposes, including without limitation commercial purposes.
+These owners may contribute to the Commons to promote the ideal of a free
+culture and the further production of creative, cultural and scientific
+works, or to gain reputation or greater distribution for their Work in
+part through the use and efforts of others.
+
+For these and/or other purposes and motivations, and without any
+expectation of additional consideration or compensation, the person
+associating CC0 with a Work (the "Affirmer"), to the extent that he or she
+is an owner of Copyright and Related Rights in the Work, voluntarily
+elects to apply CC0 to the Work and publicly distribute the Work under its
+terms, with knowledge of his or her Copyright and Related Rights in the
+Work and the meaning and intended legal effect of CC0 on those rights.
+
+1. Copyright and Related Rights. A Work made available under CC0 may be
+protected by copyright and related or neighboring rights ("Copyright and
+Related Rights"). Copyright and Related Rights include, but are not
+limited to, the following:
+
+  i. the right to reproduce, adapt, distribute, perform, display,
+     communicate, and translate a Work;
+ ii. moral rights retained by the original author(s) and/or performer(s);
+iii. publicity and privacy rights pertaining to a person's image or
+     likeness depicted in a Work;
+ iv. rights protecting against unfair competition in regards to a Work,
+     subject to the limitations in paragraph 4(a), below;
+  v. rights protecting the extraction, dissemination, use and reuse of data
+     in a Work;
+ vi. database rights (such as those arising under Directive 96/9/EC of the
+     European Parliament and of the Council of 11 March 1996 on the legal
+     protection of databases, and under any national implementation
+     thereof, including any amended or successor version of such
+     directive); and
+vii. other similar, equivalent or corresponding rights throughout the
+     world based on applicable law or treaty, and any national
+     implementations thereof.
+
+2. Waiver. To the greatest extent permitted by, but not in contravention
+of, applicable law, Affirmer hereby overtly, fully, permanently,
+irrevocably and unconditionally waives, abandons, and surrenders all of
+Affirmer's Copyright and Related Rights and associated claims and causes
+of action, whether now known or unknown (including existing as well as
+future claims and causes of action), in the Work (i) in all territories
+worldwide, (ii) for the maximum duration provided by applicable law or
+treaty (including future time extensions), (iii) in any current or future
+medium and for any number of copies, and (iv) for any purpose whatsoever,
+including without limitation commercial, advertising or promotional
+purposes (the "Waiver"). Affirmer makes the Waiver for the benefit of each
+member of the public at large and to the detriment of Affirmer's heirs and
+successors, fully intending that such Waiver shall not be subject to
+revocation, rescission, cancellation, termination, or any other legal or
+equitable action to disrupt the quiet enjoyment of the Work by the public
+as contemplated by Affirmer's express Statement of Purpose.
+
+3. Public License Fallback. Should any part of the Waiver for any reason
+be judged legally invalid or ineffective under applicable law, then the
+Waiver shall be preserved to the maximum extent permitted taking into
+account Affirmer's express Statement of Purpose. In addition, to the
+extent the Waiver is so judged Affirmer hereby grants to each affected
+person a royalty-free, non transferable, non sublicensable, non exclusive,
+irrevocable and unconditional license to exercise Affirmer's Copyright and
+Related Rights in the Work (i) in all territories worldwide, (ii) for the
+maximum duration provided by applicable law or treaty (including future
+time extensions), (iii) in any current or future medium and for any number
+of copies, and (iv) for any purpose whatsoever, including without
+limitation commercial, advertising or promotional purposes (the
+"License"). The License shall be deemed effective as of the date CC0 was
+applied by Affirmer to the Work. Should any part of the License for any
+reason be judged legally invalid or ineffective under applicable law, such
+partial invalidity or ineffectiveness shall not invalidate the remainder
+of the License, and in such case Affirmer hereby affirms that he or she
+will not (i) exercise any of his or her remaining Copyright and Related
+Rights in the Work or (ii) assert any associated claims and causes of
+action with respect to the Work, in either case contrary to Affirmer's
+express Statement of Purpose.
+
+4. Limitations and Disclaimers.
+
+ a. No trademark or patent rights held by Affirmer are waived, abandoned,
+    surrendered, licensed or otherwise affected by this document.
+ b. Affirmer offers the Work as-is and makes no representations or
+    warranties of any kind concerning the Work, express, implied,
+    statutory or otherwise, including without limitation warranties of
+    title, merchantability, fitness for a particular purpose, non
+    infringement, or the absence of latent or other defects, accuracy, or
+    the present or absence of errors, whether or not discoverable, all to
+    the greatest extent permissible under applicable law.
+ c. Affirmer disclaims responsibility for clearing rights of other persons
+    that may apply to the Work or any use thereof, including without
+    limitation any person's Copyright and Related Rights in the Work.
+    Further, Affirmer disclaims responsibility for obtaining any necessary
+    consents, permissions or other rights required for any use of the
+    Work.
+ d. Affirmer understands and acknowledges that Creative Commons is not a
+    party to this document and has no duty or obligation with respect to
+    this CC0 or use of the Work.

deps/hdr_histogram/LICENSE.txt

@@ -0,0 +1,41 @@
+The code in this repository code was Written by Gil Tene, Michael Barker,
+and Matt Warren, and released to the public domain, as explained at
+http://creativecommons.org/publicdomain/zero/1.0/
+
+For users of this code who wish to consume it under the "BSD" license
+rather than under the public domain or CC0 contribution text mentioned
+above, the code found under this directory is *also* provided under the
+following license (commonly referred to as the BSD 2-Clause License). This
+license does not detract from the above stated release of the code into
+the public domain, and simply represents an additional license granted by
+the Author.
+
+-----------------------------------------------------------------------------
+** Beginning of "BSD 2-Clause License" text. **
+
+ Copyright (c) 2012, 2013, 2014 Gil Tene
+ Copyright (c) 2014 Michael Barker
+ Copyright (c) 2014 Matt Warren
+ All rights reserved.
+
+ Redistribution and use in source and binary forms, with or without
+ modification, are permitted provided that the following conditions are met:
+
+ 1. Redistributions of source code must retain the above copyright notice,
+    this list of conditions and the following disclaimer.
+
+ 2. Redistributions in binary form must reproduce the above copyright notice,
+    this list of conditions and the following disclaimer in the documentation
+    and/or other materials provided with the distribution.
+
+ THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
+ LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
+ THE POSSIBILITY OF SUCH DAMAGE.

deps/hdr_histogram/README.md

@@ -0,0 +1,10 @@
+HdrHistogram_c v0.11.0
+
+----------------------------------------------
+
+This port contains a subset of the 'C' version of High Dynamic Range (HDR) Histogram available at [github.com/HdrHistogram/HdrHistogram_c](https://github.com/HdrHistogram/HdrHistogram_c).
+
+
+The code present on `hdr_histogram.c`, `hdr_histogram.h`, and `hdr_atomic.c` was Written by Gil Tene, Michael Barker,
+and Matt Warren, and released to the public domain, as explained at
+http://creativecommons.org/publicdomain/zero/1.0/.