diff --git a/deps/histogram/src/hdr_atomic.h b/deps/histogram/src/hdr_atomic.h new file mode 100644 index 00000000000000..ae1056a83612af --- /dev/null +++ b/deps/histogram/src/hdr_atomic.h @@ -0,0 +1,146 @@ +/** + * hdr_atomic.h + * Written by Philip Orwig and released to the public domain, + * as explained at http://creativecommons.org/publicdomain/zero/1.0/ + */ + +#ifndef HDR_ATOMIC_H__ +#define HDR_ATOMIC_H__ + + +#if defined(_MSC_VER) + +#include +#include +#include + +static void __inline * hdr_atomic_load_pointer(void** pointer) +{ + _ReadBarrier(); + return *pointer; +} + +static void hdr_atomic_store_pointer(void** pointer, void* value) +{ + _WriteBarrier(); + *pointer = value; +} + +static int64_t __inline hdr_atomic_load_64(int64_t* field) +{ + _ReadBarrier(); + return *field; +} + +static void __inline hdr_atomic_store_64(int64_t* field, int64_t value) +{ + _WriteBarrier(); + *field = value; +} + +static int64_t __inline hdr_atomic_exchange_64(volatile int64_t* field, int64_t value) +{ +#if defined(_WIN64) + return _InterlockedExchange64(field, value); +#else + int64_t comparand; + int64_t initial_value = *field; + do + { + comparand = initial_value; + initial_value = _InterlockedCompareExchange64(field, value, comparand); + } + while (comparand != initial_value); + + return initial_value; +#endif +} + +static int64_t __inline hdr_atomic_add_fetch_64(volatile int64_t* field, int64_t value) +{ +#if defined(_WIN64) + return _InterlockedExchangeAdd64(field, value) + value; +#else + int64_t comparand; + int64_t initial_value = *field; + do + { + comparand = initial_value; + initial_value = _InterlockedCompareExchange64(field, comparand + value, comparand); + } + while (comparand != initial_value); + + return initial_value + value; +#endif +} + +static bool __inline hdr_atomic_compare_exchange_64(volatile int64_t* field, int64_t* expected, int64_t desired) +{ + return *expected == _InterlockedCompareExchange64(field, desired, *expected); +} + +#elif defined(__ATOMIC_SEQ_CST) + +#define hdr_atomic_load_pointer(x) __atomic_load_n(x, __ATOMIC_SEQ_CST) +#define hdr_atomic_store_pointer(f,v) __atomic_store_n(f,v, __ATOMIC_SEQ_CST) +#define hdr_atomic_load_64(x) __atomic_load_n(x, __ATOMIC_SEQ_CST) +#define hdr_atomic_store_64(f,v) __atomic_store_n(f,v, __ATOMIC_SEQ_CST) +#define hdr_atomic_exchange_64(f,i) __atomic_exchange_n(f,i, __ATOMIC_SEQ_CST) +#define hdr_atomic_add_fetch_64(field, value) __atomic_add_fetch(field, value, __ATOMIC_SEQ_CST) +#define hdr_atomic_compare_exchange_64(field, expected, desired) __atomic_compare_exchange_n(field, expected, desired, false, __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST) + +#elif defined(__x86_64__) + +#include +#include + +static inline void* hdr_atomic_load_pointer(void** pointer) +{ + void* p = *pointer; + asm volatile ("" ::: "memory"); + return p; +} + +static inline void hdr_atomic_store_pointer(void** pointer, void* value) +{ + asm volatile ("lock; xchgq %0, %1" : "+q" (value), "+m" (*pointer)); +} + +static inline int64_t hdr_atomic_load_64(int64_t* field) +{ + int64_t i = *field; + asm volatile ("" ::: "memory"); + return i; +} + +static inline void hdr_atomic_store_64(int64_t* field, int64_t value) +{ + asm volatile ("lock; xchgq %0, %1" : "+q" (value), "+m" (*field)); +} + +static inline int64_t hdr_atomic_exchange_64(volatile int64_t* field, int64_t value) +{ + int64_t result = 0; + asm volatile ("lock; xchgq %1, %2" : "=r" (result), "+q" (value), "+m" (*field)); + return result; +} + +static inline int64_t hdr_atomic_add_fetch_64(volatile int64_t* field, int64_t value) +{ + return __sync_add_and_fetch(field, value); +} + +static inline bool hdr_atomic_compare_exchange_64(volatile int64_t* field, int64_t* expected, int64_t desired) +{ + int64_t original; + asm volatile( "lock; cmpxchgq %2, %1" : "=a"(original), "+m"(*field) : "q"(desired), "0"(*expected)); + return original == *expected; +} + +#else + +#error "Unable to determine atomic operations for your platform" + +#endif + +#endif /* HDR_ATOMIC_H__ */ diff --git a/deps/histogram/src/hdr_histogram.c b/deps/histogram/src/hdr_histogram.c index d9565b802e3564..4bcfbeb049bf59 100644 --- a/deps/histogram/src/hdr_histogram.c +++ b/deps/histogram/src/hdr_histogram.c @@ -7,7 +7,6 @@ #include #include #include -#include #include #include #include @@ -16,6 +15,7 @@ #include "hdr_histogram.h" #include "hdr_tests.h" +#include "hdr_atomic.h" /* ###### ####### ## ## ## ## ######## ###### */ /* ## ## ## ## ## ## ### ## ## ## ## */ @@ -66,12 +66,49 @@ static void counts_inc_normalised( h->total_count += value; } +static void counts_inc_normalised_atomic( + struct hdr_histogram* h, int32_t index, int64_t value) +{ + int32_t normalised_index = normalize_index(h, index); + + hdr_atomic_add_fetch_64(&h->counts[normalised_index], value); + hdr_atomic_add_fetch_64(&h->total_count, value); +} + static void update_min_max(struct hdr_histogram* h, int64_t value) { h->min_value = (value < h->min_value && value != 0) ? value : h->min_value; h->max_value = (value > h->max_value) ? value : h->max_value; } +static void update_min_max_atomic(struct hdr_histogram* h, int64_t value) +{ + int64_t current_min_value; + int64_t current_max_value; + do + { + current_min_value = hdr_atomic_load_64(&h->min_value); + + if (0 == value || current_min_value <= value) + { + break; + } + } + while (!hdr_atomic_compare_exchange_64(&h->min_value, ¤t_min_value, value)); + + do + { + current_max_value = hdr_atomic_load_64(&h->max_value); + + if (value <= current_max_value) + { + break; + } + } + while (!hdr_atomic_compare_exchange_64(&h->max_value, ¤t_max_value, value)); +} + + /* ## ## ######## #### ## #### ######## ## ## */ /* ## ## ## ## ## ## ## ## ## */ /* ## ## ## ## ## ## ## #### */ @@ -91,33 +128,40 @@ static int64_t power(int64_t base, int64_t exp) } #if defined(_MSC_VER) -# if defined(_WIN64) -# pragma intrinsic(_BitScanReverse64) -# else -# pragma intrinsic(_BitScanReverse) -# endif +# if defined(_WIN64) +# pragma intrinsic(_BitScanReverse64) +# else +# pragma intrinsic(_BitScanReverse) +# endif #endif -static int32_t get_bucket_index(const struct hdr_histogram* h, int64_t value) +static int32_t count_leading_zeros_64(int64_t value) { #if defined(_MSC_VER) uint32_t leading_zero = 0; - int64_t masked_value = value | h->sub_bucket_mask; -# if defined(_WIN64) - _BitScanReverse64(&leading_zero, masked_value); -# else - uint32_t high = masked_value >> 32; - if (_BitScanReverse(&leading_zero, high)) { - leading_zero += 32; - } else { - uint32_t low = masked_value & 0x00000000FFFFFFFF; - _BitScanReverse(&leading_zero, low); +#if defined(_WIN64) + _BitScanReverse64(&leading_zero, value); +#else + uint32_t high = value >> 32; + if (_BitScanReverse(&leading_zero, high)) + { + leading_zero += 32; + } + else + { + uint32_t low = value & 0x00000000FFFFFFFF; + _BitScanReverse(&leading_zero, low); } -# endif - int32_t pow2ceiling = 64 - (63 - leading_zero); /* smallest power of 2 containing value */ +#endif + return 63 - leading_zero; /* smallest power of 2 containing value */ #else - int32_t pow2ceiling = 64 - __builtin_clzll(value | h->sub_bucket_mask); /* smallest power of 2 containing value */ + return __builtin_clzll(value); /* smallest power of 2 containing value */ #endif +} + +static int32_t get_bucket_index(const struct hdr_histogram* h, int64_t value) +{ + int32_t pow2ceiling = 64 - count_leading_zeros_64(value | h->sub_bucket_mask); /* smallest power of 2 containing value */ return pow2ceiling - h->unit_magnitude - (h->sub_bucket_half_count_magnitude + 1); } @@ -283,11 +327,8 @@ int hdr_calculate_bucket_config( int64_t largest_value_with_single_unit_resolution; if (lowest_trackable_value < 1 || - significant_figures < 1 || 5 < significant_figures) - { - return EINVAL; - } - else if (lowest_trackable_value * 2 > highest_trackable_value) + significant_figures < 1 || 5 < significant_figures || + lowest_trackable_value * 2 > highest_trackable_value) { return EINVAL; } @@ -300,8 +341,13 @@ int hdr_calculate_bucket_config( sub_bucket_count_magnitude = (int32_t) ceil(log((double)largest_value_with_single_unit_resolution) / log(2)); cfg->sub_bucket_half_count_magnitude = ((sub_bucket_count_magnitude > 1) ? sub_bucket_count_magnitude : 1) - 1; - cfg->unit_magnitude = (int32_t) floor(log((double)lowest_trackable_value) / log(2)); + double unit_magnitude = log((double)lowest_trackable_value) / log(2); + if (INT32_MAX < unit_magnitude) + { + return EINVAL; + } + cfg->unit_magnitude = (int32_t) unit_magnitude; cfg->sub_bucket_count = (int32_t) pow(2, (cfg->sub_bucket_half_count_magnitude + 1)); cfg->sub_bucket_half_count = cfg->sub_bucket_count / 2; cfg->sub_bucket_mask = ((int64_t) cfg->sub_bucket_count - 1) << cfg->unit_magnitude; @@ -352,11 +398,16 @@ int hdr_init( return r; } - counts = calloc((size_t) cfg.counts_len, sizeof(int64_t)); - histogram = calloc(1, sizeof(struct hdr_histogram)); + counts = (int64_t*) calloc((size_t) cfg.counts_len, sizeof(int64_t)); + if (!counts) + { + return ENOMEM; + } - if (!counts || !histogram) + histogram = (struct hdr_histogram*) calloc(1, sizeof(struct hdr_histogram)); + if (!histogram) { + free(counts); return ENOMEM; } @@ -370,8 +421,10 @@ int hdr_init( void hdr_close(struct hdr_histogram* h) { - free(h->counts); - free(h); + if (h) { + free(h->counts); + free(h); + } } int hdr_alloc(int64_t highest_trackable_value, int significant_figures, struct hdr_histogram** result) @@ -407,6 +460,11 @@ bool hdr_record_value(struct hdr_histogram* h, int64_t value) return hdr_record_values(h, value, 1); } +bool hdr_record_value_atomic(struct hdr_histogram* h, int64_t value) +{ + return hdr_record_values_atomic(h, value, 1); +} + bool hdr_record_values(struct hdr_histogram* h, int64_t value, int64_t count) { int32_t counts_index; @@ -429,11 +487,37 @@ bool hdr_record_values(struct hdr_histogram* h, int64_t value, int64_t count) return true; } +bool hdr_record_values_atomic(struct hdr_histogram* h, int64_t value, int64_t count) +{ + int32_t counts_index; + + if (value < 0) + { + return false; + } + + counts_index = counts_index_for(h, value); + + if (counts_index < 0 || h->counts_len <= counts_index) + { + return false; + } + + counts_inc_normalised_atomic(h, counts_index, count); + update_min_max_atomic(h, value); + + return true; +} + bool hdr_record_corrected_value(struct hdr_histogram* h, int64_t value, int64_t expected_interval) { return hdr_record_corrected_values(h, value, 1, expected_interval); } +bool hdr_record_corrected_value_atomic(struct hdr_histogram* h, int64_t value, int64_t expected_interval) +{ + return hdr_record_corrected_values_atomic(h, value, 1, expected_interval); +} bool hdr_record_corrected_values(struct hdr_histogram* h, int64_t value, int64_t count, int64_t expected_interval) { @@ -461,6 +545,32 @@ bool hdr_record_corrected_values(struct hdr_histogram* h, int64_t value, int64_t return true; } +bool hdr_record_corrected_values_atomic(struct hdr_histogram* h, int64_t value, int64_t count, int64_t expected_interval) +{ + int64_t missing_value; + + if (!hdr_record_values_atomic(h, value, count)) + { + return false; + } + + if (expected_interval <= 0 || value <= expected_interval) + { + return true; + } + + missing_value = value - expected_interval; + for (; missing_value >= expected_interval; missing_value -= expected_interval) + { + if (!hdr_record_values_atomic(h, missing_value, count)) + { + return false; + } + } + + return true; +} + int64_t hdr_add(struct hdr_histogram* h, const struct hdr_histogram* from) { struct hdr_iter iter; @@ -672,7 +782,7 @@ static bool next_value_greater_than_reporting_level_upper_bound( return peek_next_value_from_index(iter) > reporting_level_upper_bound; } -static bool _basic_iter_next(struct hdr_iter *iter) +static bool basic_iter_next(struct hdr_iter *iter) { if (!has_next(iter) || iter->counts_index >= iter->h->counts_len) { @@ -684,19 +794,19 @@ static bool _basic_iter_next(struct hdr_iter *iter) return true; } -static void _update_iterated_values(struct hdr_iter* iter, int64_t new_value_iterated_to) +static void update_iterated_values(struct hdr_iter* iter, int64_t new_value_iterated_to) { iter->value_iterated_from = iter->value_iterated_to; iter->value_iterated_to = new_value_iterated_to; } -static bool _all_values_iter_next(struct hdr_iter* iter) +static bool all_values_iter_next(struct hdr_iter* iter) { bool result = move_next(iter); if (result) { - _update_iterated_values(iter, iter->value); + update_iterated_values(iter, iter->value); } return result; @@ -715,7 +825,7 @@ void hdr_iter_init(struct hdr_iter* iter, const struct hdr_histogram* h) iter->value_iterated_from = 0; iter->value_iterated_to = 0; - iter->_next_fp = _all_values_iter_next; + iter->_next_fp = all_values_iter_next; } bool hdr_iter_next(struct hdr_iter* iter) @@ -731,7 +841,7 @@ bool hdr_iter_next(struct hdr_iter* iter) /* ## ## ## ## ## ## ## ## ### ## ## ## ## ## ## */ /* ## ######## ## ## ###### ######## ## ## ## #### ######## ######## ###### */ -static bool _percentile_iter_next(struct hdr_iter* iter) +static bool percentile_iter_next(struct hdr_iter* iter) { int64_t temp, half_distance, percentile_reporting_ticks; @@ -750,7 +860,7 @@ static bool _percentile_iter_next(struct hdr_iter* iter) return true; } - if (iter->counts_index == -1 && !_basic_iter_next(iter)) + if (iter->counts_index == -1 && !basic_iter_next(iter)) { return false; } @@ -761,7 +871,7 @@ static bool _percentile_iter_next(struct hdr_iter* iter) if (iter->count != 0 && percentiles->percentile_to_iterate_to <= current_percentile) { - _update_iterated_values(iter, highest_equivalent_value(iter->h, iter->value)); + update_iterated_values(iter, highest_equivalent_value(iter->h, iter->value)); percentiles->percentile = percentiles->percentile_to_iterate_to; temp = (int64_t)(log(100 / (100.0 - (percentiles->percentile_to_iterate_to))) / log(2)) + 1; @@ -772,7 +882,7 @@ static bool _percentile_iter_next(struct hdr_iter* iter) return true; } } - while (_basic_iter_next(iter)); + while (basic_iter_next(iter)); return true; } @@ -788,7 +898,7 @@ void hdr_iter_percentile_init(struct hdr_iter* iter, const struct hdr_histogram* iter->specifics.percentiles.percentile_to_iterate_to = 0.0; iter->specifics.percentiles.percentile = 0.0; - iter->_next_fp = _percentile_iter_next; + iter->_next_fp = percentile_iter_next; } static void format_line_string(char* str, size_t len, int significant_figures, format_type format) @@ -827,13 +937,13 @@ static void format_line_string(char* str, size_t len, int significant_figures, f /* ## ## ######## ###### ####### ## ## ######## ######## ######## */ -static bool _recorded_iter_next(struct hdr_iter* iter) +static bool recorded_iter_next(struct hdr_iter* iter) { - while (_basic_iter_next(iter)) + while (basic_iter_next(iter)) { if (iter->count != 0) { - _update_iterated_values(iter, iter->value); + update_iterated_values(iter, iter->value); iter->specifics.recorded.count_added_in_this_iteration_step = iter->count; return true; @@ -849,7 +959,7 @@ void hdr_iter_recorded_init(struct hdr_iter* iter, const struct hdr_histogram* h iter->specifics.recorded.count_added_in_this_iteration_step = 0; - iter->_next_fp = _recorded_iter_next; + iter->_next_fp = recorded_iter_next; } /* ## #### ## ## ######## ### ######## */ @@ -861,7 +971,7 @@ void hdr_iter_recorded_init(struct hdr_iter* iter, const struct hdr_histogram* h /* ######## #### ## ## ######## ## ## ## ## */ -static bool _iter_linear_next(struct hdr_iter* iter) +static bool iter_linear_next(struct hdr_iter* iter) { struct hdr_iter_linear* linear = &iter->specifics.linear; @@ -875,7 +985,7 @@ static bool _iter_linear_next(struct hdr_iter* iter) { if (iter->value >= linear->next_value_reporting_level_lowest_equivalent) { - _update_iterated_values(iter, linear->next_value_reporting_level); + update_iterated_values(iter, linear->next_value_reporting_level); linear->next_value_reporting_level += linear->value_units_per_bucket; linear->next_value_reporting_level_lowest_equivalent = @@ -907,7 +1017,7 @@ void hdr_iter_linear_init(struct hdr_iter* iter, const struct hdr_histogram* h, iter->specifics.linear.next_value_reporting_level = value_units_per_bucket; iter->specifics.linear.next_value_reporting_level_lowest_equivalent = lowest_equivalent_value(h, value_units_per_bucket); - iter->_next_fp = _iter_linear_next; + iter->_next_fp = iter_linear_next; } /* ## ####### ###### ### ######## #### ######## ## ## ## ## #### ###### */ @@ -918,7 +1028,7 @@ void hdr_iter_linear_init(struct hdr_iter* iter, const struct hdr_histogram* h, /* ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## */ /* ######## ####### ###### ## ## ## ## #### ## ## ## ## ## #### ###### */ -static bool _log_iter_next(struct hdr_iter *iter) +static bool log_iter_next(struct hdr_iter *iter) { struct hdr_iter_log* logarithmic = &iter->specifics.log; @@ -932,7 +1042,7 @@ static bool _log_iter_next(struct hdr_iter *iter) { if (iter->value >= logarithmic->next_value_reporting_level_lowest_equivalent) { - _update_iterated_values(iter, logarithmic->next_value_reporting_level); + update_iterated_values(iter, logarithmic->next_value_reporting_level); logarithmic->next_value_reporting_level *= (int64_t)logarithmic->log_base; logarithmic->next_value_reporting_level_lowest_equivalent = lowest_equivalent_value(iter->h, logarithmic->next_value_reporting_level); @@ -965,7 +1075,7 @@ void hdr_iter_log_init( iter->specifics.log.next_value_reporting_level = value_units_first_bucket; iter->specifics.log.next_value_reporting_level_lowest_equivalent = lowest_equivalent_value(h, value_units_first_bucket); - iter->_next_fp = _log_iter_next; + iter->_next_fp = log_iter_next; } /* Printing. */ @@ -977,7 +1087,6 @@ static const char* format_head_string(format_type format) case CSV: return "%s,%s,%s,%s\n"; case CLASSIC: - return "%12s %12s %12s %12s\n\n"; default: return "%12s %12s %12s %12s\n\n"; } diff --git a/deps/histogram/src/hdr_histogram.h b/deps/histogram/src/hdr_histogram.h index 4a0f4606b57172..aff51f1c11526a 100644 --- a/deps/histogram/src/hdr_histogram.h +++ b/deps/histogram/src/hdr_histogram.h @@ -110,6 +110,21 @@ size_t hdr_get_memory_size(struct hdr_histogram* h); */ bool hdr_record_value(struct hdr_histogram* h, int64_t value); +/** + * Records a value in the histogram, will round this value of to a precision at or better + * than the significant_figure specified at construction time. + * + * Will record this value atomically, however the whole structure may appear inconsistent + * when read concurrently with this update. Do NOT mix calls to this method with calls + * to non-atomic updates. + * + * @param h "This" pointer + * @param value Value to add to the histogram + * @return false if the value is larger than the highest_trackable_value and can't be recorded, + * true otherwise. + */ +bool hdr_record_value_atomic(struct hdr_histogram* h, int64_t value); + /** * Records count values in the histogram, will round this value of to a * precision at or better than the significant_figure specified at construction @@ -123,6 +138,22 @@ bool hdr_record_value(struct hdr_histogram* h, int64_t value); */ bool hdr_record_values(struct hdr_histogram* h, int64_t value, int64_t count); +/** + * Records count values in the histogram, will round this value of to a + * precision at or better than the significant_figure specified at construction + * time. + * + * Will record this value atomically, however the whole structure may appear inconsistent + * when read concurrently with this update. Do NOT mix calls to this method with calls + * to non-atomic updates. + * + * @param h "This" pointer + * @param value Value to add to the histogram + * @param count Number of 'value's to add to the histogram + * @return false if any value is larger than the highest_trackable_value and can't be recorded, + * true otherwise. + */ +bool hdr_record_values_atomic(struct hdr_histogram* h, int64_t value, int64_t count); /** * Record a value in the histogram and backfill based on an expected interval. @@ -140,6 +171,28 @@ bool hdr_record_values(struct hdr_histogram* h, int64_t value, int64_t count); * true otherwise. */ bool hdr_record_corrected_value(struct hdr_histogram* h, int64_t value, int64_t expexcted_interval); + +/** + * Record a value in the histogram and backfill based on an expected interval. + * + * Records a value in the histogram, will round this value of to a precision at or better + * than the significant_figure specified at contruction time. This is specifically used + * for recording latency. If the value is larger than the expected_interval then the + * latency recording system has experienced co-ordinated omission. This method fills in the + * values that would have occured had the client providing the load not been blocked. + * + * Will record this value atomically, however the whole structure may appear inconsistent + * when read concurrently with this update. Do NOT mix calls to this method with calls + * to non-atomic updates. + * + * @param h "This" pointer + * @param value Value to add to the histogram + * @param expected_interval The delay between recording values. + * @return false if the value is larger than the highest_trackable_value and can't be recorded, + * true otherwise. + */ +bool hdr_record_corrected_value_atomic(struct hdr_histogram* h, int64_t value, int64_t expexcted_interval); + /** * Record a value in the histogram 'count' times. Applies the same correcting logic * as 'hdr_record_corrected_value'. @@ -153,6 +206,23 @@ bool hdr_record_corrected_value(struct hdr_histogram* h, int64_t value, int64_t */ bool hdr_record_corrected_values(struct hdr_histogram* h, int64_t value, int64_t count, int64_t expected_interval); +/** + * Record a value in the histogram 'count' times. Applies the same correcting logic + * as 'hdr_record_corrected_value'. + * + * Will record this value atomically, however the whole structure may appear inconsistent + * when read concurrently with this update. Do NOT mix calls to this method with calls + * to non-atomic updates. + * + * @param h "This" pointer + * @param value Value to add to the histogram + * @param count Number of 'value's to add to the histogram + * @param expected_interval The delay between recording values. + * @return false if the value is larger than the highest_trackable_value and can't be recorded, + * true otherwise. + */ +bool hdr_record_corrected_values_atomic(struct hdr_histogram* h, int64_t value, int64_t count, int64_t expected_interval); + /** * Adds all of the values from 'from' to 'this' histogram. Will return the * number of values that are dropped when copying. Values will be dropped @@ -299,7 +369,7 @@ struct hdr_iter /** raw index into the counts array */ int32_t counts_index; /** snapshot of the length at the time the iterator is created */ - int32_t total_count; + int64_t total_count; /** value directly from array for the current counts_index */ int64_t count; /** sum of all of the counts up to and including the count at this index */ @@ -431,4 +501,4 @@ void hdr_reset_internal_counters(struct hdr_histogram* h); } #endif -#endif \ No newline at end of file +#endif