From 55fb2eb4b4bcca974294ef3fcc8ab884c1ec088a Mon Sep 17 00:00:00 2001 From: Simon Davy Date: Wed, 5 Sep 2018 12:37:17 +0100 Subject: [PATCH 1/7] Refactor MultiprocessCollector.collect to allow for arbitrary merging. Factors out a merge() method from the previous collect() method, which is parameterized, and thus can be used for arbitrary merging of samples. For motivation, see discussion in issue #275 around merging dead workers data into a single mmaped file. This basically allows us to parameterize the files to be merged, and also whether to accumulate histograms or not. Accumulation is on by default, as that is what the prometheus format expects. But it can now be disabled, which allows merged values to be correctly written back to an mmaped file. For the same reason, the order of labels is preserved via OrderedDict. Signed-off-by: Simon Davy --- prometheus_client/multiprocess.py | 28 ++++++-- tests/test_multiprocess.py | 107 +++++++++++++++++++++++++++++- 2 files changed, 129 insertions(+), 6 deletions(-) diff --git a/prometheus_client/multiprocess.py b/prometheus_client/multiprocess.py index 7dd74b2b..a7d3176d 100644 --- a/prometheus_client/multiprocess.py +++ b/prometheus_client/multiprocess.py @@ -2,7 +2,7 @@ from __future__ import unicode_literals -from collections import defaultdict +from collections import OrderedDict, defaultdict import glob import json @@ -23,8 +23,18 @@ def __init__(self, registry, path=None): registry.register(self) def collect(self): + files = glob.glob(os.path.join(self._path, '*.db')) + return self.merge(files, accumulate=True) + + def merge(self, files, accumulate=True): + """Merge metrics from given mmap files. + + By default, histograms are accumulated, as per prometheus wire format. + But if writing the merged data back to mmap files, use + accumulate=False to avoid compound accumulation. + """ metrics = {} - for f in glob.glob(os.path.join(self._path, '*.db')): + for f in files: parts = os.path.basename(f).split('_') typ = parts[0] d = core._MmapedDict(f, read_mode=True) @@ -86,9 +96,17 @@ def collect(self): for labels, values in buckets.items(): acc = 0.0 for bucket, value in sorted(values.items()): - acc += value - samples[(metric.name + '_bucket', labels + (('le', core._floatToGoString(bucket)), ))] = acc - samples[(metric.name + '_count', labels)] = acc + sample_key = ( + metric.name + '_bucket', + labels + (('le', core._floatToGoString(bucket)), ), + ) + if accumulate: + acc += value + samples[sample_key] = acc + else: + samples[sample_key] = value + if accumulate: + samples[(metric.name + '_count', labels)] = acc # Convert to correct sample format. metric.samples = [core.Sample(name, dict(labels), value) for (name, labels), value in samples.items()] diff --git a/tests/test_multiprocess.py b/tests/test_multiprocess.py index ca84913f..321eb5b0 100644 --- a/tests/test_multiprocess.py +++ b/tests/test_multiprocess.py @@ -1,5 +1,7 @@ from __future__ import unicode_literals +from collections import OrderedDict +import glob import os import shutil import tempfile @@ -25,7 +27,7 @@ def setUp(self): os.environ['prometheus_multiproc_dir'] = self.tempdir core._ValueClass = core._MultiProcessValue(lambda: 123) self.registry = CollectorRegistry() - MultiProcessCollector(self.registry, self.tempdir) + self.collector = MultiProcessCollector(self.registry, self.tempdir) def tearDown(self): del os.environ['prometheus_multiproc_dir'] @@ -137,6 +139,109 @@ def test_counter_across_forks(self): self.assertEqual(3, self.registry.get_sample_value('c_total')) self.assertEqual(1, c1._value.get()) + def test_collect(self): + pid = 0 + core._ValueClass = core._MultiProcessValue(lambda: pid) + labels = OrderedDict((i, i) for i in 'abcd') + + def add_label(key, value): + l = labels.copy() + l[key] = value + return l + + c = Counter('c', 'help', labelnames=labels.keys(), registry=None) + g = Gauge('g', 'help', labelnames=labels.keys(), registry=None) + h = Histogram('h', 'help', labelnames=labels.keys(), registry=None) + + c.labels(**labels).inc(1) + g.labels(**labels).set(1) + h.labels(**labels).observe(1) + + pid = 1 + + c.labels(**labels).inc(1) + g.labels(**labels).set(1) + h.labels(**labels).observe(5) + + metrics = dict((m.name, m) for m in self.collector.collect()) + + self.assertEqual(metrics['c'].samples, [('c', labels, 2.0)]) + metrics['g'].samples.sort(key=lambda x: x[1]['pid']) + self.assertEqual(metrics['g'].samples, [ + ('g', add_label('pid', '0'), 1.0), + ('g', add_label('pid', '1'), 1.0), + ]) + + metrics['h'].samples.sort( + key=lambda x: (x[0], float(x[1].get('le', 0))) + ) + expected_histogram = [ + ('h_bucket', add_label('le', '0.005'), 0.0), + ('h_bucket', add_label('le', '0.01'), 0.0), + ('h_bucket', add_label('le', '0.025'), 0.0), + ('h_bucket', add_label('le', '0.05'), 0.0), + ('h_bucket', add_label('le', '0.075'), 0.0), + ('h_bucket', add_label('le', '0.1'), 0.0), + ('h_bucket', add_label('le', '0.25'), 0.0), + ('h_bucket', add_label('le', '0.5'), 0.0), + ('h_bucket', add_label('le', '0.75'), 0.0), + ('h_bucket', add_label('le', '1.0'), 1.0), + ('h_bucket', add_label('le', '2.5'), 1.0), + ('h_bucket', add_label('le', '5.0'), 2.0), + ('h_bucket', add_label('le', '7.5'), 2.0), + ('h_bucket', add_label('le', '10.0'), 2.0), + ('h_bucket', add_label('le', '+Inf'), 2.0), + ('h_count', labels, 2.0), + ('h_sum', labels, 6.0), + ] + + self.assertEqual(metrics['h'].samples, expected_histogram) + + def test_merge_no_accumulate(self): + pid = 0 + core._ValueClass = core._MultiProcessValue(lambda: pid) + labels = OrderedDict((i, i) for i in 'abcd') + + def add_label(key, value): + l = labels.copy() + l[key] = value + return l + + h = Histogram('h', 'help', labelnames=labels.keys(), registry=None) + h.labels(**labels).observe(1) + pid = 1 + h.labels(**labels).observe(5) + + path = os.path.join(os.environ['prometheus_multiproc_dir'], '*.db') + files = glob.glob(path) + metrics = dict( + (m.name, m) for m in self.collector.merge(files, accumulate=False) + ) + + metrics['h'].samples.sort( + key=lambda x: (x[0], float(x[1].get('le', 0))) + ) + expected_histogram = [ + ('h_bucket', add_label('le', '0.005'), 0.0), + ('h_bucket', add_label('le', '0.01'), 0.0), + ('h_bucket', add_label('le', '0.025'), 0.0), + ('h_bucket', add_label('le', '0.05'), 0.0), + ('h_bucket', add_label('le', '0.075'), 0.0), + ('h_bucket', add_label('le', '0.1'), 0.0), + ('h_bucket', add_label('le', '0.25'), 0.0), + ('h_bucket', add_label('le', '0.5'), 0.0), + ('h_bucket', add_label('le', '0.75'), 0.0), + ('h_bucket', add_label('le', '1.0'), 1.0), + ('h_bucket', add_label('le', '2.5'), 0.0), + ('h_bucket', add_label('le', '5.0'), 1.0), + ('h_bucket', add_label('le', '7.5'), 0.0), + ('h_bucket', add_label('le', '10.0'), 0.0), + ('h_bucket', add_label('le', '+Inf'), 0.0), + ('h_sum', labels, 6.0), + ] + + self.assertEqual(metrics['h'].samples, expected_histogram) + class TestMmapedDict(unittest.TestCase): def setUp(self): From 9efd94059424461832f5bcde0e9bbaf08ec8cec6 Mon Sep 17 00:00:00 2001 From: Simon Davy Date: Wed, 5 Sep 2018 15:08:48 +0100 Subject: [PATCH 2/7] fallback for 2.6 lack of OrderedDict Signed-off-by: Simon Davy --- prometheus_client/multiprocess.py | 7 ++++++- tests/test_multiprocess.py | 5 ++++- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/prometheus_client/multiprocess.py b/prometheus_client/multiprocess.py index a7d3176d..1f88b260 100644 --- a/prometheus_client/multiprocess.py +++ b/prometheus_client/multiprocess.py @@ -2,7 +2,12 @@ from __future__ import unicode_literals -from collections import OrderedDict, defaultdict +from collections import defaultdict + +try: + from collections import OrderedDict +except ImportError: + OrderedDict = dict import glob import json diff --git a/tests/test_multiprocess.py b/tests/test_multiprocess.py index 321eb5b0..5b011224 100644 --- a/tests/test_multiprocess.py +++ b/tests/test_multiprocess.py @@ -1,6 +1,9 @@ from __future__ import unicode_literals -from collections import OrderedDict +try: + from collections import OrderedDict +except ImportError: + OrderedDict = dict import glob import os import shutil From 645526dac51089f44a281e5475125f68f620bad2 Mon Sep 17 00:00:00 2001 From: Simon Davy Date: Fri, 7 Sep 2018 12:50:19 +0100 Subject: [PATCH 3/7] Refactor mmap key to have sorted labels Signed-off-by: Simon Davy --- prometheus_client/core.py | 9 ++++++++- prometheus_client/multiprocess.py | 11 +++-------- tests/test_multiprocess.py | 8 ++------ 3 files changed, 13 insertions(+), 15 deletions(-) diff --git a/prometheus_client/core.py b/prometheus_client/core.py index da3034d0..02e68045 100644 --- a/prometheus_client/core.py +++ b/prometheus_client/core.py @@ -586,6 +586,13 @@ def close(self): self._f = None +def _mmap_key(metric_name, name, labelnames, labelvalues): + """Format a key for use in the mmap file.""" + # ensure labels are in consistent order for identity + labels = dict(zip(labelnames, labelvalues)) + return json.dumps([metric_name, name, labels], sort_keys=True) + + def _MultiProcessValue(_pidFunc=os.getpid): files = {} values = [] @@ -618,7 +625,7 @@ def __reset(self): '{0}_{1}.db'.format(file_prefix, pid['value'])) files[file_prefix] = _MmapedDict(filename) self._file = files[file_prefix] - self._key = json.dumps((metric_name, name, labelnames, labelvalues)) + self._key = _mmap_key(metric_name, name, labelnames, labelvalues) self._value = self._file.read_value(self._key) def __check_for_pid_change(self): diff --git a/prometheus_client/multiprocess.py b/prometheus_client/multiprocess.py index 1f88b260..ead70bc5 100644 --- a/prometheus_client/multiprocess.py +++ b/prometheus_client/multiprocess.py @@ -4,11 +4,6 @@ from collections import defaultdict -try: - from collections import OrderedDict -except ImportError: - OrderedDict = dict - import glob import json import os @@ -44,7 +39,7 @@ def merge(self, files, accumulate=True): typ = parts[0] d = core._MmapedDict(f, read_mode=True) for key, value in d.read_all_values(): - metric_name, name, labelnames, labelvalues = json.loads(key) + metric_name, name, labels = json.loads(key) metric = metrics.get(metric_name) if metric is None: @@ -54,10 +49,10 @@ def merge(self, files, accumulate=True): if typ == 'gauge': pid = parts[2][:-3] metric._multiprocess_mode = parts[1] - metric.add_sample(name, tuple(zip(labelnames, labelvalues)) + (('pid', pid), ), value) + metric.add_sample(name, tuple(labels.items()) + (('pid', pid), ), value) else: # The duplicates and labels are fixed in the next for. - metric.add_sample(name, tuple(zip(labelnames, labelvalues)), value) + metric.add_sample(name, tuple(labels.items()), value) d.close() for metric in metrics.values(): diff --git a/tests/test_multiprocess.py b/tests/test_multiprocess.py index 5b011224..275a343c 100644 --- a/tests/test_multiprocess.py +++ b/tests/test_multiprocess.py @@ -1,9 +1,5 @@ from __future__ import unicode_literals -try: - from collections import OrderedDict -except ImportError: - OrderedDict = dict import glob import os import shutil @@ -145,7 +141,7 @@ def test_counter_across_forks(self): def test_collect(self): pid = 0 core._ValueClass = core._MultiProcessValue(lambda: pid) - labels = OrderedDict((i, i) for i in 'abcd') + labels = dict((i, i) for i in 'abcd') def add_label(key, value): l = labels.copy() @@ -203,7 +199,7 @@ def add_label(key, value): def test_merge_no_accumulate(self): pid = 0 core._ValueClass = core._MultiProcessValue(lambda: pid) - labels = OrderedDict((i, i) for i in 'abcd') + labels = dict((i, i) for i in 'abcd') def add_label(key, value): l = labels.copy() From fbb777c5ad6c4c886177df14c8a5fc023ad095fe Mon Sep 17 00:00:00 2001 From: Simon Davy Date: Fri, 7 Sep 2018 14:06:50 +0100 Subject: [PATCH 4/7] fix py2.6 float representation Signed-off-by: Simon Davy --- prometheus_client/core.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/prometheus_client/core.py b/prometheus_client/core.py index 02e68045..9b0a9618 100644 --- a/prometheus_client/core.py +++ b/prometheus_client/core.py @@ -1016,6 +1016,11 @@ def _floatToGoString(d): return '-Inf' elif math.isnan(d): return 'NaN' + elif sys.version_info[:2] == (2, 6): + format = '%g' % float(d) + if isinstance(d, float) and '.' not in format: + format += '.0' + return format else: return repr(float(d)) From dae1fcaa11f3b79a91cdce5eb2698d0cd0b293a7 Mon Sep 17 00:00:00 2001 From: Simon Davy Date: Fri, 7 Sep 2018 14:59:42 +0100 Subject: [PATCH 5/7] skip multiprocess collect tests on 2.6, as floating point representation differences make it too difficult Signed-off-by: Simon Davy --- prometheus_client/core.py | 17 ++++++----------- tests/test_multiprocess.py | 3 +++ 2 files changed, 9 insertions(+), 11 deletions(-) diff --git a/prometheus_client/core.py b/prometheus_client/core.py index 9b0a9618..cb1e7c5b 100644 --- a/prometheus_client/core.py +++ b/prometheus_client/core.py @@ -185,7 +185,7 @@ def get_sample_value(self, name, labels=None): REGISTRY = CollectorRegistry(auto_describe=True) '''The default registry.''' -_METRIC_TYPES = ('counter', 'gauge', 'summary', 'histogram', +_METRIC_TYPES = ('counter', 'gauge', 'summary', 'histogram', 'gaugehistogram', 'unknown', 'info', 'stateset') @@ -378,8 +378,8 @@ def add_metric(self, labels, buckets, sum_value, timestamp=None): exemplar = None if len(b) == 3: exemplar = b[2] - self.samples.append(Sample(self.name + '_bucket', - dict(list(zip(self._labelnames, labels)) + [('le', bucket)]), + self.samples.append(Sample(self.name + '_bucket', + dict(list(zip(self._labelnames, labels)) + [('le', bucket)]), value, timestamp, exemplar)) # +Inf is last and provides the count value. self.samples.append(Sample(self.name + '_count', dict(zip(self._labelnames, labels)), buckets[-1][1], timestamp)) @@ -411,7 +411,7 @@ def add_metric(self, labels, buckets, timestamp=None): ''' for bucket, value in buckets: self.samples.append(Sample( - self.name + '_bucket', + self.name + '_bucket', dict(list(zip(self._labelnames, labels)) + [('le', bucket)]), value, timestamp)) @@ -438,7 +438,7 @@ def add_metric(self, labels, value, timestamp=None): labels: A list of label values value: A dict of labels ''' - self.samples.append(Sample(self.name + '_info', + self.samples.append(Sample(self.name + '_info', dict(dict(zip(self._labelnames, labels)), **value), 1, timestamp)) @@ -1016,11 +1016,6 @@ def _floatToGoString(d): return '-Inf' elif math.isnan(d): return 'NaN' - elif sys.version_info[:2] == (2, 6): - format = '%g' % float(d) - if isinstance(d, float) and '.' not in format: - format += '.0' - return format else: return repr(float(d)) @@ -1155,7 +1150,7 @@ class Enum(object): Example usage: from prometheus_client import Enum - e = Enum('task_state', 'Description of enum', + e = Enum('task_state', 'Description of enum', states=['starting', 'running', 'stopped']) e.state('running') diff --git a/tests/test_multiprocess.py b/tests/test_multiprocess.py index 275a343c..18cf9ec2 100644 --- a/tests/test_multiprocess.py +++ b/tests/test_multiprocess.py @@ -3,6 +3,7 @@ import glob import os import shutil +import sys import tempfile import unittest @@ -138,6 +139,7 @@ def test_counter_across_forks(self): self.assertEqual(3, self.registry.get_sample_value('c_total')) self.assertEqual(1, c1._value.get()) + @unittest.skipIf(sys.version_info < (2, 7), "Test requires Python 2.7+.") def test_collect(self): pid = 0 core._ValueClass = core._MultiProcessValue(lambda: pid) @@ -196,6 +198,7 @@ def add_label(key, value): self.assertEqual(metrics['h'].samples, expected_histogram) + @unittest.skipIf(sys.version_info < (2, 7), "Test requires Python 2.7+.") def test_merge_no_accumulate(self): pid = 0 core._ValueClass = core._MultiProcessValue(lambda: pid) From 369e0d53769d1230a0dc629cf020dc0a92eacd9a Mon Sep 17 00:00:00 2001 From: Simon Davy Date: Fri, 7 Sep 2018 15:11:00 +0100 Subject: [PATCH 6/7] sort sample labels when collecting, and fix unittest usage in 2.6 Signed-off-by: Simon Davy --- prometheus_client/multiprocess.py | 5 +++-- tests/test_multiprocess.py | 8 +++++++- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/prometheus_client/multiprocess.py b/prometheus_client/multiprocess.py index ead70bc5..55213153 100644 --- a/prometheus_client/multiprocess.py +++ b/prometheus_client/multiprocess.py @@ -40,6 +40,7 @@ def merge(self, files, accumulate=True): d = core._MmapedDict(f, read_mode=True) for key, value in d.read_all_values(): metric_name, name, labels = json.loads(key) + labels_key = tuple(sorted(labels.items())) metric = metrics.get(metric_name) if metric is None: @@ -49,10 +50,10 @@ def merge(self, files, accumulate=True): if typ == 'gauge': pid = parts[2][:-3] metric._multiprocess_mode = parts[1] - metric.add_sample(name, tuple(labels.items()) + (('pid', pid), ), value) + metric.add_sample(name, labels_key + (('pid', pid), ), value) else: # The duplicates and labels are fixed in the next for. - metric.add_sample(name, tuple(labels.items()), value) + metric.add_sample(name, labels_key, value) d.close() for metric in metrics.values(): diff --git a/tests/test_multiprocess.py b/tests/test_multiprocess.py index 18cf9ec2..7538d804 100644 --- a/tests/test_multiprocess.py +++ b/tests/test_multiprocess.py @@ -5,7 +5,13 @@ import shutil import sys import tempfile -import unittest + +if sys.version_info < (2, 7): + # We need the skip decorators from unittest2 on Python 2.6. + import unittest2 as unittest +else: + import unittest + from prometheus_client import core from prometheus_client.core import ( From a4370a5acad939b6bdcbb17c1ab12d29936f9bf1 Mon Sep 17 00:00:00 2001 From: Simon Davy Date: Fri, 7 Sep 2018 18:09:16 +0100 Subject: [PATCH 7/7] fix tests after rebasing on master Signed-off-by: Simon Davy --- tests/test_multiprocess.py | 75 ++++++++++++++++++++------------------ 1 file changed, 39 insertions(+), 36 deletions(-) diff --git a/tests/test_multiprocess.py b/tests/test_multiprocess.py index 7538d804..501cb62f 100644 --- a/tests/test_multiprocess.py +++ b/tests/test_multiprocess.py @@ -19,6 +19,7 @@ Counter, Gauge, Histogram, + Sample, Summary, ) from prometheus_client.multiprocess import ( @@ -172,34 +173,36 @@ def add_label(key, value): metrics = dict((m.name, m) for m in self.collector.collect()) - self.assertEqual(metrics['c'].samples, [('c', labels, 2.0)]) + self.assertEqual( + metrics['c'].samples, [Sample('c_total', labels, 2.0)] + ) metrics['g'].samples.sort(key=lambda x: x[1]['pid']) self.assertEqual(metrics['g'].samples, [ - ('g', add_label('pid', '0'), 1.0), - ('g', add_label('pid', '1'), 1.0), + Sample('g', add_label('pid', '0'), 1.0), + Sample('g', add_label('pid', '1'), 1.0), ]) metrics['h'].samples.sort( key=lambda x: (x[0], float(x[1].get('le', 0))) ) expected_histogram = [ - ('h_bucket', add_label('le', '0.005'), 0.0), - ('h_bucket', add_label('le', '0.01'), 0.0), - ('h_bucket', add_label('le', '0.025'), 0.0), - ('h_bucket', add_label('le', '0.05'), 0.0), - ('h_bucket', add_label('le', '0.075'), 0.0), - ('h_bucket', add_label('le', '0.1'), 0.0), - ('h_bucket', add_label('le', '0.25'), 0.0), - ('h_bucket', add_label('le', '0.5'), 0.0), - ('h_bucket', add_label('le', '0.75'), 0.0), - ('h_bucket', add_label('le', '1.0'), 1.0), - ('h_bucket', add_label('le', '2.5'), 1.0), - ('h_bucket', add_label('le', '5.0'), 2.0), - ('h_bucket', add_label('le', '7.5'), 2.0), - ('h_bucket', add_label('le', '10.0'), 2.0), - ('h_bucket', add_label('le', '+Inf'), 2.0), - ('h_count', labels, 2.0), - ('h_sum', labels, 6.0), + Sample('h_bucket', add_label('le', '0.005'), 0.0), + Sample('h_bucket', add_label('le', '0.01'), 0.0), + Sample('h_bucket', add_label('le', '0.025'), 0.0), + Sample('h_bucket', add_label('le', '0.05'), 0.0), + Sample('h_bucket', add_label('le', '0.075'), 0.0), + Sample('h_bucket', add_label('le', '0.1'), 0.0), + Sample('h_bucket', add_label('le', '0.25'), 0.0), + Sample('h_bucket', add_label('le', '0.5'), 0.0), + Sample('h_bucket', add_label('le', '0.75'), 0.0), + Sample('h_bucket', add_label('le', '1.0'), 1.0), + Sample('h_bucket', add_label('le', '2.5'), 1.0), + Sample('h_bucket', add_label('le', '5.0'), 2.0), + Sample('h_bucket', add_label('le', '7.5'), 2.0), + Sample('h_bucket', add_label('le', '10.0'), 2.0), + Sample('h_bucket', add_label('le', '+Inf'), 2.0), + Sample('h_count', labels, 2.0), + Sample('h_sum', labels, 6.0), ] self.assertEqual(metrics['h'].samples, expected_histogram) @@ -230,22 +233,22 @@ def add_label(key, value): key=lambda x: (x[0], float(x[1].get('le', 0))) ) expected_histogram = [ - ('h_bucket', add_label('le', '0.005'), 0.0), - ('h_bucket', add_label('le', '0.01'), 0.0), - ('h_bucket', add_label('le', '0.025'), 0.0), - ('h_bucket', add_label('le', '0.05'), 0.0), - ('h_bucket', add_label('le', '0.075'), 0.0), - ('h_bucket', add_label('le', '0.1'), 0.0), - ('h_bucket', add_label('le', '0.25'), 0.0), - ('h_bucket', add_label('le', '0.5'), 0.0), - ('h_bucket', add_label('le', '0.75'), 0.0), - ('h_bucket', add_label('le', '1.0'), 1.0), - ('h_bucket', add_label('le', '2.5'), 0.0), - ('h_bucket', add_label('le', '5.0'), 1.0), - ('h_bucket', add_label('le', '7.5'), 0.0), - ('h_bucket', add_label('le', '10.0'), 0.0), - ('h_bucket', add_label('le', '+Inf'), 0.0), - ('h_sum', labels, 6.0), + Sample('h_bucket', add_label('le', '0.005'), 0.0), + Sample('h_bucket', add_label('le', '0.01'), 0.0), + Sample('h_bucket', add_label('le', '0.025'), 0.0), + Sample('h_bucket', add_label('le', '0.05'), 0.0), + Sample('h_bucket', add_label('le', '0.075'), 0.0), + Sample('h_bucket', add_label('le', '0.1'), 0.0), + Sample('h_bucket', add_label('le', '0.25'), 0.0), + Sample('h_bucket', add_label('le', '0.5'), 0.0), + Sample('h_bucket', add_label('le', '0.75'), 0.0), + Sample('h_bucket', add_label('le', '1.0'), 1.0), + Sample('h_bucket', add_label('le', '2.5'), 0.0), + Sample('h_bucket', add_label('le', '5.0'), 1.0), + Sample('h_bucket', add_label('le', '7.5'), 0.0), + Sample('h_bucket', add_label('le', '10.0'), 0.0), + Sample('h_bucket', add_label('le', '+Inf'), 0.0), + Sample('h_sum', labels, 6.0), ] self.assertEqual(metrics['h'].samples, expected_histogram)