From 3e4528c9e0f799dec5e6d407246ee6985c1498f1 Mon Sep 17 00:00:00 2001 From: Julia <611228+hithwen@users.noreply.github.com> Date: Tue, 6 Aug 2019 14:34:43 +0200 Subject: [PATCH] Change assert all metrics covered - ci run all (#4253) * Fix assert_all_metrics_covered * Fix cockroachdb tests * Fix etcd tests * Fix mysql tests * Add exec permissions to wait-for-it * Fix rabbitMQ tests --- cockroachdb/tests/test_cockroachdb.py | 9 ++++----- .../datadog_checks/base/stubs/aggregator.py | 18 ++++++++++-------- .../scripts/wait-for-it.sh | 0 etcd/tests/test_etcd.py | 16 ++++++---------- mysql/tests/test_mysql.py | 10 ++++++++-- rabbitmq/tests/metrics.py | 8 ++++++++ rabbitmq/tests/test_rabbitmq.py | 15 +++++++++++++++ 7 files changed, 51 insertions(+), 25 deletions(-) mode change 100644 => 100755 datadog_checks_tests_helper/scripts/wait-for-it.sh diff --git a/cockroachdb/tests/test_cockroachdb.py b/cockroachdb/tests/test_cockroachdb.py index 0d79291a45f71..be61e3080c91d 100644 --- a/cockroachdb/tests/test_cockroachdb.py +++ b/cockroachdb/tests/test_cockroachdb.py @@ -1,20 +1,19 @@ # (C) Datadog, Inc. 2018 # All rights reserved # Licensed under a 3-clause BSD style license (see LICENSE) +import pytest from six import itervalues from datadog_checks.cockroachdb import CockroachdbCheck from datadog_checks.cockroachdb.metrics import METRIC_MAP +@pytest.mark.integration def test_check(aggregator, instance): check = CockroachdbCheck('cockroachdb', {}, {}, [instance]) check.check(instance) for metric in itervalues(METRIC_MAP): - try: - aggregator.assert_metric(metric) - except AssertionError: - pass + aggregator.assert_metric('cockroachdb.{}'.format(metric), at_least=0) - assert aggregator.metrics_asserted_pct > 80 + assert aggregator.metrics_asserted_pct > 80, 'Missing metrics {}'.format(aggregator.not_asserted()) diff --git a/datadog_checks_base/datadog_checks/base/stubs/aggregator.py b/datadog_checks_base/datadog_checks/base/stubs/aggregator.py index af38552c895bf..40cfbb4ae227f 100644 --- a/datadog_checks_base/datadog_checks/base/stubs/aggregator.py +++ b/datadog_checks_base/datadog_checks/base/stubs/aggregator.py @@ -235,7 +235,10 @@ def _assert(condition, msg, expected_stub, submitted_elements): assert condition, new_msg def assert_all_metrics_covered(self): - assert self.metrics_asserted_pct >= 100.0 + missing_metrics = '' + if self.metrics_asserted_pct < 100.0: + missing_metrics = self.not_asserted() + assert self.metrics_asserted_pct >= 100.0, 'Missing metrics: {}'.format(missing_metrics) def reset(self): """ @@ -250,12 +253,8 @@ def all_metrics_asserted(self): assert self.metrics_asserted_pct >= 100.0 def not_asserted(self): - metrics_not_asserted = [] - for metric in self._metrics: - metric = ensure_unicode(metric) - if metric not in self._asserted: - metrics_not_asserted.append(metric) - return metrics_not_asserted + present_metrics = {ensure_unicode(m) for m in self._metrics} + return present_metrics - set(self._asserted) def assert_metric_has_tag_prefix(self, metric_name, tag_prefix, count=None, at_least=1): candidates = [] @@ -286,7 +285,10 @@ def metrics_asserted_pct(self): else: return 0 - return num_asserted / num_metrics * 100 + # If it there have been assertions with at_least=0 the length of the num_metrics and num_asserted can match + # even if there are different metrics in each set + not_asserted = self.not_asserted() + return (num_metrics - len(not_asserted)) / num_metrics * 100 @property def metric_names(self): diff --git a/datadog_checks_tests_helper/scripts/wait-for-it.sh b/datadog_checks_tests_helper/scripts/wait-for-it.sh old mode 100644 new mode 100755 diff --git a/etcd/tests/test_etcd.py b/etcd/tests/test_etcd.py index 42c22b7af85e7..5fdeee886b803 100644 --- a/etcd/tests/test_etcd.py +++ b/etcd/tests/test_etcd.py @@ -35,6 +35,8 @@ 'watchers', ] +pytestmark = pytest.mark.integration + @preview def test_check(aggregator, instance): @@ -44,12 +46,9 @@ def test_check(aggregator, instance): tags = ['is_leader:{}'.format('true' if is_leader(URL) else 'false')] for metric in itervalues(METRIC_MAP): - try: - aggregator.assert_metric(metric, tags=tags) - except AssertionError: - pass + aggregator.assert_metric('etcd.{}'.format(metric), tags=tags, at_least=0) - assert aggregator.metrics_asserted_pct > 80 + assert aggregator.metrics_asserted_pct > 79, 'Missing metrics {}'.format(aggregator.not_asserted()) @preview @@ -61,12 +60,9 @@ def test_check_no_leader_tag(aggregator, instance): check.check(instance) for metric in itervalues(METRIC_MAP): - try: - aggregator.assert_metric(metric, tags=[]) - except AssertionError: - pass + aggregator.assert_metric('etcd.{}'.format(metric), tags=[], at_least=0) - assert aggregator.metrics_asserted_pct > 80 + assert aggregator.metrics_asserted_pct > 79, 'Missing metrics {}'.format(aggregator.not_asserted()) @preview diff --git a/mysql/tests/test_mysql.py b/mysql/tests/test_mysql.py index 48447750f3be7..7e6534358a4d0 100644 --- a/mysql/tests/test_mysql.py +++ b/mysql/tests/test_mysql.py @@ -17,6 +17,7 @@ from .common import MYSQL_VERSION_PARSED +@pytest.mark.integration @pytest.mark.usefixtures('dd_environment') def test_minimal_config(aggregator, instance_basic): mysql_check = MySql(common.CHECK_NAME, {}, {}) @@ -39,6 +40,7 @@ def test_minimal_config(aggregator, instance_basic): aggregator.assert_metric(mname, at_least=0) +@pytest.mark.integration @pytest.mark.usefixtures('dd_environment') def test_complex_config(aggregator, instance_complex): mysql_check = MySql(common.CHECK_NAME, {}, {}, instances=[instance_complex]) @@ -114,6 +116,7 @@ def _assert_complex_config(aggregator): aggregator.assert_all_metrics_covered() +@pytest.mark.integration @pytest.mark.usefixtures('dd_environment') def test_connection_failure(aggregator, instance_error): """ @@ -129,6 +132,7 @@ def test_connection_failure(aggregator, instance_error): aggregator.assert_all_metrics_covered() +@pytest.mark.integration @pytest.mark.usefixtures('dd_environment') def test_complex_config_replica(aggregator, instance_complex): mysql_check = MySql(common.CHECK_NAME, {}, {}) @@ -156,6 +160,9 @@ def test_complex_config_replica(aggregator, instance_complex): + variables.SYNTHETIC_VARS ) + if MYSQL_VERSION_PARSED >= parse_version('5.6') and environ.get('MYSQL_FLAVOR') != 'mariadb': + testable_metrics.extend(variables.PERFORMANCE_VARS) + # Test metrics for mname in testable_metrics: # These two are currently not guaranteed outside of a Linux @@ -166,9 +173,8 @@ def test_complex_config_replica(aggregator, instance_complex): continue if mname == 'mysql.performance.cpu_time' and Platform.is_windows(): continue - if mname == 'mysql.performance.query_run_time.avg': - aggregator.assert_metric(mname, tags=tags.METRIC_TAGS + ['schema:testdb'], count=1) + aggregator.assert_metric(mname, tags=tags.METRIC_TAGS + ['schema:testdb'], at_least=1) elif mname == 'mysql.info.schema.size': aggregator.assert_metric(mname, tags=tags.METRIC_TAGS + ['schema:testdb'], count=1) aggregator.assert_metric(mname, tags=tags.METRIC_TAGS + ['schema:information_schema'], count=1) diff --git a/rabbitmq/tests/metrics.py b/rabbitmq/tests/metrics.py index 560f8c1cee356..4ee9adc1864da 100644 --- a/rabbitmq/tests/metrics.py +++ b/rabbitmq/tests/metrics.py @@ -22,6 +22,14 @@ 'rabbitmq.exchange.messages.publish_out.rate', ] +# Only present in 3.5 +E_METRICS_35 = [ + 'rabbitmq.exchange.messages.confirm.count', + 'rabbitmq.exchange.messages.confirm.rate', + 'rabbitmq.exchange.messages.return_unroutable.count', + 'rabbitmq.exchange.messages.return_unroutable.rate', +] + Q_METRICS = [ 'rabbitmq.queue.consumers', 'rabbitmq.queue.bindings.count', diff --git a/rabbitmq/tests/test_rabbitmq.py b/rabbitmq/tests/test_rabbitmq.py index b55a765b4b0cd..7711c018daa5f 100644 --- a/rabbitmq/tests/test_rabbitmq.py +++ b/rabbitmq/tests/test_rabbitmq.py @@ -16,6 +16,7 @@ log = logging.getLogger(__file__) +@pytest.mark.integration @pytest.mark.usefixtures('dd_environment') def test_rabbitmq(aggregator, check): check.check(common.CONFIG) @@ -39,6 +40,8 @@ def test_rabbitmq(aggregator, check): # Exchange attributes, should be only one exchange fetched for mname in metrics.E_METRICS: aggregator.assert_metric_has_tag(mname, 'rabbitmq_exchange:test1', count=1) + for mname in metrics.E_METRICS_35: + aggregator.assert_metric_has_tag(mname, 'rabbitmq_exchange:test1', at_least=0) # Overview attributes for mname in metrics.OVERVIEW_METRICS_TOTALS: aggregator.assert_metric_has_tag(mname, 'rabbitmq_cluster:rabbitmqtest', count=1) @@ -57,6 +60,7 @@ def test_rabbitmq(aggregator, check): aggregator.assert_all_metrics_covered() +@pytest.mark.integration @pytest.mark.usefixtures('dd_environment') def test_regex(aggregator, check): check.check(common.CONFIG_REGEX) @@ -74,6 +78,10 @@ def test_regex(aggregator, check): aggregator.assert_metric_has_tag(mname, 'rabbitmq_exchange:test1', count=1) aggregator.assert_metric_has_tag(mname, 'rabbitmq_exchange:test5', count=1) aggregator.assert_metric_has_tag(mname, 'rabbitmq_exchange:tralala', count=0) + for mname in metrics.E_METRICS_35: + aggregator.assert_metric_has_tag(mname, 'rabbitmq_exchange:test1', at_least=0) + aggregator.assert_metric_has_tag(mname, 'rabbitmq_exchange:test5', at_least=0) + aggregator.assert_metric_has_tag(mname, 'rabbitmq_exchange:tralala', at_least=0) # Queue attributes for mname in metrics.Q_METRICS: @@ -96,6 +104,7 @@ def test_regex(aggregator, check): aggregator.assert_all_metrics_covered() +@pytest.mark.integration @pytest.mark.usefixtures('dd_environment') def test_limit_vhosts(aggregator, check): check.check(common.CONFIG_REGEX) @@ -114,6 +123,8 @@ def test_limit_vhosts(aggregator, check): aggregator.assert_metric_has_tag(mname, 'rabbitmq_queue:tralala', count=0) for mname in metrics.E_METRICS: aggregator.assert_metric(mname, count=2) + for mname in metrics.E_METRICS_35: + aggregator.assert_metric(mname, at_least=0) # Overview attributes for mname in metrics.OVERVIEW_METRICS_TOTALS: @@ -131,6 +142,7 @@ def test_limit_vhosts(aggregator, check): aggregator.assert_all_metrics_covered() +@pytest.mark.integration @pytest.mark.usefixtures('dd_environment') def test_family_tagging(aggregator, check): check.check(common.CONFIG_WITH_FAMILY) @@ -144,6 +156,8 @@ def test_family_tagging(aggregator, check): aggregator.assert_metric('rabbitmq.connections', tags=['rabbitmq_vhost:myothervhost'], value=0, count=1) for mname in metrics.E_METRICS: aggregator.assert_metric_has_tag(mname, 'rabbitmq_exchange_family:test', count=2) + for mname in metrics.E_METRICS_35: + aggregator.assert_metric_has_tag(mname, 'rabbitmq_exchange_family:test', at_least=0) for mname in metrics.Q_METRICS: aggregator.assert_metric_has_tag(mname, 'rabbitmq_queue_family:test', count=6) @@ -165,6 +179,7 @@ def test_family_tagging(aggregator, check): aggregator.assert_all_metrics_covered() +@pytest.mark.integration @pytest.mark.usefixtures('dd_environment') def test_connections(aggregator, check): # no connections and no 'vhosts' list in the conf don't produce 'connections' metric