Change assert all metrics covered - ci run all (#4253)

* Fix assert_all_metrics_covered
* Fix cockroachdb tests
* Fix etcd tests
* Fix mysql tests
* Add exec permissions to wait-for-it
* Fix rabbitMQ tests

cockroachdb/tests/test_cockroachdb.py

@@ -1,20 +1,19 @@
 # (C) Datadog, Inc. 2018
 # All rights reserved
 # Licensed under a 3-clause BSD style license (see LICENSE)
+import pytest
 from six import itervalues
 
 from datadog_checks.cockroachdb import CockroachdbCheck
 from datadog_checks.cockroachdb.metrics import METRIC_MAP
 
 
+@pytest.mark.integration
 def test_check(aggregator, instance):
     check = CockroachdbCheck('cockroachdb', {}, {}, [instance])
     check.check(instance)
 
     for metric in itervalues(METRIC_MAP):
-        try:
-            aggregator.assert_metric(metric)
-        except AssertionError:
-            pass
+        aggregator.assert_metric('cockroachdb.{}'.format(metric), at_least=0)
 
-    assert aggregator.metrics_asserted_pct > 80
+    assert aggregator.metrics_asserted_pct > 80, 'Missing metrics {}'.format(aggregator.not_asserted())

datadog_checks_base/datadog_checks/base/stubs/aggregator.py

@@ -235,7 +235,10 @@ def _assert(condition, msg, expected_stub, submitted_elements):
         assert condition, new_msg
 
     def assert_all_metrics_covered(self):
-        assert self.metrics_asserted_pct >= 100.0
+        missing_metrics = ''
+        if self.metrics_asserted_pct < 100.0:
+            missing_metrics = self.not_asserted()
+        assert self.metrics_asserted_pct >= 100.0, 'Missing metrics: {}'.format(missing_metrics)
 
     def reset(self):
         """
@@ -250,12 +253,8 @@ def all_metrics_asserted(self):
         assert self.metrics_asserted_pct >= 100.0
 
     def not_asserted(self):
-        metrics_not_asserted = []
-        for metric in self._metrics:
-            metric = ensure_unicode(metric)
-            if metric not in self._asserted:
-                metrics_not_asserted.append(metric)
-        return metrics_not_asserted
+        present_metrics = {ensure_unicode(m) for m in self._metrics}
+        return present_metrics - set(self._asserted)
 
     def assert_metric_has_tag_prefix(self, metric_name, tag_prefix, count=None, at_least=1):
         candidates = []
@@ -286,7 +285,10 @@ def metrics_asserted_pct(self):
             else:
                 return 0
 
-        return num_asserted / num_metrics * 100
+        # If it there have been assertions with at_least=0 the length of the num_metrics and num_asserted can match
+        # even if there are different metrics in each set
+        not_asserted = self.not_asserted()
+        return (num_metrics - len(not_asserted)) / num_metrics * 100
 
     @property
     def metric_names(self):

etcd/tests/test_etcd.py

@@ -35,6 +35,8 @@
     'watchers',
 ]
 
+pytestmark = pytest.mark.integration
+
 
 @preview
 def test_check(aggregator, instance):
@@ -44,12 +46,9 @@ def test_check(aggregator, instance):
     tags = ['is_leader:{}'.format('true' if is_leader(URL) else 'false')]
 
     for metric in itervalues(METRIC_MAP):
-        try:
-            aggregator.assert_metric(metric, tags=tags)
-        except AssertionError:
-            pass
+        aggregator.assert_metric('etcd.{}'.format(metric), tags=tags, at_least=0)
 
-    assert aggregator.metrics_asserted_pct > 80
+    assert aggregator.metrics_asserted_pct > 79, 'Missing metrics {}'.format(aggregator.not_asserted())
 
 
 @preview
@@ -61,12 +60,9 @@ def test_check_no_leader_tag(aggregator, instance):
     check.check(instance)
 
     for metric in itervalues(METRIC_MAP):
-        try:
-            aggregator.assert_metric(metric, tags=[])
-        except AssertionError:
-            pass
+        aggregator.assert_metric('etcd.{}'.format(metric), tags=[], at_least=0)
 
-    assert aggregator.metrics_asserted_pct > 80
+    assert aggregator.metrics_asserted_pct > 79, 'Missing metrics {}'.format(aggregator.not_asserted())
 
 
 @preview

mysql/tests/test_mysql.py

@@ -17,6 +17,7 @@
 from .common import MYSQL_VERSION_PARSED
 
 
+@pytest.mark.integration
 @pytest.mark.usefixtures('dd_environment')
 def test_minimal_config(aggregator, instance_basic):
     mysql_check = MySql(common.CHECK_NAME, {}, {})
@@ -39,6 +40,7 @@ def test_minimal_config(aggregator, instance_basic):
         aggregator.assert_metric(mname, at_least=0)
 
 
+@pytest.mark.integration
 @pytest.mark.usefixtures('dd_environment')
 def test_complex_config(aggregator, instance_complex):
     mysql_check = MySql(common.CHECK_NAME, {}, {}, instances=[instance_complex])
@@ -114,6 +116,7 @@ def _assert_complex_config(aggregator):
     aggregator.assert_all_metrics_covered()
 
 
+@pytest.mark.integration
 @pytest.mark.usefixtures('dd_environment')
 def test_connection_failure(aggregator, instance_error):
     """
@@ -129,6 +132,7 @@ def test_connection_failure(aggregator, instance_error):
     aggregator.assert_all_metrics_covered()
 
 
+@pytest.mark.integration
 @pytest.mark.usefixtures('dd_environment')
 def test_complex_config_replica(aggregator, instance_complex):
     mysql_check = MySql(common.CHECK_NAME, {}, {})
@@ -156,6 +160,9 @@ def test_complex_config_replica(aggregator, instance_complex):
         + variables.SYNTHETIC_VARS
     )
 
+    if MYSQL_VERSION_PARSED >= parse_version('5.6') and environ.get('MYSQL_FLAVOR') != 'mariadb':
+        testable_metrics.extend(variables.PERFORMANCE_VARS)
+
     # Test metrics
     for mname in testable_metrics:
         # These two are currently not guaranteed outside of a Linux
@@ -166,9 +173,8 @@ def test_complex_config_replica(aggregator, instance_complex):
             continue
         if mname == 'mysql.performance.cpu_time' and Platform.is_windows():
             continue
-
         if mname == 'mysql.performance.query_run_time.avg':
-            aggregator.assert_metric(mname, tags=tags.METRIC_TAGS + ['schema:testdb'], count=1)
+            aggregator.assert_metric(mname, tags=tags.METRIC_TAGS + ['schema:testdb'], at_least=1)
         elif mname == 'mysql.info.schema.size':
             aggregator.assert_metric(mname, tags=tags.METRIC_TAGS + ['schema:testdb'], count=1)
             aggregator.assert_metric(mname, tags=tags.METRIC_TAGS + ['schema:information_schema'], count=1)

rabbitmq/tests/metrics.py

@@ -22,6 +22,14 @@
     'rabbitmq.exchange.messages.publish_out.rate',
 ]
 
+# Only present in 3.5
+E_METRICS_35 = [
+    'rabbitmq.exchange.messages.confirm.count',
+    'rabbitmq.exchange.messages.confirm.rate',
+    'rabbitmq.exchange.messages.return_unroutable.count',
+    'rabbitmq.exchange.messages.return_unroutable.rate',
+]
+
 Q_METRICS = [
     'rabbitmq.queue.consumers',
     'rabbitmq.queue.bindings.count',

rabbitmq/tests/test_rabbitmq.py

@@ -16,6 +16,7 @@
 log = logging.getLogger(__file__)
 
 
+@pytest.mark.integration
 @pytest.mark.usefixtures('dd_environment')
 def test_rabbitmq(aggregator, check):
     check.check(common.CONFIG)
@@ -39,6 +40,8 @@ def test_rabbitmq(aggregator, check):
     # Exchange attributes, should be only one exchange fetched
     for mname in metrics.E_METRICS:
         aggregator.assert_metric_has_tag(mname, 'rabbitmq_exchange:test1', count=1)
+    for mname in metrics.E_METRICS_35:
+        aggregator.assert_metric_has_tag(mname, 'rabbitmq_exchange:test1', at_least=0)
     # Overview attributes
     for mname in metrics.OVERVIEW_METRICS_TOTALS:
         aggregator.assert_metric_has_tag(mname, 'rabbitmq_cluster:rabbitmqtest', count=1)
@@ -57,6 +60,7 @@ def test_rabbitmq(aggregator, check):
     aggregator.assert_all_metrics_covered()
 
 
+@pytest.mark.integration
 @pytest.mark.usefixtures('dd_environment')
 def test_regex(aggregator, check):
     check.check(common.CONFIG_REGEX)
@@ -74,6 +78,10 @@ def test_regex(aggregator, check):
         aggregator.assert_metric_has_tag(mname, 'rabbitmq_exchange:test1', count=1)
         aggregator.assert_metric_has_tag(mname, 'rabbitmq_exchange:test5', count=1)
         aggregator.assert_metric_has_tag(mname, 'rabbitmq_exchange:tralala', count=0)
+    for mname in metrics.E_METRICS_35:
+        aggregator.assert_metric_has_tag(mname, 'rabbitmq_exchange:test1', at_least=0)
+        aggregator.assert_metric_has_tag(mname, 'rabbitmq_exchange:test5', at_least=0)
+        aggregator.assert_metric_has_tag(mname, 'rabbitmq_exchange:tralala', at_least=0)
 
     # Queue attributes
     for mname in metrics.Q_METRICS:
@@ -96,6 +104,7 @@ def test_regex(aggregator, check):
     aggregator.assert_all_metrics_covered()
 
 
+@pytest.mark.integration
 @pytest.mark.usefixtures('dd_environment')
 def test_limit_vhosts(aggregator, check):
     check.check(common.CONFIG_REGEX)
@@ -114,6 +123,8 @@ def test_limit_vhosts(aggregator, check):
         aggregator.assert_metric_has_tag(mname, 'rabbitmq_queue:tralala', count=0)
     for mname in metrics.E_METRICS:
         aggregator.assert_metric(mname, count=2)
+    for mname in metrics.E_METRICS_35:
+        aggregator.assert_metric(mname, at_least=0)
 
     # Overview attributes
     for mname in metrics.OVERVIEW_METRICS_TOTALS:
@@ -131,6 +142,7 @@ def test_limit_vhosts(aggregator, check):
     aggregator.assert_all_metrics_covered()
 
 
+@pytest.mark.integration
 @pytest.mark.usefixtures('dd_environment')
 def test_family_tagging(aggregator, check):
     check.check(common.CONFIG_WITH_FAMILY)
@@ -144,6 +156,8 @@ def test_family_tagging(aggregator, check):
     aggregator.assert_metric('rabbitmq.connections', tags=['rabbitmq_vhost:myothervhost'], value=0, count=1)
     for mname in metrics.E_METRICS:
         aggregator.assert_metric_has_tag(mname, 'rabbitmq_exchange_family:test', count=2)
+    for mname in metrics.E_METRICS_35:
+        aggregator.assert_metric_has_tag(mname, 'rabbitmq_exchange_family:test', at_least=0)
 
     for mname in metrics.Q_METRICS:
         aggregator.assert_metric_has_tag(mname, 'rabbitmq_queue_family:test', count=6)
@@ -165,6 +179,7 @@ def test_family_tagging(aggregator, check):
     aggregator.assert_all_metrics_covered()
 
 
+@pytest.mark.integration
 @pytest.mark.usefixtures('dd_environment')
 def test_connections(aggregator, check):
     # no connections and no 'vhosts' list in the conf don't produce 'connections' metric