changes coverage to an opt out with NO_COVERAGE env var (#2480)

.travis.yml

@@ -20,7 +20,6 @@ matrix:
 env:
   global:
     - CONCURRENCY=2
-    - COVERAGE=true
     - NOSE_FILTER="not windows"
     - INTEGRATIONS_DIR=$HOME/embedded
     - PIP_CACHE=$HOME/.cache/pip

appveyor.yml

@@ -7,7 +7,6 @@ environment:
   VOLATILE_DIR: c:\projects
   NOSE_FILTER: windows
   FLAVORS: windows
-  COVERAGE: true
   PYWIN_PATH: C:\projects\dd-agent\.cache\pywin32-py2.7.exe
   matrix:
   - PYTHON: C:\\Python27

tests/README.md

@@ -90,6 +90,14 @@ rake ci:run[bone]
 
 To create rake tasks for a new flavor you can use this [skeleton file](../ci/skeleton.rb).
 
+# Coverage
+
+We like to see that the metrics being generated are actually covered by tests. For the integration tests you can call the method self.coverage_report() in your test to see how many of the metrics you're actually looking at.
+
+Note, however, that Travis and Appveyor enforce coverage. It will fail the test if you don't have 100% coverage.
+
+But, since the we also know that the coverage report is very verbose, and that it can be annoying to fail a test for this, we don't enforce it being called where it doesn't make sense. And we also allow you to turn off the failure locally by setting the environment variable: `NO_COVERAGE=true`.
+
 
 # Travis
 

tests/checks/common.py

@@ -300,7 +300,7 @@ def coverage_report(self):
             untested_events=pformat(untested_events),
         ))
 
-        if os.getenv('COVERAGE'):
+        if not os.getenv('NO_COVERAGE'):
             self.assertEquals(coverage_metrics, 100.0)
             self.assertEquals(coverage_events, 100.0)
             self.assertEquals(coverage_sc, 100.0)

tests/checks/integration/test_mysql.py

@@ -382,7 +382,7 @@ def test_complex_config(self):
                                      + self.OPTIONAL_STATUS_VARS
                                      + self.OPTIONAL_STATUS_VARS_5_6_6), 1)
 
-        # Raises when COVERAGE=true and coverage < 100%
+        # Raises when coverage < 100%
         self.coverage_report()
 
     def test_connection_failure(self):

tests/checks/integration/test_process.py

@@ -279,7 +279,7 @@ def test_check(self, mock_process):
             else:
                 self.assertServiceCheckOK('process.up', count=1, tags=expected_tags)
 
-        # Raises when COVERAGE=true and coverage < 100%
+        # Raises when coverage < 100%
         self.coverage_report()
 
         # Run the check a second time and check that `cpu_pct` is there

tests/checks/integration/test_supervisord.py

@@ -67,7 +67,7 @@ def test_check(self):
             self.assertServiceCheck("supervisord.can_connect", status=AgentCheck.OK,
                                     tags=instance_tags, count=1)
 
-            # Raises when COVERAGE=true and coverage < 100%
+            # Raises when coverage < 100%
             self.coverage_report()
 
             # Sleep 10s to give enough time to processes to terminate

tests/checks/mock/test_directory.py

@@ -125,7 +125,7 @@ def test_directory_metrics(self):
                 # 12 files in 'temp_dir'
                 self.assertMetric("system.disk.directory.files", tags=dir_tags, count=1, value=12)
 
-        # Raises when COVERAGE=true and coverage < 100%
+        # Raises when coverage < 100%
         self.coverage_report()
 
     def test_file_metrics(self):
@@ -169,5 +169,5 @@ def test_file_metrics(self):
             for mname in self.COMMON_METRICS:
                 self.assertMetric(mname, tags=dir_tags, count=1)
 
-        # Raises when COVERAGE=true and coverage < 100%
+        # Raises when coverage < 100%
         self.coverage_report()