More straightforward approach to configure tests suites (#521)

CONTRIBUTING.md

@@ -102,13 +102,13 @@ Then you'll need to populate it with the following template:
 
 ```python
 # Test for one implementation of the interface
-from lexicon.tests.providers.integration_tests import IntegrationTests
+from lexicon.tests.providers.integration_tests import IntegrationTestsV2
 from unittest import TestCase
 
 # Hook into testing framework by inheriting unittest.TestCase and reuse
 # the tests which *each and every* implementation of the interface must
 # pass, by inheritance from integration_tests.IntegrationTests
-class FooProviderTests(TestCase, IntegrationTests):
+class FooProviderTests(TestCase, IntegrationTestsV2):
     """Integration tests for Foo provider"""
 	provider_name = 'foo'
 	domain = 'example.com'
@@ -182,13 +182,14 @@ In your `lexicon/tests/providers/test_foo.py` file, you can use `@pytest.mark.sk
 		return
 ```
 
-You can also skip extended test suites by adding the following snipped:
+You can also skip extended test suites by inheriting your provider test class from ``IntegrationTestsV1`` instead of ``IntegrationTestsV2``:
 
 ```python
-    @pytest.fixture(autouse=True)
-    def _skip_suite(self, request):  # pylint: disable=no-self-use
-        if request.node.get_closest_marker('ext_suite_1'):
-            pytest.skip('Skipping extended suite')
+from lexicon.tests.providers.integration_tests import IntegrationTestsV1
+from unittest import TestCase
+
+class FooProviderTests(TestCase, IntegrationTestsV1):
+    """Integration tests for Foo provider"""
 ```
 
 ## CODEOWNERS file

lexicon/tests/providers/integration_tests.py

@@ -66,7 +66,7 @@ def resolve(self, config_key):
         return None
 
 
-class IntegrationTests(object):  # pylint: disable=useless-object-inheritance,too-many-public-methods
+class IntegrationTestsV1(object):  # pylint: disable=useless-object-inheritance,too-many-public-methods
     """
     https://stackoverflow.com/questions/26266481/pytest-reusable-tests-for-different-implementations-of-the-same-interface  # pylint: disable=line-too-long
     Single, reusable definition of tests for the interface. Authors of
@@ -83,12 +83,7 @@ class IntegrationTests(object):  # pylint: disable=useless-object-inheritance,to
     self._filter_post_data_parameters can be defined to provide a list of sensitive post data parameters
     self.provider_variant can be defined as a prefix for saving cassettes when a provider uses multiple variants
 
-    Extended test suites can be skipped by adding the following snippet to the test_{PROVIDER_NAME}.py file
-
-        @pytest.fixture(autouse=True)
-        def _skip_suite(self, request):  # pylint: disable=no-self-use
-            if request.node.get_closest_marker('ext_suite_1'):
-                pytest.skip('Skipping extended suite')
+    IntegrationTestsV1 is used for providers developed before IntegrationTestsV2 has been created.
     """
 
     def __init__(self):
@@ -312,83 +307,7 @@ def test_provider_when_calling_delete_record_by_filter_with_fqdn_name_should_rem
             'TXT', 'delete.testfqdn.{0}.'.format(self.domain))
         assert not records
 
-    ###########################################################################
-    # Extended Test Suite 1 - March 2018 - Validation for Create Record NOOP & Record Sets
-    ###########################################################################
-
-    @pytest.mark.ext_suite_1
-    @_vcr_integration_test
-    def test_provider_when_calling_create_record_with_duplicate_records_should_be_noop(self):
-        provider = self._construct_authenticated_provider()
-        assert provider.create_record(
-            'TXT', "_acme-challenge.noop.{0}.".format(self.domain), 'challengetoken')
-        assert provider.create_record(
-            'TXT', "_acme-challenge.noop.{0}.".format(self.domain), 'challengetoken')
-        records = provider.list_records(
-            'TXT', "_acme-challenge.noop.{0}.".format(self.domain))
-        assert len(records) == 1
-
-    @pytest.mark.ext_suite_1
-    @_vcr_integration_test
-    def test_provider_when_calling_create_record_multiple_times_should_create_record_set(self):
-        provider = self._construct_authenticated_provider()
-        assert provider.create_record(
-            'TXT', "_acme-challenge.createrecordset.{0}.".format(self.domain), 'challengetoken1')
-        assert provider.create_record(
-            'TXT', "_acme-challenge.createrecordset.{0}.".format(self.domain), 'challengetoken2')
-
-    @pytest.mark.ext_suite_1
-    @_vcr_integration_test
-    def test_provider_when_calling_list_records_with_invalid_filter_should_be_empty_list(self):
-        provider = self._construct_authenticated_provider()
-        records = provider.list_records(
-            'TXT', 'filter.thisdoesnotexist.{0}'.format(self.domain))
-        assert not records
-
-    @pytest.mark.ext_suite_1
-    @_vcr_integration_test
-    def test_provider_when_calling_list_records_should_handle_record_sets(self):
-        provider = self._construct_authenticated_provider()
-        provider.create_record(
-            'TXT', "_acme-challenge.listrecordset.{0}.".format(self.domain), 'challengetoken1')
-        provider.create_record(
-            'TXT', "_acme-challenge.listrecordset.{0}.".format(self.domain), 'challengetoken2')
-        records = provider.list_records(
-            'TXT', '_acme-challenge.listrecordset.{0}.'.format(self.domain))
-        assert len(records) == 2
-
-    @pytest.mark.ext_suite_1
-    @_vcr_integration_test
-    def test_provider_when_calling_delete_record_with_record_set_name_remove_all(self):
-        provider = self._construct_authenticated_provider()
-        assert provider.create_record(
-            'TXT', "_acme-challenge.deleterecordset.{0}.".format(self.domain), 'challengetoken1')
-        assert provider.create_record(
-            'TXT', "_acme-challenge.deleterecordset.{0}.".format(self.domain), 'challengetoken2')
-
-        assert provider.delete_record(
-            None, 'TXT', '_acme-challenge.deleterecordset.{0}.'.format(self.domain))
-        records = provider.list_records(
-            'TXT', '_acme-challenge.deleterecordset.{0}.'.format(self.domain))
-        assert not records
-
-    @pytest.mark.ext_suite_1
-    @_vcr_integration_test
-    def test_provider_when_calling_delete_record_with_record_set_by_content_should_leave_others_untouched(self):  # pylint: disable=line-too-long
-        provider = self._construct_authenticated_provider()
-        assert provider.create_record(
-            'TXT', "_acme-challenge.deleterecordinset.{0}.".format(self.domain), 'challengetoken1')
-        assert provider.create_record(
-            'TXT', "_acme-challenge.deleterecordinset.{0}.".format(self.domain), 'challengetoken2')
-
-        assert provider.delete_record(
-            None, 'TXT', '_acme-challenge.deleterecordinset.{0}.'
-            .format(self.domain), 'challengetoken1')
-        records = provider.list_records(
-            'TXT', '_acme-challenge.deleterecordinset.{0}.'.format(self.domain))
-        assert len(records) == 1
-
-    # Private helpers, mimicing the auth_* options provided by the Client
+    # Private helpers, mimicking the auth_* options provided by the Client
     # http://stackoverflow.com/questions/6229073/how-to-make-a-python-dictionary-that-returns-key-for-keys-missing-from-the-dicti
 
     def _test_config(self):
@@ -512,3 +431,78 @@ def _filter_response(self, response):  # pylint: disable=no-self-use
                             status message.
         """
         return response
+
+
+class IntegrationTestsV2(IntegrationTestsV1):
+    """
+    New version of the reusable IntegrationTests class, that includes all tests from
+    IntegrationTestsV1, plus new tests related to validation for Create Record NOOP & Record Sets.
+
+    IntegrationTestsV2 should be used for new providers, and for existing providers during
+    their refactoring.
+    """
+    @_vcr_integration_test
+    def test_provider_when_calling_create_record_with_duplicate_records_should_be_noop(self):
+        provider = self._construct_authenticated_provider()
+        assert provider.create_record(
+            'TXT', "_acme-challenge.noop.{0}.".format(self.domain), 'challengetoken')
+        assert provider.create_record(
+            'TXT', "_acme-challenge.noop.{0}.".format(self.domain), 'challengetoken')
+        records = provider.list_records(
+            'TXT', "_acme-challenge.noop.{0}.".format(self.domain))
+        assert len(records) == 1
+
+    @_vcr_integration_test
+    def test_provider_when_calling_create_record_multiple_times_should_create_record_set(self):
+        provider = self._construct_authenticated_provider()
+        assert provider.create_record(
+            'TXT', "_acme-challenge.createrecordset.{0}.".format(self.domain), 'challengetoken1')
+        assert provider.create_record(
+            'TXT', "_acme-challenge.createrecordset.{0}.".format(self.domain), 'challengetoken2')
+
+    @_vcr_integration_test
+    def test_provider_when_calling_list_records_with_invalid_filter_should_be_empty_list(self):
+        provider = self._construct_authenticated_provider()
+        records = provider.list_records(
+            'TXT', 'filter.thisdoesnotexist.{0}'.format(self.domain))
+        assert not records
+
+    @_vcr_integration_test
+    def test_provider_when_calling_list_records_should_handle_record_sets(self):
+        provider = self._construct_authenticated_provider()
+        provider.create_record(
+            'TXT', "_acme-challenge.listrecordset.{0}.".format(self.domain), 'challengetoken1')
+        provider.create_record(
+            'TXT', "_acme-challenge.listrecordset.{0}.".format(self.domain), 'challengetoken2')
+        records = provider.list_records(
+            'TXT', '_acme-challenge.listrecordset.{0}.'.format(self.domain))
+        assert len(records) == 2
+
+    @_vcr_integration_test
+    def test_provider_when_calling_delete_record_with_record_set_name_remove_all(self):
+        provider = self._construct_authenticated_provider()
+        assert provider.create_record(
+            'TXT', "_acme-challenge.deleterecordset.{0}.".format(self.domain), 'challengetoken1')
+        assert provider.create_record(
+            'TXT', "_acme-challenge.deleterecordset.{0}.".format(self.domain), 'challengetoken2')
+
+        assert provider.delete_record(
+            None, 'TXT', '_acme-challenge.deleterecordset.{0}.'.format(self.domain))
+        records = provider.list_records(
+            'TXT', '_acme-challenge.deleterecordset.{0}.'.format(self.domain))
+        assert not records
+
+    @_vcr_integration_test
+    def test_provider_when_calling_delete_record_with_record_set_by_content_should_leave_others_untouched(self):  # pylint: disable=line-too-long
+        provider = self._construct_authenticated_provider()
+        assert provider.create_record(
+            'TXT', "_acme-challenge.deleterecordinset.{0}.".format(self.domain), 'challengetoken1')
+        assert provider.create_record(
+            'TXT', "_acme-challenge.deleterecordinset.{0}.".format(self.domain), 'challengetoken2')
+
+        assert provider.delete_record(
+            None, 'TXT', '_acme-challenge.deleterecordinset.{0}.'
+            .format(self.domain), 'challengetoken1')
+        records = provider.list_records(
+            'TXT', '_acme-challenge.deleterecordinset.{0}.'.format(self.domain))
+        assert len(records) == 1

lexicon/tests/providers/test_aliyun.py

@@ -2,12 +2,13 @@
 
 # Test for one implementation of the interface
 from unittest import TestCase
-from lexicon.tests.providers.integration_tests import IntegrationTests
+from lexicon.tests.providers.integration_tests import IntegrationTestsV2
+
 
 # Hook into testing framework by inheriting unittest.TestCase and reuse
 # the tests which *each and every* implementation of the interface must
 # pass, by inheritance from integration_tests.IntegrationTests
-class AliyunProviderTests(TestCase, IntegrationTests):
+class AliyunProviderTests(TestCase, IntegrationTestsV2):
     """Integration tests for Foo provider"""
     provider_name = 'aliyun'
     domain = 'mean.space'

lexicon/tests/providers/test_aurora.py

@@ -1,13 +1,13 @@
 """Integration tests for Aurora"""
 from unittest import TestCase
 
-from lexicon.tests.providers.integration_tests import IntegrationTests
+from lexicon.tests.providers.integration_tests import IntegrationTestsV2
 
 
 # Hook into testing framework by inheriting unittest.TestCase and reuse
 # the tests which *each and every* implementation of the interface must
 # pass, by inheritance from define_tests.TheTests
-class AuroraProviderTests(TestCase, IntegrationTests):
+class AuroraProviderTests(TestCase, IntegrationTestsV2):
     """TestCase for Aurora"""
     provider_name = 'aurora'
     domain = 'example.nl'

lexicon/tests/providers/test_auto.py

@@ -4,7 +4,7 @@
 
 import mock
 import pytest
-from lexicon.tests.providers.integration_tests import IntegrationTests
+from lexicon.tests.providers.integration_tests import IntegrationTestsV2
 from lexicon.providers.auto import _get_ns_records_domains_for_domain
 
 
@@ -35,7 +35,7 @@ def _there_is_no_network():
 # Hook into testing framework by inheriting unittest.TestCase and reuse
 # the tests which *each and every* implementation of the interface must
 # pass, by inheritance from integration_tests.IntegrationTests
-class AutoProviderTests(TestCase, IntegrationTests):
+class AutoProviderTests(TestCase, IntegrationTestsV2):
     """TestCase for auto"""
     provider_name = 'auto'
     domain = 'pacalis.net'

lexicon/tests/providers/test_azure.py

@@ -3,15 +3,15 @@
 
 from unittest import TestCase
 
-from lexicon.tests.providers.integration_tests import IntegrationTests
+from lexicon.tests.providers.integration_tests import IntegrationTestsV2
 
 
 # Hook into testing framework by inheriting unittest.TestCase and reuse
 # the tests which *each and every* implementation of the interface must
 # pass, by inheritance from integration_tests.IntegrationTests
 
 
-class AzureTests(TestCase, IntegrationTests):
+class AzureTests(TestCase, IntegrationTestsV2):
     """TestCase for Google Cloud DNS"""
     provider_name = 'azure'
     domain = 'full4ir.tk'

lexicon/tests/providers/test_cloudflare.py

@@ -1,13 +1,13 @@
 """Integration tests for Cloudflare"""
 from unittest import TestCase
 
-from lexicon.tests.providers.integration_tests import IntegrationTests
+from lexicon.tests.providers.integration_tests import IntegrationTestsV2
 
 
 # Hook into testing framework by inheriting unittest.TestCase and reuse
 # the tests which *each and every* implementation of the interface must
 # pass, by inheritance from define_tests.TheTests
-class CloudflareProviderTests(TestCase, IntegrationTests):
+class CloudflareProviderTests(TestCase, IntegrationTestsV2):
     """TestCase for Cloudflare"""
     provider_name = 'cloudflare'
     domain = 'pacalis.net'

lexicon/tests/providers/test_cloudns.py

@@ -1,13 +1,13 @@
 """Integration tests for CloudNS"""
 from unittest import TestCase
 
-from lexicon.tests.providers.integration_tests import IntegrationTests
+from lexicon.tests.providers.integration_tests import IntegrationTestsV2
 
 
 # Hook into testing framework by inheriting unittest.TestCase and reuse
 # the tests which *each and every* implementation of the interface must
 # pass, by inheritance from define_tests.TheTests
-class CloudnsProviderTests(TestCase, IntegrationTests):
+class CloudnsProviderTests(TestCase, IntegrationTestsV2):
     """TestCase for CloudNS"""
     provider_name = 'cloudns'
     domain = 'api-example.com'

lexicon/tests/providers/test_cloudxns.py

@@ -2,23 +2,23 @@
 from unittest import TestCase
 
 import pytest
-from lexicon.tests.providers.integration_tests import IntegrationTests
+from lexicon.tests.providers.integration_tests import IntegrationTestsV2
 
 
 # Hook into testing framework by inheriting unittest.TestCase and reuse
 # the tests which *each and every* implementation of the interface must
 # pass, by inheritance from define_tests.TheTests
 
 
-class CloudXNSProviderTests(TestCase, IntegrationTests):
+class CloudXNSProviderTests(TestCase, IntegrationTestsV2):
     """TestCase for CloudXNS"""
     provider_name = 'cloudxns'
     domain = 'capsulecd.com'
 
     def _filter_post_data_parameters(self):
         return ['login_token']
 
-    # TODO: the following skipped suite and fixtures should be enabled
+    # TODO: enable the skipped tests
     @pytest.mark.skip(reason="new test, missing recording")
     def test_provider_when_calling_update_record_should_modify_record_name_specified(self):
         return

lexicon/tests/providers/test_conoha.py

@@ -1,13 +1,13 @@
 """Integration tests for Conoha"""
 from unittest import TestCase
 
-from lexicon.tests.providers.integration_tests import IntegrationTests
+from lexicon.tests.providers.integration_tests import IntegrationTestsV2
 
 
 # Hook into testing framework by inheriting unittest.TestCase and reuse
 # the tests which *each and every* implementation of the interface must
 # pass, by inheritance from define_tests.TheTests
-class ConohaProviderTests(TestCase, IntegrationTests):
+class ConohaProviderTests(TestCase, IntegrationTestsV2):
     """TestCase for Conoha"""
     provider_name = 'conoha'
     domain = 'narusejun.com'

lexicon/tests/providers/test_constellix.py

@@ -1,13 +1,13 @@
 """Integration tests for Constellix"""
 from unittest import TestCase
 
-from lexicon.tests.providers.integration_tests import IntegrationTests
+from lexicon.tests.providers.integration_tests import IntegrationTestsV2
 
 
 # Constellix does not currently have a sandbox and they enforce domain
 # uniqueness across the service.  You'll need your own production credentials
 # and a unique domain name if you want to run these tests natively.
-class ConstellixProviderTests(TestCase, IntegrationTests):
+class ConstellixProviderTests(TestCase, IntegrationTestsV2):
     """TestCase for Constellix"""
     provider_name = 'constellix'
     domain = 'example.org'

lexicon/tests/providers/test_digitalocean.py

@@ -2,13 +2,13 @@
 from unittest import TestCase
 
 import pytest
-from lexicon.tests.providers.integration_tests import IntegrationTests
+from lexicon.tests.providers.integration_tests import IntegrationTestsV2
 
 
 # Hook into testing framework by inheriting unittest.TestCase and reuse
 # the tests which *each and every* implementation of the interface must
 # pass, by inheritance from define_tests.TheTests
-class DigitalOceanProviderTests(TestCase, IntegrationTests):
+class DigitalOceanProviderTests(TestCase, IntegrationTestsV2):
     """TestCase for DigitalOcean"""
     provider_name = 'digitalocean'
     domain = 'foxwoodswebsites.com'