Merge pull request #41 from mattsb42-aws/dev-23

moving test selection to pytest markers

.travis.yml

@@ -3,13 +3,37 @@ language: python
 matrix:
   include:
     - python: 2.7
-      env: TOXENV=py27
+      env: TOXENV=py27-local
+    - python: 2.7
+      env: TOXENV=py27-integ
+    - python: 2.7
+      env: TOXENV=py27-accept
+    - python: 2.7
+      env: TOXENV=py27-examples
+    - python: 3.4
+      env: TOXENV=py34-local
     - python: 3.4
-      env: TOXENV=py34
+      env: TOXENV=py34-integ
+    - python: 3.4
+      env: TOXENV=py34-accept
+    - python: 3.4
+      env: TOXENV=py34-examples
+    - python: 3.5
+      env: TOXENV=py35-local
     - python: 3.5
-      env: TOXENV=py35
+      env: TOXENV=py35-integ
+    - python: 3.5
+      env: TOXENV=py35-accept
+    - python: 3.5
+      env: TOXENV=py35-examples
+    - python: 3.6
+      env: TOXENV=py36-local
     - python: 3.6
-      env: TOXENV=py36
+      env: TOXENV=py36-integ
+    - python: 3.6
+      env: TOXENV=py36-accept
+    - python: 3.6
+      env: TOXENV=py36-examples
     - python: 3.6
       env: TOXENV=bandit
     - python: 3.6
@@ -27,8 +51,6 @@ matrix:
       env: TOXENV=flake8-tests
     - python: 3.6
       env: TOXENV=pylint-tests
-    - python: 3.6
-      env: TOXENV=examples
     - python: 3.6
       env: TOXENV=flake8-examples
     - python: 3.6

examples/test/test_i_basic_encryption.py

@@ -22,10 +22,11 @@
 import pytest
 
 from basic_encryption import cycle_string
-from integration_test_utils import get_cmk_arn, SKIP_MESSAGE, skip_tests
+from integration_test_utils import get_cmk_arn
+
+pytestmark = [pytest.mark.examples]
 
 
-@pytest.mark.skipif(skip_tests(), reason=SKIP_MESSAGE)
 def test_cycle_string():
     plaintext = os.urandom(1024)
     cmk_arn = get_cmk_arn()

examples/test/test_i_basic_file_encryption_with_multiple_providers.py

@@ -23,10 +23,11 @@
 import pytest
 
 from basic_file_encryption_with_multiple_providers import cycle_file
-from integration_test_utils import get_cmk_arn, SKIP_MESSAGE, skip_tests
+from integration_test_utils import get_cmk_arn
+
+pytestmark = [pytest.mark.examples]
 
 
-@pytest.mark.skipif(skip_tests(), reason=SKIP_MESSAGE)
 def test_cycle_file():
     cmk_arn = get_cmk_arn()
     _handle, filename = tempfile.mkstemp()

examples/test/test_i_basic_file_encryption_with_raw_key_provider.py

@@ -22,10 +22,10 @@
 import pytest
 
 from basic_file_encryption_with_raw_key_provider import cycle_file
-from integration_test_utils import SKIP_MESSAGE, skip_tests
+
+pytestmark = [pytest.mark.examples]
 
 
-@pytest.mark.skipif(skip_tests(), reason=SKIP_MESSAGE)
 def test_cycle_file():
     _handle, filename = tempfile.mkstemp()
     with open(filename, 'wb') as f:

examples/test/test_i_data_key_caching_basic.py

@@ -21,10 +21,11 @@
 import pytest
 
 from data_key_caching_basic import encrypt_with_caching
-from integration_test_utils import get_cmk_arn, SKIP_MESSAGE, skip_tests
+from integration_test_utils import get_cmk_arn
+
+pytestmark = [pytest.mark.examples]
 
 
-@pytest.mark.skipif(skip_tests(), reason=SKIP_MESSAGE)
 def test_encrypt_with_caching():
     cmk_arn = get_cmk_arn()
     encrypt_with_caching(

setup.cfg

@@ -10,6 +10,15 @@ branch = True
 [coverage:report]
 show_missing = True
 
+[tool:pytest]
+markers =
+    local: superset of unit and functional (does not require network access)
+    unit: mark test as a unit test (does not require network access)
+    functional: mark test as a functional test (does not require network access)
+    integ: mark a test as an integration test (requires network access)
+    accept: mark a test as an acceptance test (requires network access)
+    examples: mark a test as an examples test (requires network access)
+
 # Flake8 Configuration
 [flake8]
 max_complexity = 10

test/functional/test_f_aws_encryption_sdk_client.py

@@ -34,6 +34,8 @@
 from aws_encryption_sdk.key_providers.raw import RawMasterKeyProvider
 from aws_encryption_sdk.materials_managers import DecryptionMaterialsRequest, EncryptionMaterialsRequest
 
+pytestmark = [pytest.mark.functional, pytest.mark.local]
+
 VALUES = {
     'frame_lengths': (  # Assuming 1280 byte plaintext:
         0,  # Non-framed
@@ -347,7 +349,10 @@ def test_encryption_cycle_raw_mkp(wrapping_algorithm, encryption_key_type, decry
     assert plaintext == VALUES['plaintext_128']
 
 
-@pytest.mark.skipif(not _mgf1_sha256_supported(), reason='MGF1-SHA256 not supported by this backend')
+@pytest.mark.skipif(
+    not _mgf1_sha256_supported(),
+    reason='MGF1-SHA256 not supported by this backend: OpenSSL required v1.0.2+'
+)
 @pytest.mark.parametrize('wrapping_algorithm, encryption_key_type, decryption_key_type', (
     (WrappingAlgorithm.RSA_OAEP_SHA256_MGF1, EncryptionKeyType.PRIVATE, EncryptionKeyType.PRIVATE),
     (WrappingAlgorithm.RSA_OAEP_SHA256_MGF1, EncryptionKeyType.PUBLIC, EncryptionKeyType.PRIVATE)

test/functional/test_f_crypto.py

@@ -20,6 +20,8 @@
 from aws_encryption_sdk.internal.crypto.authentication import Signer
 from aws_encryption_sdk.internal.crypto.elliptic_curve import _ecc_static_length_signature
 
+pytestmark = [pytest.mark.functional, pytest.mark.local]
+
 
 # Run several of each type to make get a high probability of forcing signature length correction
 @pytest.mark.parametrize('algorithm', [

test/functional/test_f_crypto_iv.py

@@ -17,6 +17,8 @@
 from aws_encryption_sdk.internal.crypto.iv import frame_iv, header_auth_iv, non_framed_body_iv
 from aws_encryption_sdk.internal.defaults import ALGORITHM, MAX_FRAME_COUNT
 
+pytestmark = [pytest.mark.functional, pytest.mark.local]
+
 VALUES = {
     'ivs': {
         'header_auth': b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00',

test/functional/test_f_xcompat.py

@@ -29,6 +29,8 @@
 from aws_encryption_sdk.internal.str_ops import to_bytes
 from aws_encryption_sdk.key_providers.raw import RawMasterKeyProvider
 
+pytestmark = [pytest.mark.accept]
+
 
 # Environment-specific test file locator.  May not always exist.
 def _file_root():

test/integration/README.rst

@@ -7,7 +7,6 @@ In order to run these integration tests successfully, these things must be confi
 #. Ensure that AWS credentials are available in one of the `automatically discoverable credential locations`_.
 #. Set environment variable ``AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID`` to valid
    `AWS KMS key id`_ to use for integration tests.
-#. Set environment variable ``AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_CONTROL`` to ``RUN``.
 
 .. _automatically discoverable credential locations: http://boto3.readthedocs.io/en/latest/guide/configuration.html
 .. _AWS KMS key id: http://docs.aws.amazon.com/kms/latest/APIReference/API_Encrypt.html

test/integration/integration_test_utils.py

@@ -10,30 +10,26 @@
 # distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
 # ANY KIND, either express or implied. See the License for the specific
 # language governing permissions and limitations under the License.
-"""Utility functions to handle configuration, credentials setup, and test skip
-decision making for integration tests."""
+"""Utility functions to handle configuration and credentials setup for integration tests."""
 import os
 
 from aws_encryption_sdk.key_providers.kms import KMSMasterKeyProvider
 
-SKIP_MESSAGE = (
-    'Required environment variables not found. Skipping integration tests.'
-    ' See integration tests README.rst for more information.'
-)
-TEST_CONTROL = 'AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_CONTROL'
 AWS_KMS_KEY_ID = 'AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID'
 
 
-def skip_tests():
-    """Only run tests if both required environment variables are found."""
-    test_control = os.environ.get(TEST_CONTROL, None)
-    key_id = os.environ.get(AWS_KMS_KEY_ID, None)
-    return not (test_control == 'RUN' and key_id is not None)
-
-
 def get_cmk_arn():
     """Retrieves the target CMK ARN from environment variable."""
-    return os.environ.get(AWS_KMS_KEY_ID)
+    arn = os.environ.get(AWS_KMS_KEY_ID, None)
+    if arn is None:
+        raise ValueError(
+            'Environment variable "{}" must be set to a valid KMS CMK ARN for integration tests to run'.format(
+                AWS_KMS_KEY_ID
+            )
+        )
+    if arn.startswith('arn:') and ':alias/' not in arn:
+        return arn
+    raise ValueError('KMS CMK ARN provided for integration tests much be a key not an alias')
 
 
 def setup_kms_master_key_provider():

test/integration/test_i_aws_encrytion_sdk_client.py

@@ -14,9 +14,13 @@
 import io
 import unittest
 
+import pytest
+
 import aws_encryption_sdk
 from aws_encryption_sdk.identifiers import Algorithm
-from .integration_test_utils import setup_kms_master_key_provider, SKIP_MESSAGE, skip_tests
+from .integration_test_utils import setup_kms_master_key_provider
+
+pytestmark = [pytest.mark.integ]
 
 
 VALUES = {
@@ -39,8 +43,6 @@
 class TestKMSThickClientIntegration(unittest.TestCase):
 
     def setUp(self):
-        if skip_tests():
-            self.skipTest(SKIP_MESSAGE)
         self.kms_master_key_provider = setup_kms_master_key_provider()
 
     def test_encryption_cycle_default_algorithm_framed_stream(self):

test/integration/test_i_thread_safety.py

@@ -22,7 +22,9 @@
 from six.moves import queue  # six.moves confuses pylint: disable=import-error
 
 import aws_encryption_sdk
-from .integration_test_utils import setup_kms_master_key_provider, SKIP_MESSAGE, skip_tests
+from .integration_test_utils import setup_kms_master_key_provider
+
+pytestmark = [pytest.mark.integ]
 
 
 PLAINTEXT = (
@@ -101,7 +103,6 @@ def random_pause_time(max_seconds=3):
     return SystemRandom().random() * 10 % max_seconds
 
 
-@pytest.mark.skipif(skip_tests(), reason=SKIP_MESSAGE)
 def test_threading_loop():
     """Test thread safety of client."""
     rounds = 20
@@ -127,7 +128,6 @@ def test_threading_loop():
     assert all(value == PLAINTEXT for value in decrypted_values)
 
 
-@pytest.mark.skipif(skip_tests(), reason=SKIP_MESSAGE)
 def test_threading_loop_with_common_cache():
     """Test thread safety of client while using common cryptographic materials cache across all threads."""
     rounds = 20

test/integration/test_i_xcompat_kms.py

@@ -17,7 +17,9 @@
 import pytest
 
 import aws_encryption_sdk
-from .integration_test_utils import setup_kms_master_key_provider, SKIP_MESSAGE, skip_tests
+from .integration_test_utils import setup_kms_master_key_provider
+
+pytestmark = [pytest.mark.accept]
 
 
 # Environment-specific test file locator.  May not always exist.
@@ -32,9 +34,6 @@ def _file_root():
 
 
 def _generate_test_cases():
-    if skip_tests():
-        return []
-
     kms_key_provider = setup_kms_master_key_provider()
     try:
         root_dir = os.path.abspath(file_root())
@@ -73,7 +72,6 @@ def _generate_test_cases():
     return _test_cases
 
 
-@pytest.mark.skipif(skip_tests(), reason=SKIP_MESSAGE)
 @pytest.mark.parametrize('plaintext_filename,ciphertext_filename,key_provider', _generate_test_cases())
 def test_decrypt_from_file(plaintext_filename, ciphertext_filename, key_provider):
     """Tests decrypt from known good files."""

test/unit/test_aws_encryption_sdk.py

@@ -14,11 +14,14 @@
 import unittest
 
 from mock import MagicMock, patch, sentinel
+import pytest
 import six
 
 import aws_encryption_sdk
 import aws_encryption_sdk.internal.defaults
 
+pytestmark = [pytest.mark.unit, pytest.mark.local]
+
 
 class TestAwsEncryptionSdk(unittest.TestCase):
 

test/unit/test_caches.py

@@ -26,6 +26,8 @@
 from aws_encryption_sdk.materials_managers import DecryptionMaterialsRequest, EncryptionMaterialsRequest
 from aws_encryption_sdk.structures import DataKey, MasterKeyInfo
 
+pytestmark = [pytest.mark.unit, pytest.mark.local]
+
 
 VALUES = {
     'basic': {

test/unit/test_caches_base.py

@@ -15,6 +15,8 @@
 
 from aws_encryption_sdk.caches.base import CryptoMaterialsCache
 
+pytestmark = [pytest.mark.unit, pytest.mark.local]
+
 
 def test_abstracts():
     with pytest.raises(TypeError) as excinfo:

test/unit/test_caches_crypto_cache_entry.py

@@ -22,6 +22,8 @@
 from aws_encryption_sdk.exceptions import NotSupportedError
 from aws_encryption_sdk.materials_managers import DecryptionMaterials, EncryptionMaterials
 
+pytestmark = [pytest.mark.unit, pytest.mark.local]
+
 _VALID_KWARGS = {
     'CryptoMaterialsCacheEntryHints': dict(lifetime=5.0),
     'CryptoMaterialsCacheEntry': dict(

test/unit/test_caches_local.py

@@ -22,6 +22,8 @@
 from aws_encryption_sdk.caches.local import _OPPORTUNISTIC_EVICTION_ROUNDS, LocalCryptoMaterialsCache
 from aws_encryption_sdk.exceptions import CacheKeyError, NotSupportedError
 
+pytestmark = [pytest.mark.unit, pytest.mark.local]
+
 
 def build_lcmc(**custom_kwargs):
     kwargs = dict(capacity=10)

test/unit/test_caches_null.py

@@ -19,6 +19,8 @@
 from aws_encryption_sdk.exceptions import CacheKeyError
 from aws_encryption_sdk.materials_managers import DecryptionMaterials, EncryptionMaterials
 
+pytestmark = [pytest.mark.unit, pytest.mark.local]
+
 
 def test_put_encryption_materials():
     cache_key = b'ex_cache_key'

test/unit/test_crypto_authentication_signer.py

@@ -20,6 +20,8 @@
 from aws_encryption_sdk.internal.defaults import ALGORITHM
 from .test_crypto import VALUES
 
+pytestmark = [pytest.mark.unit, pytest.mark.local]
+
 
 @pytest.yield_fixture
 def patch_default_backend(mocker):

test/unit/test_crypto_authentication_verifier.py

@@ -20,6 +20,8 @@
 from aws_encryption_sdk.internal.defaults import ALGORITHM
 from .test_crypto import VALUES
 
+pytestmark = [pytest.mark.unit, pytest.mark.local]
+
 
 @pytest.yield_fixture
 def patch_default_backend(mocker):

test/unit/test_crypto_data_keys.py

@@ -18,6 +18,8 @@
 import aws_encryption_sdk.internal.crypto.data_keys
 from aws_encryption_sdk.internal.crypto.data_keys import derive_data_encryption_key
 
+pytestmark = [pytest.mark.unit, pytest.mark.local]
+
 
 @pytest.yield_fixture
 def patch_default_backend(mocker):

test/unit/test_crypto_elliptic_curve.py

@@ -28,6 +28,8 @@
 )
 from .test_crypto import VALUES
 
+pytestmark = [pytest.mark.unit, pytest.mark.local]
+
 
 @pytest.yield_fixture
 def patch_default_backend(mocker):