Add --create-external-resources pytest flag to opt into running end-t…

…o-end tests (#324)

dask_cloudprovider/aws/tests/test_ec2.py

@@ -24,20 +24,23 @@ async def skip_without_credentials():
 
 
 @pytest.fixture
+@pytest.mark.external
 async def cluster():
     await skip_without_credentials()
     async with EC2Cluster(asynchronous=True) as cluster:
         yield cluster
 
 
 @pytest.fixture
+@pytest.mark.external
 async def cluster_sync():
     await skip_without_credentials()
     cluster = EC2Cluster()
     yield cluster
 
 
 @pytest.fixture
+@pytest.mark.external
 async def cluster_rapids():
     await skip_without_credentials()
     async with EC2Cluster(
@@ -54,6 +57,7 @@ async def cluster_rapids():
 
 
 @pytest.fixture
+@pytest.mark.external
 async def cluster_rapids_packer():
     await skip_without_credentials()
     async with EC2Cluster(
@@ -70,6 +74,7 @@ async def cluster_rapids_packer():
 
 
 @pytest.fixture
+@pytest.mark.external
 async def cluster_packer():
     await skip_without_credentials()
     async with EC2Cluster(
@@ -86,6 +91,7 @@ async def ec2_client():
 
 
 @pytest.mark.asyncio
+@pytest.mark.external
 async def test_init():
     cluster = EC2Cluster(asynchronous=True)
     assert cluster.status == Status.created

dask_cloudprovider/azure/tests/test_azurevm.py

@@ -48,6 +48,7 @@ async def get_config():
 
 @pytest.mark.asyncio
 @skip_without_credentials
+@pytest.mark.external
 async def test_init():
     cluster = AzureVMCluster(asynchronous=True)
     assert cluster.status == Status.created
@@ -56,6 +57,7 @@ async def test_init():
 @pytest.mark.asyncio
 @pytest.mark.timeout(1200)
 @skip_without_credentials
+@pytest.mark.external
 async def test_create_cluster():
     async with AzureVMCluster(asynchronous=True) as cluster:
         assert cluster.status == Status.running
@@ -75,6 +77,7 @@ def inc(x):
 @pytest.mark.asyncio
 @pytest.mark.timeout(1200)
 @skip_without_credentials
+@pytest.mark.external
 async def test_create_cluster_sync():
 
     with AzureVMCluster() as cluster:
@@ -92,6 +95,7 @@ def inc(x):
 @pytest.mark.asyncio
 @pytest.mark.timeout(1200)
 @skip_without_credentials
+@pytest.mark.external
 async def test_create_rapids_cluster_sync():
 
     with AzureVMCluster(

dask_cloudprovider/conftest.py

@@ -0,0 +1,28 @@
+import pytest
+
+
+def pytest_addoption(parser):
+    parser.addoption(
+        "--create-external-resources",
+        action="store_true",
+        default=False,
+        help="Run tests that create external resources.",
+    )
+
+
+def pytest_configure(config):
+    config.addinivalue_line(
+        "markers", "external: mark test as creates external resources"
+    )
+
+
+def pytest_collection_modifyitems(config, items):
+    if config.getoption("--create-external-resources"):
+        # --runslow given in cli: do not skip slow tests
+        return
+    skip_slow = pytest.mark.skip(
+        reason="need --create-external-resources option to run"
+    )
+    for item in items:
+        if "external" in item.keywords:
+            item.add_marker(skip_slow)

dask_cloudprovider/digitalocean/tests/test_droplet.py

@@ -36,20 +36,23 @@ async def config():
 
 
 @pytest.fixture
+@pytest.mark.external
 async def cluster(config):
     await skip_without_credentials(config)
     async with DropletCluster(asynchronous=True) as cluster:
         yield cluster
 
 
 @pytest.mark.asyncio
+@pytest.mark.external
 async def test_init():
     cluster = DropletCluster(asynchronous=True)
     assert cluster.status == Status.created
 
 
 @pytest.mark.asyncio
 @pytest.mark.timeout(600)
+@pytest.mark.external
 async def test_create_cluster(cluster):
     assert cluster.status == Status.running
 

dask_cloudprovider/gcp/tests/test_gcp.py

@@ -64,6 +64,7 @@ async def test_get_cloud_init():
 
 @pytest.mark.asyncio
 @pytest.mark.timeout(1200)
+@pytest.mark.external
 async def test_create_cluster():
     skip_without_credentials()
 
@@ -93,6 +94,7 @@ def check_env():
 
 @pytest.mark.asyncio
 @pytest.mark.timeout(1200)
+@pytest.mark.external
 async def test_create_cluster_sync():
     skip_without_credentials()
 
@@ -107,6 +109,7 @@ def inc(x):
 
 @pytest.mark.asyncio
 @pytest.mark.timeout(1200)
+@pytest.mark.external
 async def test_create_rapids_cluster():
     skip_without_credentials()
 
@@ -149,6 +152,7 @@ def gpu_mem():
 
 
 @pytest.mark.timeout(1200)
+@pytest.mark.external
 def test_create_rapids_cluster_sync():
     skip_without_credentials()
     cluster = GCPCluster(

dask_cloudprovider/hetzner/tests/test_vserver.py

@@ -36,6 +36,7 @@ async def config():
 
 
 @pytest.fixture
+@pytest.mark.external
 async def cluster(config):
     await skip_without_credentials(config)
     async with HetznerCluster(asynchronous=True) as cluster:

doc/source/index.rst

@@ -84,4 +84,5 @@ this code.
     :hidden:
     :caption: Developer
 
+    testing.rst
     releasing.rst

doc/source/testing.rst

@@ -0,0 +1,41 @@
+Testing
+=======
+
+Tests in ``dask-cloudprovider`` and written and run using ``pytest``.
+
+To set up your testing environment run:
+
+.. code-block:: bash
+
+    pip install -r requirements_test.txt
+
+To run tests run ``pytest`` from the root directory
+
+.. code-block:: bash
+
+    pytest
+
+You may notice that many tests will be skipped. This is because those tests create external resources on cloud providers. You can set those tests to run with the
+``--create-external-resources`` flag.
+
+.. warning::
+
+   Running tests that create external resources are slow and will cost a small amount of credit on each cloud provider.
+
+.. code-block:: bash
+
+    pytest -rs --create-external-resources
+
+It is also helpful to set the ``-rs`` flag here because tests may also skip if you do not have appropriate credentials to create those external resources.
+If this is the case the skip reason will contain instructions on how to set up those credentials. For example
+
+.. code-block::
+
+    SKIPPED [1] dask_cloudprovider/azure/tests/test_azurevm.py:49:
+        You must configure your Azure resource group and vnet to run this test.
+
+            $ export DASK_CLOUDPROVIDER__AZURE__LOCATION="<LOCATION>"
+            $ export DASK_CLOUDPROVIDER__AZURE__AZUREVM__RESOURCE_GROUP="<RESOURCE GROUP>"
+            $ export DASK_CLOUDPROVIDER__AZURE__AZUREVM__VNET="<VNET>"
+            $ export DASK_CLOUDPROVIDER__AZURE__AZUREVM__SECURITY_GROUP="<SECUROTY GROUP>"
+