From cd752df6154c403e6dccaf5e797c1d4f8396f756 Mon Sep 17 00:00:00 2001
From: Daniel Watkins <oddbloke@ubuntu.com>
Date: Wed, 18 Nov 2020 09:48:47 -0500
Subject: [PATCH] only run a subset of integration tests in CI (#672)

This introduces the "ci" mark, used to indicate a test which should run
as part of our CI integration testing run and the integration-tests-ci
tox environment, which runs only those tests. Travis has been adjusted
to use this tox environment.

(All current module tests have been marked with the "ci" mark, but the
one bug test that we have has not.)
---
 .travis.yml                                               | 2 +-
 .../modules/test_apt_configure_sources_list.py            | 1 +
 tests/integration_tests/modules/test_ntp_servers.py       | 1 +
 tests/integration_tests/modules/test_runcmd.py            | 1 +
 tests/integration_tests/modules/test_seed_random_data.py  | 1 +
 tests/integration_tests/modules/test_set_hostname.py      | 1 +
 tests/integration_tests/modules/test_set_password.py      | 2 ++
 tests/integration_tests/modules/test_snap.py              | 1 +
 .../modules/test_ssh_auth_key_fingerprints.py             | 1 +
 tests/integration_tests/modules/test_ssh_generate.py      | 1 +
 tests/integration_tests/modules/test_ssh_import_id.py     | 1 +
 tests/integration_tests/modules/test_ssh_keys_provided.py | 1 +
 tests/integration_tests/modules/test_timezone.py          | 1 +
 tests/integration_tests/modules/test_users_groups.py      | 1 +
 tests/integration_tests/modules/test_write_files.py       | 1 +
 tox.ini                                                   | 8 ++++++++
 16 files changed, 24 insertions(+), 1 deletion(-)

diff --git a/.travis.yml b/.travis.yml
index 496c1a8111a..6d25f477c99 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -199,7 +199,7 @@ matrix:
                 fi
             # Use sudo to get a new shell where we're in the sbuild group
             - sudo -E su $USER -c 'sbuild --nolog --no-run-lintian --verbose --dist=xenial cloud-init_*.dsc'
-            - sg lxd -c 'CLOUD_INIT_IMAGE_SOURCE="$(ls *.deb)" tox -e integration-tests' &
+            - sg lxd -c 'CLOUD_INIT_IMAGE_SOURCE="$(ls *.deb)" tox -e integration-tests-ci' &
             - |
               SECONDS=0
               while [ -e /proc/$! ]; do
diff --git a/tests/integration_tests/modules/test_apt_configure_sources_list.py b/tests/integration_tests/modules/test_apt_configure_sources_list.py
index d64b3956618..d2bcc61a7fe 100644
--- a/tests/integration_tests/modules/test_apt_configure_sources_list.py
+++ b/tests/integration_tests/modules/test_apt_configure_sources_list.py
@@ -39,6 +39,7 @@
 ]
 
 
+@pytest.mark.ci
 class TestAptConfigureSourcesList:
 
     @pytest.mark.user_data(USER_DATA)
diff --git a/tests/integration_tests/modules/test_ntp_servers.py b/tests/integration_tests/modules/test_ntp_servers.py
index 4cad8926dd8..e72389c123c 100644
--- a/tests/integration_tests/modules/test_ntp_servers.py
+++ b/tests/integration_tests/modules/test_ntp_servers.py
@@ -22,6 +22,7 @@
 EXPECTED_SERVERS = yaml.safe_load(USER_DATA)["ntp"]["servers"]
 
 
+@pytest.mark.ci
 @pytest.mark.user_data(USER_DATA)
 class TestNtpServers:
 
diff --git a/tests/integration_tests/modules/test_runcmd.py b/tests/integration_tests/modules/test_runcmd.py
index eabe778dc68..50d1851eff5 100644
--- a/tests/integration_tests/modules/test_runcmd.py
+++ b/tests/integration_tests/modules/test_runcmd.py
@@ -16,6 +16,7 @@
 """
 
 
+@pytest.mark.ci
 class TestRuncmd:
 
     @pytest.mark.user_data(USER_DATA)
diff --git a/tests/integration_tests/modules/test_seed_random_data.py b/tests/integration_tests/modules/test_seed_random_data.py
index db3d2193001..b365fa98ad9 100644
--- a/tests/integration_tests/modules/test_seed_random_data.py
+++ b/tests/integration_tests/modules/test_seed_random_data.py
@@ -19,6 +19,7 @@
 """
 
 
+@pytest.mark.ci
 class TestSeedRandomData:
 
     @pytest.mark.user_data(USER_DATA)
diff --git a/tests/integration_tests/modules/test_set_hostname.py b/tests/integration_tests/modules/test_set_hostname.py
index ff46feb9386..2bfa403d0af 100644
--- a/tests/integration_tests/modules/test_set_hostname.py
+++ b/tests/integration_tests/modules/test_set_hostname.py
@@ -25,6 +25,7 @@
 """
 
 
+@pytest.mark.ci
 class TestHostname:
 
     @pytest.mark.user_data(USER_DATA_HOSTNAME)
diff --git a/tests/integration_tests/modules/test_set_password.py b/tests/integration_tests/modules/test_set_password.py
index ae6fdefc654..b13f76fbfde 100644
--- a/tests/integration_tests/modules/test_set_password.py
+++ b/tests/integration_tests/modules/test_set_password.py
@@ -139,11 +139,13 @@ def test_sshd_config(self, class_client):
         assert "PasswordAuthentication yes" in sshd_config.splitlines()
 
 
+@pytest.mark.ci
 @pytest.mark.user_data(LIST_USER_DATA)
 class TestPasswordList(Mixin):
     """Launch an instance with LIST_USER_DATA, ensure Mixin tests pass."""
 
 
+@pytest.mark.ci
 @pytest.mark.user_data(STRING_USER_DATA)
 class TestPasswordListString(Mixin):
     """Launch an instance with STRING_USER_DATA, ensure Mixin tests pass."""
diff --git a/tests/integration_tests/modules/test_snap.py b/tests/integration_tests/modules/test_snap.py
index d78a0b1e472..b626f6b03ad 100644
--- a/tests/integration_tests/modules/test_snap.py
+++ b/tests/integration_tests/modules/test_snap.py
@@ -19,6 +19,7 @@
 """
 
 
+@pytest.mark.ci
 class TestSnap:
 
     @pytest.mark.user_data(USER_DATA)
diff --git a/tests/integration_tests/modules/test_ssh_auth_key_fingerprints.py b/tests/integration_tests/modules/test_ssh_auth_key_fingerprints.py
index e88d9a02eb8..b9b0d85ef3a 100644
--- a/tests/integration_tests/modules/test_ssh_auth_key_fingerprints.py
+++ b/tests/integration_tests/modules/test_ssh_auth_key_fingerprints.py
@@ -28,6 +28,7 @@
 """  # noqa
 
 
+@pytest.mark.ci
 class TestSshAuthkeyFingerprints:
 
     @pytest.mark.user_data(USER_DATA_SSH_AUTHKEY_DISABLE)
diff --git a/tests/integration_tests/modules/test_ssh_generate.py b/tests/integration_tests/modules/test_ssh_generate.py
index 8c60fb874b7..60c369826b4 100644
--- a/tests/integration_tests/modules/test_ssh_generate.py
+++ b/tests/integration_tests/modules/test_ssh_generate.py
@@ -20,6 +20,7 @@
 """
 
 
+@pytest.mark.ci
 @pytest.mark.user_data(USER_DATA)
 class TestSshKeysGenerate:
 
diff --git a/tests/integration_tests/modules/test_ssh_import_id.py b/tests/integration_tests/modules/test_ssh_import_id.py
index 2f2ac92c7a8..45d37d6ca33 100644
--- a/tests/integration_tests/modules/test_ssh_import_id.py
+++ b/tests/integration_tests/modules/test_ssh_import_id.py
@@ -17,6 +17,7 @@
 """
 
 
+@pytest.mark.ci
 class TestSshImportId:
 
     @pytest.mark.user_data(USER_DATA)
diff --git a/tests/integration_tests/modules/test_ssh_keys_provided.py b/tests/integration_tests/modules/test_ssh_keys_provided.py
index 4699518d027..dc6d2fc14b2 100644
--- a/tests/integration_tests/modules/test_ssh_keys_provided.py
+++ b/tests/integration_tests/modules/test_ssh_keys_provided.py
@@ -78,6 +78,7 @@
 """  # noqa
 
 
+@pytest.mark.ci
 @pytest.mark.user_data(USER_DATA)
 class TestSshKeysProvided:
 
diff --git a/tests/integration_tests/modules/test_timezone.py b/tests/integration_tests/modules/test_timezone.py
index 6080d79eeca..111d53f7380 100644
--- a/tests/integration_tests/modules/test_timezone.py
+++ b/tests/integration_tests/modules/test_timezone.py
@@ -15,6 +15,7 @@
 """
 
 
+@pytest.mark.ci
 class TestTimezone:
 
     @pytest.mark.user_data(USER_DATA)
diff --git a/tests/integration_tests/modules/test_users_groups.py b/tests/integration_tests/modules/test_users_groups.py
index b1fa8c22202..6a085a8fb4c 100644
--- a/tests/integration_tests/modules/test_users_groups.py
+++ b/tests/integration_tests/modules/test_users_groups.py
@@ -38,6 +38,7 @@
 """
 
 
+@pytest.mark.ci
 @pytest.mark.user_data(USER_DATA)
 class TestUsersGroups:
     @pytest.mark.parametrize(
diff --git a/tests/integration_tests/modules/test_write_files.py b/tests/integration_tests/modules/test_write_files.py
index d7032a0c675..15832ae3caf 100644
--- a/tests/integration_tests/modules/test_write_files.py
+++ b/tests/integration_tests/modules/test_write_files.py
@@ -44,6 +44,7 @@
 """.format(B64_CONTENT.decode("ascii"))
 
 
+@pytest.mark.ci
 @pytest.mark.user_data(USER_DATA)
 class TestWriteFiles:
 
diff --git a/tox.ini b/tox.ini
index 32174dee87f..4320ab879c9 100644
--- a/tox.ini
+++ b/tox.ini
@@ -146,6 +146,13 @@ passenv = CLOUD_INIT_*
 deps =
     -r{toxinidir}/integration-requirements.txt
 
+[testenv:integration-tests-ci]
+commands = {[testenv:integration-tests]commands}
+passenv = {[testenv:integration-tests]passenv}
+deps = {[testenv:integration-tests]deps}
+setenv =
+    PYTEST_ADDOPTS="-k ci"
+
 [pytest]
 # TODO: s/--strict/--strict-markers/ once xenial support is dropped
 testpaths = cloudinit tests/unittests
@@ -153,6 +160,7 @@ addopts = --strict
 markers =
     allow_subp_for: allow subp usage for the given commands (disable_subp_usage)
     allow_all_subp: allow all subp usage (disable_subp_usage)
+    ci: run this integration test as part of CI test runs
     ds_sys_cfg: a sys_cfg dict to be used by datasource fixtures
     ec2: test will only run on EC2 platform
     gce: test will only run on GCE platform