Revamp integration tests, run in parallel

.github/workflows/lint-and-test.yml

@@ -161,6 +161,7 @@ jobs:
     - name: pytest tests/unit_tests
       run: |
         poetry run pytest \
+          -rsxfE \
           --cov=hvac \
           --cov-report=xml:reports/coverage_units_py${{ matrix.python-version }}.xml \
           tests/unit_tests
@@ -242,6 +243,7 @@ jobs:
         COVFILE: coverage_integration_py${{ matrix.python-version }}_${{ matrix.vault-version }}.xml
       run: |
         poetry run pytest \
+          -rsxfE \
           --cov=hvac \
           --cov-report=xml:reports/${COVFILE//[^A-Za-z0-9\-_\.]/_} \
           tests/integration_tests

docs/overview.rst

@@ -109,13 +109,13 @@ Read and write to secrets engines
 KV Secrets Engine - Version 2
 """""""""""""""""""""""""""""
 
+.. testsetup:: kvv2
+
+    client = manager.client
 
 .. doctest:: kvv2
    :skipif: client.sys.retrieve_mount_option('secret', 'version', '1') != '2'
 
-    >>> # Retrieve an authenticated hvac.Client() instance
-    >>> client = test_utils.create_client()
-    >>>
     >>> # Write a k/v pair under path: secret/foo
     >>> create_response = client.secrets.kv.v2.create_or_update_secret(
     ...     path='foo',

pyproject.toml

@@ -74,6 +74,10 @@ greenlet = "^3.0.0"
 jwcrypto = "^1.5.0"
 typos = "^1.16.11"
 pytest-mock = "^3.11.1"
+pytest-xdist = "^3.3.1"
+
+[tool.pytest.ini_options]
+addopts = "-n auto --dist worksteal"
 
 [tool.typos.default.extend-words]
 Hashi = "Hashi"

tests/config_files/generated/.gitignore

@@ -0,0 +1,2 @@
+*
+!.gitignore

tests/config_files/vault-doctest.hcl

@@ -2,6 +2,7 @@ backend "inmem" {
 }
 
 listener "tcp" {
+  address = "127.0.0.1:8200"
   tls_cert_file = "../tests/config_files/server-cert.pem"
   tls_key_file  = "../tests/config_files/server-key.pem"
 }

tests/config_files/vault-ha-node1.hcl

@@ -1,5 +1,5 @@
 listener "tcp" {
-  address = "127.0.0.1:8200"
+  // address = "127.0.0.1:8200"
   tls_cert_file = "tests/config_files/server-cert.pem"
   tls_key_file  = "tests/config_files/server-key.pem"
 }
@@ -11,5 +11,5 @@ max_lease_ttl = "768h"
 
 storage "consul" {
   address = "127.0.0.1:8500"
-  path    = "vault"
+  path    = "vault_123/"
 }

tests/config_files/vault-ha-node2.hcl

@@ -1,6 +1,6 @@
 listener "tcp" {
-  address = "127.0.0.1:8199"
-  cluster_address = "127.0.0.1:8201"
+  // address = "127.0.0.1:8198"
+  # cluster_address = "127.0.0.1:8201"
   tls_cert_file = "tests/config_files/server-cert.pem"
   tls_key_file  = "tests/config_files/server-key.pem"
 }
@@ -12,5 +12,5 @@ max_lease_ttl = "768h"
 
 storage "consul" {
   address = "127.0.0.1:8500"
-  path    = "vault"
+  path    = "vault_123/"
 }

tests/doctest/__init__.py

@@ -11,23 +11,24 @@
 
 
 def doctest_global_setup():
-    client = test_utils.create_client()
     manager = ServerManager(
         config_paths=[test_utils.get_config_file_path("vault-doctest.hcl")],
-        client=client,
+        patch_config=False,
     )
     manager.start()
     manager.initialize()
     manager.unseal()
 
+    client = manager.client
+
     mocker = Mocker(real_http=True)
     mocker.start()
 
     auth_method_paths = [
         f"ldap/login/{MockLdapServer.ldap_user_name}",
     ]
     for auth_method_path in auth_method_paths:
-        mock_url = f"https://127.0.0.1:8200/v1/auth/{auth_method_path}"
+        mock_url = f"{client.url}/v1/auth/{auth_method_path}"
         mock_response = {
             "auth": {
                 "client_token": manager.root_token,

tests/integration_tests/api/auth_methods/test_cert.py

@@ -10,7 +10,7 @@ class TestCert(HvacIntegrationTestCase, TestCase):
     TEST_MOUNT_POINT = "cert-test"
     TEST_ROLE_NAME = "testrole"
     TEST_CLIENT_CERTIFICATE_FILE = utils.get_config_file_path("client-cert.pem")
-    cert = utils.create_client()._adapter._kwargs.get("cert")
+    cert = utils.create_client(url="fake")._adapter._kwargs.get("cert")
     with open(TEST_CLIENT_CERTIFICATE_FILE, "r") as fp:
         TEST_CERTIFICATE = fp.read()
 

tests/integration_tests/api/auth_methods/test_jwt.py

@@ -37,11 +37,11 @@ def tearDown(self):
         [
             param(
                 "configure using vault identity OIDC",
-                issuer="https://localhost:8200",
             ),
         ]
     )
-    def test_configure(self, label, issuer):
+    def test_configure(self, label):
+        issuer = self.client.url
         oidc_discovery_url = f"{issuer}/v1/identity/oidc"
         self.client.secrets.identity.configure_tokens_backend(
             issuer=issuer,
@@ -63,11 +63,11 @@ def test_configure(self, label, issuer):
         [
             param(
                 "configure using vault identity OIDC",
-                issuer="https://localhost:8200",
             ),
         ]
     )
-    def test_read_config(self, label, issuer):
+    def test_read_config(self, label):
+        issuer = self.client.url
         oidc_discovery_url = f"{issuer}/v1/identity/oidc"
         self.client.secrets.identity.configure_tokens_backend(
             issuer=issuer,
@@ -94,12 +94,15 @@ def test_read_config(self, label, issuer):
             param(
                 "success",
                 role_name="hvac",
-                allowed_redirect_uris=["https://localhost:8200/jwt-test/callback"],
+                allowed_redirect_uris=["{url}/jwt-test/callback"],
                 user_claim="https://vault/user",
             ),
         ]
     )
     def test_create_role(self, label, role_name, allowed_redirect_uris, user_claim):
+        allowed_redirect_uris = [
+            uri.format(url=self.client.url) for uri in allowed_redirect_uris
+        ]
         response = self.client.auth.jwt.create_role(
             name=role_name,
             allowed_redirect_uris=allowed_redirect_uris,
@@ -124,12 +127,15 @@ def test_create_role(self, label, role_name, allowed_redirect_uris, user_claim):
             param(
                 "success",
                 role_name="hvac",
-                allowed_redirect_uris=["https://localhost:8200/jwt-test/callback"],
+                allowed_redirect_uris=["{url}/jwt-test/callback"],
                 user_claim="https://vault/user",
             ),
         ]
     )
     def test_read_role(self, label, role_name, allowed_redirect_uris, user_claim):
+        allowed_redirect_uris = [
+            uri.format(url=self.client.url) for uri in allowed_redirect_uris
+        ]
         create_role_response = self.client.auth.jwt.create_role(
             name=role_name,
             allowed_redirect_uris=allowed_redirect_uris,
@@ -153,12 +159,15 @@ def test_read_role(self, label, role_name, allowed_redirect_uris, user_claim):
             param(
                 "success",
                 role_name="hvac",
-                allowed_redirect_uris=["https://localhost:8200/jwt-test/callback"],
+                allowed_redirect_uris=["{url}/jwt-test/callback"],
                 user_claim="https://vault/user",
             ),
         ]
     )
     def test_list_roles(self, label, role_name, allowed_redirect_uris, user_claim):
+        allowed_redirect_uris = [
+            uri.format(url=self.client.url) for uri in allowed_redirect_uris
+        ]
         create_role_response = self.client.auth.jwt.create_role(
             name=role_name,
             allowed_redirect_uris=allowed_redirect_uris,
@@ -181,12 +190,15 @@ def test_list_roles(self, label, role_name, allowed_redirect_uris, user_claim):
             param(
                 "success",
                 role_name="hvac",
-                allowed_redirect_uris=["https://localhost:8200/jwt-test/callback"],
+                allowed_redirect_uris=["{url}/jwt-test/callback"],
                 user_claim="https://vault/user",
             ),
         ]
     )
     def test_delete_role(self, label, role_name, allowed_redirect_uris, user_claim):
+        allowed_redirect_uris = [
+            uri.format(url=self.client.url) for uri in allowed_redirect_uris
+        ]
         create_role_response = self.client.auth.jwt.create_role(
             name=role_name,
             allowed_redirect_uris=allowed_redirect_uris,
@@ -209,16 +221,17 @@ def test_delete_role(self, label, role_name, allowed_redirect_uris, user_claim):
         [
             param(
                 "success",
-                issuer="https://localhost:8200",
                 role_name="hvac-jwt",
-                allowed_redirect_uris=["https://localhost:8200/jwt-test/oidc/callback"],
+                allowed_redirect_uris=["{url}/jwt-test/oidc/callback"],
                 user_claim="sub",
             ),
         ]
     )
-    def test_jwt_login(
-        self, label, issuer, role_name, allowed_redirect_uris, user_claim
-    ):
+    def test_jwt_login(self, label, role_name, allowed_redirect_uris, user_claim):
+        issuer = self.client.url
+        allowed_redirect_uris = [
+            uri.format(url=self.client.url) for uri in allowed_redirect_uris
+        ]
         if "%s/" % self.TEST_APPROLE_PATH not in self.client.sys.list_auth_methods():
             self.client.sys.enable_auth_method(
                 method_type="approle",
@@ -253,10 +266,10 @@ def test_jwt_login(
         logging.debug("create_named_key response: %s" % create_named_key_response)
 
         self.client.secrets.identity.configure_tokens_backend(
-            issuer="https://localhost:8200",
+            issuer=issuer,
         )
         response = self.client.auth.jwt.configure(
-            jwks_url="https://localhost:8200/v1/identity/oidc/.well-known/keys",
+            jwks_url=f"{issuer}/v1/identity/oidc/.well-known/keys",
             jwks_ca_pem="".join(
                 open(utils.get_config_file_path("server-cert.pem")).readlines()
             ),