From 5d457897265a42243034592c6db9cea72e6a0a6a Mon Sep 17 00:00:00 2001 From: Luciano Lo Giudice Date: Fri, 5 Apr 2024 18:05:40 -0300 Subject: [PATCH 01/43] Implement tests for ceph-mon's rotate-key action This PR implements tests for the ceph-mon charm's action 'rotate-key'. For now, the charm only supports key rotation for managers, but more entities will be added. This PR will be used to test those new entities as well. --- zaza/openstack/charm_tests/ceph/tests.py | 36 ++++++++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/zaza/openstack/charm_tests/ceph/tests.py b/zaza/openstack/charm_tests/ceph/tests.py index 46911c07c..83d7ccd52 100644 --- a/zaza/openstack/charm_tests/ceph/tests.py +++ b/zaza/openstack/charm_tests/ceph/tests.py @@ -1748,3 +1748,39 @@ def test_persistent_config(self): ) data = json.loads(result['Stdout']) assert data['loglevel'] == 2 + + +class CephMonKeyRotationTests(test_utils.BaseCharmTest): + """Tests for the rotate-key action.""" + + def _get_key_for_mgr(self, unit, mgr_id): + cmd = 'sudo ceph auth ls' + result = zaza_model.run_on_unit(unit, cmd) + # Don't use json formatting, as it's buggy upstream. + data = result['Stdout'].split() + + for ix, line in enumerate(data): + # Structure: + # mgr.XXXX + # key: + # key contents + # That's why we need to move 2 positions ahead. + if line.startswith('mgr.') and line.endswith(mgr_id): + return data[ix + 2] + + def test_mgr_key_rotate(self): + """Test that rotating the manager key actually changes it.""" + unit = 'ceph-mon/0' + mgr_id = unit[-1] + old_key = self._get_key_for_mgr(unit, mgr_id) + self.assertIsNotNone(old_key) + + action_obj = zaza_model.run_action( + unit_name=unit, + action_name='rotate-key', + action_params={'entity': 'mgr'} + ) + zaza_utils.assertActionRanOk(action_obj) + new_key = self._get_key_for_mgr(unit, mgr_id) + self.assertIsNotNone(new_key) + self.assertNotEqual(old_key, new_key) From 3989d543f6fccbc7541cc960db5a59797390340a Mon Sep 17 00:00:00 2001 From: Luciano Lo Giudice Date: Mon, 8 Apr 2024 11:20:48 -0300 Subject: [PATCH 02/43] Get the keys for all managers to compare --- zaza/openstack/charm_tests/ceph/tests.py | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/zaza/openstack/charm_tests/ceph/tests.py b/zaza/openstack/charm_tests/ceph/tests.py index 83d7ccd52..5c2b7217f 100644 --- a/zaza/openstack/charm_tests/ceph/tests.py +++ b/zaza/openstack/charm_tests/ceph/tests.py @@ -1753,11 +1753,12 @@ def test_persistent_config(self): class CephMonKeyRotationTests(test_utils.BaseCharmTest): """Tests for the rotate-key action.""" - def _get_key_for_mgr(self, unit, mgr_id): + def _get_mgrs_keys(self, unit): cmd = 'sudo ceph auth ls' result = zaza_model.run_on_unit(unit, cmd) # Don't use json formatting, as it's buggy upstream. data = result['Stdout'].split() + ret = set() for ix, line in enumerate(data): # Structure: @@ -1765,22 +1766,20 @@ def _get_key_for_mgr(self, unit, mgr_id): # key: # key contents # That's why we need to move 2 positions ahead. - if line.startswith('mgr.') and line.endswith(mgr_id): - return data[ix + 2] + if line.startswith('mgr.'): + ret.add(data[ix + 2]) + return ret def test_mgr_key_rotate(self): """Test that rotating the manager key actually changes it.""" unit = 'ceph-mon/0' - mgr_id = unit[-1] - old_key = self._get_key_for_mgr(unit, mgr_id) - self.assertIsNotNone(old_key) + old_keys = self._get_mgrs_keys(unit) action_obj = zaza_model.run_action( unit_name=unit, action_name='rotate-key', action_params={'entity': 'mgr'} ) - zaza_utils.assertActionRanOk(action_obj) - new_key = self._get_key_for_mgr(unit, mgr_id) - self.assertIsNotNone(new_key) - self.assertNotEqual(old_key, new_key) + zaza_utils.assertActionRanOK(action_obj) + new_keys = self._get_mgrs_keys(unit) + self.assertNotEqual(old_keys, new_keys) From 99456d9229c26a526d83b20c2412dd9bc1ffc36a Mon Sep 17 00:00:00 2001 From: Luciano Lo Giudice Date: Tue, 9 Apr 2024 11:59:43 -0300 Subject: [PATCH 03/43] Adjust test so it can be used for more than managers --- zaza/openstack/charm_tests/ceph/tests.py | 42 +++++++++++++++++------- 1 file changed, 31 insertions(+), 11 deletions(-) diff --git a/zaza/openstack/charm_tests/ceph/tests.py b/zaza/openstack/charm_tests/ceph/tests.py index 5c2b7217f..049be0797 100644 --- a/zaza/openstack/charm_tests/ceph/tests.py +++ b/zaza/openstack/charm_tests/ceph/tests.py @@ -1753,7 +1753,7 @@ def test_persistent_config(self): class CephMonKeyRotationTests(test_utils.BaseCharmTest): """Tests for the rotate-key action.""" - def _get_mgrs_keys(self, unit): + def _get_all_keys(self, unit, entity_filter): cmd = 'sudo ceph auth ls' result = zaza_model.run_on_unit(unit, cmd) # Don't use json formatting, as it's buggy upstream. @@ -1762,24 +1762,44 @@ def _get_mgrs_keys(self, unit): for ix, line in enumerate(data): # Structure: - # mgr.XXXX + # $ENTITY # key: # key contents - # That's why we need to move 2 positions ahead. - if line.startswith('mgr.'): - ret.add(data[ix + 2]) + # That's why we need to move one position ahead. + if 'key:' in line and entity_filter(line[ix - 1]): + ret.add(data[ix + 1]) return ret - def test_mgr_key_rotate(self): - """Test that rotating the manager key actually changes it.""" - unit = 'ceph-mon/0' - old_keys = self._get_mgrs_keys(unit) + def _check_key_rotation(self, entity, unit): + def entity_filter(name): + return name.startswith(entity) + old_keys = self._get_all_keys(unit, entity_filter) action_obj = zaza_model.run_action( unit_name=unit, action_name='rotate-key', - action_params={'entity': 'mgr'} + action_params={'entity': entity} ) zaza_utils.assertActionRanOK(action_obj) - new_keys = self._get_mgrs_keys(unit) + zaza_model.wait_for_application_states() + new_keys = self._get_all_keys(unit, entity_filter) self.assertNotEqual(old_keys, new_keys) + self.assertEqual(new_keys - old_keys, 1) + + def _get_rgw_client(self, unit): + cmd = 'sudo ceph auth ls | grep client.rgw' + result = zaza_model.run_on_unit(unit, cmd) + return result['Stdout'].strip() + + def test_key_rotate(self): + """Test that rotating the keys actually changes them.""" + unit = 'ceph-mon/0' + self._check_key_rotation('mgr', unit) + + try: + zaza_model.get_application('ceph-radosgw') + rgw_client = self._get_rgw_client('ceph-radosgw/0') + if rgw_client: + self._check_key_rotation(rgw_client, unit) + except KeyError: + pass From b99a6c428a078e133678abd8b464925e77abb9d1 Mon Sep 17 00:00:00 2001 From: Luciano Lo Giudice Date: Tue, 9 Apr 2024 13:47:25 -0300 Subject: [PATCH 04/43] Fix access to keys --- zaza/openstack/charm_tests/ceph/tests.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/zaza/openstack/charm_tests/ceph/tests.py b/zaza/openstack/charm_tests/ceph/tests.py index 049be0797..4e51d3cf3 100644 --- a/zaza/openstack/charm_tests/ceph/tests.py +++ b/zaza/openstack/charm_tests/ceph/tests.py @@ -1766,7 +1766,7 @@ def _get_all_keys(self, unit, entity_filter): # key: # key contents # That's why we need to move one position ahead. - if 'key:' in line and entity_filter(line[ix - 1]): + if 'key:' in line and entity_filter(data[ix - 1]): ret.add(data[ix + 1]) return ret @@ -1784,7 +1784,7 @@ def entity_filter(name): zaza_model.wait_for_application_states() new_keys = self._get_all_keys(unit, entity_filter) self.assertNotEqual(old_keys, new_keys) - self.assertEqual(new_keys - old_keys, 1) + self.assertEqual(len(new_keys - old_keys), 1) def _get_rgw_client(self, unit): cmd = 'sudo ceph auth ls | grep client.rgw' From 761913176c50c5e2b953e944341634c3f99fa0c8 Mon Sep 17 00:00:00 2001 From: Luciano Lo Giudice Date: Tue, 9 Apr 2024 19:28:40 -0300 Subject: [PATCH 05/43] Make sure the result is the type of entity we're after --- zaza/openstack/charm_tests/ceph/tests.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/zaza/openstack/charm_tests/ceph/tests.py b/zaza/openstack/charm_tests/ceph/tests.py index 4e51d3cf3..c12188414 100644 --- a/zaza/openstack/charm_tests/ceph/tests.py +++ b/zaza/openstack/charm_tests/ceph/tests.py @@ -1767,7 +1767,7 @@ def _get_all_keys(self, unit, entity_filter): # key contents # That's why we need to move one position ahead. if 'key:' in line and entity_filter(data[ix - 1]): - ret.add(data[ix + 1]) + ret.add((data[ix - 1], data[ix + 1])) return ret def _check_key_rotation(self, entity, unit): @@ -1784,7 +1784,13 @@ def entity_filter(name): zaza_model.wait_for_application_states() new_keys = self._get_all_keys(unit, entity_filter) self.assertNotEqual(old_keys, new_keys) - self.assertEqual(len(new_keys - old_keys), 1) + diff = new_keys - old_keys + self.assertEqual(len(diff), 1) + first = next(iter(diff)) + # Check that the entity matches. The 'entity_filter' + # callable will return a true-like value if it + # matches the type of entity we're after (i.e: 'mgr') + self.assertTrue(entity_filter(first[0])) def _get_rgw_client(self, unit): cmd = 'sudo ceph auth ls | grep client.rgw' @@ -1801,5 +1807,7 @@ def test_key_rotate(self): rgw_client = self._get_rgw_client('ceph-radosgw/0') if rgw_client: self._check_key_rotation(rgw_client, unit) + else: + logging.info('ceph-radosgw units present, but no RGW service') except KeyError: pass From 8fe9a110edbce62acdbddf4013b1ae8dec78659e Mon Sep 17 00:00:00 2001 From: Luciano Lo Giudice Date: Tue, 9 Apr 2024 19:47:24 -0300 Subject: [PATCH 06/43] Also test key rotation in OSD's --- zaza/openstack/charm_tests/ceph/tests.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/zaza/openstack/charm_tests/ceph/tests.py b/zaza/openstack/charm_tests/ceph/tests.py index c12188414..06843b97d 100644 --- a/zaza/openstack/charm_tests/ceph/tests.py +++ b/zaza/openstack/charm_tests/ceph/tests.py @@ -1811,3 +1811,9 @@ def test_key_rotate(self): logging.info('ceph-radosgw units present, but no RGW service') except KeyError: pass + + try: + zaza_model.get_application('ceph-osd') + self._check_key_rotation('osd.0', unit) + except KeyError: + pass From 99167326cec2aa1378c77ac1379fd2a81a227ab1 Mon Sep 17 00:00:00 2001 From: Peter Sabaini Date: Wed, 10 Apr 2024 11:50:12 +0200 Subject: [PATCH 07/43] Fix race in setting up certs Closes bug #2060785 --- zaza/openstack/charm_tests/ceph/tests.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/zaza/openstack/charm_tests/ceph/tests.py b/zaza/openstack/charm_tests/ceph/tests.py index 46911c07c..9e6f327b1 100644 --- a/zaza/openstack/charm_tests/ceph/tests.py +++ b/zaza/openstack/charm_tests/ceph/tests.py @@ -1264,7 +1264,13 @@ def test_005_virtual_hosted_bucket(self): endpoint_url=primary_endpoint, aws_access_key_id=access_key, aws_secret_access_key=secret_key) - primary_client.Bucket(container_name).create() + # We may not have certs for the pub hostname yet, so retry a few times. + for attempt in tenacity.Retrying( + stop=tenacity.stop_after_attempt(10), + wait=tenacity.wait_fixed(4), + ): + with attempt: + primary_client.Bucket(container_name).create() primary_object_one = primary_client.Object( container_name, obj_name From f411c0b0b1adfd1a883b54e7da46613d85146ff3 Mon Sep 17 00:00:00 2001 From: Luciano Lo Giudice Date: Wed, 10 Apr 2024 13:08:41 -0300 Subject: [PATCH 08/43] Use already existing functionality to fetch RGW client --- zaza/openstack/charm_tests/ceph/tests.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/zaza/openstack/charm_tests/ceph/tests.py b/zaza/openstack/charm_tests/ceph/tests.py index 06843b97d..8bf4bf425 100644 --- a/zaza/openstack/charm_tests/ceph/tests.py +++ b/zaza/openstack/charm_tests/ceph/tests.py @@ -1793,9 +1793,10 @@ def entity_filter(name): self.assertTrue(entity_filter(first[0])) def _get_rgw_client(self, unit): - cmd = 'sudo ceph auth ls | grep client.rgw' - result = zaza_model.run_on_unit(unit, cmd) - return result['Stdout'].strip() + ret = self._get_all_keys(unit, 'client.rgw') + if not ret: + return None + return next(iter(ret))[0] def test_key_rotate(self): """Test that rotating the keys actually changes them.""" From d10c243f8d1db7091730f813ea72a5b43c560e71 Mon Sep 17 00:00:00 2001 From: Luciano Lo Giudice Date: Wed, 10 Apr 2024 13:41:40 -0300 Subject: [PATCH 09/43] Comment out OSD tests until PR is ready --- zaza/openstack/charm_tests/ceph/tests.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/zaza/openstack/charm_tests/ceph/tests.py b/zaza/openstack/charm_tests/ceph/tests.py index 8bf4bf425..c39434560 100644 --- a/zaza/openstack/charm_tests/ceph/tests.py +++ b/zaza/openstack/charm_tests/ceph/tests.py @@ -1813,8 +1813,8 @@ def test_key_rotate(self): except KeyError: pass - try: - zaza_model.get_application('ceph-osd') - self._check_key_rotation('osd.0', unit) - except KeyError: - pass + #try: + # zaza_model.get_application('ceph-osd') + # self._check_key_rotation('osd.0', unit) + #except KeyError: + # pass From eee7611ccbbb8084e70f9d33c899fc61c11a4208 Mon Sep 17 00:00:00 2001 From: Luciano Lo Giudice Date: Wed, 10 Apr 2024 13:45:12 -0300 Subject: [PATCH 10/43] PEP8 fix --- zaza/openstack/charm_tests/ceph/tests.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/zaza/openstack/charm_tests/ceph/tests.py b/zaza/openstack/charm_tests/ceph/tests.py index c39434560..88f6ad6e0 100644 --- a/zaza/openstack/charm_tests/ceph/tests.py +++ b/zaza/openstack/charm_tests/ceph/tests.py @@ -1813,8 +1813,8 @@ def test_key_rotate(self): except KeyError: pass - #try: - # zaza_model.get_application('ceph-osd') - # self._check_key_rotation('osd.0', unit) - #except KeyError: - # pass + # try: + # zaza_model.get_application('ceph-osd') + # self._check_key_rotation('osd.0', unit) + # except KeyError: + # pass From cbc117208e32a291d7478792a4a17d018607af5e Mon Sep 17 00:00:00 2001 From: Luciano Lo Giudice Date: Thu, 11 Apr 2024 13:14:45 -0300 Subject: [PATCH 11/43] Use the right unit to get the RGW client --- zaza/openstack/charm_tests/ceph/tests.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zaza/openstack/charm_tests/ceph/tests.py b/zaza/openstack/charm_tests/ceph/tests.py index e000f42b2..d693276c7 100644 --- a/zaza/openstack/charm_tests/ceph/tests.py +++ b/zaza/openstack/charm_tests/ceph/tests.py @@ -1811,7 +1811,7 @@ def test_key_rotate(self): try: zaza_model.get_application('ceph-radosgw') - rgw_client = self._get_rgw_client('ceph-radosgw/0') + rgw_client = self._get_rgw_client(unit) if rgw_client: self._check_key_rotation(rgw_client, unit) else: From 2f557c531d4565b16864aee7ddb0f90744f707db Mon Sep 17 00:00:00 2001 From: Luciano Lo Giudice Date: Thu, 11 Apr 2024 16:06:19 -0300 Subject: [PATCH 12/43] Use the correct callable --- zaza/openstack/charm_tests/ceph/tests.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zaza/openstack/charm_tests/ceph/tests.py b/zaza/openstack/charm_tests/ceph/tests.py index d693276c7..f882c677c 100644 --- a/zaza/openstack/charm_tests/ceph/tests.py +++ b/zaza/openstack/charm_tests/ceph/tests.py @@ -1799,7 +1799,7 @@ def entity_filter(name): self.assertTrue(entity_filter(first[0])) def _get_rgw_client(self, unit): - ret = self._get_all_keys(unit, 'client.rgw') + ret = self._get_all_keys(unit, lambda x: x.startswith('client.rgw')) if not ret: return None return next(iter(ret))[0] From 06e876a7ad52ae457c407f81920ef0bfa685b7a1 Mon Sep 17 00:00:00 2001 From: Luciano Lo Giudice Date: Fri, 12 Apr 2024 13:34:11 -0300 Subject: [PATCH 13/43] Uncomment ceph-osd tests --- zaza/openstack/charm_tests/ceph/tests.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/zaza/openstack/charm_tests/ceph/tests.py b/zaza/openstack/charm_tests/ceph/tests.py index f882c677c..4acf217a6 100644 --- a/zaza/openstack/charm_tests/ceph/tests.py +++ b/zaza/openstack/charm_tests/ceph/tests.py @@ -1819,8 +1819,8 @@ def test_key_rotate(self): except KeyError: pass - # try: - # zaza_model.get_application('ceph-osd') - # self._check_key_rotation('osd.0', unit) - # except KeyError: - # pass + try: + zaza_model.get_application('ceph-osd') + self._check_key_rotation('osd.0', unit) + except KeyError: + pass From e574fc00191cb6838815e551101360c58d9eef78 Mon Sep 17 00:00:00 2001 From: Luciano Lo Giudice Date: Mon, 15 Apr 2024 11:18:08 -0300 Subject: [PATCH 14/43] Temporarily revert OSD testing to get RGW changes merged --- zaza/openstack/charm_tests/ceph/tests.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/zaza/openstack/charm_tests/ceph/tests.py b/zaza/openstack/charm_tests/ceph/tests.py index 4acf217a6..f882c677c 100644 --- a/zaza/openstack/charm_tests/ceph/tests.py +++ b/zaza/openstack/charm_tests/ceph/tests.py @@ -1819,8 +1819,8 @@ def test_key_rotate(self): except KeyError: pass - try: - zaza_model.get_application('ceph-osd') - self._check_key_rotation('osd.0', unit) - except KeyError: - pass + # try: + # zaza_model.get_application('ceph-osd') + # self._check_key_rotation('osd.0', unit) + # except KeyError: + # pass From aa7d619a249b4ec66ffc335f9342a146a13e6197 Mon Sep 17 00:00:00 2001 From: Luciano Lo Giudice Date: Mon, 15 Apr 2024 13:55:44 -0300 Subject: [PATCH 15/43] Add some time to get the unit to settle --- zaza/openstack/charm_tests/ceph/tests.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/zaza/openstack/charm_tests/ceph/tests.py b/zaza/openstack/charm_tests/ceph/tests.py index f882c677c..ca76984db 100644 --- a/zaza/openstack/charm_tests/ceph/tests.py +++ b/zaza/openstack/charm_tests/ceph/tests.py @@ -23,6 +23,7 @@ ) import requests import tempfile +import time import boto3 import botocore.exceptions import urllib3 @@ -1788,6 +1789,7 @@ def entity_filter(name): ) zaza_utils.assertActionRanOK(action_obj) zaza_model.wait_for_application_states() + time.sleep(5) # Give it time to finish. new_keys = self._get_all_keys(unit, entity_filter) self.assertNotEqual(old_keys, new_keys) diff = new_keys - old_keys From da3463cb94ded5c8197c93d0b2de10b979c3124b Mon Sep 17 00:00:00 2001 From: Peter Sabaini Date: Wed, 17 Apr 2024 16:38:39 +0200 Subject: [PATCH 16/43] Fix: ceph-radosgw test should only check radosgw app Don't block on global app state waiting but only check the SUT app state. --- zaza/openstack/charm_tests/ceph/tests.py | 16 +++++----------- 1 file changed, 5 insertions(+), 11 deletions(-) diff --git a/zaza/openstack/charm_tests/ceph/tests.py b/zaza/openstack/charm_tests/ceph/tests.py index 9e6f327b1..0beac49e6 100644 --- a/zaza/openstack/charm_tests/ceph/tests.py +++ b/zaza/openstack/charm_tests/ceph/tests.py @@ -1236,17 +1236,11 @@ def test_005_virtual_hosted_bucket(self): # 0. Configure virtual hosted bucket self.enable_virtual_hosted_bucket() - assert_state = { - self.primary_rgw_app: { - "workload-status": "blocked", - "workload-status-message-prefix": - "os-public-hostname must have a value " - "when virtual hosted bucket is enabled" - } - } - zaza_model.wait_for_application_states(self.model_name, - states=assert_state, - timeout=900) + zaza_model.block_until_wl_status_info_starts_with( + self.primary_rgw_app, + 'os-public-hostname must have a value', + timeout=900 + ) self.set_os_public_hostname() zaza_model.block_until_all_units_idle(self.model_name) container_name = 'zaza-bucket' From f090ca685131894fc00da6e8bcef5419db053e6f Mon Sep 17 00:00:00 2001 From: Myles Penner Date: Thu, 18 Apr 2024 11:43:53 -0700 Subject: [PATCH 17/43] Add cinder tests for audit middleware --- zaza/openstack/charm_tests/cinder/tests.py | 40 ++++++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/zaza/openstack/charm_tests/cinder/tests.py b/zaza/openstack/charm_tests/cinder/tests.py index 421d30631..7668127dc 100644 --- a/zaza/openstack/charm_tests/cinder/tests.py +++ b/zaza/openstack/charm_tests/cinder/tests.py @@ -305,6 +305,46 @@ def verify(stdin, stdout, stderr): 'lsblk -sn -o SIZE /dev/vdb', privkey=privkey, verify=verify) + def test_300_apipaste_includes_audit_section(self): + """Test api-paste.ini renders audit section when enabled.""" + service_name = 'cinder' + api_paste_ini_path = f"/etc/{service_name}/api-paste.ini" + expected_content = [ + "[filter:audit]", + "paste.filter_factory = keystonemiddleware.audit:filter_factory", + f"audit_map_file = /etc/{service_name}/api_audit_map.conf", + f"service_name = {service_name}" + ] + + set_default = {'audit-middleware': False} + set_alternate = {'audit-middleware': True} + + with self.config_change(set_default, set_alternate): + try: + api_paste_content = zaza.model.file_contents( + self.lead_unit, + api_paste_ini_path, + ) + except Exception as e: + self.fail("Error fetching api-paste.ini: {}".format(str(e))) + for line in expected_content: + self.assertIn(line, api_paste_content) + + def test_301_apipaste_excludes_audit_section(self): + """Test api_paste.ini does not render audit section when disabled.""" + service_name = 'cinder' + section_heading = '[filter:audit]' + api_paste_ini_path = f"/etc/{service_name}/api-paste.ini" + + try: + api_paste_content = zaza.model.file_contents( + self.lead_unit, + api_paste_ini_path + ) + except Exception as e: + self.fail("Error fetching api-paste.ini: {}".format(str(e))) + self.assertNotIn(section_heading, api_paste_content) + @property def services(self): """Return a list services for the selected OpenStack release.""" From e083e56ee7f8d11cec4ba8f0173cd0c01c7b80e1 Mon Sep 17 00:00:00 2001 From: Luciano Lo Giudice Date: Fri, 19 Apr 2024 15:36:00 -0300 Subject: [PATCH 18/43] Remove OSD tests for initial release --- zaza/openstack/charm_tests/ceph/tests.py | 8 -------- 1 file changed, 8 deletions(-) diff --git a/zaza/openstack/charm_tests/ceph/tests.py b/zaza/openstack/charm_tests/ceph/tests.py index ca76984db..03c2e90c0 100644 --- a/zaza/openstack/charm_tests/ceph/tests.py +++ b/zaza/openstack/charm_tests/ceph/tests.py @@ -23,7 +23,6 @@ ) import requests import tempfile -import time import boto3 import botocore.exceptions import urllib3 @@ -1789,7 +1788,6 @@ def entity_filter(name): ) zaza_utils.assertActionRanOK(action_obj) zaza_model.wait_for_application_states() - time.sleep(5) # Give it time to finish. new_keys = self._get_all_keys(unit, entity_filter) self.assertNotEqual(old_keys, new_keys) diff = new_keys - old_keys @@ -1820,9 +1818,3 @@ def test_key_rotate(self): logging.info('ceph-radosgw units present, but no RGW service') except KeyError: pass - - # try: - # zaza_model.get_application('ceph-osd') - # self._check_key_rotation('osd.0', unit) - # except KeyError: - # pass From a15832695ca4c5db5cd2fcb858a527f93d882277 Mon Sep 17 00:00:00 2001 From: Luciano Lo Giudice Date: Mon, 22 Apr 2024 14:43:46 -0300 Subject: [PATCH 19/43] Implement key rotation tests for OSD units This PR adds OSD tests to the key rotation class for the ceph-mon charm. It will be tested in the following merge requests: https://review.opendev.org/c/openstack/charm-ceph-mon/+/915925 https://review.opendev.org/c/openstack/charm-ceph-osd/+/915926 --- zaza/openstack/charm_tests/ceph/tests.py | 1 + 1 file changed, 1 insertion(+) diff --git a/zaza/openstack/charm_tests/ceph/tests.py b/zaza/openstack/charm_tests/ceph/tests.py index 85de53dcf..10294d6ac 100644 --- a/zaza/openstack/charm_tests/ceph/tests.py +++ b/zaza/openstack/charm_tests/ceph/tests.py @@ -1802,6 +1802,7 @@ def test_key_rotate(self): """Test that rotating the keys actually changes them.""" unit = 'ceph-mon/0' self._check_key_rotation('mgr', unit) + self._check_key_rotation('osd.0', unit) try: zaza_model.get_application('ceph-radosgw') From c0f04747620bfd2d62dd50959c49a96919b90b84 Mon Sep 17 00:00:00 2001 From: Ionut Balutoiu Date: Tue, 19 Mar 2024 00:44:16 +0200 Subject: [PATCH 20/43] CephRGWTest: Reorder test cases * Rename `test_004_migration_and_multisite_failover` to `test_100_migration_and_multisite_failover` * This will allow us to insert more multi-site tests after `test_003`, before scale-down scenario is run in `test_100_migration_and_multisite_failover`. * Rename `test_005_virtual_hosted_bucket` to `test_101_virtual_hosted_bucket`. * This was previously run after `test_004_migration_and_multisite_failover`. So, we rename the test case to `test_101` to keep the same order. Signed-off-by: Ionut Balutoiu --- zaza/openstack/charm_tests/ceph/tests.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/zaza/openstack/charm_tests/ceph/tests.py b/zaza/openstack/charm_tests/ceph/tests.py index 85de53dcf..5c047c7cb 100644 --- a/zaza/openstack/charm_tests/ceph/tests.py +++ b/zaza/openstack/charm_tests/ceph/tests.py @@ -648,7 +648,7 @@ class CephRGWTest(test_utils.BaseCharmTest): This Testset is not idempotent, because we don't support scale down from multisite to singlesite (yet). Tests can be performed independently. - However, If test_004 has completed migration, retriggering the test-set + However, If test_100 has completed migration, retriggering the test-set would cause a time-out in test_003. """ @@ -1066,7 +1066,7 @@ def test_003_object_storage_and_secondary_block(self): zaza_model.block_until_unit_wl_status(self.secondary_rgw_unit, 'active') - def test_004_migration_and_multisite_failover(self): + def test_100_migration_and_multisite_failover(self): """Perform multisite migration and verify failover.""" container_name = 'zaza-container' obj_data = 'Test data from Zaza' @@ -1224,7 +1224,7 @@ def test_004_migration_and_multisite_failover(self): self.purge_bucket(self.secondary_rgw_app, 'zaza-container') self.purge_bucket(self.secondary_rgw_app, 'failover-container') - def test_005_virtual_hosted_bucket(self): + def test_101_virtual_hosted_bucket(self): """Test virtual hosted bucket.""" primary_rgw_unit = zaza_model.get_unit_from_name(self.primary_rgw_unit) if primary_rgw_unit.workload_status != "active": From 3bf0ec72b99053277e5fe4e86877f5fc84c8d6a9 Mon Sep 17 00:00:00 2001 From: Luciano Lo Giudice Date: Wed, 24 Apr 2024 11:29:33 -0300 Subject: [PATCH 21/43] Implement ceph-fs key rotation tests This PR implements tests for key rotation in ceph-fs units. It works similarly to how RGW tests do. --- zaza/openstack/charm_tests/ceph/tests.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/zaza/openstack/charm_tests/ceph/tests.py b/zaza/openstack/charm_tests/ceph/tests.py index 85de53dcf..9e18ca872 100644 --- a/zaza/openstack/charm_tests/ceph/tests.py +++ b/zaza/openstack/charm_tests/ceph/tests.py @@ -1812,3 +1812,13 @@ def test_key_rotate(self): logging.info('ceph-radosgw units present, but no RGW service') except KeyError: pass + + try: + zaza_model.get_application('ceph-fs') + fs_entity = self._get_all_keys(unit, lambda x: x.startswith('mds.')) + if fs_entity is not None: + self._check_key_rotation(next(iter(fs_entity))[0], unit) + else: + logging.info('ceph-fs units present, but no MDS service') + except KeyError: + pass From 42966304799465bf13e8cfe5c80dfe2647862a54 Mon Sep 17 00:00:00 2001 From: Luciano Lo Giudice Date: Wed, 24 Apr 2024 11:38:12 -0300 Subject: [PATCH 22/43] PEP8 change --- zaza/openstack/charm_tests/ceph/tests.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/zaza/openstack/charm_tests/ceph/tests.py b/zaza/openstack/charm_tests/ceph/tests.py index 9e18ca872..86fcf33c9 100644 --- a/zaza/openstack/charm_tests/ceph/tests.py +++ b/zaza/openstack/charm_tests/ceph/tests.py @@ -1815,9 +1815,9 @@ def test_key_rotate(self): try: zaza_model.get_application('ceph-fs') - fs_entity = self._get_all_keys(unit, lambda x: x.startswith('mds.')) - if fs_entity is not None: - self._check_key_rotation(next(iter(fs_entity))[0], unit) + fs_svc = self._get_all_keys(unit, lambda x: x.startswith('mds.')) + if fs_svc is not None: + self._check_key_rotation(next(iter(fs_svc))[0], unit) else: logging.info('ceph-fs units present, but no MDS service') except KeyError: From e68c902fc868f6631b2c447c2a493b98a1e13887 Mon Sep 17 00:00:00 2001 From: Ionut Balutoiu Date: Wed, 24 Apr 2024 19:20:33 +0300 Subject: [PATCH 23/43] CephRGWTest: Wait for sites to be syncronised before recovery scenario If `self.promote_rgw_to_primary(self.primary_rgw_app)` is executed before sites are syncronised, the sites will not be syncronised after. Signed-off-by: Ionut Balutoiu --- zaza/openstack/charm_tests/ceph/tests.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/zaza/openstack/charm_tests/ceph/tests.py b/zaza/openstack/charm_tests/ceph/tests.py index 5c047c7cb..546aa569b 100644 --- a/zaza/openstack/charm_tests/ceph/tests.py +++ b/zaza/openstack/charm_tests/ceph/tests.py @@ -1167,6 +1167,10 @@ def test_100_migration_and_multisite_failover(self): 'Body' ].read().decode('UTF-8') + # Wait for Sites to be syncronised. + self.wait_for_status(self.primary_rgw_app, is_primary=False) + self.wait_for_status(self.secondary_rgw_app, is_primary=True) + # Recovery scenario, reset ceph-rgw as primary. self.promote_rgw_to_primary(self.primary_rgw_app) self.wait_for_status(self.primary_rgw_app, is_primary=True) From 5054a7992cd3c48eedba99dc13197931347f0f93 Mon Sep 17 00:00:00 2001 From: Ionut Balutoiu Date: Tue, 23 Apr 2024 23:04:30 +0300 Subject: [PATCH 24/43] CephRGWTest: Add integration test for multisite sync policies feature Signed-off-by: Ionut Balutoiu --- zaza/openstack/charm_tests/ceph/tests.py | 224 +++++++++++++++++++++++ 1 file changed, 224 insertions(+) diff --git a/zaza/openstack/charm_tests/ceph/tests.py b/zaza/openstack/charm_tests/ceph/tests.py index 546aa569b..d905b9a6a 100644 --- a/zaza/openstack/charm_tests/ceph/tests.py +++ b/zaza/openstack/charm_tests/ceph/tests.py @@ -1066,6 +1066,230 @@ def test_003_object_storage_and_secondary_block(self): zaza_model.block_until_unit_wl_status(self.secondary_rgw_unit, 'active') + def test_004_multisite_directional_sync_policy(self): + """Verify Multisite Directional Sync Policy.""" + # Skip multisite tests if not compatible with bundle. + if not self.multisite: + logging.info('Skipping multisite sync policy verification') + return + + container_name = 'zaza-container' + primary_obj_name = 'primary-testfile' + primary_obj_data = 'Primary test data' + secondary_directional_obj_name = 'secondary-directional-testfile' + secondary_directional_obj_data = 'Secondary directional test data' + secondary_symmetrical_obj_name = 'secondary-symmetrical-testfile' + secondary_symmetrical_obj_data = 'Secondary symmetrical test data' + + logging.info('Verifying multisite directional sync policy') + + # Set default sync policy to "allowed", which allows buckets to sync, + # but the sync is disabled by default in the zone group. Also, set the + # secondary zone sync policy flow type policy to "directional". + zaza_model.set_application_config( + self.primary_rgw_app, + { + "sync-policy-state": "allowed", + } + ) + zaza_model.set_application_config( + self.secondary_rgw_app, + { + "sync-policy-flow-type": "directional", + } + ) + zaza_model.wait_for_unit_idle(self.secondary_rgw_unit) + zaza_model.wait_for_unit_idle(self.primary_rgw_unit) + + # Setup multisite relation. + multisite_relation = zaza_model.get_relation_id( + self.primary_rgw_app, self.secondary_rgw_app, + remote_interface_name='secondary' + ) + if multisite_relation is None: + logging.info('Configuring Multisite') + self.configure_rgw_apps_for_multisite() + zaza_model.add_relation( + self.primary_rgw_app, + self.primary_rgw_app + ":primary", + self.secondary_rgw_app + ":secondary" + ) + zaza_model.block_until_unit_wl_status( + self.secondary_rgw_unit, "waiting" + ) + zaza_model.block_until_unit_wl_status( + self.primary_rgw_unit, "active" + ) + zaza_model.block_until_unit_wl_status( + self.secondary_rgw_unit, "active" + ) + zaza_model.wait_for_unit_idle(self.secondary_rgw_unit) + zaza_model.wait_for_unit_idle(self.primary_rgw_unit) + + logging.info('Waiting for Data and Metadata to Synchronize') + # NOTE: We only check the secondary zone, because the sync policy flow + # type is set to "directional" between the primary and secondary zones. + self.wait_for_status(self.secondary_rgw_app, is_primary=False) + + # Create bucket on primary RGW zone. + logging.info('Creating bucket on primary zone') + primary_endpoint = self.get_rgw_endpoint(self.primary_rgw_unit) + self.assertNotEqual(primary_endpoint, None) + + access_key, secret_key = self.get_client_keys() + primary_client = boto3.resource("s3", + verify=False, + endpoint_url=primary_endpoint, + aws_access_key_id=access_key, + aws_secret_access_key=secret_key) + primary_client.Bucket(container_name).create() + + # Enable sync on the bucket. + logging.info('Enabling sync on the bucket from the primary zone') + zaza_model.run_action_on_leader( + self.primary_rgw_app, + 'enable-buckets-sync', + action_params={ + 'buckets': container_name, + }, + raise_on_failure=True, + ) + + # Check that sync cannot be enabled using secondary Juju RGW app. + with self.assertRaises(zaza_model.ActionFailed): + zaza_model.run_action_on_leader( + self.secondary_rgw_app, + 'enable-buckets-sync', + action_params={ + 'buckets': container_name, + }, + raise_on_failure=True, + ) + + logging.info('Waiting for Data and Metadata to Synchronize') + self.wait_for_status(self.secondary_rgw_app, is_primary=False) + + # Perform IO on primary zone bucket. + logging.info('Performing IO on primary zone bucket') + primary_object = primary_client.Object( + container_name, + primary_obj_name + ) + primary_object.put(Body=primary_obj_data) + + # Verify that the object is replicated to the secondary zone. + logging.info('Verifying that the object is replicated to the ' + 'secondary zone') + secondary_endpoint = self.get_rgw_endpoint(self.secondary_rgw_unit) + self.assertNotEqual(secondary_endpoint, None) + + secondary_client = boto3.resource("s3", + verify=False, + endpoint_url=secondary_endpoint, + aws_access_key_id=access_key, + aws_secret_access_key=secret_key) + secondary_data = self.fetch_rgw_object( + secondary_client, + container_name, + primary_obj_name + ) + self.assertEqual(secondary_data, primary_obj_data) + + # Write object to the secondary zone bucket, when the sync policy + # flow type is set to "directional" between the zones. + logging.info('Writing object to the secondary zone bucket, which ' + 'should not be replicated to the primary zone') + secondary_object = secondary_client.Object( + container_name, + secondary_directional_obj_name + ) + secondary_object.put(Body=secondary_directional_obj_data) + + # Verify that the object is not replicated to the primary zone. + logging.info('Verifying that the object is not replicated to the ' + 'primary zone') + with self.assertRaises(botocore.exceptions.ClientError): + self.fetch_rgw_object( + primary_client, + container_name, + secondary_directional_obj_name + ) + + logging.info('Setting sync policy flow to "symmetrical" on the ' + 'secondary RGW zone') + zaza_model.set_application_config( + self.secondary_rgw_app, + { + "sync-policy-flow-type": "symmetrical", + } + ) + zaza_model.wait_for_unit_idle(self.secondary_rgw_unit) + zaza_model.wait_for_unit_idle(self.primary_rgw_unit) + + # Write another object to the secondary zone bucket. + logging.info('Writing another object to the secondary zone bucket.') + secondary_object = secondary_client.Object( + container_name, + secondary_symmetrical_obj_name + ) + secondary_object.put(Body=secondary_symmetrical_obj_data) + + logging.info('Waiting for Data and Metadata to Synchronize') + # NOTE: This time, we check both the primary and secondary zones, + # because the sync policy flow type is set to "symmetrical" between + # the zones. + self.wait_for_status(self.secondary_rgw_app, is_primary=False) + self.wait_for_status(self.primary_rgw_app, is_primary=True) + + # Verify that all objects are replicated to the primary zone. + logging.info('Verifying that all objects are replicated to the ' + 'primary zone (including older objects).') + test_cases = [ + { + 'obj_name': primary_obj_name, + 'obj_data': primary_obj_data, + }, + { + 'obj_name': secondary_directional_obj_name, + 'obj_data': secondary_directional_obj_data, + }, + { + 'obj_name': secondary_symmetrical_obj_name, + 'obj_data': secondary_symmetrical_obj_data, + }, + ] + for tc in test_cases: + logging.info('Verifying that object "{}" is replicated'.format( + tc['obj_name'])) + primary_data = self.fetch_rgw_object( + primary_client, + container_name, + tc['obj_name'] + ) + self.assertEqual(primary_data, tc['obj_data']) + + # Cleanup. + logging.info('Cleaning up buckets after test case') + self.purge_bucket(self.primary_rgw_app, container_name) + self.purge_bucket(self.secondary_rgw_app, container_name) + + logging.info('Waiting for Data and Metadata to Synchronize') + self.wait_for_status(self.secondary_rgw_app, is_primary=False) + self.wait_for_status(self.primary_rgw_app, is_primary=True) + + # Set multisite sync policy state to "enabled" on the primary RGW app. + # Paired with "symmetrical" sync policy flow on the secondary RGW app, + # this enables bidirectional sync between the zones (which is the + # default behaviour without multisite sync policies configured). + logging.info('Setting sync policy state to "enabled".') + zaza_model.set_application_config( + self.primary_rgw_app, + { + "sync-policy-state": "enabled", + } + ) + zaza_model.wait_for_unit_idle(self.primary_rgw_unit) + def test_100_migration_and_multisite_failover(self): """Perform multisite migration and verify failover.""" container_name = 'zaza-container' From 1ecd7cc9c6ebcf8f425de767c33c145efd23046c Mon Sep 17 00:00:00 2001 From: Luciano Lo Giudice Date: Wed, 24 Apr 2024 14:04:29 -0300 Subject: [PATCH 25/43] Use specific workload messages for the Ubuntu units --- zaza/openstack/charm_tests/ceph/tests.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/zaza/openstack/charm_tests/ceph/tests.py b/zaza/openstack/charm_tests/ceph/tests.py index 86fcf33c9..1bca97355 100644 --- a/zaza/openstack/charm_tests/ceph/tests.py +++ b/zaza/openstack/charm_tests/ceph/tests.py @@ -1770,7 +1770,7 @@ def _get_all_keys(self, unit, entity_filter): ret.add((data[ix - 1], data[ix + 1])) return ret - def _check_key_rotation(self, entity, unit): + def _check_key_rotation(self, entity, unit, states=None): def entity_filter(name): return name.startswith(entity) @@ -1781,7 +1781,7 @@ def entity_filter(name): action_params={'entity': entity} ) zaza_utils.assertActionRanOK(action_obj) - zaza_model.wait_for_application_states() + zaza_model.wait_for_application_states(states=states) new_keys = self._get_all_keys(unit, entity_filter) self.assertNotEqual(old_keys, new_keys) diff = new_keys - old_keys @@ -1817,7 +1817,14 @@ def test_key_rotate(self): zaza_model.get_application('ceph-fs') fs_svc = self._get_all_keys(unit, lambda x: x.startswith('mds.')) if fs_svc is not None: - self._check_key_rotation(next(iter(fs_svc))[0], unit) + ubuntu_states = { + 'ubuntu': { + 'workload-status': 'active', + 'workload-status-message': '' + } + } + self._check_key_rotation(next(iter(fs_svc))[0], unit, + ubuntu_states) else: logging.info('ceph-fs units present, but no MDS service') except KeyError: From d2b38e6279aaba4d22300ac3e0e06c19b3efebfb Mon Sep 17 00:00:00 2001 From: Ionut Balutoiu Date: Thu, 25 Apr 2024 10:41:43 +0300 Subject: [PATCH 26/43] CephRGWTest: Deduplicate code to create multi-site relation Signed-off-by: Ionut Balutoiu --- zaza/openstack/charm_tests/ceph/tests.py | 69 ++++++++++-------------- 1 file changed, 29 insertions(+), 40 deletions(-) diff --git a/zaza/openstack/charm_tests/ceph/tests.py b/zaza/openstack/charm_tests/ceph/tests.py index d905b9a6a..3fe2762a2 100644 --- a/zaza/openstack/charm_tests/ceph/tests.py +++ b/zaza/openstack/charm_tests/ceph/tests.py @@ -878,6 +878,33 @@ def configure_rgw_apps_for_multisite(self): } ) + def configure_rgw_multisite_relation(self): + """Configure multi-site relation between primary and secondary apps.""" + multisite_relation = zaza_model.get_relation_id( + self.primary_rgw_app, self.secondary_rgw_app, + remote_interface_name='secondary' + ) + if multisite_relation is None: + logging.info('Configuring Multisite') + self.configure_rgw_apps_for_multisite() + zaza_model.add_relation( + self.primary_rgw_app, + self.primary_rgw_app + ":primary", + self.secondary_rgw_app + ":secondary" + ) + zaza_model.block_until_unit_wl_status( + self.secondary_rgw_unit, "waiting" + ) + + zaza_model.block_until_unit_wl_status( + self.secondary_rgw_unit, "active" + ) + zaza_model.block_until_unit_wl_status( + self.primary_rgw_unit, "active" + ) + zaza_model.wait_for_unit_idle(self.secondary_rgw_unit) + zaza_model.wait_for_unit_idle(self.primary_rgw_unit) + def clean_rgw_multisite_config(self, app_name): """Clear Multisite Juju config values to default. @@ -1102,29 +1129,7 @@ def test_004_multisite_directional_sync_policy(self): zaza_model.wait_for_unit_idle(self.primary_rgw_unit) # Setup multisite relation. - multisite_relation = zaza_model.get_relation_id( - self.primary_rgw_app, self.secondary_rgw_app, - remote_interface_name='secondary' - ) - if multisite_relation is None: - logging.info('Configuring Multisite') - self.configure_rgw_apps_for_multisite() - zaza_model.add_relation( - self.primary_rgw_app, - self.primary_rgw_app + ":primary", - self.secondary_rgw_app + ":secondary" - ) - zaza_model.block_until_unit_wl_status( - self.secondary_rgw_unit, "waiting" - ) - zaza_model.block_until_unit_wl_status( - self.primary_rgw_unit, "active" - ) - zaza_model.block_until_unit_wl_status( - self.secondary_rgw_unit, "active" - ) - zaza_model.wait_for_unit_idle(self.secondary_rgw_unit) - zaza_model.wait_for_unit_idle(self.primary_rgw_unit) + self.configure_rgw_multisite_relation() logging.info('Waiting for Data and Metadata to Synchronize') # NOTE: We only check the secondary zone, because the sync policy flow @@ -1317,24 +1322,8 @@ def test_100_migration_and_multisite_failover(self): ).put(Body=obj_data) # If Primary/Secondary relation does not exist, add it. - if zaza_model.get_relation_id( - self.primary_rgw_app, self.secondary_rgw_app, - remote_interface_name='secondary' - ) is None: - logging.info('Configuring Multisite') - self.configure_rgw_apps_for_multisite() - zaza_model.add_relation( - self.primary_rgw_app, - self.primary_rgw_app + ":primary", - self.secondary_rgw_app + ":secondary" - ) - zaza_model.block_until_unit_wl_status( - self.secondary_rgw_unit, "waiting" - ) + self.configure_rgw_multisite_relation() - zaza_model.block_until_unit_wl_status( - self.secondary_rgw_unit, "active" - ) logging.info('Waiting for Data and Metadata to Synchronize') self.wait_for_status(self.secondary_rgw_app, is_primary=False) self.wait_for_status(self.primary_rgw_app, is_primary=True) From f8136a6c1579c5ef27b9fa4af10cc2716796dcec Mon Sep 17 00:00:00 2001 From: Luciano Lo Giudice Date: Thu, 25 Apr 2024 11:30:08 -0300 Subject: [PATCH 27/43] Use a different waiting strategy for ceph-fs tests --- zaza/openstack/charm_tests/ceph/tests.py | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/zaza/openstack/charm_tests/ceph/tests.py b/zaza/openstack/charm_tests/ceph/tests.py index 1bca97355..638a31fe8 100644 --- a/zaza/openstack/charm_tests/ceph/tests.py +++ b/zaza/openstack/charm_tests/ceph/tests.py @@ -1770,7 +1770,7 @@ def _get_all_keys(self, unit, entity_filter): ret.add((data[ix - 1], data[ix + 1])) return ret - def _check_key_rotation(self, entity, unit, states=None): + def _check_key_rotation(self, entity, unit, wait_for_unit=None): def entity_filter(name): return name.startswith(entity) @@ -1781,7 +1781,11 @@ def entity_filter(name): action_params={'entity': entity} ) zaza_utils.assertActionRanOK(action_obj) - zaza_model.wait_for_application_states(states=states) + if wait_for_unit is None: + # Wait for all applications + zaza_model.wait_for_application_states() + else: + zaza_model.wait_for_unit_idle(wait_for_unit) new_keys = self._get_all_keys(unit, entity_filter) self.assertNotEqual(old_keys, new_keys) diff = new_keys - old_keys @@ -1817,14 +1821,11 @@ def test_key_rotate(self): zaza_model.get_application('ceph-fs') fs_svc = self._get_all_keys(unit, lambda x: x.startswith('mds.')) if fs_svc is not None: - ubuntu_states = { - 'ubuntu': { - 'workload-status': 'active', - 'workload-status-message': '' - } - } + # Only wait for ceph-fs, as this model includes 'ubuntu' + # units, and those don't play nice with zaza (they don't + # set the workload-status-message correctly). self._check_key_rotation(next(iter(fs_svc))[0], unit, - ubuntu_states) + 'ceph-fs/0') else: logging.info('ceph-fs units present, but no MDS service') except KeyError: From 08b1e4902fb133cf5b132d572986a7d309bef744 Mon Sep 17 00:00:00 2001 From: Luciano Lo Giudice Date: Thu, 25 Apr 2024 16:13:03 -0300 Subject: [PATCH 28/43] Use a more thorough stategy --- zaza/openstack/charm_tests/ceph/tests.py | 25 ++++++++++++++++-------- 1 file changed, 17 insertions(+), 8 deletions(-) diff --git a/zaza/openstack/charm_tests/ceph/tests.py b/zaza/openstack/charm_tests/ceph/tests.py index 638a31fe8..9b5e99598 100644 --- a/zaza/openstack/charm_tests/ceph/tests.py +++ b/zaza/openstack/charm_tests/ceph/tests.py @@ -1753,6 +1753,20 @@ def test_persistent_config(self): class CephMonKeyRotationTests(test_utils.BaseCharmTest): """Tests for the rotate-key action.""" + def setUp(self): + """Initialize key rotation test class.""" + super(CephMonKeyRotationTests, self).setUp() + try: + # Workaround for ubuntu units that don't play nicely with zaza. + zaza_model.get_application('ubuntu') + self.app_states = { + 'ubuntu': { + 'workload-status-message': '' + } + } + except KeyError: + self.app_states = None + def _get_all_keys(self, unit, entity_filter): cmd = 'sudo ceph auth ls' result = zaza_model.run_on_unit(unit, cmd) @@ -1770,7 +1784,7 @@ def _get_all_keys(self, unit, entity_filter): ret.add((data[ix - 1], data[ix + 1])) return ret - def _check_key_rotation(self, entity, unit, wait_for_unit=None): + def _check_key_rotation(self, entity, unit): def entity_filter(name): return name.startswith(entity) @@ -1781,11 +1795,7 @@ def entity_filter(name): action_params={'entity': entity} ) zaza_utils.assertActionRanOK(action_obj) - if wait_for_unit is None: - # Wait for all applications - zaza_model.wait_for_application_states() - else: - zaza_model.wait_for_unit_idle(wait_for_unit) + zaza_model.wait_for_application_states(states=self.app_states) new_keys = self._get_all_keys(unit, entity_filter) self.assertNotEqual(old_keys, new_keys) diff = new_keys - old_keys @@ -1824,8 +1834,7 @@ def test_key_rotate(self): # Only wait for ceph-fs, as this model includes 'ubuntu' # units, and those don't play nice with zaza (they don't # set the workload-status-message correctly). - self._check_key_rotation(next(iter(fs_svc))[0], unit, - 'ceph-fs/0') + self._check_key_rotation(next(iter(fs_svc))[0], unit) else: logging.info('ceph-fs units present, but no MDS service') except KeyError: From 6f87be2b5f8515d95cd4d9dfdc9460aa02dec43d Mon Sep 17 00:00:00 2001 From: Luciano Lo Giudice Date: Thu, 25 Apr 2024 18:18:51 -0300 Subject: [PATCH 29/43] Filter master MDS client --- zaza/openstack/charm_tests/ceph/tests.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/zaza/openstack/charm_tests/ceph/tests.py b/zaza/openstack/charm_tests/ceph/tests.py index 9b5e99598..27ad29738 100644 --- a/zaza/openstack/charm_tests/ceph/tests.py +++ b/zaza/openstack/charm_tests/ceph/tests.py @@ -1812,6 +1812,13 @@ def _get_rgw_client(self, unit): return None return next(iter(ret))[0] + def _get_fs_client(self, unit): + ret = self._get_all_keys(unit, lambda x: (x.startswith('mds.') and + x != 'mds.ceph-fs')) + if not ret: + return None + return next(iter(ret))[0] + def test_key_rotate(self): """Test that rotating the keys actually changes them.""" unit = 'ceph-mon/0' @@ -1829,7 +1836,7 @@ def test_key_rotate(self): try: zaza_model.get_application('ceph-fs') - fs_svc = self._get_all_keys(unit, lambda x: x.startswith('mds.')) + fs_svc = self._get_fs_client(unit) if fs_svc is not None: # Only wait for ceph-fs, as this model includes 'ubuntu' # units, and those don't play nice with zaza (they don't From 987e048c11e314481f37ee8fddb6f128756d3dd1 Mon Sep 17 00:00:00 2001 From: Luciano Lo Giudice Date: Thu, 25 Apr 2024 19:39:52 -0300 Subject: [PATCH 30/43] Fix fetching of FS service --- zaza/openstack/charm_tests/ceph/tests.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zaza/openstack/charm_tests/ceph/tests.py b/zaza/openstack/charm_tests/ceph/tests.py index 27ad29738..b8be90d52 100644 --- a/zaza/openstack/charm_tests/ceph/tests.py +++ b/zaza/openstack/charm_tests/ceph/tests.py @@ -1841,7 +1841,7 @@ def test_key_rotate(self): # Only wait for ceph-fs, as this model includes 'ubuntu' # units, and those don't play nice with zaza (they don't # set the workload-status-message correctly). - self._check_key_rotation(next(iter(fs_svc))[0], unit) + self._check_key_rotation(fs_svc, unit) else: logging.info('ceph-fs units present, but no MDS service') except KeyError: From 3cf2ed6d1519b450ca02c42b55383b0a31821e74 Mon Sep 17 00:00:00 2001 From: Luciano Lo Giudice Date: Mon, 6 May 2024 12:24:02 -0300 Subject: [PATCH 31/43] Retry on key rotation --- zaza/openstack/charm_tests/ceph/tests.py | 21 ++++++++++++++++++--- 1 file changed, 18 insertions(+), 3 deletions(-) diff --git a/zaza/openstack/charm_tests/ceph/tests.py b/zaza/openstack/charm_tests/ceph/tests.py index 4762d5eb3..b4bf1dc85 100644 --- a/zaza/openstack/charm_tests/ceph/tests.py +++ b/zaza/openstack/charm_tests/ceph/tests.py @@ -1795,9 +1795,24 @@ def entity_filter(name): action_params={'entity': entity} ) zaza_utils.assertActionRanOK(action_obj) - zaza_model.wait_for_application_states(states=self.app_states) - new_keys = self._get_all_keys(unit, entity_filter) - self.assertNotEqual(old_keys, new_keys) + # NOTE(lmlg): There's a nasty race going on here. Essentially, + # since this action involves 2 different applications, what + # happens is as follows: + # (1) (2) (3) (4) + # ceph-mon rotates key | (idle) | remote-unit rotates key | (idle) + # Between (2) and (3), there's a window where all units are + # idle, _but_ the key hasn't been rotated in the other unit. + # As such, we retry the checks a couple of times instead of + # using the `wait_for_application_states` interface. + + for attempt in tenacity.Retrying( + wait=tenacity.wait_exponential(multiplier=2, max=32), + reraise=True, stop=tenacity.stop_after_attempt(10), + retry=tenacity.retry_if_exception_type(AssertionError) + ): + new_keys = self._get_all_keys(unit, entity_filter) + self.assertNotEqual(old_keys, new_keys) + diff = new_keys - old_keys self.assertEqual(len(diff), 1) first = next(iter(diff)) From 9d931d373e670c90231724444a5a07150f31499a Mon Sep 17 00:00:00 2001 From: Luciano Lo Giudice Date: Mon, 6 May 2024 15:09:31 -0300 Subject: [PATCH 32/43] Sleep instead of retrying --- zaza/openstack/charm_tests/ceph/tests.py | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/zaza/openstack/charm_tests/ceph/tests.py b/zaza/openstack/charm_tests/ceph/tests.py index b4bf1dc85..66940b154 100644 --- a/zaza/openstack/charm_tests/ceph/tests.py +++ b/zaza/openstack/charm_tests/ceph/tests.py @@ -28,6 +28,7 @@ import urllib3 import tenacity +import time import zaza.charm_lifecycle.utils as lifecycle_utils import zaza.openstack.charm_tests.test_utils as test_utils @@ -1802,16 +1803,12 @@ def entity_filter(name): # ceph-mon rotates key | (idle) | remote-unit rotates key | (idle) # Between (2) and (3), there's a window where all units are # idle, _but_ the key hasn't been rotated in the other unit. - # As such, we retry the checks a couple of times instead of - # using the `wait_for_application_states` interface. + # As such, we sleep for a short time instead of using the + # `wait_for_application_states` interface. - for attempt in tenacity.Retrying( - wait=tenacity.wait_exponential(multiplier=2, max=32), - reraise=True, stop=tenacity.stop_after_attempt(10), - retry=tenacity.retry_if_exception_type(AssertionError) - ): - new_keys = self._get_all_keys(unit, entity_filter) - self.assertNotEqual(old_keys, new_keys) + time.sleep(10) + new_keys = self._get_all_keys(unit, entity_filter) + self.assertNotEqual(old_keys, new_keys) diff = new_keys - old_keys self.assertEqual(len(diff), 1) From 817534af7ece20b7974a0fd30b151d3ace345015 Mon Sep 17 00:00:00 2001 From: Luciano Lo Giudice Date: Mon, 6 May 2024 18:00:12 -0300 Subject: [PATCH 33/43] Fix tenacity usage --- zaza/openstack/charm_tests/ceph/tests.py | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/zaza/openstack/charm_tests/ceph/tests.py b/zaza/openstack/charm_tests/ceph/tests.py index 66940b154..7cc4aa846 100644 --- a/zaza/openstack/charm_tests/ceph/tests.py +++ b/zaza/openstack/charm_tests/ceph/tests.py @@ -28,7 +28,6 @@ import urllib3 import tenacity -import time import zaza.charm_lifecycle.utils as lifecycle_utils import zaza.openstack.charm_tests.test_utils as test_utils @@ -1803,12 +1802,17 @@ def entity_filter(name): # ceph-mon rotates key | (idle) | remote-unit rotates key | (idle) # Between (2) and (3), there's a window where all units are # idle, _but_ the key hasn't been rotated in the other unit. - # As such, we sleep for a short time instead of using the + # As such, we retry a few times instead of using the # `wait_for_application_states` interface. - time.sleep(10) - new_keys = self._get_all_keys(unit, entity_filter) - self.assertNotEqual(old_keys, new_keys) + for attempt in tenacity.Retrying( + wait=tenacity.wait_exponential(multiplier=2, max=32), + reraise=True, stop=tenacity.stop_after_attempt(10), + retry=tenacity.retry_if_exception_type(AssertionError) + ): + with attempt: + new_keys = self._get_all_keys(unit, entity_filter) + self.assertNotEqual(old_keys, new_keys) diff = new_keys - old_keys self.assertEqual(len(diff), 1) @@ -1834,7 +1838,6 @@ def _get_fs_client(self, unit): def test_key_rotate(self): """Test that rotating the keys actually changes them.""" unit = 'ceph-mon/0' - self._check_key_rotation('mgr', unit) self._check_key_rotation('osd.0', unit) try: From 239d028fdcc98e8e571089a9f24ec06d3e038fd4 Mon Sep 17 00:00:00 2001 From: Luciano Lo Giudice Date: Mon, 6 May 2024 20:26:21 -0300 Subject: [PATCH 34/43] Increase retries --- zaza/openstack/charm_tests/ceph/tests.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zaza/openstack/charm_tests/ceph/tests.py b/zaza/openstack/charm_tests/ceph/tests.py index 7cc4aa846..58719c216 100644 --- a/zaza/openstack/charm_tests/ceph/tests.py +++ b/zaza/openstack/charm_tests/ceph/tests.py @@ -1807,7 +1807,7 @@ def entity_filter(name): for attempt in tenacity.Retrying( wait=tenacity.wait_exponential(multiplier=2, max=32), - reraise=True, stop=tenacity.stop_after_attempt(10), + reraise=True, stop=tenacity.stop_after_attempt(20), retry=tenacity.retry_if_exception_type(AssertionError) ): with attempt: From ae9a99feaa41bd588cdac86f7c7961a9d1a60018 Mon Sep 17 00:00:00 2001 From: Felipe Reyes Date: Wed, 8 May 2024 17:51:00 -0400 Subject: [PATCH 35/43] Add caracal to codename maps jammy-caracal, noble-caracal, 2024.1 and noble are being added to their respective codename maps to allow the testing to recognize caracal --- zaza/openstack/utilities/os_versions.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/zaza/openstack/utilities/os_versions.py b/zaza/openstack/utilities/os_versions.py index 2d7639f20..c1d819ccb 100644 --- a/zaza/openstack/utilities/os_versions.py +++ b/zaza/openstack/utilities/os_versions.py @@ -42,6 +42,7 @@ ('kinetic', 'zed'), ('lunar', 'antelope'), ('mantic', 'bobcat'), + ('noble', 'caracal'), ]) @@ -71,6 +72,7 @@ ('2022.2', 'zed'), ('2023.1', 'antelope'), ('2023.2', 'bobcat'), + ('2024.1', 'caracal'), ]) OPENSTACK_RELEASES_PAIRS = [ @@ -87,6 +89,7 @@ 'focal_yoga', 'jammy_yoga', 'jammy_zed', 'kinetic_zed', 'jammy_antelope', 'lunar_antelope', 'jammy_bobcat', 'mantic_bobcat', + 'jammy_caracal', 'noble_caracal', ] SWIFT_CODENAMES = OrderedDict([ @@ -320,6 +323,7 @@ 'kinetic', 'lunar', 'mantic', + 'noble', ) From 5541905935a44f317847dbffdbc875258a21e24c Mon Sep 17 00:00:00 2001 From: Luciano Lo Giudice Date: Mon, 13 May 2024 06:12:15 -0300 Subject: [PATCH 36/43] Fix test for ceph-fs key rotation The ceph-fs charm sometimes creates an entity called 'mds.None'. This is probably a bug, but in the meantime, we filter it out in the key rotation test class. --- zaza/openstack/charm_tests/ceph/tests.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/zaza/openstack/charm_tests/ceph/tests.py b/zaza/openstack/charm_tests/ceph/tests.py index 58719c216..f0c00a693 100644 --- a/zaza/openstack/charm_tests/ceph/tests.py +++ b/zaza/openstack/charm_tests/ceph/tests.py @@ -1829,8 +1829,11 @@ def _get_rgw_client(self, unit): return next(iter(ret))[0] def _get_fs_client(self, unit): - ret = self._get_all_keys(unit, lambda x: (x.startswith('mds.') and - x != 'mds.ceph-fs')) + def _filter_fs(name): + return (name.startswith('mds.') and + name not in ('mds.ceph-fs', 'mds.None')) + + ret = self._get_all_keys(unit, _filter_fs) if not ret: return None return next(iter(ret))[0] From 418c98ee57e614f39813b579e743299ede3d6abd Mon Sep 17 00:00:00 2001 From: Peter Sabaini Date: Mon, 13 May 2024 21:46:18 +0200 Subject: [PATCH 37/43] Add squid package name --- zaza/openstack/utilities/os_versions.py | 1 + 1 file changed, 1 insertion(+) diff --git a/zaza/openstack/utilities/os_versions.py b/zaza/openstack/utilities/os_versions.py index c1d819ccb..7930d5f7d 100644 --- a/zaza/openstack/utilities/os_versions.py +++ b/zaza/openstack/utilities/os_versions.py @@ -285,6 +285,7 @@ ('16', 'victoria'), # pacific ('17', 'yoga'), # quincy ('18', 'bobcat'), # reef + ('19', 'caracal'), # squid ]), 'placement-common': OrderedDict([ ('2', 'train'), From 2a7c49b4985c8a17482935fd488f1199a7467ab9 Mon Sep 17 00:00:00 2001 From: Myles Penner Date: Fri, 24 May 2024 15:27:39 -0700 Subject: [PATCH 38/43] Add class for keystone audit middleware testing Added general class for testing keystone audit middleware functionality in charms. Tests correct rendering of api-paste.ini file. Example usage with charm Heat: tests/tests.yaml tests: - zaza.openstack.charm_tests.audit.tests.KeystoneAuditMiddlewareTest tests_options: audit-middleware: service: heat --- zaza/openstack/charm_tests/audit/__init__.py | 20 +++++ zaza/openstack/charm_tests/audit/tests.py | 90 ++++++++++++++++++++ 2 files changed, 110 insertions(+) create mode 100644 zaza/openstack/charm_tests/audit/__init__.py create mode 100644 zaza/openstack/charm_tests/audit/tests.py diff --git a/zaza/openstack/charm_tests/audit/__init__.py b/zaza/openstack/charm_tests/audit/__init__.py new file mode 100644 index 000000000..0c3a4c526 --- /dev/null +++ b/zaza/openstack/charm_tests/audit/__init__.py @@ -0,0 +1,20 @@ +# Copyright 2024 Canonical Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Keystone audit middleware. + +Collection of code for setting up and testing Keystone audit middleware +functionality. +""" diff --git a/zaza/openstack/charm_tests/audit/tests.py b/zaza/openstack/charm_tests/audit/tests.py new file mode 100644 index 000000000..146c1d45e --- /dev/null +++ b/zaza/openstack/charm_tests/audit/tests.py @@ -0,0 +1,90 @@ +#!/usr/bin/env python3 +# +# Copyright 2024 Canonical Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Keystone audit middleware API logging testing. + +These methods test the +rendering of the charm's api-paste.ini file to ensure the appropriate sections +are rendered or not rendered depending on the state of the audit-middleware +configuration option. +""" + +import logging +import zaza.model +import zaza.openstack.charm_tests.test_utils as test_utils + + +class KeystoneAuditMiddlewareBaseTest(test_utils.OpenStackBaseTest): + """Base class for Keystone audit middleware tests.""" + + @classmethod + def setUpClass(cls): + """Run class setup for Keystone audit middleware tests.""" + super(KeystoneAuditMiddlewareBaseTest, cls).setUpClass() + test_config = cls.test_config['tests_options']['audit-middleware'] + cls.service_name = test_config['service'] + cls.lead_unit = zaza.model.get_lead_unit_name( + cls.service_name, + model_name=cls.model_name + ) + logging.info('Leader unit is %s', cls.lead_unit) + logging.info('Service name is %s', cls.service_name) + + def fetch_api_paste_content(self): + """Fetch content of api-paste.ini file.""" + api_paste_ini_path = f"/etc/{self.service_name}/api-paste.ini" + try: + return zaza.model.file_contents( + self.lead_unit, + api_paste_ini_path, + ) + except zaza.model.CommandRunFailed as e: + self.fail("Error fetching api-paste.ini: {}".format(str(e))) + + +class KeystoneAuditMiddlewareTest(KeystoneAuditMiddlewareBaseTest): + """Keystone audit middleware API logging feature tests.""" + + def test_101_apipaste_includes_audit_section(self): + """Test api-paste.ini renders audit section when enabled.""" + expected_content = [ + "[filter:audit]", + "paste.filter_factory = keystonemiddleware.audit:filter_factory", + f"audit_map_file = /etc/{self.service_name}/api_audit_map.conf", + f"service_name = {self.service_name}" + ] + + set_default = {'audit-middleware': False} + set_alternate = {'audit-middleware': True} + + with self.config_change(default_config=set_default, + alternate_config=set_alternate, + application_name=self.service_name): + api_paste_content = self.fetch_api_paste_content() + for line in expected_content: + self.assertIn(line, api_paste_content) + + def test_102_apipaste_excludes_audit_section(self): + """Test api_paste.ini does not render audit section when disabled.""" + section_heading = '[filter:audit]' + + if not self.config_current(self.service_name)['audit-middleware']: + api_paste_content = self.fetch_api_paste_content() + self.assertNotIn(section_heading, api_paste_content) + else: + self.fail("Failed due to audit-middleware being incorrectly set to\ + True") From df3858d0d198b9665520d2e549ae128f32e40284 Mon Sep 17 00:00:00 2001 From: Myles Penner Date: Mon, 27 May 2024 10:11:13 -0700 Subject: [PATCH 39/43] Add fix for when service and app name differ --- zaza/openstack/charm_tests/audit/tests.py | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/zaza/openstack/charm_tests/audit/tests.py b/zaza/openstack/charm_tests/audit/tests.py index 146c1d45e..02b05b12e 100644 --- a/zaza/openstack/charm_tests/audit/tests.py +++ b/zaza/openstack/charm_tests/audit/tests.py @@ -37,10 +37,19 @@ def setUpClass(cls): super(KeystoneAuditMiddlewareBaseTest, cls).setUpClass() test_config = cls.test_config['tests_options']['audit-middleware'] cls.service_name = test_config['service'] + + # For cases when service name and application name differ + if not test_config["application"]: + cls.application_name = cls.service_name + else: + cls.application_name = test_config['application'] + print(test_config) + print(cls.model_name) cls.lead_unit = zaza.model.get_lead_unit_name( - cls.service_name, + cls.application_name, model_name=cls.model_name ) + print(cls.lead_unit) logging.info('Leader unit is %s', cls.lead_unit) logging.info('Service name is %s', cls.service_name) @@ -73,7 +82,7 @@ def test_101_apipaste_includes_audit_section(self): with self.config_change(default_config=set_default, alternate_config=set_alternate, - application_name=self.service_name): + application_name=self.application_name): api_paste_content = self.fetch_api_paste_content() for line in expected_content: self.assertIn(line, api_paste_content) @@ -82,9 +91,8 @@ def test_102_apipaste_excludes_audit_section(self): """Test api_paste.ini does not render audit section when disabled.""" section_heading = '[filter:audit]' - if not self.config_current(self.service_name)['audit-middleware']: + if not self.config_current(self.application_name)['audit-middleware']: api_paste_content = self.fetch_api_paste_content() self.assertNotIn(section_heading, api_paste_content) else: - self.fail("Failed due to audit-middleware being incorrectly set to\ - True") + self.fail("Config option audit-middleware incorrectly set to true") From 38a7c9b0c2f9ce0ed7878e1d4dcf5d91712d7ce1 Mon Sep 17 00:00:00 2001 From: Myles Penner Date: Tue, 28 May 2024 10:57:47 -0700 Subject: [PATCH 40/43] Make corrections as per review comments --- zaza/openstack/charm_tests/audit/tests.py | 81 +++++++++++++---------- 1 file changed, 47 insertions(+), 34 deletions(-) diff --git a/zaza/openstack/charm_tests/audit/tests.py b/zaza/openstack/charm_tests/audit/tests.py index 02b05b12e..3872777fa 100644 --- a/zaza/openstack/charm_tests/audit/tests.py +++ b/zaza/openstack/charm_tests/audit/tests.py @@ -17,65 +17,77 @@ """ Keystone audit middleware API logging testing. -These methods test the -rendering of the charm's api-paste.ini file to ensure the appropriate sections -are rendered or not rendered depending on the state of the audit-middleware -configuration option. +These methods test the rendering of the charm api-paste.ini file to +ensure the appropriate sections are rendered or not rendered depending +on the state of the audit-middleware configuration option. """ +import textwrap import logging import zaza.model import zaza.openstack.charm_tests.test_utils as test_utils -class KeystoneAuditMiddlewareBaseTest(test_utils.OpenStackBaseTest): - """Base class for Keystone audit middleware tests.""" +class KeystoneAuditMiddlewareTest(test_utils.OpenStackBaseTest): + """Keystone audit middleware test class.""" @classmethod def setUpClass(cls): """Run class setup for Keystone audit middleware tests.""" - super(KeystoneAuditMiddlewareBaseTest, cls).setUpClass() + super(KeystoneAuditMiddlewareTest, cls).setUpClass() test_config = cls.test_config['tests_options']['audit-middleware'] cls.service_name = test_config['service'] - # For cases when service name and application name differ - if not test_config["application"]: - cls.application_name = cls.service_name - else: + if test_config['application']: cls.application_name = test_config['application'] - print(test_config) - print(cls.model_name) - cls.lead_unit = zaza.model.get_lead_unit_name( + else: + cls.application_name = cls.service_name + logging.info('Application name not set, using service name:\ + %s', cls.service_name) + + cls.initial_audit_middleware = zaza.model.get_application_config( + cls.application_name)['audit-middleware']['value'] + + @classmethod + def tearDownClass(cls): + """Restore the audit-middleware configuration to its original state.""" + super(KeystoneAuditMiddlewareTest, cls).tearDownClass() + logging.info("Running teardown on %s" % cls.application_name) + zaza.model.set_application_config( cls.application_name, + {'audit-middleware': str(cls.initial_audit_middleware)}, + model_name=cls.model_name + ) + zaza.model.wait_for_application_states( + states={cls.application_name: { + 'workload-status': 'active', + 'workload-status-message': 'Unit is ready'}}, model_name=cls.model_name ) - print(cls.lead_unit) - logging.info('Leader unit is %s', cls.lead_unit) - logging.info('Service name is %s', cls.service_name) def fetch_api_paste_content(self): """Fetch content of api-paste.ini file.""" api_paste_ini_path = f"/etc/{self.service_name}/api-paste.ini" + lead_unit = zaza.model.get_lead_unit_name( + self.application_name, + model_name=self.model_name + ) try: return zaza.model.file_contents( - self.lead_unit, + lead_unit, api_paste_ini_path, ) except zaza.model.CommandRunFailed as e: - self.fail("Error fetching api-paste.ini: {}".format(str(e))) - - -class KeystoneAuditMiddlewareTest(KeystoneAuditMiddlewareBaseTest): - """Keystone audit middleware API logging feature tests.""" + self.fail("Error fetching api-paste.ini: %s" % e) def test_101_apipaste_includes_audit_section(self): """Test api-paste.ini renders audit section when enabled.""" - expected_content = [ - "[filter:audit]", - "paste.filter_factory = keystonemiddleware.audit:filter_factory", - f"audit_map_file = /etc/{self.service_name}/api_audit_map.conf", - f"service_name = {self.service_name}" - ] + expected_content = textwrap.dedent(f"""\ + [filter:audit] + paste.filter_factory = keystonemiddleware.audit:filter_factory + audit_map_file = /etc/{self.service_name}/api_audit_map.conf + service_name = {self.service_name} + """) set_default = {'audit-middleware': False} set_alternate = {'audit-middleware': True} @@ -84,15 +96,16 @@ def test_101_apipaste_includes_audit_section(self): alternate_config=set_alternate, application_name=self.application_name): api_paste_content = self.fetch_api_paste_content() - for line in expected_content: - self.assertIn(line, api_paste_content) + self.assertIn(expected_content, api_paste_content) def test_102_apipaste_excludes_audit_section(self): """Test api_paste.ini does not render audit section when disabled.""" section_heading = '[filter:audit]' + set_default = {'audit-middleware': True} + set_alternate = {'audit-middleware': False} - if not self.config_current(self.application_name)['audit-middleware']: + with self.config_change(default_config=set_default, + alternate_config=set_alternate, + application_name=self.application_name): api_paste_content = self.fetch_api_paste_content() self.assertNotIn(section_heading, api_paste_content) - else: - self.fail("Config option audit-middleware incorrectly set to true") From 3af8d41067facd258426f0625c7ca7eee26b4d49 Mon Sep 17 00:00:00 2001 From: Myles Penner Date: Wed, 29 May 2024 08:38:49 -0700 Subject: [PATCH 41/43] Fix dictionary lookup method --- zaza/openstack/charm_tests/audit/tests.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/zaza/openstack/charm_tests/audit/tests.py b/zaza/openstack/charm_tests/audit/tests.py index 3872777fa..8b8f5f62e 100644 --- a/zaza/openstack/charm_tests/audit/tests.py +++ b/zaza/openstack/charm_tests/audit/tests.py @@ -38,12 +38,8 @@ def setUpClass(cls): test_config = cls.test_config['tests_options']['audit-middleware'] cls.service_name = test_config['service'] - if test_config['application']: - cls.application_name = test_config['application'] - else: - cls.application_name = cls.service_name - logging.info('Application name not set, using service name:\ - %s', cls.service_name) + cls.application_name = test_config.get('application', cls.service_name) + logging.info('Using application name: %s', cls.application_name) cls.initial_audit_middleware = zaza.model.get_application_config( cls.application_name)['audit-middleware']['value'] From 2ddde4307e6a14f310e3835a410dedf94838d481 Mon Sep 17 00:00:00 2001 From: Myles Penner Date: Wed, 29 May 2024 15:11:20 -0700 Subject: [PATCH 42/43] Remove redundant cinder tests --- zaza/openstack/charm_tests/cinder/tests.py | 40 ---------------------- 1 file changed, 40 deletions(-) diff --git a/zaza/openstack/charm_tests/cinder/tests.py b/zaza/openstack/charm_tests/cinder/tests.py index 7668127dc..421d30631 100644 --- a/zaza/openstack/charm_tests/cinder/tests.py +++ b/zaza/openstack/charm_tests/cinder/tests.py @@ -305,46 +305,6 @@ def verify(stdin, stdout, stderr): 'lsblk -sn -o SIZE /dev/vdb', privkey=privkey, verify=verify) - def test_300_apipaste_includes_audit_section(self): - """Test api-paste.ini renders audit section when enabled.""" - service_name = 'cinder' - api_paste_ini_path = f"/etc/{service_name}/api-paste.ini" - expected_content = [ - "[filter:audit]", - "paste.filter_factory = keystonemiddleware.audit:filter_factory", - f"audit_map_file = /etc/{service_name}/api_audit_map.conf", - f"service_name = {service_name}" - ] - - set_default = {'audit-middleware': False} - set_alternate = {'audit-middleware': True} - - with self.config_change(set_default, set_alternate): - try: - api_paste_content = zaza.model.file_contents( - self.lead_unit, - api_paste_ini_path, - ) - except Exception as e: - self.fail("Error fetching api-paste.ini: {}".format(str(e))) - for line in expected_content: - self.assertIn(line, api_paste_content) - - def test_301_apipaste_excludes_audit_section(self): - """Test api_paste.ini does not render audit section when disabled.""" - service_name = 'cinder' - section_heading = '[filter:audit]' - api_paste_ini_path = f"/etc/{service_name}/api-paste.ini" - - try: - api_paste_content = zaza.model.file_contents( - self.lead_unit, - api_paste_ini_path - ) - except Exception as e: - self.fail("Error fetching api-paste.ini: {}".format(str(e))) - self.assertNotIn(section_heading, api_paste_content) - @property def services(self): """Return a list services for the selected OpenStack release.""" From d83e01f86270dd7005c0fab26a4e387977726701 Mon Sep 17 00:00:00 2001 From: Alex Kavanagh Date: Thu, 30 May 2024 10:56:37 +0100 Subject: [PATCH 43/43] Remove designate-agent for caracal+ testing The designate-agent service was removed at caracal. This patch removes the designate-agent from the list of services that designate supports so that the pause/resume testing works appropriately. --- zaza/openstack/charm_tests/designate/tests.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/zaza/openstack/charm_tests/designate/tests.py b/zaza/openstack/charm_tests/designate/tests.py index ed68a8c93..2203c3a4d 100644 --- a/zaza/openstack/charm_tests/designate/tests.py +++ b/zaza/openstack/charm_tests/designate/tests.py @@ -44,8 +44,15 @@ def setUpClass(cls, application_name=None, model_alias=None): model_alias = model_alias or "" super(BaseDesignateTest, cls).setUpClass(application_name, model_alias) os_release = openstack_utils.get_os_release + current_release = os_release() - if os_release() >= os_release('bionic_rocky'): + if current_release >= os_release('jammy_caracal'): + cls.designate_svcs = [ + 'designate-api', 'designate-central', + 'designate-mdns', 'designate-worker', 'designate-sink', + 'designate-producer', + ] + elif current_release >= os_release('bionic_rocky'): cls.designate_svcs = [ 'designate-agent', 'designate-api', 'designate-central', 'designate-mdns', 'designate-worker', 'designate-sink',