diff --git a/zaza/openstack/charm_tests/audit/__init__.py b/zaza/openstack/charm_tests/audit/__init__.py new file mode 100644 index 000000000..0c3a4c526 --- /dev/null +++ b/zaza/openstack/charm_tests/audit/__init__.py @@ -0,0 +1,20 @@ +# Copyright 2024 Canonical Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Keystone audit middleware. + +Collection of code for setting up and testing Keystone audit middleware +functionality. +""" diff --git a/zaza/openstack/charm_tests/audit/tests.py b/zaza/openstack/charm_tests/audit/tests.py new file mode 100644 index 000000000..8b8f5f62e --- /dev/null +++ b/zaza/openstack/charm_tests/audit/tests.py @@ -0,0 +1,107 @@ +#!/usr/bin/env python3 +# +# Copyright 2024 Canonical Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Keystone audit middleware API logging testing. + +These methods test the rendering of the charm api-paste.ini file to +ensure the appropriate sections are rendered or not rendered depending +on the state of the audit-middleware configuration option. +""" + +import textwrap +import logging +import zaza.model +import zaza.openstack.charm_tests.test_utils as test_utils + + +class KeystoneAuditMiddlewareTest(test_utils.OpenStackBaseTest): + """Keystone audit middleware test class.""" + + @classmethod + def setUpClass(cls): + """Run class setup for Keystone audit middleware tests.""" + super(KeystoneAuditMiddlewareTest, cls).setUpClass() + test_config = cls.test_config['tests_options']['audit-middleware'] + cls.service_name = test_config['service'] + + cls.application_name = test_config.get('application', cls.service_name) + logging.info('Using application name: %s', cls.application_name) + + cls.initial_audit_middleware = zaza.model.get_application_config( + cls.application_name)['audit-middleware']['value'] + + @classmethod + def tearDownClass(cls): + """Restore the audit-middleware configuration to its original state.""" + super(KeystoneAuditMiddlewareTest, cls).tearDownClass() + logging.info("Running teardown on %s" % cls.application_name) + zaza.model.set_application_config( + cls.application_name, + {'audit-middleware': str(cls.initial_audit_middleware)}, + model_name=cls.model_name + ) + zaza.model.wait_for_application_states( + states={cls.application_name: { + 'workload-status': 'active', + 'workload-status-message': 'Unit is ready'}}, + model_name=cls.model_name + ) + + def fetch_api_paste_content(self): + """Fetch content of api-paste.ini file.""" + api_paste_ini_path = f"/etc/{self.service_name}/api-paste.ini" + lead_unit = zaza.model.get_lead_unit_name( + self.application_name, + model_name=self.model_name + ) + try: + return zaza.model.file_contents( + lead_unit, + api_paste_ini_path, + ) + except zaza.model.CommandRunFailed as e: + self.fail("Error fetching api-paste.ini: %s" % e) + + def test_101_apipaste_includes_audit_section(self): + """Test api-paste.ini renders audit section when enabled.""" + expected_content = textwrap.dedent(f"""\ + [filter:audit] + paste.filter_factory = keystonemiddleware.audit:filter_factory + audit_map_file = /etc/{self.service_name}/api_audit_map.conf + service_name = {self.service_name} + """) + + set_default = {'audit-middleware': False} + set_alternate = {'audit-middleware': True} + + with self.config_change(default_config=set_default, + alternate_config=set_alternate, + application_name=self.application_name): + api_paste_content = self.fetch_api_paste_content() + self.assertIn(expected_content, api_paste_content) + + def test_102_apipaste_excludes_audit_section(self): + """Test api_paste.ini does not render audit section when disabled.""" + section_heading = '[filter:audit]' + set_default = {'audit-middleware': True} + set_alternate = {'audit-middleware': False} + + with self.config_change(default_config=set_default, + alternate_config=set_alternate, + application_name=self.application_name): + api_paste_content = self.fetch_api_paste_content() + self.assertNotIn(section_heading, api_paste_content) diff --git a/zaza/openstack/charm_tests/ceph/tests.py b/zaza/openstack/charm_tests/ceph/tests.py index 46911c07c..178e2f88a 100644 --- a/zaza/openstack/charm_tests/ceph/tests.py +++ b/zaza/openstack/charm_tests/ceph/tests.py @@ -648,7 +648,7 @@ class CephRGWTest(test_utils.BaseCharmTest): This Testset is not idempotent, because we don't support scale down from multisite to singlesite (yet). Tests can be performed independently. - However, If test_004 has completed migration, retriggering the test-set + However, If test_100 has completed migration, retriggering the test-set would cause a time-out in test_003. """ @@ -878,6 +878,33 @@ def configure_rgw_apps_for_multisite(self): } ) + def configure_rgw_multisite_relation(self): + """Configure multi-site relation between primary and secondary apps.""" + multisite_relation = zaza_model.get_relation_id( + self.primary_rgw_app, self.secondary_rgw_app, + remote_interface_name='secondary' + ) + if multisite_relation is None: + logging.info('Configuring Multisite') + self.configure_rgw_apps_for_multisite() + zaza_model.add_relation( + self.primary_rgw_app, + self.primary_rgw_app + ":primary", + self.secondary_rgw_app + ":secondary" + ) + zaza_model.block_until_unit_wl_status( + self.secondary_rgw_unit, "waiting" + ) + + zaza_model.block_until_unit_wl_status( + self.secondary_rgw_unit, "active" + ) + zaza_model.block_until_unit_wl_status( + self.primary_rgw_unit, "active" + ) + zaza_model.wait_for_unit_idle(self.secondary_rgw_unit) + zaza_model.wait_for_unit_idle(self.primary_rgw_unit) + def clean_rgw_multisite_config(self, app_name): """Clear Multisite Juju config values to default. @@ -1066,7 +1093,209 @@ def test_003_object_storage_and_secondary_block(self): zaza_model.block_until_unit_wl_status(self.secondary_rgw_unit, 'active') - def test_004_migration_and_multisite_failover(self): + def test_004_multisite_directional_sync_policy(self): + """Verify Multisite Directional Sync Policy.""" + # Skip multisite tests if not compatible with bundle. + if not self.multisite: + logging.info('Skipping multisite sync policy verification') + return + + container_name = 'zaza-container' + primary_obj_name = 'primary-testfile' + primary_obj_data = 'Primary test data' + secondary_directional_obj_name = 'secondary-directional-testfile' + secondary_directional_obj_data = 'Secondary directional test data' + secondary_symmetrical_obj_name = 'secondary-symmetrical-testfile' + secondary_symmetrical_obj_data = 'Secondary symmetrical test data' + + logging.info('Verifying multisite directional sync policy') + + # Set default sync policy to "allowed", which allows buckets to sync, + # but the sync is disabled by default in the zone group. Also, set the + # secondary zone sync policy flow type policy to "directional". + zaza_model.set_application_config( + self.primary_rgw_app, + { + "sync-policy-state": "allowed", + } + ) + zaza_model.set_application_config( + self.secondary_rgw_app, + { + "sync-policy-flow-type": "directional", + } + ) + zaza_model.wait_for_unit_idle(self.secondary_rgw_unit) + zaza_model.wait_for_unit_idle(self.primary_rgw_unit) + + # Setup multisite relation. + self.configure_rgw_multisite_relation() + + logging.info('Waiting for Data and Metadata to Synchronize') + # NOTE: We only check the secondary zone, because the sync policy flow + # type is set to "directional" between the primary and secondary zones. + self.wait_for_status(self.secondary_rgw_app, is_primary=False) + + # Create bucket on primary RGW zone. + logging.info('Creating bucket on primary zone') + primary_endpoint = self.get_rgw_endpoint(self.primary_rgw_unit) + self.assertNotEqual(primary_endpoint, None) + + access_key, secret_key = self.get_client_keys() + primary_client = boto3.resource("s3", + verify=False, + endpoint_url=primary_endpoint, + aws_access_key_id=access_key, + aws_secret_access_key=secret_key) + primary_client.Bucket(container_name).create() + + # Enable sync on the bucket. + logging.info('Enabling sync on the bucket from the primary zone') + zaza_model.run_action_on_leader( + self.primary_rgw_app, + 'enable-buckets-sync', + action_params={ + 'buckets': container_name, + }, + raise_on_failure=True, + ) + + # Check that sync cannot be enabled using secondary Juju RGW app. + with self.assertRaises(zaza_model.ActionFailed): + zaza_model.run_action_on_leader( + self.secondary_rgw_app, + 'enable-buckets-sync', + action_params={ + 'buckets': container_name, + }, + raise_on_failure=True, + ) + + logging.info('Waiting for Data and Metadata to Synchronize') + self.wait_for_status(self.secondary_rgw_app, is_primary=False) + + # Perform IO on primary zone bucket. + logging.info('Performing IO on primary zone bucket') + primary_object = primary_client.Object( + container_name, + primary_obj_name + ) + primary_object.put(Body=primary_obj_data) + + # Verify that the object is replicated to the secondary zone. + logging.info('Verifying that the object is replicated to the ' + 'secondary zone') + secondary_endpoint = self.get_rgw_endpoint(self.secondary_rgw_unit) + self.assertNotEqual(secondary_endpoint, None) + + secondary_client = boto3.resource("s3", + verify=False, + endpoint_url=secondary_endpoint, + aws_access_key_id=access_key, + aws_secret_access_key=secret_key) + secondary_data = self.fetch_rgw_object( + secondary_client, + container_name, + primary_obj_name + ) + self.assertEqual(secondary_data, primary_obj_data) + + # Write object to the secondary zone bucket, when the sync policy + # flow type is set to "directional" between the zones. + logging.info('Writing object to the secondary zone bucket, which ' + 'should not be replicated to the primary zone') + secondary_object = secondary_client.Object( + container_name, + secondary_directional_obj_name + ) + secondary_object.put(Body=secondary_directional_obj_data) + + # Verify that the object is not replicated to the primary zone. + logging.info('Verifying that the object is not replicated to the ' + 'primary zone') + with self.assertRaises(botocore.exceptions.ClientError): + self.fetch_rgw_object( + primary_client, + container_name, + secondary_directional_obj_name + ) + + logging.info('Setting sync policy flow to "symmetrical" on the ' + 'secondary RGW zone') + zaza_model.set_application_config( + self.secondary_rgw_app, + { + "sync-policy-flow-type": "symmetrical", + } + ) + zaza_model.wait_for_unit_idle(self.secondary_rgw_unit) + zaza_model.wait_for_unit_idle(self.primary_rgw_unit) + + # Write another object to the secondary zone bucket. + logging.info('Writing another object to the secondary zone bucket.') + secondary_object = secondary_client.Object( + container_name, + secondary_symmetrical_obj_name + ) + secondary_object.put(Body=secondary_symmetrical_obj_data) + + logging.info('Waiting for Data and Metadata to Synchronize') + # NOTE: This time, we check both the primary and secondary zones, + # because the sync policy flow type is set to "symmetrical" between + # the zones. + self.wait_for_status(self.secondary_rgw_app, is_primary=False) + self.wait_for_status(self.primary_rgw_app, is_primary=True) + + # Verify that all objects are replicated to the primary zone. + logging.info('Verifying that all objects are replicated to the ' + 'primary zone (including older objects).') + test_cases = [ + { + 'obj_name': primary_obj_name, + 'obj_data': primary_obj_data, + }, + { + 'obj_name': secondary_directional_obj_name, + 'obj_data': secondary_directional_obj_data, + }, + { + 'obj_name': secondary_symmetrical_obj_name, + 'obj_data': secondary_symmetrical_obj_data, + }, + ] + for tc in test_cases: + logging.info('Verifying that object "{}" is replicated'.format( + tc['obj_name'])) + primary_data = self.fetch_rgw_object( + primary_client, + container_name, + tc['obj_name'] + ) + self.assertEqual(primary_data, tc['obj_data']) + + # Cleanup. + logging.info('Cleaning up buckets after test case') + self.purge_bucket(self.primary_rgw_app, container_name) + self.purge_bucket(self.secondary_rgw_app, container_name) + + logging.info('Waiting for Data and Metadata to Synchronize') + self.wait_for_status(self.secondary_rgw_app, is_primary=False) + self.wait_for_status(self.primary_rgw_app, is_primary=True) + + # Set multisite sync policy state to "enabled" on the primary RGW app. + # Paired with "symmetrical" sync policy flow on the secondary RGW app, + # this enables bidirectional sync between the zones (which is the + # default behaviour without multisite sync policies configured). + logging.info('Setting sync policy state to "enabled".') + zaza_model.set_application_config( + self.primary_rgw_app, + { + "sync-policy-state": "enabled", + } + ) + zaza_model.wait_for_unit_idle(self.primary_rgw_unit) + + def test_100_migration_and_multisite_failover(self): """Perform multisite migration and verify failover.""" container_name = 'zaza-container' obj_data = 'Test data from Zaza' @@ -1093,24 +1322,8 @@ def test_004_migration_and_multisite_failover(self): ).put(Body=obj_data) # If Primary/Secondary relation does not exist, add it. - if zaza_model.get_relation_id( - self.primary_rgw_app, self.secondary_rgw_app, - remote_interface_name='secondary' - ) is None: - logging.info('Configuring Multisite') - self.configure_rgw_apps_for_multisite() - zaza_model.add_relation( - self.primary_rgw_app, - self.primary_rgw_app + ":primary", - self.secondary_rgw_app + ":secondary" - ) - zaza_model.block_until_unit_wl_status( - self.secondary_rgw_unit, "waiting" - ) + self.configure_rgw_multisite_relation() - zaza_model.block_until_unit_wl_status( - self.secondary_rgw_unit, "active" - ) logging.info('Waiting for Data and Metadata to Synchronize') self.wait_for_status(self.secondary_rgw_app, is_primary=False) self.wait_for_status(self.primary_rgw_app, is_primary=True) @@ -1167,6 +1380,10 @@ def test_004_migration_and_multisite_failover(self): 'Body' ].read().decode('UTF-8') + # Wait for Sites to be syncronised. + self.wait_for_status(self.primary_rgw_app, is_primary=False) + self.wait_for_status(self.secondary_rgw_app, is_primary=True) + # Recovery scenario, reset ceph-rgw as primary. self.promote_rgw_to_primary(self.primary_rgw_app) self.wait_for_status(self.primary_rgw_app, is_primary=True) @@ -1224,7 +1441,7 @@ def test_004_migration_and_multisite_failover(self): self.purge_bucket(self.secondary_rgw_app, 'zaza-container') self.purge_bucket(self.secondary_rgw_app, 'failover-container') - def test_005_virtual_hosted_bucket(self): + def test_101_virtual_hosted_bucket(self): """Test virtual hosted bucket.""" primary_rgw_unit = zaza_model.get_unit_from_name(self.primary_rgw_unit) if primary_rgw_unit.workload_status != "active": @@ -1236,17 +1453,11 @@ def test_005_virtual_hosted_bucket(self): # 0. Configure virtual hosted bucket self.enable_virtual_hosted_bucket() - assert_state = { - self.primary_rgw_app: { - "workload-status": "blocked", - "workload-status-message-prefix": - "os-public-hostname must have a value " - "when virtual hosted bucket is enabled" - } - } - zaza_model.wait_for_application_states(self.model_name, - states=assert_state, - timeout=900) + zaza_model.block_until_wl_status_info_starts_with( + self.primary_rgw_app, + 'os-public-hostname must have a value', + timeout=900 + ) self.set_os_public_hostname() zaza_model.block_until_all_units_idle(self.model_name) container_name = 'zaza-bucket' @@ -1264,7 +1475,13 @@ def test_005_virtual_hosted_bucket(self): endpoint_url=primary_endpoint, aws_access_key_id=access_key, aws_secret_access_key=secret_key) - primary_client.Bucket(container_name).create() + # We may not have certs for the pub hostname yet, so retry a few times. + for attempt in tenacity.Retrying( + stop=tenacity.stop_after_attempt(10), + wait=tenacity.wait_fixed(4), + ): + with attempt: + primary_client.Bucket(container_name).create() primary_object_one = primary_client.Object( container_name, obj_name @@ -1748,3 +1965,120 @@ def test_persistent_config(self): ) data = json.loads(result['Stdout']) assert data['loglevel'] == 2 + + +class CephMonKeyRotationTests(test_utils.BaseCharmTest): + """Tests for the rotate-key action.""" + + def setUp(self): + """Initialize key rotation test class.""" + super(CephMonKeyRotationTests, self).setUp() + try: + # Workaround for ubuntu units that don't play nicely with zaza. + zaza_model.get_application('ubuntu') + self.app_states = { + 'ubuntu': { + 'workload-status-message': '' + } + } + except KeyError: + self.app_states = None + + def _get_all_keys(self, unit, entity_filter): + cmd = 'sudo ceph auth ls' + result = zaza_model.run_on_unit(unit, cmd) + # Don't use json formatting, as it's buggy upstream. + data = result['Stdout'].split() + ret = set() + + for ix, line in enumerate(data): + # Structure: + # $ENTITY + # key: + # key contents + # That's why we need to move one position ahead. + if 'key:' in line and entity_filter(data[ix - 1]): + ret.add((data[ix - 1], data[ix + 1])) + return ret + + def _check_key_rotation(self, entity, unit): + def entity_filter(name): + return name.startswith(entity) + + old_keys = self._get_all_keys(unit, entity_filter) + action_obj = zaza_model.run_action( + unit_name=unit, + action_name='rotate-key', + action_params={'entity': entity} + ) + zaza_utils.assertActionRanOK(action_obj) + # NOTE(lmlg): There's a nasty race going on here. Essentially, + # since this action involves 2 different applications, what + # happens is as follows: + # (1) (2) (3) (4) + # ceph-mon rotates key | (idle) | remote-unit rotates key | (idle) + # Between (2) and (3), there's a window where all units are + # idle, _but_ the key hasn't been rotated in the other unit. + # As such, we retry a few times instead of using the + # `wait_for_application_states` interface. + + for attempt in tenacity.Retrying( + wait=tenacity.wait_exponential(multiplier=2, max=32), + reraise=True, stop=tenacity.stop_after_attempt(20), + retry=tenacity.retry_if_exception_type(AssertionError) + ): + with attempt: + new_keys = self._get_all_keys(unit, entity_filter) + self.assertNotEqual(old_keys, new_keys) + + diff = new_keys - old_keys + self.assertEqual(len(diff), 1) + first = next(iter(diff)) + # Check that the entity matches. The 'entity_filter' + # callable will return a true-like value if it + # matches the type of entity we're after (i.e: 'mgr') + self.assertTrue(entity_filter(first[0])) + + def _get_rgw_client(self, unit): + ret = self._get_all_keys(unit, lambda x: x.startswith('client.rgw')) + if not ret: + return None + return next(iter(ret))[0] + + def _get_fs_client(self, unit): + def _filter_fs(name): + return (name.startswith('mds.') and + name not in ('mds.ceph-fs', 'mds.None')) + + ret = self._get_all_keys(unit, _filter_fs) + if not ret: + return None + return next(iter(ret))[0] + + def test_key_rotate(self): + """Test that rotating the keys actually changes them.""" + unit = 'ceph-mon/0' + self._check_key_rotation('osd.0', unit) + + try: + zaza_model.get_application('ceph-radosgw') + rgw_client = self._get_rgw_client(unit) + if rgw_client: + self._check_key_rotation(rgw_client, unit) + else: + logging.info('ceph-radosgw units present, but no RGW service') + except KeyError: + pass + + try: + zaza_model.get_application('ceph-fs') + fs_svc = self._get_fs_client(unit) + if fs_svc is not None: + # Only wait for ceph-fs, as this model includes 'ubuntu' + # units, and those don't play nice with zaza (they don't + # set the workload-status-message correctly). + self._check_key_rotation(fs_svc, unit) + else: + logging.info('ceph-fs units present, but no MDS service') + except KeyError: + pass diff --git a/zaza/openstack/charm_tests/designate/tests.py b/zaza/openstack/charm_tests/designate/tests.py index ed68a8c93..2203c3a4d 100644 --- a/zaza/openstack/charm_tests/designate/tests.py +++ b/zaza/openstack/charm_tests/designate/tests.py @@ -44,8 +44,15 @@ def setUpClass(cls, application_name=None, model_alias=None): model_alias = model_alias or "" super(BaseDesignateTest, cls).setUpClass(application_name, model_alias) os_release = openstack_utils.get_os_release + current_release = os_release() - if os_release() >= os_release('bionic_rocky'): + if current_release >= os_release('jammy_caracal'): + cls.designate_svcs = [ + 'designate-api', 'designate-central', + 'designate-mdns', 'designate-worker', 'designate-sink', + 'designate-producer', + ] + elif current_release >= os_release('bionic_rocky'): cls.designate_svcs = [ 'designate-agent', 'designate-api', 'designate-central', 'designate-mdns', 'designate-worker', 'designate-sink', diff --git a/zaza/openstack/utilities/os_versions.py b/zaza/openstack/utilities/os_versions.py index 2d7639f20..7930d5f7d 100644 --- a/zaza/openstack/utilities/os_versions.py +++ b/zaza/openstack/utilities/os_versions.py @@ -42,6 +42,7 @@ ('kinetic', 'zed'), ('lunar', 'antelope'), ('mantic', 'bobcat'), + ('noble', 'caracal'), ]) @@ -71,6 +72,7 @@ ('2022.2', 'zed'), ('2023.1', 'antelope'), ('2023.2', 'bobcat'), + ('2024.1', 'caracal'), ]) OPENSTACK_RELEASES_PAIRS = [ @@ -87,6 +89,7 @@ 'focal_yoga', 'jammy_yoga', 'jammy_zed', 'kinetic_zed', 'jammy_antelope', 'lunar_antelope', 'jammy_bobcat', 'mantic_bobcat', + 'jammy_caracal', 'noble_caracal', ] SWIFT_CODENAMES = OrderedDict([ @@ -282,6 +285,7 @@ ('16', 'victoria'), # pacific ('17', 'yoga'), # quincy ('18', 'bobcat'), # reef + ('19', 'caracal'), # squid ]), 'placement-common': OrderedDict([ ('2', 'train'), @@ -320,6 +324,7 @@ 'kinetic', 'lunar', 'mantic', + 'noble', )