diff --git a/actions.yaml b/actions.yaml new file mode 100644 index 0000000..3e564b5 --- /dev/null +++ b/actions.yaml @@ -0,0 +1,2 @@ +sync-images: + description: "Sync all images into local OpenStack Cloud" diff --git a/actions/sync-images b/actions/sync-images new file mode 100755 index 0000000..0c19053 --- /dev/null +++ b/actions/sync-images @@ -0,0 +1,5 @@ +#!/bin/bash + +set -e + +/usr/share/glance-simplestreams-sync/glance-simplestreams-sync.sh diff --git a/config.yaml b/config.yaml index 42df070..5d3ec19 100644 --- a/config.yaml +++ b/config.yaml @@ -4,14 +4,14 @@ options: default: "[{url: 'http://cloud-images.ubuntu.com/releases/', name_prefix: 'ubuntu:released', path: 'streams/v1/index.sjson', max: 1, - item_filters: ['release~(trusty|xenial|bionic)', 'arch~(x86_64|amd64)', 'ftype~(disk1.img|disk.img)']}]" + item_filters: ['release~(trusty|xenial|bionic|focal)', 'arch~(x86_64|amd64)', 'ftype~(disk1.img|disk.img)']}]" description: > YAML-formatted list of simplestreams mirrors and their configuration properties. Defaults to downloading the released images from cloud-images.ubuntu.com. run: type: boolean - default: True + default: False description: "Should the sync be running or not?" use_swift: type: boolean @@ -41,10 +41,11 @@ options: description: "This is prefixed to the object name when uploading to glance." custom_properties: type: string - default: "" + default: description: > - YAML-formatted list of any custom properties to be set in glance - for the synced image, e.g. architecture, hypervisor_type. + Space separated list of custom properties (format key=value) to be + set in glance for all synced images e.g. hw_firmware_type, + hw_vif_multiqueue_enabled. content_id_template: type: string default: "auto.sync" @@ -99,21 +100,11 @@ options: source: type: string default: - description: | - Optional configuration to support use of additional sources such as: - - - ppa:myteam/ppa - - cloud:trusty-proposed/kilo - - http://my.archive.com/ubuntu main - - The last option should be used in conjunction with the key configuration - option. + description: DEPRECATED - option no longer used and will be removed key: type: string default: - description: | - Key ID to import to the apt keyring to support use with arbitary source - configuration from outside of Launchpad archives or PPA's. + description: DEPRECATED - option no longer used and will be removed hypervisor_mapping: type: boolean default: false @@ -121,3 +112,7 @@ options: Enable configuration of hypervisor-type on synced images. . This is useful in multi-hypervisor clouds supporting both LXD and KVM. + snap-channel: + type: string + default: stable + description: Snap channel to install simplestreams snap from diff --git a/files/glance-simplestreams-sync.sh b/files/glance-simplestreams-sync.sh index d29c438..3d8f0e8 100755 --- a/files/glance-simplestreams-sync.sh +++ b/files/glance-simplestreams-sync.sh @@ -1,4 +1,11 @@ #!/bin/bash + +if [ -z "$HOME" ]; then + export HOME=/root +fi + +set -e + if [ -f /etc/profile.d/juju-proxy.sh ]; then source /etc/profile.d/juju-proxy.sh elif [ -f /etc/juju-proxy.conf ]; then @@ -8,10 +15,10 @@ elif [ -f /home/ubuntu/.juju-proxy ]; then fi source /etc/lsb-release -if [[ $DISTRIB_RELEASE > "18.04" ]]; then +if dpkg --compare-versions $DISTRIB_RELEASE gt "18.04"; then PYTHON=python3 else PYTHON=python fi -$PYTHON /usr/share/glance-simplestreams-sync/glance-simplestreams-sync.py +$PYTHON /usr/share/glance-simplestreams-sync/glance_simplestreams_sync.py diff --git a/files/glance-simplestreams-sync.py b/files/glance_simplestreams_sync.py similarity index 77% rename from files/glance-simplestreams-sync.py rename to files/glance_simplestreams_sync.py index f1a3dc9..fb0821d 100755 --- a/files/glance-simplestreams-sync.py +++ b/files/glance_simplestreams_sync.py @@ -27,6 +27,8 @@ import copy import logging import os +import shutil +import tempfile def setup_logging(): @@ -59,10 +61,6 @@ def setup_logging(): from keystoneclient.v3 import client as keystone_v3_client import keystoneclient.exceptions as keystone_exceptions import kombu -from simplestreams.mirrors import glance, UrlMirrorReader -from simplestreams.objectstores.swift import SwiftObjectStore -from simplestreams.objectstores import FileStore -from simplestreams.util import read_signed, path_from_mirror_url import sys import time import traceback @@ -92,7 +90,11 @@ def setup_logging(): CRON_POLL_FILENAME = '/etc/cron.d/glance_simplestreams_sync_fastpoll' -CACERT_FILE = os.path.join(CONF_FILE_DIR, 'cacert.pem') +SSTREAM_SNAP_COMMON = '/var/snap/simplestreams/common' +SSTREAM_LOG_FILE = os.path.join(SSTREAM_SNAP_COMMON, + 'sstream-mirror-glance.log') + +CACERT_FILE = os.path.join(SSTREAM_SNAP_COMMON, 'cacert.pem') SYSTEM_CACERT_FILE = '/etc/ssl/certs/ca-certificates.crt' # TODOs: @@ -103,63 +105,6 @@ def setup_logging(): # - figure out what content_id is and whether we should allow users to # set it -try: - from simplestreams.util import ProgressAggregator - SIMPLESTREAMS_HAS_PROGRESS = True -except ImportError: - class ProgressAggregator: - "Dummy class to allow charm to load with old simplestreams" - SIMPLESTREAMS_HAS_PROGRESS = False - - -class GlanceMirrorWithCustomProperties(glance.GlanceMirror): - def __init__(self, *args, **kwargs): - custom_properties = kwargs.pop('custom_properties', {}) - super(GlanceMirrorWithCustomProperties, self).__init__(*args, **kwargs) - self.custom_properties = custom_properties - - def prepare_glance_arguments(self, *args, **kwargs): - - glance_args = (super(GlanceMirrorWithCustomProperties, self) - .prepare_glance_arguments(*args, **kwargs)) - - if self.custom_properties: - log.info('Setting custom image properties: {}'.format( - self.custom_properties)) - props = glance_args.get('properties', {}) - props.update(self.custom_properties) - glance_args['properties'] = props - - return glance_args - - -class StatusMessageProgressAggregator(ProgressAggregator): - def __init__(self, remaining_items, send_status_message): - super(StatusMessageProgressAggregator, self).__init__(remaining_items) - self.send_status_message = send_status_message - - def emit(self, progress): - size = float(progress['size']) - written = float(progress['written']) - cur = self.total_image_count - len(self.remaining_items) + 1 - totpct = float(self.total_written) / self.total_size - msg = "{name} {filepct:.0%}\n"\ - "({cur} of {tot} images) total: "\ - "{totpct:.0%}".format(name=progress['name'], - filepct=(written / size), - cur=cur, - tot=self.total_image_count, - totpct=totpct) - self.send_status_message(dict(status="Syncing", - message=msg)) - - -def policy(content, path): - if path.endswith('sjson'): - return read_signed(content, keyring=KEYRING) - else: - return content - def read_conf(filename): with open(filename) as f: @@ -292,54 +237,68 @@ def do_sync(charm_conf, status_exchange): # user_agent = charm_conf.get("user_agent") for mirror_info in charm_conf['mirror_list']: - mirror_url, initial_path = path_from_mirror_url(mirror_info['url'], - mirror_info['path']) - - log.info("configuring sync for url {}".format(mirror_info)) - - smirror = UrlMirrorReader( - mirror_url, policy=policy) - - if charm_conf['use_swift']: - store = SwiftObjectStore(SWIFT_DATA_DIR) - else: - # Use the local apache server to serve product streams - store = FileStore(prefix=APACHE_DATA_DIR) - - content_id = charm_conf['content_id_template'].format( - region=charm_conf['region']) - - config = {'max_items': mirror_info['max'], - 'modify_hook': charm_conf['modify_hook_scripts'], - 'keep_items': True, - 'content_id': content_id, - 'cloud_name': charm_conf['cloud_name'], - 'item_filters': mirror_info['item_filters'], - 'hypervisor_mapping': charm_conf.get('hypervisor_mapping', - False)} - - mirror_args = dict(config=config, objectstore=store, - name_prefix=charm_conf['name_prefix']) - mirror_args['custom_properties'] = charm_conf.get('custom_properties', - {}) - - if SIMPLESTREAMS_HAS_PROGRESS: - log.info("Calling DryRun mirror to get item list") - - drmirror = glance.ItemInfoDryRunMirror(config=config, - objectstore=store) - drmirror.sync(smirror, path=initial_path) - p = StatusMessageProgressAggregator(drmirror.items, - status_exchange.send_message) - mirror_args['progress_callback'] = p.progress_callback - else: - log.info("Detected simplestreams version without progress" - " update support. Only limited feedback available.") - - tmirror = GlanceMirrorWithCustomProperties(**mirror_args) - - log.info("calling GlanceMirror.sync") - tmirror.sync(smirror, path=initial_path) + # NOTE: output directory must be under HOME + # or snap cannot access it for stream files + tmpdir = tempfile.mkdtemp(dir=os.environ['HOME']) + try: + log.info("Configuring sync for url {}".format(mirror_info)) + content_id = charm_conf['content_id_template'].format( + region=charm_conf['region']) + + sync_command = [ + "/snap/bin/simplestreams.sstream-mirror-glance", + "-vv", + "--keep", + "--max", str(mirror_info['max']), + "--content-id", content_id, + "--cloud-name", charm_conf['cloud_name'], + "--path", mirror_info['path'], + "--name-prefix", charm_conf['name_prefix'], + "--keyring", KEYRING, + "--log-file", SSTREAM_LOG_FILE, + ] + + if charm_conf['use_swift']: + sync_command += [ + '--output-swift', + SWIFT_DATA_DIR + ] + else: + sync_command += [ + "--output-dir", + tmpdir + ] + + if charm_conf.get('hypervisor_mapping', False): + sync_command += [ + '--hypervisor-mapping' + ] + if charm_conf.get('custom_properties'): + custom_properties = charm_conf.get('custom_properties').split() + for custom_property in custom_properties: + sync_command += [ + '--custom-property', + custom_property + ] + + sync_command += [ + mirror_info['url'], + ] + sync_command += mirror_info['item_filters'] + + log.info("calling sstream-mirror-glance") + log.debug("command: {}".format(" ".join(sync_command))) + subprocess.check_call(sync_command) + + if not charm_conf['use_swift']: + # Sync output directory to APACHE_DATA_DIR + subprocess.check_call([ + 'rsync', '-avz', + os.path.join(tmpdir, charm_conf['region'], 'streams'), + APACHE_DATA_DIR + ]) + finally: + shutil.rmtree(tmpdir) def update_product_streams_service(ksc, services, region): @@ -367,18 +326,32 @@ def update_product_streams_service(ksc, services, region): def juju_run_cmd(cmd): - '''Execute the passed commands under the local unit context''' - id_conf, _ = get_conf() - unit_name = id_conf['unit_name'] - _cmd = ['juju-run', unit_name, ' '.join(cmd)] + '''Execute the passed commands under the local unit context if required''' + # NOTE: determine whether juju-run is actually required + # supporting execution via actions. + if not os.environ.get('JUJU_CONTEXT_ID'): + id_conf, _ = get_conf() + unit_name = id_conf['unit_name'] + _cmd = ['juju-run', unit_name, ' '.join(cmd)] + else: + _cmd = cmd log.info("Executing command: {}".format(_cmd)) return subprocess.check_output(_cmd) def status_set(status, message): try: - juju_run_cmd(['status-set', status, - '"{}"'.format(message)]) + # NOTE: format of message is different for out of + # context execution. + if not os.environ.get('JUJU_CONTEXT_ID'): + juju_run_cmd(['status-set', status, + '"{}"'.format(message)]) + else: + subprocess.check_output([ + 'status-set', + status, + message + ]) except subprocess.CalledProcessError: log.info(message) diff --git a/hooks/hooks.py b/hooks/hooks.py index b9967c0..86b391a 100755 --- a/hooks/hooks.py +++ b/hooks/hooks.py @@ -14,6 +14,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +import base64 import glob import os import shutil @@ -32,6 +33,7 @@ def _add_path(path): _add_path(_root) from charmhelpers.fetch import add_source, apt_install, apt_update +from charmhelpers.fetch.snap import snap_install from charmhelpers.core import hookenv from charmhelpers.payload.execd import execd_preinstall @@ -66,7 +68,7 @@ def _add_path(path): MIRRORS_CONF_FILE_NAME = os.path.join(CONF_FILE_DIR, 'mirrors.yaml') ID_CONF_FILE_NAME = os.path.join(CONF_FILE_DIR, 'identity.yaml') -SYNC_SCRIPT_NAME = "glance-simplestreams-sync.py" +SYNC_SCRIPT_NAME = "glance_simplestreams_sync.py" SCRIPT_WRAPPER_NAME = "glance-simplestreams-sync.sh" CRON_D = '/etc/cron.d/' @@ -76,16 +78,18 @@ def _add_path(path): ERR_FILE_EXISTS = 17 -PACKAGES = ['python-simplestreams', 'python-glanceclient', +PACKAGES = ['python-glanceclient', 'python-yaml', 'python-keystoneclient', 'python-kombu', - 'python-swiftclient', 'ubuntu-cloudimage-keyring'] + 'python-swiftclient', 'ubuntu-cloudimage-keyring', 'snapd'] -PY3_PACKAGES = ['python3-simplestreams', 'python3-glanceclient', +PY3_PACKAGES = ['python3-glanceclient', 'python3-yaml', 'python3-keystoneclient', 'python3-kombu', 'python3-swiftclient'] +JUJU_CA_CERT = "/usr/local/share/ca-certificates/keystone_juju_ca_cert.crt" + hooks = hookenv.Hooks() @@ -110,8 +114,12 @@ class SSLIdentityServiceContext(IdentityServiceContext): def __call__(self): ctxt = super(SSLIdentityServiceContext, self).__call__() ssl_ca = hookenv.config('ssl_ca') - if ctxt and ssl_ca: - ctxt['ssl_ca'] = ssl_ca + if ctxt: + if ssl_ca: + ctxt['ssl_ca'] = ssl_ca + elif os.path.exists(JUJU_CA_CERT): + with open(JUJU_CA_CERT, 'rb') as ca_cert: + ctxt['ssl_ca'] = base64.b64encode(ca_cert.read()).decode() return ctxt @@ -184,6 +192,12 @@ def get_configs(): return configs +def install_gss_wrappers(): + """Installs wrapper scripts for execution of simplestreams sync.""" + for fn in [SYNC_SCRIPT_NAME, SCRIPT_WRAPPER_NAME]: + shutil.copy(os.path.join("files", fn), USR_SHARE_DIR) + + def install_cron_script(): """Installs cron job in /etc/cron.$frequency/ for repeating sync @@ -191,9 +205,6 @@ def install_cron_script(): up-to-date. """ - for fn in [SYNC_SCRIPT_NAME, SCRIPT_WRAPPER_NAME]: - shutil.copy(os.path.join("files", fn), USR_SHARE_DIR) - config = hookenv.config() installed_script = os.path.join(USR_SHARE_DIR, SCRIPT_WRAPPER_NAME) linkname = '/etc/cron.{f}/{s}'.format(f=config['frequency'], @@ -283,6 +294,11 @@ def install(): apt_install(_packages) + snap_install('simplestreams', + *['--channel={}'.format(hookenv.config('snap-channel'))]) + + install_gss_wrappers() + hookenv.log('end install hook.') diff --git a/metadata.yaml b/metadata.yaml index 83ea487..06a1e04 100644 --- a/metadata.yaml +++ b/metadata.yaml @@ -8,7 +8,6 @@ tags: - misc - openstack series: -- trusty - xenial - bionic - focal diff --git a/tests/bundles/xenial-pike.yaml b/tests/bundles/bionic-ussuri.yaml similarity index 89% rename from tests/bundles/xenial-pike.yaml rename to tests/bundles/bionic-ussuri.yaml index a753717..89c650a 100644 --- a/tests/bundles/xenial-pike.yaml +++ b/tests/bundles/bionic-ussuri.yaml @@ -1,11 +1,10 @@ -series: xenial +series: bionic comment: - 'machines section to decide order of deployment. database sooner = faster' machines: '0': - constraints: mem=3072M '1': '2': '3': @@ -43,24 +42,24 @@ applications: to: - '2' keystone: + series: bionic charm: cs:~openstack-charmers-next/keystone num_units: 1 options: - openstack-origin: cloud:xenial-pike + openstack-origin: cloud:bionic-ussuri to: - '3' glance: charm: cs:~openstack-charmers-next/glance num_units: 1 options: - openstack-origin: cloud:xenial-pike + openstack-origin: cloud:bionic-ussuri to: - '4' glance-simplestreams-sync: charm: ../../glance-simplestreams-sync num_units: 1 options: - source: ppa:simplestreams-dev/trunk use_swift: False to: - '5' diff --git a/tests/bundles/trusty-mitaka.yaml b/tests/bundles/trusty-mitaka.yaml deleted file mode 100644 index a6e8b58..0000000 --- a/tests/bundles/trusty-mitaka.yaml +++ /dev/null @@ -1,59 +0,0 @@ -series: trusty - -comment: - - 'machines section to decide order of deployment. database sooner = faster' - - 'no ssl for this bundle since charm-vault does not support trusty' - -machines: - '0': - constraints: mem=3072M - # series "trusty" not supported by mysql charm - series: xenial - '1': - '2': - '3': - '4': - -relations: - - ['keystone:shared-db', 'mysql:shared-db'] - - ['glance:shared-db', 'mysql:shared-db'] - - ['glance:amqp', 'rabbitmq-server:amqp'] - - ['glance-simplestreams-sync:amqp', 'rabbitmq-server:amqp'] - - ['glance:identity-service', 'keystone:identity-service'] - - ['glance-simplestreams-sync:identity-service', 'keystone:identity-service'] - -applications: - mysql: - charm: cs:~openstack-charmers-next/percona-cluster - # series "trusty" not supported by mysql charm - series: xenial - num_units: 1 - to: - - '0' - rabbitmq-server: - charm: cs:~openstack-charmers-next/rabbitmq-server - num_units: 1 - to: - - '1' - keystone: - charm: cs:~openstack-charmers-next/keystone - num_units: 1 - options: - openstack-origin: cloud:trusty-mitaka - to: - - '2' - glance: - charm: cs:~openstack-charmers-next/glance - num_units: 1 - options: - openstack-origin: cloud:trusty-mitaka - to: - - '3' - glance-simplestreams-sync: - charm: ../../glance-simplestreams-sync - num_units: 1 - options: - source: ppa:simplestreams-dev/trunk - use_swift: False - to: - - '4' diff --git a/tests/bundles/xenial-ocata.yaml b/tests/bundles/xenial-ocata.yaml deleted file mode 100644 index 10df792..0000000 --- a/tests/bundles/xenial-ocata.yaml +++ /dev/null @@ -1,66 +0,0 @@ -series: xenial - -comment: - - 'machines section to decide order of deployment. database sooner = faster' - -machines: - '0': - constraints: mem=3072M - '1': - '2': - '3': - '4': - '5': - -relations: - - ['vault:shared-db', 'mysql:shared-db'] - - ['keystone:shared-db', 'mysql:shared-db'] - - ['glance:shared-db', 'mysql:shared-db'] - - ['glance:amqp', 'rabbitmq-server:amqp'] - - ['glance-simplestreams-sync:amqp', 'rabbitmq-server:amqp'] - - ['keystone:certificates', 'vault:certificates'] - - ['glance:certificates', 'vault:certificates'] - - ['glance-simplestreams-sync:certificates', 'vault:certificates'] - - ['glance:identity-service', 'keystone:identity-service'] - - ['glance-simplestreams-sync:identity-service', 'keystone:identity-service'] - -applications: - mysql: - charm: cs:~openstack-charmers-next/percona-cluster - num_units: 1 - to: - - '0' - rabbitmq-server: - charm: cs:~openstack-charmers-next/rabbitmq-server - num_units: 1 - options: - ssl: 'on' # must be str(in quote), otherwise it's bool - to: - - '1' - vault: - charm: cs:~openstack-charmers-next/vault - num_units: 1 - to: - - '2' - keystone: - charm: cs:~openstack-charmers-next/keystone - num_units: 1 - options: - openstack-origin: cloud:xenial-ocata - to: - - '3' - glance: - charm: cs:~openstack-charmers-next/glance - num_units: 1 - options: - openstack-origin: cloud:xenial-ocata - to: - - '4' - glance-simplestreams-sync: - charm: ../../glance-simplestreams-sync - num_units: 1 - options: - source: ppa:simplestreams-dev/trunk - use_swift: False - to: - - '5' diff --git a/tests/tests.yaml b/tests/tests.yaml index 601d7d2..172e5ac 100644 --- a/tests/tests.yaml +++ b/tests/tests.yaml @@ -6,15 +6,13 @@ comment: # functest-run-suite ... # functest-deploy --bundle /path/to/gate/bundle gate_bundles: - - model_alias_trusty: trusty-mitaka - xenial-mitaka - - xenial-ocata - - xenial-pike - xenial-queens - bionic-queens - bionic-rocky - bionic-stein - bionic-train + - bionic-ussuri - focal-ussuri tests_options: @@ -25,7 +23,7 @@ tests_options: # functest-deploy --bundle /path/to/smoke/bundle # smoke bundle should (Ubuntu LTS latest)-(OpenStack latest) smoke_bundles: - - bionic-train + - bionic-ussuri # functest-run-suite --dev ... # functest-deploy --bundle /path/to/dev/bundle @@ -43,14 +41,13 @@ target_deploy_status: glance-simplestreams-sync: # gss will be blocked since glance and rabbitmq don't have their # certificates yet. This should be fixed after vault initialization - workload-status: blocked - workload-status-message: Image sync failed, retrying soon. + workload-status: unknown + workload-status-message: "" # functest-configure configure: - zaza.openstack.charm_tests.vault.setup.auto_initialize - # skip vault init for trusty since vault doesn't suport trusty - - model_alias_trusty: [] + - zaza.openstack.charm_tests.glance_simplestreams_sync.setup.sync_images # functest-test tests: diff --git a/unit_tests/test_hooks.py b/unit_tests/test_hooks.py index a11dbb6..637f549 100644 --- a/unit_tests/test_hooks.py +++ b/unit_tests/test_hooks.py @@ -57,6 +57,8 @@ def test_default_config(self, local_unit, nrpe_config, nag_host, setattr(self.test_config, "changed", lambda x: False) config.return_value = self.test_config + self.test_config.set('run', True) + hooks.config_changed() symlink.assert_any_call(os.path.join(self.sharedir, @@ -92,10 +94,9 @@ def test_custom_properties_config(self, local_unit, nrpe_config, nag_host, nrpe_config.return_value = self.test_config setattr(self.test_config, "changed", lambda x: False) - self.test_config.config["custom_properties"] = { - 'hypervisor_type': 'kvm', - 'hw_firmware_type': 'uefi' - } + self.test_config.config["custom_properties"] = ( + "hypervisor_type=kvm hw_firmware_type=uefi" + ) config.return_value = self.test_config hooks.config_changed()