From 73e1ee229eefcdb88bbd4db1826aa80c597fe6ef Mon Sep 17 00:00:00 2001 From: "Chandan Kumar (raukadah)" Date: Mon, 28 Jun 2021 18:27:21 +0530 Subject: [PATCH] Added dockerfile_client for uploading dockerfiles For Downstream, on image server we host both images and dockerfiles on the same image server. It introduces dockerfile_client as a additional client which can be enabled in rdo/defaults.yaml file under allowed_clients. It is disabled by default. Change-Id: I71de9500da18ab04fc8f511288383a705f68b2bb Signed-off-by: Chandan Kumar (raukadah) Co-authored-by: Amol Kahat Signed-off-by: Amol Kahat --- .../config_environments/global_defaults.yaml | 1 + .../config_environments/rdo/defaults.yaml | 1 + .../config_environments/staging/defaults.yaml | 1 + .../dlrnapi_promoter/dockerfile_client.py | 199 ++++++++++++++++++ ci-scripts/dlrnapi_promoter/logic.py | 2 + 5 files changed, 204 insertions(+) create mode 100644 ci-scripts/dlrnapi_promoter/dockerfile_client.py diff --git a/ci-scripts/dlrnapi_promoter/config_environments/global_defaults.yaml b/ci-scripts/dlrnapi_promoter/config_environments/global_defaults.yaml index 4cec2f0b2..9212a14bd 100644 --- a/ci-scripts/dlrnapi_promoter/config_environments/global_defaults.yaml +++ b/ci-scripts/dlrnapi_promoter/config_environments/global_defaults.yaml @@ -1,4 +1,5 @@ log_level: INFO +#allowed_clients: registries_client,qcow_client,dockerfile_client,dlrn_client allowed_clients: registries_client,qcow_client,dlrn_client # relative paths are relative to code_root global_config_root: config_environments diff --git a/ci-scripts/dlrnapi_promoter/config_environments/rdo/defaults.yaml b/ci-scripts/dlrnapi_promoter/config_environments/rdo/defaults.yaml index 21b9ddd1e..6572044cf 100644 --- a/ci-scripts/dlrnapi_promoter/config_environments/rdo/defaults.yaml +++ b/ci-scripts/dlrnapi_promoter/config_environments/rdo/defaults.yaml @@ -28,6 +28,7 @@ overcloud_images: - overcloud-full.tar - overcloud-full.tar.md5 default_qcow_server: promoter +dockerfile_root: dockerfiles/ containers: containers_list_base_url: "https://opendev.org/openstack/tripleo-common/raw/commit/" diff --git a/ci-scripts/dlrnapi_promoter/config_environments/staging/defaults.yaml b/ci-scripts/dlrnapi_promoter/config_environments/staging/defaults.yaml index 0a60662fe..7a91e4ecb 100644 --- a/ci-scripts/dlrnapi_promoter/config_environments/staging/defaults.yaml +++ b/ci-scripts/dlrnapi_promoter/config_environments/staging/defaults.yaml @@ -30,6 +30,7 @@ overcloud_images: - undercloud.qcow2 - undercloud.qcow2.md5 default_qcow_server: local +dockerfile_root: dockerfile/ # Stage configuration file for the promoter staging # This versions creates only insecure registries diff --git a/ci-scripts/dlrnapi_promoter/dockerfile_client.py b/ci-scripts/dlrnapi_promoter/dockerfile_client.py new file mode 100644 index 000000000..778e57bf5 --- /dev/null +++ b/ci-scripts/dlrnapi_promoter/dockerfile_client.py @@ -0,0 +1,199 @@ +""" +This file contains classes and functionto interact with qcow images servers +to upload dockerfiles +""" +import logging +import os + +import paramiko +from common import PromotionError + + +class DockerfileConnectionClient(object): + """ + Proxy class for client connection + """ + + _log = logging.getLogger("promoter") + + def __init__(self, server_conf): + self._host = server_conf['host'] + self._user = server_conf['user'] + self._client_type = server_conf['client'] + self._keypath = server_conf['keypath'] + self._client = os + if self._client_type == "sftp": + client = paramiko.SSHClient() + client.load_system_host_keys() + client.set_missing_host_key_policy(paramiko.AutoAddPolicy()) + + keypath = os.path.expanduser(self._keypath) + self.key = paramiko.RSAKey.from_private_key_file(filename=keypath) + self.kwargs = {} + if self._user is not None: + self.kwargs['username'] = self._user + else: + self.kwargs['username'] = os.environ.get("USER") + + self._log.debug("Connecting to %s as user %s", self._host, + self._user) + + self.ssh_client = client + + def connect(self): + if hasattr(self, 'ssh_client'): + self.ssh_client.connect(self._host, pkey=self.key, **self.kwargs) + self._client = self.ssh_client.open_sftp() + + def __getattr__(self, item): + return getattr(self._client, item) + + def close(self): + if self._client_type == "sftp": + self._client.close() + + +class DockerfileClient(object): + """ + This class interacts with qcow images servers + """ + log = logging.getLogger("promoter") + + def __init__(self, config): + self.config = config + self.release = config.release + self.git_root = self.config.git_root + self.distro_name = self.config.distro_name + self.distro_version = self.config.distro_version + self.rollback_links = {} + server_conf = self.config.overcloud_images.get('qcow_servers') + qcow_server = self.config.default_qcow_server + self.user = server_conf[qcow_server]['user'] + self.root = self.config.dockerfile_root + self.host = server_conf[qcow_server]['host'] + downstream_release_map = {'osp16-2': 'rhos-16.2', + 'osp17': 'rhos-17'} + if self.config.release.startswith('osp'): + self.release = downstream_release_map[self.release] + + self.client = DockerfileConnectionClient(server_conf[qcow_server]) + self.images_dir = os.path.join( + os.path.join(config.stage_root, self.root), + config.distro, self.release, "rdo_trunk") + if self.config.release.startswith('osp'): + self.images_dir = self.images_dir.rstrip("/rdo_trunk") + + def promote(self, candidate_hash, target_label, candidate_label=None, + create_previous=True): + """ + Effective promotion of the dockerfiles. This method will handle symbolic + links to the dir containing dockerfiles from the candidate hash, + optionally saving the current link as previous + :param candidate_hash: The dlrn hash to promote + :param target_label: The name of the link to create + :param candidate_label: Currently unused + :param create_previous: A bool to determine if previous link is created + :return: None + """ + + self.client.connect() + + self.client.chdir(self.images_dir) + self.log.debug("Changing dir: {}".format(self.images_dir)) + log_header = "Qcow promote '{}' to {}:".format(candidate_hash, + target_label) + self.log.info("%s Attempting promotion", log_header) + + # Check if candidate_hash dir is present + try: + self.client.stat(candidate_hash.full_hash) + self.log.debug("Checking candidate hash dir: " + "{}".format(candidate_hash.full_hash)) + except EnvironmentError as ex: + self.log.error("%s images dir for hash %s not present or not " + "accessible", log_header, candidate_hash) + self.log.exception(ex) + self.client.close() + raise PromotionError("{} No images dir for hash {}" + "".format(log_header, candidate_hash)) + + # Check if the target label exists and points to a hash dir + current_hash = None + try: + current_hash = self.client.readlink(target_label) + self.log.debug("Checking target link: {}".format(current_hash)) + except EnvironmentError: + self.log.debug("%s No link named %s exists", log_header, + target_label) + + # If this exists Check if we can remove the symlink + if current_hash: + self.rollback_links['target_label'] = current_hash + try: + self.client.remove(target_label) + self.log.debug("Removing label: {}".format(target_label)) + except EnvironmentError as ex: + self.log.debug("Unable to remove the target_label: %s", + target_label) + self.log.exception(ex) + self.client.close() + raise + + # Check if a previous link exists and points to an hash-dir + previous_label = "previous-{}".format(target_label) + previous_hash = None + try: + previous_hash = self.client.readlink(previous_label) + self.log.debug("Previous hash: {}".format(previous_hash)) + except EnvironmentError: + self.log.debug("%s No previous-link named %s exists", + log_header, + previous_label) + self.log.debug("Previous hash %s", previous_hash) + # If it exists and we are handling it, check if we can remove and + # reassign it + if current_hash and previous_hash and create_previous: + self.rollback_links[previous_label] = previous_hash + try: + self.client.remove(previous_label) + self.log.debug("Removing previous label: " + "{}".format(previous_label)) + except EnvironmentError as ex: + self.log.debug("Unable to remove the target_label: %s", + target_label) + self.log.exception(ex) + self.client.close() + # Rollback is not tested, we enable it later, when tests are + # easier to add + # self.rollback() + raise + try: + self.client.symlink(current_hash, previous_label) + self.log.debug("Created symlink: " + "{} -> {}".format(current_hash, previous_label)) + except EnvironmentError as ex: + self.log.error("%s failed to link %s to %s", log_header, + previous_label, current_hash) + self.log.exception(ex) + # Rollback is not tested, we enable it later, when tests are + # easier to add + # self.rollback() + self.client.close() + raise + + # Finally the effective promotion + try: + self.client.symlink(candidate_hash.full_hash, target_label) + self.log.debug("Created symlink {} -> {}".format( + candidate_hash.full_hash, target_label)) + except EnvironmentError as ex: + self.log.error("%s failed to link %s to %s", log_header, + target_label, candidate_hash.full_hash) + self.log.exception(ex) + # Rollback is not tested, we enable it later, when tests are + # easier to add + # self.rollback() + finally: + self.client.close() + + self.log.info("%s Successful promotion", log_header) diff --git a/ci-scripts/dlrnapi_promoter/logic.py b/ci-scripts/dlrnapi_promoter/logic.py index 458dad7f4..78e360838 100644 --- a/ci-scripts/dlrnapi_promoter/logic.py +++ b/ci-scripts/dlrnapi_promoter/logic.py @@ -6,6 +6,7 @@ from common import PromotionError from dlrn_client import DlrnClient +from dockerfile_client import DockerfileClient from qcow_client import QcowClient from registries_client import RegistriesClient @@ -29,6 +30,7 @@ def __init__(self, config): self.dlrn_client = DlrnClient(self.config) self.registries_client = RegistriesClient(self.config) self.qcow_client = QcowClient(self.config) + self.dockerfile_client = DockerfileClient(self.config) def select_candidates(self, candidate_label, target_label): """