From 8f507f0deb8633f24903896b09574e1b7f1ae545 Mon Sep 17 00:00:00 2001 From: Jason Boutte Date: Fri, 22 Jan 2021 14:22:56 -0800 Subject: [PATCH 1/9] Fixes customizable storage class when retoring to k8s --- images/benji-k8s/bin/benji-restore-pvc | 4 +++- src/benji/helpers/kubernetes.py | 4 ++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/images/benji-k8s/bin/benji-restore-pvc b/images/benji-k8s/bin/benji-restore-pvc index 3ef4dee8..6992051a 100755 --- a/images/benji-k8s/bin/benji-restore-pvc +++ b/images/benji-k8s/bin/benji-restore-pvc @@ -25,6 +25,7 @@ parser.add_argument('-f', parser.add_argument(metavar='version_uid', dest='version_uid', help='Version uid') parser.add_argument(metavar='pvc_namespace', dest='pvc_namespace', help='PVC namespace') parser.add_argument(metavar='pvc_name', dest='pvc_name', help='PVC name') +parser.add_argument(metavar='pvc_storage_class', dest='pvc_storage_class', help='PVC storageclassname') args = parser.parse_args() @@ -57,7 +58,8 @@ except ApiException as exception: raise RuntimeError(f'Unexpected Kubernetes API exception: {str(exception)}') if pvc is None: - pvc = benji.helpers.kubernetes.create_pvc(args.pvc_name, args.pvc_namespace, version_size) + pvc = benji.helpers.kubernetes.create_pvc(args.pvc_name, args.pvc_namespace, + version_size, args.pvc_storage_class) else: if not args.force: raise RuntimeError('PVC already exists. Will not overwrite it unless forced.') diff --git a/src/benji/helpers/kubernetes.py b/src/benji/helpers/kubernetes.py index bfcff5f3..7ff2a887 100644 --- a/src/benji/helpers/kubernetes.py +++ b/src/benji/helpers/kubernetes.py @@ -172,7 +172,7 @@ def create_pvc_event(*, type: str, reason: str, message: str, pvc_namespace: str def create_pvc(pvc_name: str, pvc_namespace: int, - pvc_size: str) -> kubernetes.client.models.v1_persistent_volume_claim.V1PersistentVolumeClaim: + pvc_size: str, pvc_storage_class: str) -> kubernetes.client.models.v1_persistent_volume_claim.V1PersistentVolumeClaim: pvc = { 'kind': 'PersistentVolumeClaim', 'apiVersion': 'v1', @@ -181,7 +181,7 @@ def create_pvc(pvc_name: str, pvc_namespace: int, 'name': pvc_name, }, 'spec': { - 'storageClassName': 'rbd', + 'storageClassName': pvc_storage_class, 'accessModes': ['ReadWriteOnce'], 'resources': { 'requests': { From 5d5ef57afdddee549d9ed3bdde9773e00f4b3277 Mon Sep 17 00:00:00 2001 From: Jason Boutte Date: Fri, 22 Jan 2021 14:31:49 -0800 Subject: [PATCH 2/9] Fixes storageclass to be optional --- images/benji-k8s/bin/benji-restore-pvc | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/images/benji-k8s/bin/benji-restore-pvc b/images/benji-k8s/bin/benji-restore-pvc index 6992051a..1af1f35f 100755 --- a/images/benji-k8s/bin/benji-restore-pvc +++ b/images/benji-k8s/bin/benji-restore-pvc @@ -25,7 +25,8 @@ parser.add_argument('-f', parser.add_argument(metavar='version_uid', dest='version_uid', help='Version uid') parser.add_argument(metavar='pvc_namespace', dest='pvc_namespace', help='PVC namespace') parser.add_argument(metavar='pvc_name', dest='pvc_name', help='PVC name') -parser.add_argument(metavar='pvc_storage_class', dest='pvc_storage_class', help='PVC storageclassname') +parser.add_argument('--pvc-storage-class', metavar='pvc_storage_class', dest='pvc_storage_class', + help='PVC storageclassname', default='rbd') args = parser.parse_args() From a48a3be6f4b7dd842d74938aeb7665ed1fbc8ca6 Mon Sep 17 00:00:00 2001 From: Jason Boutte Date: Fri, 22 Jan 2021 14:38:58 -0800 Subject: [PATCH 3/9] Fixes determining pv pool and image when restoring --- images/benji-k8s/bin/benji-restore-pvc | 32 +++++++++++++++++++++++++- 1 file changed, 31 insertions(+), 1 deletion(-) diff --git a/images/benji-k8s/bin/benji-restore-pvc b/images/benji-k8s/bin/benji-restore-pvc index 1af1f35f..a70b56df 100755 --- a/images/benji-k8s/bin/benji-restore-pvc +++ b/images/benji-k8s/bin/benji-restore-pvc @@ -81,8 +81,38 @@ while True: pv = core_v1_api.read_persistent_volume(pvc.spec.volume_name) +pool, image = None, None + +# Taken from backup code, maybe refactor? +if hasattr(pv.spec, 'rbd') and hasattr(pv.spec.rbd, 'pool') and hasattr(pv.spec.rbd, 'image'): + # Native Kubernetes RBD PV + pool, image = pv.spec.rbd.pool, pv.spec.rbd.image +elif hasattr(pv.spec, 'flex_volume') and hasattr(pv.spec.flex_volume, 'options') and hasattr( + pv.spec.flex_volume, 'driver'): + # Rook Ceph PV + options = pv.spec.flex_volume.options + driver = pv.spec.flex_volume.driver + if driver.startswith('ceph.rook.io/') and options.get('pool') and options.get('image'): + pool, image = options['pool'], options['image'] +elif (hasattr(pv.spec, 'csi') + and hasattr(pv.spec.csi, 'driver') + and pv.spec.csi.driver == 'rook-ceph.rbd.csi.ceph.com' + and hasattr(pv.spec.csi, 'volume_handle') + and pv.spec.csi.volume_handle + and hasattr(pv.spec.csi, 'volume_attributes') + and pv.spec.csi.volume_attributes.get('pool')): + attributes = pv.spec.csi.volume_attributes + volume_handle_parts = pv.spec.csi.volume_handle.split('-') + if len(volume_handle_parts) >= 9: + image_prefix = attributes.get('volumeNamePrefix', 'csi-vol-') + image_suffix = '-'.join(volume_handle_parts[len(volume_handle_parts)-5:]) + pool, image = attributes['pool'], image_prefix + image_suffix + +if pool is None or image is None: + raise RuntimeError(f'Unable to determine PersistentVolume pool or image for {pv.metadata.name}') + utils.subprocess_run([ 'benji', '--log-level', settings.benji_log_level, 'restore', '--sparse', '--force', args.version_uid, - f'rbd:{pv.spec.rbd.pool}/{pv.spec.rbd.image}' + f'rbd:{pool}/{image}', ]) sys.exit(0) From c7c57bea5029b09403f1a4bebfc3405776fd7022 Mon Sep 17 00:00:00 2001 From: Jason Boutte Date: Fri, 22 Jan 2021 14:57:49 -0800 Subject: [PATCH 4/9] Fixes configurable io when restoring --- images/benji-k8s/bin/benji-restore-pvc | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/images/benji-k8s/bin/benji-restore-pvc b/images/benji-k8s/bin/benji-restore-pvc index a70b56df..ac35b6c4 100755 --- a/images/benji-k8s/bin/benji-restore-pvc +++ b/images/benji-k8s/bin/benji-restore-pvc @@ -27,6 +27,8 @@ parser.add_argument(metavar='pvc_namespace', dest='pvc_namespace', help='PVC nam parser.add_argument(metavar='pvc_name', dest='pvc_name', help='PVC name') parser.add_argument('--pvc-storage-class', metavar='pvc_storage_class', dest='pvc_storage_class', help='PVC storageclassname', default='rbd') +parser.add_argument('--ios-name', metavar='ios_name', dest='ios_name', help='Name of I/O', + default='rbd') args = parser.parse_args() @@ -113,6 +115,6 @@ if pool is None or image is None: utils.subprocess_run([ 'benji', '--log-level', settings.benji_log_level, 'restore', '--sparse', '--force', args.version_uid, - f'rbd:{pool}/{image}', + f'{args.ios_name}:{pool}/{image}', ]) sys.exit(0) From 0ad07dd4884da7d4a6ff8034ef10c56d546c7c97 Mon Sep 17 00:00:00 2001 From: Lars Fenneberg Date: Tue, 16 Feb 2021 20:27:24 +0100 Subject: [PATCH 5/9] Refactor K8s tools in benji-k8s container image --- images/benji-k8s/Dockerfile | 6 +- images/benji-k8s/bin/benji-command | 32 --- images/benji-k8s/bin/benji-restore-pvc | 120 ----------- images/benji-k8s/bin/benji-versions-status | 33 --- images/benji-k8s/k8s-tools/MANIFEST.in | 2 + images/benji-k8s/k8s-tools/setup.py | 24 +++ .../benji-k8s/k8s-tools/src/benji/__init__.py | 0 .../k8s-tools/src/benji/k8s_tools/__init__.py | 0 .../src/benji/k8s_tools/scripts/__init__.py | 0 .../benji/k8s_tools/scripts/backup_pvc.py} | 195 ++++++++---------- .../src/benji/k8s_tools/scripts/command.py | 33 +++ .../benji/k8s_tools/scripts/restore_pvc.py | 106 ++++++++++ .../k8s_tools/scripts/versions_status.py | 35 ++++ images/benji/Dockerfile | 2 +- src/benji/helpers/kubernetes.py | 40 +++- 15 files changed, 326 insertions(+), 302 deletions(-) delete mode 100644 images/benji-k8s/bin/benji-command delete mode 100755 images/benji-k8s/bin/benji-restore-pvc delete mode 100755 images/benji-k8s/bin/benji-versions-status create mode 100644 images/benji-k8s/k8s-tools/MANIFEST.in create mode 100644 images/benji-k8s/k8s-tools/setup.py create mode 100644 images/benji-k8s/k8s-tools/src/benji/__init__.py create mode 100644 images/benji-k8s/k8s-tools/src/benji/k8s_tools/__init__.py create mode 100644 images/benji-k8s/k8s-tools/src/benji/k8s_tools/scripts/__init__.py rename images/benji-k8s/{bin/benji-backup-pvc => k8s-tools/src/benji/k8s_tools/scripts/backup_pvc.py} (71%) create mode 100644 images/benji-k8s/k8s-tools/src/benji/k8s_tools/scripts/command.py create mode 100644 images/benji-k8s/k8s-tools/src/benji/k8s_tools/scripts/restore_pvc.py create mode 100644 images/benji-k8s/k8s-tools/src/benji/k8s_tools/scripts/versions_status.py diff --git a/images/benji-k8s/Dockerfile b/images/benji-k8s/Dockerfile index a6335d97..9c3c5476 100644 --- a/images/benji-k8s/Dockerfile +++ b/images/benji-k8s/Dockerfile @@ -18,8 +18,10 @@ LABEL org.label-schema.schema-version="1.0" \ RUN curl -o /usr/bin/kubectl -sSL https://storage.googleapis.com/kubernetes-release/release/$(curl -s https://storage.googleapis.com/kubernetes-release/release/stable.txt)/bin/linux/amd64/kubectl && \ chmod a+x /usr/bin/kubectl -COPY images/benji-k8s/bin/ $VENV_DIR/bin/ -RUN chmod a+x $VENV_DIR/bin/* +COPY images/benji-k8s/k8s-tools /k8s-tools-source +RUN . $VENV_DIR/bin/activate && \ + pip install /k8s-tools-source && \ + rm -rf /k8s-tools-source ENTRYPOINT ["/bin/bash"] CMD ["-c", "sleep 3650d"] diff --git a/images/benji-k8s/bin/benji-command b/images/benji-k8s/bin/benji-command deleted file mode 100644 index eb8f5205..00000000 --- a/images/benji-k8s/bin/benji-command +++ /dev/null @@ -1,32 +0,0 @@ -#!/usr/bin/env python3 -import sys -import time -from typing import Optional - -import benji.helpers.prometheus as prometheus -import benji.helpers.settings as settings -import benji.helpers.utils as utils - -utils.setup_logging() - -exception: Optional[Exception] = None -command = ' '.join(sys.argv[1:]) -start_time = time.time() - -prometheus.command_start_time.labels(command=command).set(start_time) -try: - utils.subprocess_run(['benji', '--log-level', settings.benji_log_level] + sys.argv[1:]) -except Exception as exception: - prometheus.command_status_failed.labels(command=command).set(1) - completion_time = time.time() - prometheus.command_completion_time.labels(command=command).set(completion_time) - prometheus.command_runtime_seconds.labels(command=command).set(completion_time - start_time) - prometheus.push(prometheus.command_registry) - raise exception -else: - prometheus.command_status_succeeded.labels(command=command).set(1) - completion_time = time.time() - prometheus.command_completion_time.labels(command=command).set(completion_time) - prometheus.command_runtime_seconds.labels(command=command).set(completion_time - start_time) - prometheus.push(prometheus.command_registry) - sys.exit(0) diff --git a/images/benji-k8s/bin/benji-restore-pvc b/images/benji-k8s/bin/benji-restore-pvc deleted file mode 100755 index ac35b6c4..00000000 --- a/images/benji-k8s/bin/benji-restore-pvc +++ /dev/null @@ -1,120 +0,0 @@ -#!/usr/bin/env python3 -import argparse -import logging -import sys -import time - -import kubernetes -from kubernetes.client.rest import ApiException - -import benji.helpers.kubernetes -import benji.helpers.settings as settings -import benji.helpers.utils as utils - -utils.setup_logging() -logger = logging.getLogger() - -parser = argparse.ArgumentParser(formatter_class=argparse.ArgumentDefaultsHelpFormatter, allow_abbrev=False) - -parser.add_argument('-f', - '--force', - dest='force', - action='store_true', - default=False, - help='Overwrite content of existing persistent volumes') -parser.add_argument(metavar='version_uid', dest='version_uid', help='Version uid') -parser.add_argument(metavar='pvc_namespace', dest='pvc_namespace', help='PVC namespace') -parser.add_argument(metavar='pvc_name', dest='pvc_name', help='PVC name') -parser.add_argument('--pvc-storage-class', metavar='pvc_storage_class', dest='pvc_storage_class', - help='PVC storageclassname', default='rbd') -parser.add_argument('--ios-name', metavar='ios_name', dest='ios_name', help='Name of I/O', - default='rbd') - -args = parser.parse_args() - -benji.helpers.kubernetes.load_config() -core_v1_api = kubernetes.client.CoreV1Api() - -logger.info(f'Restoring version {args.version_uid} to PVC {args.pvc_namespace}/{args.pvc_name}.') - -benji_ls = utils.subprocess_run( - ['benji', '--machine-output', '--log-level', settings.benji_log_level, 'ls', f'uid == "{args.version_uid}"'], - decode_json=True) -assert isinstance(benji_ls, dict) -assert 'versions' in benji_ls -assert isinstance(benji_ls['versions'], list) - -if len(benji_ls['versions']) == 0: - raise RuntimeError(f'Size of {args.version_uid} could not be determined.') - -assert isinstance(benji_ls['versions'][0], dict) -assert isinstance(benji_ls['versions'][0]['size'], int) -version_size = benji_ls['versions'][0]['size'] - -# This assumes that the Kubernetes client has already been initialized -core_v1_api = kubernetes.client.CoreV1Api() -pvc = None -try: - pvc = core_v1_api.read_namespaced_persistent_volume_claim(args.pvc_name, args.pvc_namespace) -except ApiException as exception: - if exception.status != 404: - raise RuntimeError(f'Unexpected Kubernetes API exception: {str(exception)}') - -if pvc is None: - pvc = benji.helpers.kubernetes.create_pvc(args.pvc_name, args.pvc_namespace, - version_size, args.pvc_storage_class) -else: - if not args.force: - raise RuntimeError('PVC already exists. Will not overwrite it unless forced.') - - # I don't really understand why capacity is a regular dict and not an object. Oh, well. - pvc_size = int(benji.helpers.kubernetes.parse_quantity(pvc.status.capacity['storage'])) - if pvc_size < version_size: - raise RuntimeError(f'Existing PVC is too small to hold version {args.version_uid} ({pvc_size} < {version_size}).') - elif pvc_size > version_size: - logger.warning(f'Existing PVC is {pvc_size - version_size} bytes bigger than version {args.version_uid}.') - -while True: - pvc = core_v1_api.read_namespaced_persistent_volume_claim(args.pvc_name, args.pvc_namespace) - if pvc.status.phase == 'Bound': - break - logger.info('Waiting for persistent volume creation.') - time.sleep(1) - -pv = core_v1_api.read_persistent_volume(pvc.spec.volume_name) - -pool, image = None, None - -# Taken from backup code, maybe refactor? -if hasattr(pv.spec, 'rbd') and hasattr(pv.spec.rbd, 'pool') and hasattr(pv.spec.rbd, 'image'): - # Native Kubernetes RBD PV - pool, image = pv.spec.rbd.pool, pv.spec.rbd.image -elif hasattr(pv.spec, 'flex_volume') and hasattr(pv.spec.flex_volume, 'options') and hasattr( - pv.spec.flex_volume, 'driver'): - # Rook Ceph PV - options = pv.spec.flex_volume.options - driver = pv.spec.flex_volume.driver - if driver.startswith('ceph.rook.io/') and options.get('pool') and options.get('image'): - pool, image = options['pool'], options['image'] -elif (hasattr(pv.spec, 'csi') - and hasattr(pv.spec.csi, 'driver') - and pv.spec.csi.driver == 'rook-ceph.rbd.csi.ceph.com' - and hasattr(pv.spec.csi, 'volume_handle') - and pv.spec.csi.volume_handle - and hasattr(pv.spec.csi, 'volume_attributes') - and pv.spec.csi.volume_attributes.get('pool')): - attributes = pv.spec.csi.volume_attributes - volume_handle_parts = pv.spec.csi.volume_handle.split('-') - if len(volume_handle_parts) >= 9: - image_prefix = attributes.get('volumeNamePrefix', 'csi-vol-') - image_suffix = '-'.join(volume_handle_parts[len(volume_handle_parts)-5:]) - pool, image = attributes['pool'], image_prefix + image_suffix - -if pool is None or image is None: - raise RuntimeError(f'Unable to determine PersistentVolume pool or image for {pv.metadata.name}') - -utils.subprocess_run([ - 'benji', '--log-level', settings.benji_log_level, 'restore', '--sparse', '--force', args.version_uid, - f'{args.ios_name}:{pool}/{image}', -]) -sys.exit(0) diff --git a/images/benji-k8s/bin/benji-versions-status b/images/benji-k8s/bin/benji-versions-status deleted file mode 100755 index 8b82caa2..00000000 --- a/images/benji-k8s/bin/benji-versions-status +++ /dev/null @@ -1,33 +0,0 @@ -#!/usr/bin/env python3 -import sys - -import benji.helpers.prometheus as prometheus -import benji.helpers.settings as settings -import benji.helpers.utils as utils - -utils.setup_logging() - -incomplete_versions = utils.subprocess_run([ - 'benji', - '--machine-output', - '--log-level', - settings.benji_log_level, - 'ls', - 'status == "incomplete" and date < "1 day ago"', -], - decode_json=True) - -invalid_versions = utils.subprocess_run([ - 'benji', - '--machine-output', - '--log-level', - settings.benji_log_level, - 'ls', - 'status == "invalid"', -], - decode_json=True) - -prometheus.older_incomplete_versions.set(len(incomplete_versions['versions'])) -prometheus.invalid_versions.set(len(invalid_versions['versions'])) -prometheus.push(prometheus.version_status_registry) -sys.exit(0) diff --git a/images/benji-k8s/k8s-tools/MANIFEST.in b/images/benji-k8s/k8s-tools/MANIFEST.in new file mode 100644 index 00000000..db566413 --- /dev/null +++ b/images/benji-k8s/k8s-tools/MANIFEST.in @@ -0,0 +1,2 @@ +include MANIFEST.in +recursive-include src *.py diff --git a/images/benji-k8s/k8s-tools/setup.py b/images/benji-k8s/k8s-tools/setup.py new file mode 100644 index 00000000..1d8f40bd --- /dev/null +++ b/images/benji-k8s/k8s-tools/setup.py @@ -0,0 +1,24 @@ +from setuptools import setup, find_packages + +setup(name='benji-k8s-tools', + version='0.1', + description='Small tools for using Benji with Kubernetes', + url='https://github.com/elemental-lf/benji', + author='Lars Fenneberg', + author_email='lf@elemental.net', + license='LGPG-3', + python_requires='~=3.6', + packages=find_packages('src'), + package_dir={ + '': 'src', + }, + install_requires=[ + 'benji', + ], + entry_points=""" + [console_scripts] + benji-backup-pvc = benji.k8s_tools.scripts.backup_pvc:main + benji-command = benji.k8s_tools.scripts.command:main + benji-restore-pvc = benji.k8s_tools.scripts.restore_pvc:main + benji-versions-status = benji.k8s_tools.scripts.versions_status:main + """) diff --git a/images/benji-k8s/k8s-tools/src/benji/__init__.py b/images/benji-k8s/k8s-tools/src/benji/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/images/benji-k8s/k8s-tools/src/benji/k8s_tools/__init__.py b/images/benji-k8s/k8s-tools/src/benji/k8s_tools/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/images/benji-k8s/k8s-tools/src/benji/k8s_tools/scripts/__init__.py b/images/benji-k8s/k8s-tools/src/benji/k8s_tools/scripts/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/images/benji-k8s/bin/benji-backup-pvc b/images/benji-k8s/k8s-tools/src/benji/k8s_tools/scripts/backup_pvc.py similarity index 71% rename from images/benji-k8s/bin/benji-backup-pvc rename to images/benji-k8s/k8s-tools/src/benji/k8s_tools/scripts/backup_pvc.py index a1210e40..147f9e2b 100644 --- a/images/benji-k8s/bin/benji-backup-pvc +++ b/images/benji-k8s/k8s-tools/src/benji/k8s_tools/scripts/backup_pvc.py @@ -250,114 +250,89 @@ def ceph_backup_post_error(sender: str, volume: str, pool: str, image: str, vers raise exception -# This arguments parser tries to mimic kubectl -parser = argparse.ArgumentParser(formatter_class=argparse.ArgumentDefaultsHelpFormatter, allow_abbrev=False) - -parser.add_argument('-n', - '--namespace', - metavar='namespace', - dest='namespace', - default=None, - help='Filter on namespace') -parser.add_argument('-l', - '--selector', - metavar='label-selector', - dest='labels', - action='append', - default=[], - help='Filter PVCs on label selector') -parser.add_argument('--field-selector', - metavar='field-selector', - dest='fields', - action='append', - default=[], - help='Filter PVCs on field selector') - -args = parser.parse_args() - -benji.helpers.kubernetes.load_config() -core_v1_api = kubernetes.client.CoreV1Api() - -labels = ','.join(args.labels) -fields = ','.join(args.fields) - -if args.namespace is not None: - logger.info(f'Backing up all PVCs in namespace {args.namespace}.') -else: - logger.info(f'Backing up all PVCs in all namespaces.') -if labels != '': - logger.info(f'Matching label(s) {labels}.') -if fields != '': - logger.info(f'Matching field(s) {fields}.') - -if args.namespace is not None: - pvcs = core_v1_api.list_namespaced_persistent_volume_claim(args.namespace, - watch=False, - label_selector=labels, - field_selector=fields).items -else: - pvcs = core_v1_api.list_persistent_volume_claim_for_all_namespaces(watch=False, - label_selector=labels, - field_selector=fields).items -if len(pvcs) == 0: - logger.info('Not matching PVCs found.') - sys.exit(0) +def main(): + # This arguments parser tries to mimic kubectl + parser = argparse.ArgumentParser(formatter_class=argparse.ArgumentDefaultsHelpFormatter, allow_abbrev=False) + + parser.add_argument('-n', + '--namespace', + metavar='namespace', + dest='namespace', + default=None, + help='Filter on namespace') + parser.add_argument('-l', + '--selector', + metavar='label-selector', + dest='labels', + action='append', + default=[], + help='Filter PVCs on label selector') + parser.add_argument('--field-selector', + metavar='field-selector', + dest='fields', + action='append', + default=[], + help='Filter PVCs on field selector') + + args = parser.parse_args() + + benji.helpers.kubernetes.load_config() + core_v1_api = kubernetes.client.CoreV1Api() + + labels = ','.join(args.labels) + fields = ','.join(args.fields) -for pvc in pvcs: - if not hasattr(pvc.spec, 'volume_name') or pvc.spec.volume_name in (None, ''): - continue - - pv = core_v1_api.read_persistent_volume(pvc.spec.volume_name) - - pool, image = None, None - - if hasattr(pv.spec, 'rbd') and hasattr(pv.spec.rbd, 'pool') and hasattr(pv.spec.rbd, 'image'): - # Native Kubernetes RBD PV - pool, image = pv.spec.rbd.pool, pv.spec.rbd.image - elif hasattr(pv.spec, 'flex_volume') and hasattr(pv.spec.flex_volume, 'options') and hasattr( - pv.spec.flex_volume, 'driver'): - # Rook Ceph PV - options = pv.spec.flex_volume.options - driver = pv.spec.flex_volume.driver - if driver.startswith('ceph.rook.io/') and options.get('pool') and options.get('image'): - pool, image = options['pool'], options['image'] - elif (hasattr(pv.spec, 'csi') - and hasattr(pv.spec.csi, 'driver') - and pv.spec.csi.driver in ('rook-ceph.rbd.csi.ceph.com', 'rbd.csi.ceph.com') - and hasattr(pv.spec.csi, 'volume_handle') - and pv.spec.csi.volume_handle - and hasattr(pv.spec.csi, 'volume_attributes') - and pv.spec.csi.volume_attributes.get('pool')): - attributes = pv.spec.csi.volume_attributes - volume_handle_parts = pv.spec.csi.volume_handle.split('-') - if len(volume_handle_parts) >= 9: - image_prefix = attributes.get('volumeNamePrefix', 'csi-vol-') - image_suffix = '-'.join(volume_handle_parts[len(volume_handle_parts)-5:]) - pool, image = attributes['pool'], image_prefix + image_suffix - - if pool is None or image is None: - continue - - volume = f'{pvc.metadata.namespace}/{pvc.metadata.name}' - # Limit the version_uid to 253 characters so that it is a compatible Kubernetes resource name. - version_uid = '{}-{}'.format(f'{pvc.metadata.namespace}-{pvc.metadata.name}' [:246], _random_string(6)) - - version_labels = { - 'benji-backup.me/instance': settings.benji_instance, - 'benji-backup.me/ceph-pool': pool, - 'benji-backup.me/ceph-rbd-image': image, - 'benji-backup.me/k8s-pvc-namespace': pvc.metadata.namespace, - 'benji-backup.me/k8s-pvc': pvc.metadata.name, - 'benji-backup.me/k8s-pv': pv.metadata.name - } - - context = {'pvc': pvc} - ceph.backup(volume=volume, - pool=pool, - image=image, - version_uid=version_uid, - version_labels=version_labels, - context=context) - -prometheus.push(prometheus.backup_registry) -sys.exit(0) + if args.namespace is not None: + logger.info(f'Backing up all PVCs in namespace {args.namespace}.') + else: + logger.info(f'Backing up all PVCs in all namespaces.') + if labels != '': + logger.info(f'Matching label(s) {labels}.') + if fields != '': + logger.info(f'Matching field(s) {fields}.') + + if args.namespace is not None: + pvcs = core_v1_api.list_namespaced_persistent_volume_claim(args.namespace, + watch=False, + label_selector=labels, + field_selector=fields).items + else: + pvcs = core_v1_api.list_persistent_volume_claim_for_all_namespaces(watch=False, + label_selector=labels, + field_selector=fields).items + if len(pvcs) == 0: + logger.info('Not matching PVCs found.') + sys.exit(0) + + for pvc in pvcs: + if not hasattr(pvc.spec, 'volume_name') or pvc.spec.volume_name in (None, ''): + continue + + pv = core_v1_api.read_persistent_volume(pvc.spec.volume_name) + pool, image = benji.helpers.kubernetes.determine_rbd_image_from_pv(pv) + if pool is None or image is None: + continue + + volume = f'{pvc.metadata.namespace}/{pvc.metadata.name}' + # Limit the version_uid to 253 characters so that it is a compatible Kubernetes resource name. + version_uid = '{}-{}'.format(f'{pvc.metadata.namespace}-{pvc.metadata.name}'[:246], _random_string(6)) + + version_labels = { + 'benji-backup.me/instance': settings.benji_instance, + 'benji-backup.me/ceph-pool': pool, + 'benji-backup.me/ceph-rbd-image': image, + 'benji-backup.me/k8s-pvc-namespace': pvc.metadata.namespace, + 'benji-backup.me/k8s-pvc': pvc.metadata.name, + 'benji-backup.me/k8s-pv': pv.metadata.name + } + + context = {'pvc': pvc} + ceph.backup(volume=volume, + pool=pool, + image=image, + version_uid=version_uid, + version_labels=version_labels, + context=context) + + prometheus.push(prometheus.backup_registry) + sys.exit(0) diff --git a/images/benji-k8s/k8s-tools/src/benji/k8s_tools/scripts/command.py b/images/benji-k8s/k8s-tools/src/benji/k8s_tools/scripts/command.py new file mode 100644 index 00000000..c72534e8 --- /dev/null +++ b/images/benji-k8s/k8s-tools/src/benji/k8s_tools/scripts/command.py @@ -0,0 +1,33 @@ +#!/usr/bin/env python3 +import sys +import time +from typing import Optional + +import benji.helpers.prometheus as prometheus +import benji.helpers.settings as settings +import benji.helpers.utils as utils + +utils.setup_logging() + + +def main(): + command = ' '.join(sys.argv[1:]) + start_time = time.time() + + prometheus.command_start_time.labels(command=command).set(start_time) + try: + utils.subprocess_run(['benji', '--log-level', settings.benji_log_level] + sys.argv[1:]) + except Exception as exception: + prometheus.command_status_failed.labels(command=command).set(1) + completion_time = time.time() + prometheus.command_completion_time.labels(command=command).set(completion_time) + prometheus.command_runtime_seconds.labels(command=command).set(completion_time - start_time) + prometheus.push(prometheus.command_registry) + raise exception + else: + prometheus.command_status_succeeded.labels(command=command).set(1) + completion_time = time.time() + prometheus.command_completion_time.labels(command=command).set(completion_time) + prometheus.command_runtime_seconds.labels(command=command).set(completion_time - start_time) + prometheus.push(prometheus.command_registry) + sys.exit(0) diff --git a/images/benji-k8s/k8s-tools/src/benji/k8s_tools/scripts/restore_pvc.py b/images/benji-k8s/k8s-tools/src/benji/k8s_tools/scripts/restore_pvc.py new file mode 100644 index 00000000..45344608 --- /dev/null +++ b/images/benji-k8s/k8s-tools/src/benji/k8s_tools/scripts/restore_pvc.py @@ -0,0 +1,106 @@ +#!/usr/bin/env python3 +import argparse +import logging +import sys +import time + +import kubernetes +from kubernetes.client.rest import ApiException + +import benji.helpers.kubernetes +import benji.helpers.settings as settings +import benji.helpers.utils as utils + +utils.setup_logging() +logger = logging.getLogger() + + +def main(): + parser = argparse.ArgumentParser(formatter_class=argparse.ArgumentDefaultsHelpFormatter, allow_abbrev=False) + + parser.add_argument('-f', + '--force', + dest='force', + action='store_true', + default=False, + help='Overwrite content of existing persistent volumes') + parser.add_argument('--pvc-storage-class', + metavar='pvc_storage_class', + dest='pvc_storage_class', + default='rbd', + help='PVC storage class (only takes effect if the PVC does not exist yet)') + parser.add_argument('--restore-url-template', + metavar='restore_url_template', + dest='restore_url_template', + help='Template to use for constructing URL for benji restore call', + default='rbd:{pool}/{image}') + parser.add_argument(metavar='version_uid', dest='version_uid', help='Version uid') + parser.add_argument(metavar='pvc_namespace', dest='pvc_namespace', help='PVC namespace') + parser.add_argument(metavar='pvc_name', dest='pvc_name', help='PVC name') + + args = parser.parse_args() + + benji.helpers.kubernetes.load_config() + + logger.info(f'Restoring version {args.version_uid} to PVC {args.pvc_namespace}/{args.pvc_name}.') + + benji_ls = utils.subprocess_run( + ['benji', '--machine-output', '--log-level', settings.benji_log_level, 'ls', f'uid == "{args.version_uid}"'], + decode_json=True) + assert isinstance(benji_ls, dict) + assert 'versions' in benji_ls + assert isinstance(benji_ls['versions'], list) + + if len(benji_ls['versions']) == 0: + raise RuntimeError(f'Size of {args.version_uid} could not be determined.') + + assert isinstance(benji_ls['versions'][0], dict) + assert isinstance(benji_ls['versions'][0]['size'], int) + version_size = benji_ls['versions'][0]['size'] + + # This assumes that the Kubernetes client has already been initialized + core_v1_api = kubernetes.client.CoreV1Api() + pvc = None + try: + pvc = core_v1_api.read_namespaced_persistent_volume_claim(args.pvc_name, args.pvc_namespace) + except ApiException as exception: + if exception.status != 404: + raise RuntimeError(f'Unexpected Kubernetes API exception: {str(exception)}') + + if pvc is None: + pvc = benji.helpers.kubernetes.create_pvc(args.pvc_name, args.pvc_namespace, version_size, + args.pvc_storage_class) + else: + if not args.force: + raise RuntimeError('PVC already exists. Will not overwrite it unless forced.') + + # I don't really understand why capacity is a regular dict and not an object. Oh, well. + pvc_size = int(benji.helpers.kubernetes.parse_quantity(pvc.status.capacity['storage'])) + if pvc_size < version_size: + raise RuntimeError(f'Existing PVC is too small to hold version {args.version_uid} ({pvc_size} < {version_size}).') + elif pvc_size > version_size: + logger.warning(f'Existing PVC is {pvc_size - version_size} bytes bigger than version {args.version_uid}.') + + while True: + pvc = core_v1_api.read_namespaced_persistent_volume_claim(args.pvc_name, args.pvc_namespace) + if pvc.status.phase == 'Bound': + break + logger.info('Waiting for persistent volume creation.') + time.sleep(1) + + pv = core_v1_api.read_persistent_volume(pvc.spec.volume_name) + pool, image = benji.helpers.kubernetes.determine_rbd_image_from_pv(pv) + if pool is None or image is None: + raise RuntimeError(f'Unable to determine PersistentVolume pool or image for {pv.metadata.name}') + + utils.subprocess_run([ + 'benji', + '--log-level', + settings.benji_log_level, + 'restore', + '--sparse', + '--force', + args.version_uid, + args.restore_url_template.format(pool=pool, image=image), + ]) + sys.exit(0) diff --git a/images/benji-k8s/k8s-tools/src/benji/k8s_tools/scripts/versions_status.py b/images/benji-k8s/k8s-tools/src/benji/k8s_tools/scripts/versions_status.py new file mode 100644 index 00000000..5b4f0368 --- /dev/null +++ b/images/benji-k8s/k8s-tools/src/benji/k8s_tools/scripts/versions_status.py @@ -0,0 +1,35 @@ +#!/usr/bin/env python3 +import sys + +import benji.helpers.prometheus as prometheus +import benji.helpers.settings as settings +import benji.helpers.utils as utils + +utils.setup_logging() + + +def main(): + incomplete_versions = utils.subprocess_run([ + 'benji', + '--machine-output', + '--log-level', + settings.benji_log_level, + 'ls', + 'status == "incomplete" and date < "1 day ago"', + ], + decode_json=True) + + invalid_versions = utils.subprocess_run([ + 'benji', + '--machine-output', + '--log-level', + settings.benji_log_level, + 'ls', + 'status == "invalid"', + ], + decode_json=True) + + prometheus.older_incomplete_versions.set(len(incomplete_versions['versions'])) + prometheus.invalid_versions.set(len(invalid_versions['versions'])) + prometheus.push(prometheus.version_status_registry) + sys.exit(0) diff --git a/images/benji/Dockerfile b/images/benji/Dockerfile index dd8e277d..e858f209 100644 --- a/images/benji/Dockerfile +++ b/images/benji/Dockerfile @@ -15,7 +15,7 @@ RUN rpm --import 'https://download.ceph.com/keys/release.asc' && \ python36-devel python36-pip python36-libs python36-setuptools \ python36-rbd python36-rados -ADD . /benji-source/ +COPY . /benji-source/ RUN python3.6 -m venv --system-site-packages $VENV_DIR && \ . $VENV_DIR/bin/activate && \ diff --git a/src/benji/helpers/kubernetes.py b/src/benji/helpers/kubernetes.py index 7ff2a887..ddfbed29 100644 --- a/src/benji/helpers/kubernetes.py +++ b/src/benji/helpers/kubernetes.py @@ -5,7 +5,7 @@ import time import uuid from subprocess import TimeoutExpired, CalledProcessError -from typing import List, Union, Tuple +from typing import List, Union, Tuple, Optional import kubernetes from kubernetes.stream import stream @@ -49,7 +49,11 @@ def service_account_namespace() -> str: # connect_post_namespaced_pod_exec. The examples from the kubernetes client use connect_get_namespaced_pod_exec instead. # There shouldn't be any differences in functionality but the settings in the RBAC role are different (create vs. get) # which is why we follow the kubectl implementation here. -def pod_exec(args: List[str], *, name: str, namespace: str, container: str = None, +def pod_exec(args: List[str], + *, + name: str, + namespace: str, + container: str = None, timeout: float = float("inf")) -> Tuple[str, str]: core_v1_api = kubernetes.client.CoreV1Api() logger.debug('Running command in pod {}/{}: {}.'.format(namespace, name, ' '.join(args))) @@ -171,8 +175,8 @@ def create_pvc_event(*, type: str, reason: str, message: str, pvc_namespace: str return core_v1_api.create_namespaced_event(namespace=pvc_namespace, body=event) -def create_pvc(pvc_name: str, pvc_namespace: int, - pvc_size: str, pvc_storage_class: str) -> kubernetes.client.models.v1_persistent_volume_claim.V1PersistentVolumeClaim: +def create_pvc(pvc_name: str, pvc_namespace: int, pvc_size: str, + pvc_storage_class: str) -> kubernetes.client.models.v1_persistent_volume_claim.V1PersistentVolumeClaim: pvc = { 'kind': 'PersistentVolumeClaim', 'apiVersion': 'v1', @@ -195,6 +199,34 @@ def create_pvc(pvc_name: str, pvc_namespace: int, return core_v1_api.create_namespaced_persistent_volume_claim(namespace=pvc_namespace, body=pvc) +def determine_rbd_image_from_pv( + pv: kubernetes.client.models.v1_persistent_volume.V1PersistentVolume) -> Tuple[Optional[str], Optional[str]]: + pool, image = None, None + + if hasattr(pv.spec, 'rbd') and hasattr(pv.spec.rbd, 'pool') and hasattr(pv.spec.rbd, 'image'): + # Native Kubernetes RBD PV + pool, image = pv.spec.rbd.pool, pv.spec.rbd.image + elif hasattr(pv.spec, 'flex_volume') and hasattr(pv.spec.flex_volume, 'options') and hasattr( + pv.spec.flex_volume, 'driver'): + # Rook Ceph PV + options = pv.spec.flex_volume.options + driver = pv.spec.flex_volume.driver + if driver.startswith('ceph.rook.io/') and options.get('pool') and options.get('image'): + pool, image = options['pool'], options['image'] + elif (hasattr(pv.spec, 'csi') and hasattr(pv.spec.csi, 'driver') and + pv.spec.csi.driver in ('rook-ceph.rbd.csi.ceph.com', 'rbd.csi.ceph.com') and + hasattr(pv.spec.csi, 'volume_handle') and pv.spec.csi.volume_handle and + hasattr(pv.spec.csi, 'volume_attributes') and pv.spec.csi.volume_attributes.get('pool')): + attributes = pv.spec.csi.volume_attributes + volume_handle_parts = pv.spec.csi.volume_handle.split('-') + if len(volume_handle_parts) >= 9: + image_prefix = attributes.get('volumeNamePrefix', 'csi-vol-') + image_suffix = '-'.join(volume_handle_parts[len(volume_handle_parts) - 5:]) + pool, image = attributes['pool'], image_prefix + image_suffix + + return pool, image + + # This is taken from https://github.com/kubernetes-client/python/pull/855 with minimal changes. # # Copyright 2019 The Kubernetes Authors. From a7289862bdea34f1e8ab34c688bae0bb4922b7f9 Mon Sep 17 00:00:00 2001 From: Lars Fenneberg Date: Wed, 3 Mar 2021 15:03:46 +0100 Subject: [PATCH 6/9] Add proper magic for namespace packages --- images/benji-k8s/k8s-tools/src/benji/__init__.py | 1 + src/benji/__init__.py | 2 ++ 2 files changed, 3 insertions(+) diff --git a/images/benji-k8s/k8s-tools/src/benji/__init__.py b/images/benji-k8s/k8s-tools/src/benji/__init__.py index e69de29b..69e3be50 100644 --- a/images/benji-k8s/k8s-tools/src/benji/__init__.py +++ b/images/benji-k8s/k8s-tools/src/benji/__init__.py @@ -0,0 +1 @@ +__path__ = __import__('pkgutil').extend_path(__path__, __name__) diff --git a/src/benji/__init__.py b/src/benji/__init__.py index a9552ac8..49075c96 100644 --- a/src/benji/__init__.py +++ b/src/benji/__init__.py @@ -1,3 +1,5 @@ +__path__ = __import__('pkgutil').extend_path(__path__, __name__) + __all__ = ['__version__'] from ._version import __version__ del _version # remove to avoid confusion with __version__ From 9256653009d2230d3c2a176798f1489a3d799c82 Mon Sep 17 00:00:00 2001 From: Lars Fenneberg Date: Wed, 3 Mar 2021 15:04:36 +0100 Subject: [PATCH 7/9] Refactor: Move K8s helper functions to benji-k8s-tools package --- images/benji-k8s/k8s-tools/setup.py | 4 +- .../src/benji/k8s_tools}/kubernetes.py | 0 .../src/benji/k8s_tools/scripts/backup_pvc.py | 70 +++++++++---------- .../benji/k8s_tools/scripts/restore_pvc.py | 12 ++-- setup.py | 2 +- 5 files changed, 43 insertions(+), 45 deletions(-) rename {src/benji/helpers => images/benji-k8s/k8s-tools/src/benji/k8s_tools}/kubernetes.py (100%) diff --git a/images/benji-k8s/k8s-tools/setup.py b/images/benji-k8s/k8s-tools/setup.py index 1d8f40bd..6223aafa 100644 --- a/images/benji-k8s/k8s-tools/setup.py +++ b/images/benji-k8s/k8s-tools/setup.py @@ -12,9 +12,7 @@ package_dir={ '': 'src', }, - install_requires=[ - 'benji', - ], + install_requires=['benji', 'kubernetes>=10.0.0,<11'], entry_points=""" [console_scripts] benji-backup-pvc = benji.k8s_tools.scripts.backup_pvc:main diff --git a/src/benji/helpers/kubernetes.py b/images/benji-k8s/k8s-tools/src/benji/k8s_tools/kubernetes.py similarity index 100% rename from src/benji/helpers/kubernetes.py rename to images/benji-k8s/k8s-tools/src/benji/k8s_tools/kubernetes.py diff --git a/images/benji-k8s/k8s-tools/src/benji/k8s_tools/scripts/backup_pvc.py b/images/benji-k8s/k8s-tools/src/benji/k8s_tools/scripts/backup_pvc.py index 147f9e2b..61bf0768 100644 --- a/images/benji-k8s/k8s-tools/src/benji/k8s_tools/scripts/backup_pvc.py +++ b/images/benji-k8s/k8s-tools/src/benji/k8s_tools/scripts/backup_pvc.py @@ -11,10 +11,10 @@ import kubernetes.stream import benji.helpers.ceph as ceph -import benji.helpers.kubernetes import benji.helpers.prometheus as prometheus import benji.helpers.settings as settings import benji.helpers.utils as utils +import benji.k8s_tools.kubernetes FSFREEZE_TIMEOUT = 15 FSFREEZE_UNFREEZE_TRIES = (0, 1, 1, 1, 15, 30) @@ -40,7 +40,7 @@ def _determine_fsfreeze_info(pvc_namespace: str, pvc_name: str, core_v1_api = kubernetes.client.CoreV1Api() pvc = core_v1_api.read_namespaced_persistent_volume_claim(pvc_name, pvc_namespace) - service_account_namespace = benji.helpers.kubernetes.service_account_namespace() + service_account_namespace = benji.k8s_tools.kubernetes.service_account_namespace() if hasattr(pvc.metadata, 'annotations') and FSFREEZE_ANNOTATION in pvc.metadata.annotations and pvc.metadata.annotations[FSFREEZE_ANNOTATION] == 'yes': pods = core_v1_api.list_namespaced_pod(service_account_namespace, watch=False).items @@ -59,7 +59,7 @@ def _determine_fsfreeze_info(pvc_namespace: str, pvc_name: str, break if pv_fsfreeze: - pods = core_v1_api.list_namespaced_pod(benji.helpers.kubernetes.service_account_namespace(), + pods = core_v1_api.list_namespaced_pod(benji.k8s_tools.kubernetes.service_account_namespace(), label_selector=FSFREEZE_POD_LABEL_SELECTOR).items if not pods: @@ -105,21 +105,21 @@ def ceph_snapshot_create_pre(sender: str, volume: str, pool: str, image: str, sn logger.info(f'Freezing filesystem {pv_mount_point} on host {pv_host_ip} (pod {pv_fsfreeze_pod}).') - service_account_namespace = benji.helpers.kubernetes.service_account_namespace() + service_account_namespace = benji.k8s_tools.kubernetes.service_account_namespace() try: - benji.helpers.kubernetes.pod_exec(['fsfreeze', '--freeze', pv_mount_point], - name=pv_fsfreeze_pod, - namespace=service_account_namespace, - container=FSFREEZE_CONTAINER_NAME, - timeout=FSFREEZE_TIMEOUT) + benji.k8s_tools.kubernetes.pod_exec(['fsfreeze', '--freeze', pv_mount_point], + name=pv_fsfreeze_pod, + namespace=service_account_namespace, + container=FSFREEZE_CONTAINER_NAME, + timeout=FSFREEZE_TIMEOUT) except Exception as exception: # Try to unfreeze in any case try: - benji.helpers.kubernetes.pod_exec(['fsfreeze', '--unfreeze', pv_mount_point], - name=pv_fsfreeze_pod, - namespace=service_account_namespace, - container=FSFREEZE_CONTAINER_NAME, - timeout=FSFREEZE_TIMEOUT) + benji.k8s_tools.kubernetes.pod_exec(['fsfreeze', '--unfreeze', pv_mount_point], + name=pv_fsfreeze_pod, + namespace=service_account_namespace, + container=FSFREEZE_CONTAINER_NAME, + timeout=FSFREEZE_TIMEOUT) except Exception as exception_2: raise exception_2 from exception else: @@ -142,17 +142,17 @@ def ceph_snapshot_create_post_success(sender: str, volume: str, pool: str, image logger.info(f'Unfreezing filesystem {pv_mount_point} on host {pv_host_ip}.') - service_account_namespace = benji.helpers.kubernetes.service_account_namespace() + service_account_namespace = benji.k8s_tools.kubernetes.service_account_namespace() for delay in FSFREEZE_UNFREEZE_TRIES: if delay > 0: time.sleep(delay) try: - benji.helpers.kubernetes.pod_exec(['fsfreeze', '--unfreeze', pv_mount_point], - name=pv_fsfreeze_pod, - namespace=service_account_namespace, - container=FSFREEZE_CONTAINER_NAME, - timeout=FSFREEZE_TIMEOUT) + benji.k8s_tools.kubernetes.pod_exec(['fsfreeze', '--unfreeze', pv_mount_point], + name=pv_fsfreeze_pod, + namespace=service_account_namespace, + container=FSFREEZE_CONTAINER_NAME, + timeout=FSFREEZE_TIMEOUT) except Exception: pass else: @@ -185,12 +185,12 @@ def _k8s_create_pvc_event(type: str, reason: str, message: str, context: Dict[st pvc_uid = context['pvc'].metadata.uid try: - benji.helpers.kubernetes.create_pvc_event(type=type, - reason=reason, - message=message, - pvc_namespace=pvc_namespace, - pvc_name=pvc_name, - pvc_uid=pvc_uid) + benji.k8s_tools.kubernetes.create_pvc_event(type=type, + reason=reason, + message=message, + pvc_namespace=pvc_namespace, + pvc_name=pvc_name, + pvc_uid=pvc_uid) except Exception as exception: logger.error(f'Creating Kubernetes event for {pvc_namespace}/{pvc_name} failed with a {exception.__class__.__name__} exception: {str(exception)}') pass @@ -213,7 +213,7 @@ def ceph_backup_post_success(sender: str, volume: str, pool: str, image: str, ve prometheus.backup_status_succeeded.labels(volume=volume).set(1) try: - benji.helpers.kubernetes.create_pvc_event( + benji.k8s_tools.kubernetes.create_pvc_event( type='Normal', reason='SuccessfulBackup', message=f'Backup to {version["uid"]} completed successfully (took {completion_time - start_time:.0f} seconds).', @@ -240,12 +240,12 @@ def ceph_backup_post_error(sender: str, volume: str, pool: str, image: str, vers prometheus.backup_runtime_seconds.labels(volume=volume).set(completion_time - start_time) prometheus.backup_status_failed.labels(volume=volume).set(1) - benji.helpers.kubernetes.create_pvc_event(type='Warning', - reason='FailedBackup', - message=f'Backup failed: {exception.__class__.__name__} {str(exception)}', - pvc_namespace=pvc_namespace, - pvc_name=pvc_name, - pvc_uid=pvc_uid) + benji.k8s_tools.kubernetes.create_pvc_event(type='Warning', + reason='FailedBackup', + message=f'Backup failed: {exception.__class__.__name__} {str(exception)}', + pvc_namespace=pvc_namespace, + pvc_name=pvc_name, + pvc_uid=pvc_uid) raise exception @@ -276,7 +276,7 @@ def main(): args = parser.parse_args() - benji.helpers.kubernetes.load_config() + benji.k8s_tools.kubernetes.load_config() core_v1_api = kubernetes.client.CoreV1Api() labels = ','.join(args.labels) @@ -309,7 +309,7 @@ def main(): continue pv = core_v1_api.read_persistent_volume(pvc.spec.volume_name) - pool, image = benji.helpers.kubernetes.determine_rbd_image_from_pv(pv) + pool, image = benji.k8s_tools.kubernetes.determine_rbd_image_from_pv(pv) if pool is None or image is None: continue diff --git a/images/benji-k8s/k8s-tools/src/benji/k8s_tools/scripts/restore_pvc.py b/images/benji-k8s/k8s-tools/src/benji/k8s_tools/scripts/restore_pvc.py index 45344608..fc5bde60 100644 --- a/images/benji-k8s/k8s-tools/src/benji/k8s_tools/scripts/restore_pvc.py +++ b/images/benji-k8s/k8s-tools/src/benji/k8s_tools/scripts/restore_pvc.py @@ -7,9 +7,9 @@ import kubernetes from kubernetes.client.rest import ApiException -import benji.helpers.kubernetes import benji.helpers.settings as settings import benji.helpers.utils as utils +import benji.k8s_tools.kubernetes utils.setup_logging() logger = logging.getLogger() @@ -40,7 +40,7 @@ def main(): args = parser.parse_args() - benji.helpers.kubernetes.load_config() + benji.k8s_tools.kubernetes.load_config() logger.info(f'Restoring version {args.version_uid} to PVC {args.pvc_namespace}/{args.pvc_name}.') @@ -68,14 +68,14 @@ def main(): raise RuntimeError(f'Unexpected Kubernetes API exception: {str(exception)}') if pvc is None: - pvc = benji.helpers.kubernetes.create_pvc(args.pvc_name, args.pvc_namespace, version_size, - args.pvc_storage_class) + pvc = benji.k8s_tools.kubernetes.create_pvc(args.pvc_name, args.pvc_namespace, version_size, + args.pvc_storage_class) else: if not args.force: raise RuntimeError('PVC already exists. Will not overwrite it unless forced.') # I don't really understand why capacity is a regular dict and not an object. Oh, well. - pvc_size = int(benji.helpers.kubernetes.parse_quantity(pvc.status.capacity['storage'])) + pvc_size = int(benji.k8s_tools.kubernetes.parse_quantity(pvc.status.capacity['storage'])) if pvc_size < version_size: raise RuntimeError(f'Existing PVC is too small to hold version {args.version_uid} ({pvc_size} < {version_size}).') elif pvc_size > version_size: @@ -89,7 +89,7 @@ def main(): time.sleep(1) pv = core_v1_api.read_persistent_volume(pvc.spec.volume_name) - pool, image = benji.helpers.kubernetes.determine_rbd_image_from_pv(pv) + pool, image = benji.k8s_tools.kubernetes.determine_rbd_image_from_pv(pv) if pool is None or image is None: raise RuntimeError(f'Unable to determine PersistentVolume pool or image for {pv.metadata.name}') diff --git a/setup.py b/setup.py index 4b9dba56..fa61f34c 100644 --- a/setup.py +++ b/setup.py @@ -75,7 +75,7 @@ def get_version_and_cmdclass(package_path): #'rbd': ['rados', 'rbd'], 'dev': ['parameterized', 'wheel', 'yapf', 'mypy'], 'doc': ['sphinx', 'sphinx_rtd_theme', 'sphinxcontrib-programoutput'], - 'helpers': ['blinker>=1.4,<2', 'prometheus_client>=0.7.0,<1', 'kubernetes>=10.0.0,<11'], + 'helpers': ['blinker>=1.4,<2', 'prometheus_client>=0.7.0,<1'], 'rest-api': ['bottle>=0.12.16,<0.13.0', 'gunicorn>=19.9.0,<20', 'webargs>=5.3.1,<6', 'requests>=2.22.0,<3'], }, python_requires='~=3.6', From 536b35625fd9771505f5ee0afde2a102365c0224 Mon Sep 17 00:00:00 2001 From: Lars Fenneberg Date: Wed, 3 Mar 2021 15:21:46 +0100 Subject: [PATCH 8/9] Fix MANIFEST.in to include new files --- MANIFEST.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/MANIFEST.in b/MANIFEST.in index 6c26614e..cc0f9541 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -2,7 +2,7 @@ include MANIFEST.in README.rst TODO.rst LICENSE.txt .dockerignore include .style.yapf alembic recursive-include docs *.rst *.html Makefile *.gif *.png *.cast *.js *.css recursive-include src *.py *.ini *.yaml -recursive-include images Dockerfile *.sh +recursive-include images Dockerfile ceph.repo bashrc *.sh *.in *.py recursive-include scripts *.sh recursive-include charts *.yaml *.tpl .helmignore recursive-include tests Makefile *.yaml From d4914033a30a2f2ff18cc2c81d86f332319e776e Mon Sep 17 00:00:00 2001 From: Lars Fenneberg Date: Wed, 3 Mar 2021 15:22:14 +0100 Subject: [PATCH 9/9] Upgrade pip and setuptools in container image Fixes a problem with a dependency requiring a newer setuptools version. --- images/benji/Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/images/benji/Dockerfile b/images/benji/Dockerfile index e858f209..52b6728a 100644 --- a/images/benji/Dockerfile +++ b/images/benji/Dockerfile @@ -19,6 +19,7 @@ COPY . /benji-source/ RUN python3.6 -m venv --system-site-packages $VENV_DIR && \ . $VENV_DIR/bin/activate && \ + pip install --upgrade pip setuptools && \ pip install git+https://github.com/elemental-lf/libiscsi-python && \ pip install '/benji-source/[compression,s3,b2,helpers]'