Add backup/restore functionality

Can be used for migrating OSBS instances.

docs/backups.md

@@ -1,6 +1,32 @@
 # OSBS Backups
 
-## Creating Backups
+## Using osbs-client
+
+OSBS client has `backup-builder` and `restore-builder` subcommands that automate most of what needs to be done.
+
+### backup-builder
+
+To backup OSBS builder, simply run (you may have to add authentication and instance/namespace selection options if needed):
+
+    osbs backup-builder
+
+The command creates backup file named `osbs-backup-<instance>-<namespace>-<timestamp>.tar.bz2`. You can use the `--filename` argument to override the file name or write the backup to standard output.
+
+Please note that you need to be able to create/delete `resourcequotas` on the builder in order to prevent new builds from being created while backup is in progress, and read permission on `builds`, `buildconfigs` and `imagestreams`.
+
+### restore-builder
+
+Restoring builder data from a backup is as simple as:
+
+    osbs restore-builder <osbs-backup-file>
+
+The backup is read from standard input if you use `-` as a file name. You need the permission to create/delete `resourcequotas`, `builds`, `buildconfigs` and `imagestreams`.
+
+It is recommended to perform restore on freshly installed OpenShift with no data, otherwise you'll end up with mix of original and restored data, or an error in case some resource that you want to restore has the same name as one that is already present. You can use the `--continue-on-error` flag if you want to ignore such name clashes (and other errors) and import only the resources that do not raise an error.
+
+## Manually
+
+### Creating Backups
 
 Creating OSBS backups is very simple. To backup all relevant data, one needs to:
 
@@ -13,7 +39,7 @@ Creating OSBS backups is very simple. To backup all relevant data, one needs to:
 
 Note: pausing builds is recommended, since builds can create new ImageStream objects during their execution and the backup data could theoretically get inconsistent - e.g. we could get a BuildConfig without an ImageStream for the image it creates or vice versa, depending on the order of `oc export` commands.
 
-## Restoring from Backups
+### Restoring from Backups
 
 The assumption is, that we're importing data into a fresh OpenShift instance that is not yet accepting builds. To restore the data, run:
 

osbs/api.py

@@ -539,3 +539,40 @@ def resume_builds(self, namespace=DEFAULT_NAMESPACE):
 
         name = quota_json['metadata']['name']
         self.os.delete_resource_quota(name, namespace=namespace)
+
+    # implements subset of OpenShift's export logic in pkg/cmd/cli/cmd/exporter.go
+    @staticmethod
+    def _prepare_resource(resource_type, resource):
+        utils.graceful_chain_del(resource, 'metadata', 'resourceVersion')
+
+        if resource_type == 'buildconfigs':
+            utils.graceful_chain_del(resource, 'status', 'lastVersion')
+
+            triggers = utils.graceful_chain_get(resource, 'spec', 'triggers') or ()
+            for t in triggers:
+                utils.graceful_chain_del(t, 'imageChange', 'lastTrigerredImageID')
+
+    @osbsapi
+    def dump_resource(self, resource_type, namespace=DEFAULT_NAMESPACE):
+        return self.os.dump_resource(resource_type, namespace=namespace).json()
+
+    @osbsapi
+    def restore_resource(self, resource_type, resources, continue_on_error=False,
+                         namespace=DEFAULT_NAMESPACE):
+        nfailed = 0
+        for r in resources["items"]:
+            name = utils.graceful_chain_get(r, 'metadata', 'name') or '(no name)'
+            logger.debug("restoring %s/%s", resource_type, name)
+            try:
+                self._prepare_resource(resource_type, r)
+                self.os.restore_resource(resource_type, r, namespace=namespace)
+            except Exception:
+                if continue_on_error:
+                    logger.exception("failed to restore %s/%s", resource_type, name)
+                    nfailed += 1
+                else:
+                    raise
+
+        if continue_on_error:
+            ntotal = len(resources["items"])
+            logger.info("restored %s/%s %s", ntotal - nfailed, ntotal, resource_type)

osbs/cli/main.py

@@ -12,17 +12,20 @@
 import logging
 
 from os import uname
+import codecs
+import time
+import os.path
 import sys
 import argparse
 from osbs import set_logging
 from osbs.api import OSBS
 from osbs.cli.render import TablePrinter
 from osbs.conf import Configuration
 from osbs.constants import (DEFAULT_CONFIGURATION_FILE, DEFAULT_CONFIGURATION_SECTION,
-                            CLI_LIST_BUILDS_DEFAULT_COLS, PY3)
+                            CLI_LIST_BUILDS_DEFAULT_COLS, PY3, BACKUP_RESOURCES)
 from osbs.exceptions import OsbsNetworkException, OsbsException, OsbsAuthException, OsbsResponseException
 from osbs.cli.capture import setup_json_capture
-from osbs.utils import strip_registry_from_image
+from osbs.utils import strip_registry_from_image, paused_builds, TarReader, TarWriter
 
 logger = logging.getLogger('osbs')
 
@@ -292,6 +295,50 @@ def cmd_resume_builds(args, osbs):
     osbs.resume_builds(namespace=args.namespace)
 
 
+def cmd_backup(args, osbs):
+    dirname = time.strftime("osbs-backup-{0}-{1}-%Y-%m-%d-%H%M%S"
+                            .format(args.instance, args.namespace))
+    if args.filename == '-':
+        outfile = sys.stdout.buffer if PY3 else sys.stdout
+    elif args.filename:
+        outfile = args.filename
+    else:
+        outfile = dirname + ".tar.bz2"
+
+    with paused_builds(osbs, args.namespace):
+        with TarWriter(outfile, dirname) as t:
+            for resource_type in BACKUP_RESOURCES:
+                logger.info("dumping %s" % resource_type)
+                resources = osbs.dump_resource(resource_type, namespace=args.namespace)
+                t.write_file(resource_type + ".json", json.dumps(resources).encode('ascii'))
+
+    if not hasattr(outfile, "write"):
+        logger.info("backup archive created: %s", outfile)
+
+
+def cmd_restore(args, osbs):
+    if args.BACKUP_ARCHIVE == '-':
+        infile = sys.stdin.buffer if PY3 else sys.stdin
+    else:
+        infile = args.BACKUP_ARCHIVE
+    asciireader = codecs.getreader('ascii')
+
+    with paused_builds(osbs, args.namespace):
+        for f in TarReader(infile):
+            resource_type = os.path.basename(f.filename).split('.')[0]
+            if resource_type not in BACKUP_RESOURCES:
+                logger.warning("Unknown resource type for %s, skipping", f.filename)
+                continue
+
+            logger.info("restoring %s" % resource_type)
+            osbs.restore_resource(resource_type, json.load(asciireader(f.fileobj)),
+                                                 continue_on_error=args.continue_on_error,
+                                                 namespace=args.namespace)
+            f.fileobj.close()
+
+    logger.info("backup recovery complete!")
+
+
 def str_on_2_unicode_on_3(s):
     """
     argparse is way too awesome when doing repr() on choices when printing usage
@@ -423,6 +470,20 @@ def cli():
                                           description='allow builds to be scheduled again after pause-builds')
     resume_builds.set_defaults(func=cmd_resume_builds)
 
+    backup_builder = subparsers.add_parser(str_on_2_unicode_on_3('backup-builder'),
+                                           help='dump builder data (admin)',
+                                           description='create backup of all OSBS data')
+    backup_builder.add_argument("-f", "--filename", help="name of the resulting tar.bz2 file (use - for stdout)")
+    backup_builder.set_defaults(func=cmd_backup)
+
+    restore_builder = subparsers.add_parser(str_on_2_unicode_on_3('restore-builder'),
+                                            help='restore builder data (admin)',
+                                            description='restore OSBS data from backup')
+    restore_builder.add_argument("BACKUP_ARCHIVE", help="name of the tar.bz2 archive to restore (use - for stdin)")
+    restore_builder.add_argument("--continue-on-error", action='store_true',
+                                 help="don't stop when restoring a resource fails")
+    restore_builder.set_defaults(func=cmd_restore)
+
     parser.add_argument("--openshift-uri", action='store', metavar="URL",
                         help="openshift URL to remote API")
     parser.add_argument("--registry-uri", action='store', metavar="URL",

osbs/constants.py

@@ -53,4 +53,7 @@
 # Where will secrets be mounted?
 SECRETS_PATH = "/var/run/secrets/atomic-reactor"
 
+# Backup/restore
+BACKUP_RESOURCES = ('buildconfigs', 'imagestreams', 'builds',)
+
 CLI_LIST_BUILDS_DEFAULT_COLS = ["name", "status", "image"]

osbs/core.py

@@ -595,6 +595,19 @@ def import_image(self, name, namespace=DEFAULT_NAMESPACE):
                         logger.info("ImageStream updated")
                         break
 
+    def dump_resource(self, resource_type, namespace=DEFAULT_NAMESPACE):
+        url = self._build_url("namespaces/%s/%s" % (namespace, resource_type))
+        response = self._get(url)
+        check_response(response)
+        return response
+
+    def restore_resource(self, resource_type, resource, namespace=DEFAULT_NAMESPACE):
+        url = self._build_url("namespaces/%s/%s" % (namespace, resource_type))
+        response = self._post(url, data=json.dumps(resource),
+                              headers={"Content-Type": "application/json"})
+        check_response(response)
+        return response
+
 
 if __name__ == '__main__':
     o = Openshift(openshift_api_url="https://localhost:8443/oapi/v1/",

osbs/utils.py

@@ -9,14 +9,19 @@
 
 import contextlib
 import copy
+import logging
 import os
+import os.path
 import re
 import shutil
 import string
 import subprocess
 import sys
 import tempfile
+import tarfile
+from collections import namedtuple
 from datetime import datetime
+from io import BytesIO
 
 try:
     # py3
@@ -32,6 +37,7 @@
 from dockerfile_parse import DockerfileParser
 from osbs.exceptions import OsbsException
 
+logger = logging.getLogger(__name__)
 
 class RegistryURI(object):
     # Group 0: URI without path -- allowing empty value -- including:
@@ -52,6 +58,59 @@ def uri(self):
         return self.scheme + self.docker_uri
 
 
+class TarWriter(object):
+    def __init__(self, outfile, directory=None):
+        mode = "w|bz2"
+        if hasattr(outfile, "write"):
+            self.tarfile = tarfile.open(fileobj=outfile, mode=mode)
+        else:
+            self.tarfile = tarfile.open(name=outfile, mode=mode)
+        self.directory = directory or ""
+
+    def __enter__(self):
+        return self
+
+    def __exit__(self, typ, val, tb):
+        self.tarfile.close()
+
+    def write_file(self, name, content):
+        buf = BytesIO(content)
+        arcname = os.path.join(self.directory, name)
+
+        ti = tarfile.TarInfo(arcname)
+        ti.size = len(content)
+        self.tarfile.addfile(ti, fileobj=buf)
+
+
+class TarReader(object):
+    TarFile = namedtuple('TarFile', ['filename', 'fileobj'])
+
+    def __init__(self, infile):
+        mode = "r|bz2"
+        if hasattr(infile, "read"):
+            self.tarfile = tarfile.open(fileobj=infile, mode=mode)
+        else:
+            self.tarfile = tarfile.open(name=infile, mode=mode)
+
+    def __iter__(self):
+        return self
+
+    def __next__(self):
+        return self.next()
+
+    def next(self):
+        ti = self.tarfile.next()
+
+        if ti is None:
+            self.close()
+            raise StopIteration()
+
+        return self.TarFile(ti.name, self.tarfile.extractfile(ti))
+
+    def close(self):
+        self.tarfile.close()
+
+
 def graceful_chain_get(d, *args):
     if not d:
         return None
@@ -64,6 +123,20 @@ def graceful_chain_get(d, *args):
     return t
 
 
+def graceful_chain_del(d, *args):
+    if not d:
+        return
+    for arg in args[:-1]:
+        try:
+            d = d[arg]
+        except (IndexError, KeyError):
+            return
+    try:
+        del d[args[-1]]
+    except (IndexError, KeyError):
+        pass
+
+
 def buildconfig_update(orig, new, remove_nonexistent_keys=False):
     """Performs update of given `orig` BuildConfig with values from `new` BuildConfig.
     Both BuildConfigs have to be represented as `dict`s.
@@ -121,6 +194,17 @@ def checkout_git_repo(git_uri, git_ref, git_branch=None):
         shutil.rmtree(tmpdir)
 
 
+@contextlib.contextmanager
+def paused_builds(osbs, namespace):
+    logger.info("pausing builds")
+    osbs.pause_builds(namespace=namespace)
+    try:
+        yield osbs
+    finally:
+        logger.info("resuming builds")
+        osbs.resume_builds(namespace=namespace)
+
+
 def looks_like_git_hash(git_ref):
     return all(ch in string.hexdigits for ch in git_ref) and len(git_ref) == 40