ipa-restore: adapt for 389-ds switch to LMDB

ipa-restore is relying on the presence of specific directories,
e.g. /var/lib/dirsrv/slapd-/db/ipaca, to detect
which backends are in use (userRoot or ipaca).

With the switch to LMDB, these directories do not exist and the
restore fails finding the ipaca backend.

Use lib389.cli_ctl.dblib.run_dbscan utility instead to
check which backends are present.

This method was been introduced in 389ds 2.1.0 and works with
Berkeley DB and LMDB.

Add a --data option to the ipa-backup and ipa-restore tasks to do
only an LDIF backup and restore. Also add the ability to restore by
backend.

Add new tests to do a data-only backup and restore.

Fixes: https://pagure.io/freeipa/issue/9526

Signed-off-by: Rob Crittenden <rcritten@redhat.com>
Reviewed-By: Florence Blanc-Renaud <flo@redhat.com>

ipaserver/install/ipa_restore.py

@@ -50,6 +50,8 @@
 from ipaplatform import services
 from ipaplatform.paths import paths
 
+from lib389.cli_ctl.dblib import run_dbscan
+
 try:
     from ipaserver.install import adtrustinstance
 except ImportError:
@@ -65,6 +67,29 @@
 
 logger = logging.getLogger(__name__)
 
+backends = []  # global to save running dbscan multiple times
+
+
+def get_backends(db_dir):
+    """Retrieve the set of backends directly from the current database"""
+    global backends
+
+    if backends:
+        return backends
+
+    output = run_dbscan(['-L', db_dir])
+    output = output.replace(db_dir + '/', '')
+    output = output.split('\n')
+    for line in output:
+        if '/' not in line:
+            continue
+        backends.append(line.split('/')[0].strip().lower())
+    backends = set(backends)
+    if 'changelog' in backends:
+        backends.remove('changelog')
+
+    return backends
+
 
 def recursive_chown(path, uid, gid):
     '''
@@ -295,24 +320,30 @@ def run(self):
             if options.backend:
                 for instance in self.instances:
                     db_dir = (paths.SLAPD_INSTANCE_DB_DIR_TEMPLATE %
-                              (instance, options.backend))
-                    if os.path.exists(db_dir):
+                              (instance, ""))
+                    backends = get_backends(db_dir)
+                    if options.backend.lower() in backends:
                         break
                 else:
                     raise admintool.ScriptError(
                         "Backend %s does not exist" % options.backend)
 
                 self.backends = [options.backend]
 
+            missing_backends = []
             for instance, backend in itertools.product(self.instances,
                                                        self.backends):
                 db_dir = (paths.SLAPD_INSTANCE_DB_DIR_TEMPLATE %
-                          (instance, backend))
-                if os.path.exists(db_dir):
-                    break
-            else:
+                          (instance, ""))
+                backends = get_backends(db_dir)
+                if backend.lower() not in backends:
+                    missing_backends.append(backend)
+
+            if missing_backends:
                 raise admintool.ScriptError(
-                    "Cannot restore a data backup into an empty system")
+                    "Cannot restore a data backup into an empty system. "
+                    "Missing backend(s) %s" % ', '.join(missing_backends)
+                )
 
         logger.info("Performing %s restore from %s backup",
                     restore_type, self.backup_type)
@@ -381,6 +412,10 @@ def run(self):
                     ldiffile = os.path.join(self.dir, '%s-%s.ldif' % database)
                     if os.path.exists(ldiffile):
                         databases.append(database)
+                    else:
+                        logger.warning(
+                            "LDIF file '%s-%s.ldif' not found in backup",
+                            instance, backend)
 
             if options.instance:
                 for instance, backend in databases:

ipatests/pytest_ipa/integration/tasks.py

@@ -1617,12 +1617,15 @@ def resolve_record(nameserver, query, rtype="SOA", retry=True, timeout=100):
     raise errors.DNSResolverError(exception=ValueError("Record not found"))
 
 
-def ipa_backup(host, disable_role_check=False, raiseonerr=True):
+def ipa_backup(host, disable_role_check=False, data_only=False,
+               raiseonerr=True):
     """Run backup on host and return the run_command result.
     """
     cmd = ['ipa-backup', '-v']
     if disable_role_check:
         cmd.append('--disable-role-check')
+    if data_only:
+        cmd.append('--data')
     result = host.run_command(cmd, raiseonerr=raiseonerr)
 
     # Test for ticket 7632: check that services are restarted
@@ -1652,10 +1655,10 @@ def ipa_epn(
     return host.run_command(cmd, raiseonerr=raiseonerr)
 
 
-def get_backup_dir(host, raiseonerr=True):
+def get_backup_dir(host, data_only=False, raiseonerr=True):
     """Wrapper around ipa_backup: returns the backup directory.
     """
-    result = ipa_backup(host, raiseonerr)
+    result = ipa_backup(host, data_only=data_only, raiseonerr=raiseonerr)
 
     # Get the backup location from the command's output
     for line in result.stderr_text.splitlines():
@@ -1671,10 +1674,13 @@ def get_backup_dir(host, raiseonerr=True):
             return None
 
 
-def ipa_restore(master, backup_path):
-    master.run_command(["ipa-restore", "-U",
-                        "-p", master.config.dirman_password,
-                        backup_path])
+def ipa_restore(master, backup_path, backend=None):
+    cmd = ["ipa-restore", "-U",
+           "-p", master.config.dirman_password,
+           backup_path]
+    if backend:
+        cmd.extend(["--data", "--backend", backend])
+    master.run_command(cmd)
 
 
 def install_kra(host, domain_level=None,

ipatests/test_integration/test_backup_and_restore.py

@@ -210,6 +210,40 @@ def test_full_backup_and_restore(self):
                                            '"%a %G:%U"', log_path])
             assert "770 dirsrv:dirsrv" in cmd.stdout_text
 
+    def test_data_backup_and_restore(self):
+        """backup data only then restore"""
+        with restore_checker(self.master):
+            backup_path = tasks.get_backup_dir(self.master, data_only=True)
+
+            self.master.run_command(['ipa', 'user-add', 'testuser',
+                                     '--first', 'test',
+                                     '--last', 'user'])
+
+            tasks.ipa_restore(self.master, backup_path)
+
+            # the user added in the interim should now be gone
+            result = self.master.run_command(
+                ['ipa', 'user-show', 'test-user'], raiseonerr=False
+            )
+            assert 'user not found' in result.stderr_text
+
+    def test_data_backup_and_restore_backend(self):
+        """backup data only then restore"""
+        with restore_checker(self.master):
+            backup_path = tasks.get_backup_dir(self.master, data_only=True)
+
+            self.master.run_command(['ipa', 'user-add', 'testuser',
+                                     '--first', 'test',
+                                     '--last', 'user'])
+
+            tasks.ipa_restore(self.master, backup_path, backend='userRoot')
+
+            # the user added in the interim should now be gone
+            result = self.master.run_command(
+                ['ipa', 'user-show', 'test-user'], raiseonerr=False
+            )
+            assert 'user not found' in result.stderr_text
+
     def test_full_backup_and_restore_with_removed_users(self):
         """regression test for https://fedorahosted.org/freeipa/ticket/3866"""
         with restore_checker(self.master):