[Backport][ipa-4-6] Auto-retry failed certmonger requests

ipaclient/install/client.py

@@ -776,6 +776,7 @@ def configure_certmonger(
     cmonger = services.knownservices.certmonger
     try:
         cmonger.enable()
+        cmonger.start()
     except Exception as e:
         logger.error(
             "Failed to configure automatic startup of the %s daemon: %s",
@@ -787,14 +788,24 @@ def configure_certmonger(
     subject = str(DN(('CN', hostname), subject_base))
     passwd_fname = os.path.join(paths.IPA_NSSDB_DIR, 'pwdfile.txt')
     try:
-        certmonger.request_cert(
+        certmonger.request_and_wait_for_cert(
             certpath=paths.IPA_NSSDB_DIR,
-            nickname='Local IPA host', subject=subject, dns=[hostname],
-            principal=principal, passwd_fname=passwd_fname)
-    except Exception as ex:
-        logger.error(
-            "%s request for host certificate failed: %s",
-            cmonger.service_name, ex)
+            storage='NSSDB',
+            nickname='Local IPA host',
+            subject=subject,
+            dns=[hostname],
+            principal=principal,
+            passwd_fname=passwd_fname,
+            resubmit_timeout=120,
+        )
+    except Exception as e:
+        logger.exception("certmonger request failed")
+        raise ScriptError(
+            "{} request for host certificate failed: {}".format(
+                cmonger.service_name, e
+            ),
+            rval=CLIENT_INSTALL_ERROR
+        )
 
 
 def configure_sssd_conf(

ipalib/install/certmonger.py

@@ -305,20 +305,56 @@ def add_subject(request_id, subject):
 def request_and_wait_for_cert(
         certpath, subject, principal, nickname=None, passwd_fname=None,
         dns=None, ca='IPA', profile=None,
-        pre_command=None, post_command=None, storage='NSSDB', perms=None):
-    """
-    Execute certmonger to request a server certificate.
-
-    The method also waits for the certificate to be available.
-    """
-    reqId = request_cert(certpath, subject, principal, nickname,
-                         passwd_fname, dns, ca, profile,
-                         pre_command, post_command, storage, perms)
-    state = wait_for_request(reqId, api.env.startup_timeout)
-    ca_error = get_request_value(reqId, 'ca-error')
-    if state != 'MONITORING' or ca_error:
-        raise RuntimeError("Certificate issuance failed ({})".format(state))
-    return reqId
+        pre_command=None, post_command=None, storage='NSSDB', perms=None,
+        resubmit_timeout=0):
+    """Request certificate, wait and possibly resubmit failing requests
+
+    Submit a cert request to certmonger and wait until the request has
+    finished.
+
+    With timeout, a failed request is resubmitted. During parallel replica
+    installation, a request sometimes fails with CA_REJECTED or
+    CA_UNREACHABLE. The error occurs when the master is either busy or some
+    information haven't been replicated yet. Even a stuck request can be
+    recovered, e.g. when permission and group information have been
+    replicated.
+    """
+    req_id = request_cert(
+        certpath, subject, principal, nickname, passwd_fname, dns, ca,
+        profile, pre_command, post_command, storage, perms
+    )
+
+    deadline = time.time() + resubmit_timeout
+    while True:  # until success, timeout, or error
+        state = wait_for_request(req_id, api.env.replication_wait_timeout)
+        ca_error = get_request_value(req_id, 'ca-error')
+        if state == 'MONITORING' and ca_error is None:
+            # we got a winner, exiting
+            logger.debug("Cert request %s was successful", req_id)
+            return req_id
+
+        logger.debug(
+            "Cert request %s failed: %s (%s)", req_id, state, ca_error
+        )
+        if state not in {'CA_REJECTED', 'CA_UNREACHABLE'}:
+            # probably unrecoverable error
+            logger.debug("Giving up on cert request %s", req_id)
+            break
+        elif not resubmit_timeout:
+            # no resubmit
+            break
+        elif time.time() > deadline:
+            logger.debug("Request %s reached resubmit dead line", req_id)
+            break
+        else:
+            # sleep and resubmit
+            logger.debug("Sleep and resubmit cert request %s", req_id)
+            time.sleep(10)
+            resubmit_request(req_id)
+
+    raise RuntimeError(
+        "Certificate issuance failed ({}: {})".format(state, ca_error)
+    )
 
 
 def request_cert(

ipaserver/install/cainstance.py

@@ -914,7 +914,9 @@ def __request_ra_certificate(self):
                 profile='caServerCert',
                 pre_command='renew_ra_cert_pre',
                 post_command='renew_ra_cert',
-                storage="FILE")
+                storage="FILE",
+                resubmit_timeout=api.env.replication_wait_timeout
+            )
             self.__set_ra_cert_perms()
 
             self.requestId = str(reqId)

ipaserver/install/certs.py

@@ -600,12 +600,19 @@ def init_from_pkcs12(self, pkcs12_fname, pkcs12_passwd,
     def export_pem_cert(self, nickname, location):
         return self.nssdb.export_pem_cert(nickname, location)
 
-    def request_service_cert(self, nickname, principal, host):
-        certmonger.request_and_wait_for_cert(certpath=self.secdir,
-                                             nickname=nickname,
-                                             principal=principal,
-                                             subject=host,
-                                             passwd_fname=self.passwd_fname)
+    def request_service_cert(self, nickname, principal, host,
+                             resubmit_timeout=None):
+        if resubmit_timeout is None:
+            resubmit_timeout = api.env.replication_wait_timeout
+        return certmonger.request_and_wait_for_cert(
+            certpath=self.secdir,
+            storage='NSSDB',
+            nickname=nickname,
+            principal=principal,
+            subject=host,
+            passwd_fname=self.passwd_fname,
+            resubmit_timeout=resubmit_timeout
+        )
 
     def is_ipa_issued_cert(self, api, nickname):
         """

ipaserver/install/dsinstance.py

@@ -839,7 +839,9 @@ def __enable_ssl(self):
                     ca='IPA',
                     profile=dogtag.DEFAULT_PROFILE,
                     dns=[self.fqdn],
-                    post_command=cmd)
+                    post_command=cmd,
+                    resubmit_timeout=api.env.replication_wait_timeout
+                )
             finally:
                 if prev_helper is not None:
                     certmonger.modify_ca_helper('IPA', prev_helper)

ipaserver/install/httpinstance.py

@@ -450,7 +450,10 @@ def __setup_ssl(self):
                     ca='IPA',
                     profile=dogtag.DEFAULT_PROFILE,
                     dns=[self.fqdn],
-                    post_command='restart_httpd')
+                    post_command='restart_httpd',
+                    storage='NSSDB',
+                    resubmit_timeout=api.env.replication_wait_timeout
+                )
             finally:
                 if prev_helper is not None:
                     certmonger.modify_ca_helper('IPA', prev_helper)

ipaserver/install/krbinstance.py

@@ -445,7 +445,9 @@ def _call_certmonger(self, certmonger_ca='IPA'):
                 storage='FILE',
                 profile=KDC_PROFILE,
                 post_command='renew_kdc_cert',
-                perms=(0o644, 0o600))
+                perms=(0o644, 0o600),
+                resubmit_timeout=api.env.replication_wait_timeout
+            )
         except dbus.DBusException as e:
             # if the certificate is already tracked, ignore the error
             name = e.get_dbus_name()

ipatests/test_integration/test_user_permissions.py

@@ -142,10 +142,23 @@ def test_installclient_as_user_admin(self):
         user_kinit = "%s\n%s\n%s\n" % (password, password, password)
         self.master.run_command(['kinit', username],
                                 stdin_text=user_kinit)
-        tasks.install_client(self.master, client, user=username,
-                             password=password)
+        tasks.install_client(
+            self.master, client,
+            extra_args=['--request-cert'],
+            user=username, password=password
+        )
         msg = "args=['/usr/bin/getent', 'passwd', '%s@%s']" % \
               (username, client.domain.name)
         install_log = client.get_file_contents(paths.IPACLIENT_INSTALL_LOG,
                                                encoding='utf-8')
         assert msg in install_log
+
+        # check that user is able to request a host cert, too
+        result = tasks.run_certutil(client, ['-L'], paths.IPA_NSSDB_DIR)
+        assert 'Local IPA host' in result.stdout_text
+        result = tasks.run_certutil(
+            client,
+            ['-K', '-f', paths.IPA_NSSDB_PWDFILE_TXT],
+            paths.IPA_NSSDB_DIR
+        )
+        assert 'Local IPA host' in result.stdout_text