From 984542f40c364f56b4c5ddf1a6456f89a5e52058 Mon Sep 17 00:00:00 2001 From: Christian Heimes Date: Sun, 8 Jul 2018 11:53:58 +0200 Subject: [PATCH 1/2] Auto-retry failed certmonger requests During parallel replica installation, a request sometimes fails with CA_REJECTED or CA_UNREACHABLE. The error occur when the master is either busy or some information haven't been replicated yet. Even a stuck request can be recovered, e.g. when permission and group information have been replicated. A new function request_and_retry_cert() automatically resubmits failing requests until it times out. Fixes: https://pagure.io/freeipa/issue/7623 Signed-off-by: Christian Heimes --- ipalib/install/certmonger.py | 64 ++++++++++++++++++++++++------- ipaserver/install/cainstance.py | 4 +- ipaserver/install/certs.py | 19 ++++++--- ipaserver/install/dsinstance.py | 4 +- ipaserver/install/httpinstance.py | 5 ++- ipaserver/install/krbinstance.py | 4 +- 6 files changed, 76 insertions(+), 24 deletions(-) diff --git a/ipalib/install/certmonger.py b/ipalib/install/certmonger.py index c07242412af..3e1862192eb 100644 --- a/ipalib/install/certmonger.py +++ b/ipalib/install/certmonger.py @@ -305,20 +305,56 @@ def add_subject(request_id, subject): def request_and_wait_for_cert( certpath, subject, principal, nickname=None, passwd_fname=None, dns=None, ca='IPA', profile=None, - pre_command=None, post_command=None, storage='NSSDB', perms=None): - """ - Execute certmonger to request a server certificate. - - The method also waits for the certificate to be available. - """ - reqId = request_cert(certpath, subject, principal, nickname, - passwd_fname, dns, ca, profile, - pre_command, post_command, storage, perms) - state = wait_for_request(reqId, api.env.startup_timeout) - ca_error = get_request_value(reqId, 'ca-error') - if state != 'MONITORING' or ca_error: - raise RuntimeError("Certificate issuance failed ({})".format(state)) - return reqId + pre_command=None, post_command=None, storage='NSSDB', perms=None, + resubmit_timeout=0): + """Request certificate, wait and possibly resubmit failing requests + + Submit a cert request to certmonger and wait until the request has + finished. + + With timeout, a failed request is resubmitted. During parallel replica + installation, a request sometimes fails with CA_REJECTED or + CA_UNREACHABLE. The error occurs when the master is either busy or some + information haven't been replicated yet. Even a stuck request can be + recovered, e.g. when permission and group information have been + replicated. + """ + req_id = request_cert( + certpath, subject, principal, nickname, passwd_fname, dns, ca, + profile, pre_command, post_command, storage, perms + ) + + deadline = time.time() + resubmit_timeout + while True: # until success, timeout, or error + state = wait_for_request(req_id, api.env.replication_wait_timeout) + ca_error = get_request_value(req_id, 'ca-error') + if state == 'MONITORING' and ca_error is None: + # we got a winner, exiting + logger.debug("Cert request %s was successful", req_id) + return req_id + + logger.debug( + "Cert request %s failed: %s (%s)", req_id, state, ca_error + ) + if state not in {'CA_REJECTED', 'CA_UNREACHABLE'}: + # probably unrecoverable error + logger.debug("Giving up on cert request %s", req_id) + break + elif not resubmit_timeout: + # no resubmit + break + elif time.time() > deadline: + logger.debug("Request %s reached resubmit dead line", req_id) + break + else: + # sleep and resubmit + logger.debug("Sleep and resubmit cert request %s", req_id) + time.sleep(10) + resubmit_request(req_id) + + raise RuntimeError( + "Certificate issuance failed ({}: {})".format(state, ca_error) + ) def request_cert( diff --git a/ipaserver/install/cainstance.py b/ipaserver/install/cainstance.py index e114ea6f71a..9bcd6874f04 100644 --- a/ipaserver/install/cainstance.py +++ b/ipaserver/install/cainstance.py @@ -914,7 +914,9 @@ def __request_ra_certificate(self): profile='caServerCert', pre_command='renew_ra_cert_pre', post_command='renew_ra_cert', - storage="FILE") + storage="FILE", + resubmit_timeout=api.env.replication_wait_timeout + ) self.__set_ra_cert_perms() self.requestId = str(reqId) diff --git a/ipaserver/install/certs.py b/ipaserver/install/certs.py index 3f843399f4f..30b2aa0d3e7 100644 --- a/ipaserver/install/certs.py +++ b/ipaserver/install/certs.py @@ -600,12 +600,19 @@ def init_from_pkcs12(self, pkcs12_fname, pkcs12_passwd, def export_pem_cert(self, nickname, location): return self.nssdb.export_pem_cert(nickname, location) - def request_service_cert(self, nickname, principal, host): - certmonger.request_and_wait_for_cert(certpath=self.secdir, - nickname=nickname, - principal=principal, - subject=host, - passwd_fname=self.passwd_fname) + def request_service_cert(self, nickname, principal, host, + resubmit_timeout=None): + if resubmit_timeout is None: + resubmit_timeout = api.env.replication_wait_timeout + return certmonger.request_and_wait_for_cert( + certpath=self.secdir, + storage='NSSDB', + nickname=nickname, + principal=principal, + subject=host, + passwd_fname=self.passwd_fname, + resubmit_timeout=resubmit_timeout + ) def is_ipa_issued_cert(self, api, nickname): """ diff --git a/ipaserver/install/dsinstance.py b/ipaserver/install/dsinstance.py index 06c1273dc21..24d17dc46a3 100644 --- a/ipaserver/install/dsinstance.py +++ b/ipaserver/install/dsinstance.py @@ -839,7 +839,9 @@ def __enable_ssl(self): ca='IPA', profile=dogtag.DEFAULT_PROFILE, dns=[self.fqdn], - post_command=cmd) + post_command=cmd, + resubmit_timeout=api.env.replication_wait_timeout + ) finally: if prev_helper is not None: certmonger.modify_ca_helper('IPA', prev_helper) diff --git a/ipaserver/install/httpinstance.py b/ipaserver/install/httpinstance.py index 0701d68ef98..3f8b18c4e84 100644 --- a/ipaserver/install/httpinstance.py +++ b/ipaserver/install/httpinstance.py @@ -450,7 +450,10 @@ def __setup_ssl(self): ca='IPA', profile=dogtag.DEFAULT_PROFILE, dns=[self.fqdn], - post_command='restart_httpd') + post_command='restart_httpd', + storage='NSSDB', + resubmit_timeout=api.env.replication_wait_timeout + ) finally: if prev_helper is not None: certmonger.modify_ca_helper('IPA', prev_helper) diff --git a/ipaserver/install/krbinstance.py b/ipaserver/install/krbinstance.py index f77967f1054..a3079bd6304 100644 --- a/ipaserver/install/krbinstance.py +++ b/ipaserver/install/krbinstance.py @@ -445,7 +445,9 @@ def _call_certmonger(self, certmonger_ca='IPA'): storage='FILE', profile=KDC_PROFILE, post_command='renew_kdc_cert', - perms=(0o644, 0o600)) + perms=(0o644, 0o600), + resubmit_timeout=api.env.replication_wait_timeout + ) except dbus.DBusException as e: # if the certificate is already tracked, ignore the error name = e.get_dbus_name() From 183b233243beb0d23d40184865e8de643a0fc133 Mon Sep 17 00:00:00 2001 From: Christian Heimes Date: Mon, 9 Jul 2018 13:53:44 +0200 Subject: [PATCH 2/2] Wait for client certificates ipa-client-install --request-cert now waits until certmonger has provided a host certificate. In case of an error, ipa-client-install no longer pretents to success but fails with an error code. The --request-cert option also ensures that certmonger is enabled and running. See: Fixes: https://pagure.io/freeipa/issue/7623 Signed-off-by: Christian Heimes --- ipaclient/install/client.py | 25 +++++++++++++------ .../test_integration/test_user_permissions.py | 17 +++++++++++-- 2 files changed, 33 insertions(+), 9 deletions(-) diff --git a/ipaclient/install/client.py b/ipaclient/install/client.py index f7e6ef378f0..0126ab77e24 100644 --- a/ipaclient/install/client.py +++ b/ipaclient/install/client.py @@ -776,6 +776,7 @@ def configure_certmonger( cmonger = services.knownservices.certmonger try: cmonger.enable() + cmonger.start() except Exception as e: logger.error( "Failed to configure automatic startup of the %s daemon: %s", @@ -787,14 +788,24 @@ def configure_certmonger( subject = str(DN(('CN', hostname), subject_base)) passwd_fname = os.path.join(paths.IPA_NSSDB_DIR, 'pwdfile.txt') try: - certmonger.request_cert( + certmonger.request_and_wait_for_cert( certpath=paths.IPA_NSSDB_DIR, - nickname='Local IPA host', subject=subject, dns=[hostname], - principal=principal, passwd_fname=passwd_fname) - except Exception as ex: - logger.error( - "%s request for host certificate failed: %s", - cmonger.service_name, ex) + storage='NSSDB', + nickname='Local IPA host', + subject=subject, + dns=[hostname], + principal=principal, + passwd_fname=passwd_fname, + resubmit_timeout=120, + ) + except Exception as e: + logger.exception("certmonger request failed") + raise ScriptError( + "{} request for host certificate failed: {}".format( + cmonger.service_name, e + ), + rval=CLIENT_INSTALL_ERROR + ) def configure_sssd_conf( diff --git a/ipatests/test_integration/test_user_permissions.py b/ipatests/test_integration/test_user_permissions.py index 3cd4655365e..3236cb5941b 100644 --- a/ipatests/test_integration/test_user_permissions.py +++ b/ipatests/test_integration/test_user_permissions.py @@ -142,10 +142,23 @@ def test_installclient_as_user_admin(self): user_kinit = "%s\n%s\n%s\n" % (password, password, password) self.master.run_command(['kinit', username], stdin_text=user_kinit) - tasks.install_client(self.master, client, user=username, - password=password) + tasks.install_client( + self.master, client, + extra_args=['--request-cert'], + user=username, password=password + ) msg = "args=['/usr/bin/getent', 'passwd', '%s@%s']" % \ (username, client.domain.name) install_log = client.get_file_contents(paths.IPACLIENT_INSTALL_LOG, encoding='utf-8') assert msg in install_log + + # check that user is able to request a host cert, too + result = tasks.run_certutil(client, ['-L'], paths.IPA_NSSDB_DIR) + assert 'Local IPA host' in result.stdout_text + result = tasks.run_certutil( + client, + ['-K', '-f', paths.IPA_NSSDB_PWDFILE_TXT], + paths.IPA_NSSDB_DIR + ) + assert 'Local IPA host' in result.stdout_text