From c77847f2be37c33d0a903f39cf45fa8b60a02811 Mon Sep 17 00:00:00 2001 From: Sage Weil Date: Wed, 5 Oct 2016 11:09:19 -0400 Subject: [PATCH 1/2] auth/cephx: tolerate missing rotating keys During an upgrade, we may have a client requesting an MGR service key but not have one in the database yet, either because we *just* upgraded and haven't generated one yet, or because the leader mon hasn't been upgraded yet. Fix this by silently tolerating a missing key as long as one or more other service keys were present and we have something to give to the client. Signed-off-by: Sage Weil --- src/auth/cephx/CephxServiceHandler.cc | 25 +++++++++++++++++++------ 1 file changed, 19 insertions(+), 6 deletions(-) diff --git a/src/auth/cephx/CephxServiceHandler.cc b/src/auth/cephx/CephxServiceHandler.cc index 914fea712760e..15d27f540c767 100644 --- a/src/auth/cephx/CephxServiceHandler.cc +++ b/src/auth/cephx/CephxServiceHandler.cc @@ -163,19 +163,32 @@ int CephxServiceHandler::handle_request(bufferlist::iterator& indata, bufferlist ret = 0; vector info_vec; - for (uint32_t service_id = 1; service_id <= ticket_req.keys; service_id <<= 1) { + int found_services = 0; + int service_err = 0; + for (uint32_t service_id = 1; service_id <= ticket_req.keys; + service_id <<= 1) { if (ticket_req.keys & service_id) { - ldout(cct, 10) << " adding key for service " << ceph_entity_type_name(service_id) << dendl; + ldout(cct, 10) << " adding key for service " + << ceph_entity_type_name(service_id) << dendl; CephXSessionAuthInfo info; - int r = key_server->build_session_auth_info(service_id, auth_ticket_info, info); + int r = key_server->build_session_auth_info(service_id, + auth_ticket_info, info); + // tolerate missing MGR rotating key for the purposes of upgrades. if (r < 0) { - ret = r; - break; - } + ldout(cct, 10) << " missing key for service " + << ceph_entity_type_name(service_id) << dendl; + service_err = r; + continue; + } info.validity += cct->_conf->auth_service_ticket_ttl; info_vec.push_back(info); + ++found_services; } } + if (!found_services && service_err) { + ldout(cct, 10) << __func__ << " did not find any service keys" << dendl; + ret = service_err; + } CryptoKey no_key; build_cephx_response_header(cephx_header.request_type, ret, result_bl); cephx_build_service_ticket_reply(cct, auth_ticket_info.session_key, info_vec, false, no_key, result_bl); From 4b789807d5ba7de198cae55896e3bdcde5c2129e Mon Sep 17 00:00:00 2001 From: Sage Weil Date: Wed, 5 Oct 2016 11:10:30 -0400 Subject: [PATCH 2/2] auth/cephx: do not re-request *only* the MGR key If we request a bunch of service keys, we may not get back a MGR key because of an in-progress upgrade. If we have everything we need except for just the MGR key, do not bother re-requesting it. Instead just continue and we'll re-request it later when the secrets rotate. Signed-off-by: Sage Weil --- src/auth/cephx/CephxClientHandler.cc | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/auth/cephx/CephxClientHandler.cc b/src/auth/cephx/CephxClientHandler.cc index ced5ff3a41b16..2ecc8ec8a29a0 100644 --- a/src/auth/cephx/CephxClientHandler.cc +++ b/src/auth/cephx/CephxClientHandler.cc @@ -74,7 +74,11 @@ int CephxClientHandler::build_request(bufferlist& bl) const return 0; } - if (need) { + // do not bother (re)requesting tickets if we *only* need the MGR + // ticket; that can happen during an upgrade and we want to avoid a + // loop. we'll end up re-requesting it later when the secrets + // rotating. + if (need && need != CEPH_ENTITY_TYPE_MGR) { /* get service tickets */ ldout(cct, 10) << "get service keys: want=" << want << " need=" << need << " have=" << have << dendl;