ceph: do not check ok-to-stop when OSDs are in CLBO

pkg/daemon/ceph/client/upgrade.go

@@ -37,7 +37,8 @@ const (
 var (
 	// we don't perform any checks on these daemons
 	// they don't have any "ok-to-stop" command implemented
-	daemonNoCheck = []string{"mgr", "rgw", "rbd-mirror", "nfs", "fs-mirror"}
+	daemonNoCheck    = []string{"mgr", "rgw", "rbd-mirror", "nfs", "fs-mirror"}
+	errNoHostInCRUSH = errors.New("no host in crush map yet?")
 )
 
 func getCephMonVersionString(context *clusterd.Context, clusterInfo *ClusterInfo) (string, error) {
@@ -311,7 +312,7 @@ func allOSDsSameHost(context *clusterd.Context, clusterInfo *ClusterInfo) (bool,
 
 	hostOsdNodes := len(hostOsdTree.Nodes)
 	if hostOsdNodes == 0 {
-		return false, errors.New("no host in crush map yet?")
+		return false, errNoHostInCRUSH
 	}
 
 	// If the number of OSD node is 1, chances are this is simple setup with all OSDs on it
@@ -369,6 +370,10 @@ func OSDUpdateShouldCheckOkToStop(context *clusterd.Context, clusterInfo *Cluste
 	// aio means all in one
 	aio, err := allOSDsSameHost(context, clusterInfo)
 	if err != nil {
+		if errors.Is(err, errNoHostInCRUSH) {
+			logger.Warning("the CRUSH map has no 'host' entries so not performing ok-to-stop checks")
+			return false
+		}
 		logger.Warningf("failed to determine if all osds are running on the same host. will check if OSDs are ok-to-stop. if all OSDs are running on one host %s. %v", userIntervention, err)
 		return true
 	}
@@ -401,6 +406,18 @@ func osdDoNothing(context *clusterd.Context, clusterInfo *ClusterInfo) bool {
 	// aio means all in one
 	aio, err := allOSDsSameHost(context, clusterInfo)
 	if err != nil {
+		// We return true so that we can continue without a retry and subsequently not test if the
+		// osd can be stopped This handles the scenario where the OSDs have been created but not yet
+		// started due to a wrong CR configuration For instance, when OSDs are encrypted and Vault
+		// is used to store encryption keys, if the KV version is incorrect during the cluster
+		// initialization the OSDs will fail to start and stay in CLBO until the CR is updated again
+		// with the correct KV version so that it can start For this scenario we don't need to go
+		// through the path where the check whether the OSD can be stopped or not, so it will always
+		// fail and make us wait for nothing
+		if errors.Is(err, errNoHostInCRUSH) {
+			logger.Warning("the CRUSH map has no 'host' entries so not performing ok-to-stop checks")
+			return true
+		}
 		logger.Warningf("failed to determine if all osds are running on the same host, performing upgrade check anyways. %v", err)
 		return false
 	}

pkg/daemon/ceph/client/upgrade_test.go

@@ -357,4 +357,11 @@ func TestOSDUpdateShouldCheckOkToStop(t *testing.T) {
 		treeOutput = fake.OsdTreeOutput(0, 0)
 		assert.False(t, OSDUpdateShouldCheckOkToStop(context, clusterInfo))
 	})
+
+	// degraded case, OSDs are failing to start so they haven't registered in the CRUSH map yet
+	t.Run("0 nodes with down OSDs", func(t *testing.T) {
+		lsOutput = fake.OsdLsOutput(3)
+		treeOutput = fake.OsdTreeOutput(0, 1)
+		assert.False(t, OSDUpdateShouldCheckOkToStop(context, clusterInfo))
+	})
 }