ceph: add ClusterID and PoolID mappings between local and peer cluster

.github/workflows/canary-integration-test.yml

@@ -826,6 +826,20 @@ jobs:
         kubectl exec -n rook-ceph deploy/rook-ceph-tools -t -- rbd -p replicapool info test
         kubectl exec -n rook-ceph deploy/rook-ceph-tools -t -- rbd -p replicapool2 info test
 
+    - name: copy block mirror peer secret into the primary cluster for replicapool
+      run: |
+        kubectl -n rook-ceph-secondary get secret pool-peer-token-replicapool -o yaml |\
+        sed 's/namespace: rook-ceph-secondary/namespace: rook-ceph/g; s/name: pool-peer-token-replicapool/name: pool-peer-token-replicapool-config/g' |\
+        kubectl create --namespace=rook-ceph -f -
+
+    - name: add block mirror peer secret to the primary cluster for replicapool
+      run: |
+        kubectl -n rook-ceph patch cephblockpool replicapool --type merge -p '{"spec":{"mirroring":{"peers": {"secretNames": ["pool-peer-token-replicapool-config"]}}}}'
+
+    - name: wait for rook-ceph-csi-mapping-config to be updated with cluster ID
+      run: |
+        timeout 60 sh -c 'until [ "$(kubectl get cm -n rook-ceph rook-ceph-csi-mapping-config  -o jsonpath='{.data.csi-mapping-config-json}' | grep -c "rook-ceph-secondary")" -eq 1 ]; do echo "waiting for rook-ceph-csi-mapping-config to be created with cluster ID mappings" && sleep 1; done'
+
     - name: create replicated mirrored filesystem on cluster 1
       run: |
         PRIMARY_YAML=cluster/examples/kubernetes/ceph/filesystem-test-primary.yaml

cluster/examples/kubernetes/ceph/csi/template/rbd/csi-rbdplugin-provisioner-dep.yaml

@@ -103,7 +103,7 @@ spec:
                   fieldPath: metadata.namespace
           imagePullPolicy: "IfNotPresent"
           volumeMounts:
-            - name: ceph-csi-config
+            - name: ceph-csi-configs
               mountPath: /etc/ceph-csi-config/
             - name: keys-tmp-dir
               mountPath: /tmp/csi/keys
@@ -166,7 +166,7 @@ spec:
             - mountPath: /lib/modules
               name: lib-modules
               readOnly: true
-            - name: ceph-csi-config
+            - name: ceph-csi-configs
               mountPath: /etc/ceph-csi-config/
             - name: keys-tmp-dir
               mountPath: /tmp/csi/keys
@@ -204,12 +204,21 @@ spec:
           emptyDir: {
             medium: "Memory"
           }
-        - name: ceph-csi-config
-          configMap:
-            name: rook-ceph-csi-config
-            items:
-              - key: csi-cluster-config-json
-                path: config.json
+        - name: ceph-csi-configs
+          projected:
+            sources:
+              - name: ceph-csi-config
+                configMap:
+                  name: rook-ceph-csi-config
+                  items:
+                    - key: csi-cluster-config-json
+                      path: config.json
+              - name: ceph-csi-mapping-config
+                configMap:
+                  name: rook-ceph-csi-mapping-config
+                  items:
+                    - key: csi-mapping-config-json
+                      path: cluster-mapping.json
         - name: keys-tmp-dir
           emptyDir: {
             medium: "Memory"

cluster/examples/kubernetes/ceph/csi/template/rbd/csi-rbdplugin.yaml

@@ -100,7 +100,7 @@ spec:
             - mountPath: /lib/modules
               name: lib-modules
               readOnly: true
-            - name: ceph-csi-config
+            - name: ceph-csi-configs
               mountPath: /etc/ceph-csi-config/
             - name: keys-tmp-dir
               mountPath: /tmp/csi/keys
@@ -154,12 +154,21 @@ spec:
         - name: lib-modules
           hostPath:
             path: /lib/modules
-        - name: ceph-csi-config
-          configMap:
-            name: rook-ceph-csi-config
-            items:
-              - key: csi-cluster-config-json
-                path: config.json
+        - name: ceph-csi-configs
+          projected:
+            sources:
+              - name: ceph-csi-config
+                configMap:
+                  name: rook-ceph-csi-config
+                  items:
+                    - key: csi-cluster-config-json
+                      path: config.json
+              - name: ceph-csi-mapping-config
+                configMap:
+                  name: rook-ceph-csi-mapping-config
+                  items:
+                    - key: csi-mapping-config-json
+                      path: cluster-mapping.json
         - name: keys-tmp-dir
           emptyDir: {
             medium: "Memory"

pkg/daemon/ceph/client/pool.go

@@ -124,6 +124,11 @@ func GetPoolDetails(context *clusterd.Context, clusterInfo *ClusterInfo, name st
 		return CephStoragePoolDetails{}, errors.Wrapf(err, "failed to get pool %s details. %s", name, string(output))
 	}
 
+	return ParsePoolDetails(output)
+}
+
+func ParsePoolDetails(in []byte) (CephStoragePoolDetails, error) {
+
 	// The response for osd pool get when passing var=all is actually malformed JSON similar to:
 	// {"pool":"rbd","size":1}{"pool":"rbd","min_size":2}...
 	// Note the multiple top level entities, one for each property returned.  To workaround this,
@@ -132,7 +137,7 @@ func GetPoolDetails(context *clusterd.Context, clusterInfo *ClusterInfo, name st
 	// Since previously set fields remain intact if they are not overwritten, the result is the JSON
 	// unmarshalling of all properties in the response.
 	var poolDetails CephStoragePoolDetails
-	poolDetailsUnits := strings.Split(string(output), "}{")
+	poolDetailsUnits := strings.Split(string(in), "}{")
 	for i := range poolDetailsUnits {
 		pdu := poolDetailsUnits[i]
 		if !strings.HasPrefix(pdu, "{") {
@@ -143,7 +148,7 @@ func GetPoolDetails(context *clusterd.Context, clusterInfo *ClusterInfo, name st
 		}
 		err := json.Unmarshal([]byte(pdu), &poolDetails)
 		if err != nil {
-			return CephStoragePoolDetails{}, errors.Wrapf(err, "unmarshal failed raw buffer response %s", string(output))
+			return CephStoragePoolDetails{}, errors.Wrapf(err, "unmarshal failed raw buffer response %s", string(in))
 		}
 	}
 

pkg/operator/ceph/cluster/mgr/dashboard.go

@@ -21,7 +21,6 @@ import (
 	"context"
 	"crypto/rand"
 	"fmt"
-	"io/ioutil"
 	"os"
 	"strconv"
 	"syscall"
@@ -32,6 +31,7 @@ import (
 	"github.com/rook/rook/pkg/operator/ceph/config"
 	cephver "github.com/rook/rook/pkg/operator/ceph/version"
 	"github.com/rook/rook/pkg/operator/k8sutil"
+	"github.com/rook/rook/pkg/util"
 	"github.com/rook/rook/pkg/util/exec"
 	v1 "k8s.io/api/core/v1"
 	kerrors "k8s.io/apimachinery/pkg/api/errors"
@@ -217,21 +217,6 @@ func FileBasedPasswordSupported(c *client.ClusterInfo) bool {
 	return false
 }
 
-func CreateTempPasswordFile(password string) (*os.File, error) {
-	// Generate a temp file
-	file, err := ioutil.TempFile("", "")
-	if err != nil {
-		return nil, errors.Wrap(err, "failed to generate temp file")
-	}
-
-	// Write password into file
-	err = ioutil.WriteFile(file.Name(), []byte(password), 0440)
-	if err != nil {
-		return nil, errors.Wrap(err, "failed to write dashboard password into file")
-	}
-	return file, nil
-}
-
 func (c *Cluster) setLoginCredentials(password string) error {
 	// Set the login credentials. Write the command/args to the debug log so we don't write the password by default to the log.
 	logger.Infof("setting ceph dashboard %q login creds", dashboardUsername)
@@ -240,7 +225,7 @@ func (c *Cluster) setLoginCredentials(password string) error {
 	// for latest Ceph versions
 	if FileBasedPasswordSupported(c.clusterInfo) {
 		// Generate a temp file
-		file, err := CreateTempPasswordFile(password)
+		file, err := util.CreateTempFile(password)
 		if err != nil {
 			return errors.Wrap(err, "failed to create a temporary dashboard password file")
 		}