refact the data migration

Signed-off-by: Liang Sun <sunliang7927@gmail.com>

.github/workflows/release-tools.yml

@@ -0,0 +1,34 @@
+name: Release Tools
+
+on:
+  workflow_dispatch:
+
+jobs:
+  release-image:
+    runs-on: ubuntu-latest
+    permissions:
+      packages: write
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+      - name: Set up QEMU
+        id: qemu
+        uses: docker/setup-qemu-action@v1
+        with:
+          image: tonistiigi/binfmt:latest
+          platforms: all
+      - name: Available platforms
+        run: echo ${{ steps.qemu.outputs.platforms }}
+      - name: Login registry
+        run: |
+          echo "${{ secrets.GITHUB_TOKEN }}" | docker login ghcr.io -u ${{ github.actor }} --password-stdin
+      - name: Set env
+        run: |
+          echo "RELEASE_TAG=`echo $(git describe --tags --abbrev=0)`" >> $GITHUB_ENV
+          echo "PROJECT_SOURCE_CODE_DIR=`echo ${GITHUB_WORKSPACE}`" >> $GITHUB_ENV
+          echo "IMAGE_REGISTRY=`echo ghcr.io/$(echo ${{ github.repository_owner }} | tr 'A-Z' 'a-z')`" >> $GITHUB_ENV
+      - name: Release tools
+        run: |
+          PROJECT_SOURCE_CODE_DIR=${PROJECT_SOURCE_CODE_DIR} IMAGE_REGISTRY=${IMAGE_REGISTRY} RELEASE_TAG=${RELEASE_TAG} make tools

Makefile

@@ -406,6 +406,18 @@ builder:
 	docker build -t ${BUILDER_NAME}:${BUILDER_TAG} -f ${BUILDER_DOCKERFILE} ${PROJECT_SOURCE_CODE_DIR}
 	docker push ${BUILDER_NAME}:${BUILDER_TAG}
 
+.PHONY: tools
+tools: juicesync
+
+.PHONY: juicesync
+juicesync:
+	${DOCKER_BUILDX_CMD_AMD64} -t ${JUICESYNC_NAME}:${JUICESYNC_TAG}-amd64 -f ${JUICESYNC_DOCKERFILE}.amd64 ${PROJECT_SOURCE_CODE_DIR}
+	# build for arm64 version
+	${DOCKER_BUILDX_CMD_ARM64} -t ${JUICESYNC_NAME}:${JUICESYNC_TAG}-arm64 -f ${JUICESYNC_DOCKERFILE}.arm64 ${PROJECT_SOURCE_CODE_DIR}
+	# push to a public registry
+	${MUILT_ARCH_PUSH_CMD} -i ${JUICESYNC_NAME}:${JUICESYNC_TAG}
+
+
 .PHONY: _gen-apis
 _gen-apis:
 	${OPERATOR_CMD} generate k8s

Makefile.variables

@@ -89,6 +89,13 @@ BUILDER_TAG = latest
 BUILDER_DOCKERFILE = ${PROJECT_SOURCE_CODE_DIR}/build/builder/Dockerfile
 BUILDER_MOUNT_DST_DIR = /go/src/github.com/hwameistor/hwameistor
 
+# [ JUICESYNC ]
+#----------------
+JUICESYNC_NAME = ${IMAGE_REGISTRY}/${MODULE_NAME}-juicesync
+JUICESYNC_TAG = v1.0.4-01
+JUICESYNC_DOCKERFILE = ${PROJECT_SOURCE_CODE_DIR}/build/juicesync/Dockerfile
+JUICESYNC_MOUNT_DST_DIR = /go/src/github.com/hwameistor/hwameistor
+
 # [OPERATOR]
 #----------------
 # Operator tools

build/juicesync/Dockerfile.amd64

@@ -0,0 +1,6 @@
+FROM alpine:latest
+
+RUN cd /tmp && wget https://github.com/juicedata/juicesync/releases/download/v1.0.4/juicesync-1.0.4-linux-amd64.tar.gz && tar zxf juicesync-1.0.4-linux-amd64.tar.gz && mv juicesync /usr/bin
+
+COPY build/juicesync/sync_hwameistor_volumes.sh /usr/bin
+RUN chmod +x /usr/bin/sync_hwameistor_volumes.sh

build/juicesync/Dockerfile.arm64

@@ -0,0 +1,6 @@
+FROM alpine:latest
+
+RUN cd /tmp && wget https://github.com/juicedata/juicesync/releases/download/v1.0.4/juicesync-1.0.4-linux-arm64.tar.gz && tar zxf juicesync-1.0.4-linux-arm64.tar.gz && mv juicesync /usr/bin
+
+COPY build/juicesync/sync_hwameistor_volumes.sh /usr/bin
+RUN chmod +x /usr/bin/sync_hwameistor_volumes.sh

build/juicesync/sync_hwameistor_volumes.sh

@@ -0,0 +1 @@
+SSH_PRIVATE_KEY_PATH=/root/.ssh/id_rsa juicesync --links --dirs --perms --force-update root@${sourceNode}:${sourceMountPoint}/ root@${targetNode}:${targetMountPoint}/

cmd/local-storage/storage.go

@@ -32,15 +32,15 @@ const (
 
 	defaultDRBDStartPort      = 43001
 	defaultHAVolumeTotalCount = 1000
-	defaultDataSyncToolName   = datacopy.SyncToolRClone
+	defaultDataSyncToolName   = datacopy.SyncToolJuiceSync
 )
 
 var (
 	nodeName           = flag.String("nodename", "", "Node name")
 	namespace          = flag.String("namespace", "", "Namespace of the Pod")
 	csiSockAddr        = flag.String("csi-address", "", "CSI endpoint")
 	systemMode         = flag.String("system-mode", string(apisv1alpha1.SystemModeDRBD), "dlocal system mode")
-	dataSyncToolName   = flag.String("data-sync-tool", defaultDataSyncToolName, "tool to sync the data across the nodes, e.g. rclone, juicesync")
+	dataSyncToolName   = flag.String("data-sync-tool", defaultDataSyncToolName, "tool to sync the data across the nodes, e.g. juicesync")
 	drbdStartPort      = flag.Int("drbd-start-port", defaultDRBDStartPort, "drbd start port, end port=start-port+volume-count-1")
 	haVolumeTotalCount = flag.Int("max-ha-volume-count", defaultHAVolumeTotalCount, "max HA volume count")
 	httpPort           = flag.Int("http-port", restServerDefaultPort, "HTTP port for REST server")

deploy/crds/hwameistor.io_localvolumemigrates_crd.yaml

@@ -58,7 +58,7 @@ spec:
                 default: false
                 type: boolean
               migrateAllVols:
-                default: false
+                default: true
                 type: boolean
               sourceNode:
                 description: source NodeNames

helm/hwameistor/templates/local-storage.yaml

@@ -58,6 +58,9 @@ spec:
         - --csi-address=$(CSI_ENDPOINT)
         - --http-port=80
         - --v=5
+        {{- if .Values.localStorage.migrate.tool }}
+        - --data-sync-tool={{ .Values.localStorage.migrate.tool }}
+        {{- end }}
         {{- if .Values.localStorage.member.config.drbdStartPort }}
         - --drbd-start-port={{ .Values.localStorage.member.config.drbdStartPort }}
         {{- end }}
@@ -84,9 +87,9 @@ spec:
           value: unix:/{{ template "hwameistor.kubeletRootDir" . }}/plugins/lvm.hwameistor.io/csi.sock
         - name: NODE_ANNOTATION_KEY_STORAGE_IPV4
           value: localstorage.hwameistor.io/storage-ipv4
-        - name: MIGRAGE_RCLONE_IMAGE
-          value: {{ .Values.localStorage.migrate.rclone.imageRepository }}:{{ .Values.localStorage.migrate.rclone.tag }}
-        image: {{ .Values.global.hwameistorImageRegistry}}/{{ .Values.localStorage.member.imageRepository}}:{{ template "hwameistor.localstorageImageTag" . }}
+        - name: MIGRAGE_JUICESYNC_IMAGE
+          value: {{ .Values.global.hwameistorImageRegistry }}/{{ .Values.localStorage.migrate.juicesync.imageRepository }}:{{ .Values.localStorage.migrate.juicesync.tag }}
+        image: {{ .Values.global.hwameistorImageRegistry }}/{{ .Values.localStorage.member.imageRepository }}:{{ template "hwameistor.localstorageImageTag" . }}
         imagePullPolicy: IfNotPresent
         name: member
         ports:
@@ -103,7 +106,7 @@ spec:
           periodSeconds: 2
           successThreshold: 1
           timeoutSeconds: 3
-        resources: 
+        resources:
           {{- toYaml .Values.localStorage.member.resources | nindent 12 }}
         securityContext:
           #  allowPrivilegeEscalation: true

helm/hwameistor/values.yaml

@@ -133,10 +133,9 @@ localStorage:
     tag: ""
     resources: {}
   migrate:
-    rclone:
-      # can use daocloud.io/daocloud/hwameistor-migrate-rclone:v1.1.2 which is exactly same as the following
-      imageRepository: rclone/rclone
-      tag: 1.53.2
+    juicesync:
+      imageRepository: hwameistor/hwameistor-juicesync
+      tag: v1.0.4-01
   hostPaths:
     sshDir: /root/.ssh
     drbdDir: /etc/drbd.d

pkg/apis/hwameistor/v1alpha1/localvolumemigrate_types.go

@@ -25,7 +25,7 @@ type LocalVolumeMigrateSpec struct {
 
 	// *** common section of all the operations ***
 
-	// +kubebuilder:default:=false
+	// +kubebuilder:default:=true
 	MigrateAllVols bool `json:"migrateAllVols,omitempty"`
 
 	// +kubebuilder:default:=false

pkg/local-storage/member/controller/manager.go

@@ -446,10 +446,10 @@ func (m *manager) handlePodUpdateEvent(_, nObj interface{}) {
 	pod, _ := nObj.(*corev1.Pod)
 
 	// this is for the pod orphan pod which is abandoned by migration rclone job
-	m.rclonePodGC(pod)
+	m.gcSyncJobPod(pod)
 }
 
 func (m *manager) handlePodAddEvent(obj interface{}) {
 	pod, _ := obj.(*corev1.Pod)
-	m.rclonePodGC(pod)
+	m.gcSyncJobPod(pod)
 }

pkg/local-storage/member/controller/volume_migrate_task_worker.go

@@ -446,7 +446,7 @@ func (m *manager) getAllVolumesInGroup(lvg *apisv1alpha1.LocalVolumeGroup) ([]*a
 	return vols, nil
 }
 
-func (m *manager) rclonePodGC(pod *corev1.Pod) error {
+func (m *manager) gcSyncJobPod(pod *corev1.Pod) error {
 	if pod.Namespace == m.namespace && pod.Labels["app"] == datacopy.SyncJobLabelApp && len(pod.OwnerReferences) == 0 && pod.Status.Phase == corev1.PodSucceeded {
 		return m.apiClient.Delete(context.TODO(), pod)
 	}

pkg/local-storage/member/node/datacopy_volume_mount_worker.go

@@ -74,13 +74,13 @@ func (m *manager) processSyncVolumeMount(lvName string) error {
 		return err
 	}
 
-	sourceNodeName := cm.Data[datacopy.SyncConfigSrcNodeNameKey]
-	targetNodeName := cm.Data[datacopy.SyncConfigDstNodeNameKey]
+	sourceNodeName := cm.Data[datacopy.SyncConfigSourceNodeNameKey]
+	targetNodeName := cm.Data[datacopy.SyncConfigTargetNodeNameKey]
 	mountPoint := ""
 	if m.name == sourceNodeName {
-		mountPoint = datacopy.SyncSrcMountPoint + lvName
+		mountPoint = datacopy.SyncSourceMountPoint + lvName
 	} else if m.name == targetNodeName {
-		mountPoint = datacopy.SyncDstMountPoint + lvName
+		mountPoint = datacopy.SyncTargetMountPoint + lvName
 	} else {
 		return nil
 	}
@@ -114,11 +114,13 @@ func (m *manager) processSyncVolumeMount(lvName string) error {
 			return nil
 		}
 		cm.Data[datacopy.SyncConfigSourceNodeReadyKey] = datacopy.SyncTrue
+		cm.Data[datacopy.SyncConfigSourceMountPointKey] = mountPoint
 	} else {
-		if cm.Data[datacopy.SyncConfigRemoteNodeReadyKey] == datacopy.SyncTrue {
+		if cm.Data[datacopy.SyncConfigTargetNodeReadyKey] == datacopy.SyncTrue {
 			return nil
 		}
-		cm.Data[datacopy.SyncConfigRemoteNodeReadyKey] = datacopy.SyncTrue
+		cm.Data[datacopy.SyncConfigTargetNodeReadyKey] = datacopy.SyncTrue
+		cm.Data[datacopy.SyncConfigTargetMountPointKey] = mountPoint
 	}
 	if err := m.apiClient.Update(ctx, cm); err != nil {
 		m.logger.WithField("configmap", cm.Name).WithError(err).Error("Failed to update rclone's config")

pkg/local-storage/member/node/manager.go

@@ -5,7 +5,6 @@ import (
 	"fmt"
 	"net"
 	"os"
-	"path/filepath"
 	"strings"
 	"sync"
 
@@ -195,7 +194,7 @@ func (m *manager) isPhysicalNode() bool {
 */
 
 func (m *manager) initForDataSync() {
-	keyFilePath := filepath.Join(datacopy.SyncKeyDir, "authorized_keys")
+	keyFilePath := "/root/.ssh/authorized_keys"
 	if err := utils.TouchFile(keyFilePath); err != nil {
 		m.logger.WithField("file", keyFilePath).WithError(err).Panic("Failed to create a keys file for data copy")
 	}