feat(mount): request remount when volume mount becomes read only

ref: longhorn/longhorn 6386 Signed-off-by: Jack Lin <jack.lin@suse.com>
longhorn · Dec 26, 2023 · 26e4f39 · 26e4f39
1 parent 8cfda3e
commit 26e4f39
Show file tree

Hide file tree

Showing 8 changed files with 65 additions and 14 deletions.
diff --git a/controller/instance_handler.go b/controller/instance_handler.go
@@ -171,6 +171,14 @@ func (h *InstanceHandler) syncStatusWithInstanceManager(im *longhorn.InstanceMan
 		if status.CurrentImage == "" {
 			status.CurrentImage = spec.Image
 		}
+		for condition, flag := range instance.Status.Conditions {
+			conditionStatus := longhorn.ConditionStatusFalse
+			if flag {
+				conditionStatus = longhorn.ConditionStatusTrue
+			}
+			status.Conditions = types.SetCondition(status.Conditions, condition, conditionStatus, "", "")
+		}
+
 	case longhorn.InstanceStateStopping:
 		if status.Started {
 			status.CurrentState = longhorn.InstanceStateError

diff --git a/controller/volume_controller.go b/controller/volume_controller.go
@@ -29,6 +29,7 @@ import (
 
 	"github.com/longhorn/backupstore"
 
+	imtypes "github.com/longhorn/longhorn-instance-manager/pkg/types"
 	imutil "github.com/longhorn/longhorn-instance-manager/pkg/util"
 
 	"github.com/longhorn/longhorn-manager/constant"
@@ -1380,6 +1381,9 @@ func (c *VolumeController) ReconcileVolumeState(v *longhorn.Volume, es map[strin
 		}
 	}
 
+	// check volume mount status
+	c.requestRemountIfFileSystemReadOnly(v, e)
+
 	if err := c.reconcileAttachDetachStateMachine(v, e, rs, isNewVolume, log); err != nil {
 		return err
 	}
@@ -1403,6 +1407,20 @@ func (c *VolumeController) ReconcileVolumeState(v *longhorn.Volume, es map[strin
 	return c.checkAndFinishVolumeRestore(v, e, rs)
 }
 
+func (c *VolumeController) requestRemountIfFileSystemReadOnly(v *longhorn.Volume, e *longhorn.Engine) {
+	log := getLoggerForVolume(c.logger, v)
+	if v.Status.State == longhorn.VolumeStateAttached && e.Status.CurrentState == longhorn.InstanceStateRunning {
+		fileSystemReadOnlyCondition := types.GetCondition(e.Status.Conditions, imtypes.EngineConditionFilesystemReadOnly)
+
+		if fileSystemReadOnlyCondition.Status == longhorn.ConditionStatusTrue {
+			v.Status.RemountRequestedAt = c.nowHandler()
+			log.Infof("Volume request remount at %v due to engine detected read-only filesystem", v.Status.RemountRequestedAt)
+			msg := fmt.Sprintf("Volume %s requested remount at %v due to engine detected read-only filesystem", v.Name, v.Status.RemountRequestedAt)
+			c.eventRecorder.Eventf(v, corev1.EventTypeNormal, constant.EventReasonRemount, msg)
+		}
+	}
+}
+
 func (c *VolumeController) reconcileAttachDetachStateMachine(v *longhorn.Volume, e *longhorn.Engine, rs map[string]*longhorn.Replica, isNewVolume bool, log *logrus.Entry) error {
 	// TODO: link the state machine graph here
 

diff --git a/engineapi/instance_manager.go b/engineapi/instance_manager.go
@@ -250,11 +250,12 @@ func parseInstance(p *imapi.Instance) *longhorn.InstanceProcess {
 			BackendStoreDriver: longhorn.BackendStoreDriverType(p.BackendStoreDriver),
 		},
 		Status: longhorn.InstanceProcessStatus{
-			Type:      getTypeForInstance(longhorn.InstanceType(p.Type), p.PortCount),
-			State:     longhorn.InstanceState(p.InstanceStatus.State),
-			ErrorMsg:  p.InstanceStatus.ErrorMsg,
-			PortStart: p.InstanceStatus.PortStart,
-			PortEnd:   p.InstanceStatus.PortEnd,
+			Type:       getTypeForInstance(longhorn.InstanceType(p.Type), p.PortCount),
+			State:      longhorn.InstanceState(p.InstanceStatus.State),
+			ErrorMsg:   p.InstanceStatus.ErrorMsg,
+			Conditions: p.InstanceStatus.Conditions,
+			PortStart:  p.InstanceStatus.PortStart,
+			PortEnd:    p.InstanceStatus.PortEnd,
 
 			// These fields are not used, maybe we can deprecate them later.
 			Listen:   "",
@@ -274,11 +275,12 @@ func parseProcess(p *imapi.Process) *longhorn.InstanceProcess {
 			BackendStoreDriver: longhorn.BackendStoreDriverTypeV1,
 		},
 		Status: longhorn.InstanceProcessStatus{
-			Type:      getTypeForProcess(p.PortCount),
-			State:     longhorn.InstanceState(p.ProcessStatus.State),
-			ErrorMsg:  p.ProcessStatus.ErrorMsg,
-			PortStart: p.ProcessStatus.PortStart,
-			PortEnd:   p.ProcessStatus.PortEnd,
+			Type:       getTypeForProcess(p.PortCount),
+			State:      longhorn.InstanceState(p.ProcessStatus.State),
+			ErrorMsg:   p.ProcessStatus.ErrorMsg,
+			Conditions: p.ProcessStatus.Conditions,
+			PortStart:  p.ProcessStatus.PortStart,
+			PortEnd:    p.ProcessStatus.PortEnd,
 		},
 	}
 }

diff --git a/go.mod b/go.mod
@@ -3,6 +3,7 @@ module github.com/longhorn/longhorn-manager
 go 1.21
 
 replace (
+	github.com/longhorn/longhorn-instance-manager v1.4.0-rc1.0.20231117031020-055db7ef0240 => github.com/ChanYiLin/longhorn-instance-manager v1.4.0-rc1.0.20231124094442-38f78aca1512
 	k8s.io/api => k8s.io/api v0.28.2
 	k8s.io/apiextensions-apiserver => k8s.io/apiextensions-apiserver v0.28.2
 	k8s.io/apimachinery => k8s.io/apimachinery v0.28.2

diff --git a/k8s/crds.yaml b/k8s/crds.yaml
@@ -1694,6 +1694,10 @@ spec:
                       type: object
                     status:
                       properties:
+                        conditions:
+                          additionalProperties:
+                            type: boolean
+                          type: object
                         endpoint:
                           type: string
                         errorMsg:
@@ -1729,6 +1733,10 @@ spec:
                       type: object
                     status:
                       properties:
+                        conditions:
+                          additionalProperties:
+                            type: boolean
+                          type: object
                         endpoint:
                           type: string
                         errorMsg:
@@ -1764,6 +1772,10 @@ spec:
                       type: object
                     status:
                       properties:
+                        conditions:
+                          additionalProperties:
+                            type: boolean
+                          type: object
                         endpoint:
                           type: string
                         errorMsg:

diff --git a/k8s/pkg/apis/longhorn/v1beta2/instancemanager.go b/k8s/pkg/apis/longhorn/v1beta2/instancemanager.go
@@ -117,6 +117,8 @@ type InstanceProcessStatus struct {
 	Endpoint string `json:"endpoint"`
 	// +optional
 	ErrorMsg string `json:"errorMsg"`
+	//+optional
+	Conditions map[string]bool `json:"conditions"`
 	// +optional
 	Listen string `json:"listen"`
 	// +optional

diff --git a/k8s/pkg/apis/longhorn/v1beta2/zz_generated.deepcopy.go b/k8s/pkg/apis/longhorn/v1beta2/zz_generated.deepcopy.go
diff --git a/vendor/modules.txt b/vendor/modules.txt
@@ -1548,6 +1548,7 @@ sigs.k8s.io/structured-merge-diff/v4/value
 # sigs.k8s.io/yaml v1.3.0
 ## explicit; go 1.12
 sigs.k8s.io/yaml
+# github.com/longhorn/longhorn-instance-manager v1.4.0-rc1.0.20231117031020-055db7ef0240 => github.com/ChanYiLin/longhorn-instance-manager v1.4.0-rc1.0.20231124094442-38f78aca1512
 # k8s.io/api => k8s.io/api v0.28.2
 # k8s.io/apiextensions-apiserver => k8s.io/apiextensions-apiserver v0.28.2
 # k8s.io/apimachinery => k8s.io/apimachinery v0.28.2