Skip to content

Commit

Permalink
Support v1 and v2 data engines enablement and disablement independently
Browse files Browse the repository at this point in the history 
Longhorn 7095

Signed-off-by: Derek Su <derek.su@suse.com>
  • Loading branch information
@derekbit
derekbit authored and David Ko committed Dec 28, 2023
1 parent a5bde2f commit bf7f801
Show file tree
Hide file tree
Showing 12 changed files with 163 additions and 137 deletions.
23 changes: 15 additions & 8 deletions controller/monitor/disk_monitor.go
View file
Expand Up @@ -141,15 +141,22 @@ func (m *NodeMonitor) run(value interface{}) error {
}

func (m *NodeMonitor) getBackendStoreDrivers() []longhorn.BackendStoreDriverType {
backendStoreDrivers := []longhorn.BackendStoreDriverType{longhorn.BackendStoreDriverTypeV1}
backendStoreDrivers := []longhorn.BackendStoreDriverType{}

v2DataEngineEnabled, err := m.ds.GetSettingAsBool(types.SettingNameV2DataEngine)
if err != nil {
m.logger.WithError(err).Errorf("Failed to get setting %v", types.SettingNameV2DataEngine)
return backendStoreDrivers
}
if v2DataEngineEnabled {
backendStoreDrivers = append(backendStoreDrivers, longhorn.BackendStoreDriverTypeV2)
for _, setting := range []types.SettingName{types.SettingNameV1DataEngine, types.SettingNameV2DataEngine} {
dataEngineEnabled, err := m.ds.GetSettingAsBool(setting)
if err != nil {
m.logger.WithError(err).Warnf("Failed to get setting %v", setting)
continue
}
if dataEngineEnabled {
switch setting {
case types.SettingNameV1DataEngine:
backendStoreDrivers = append(backendStoreDrivers, longhorn.BackendStoreDriverTypeV1)
case types.SettingNameV2DataEngine:
backendStoreDrivers = append(backendStoreDrivers, longhorn.BackendStoreDriverTypeV2)
}
}
}

return backendStoreDrivers
Expand Down
58 changes: 28 additions & 30 deletions controller/node_controller.go
View file
Expand Up @@ -978,41 +978,39 @@ func (nc *NodeController) syncNodeStatus(pod *corev1.Pod, node *longhorn.Node) e
func (nc *NodeController) getImTypeBackendStoreDrivers(node *longhorn.Node) map[longhorn.InstanceManagerType][]longhorn.BackendStoreDriverType {
log := getLoggerForNode(nc.logger, node)

// Check if v1 data engine is enabled
// TODO: remove InstanceManagerTypeEngine and InstanceManagerTypeReplica in the future.
backendStoreDrivers := map[longhorn.InstanceManagerType][]longhorn.BackendStoreDriverType{
longhorn.InstanceManagerTypeAllInOne: {
longhorn.BackendStoreDriverTypeV1,
},
longhorn.InstanceManagerTypeEngine: {
longhorn.BackendStoreDriverTypeV1,
},
}
if len(node.Spec.Disks) != 0 {
backendStoreDrivers[longhorn.InstanceManagerTypeReplica] = []longhorn.BackendStoreDriverType{
longhorn.BackendStoreDriverTypeV1,
}
}

// Check if v2 data engine is enabled
v2DataEngineEnabled, err := nc.ds.GetSettingAsBool(types.SettingNameV2DataEngine)
if err != nil {
log.WithError(err).Warnf("Failed to get %v setting", types.SettingNameV2DataEngine)
return backendStoreDrivers
longhorn.InstanceManagerTypeAllInOne: {},
longhorn.InstanceManagerTypeEngine: {},
longhorn.InstanceManagerTypeReplica: {},
}

if !v2DataEngineEnabled {
return backendStoreDrivers
}
for _, setting := range []types.SettingName{types.SettingNameV1DataEngine, types.SettingNameV2DataEngine} {
enabled, err := nc.ds.GetSettingAsBool(setting)
if err != nil {
log.WithError(err).Warnf("Failed to get %v setting", setting)
continue
}
if !enabled {
continue
}

err = nc.ds.ValidateV2DataEngine(v2DataEngineEnabled)
if err != nil {
log.WithError(err).Warnf("Failed to validate %v setting", types.SettingNameV2DataEngine)
return backendStoreDrivers
switch setting {
case types.SettingNameV1DataEngine:
backendStoreDrivers[longhorn.InstanceManagerTypeAllInOne] = append(backendStoreDrivers[longhorn.InstanceManagerTypeAllInOne], longhorn.BackendStoreDriverTypeV1)
backendStoreDrivers[longhorn.InstanceManagerTypeEngine] = append(backendStoreDrivers[longhorn.InstanceManagerTypeEngine], longhorn.BackendStoreDriverTypeV1)
if len(node.Spec.Disks) != 0 {
backendStoreDrivers[longhorn.InstanceManagerTypeReplica] = append(backendStoreDrivers[longhorn.InstanceManagerTypeReplica], longhorn.BackendStoreDriverTypeV1)
}
case types.SettingNameV2DataEngine:
if err := nc.ds.ValidateV2DataEngineEnabled(enabled); err == nil {
backendStoreDrivers[longhorn.InstanceManagerTypeAllInOne] = append(backendStoreDrivers[longhorn.InstanceManagerTypeAllInOne], longhorn.BackendStoreDriverTypeV2)
} else {
log.WithError(err).Warnf("Failed to validate %v setting", types.SettingNameV2DataEngine)
}
}
}

backendStoreDrivers[longhorn.InstanceManagerTypeAllInOne] =
append(backendStoreDrivers[longhorn.InstanceManagerTypeAllInOne], longhorn.BackendStoreDriverTypeV2)

return backendStoreDrivers
}

Expand Down Expand Up @@ -1060,7 +1058,7 @@ func (nc *NodeController) syncInstanceManagers(node *longhorn.Node) error {
}
for _, im := range imMap {
if im.Labels[types.GetLonghornLabelKey(types.LonghornLabelNode)] != im.Spec.NodeID {
return fmt.Errorf("instance manager %v NodeID %v is not consistent with the label %v=%v",
return fmt.Errorf("instance manager %v nodeID %v is not consistent with the label %v=%v",
im.Name, im.Spec.NodeID, types.GetLonghornLabelKey(types.LonghornLabelNode), im.Labels[types.GetLonghornLabelKey(types.LonghornLabelNode)])
}
cleanupRequired := true
Expand Down
44 changes: 29 additions & 15 deletions controller/setting_controller.go
View file
Expand Up @@ -259,8 +259,8 @@ func (sc *SettingController) syncSetting(key string) (err error) {
if err := sc.updateLogLevel(); err != nil {
return err
}
case string(types.SettingNameV2DataEngine):
if err := sc.updateV2DataEngine(); err != nil {
case string(types.SettingNameV1DataEngine), string(types.SettingNameV2DataEngine):
if err := sc.updateDataEngine(types.SettingName(name)); err != nil {
return err
}
}
Expand Down Expand Up @@ -752,7 +752,7 @@ func (sc *SettingController) updateCNI() error {
return err
}

volumesDetached, err := sc.ds.AreAllVolumesDetached()
volumesDetached, err := sc.ds.AreAllVolumesDetached(longhorn.BackendStoreDriverTypeAll)
if err != nil {
return errors.Wrapf(err, "failed to check volume detachment for %v setting update", types.SettingNameStorageNetwork)
}
Expand Down Expand Up @@ -804,36 +804,49 @@ func (sc *SettingController) updateLogLevel() error {
return nil
}

func (sc *SettingController) updateV2DataEngine() error {
v2DataEngineEnabled, err := sc.ds.GetSettingAsBool(types.SettingNameV2DataEngine)
func (sc *SettingController) updateDataEngine(setting types.SettingName) error {
enabled, err := sc.ds.GetSettingAsBool(setting)
if err != nil {
return err
return errors.Wrapf(err, "failed to get %v setting for updating data engine", setting)
}

if err := sc.ds.ValidateV2DataEngine(v2DataEngineEnabled); err != nil {
return err
var backednStoreDriver longhorn.BackendStoreDriverType
switch setting {
case types.SettingNameV1DataEngine:
backednStoreDriver = longhorn.BackendStoreDriverTypeV1
if err := sc.ds.ValidateV1DataEngineEnabled(enabled); err != nil {
return err
}
case types.SettingNameV2DataEngine:
backednStoreDriver = longhorn.BackendStoreDriverTypeV2
if err := sc.ds.ValidateV2DataEngineEnabled(enabled); err != nil {
return err
}
default:
return fmt.Errorf("unknown setting %v for updating data engine", setting)
}

if !v2DataEngineEnabled {
return sc.cleanupInstanceManagerForV2DataEngine()
if !enabled {
return sc.cleanupInstanceManager(backednStoreDriver)
}

return nil
}

func (sc *SettingController) cleanupInstanceManagerForV2DataEngine() error {
imMap, err := sc.ds.ListInstanceManagersBySelectorRO("", "", longhorn.InstanceManagerTypeAllInOne, longhorn.BackendStoreDriverTypeV2)
func (sc *SettingController) cleanupInstanceManager(backendStoreDriver longhorn.BackendStoreDriverType) error {
sc.logger.Infof("Cleaning up the instance manager for %v data engine", backendStoreDriver)
imMap, err := sc.ds.ListInstanceManagersBySelectorRO("", "", longhorn.InstanceManagerTypeAllInOne, backendStoreDriver)
if err != nil {
return err
return errors.Wrapf(err, "failed to list instance managers for cleaning up %v data engine", backendStoreDriver)
}

for _, im := range imMap {
if len(im.Status.InstanceEngines) != 0 || len(im.Status.InstanceReplicas) != 0 || len(im.Status.Instances) != 0 {
sc.logger.Infof("Skipping cleaning up the instance manager %v for v2 data engine since there are still instances running on it", im.Name)
sc.logger.Infof("Skipping cleaning up the instance manager %v for %v data engine since there are still instances running on it", im.Name, backendStoreDriver)
continue
}

sc.logger.Infof("Cleaning up the instance manager %v for v2 data engine", im.Name)
sc.logger.Infof("Cleaning up the instance manager %v for %v data engine", im.Name, backendStoreDriver)
if err := sc.ds.DeleteInstanceManager(im.Name); err != nil {
return err
}
Expand Down Expand Up @@ -1473,6 +1486,7 @@ func (info *ClusterInfo) collectSettings() error {
types.SettingNameStorageReservedPercentageForDefaultDisk: true,
types.SettingNameSupportBundleFailedHistoryLimit: true,
types.SettingNameSystemManagedPodsImagePullPolicy: true,
types.SettingNameV1DataEngine: true,
types.SettingNameV2DataEngine: true,
types.SettingNameV2DataEngineGuaranteedInstanceManagerCPU: true,
types.SettingNameOfflineReplicaRebuilding: true,
Expand Down
88 changes: 46 additions & 42 deletions datastore/longhorn.go
View file
Expand Up @@ -358,24 +358,39 @@ func (s *DataStore) ValidateSetting(name, value string) (err error) {
return err
}
case types.SettingNameStorageNetwork:
volumesDetached, err := s.AreAllVolumesDetached()
volumesDetached, err := s.AreAllVolumesDetached(longhorn.BackendStoreDriverTypeAll)
if err != nil {
return errors.Wrapf(err, "failed to check volume detachment for %v setting update", name)
}
if !volumesDetached {
return &types.ErrorInvalidState{Reason: fmt.Sprintf("cannot apply %v setting to Longhorn workloads when there are attached volumes", name)}
}
case types.SettingNameV1DataEngine:
old, err := s.GetSettingWithAutoFillingRO(types.SettingNameV1DataEngine)
if err != nil {
return err
}
if old.Value != value {
dataEngineEnabled, err := strconv.ParseBool(value)
if err != nil {
return err
}
err = s.ValidateV1DataEngineEnabled(dataEngineEnabled)
if err != nil {
return err
}
}
case types.SettingNameV2DataEngine:
old, err := s.GetSettingWithAutoFillingRO(types.SettingNameV2DataEngine)
if err != nil {
return err
}
if old.Value != value {
v2DataEngineEnabled, err := strconv.ParseBool(value)
dataEngineEnabled, err := strconv.ParseBool(value)
if err != nil {
return err
}
err = s.ValidateV2DataEngine(v2DataEngineEnabled)
err = s.ValidateV2DataEngineEnabled(dataEngineEnabled)
if err != nil {
return err
}
Expand All @@ -400,21 +415,35 @@ func (s *DataStore) ValidateSetting(name, value string) (err error) {
return nil
}

func (s *DataStore) ValidateV2DataEngine(v2DataEngineEnabled bool) error {
if !v2DataEngineEnabled {
allV2VolumesDetached, err := s.AreAllV2VolumesDetached()
func (s *DataStore) ValidateV1DataEngineEnabled(dataEngineEnabled bool) error {
if !dataEngineEnabled {
allVolumesDetached, err := s.AreAllVolumesDetached(longhorn.BackendStoreDriverTypeV1)
if err != nil {
return errors.Wrapf(err, "failed to check volume detachment for %v setting update", types.SettingNameV1DataEngine)
}
if !allVolumesDetached {
return &types.ErrorInvalidState{Reason: fmt.Sprintf("cannot apply %v setting to Longhorn workloads when there are attached v1 volumes", types.SettingNameV1DataEngine)}
}
}

return nil
}

func (s *DataStore) ValidateV2DataEngineEnabled(dataEngineEnabled bool) error {
if !dataEngineEnabled {
allVolumesDetached, err := s.AreAllVolumesDetached(longhorn.BackendStoreDriverTypeV2)
if err != nil {
return errors.Wrapf(err, "failed to check volume detachment for %v setting update", types.SettingNameV2DataEngine)
}
if !allV2VolumesDetached {
return &types.ErrorInvalidState{Reason: fmt.Sprintf("cannot apply %v setting to Longhorn workloads when there are attached volumes", types.SettingNameV2DataEngine)}
if !allVolumesDetached {
return &types.ErrorInvalidState{Reason: fmt.Sprintf("cannot apply %v setting to Longhorn workloads when there are attached v2 volumes", types.SettingNameV2DataEngine)}
}

allBlockTypeDisksRemoved, err := s.AreAllBlockTypeDisksRemoved()
allDisksRemoved, err := s.AreAllDisksRemovedByDiskType(longhorn.DiskTypeBlock)
if err != nil {
return errors.Wrapf(err, "failed to check block-type disk removal for %v setting update", types.SettingNameV2DataEngine)
}
if !allBlockTypeDisksRemoved {
if !allDisksRemoved {
return &types.ErrorInvalidState{Reason: fmt.Sprintf("cannot apply %v setting to Longhorn workloads when there are block-type disks", types.SettingNameV2DataEngine)}
}
return nil
Expand All @@ -441,7 +470,7 @@ func (s *DataStore) ValidateV2DataEngine(v2DataEngineEnabled bool) error {
continue
}

if v2DataEngineEnabled {
if dataEngineEnabled {
capacity, ok := node.Status.Capacity["hugepages-2Mi"]
if !ok {
return errors.Errorf("failed to get hugepages-2Mi capacity for node %v", node.Name)
Expand Down Expand Up @@ -469,19 +498,19 @@ func (s *DataStore) ValidateV2DataEngine(v2DataEngineEnabled bool) error {
return nil
}

func (s *DataStore) AreAllVolumesDetached() (bool, error) {
func (s *DataStore) AreAllVolumesDetached(backendStoreDriver longhorn.BackendStoreDriverType) (bool, error) {
nodes, err := s.ListNodes()
if err != nil {
return false, err
}

for node := range nodes {
engineInstanceManagers, err := s.ListInstanceManagersBySelectorRO(node, "", longhorn.InstanceManagerTypeEngine, longhorn.BackendStoreDriverTypeV1)

engineInstanceManagers, err := s.ListInstanceManagersBySelectorRO(node, "", longhorn.InstanceManagerTypeEngine, backendStoreDriver)
if err != nil && !ErrorIsNotFound(err) {
return false, err
}

aioInstanceManagers, err := s.ListInstanceManagersBySelectorRO(node, "", longhorn.InstanceManagerTypeAllInOne, "")
aioInstanceManagers, err := s.ListInstanceManagersBySelectorRO(node, "", longhorn.InstanceManagerTypeAllInOne, backendStoreDriver)
if err != nil && !ErrorIsNotFound(err) {
return false, err
}
Expand All @@ -493,41 +522,18 @@ func (s *DataStore) AreAllVolumesDetached() (bool, error) {
}
}
}

return true, nil
}

func (s *DataStore) AreAllV2VolumesDetached() (bool, error) {
nodes, err := s.ListNodesRO()
if err != nil {
return false, err
}

for _, node := range nodes {
aioInstanceManagers, err := s.ListInstanceManagersBySelectorRO(node.Name, "", longhorn.InstanceManagerTypeAllInOne, longhorn.BackendStoreDriverTypeV2)
if err != nil && !ErrorIsNotFound(err) {
return false, err
}

for _, instanceManager := range aioInstanceManagers {
if len(instanceManager.Status.InstanceEngines)+len(instanceManager.Status.Instances) > 0 {
return false, nil
}
}
}

return true, nil
}

func (s *DataStore) AreAllBlockTypeDisksRemoved() (bool, error) {
func (s *DataStore) AreAllDisksRemovedByDiskType(diskType longhorn.DiskType) (bool, error) {
nodes, err := s.ListNodesRO()
if err != nil {
return false, err
}

for _, node := range nodes {
for _, disk := range node.Spec.Disks {
if disk.Type == longhorn.DiskTypeBlock {
if disk.Type == diskType {
return false, nil
}
}
Expand Down Expand Up @@ -1747,11 +1753,9 @@ func (s *DataStore) CheckEngineImageCompatiblityByImage(image string) error {
// CheckEngineImageReadiness return true if the engine IMAGE is deployed on all nodes in the NODES list
func (s *DataStore) CheckEngineImageReadiness(image string, nodes ...string) (isReady bool, err error) {
if len(nodes) == 0 {
logrus.Warnf("CheckEngineImageReadiness: no nodes specified")
return false, nil
}
if len(nodes) == 1 && nodes[0] == "" {
logrus.Warnf("CheckEngineImageReadiness: only one node specified and it's empty")
return false, nil
}
ei, err := s.GetEngineImageRO(types.GetEngineImageChecksumName(image))
Expand Down
5 changes: 3 additions & 2 deletions k8s/pkg/apis/longhorn/v1beta2/volume.go
View file
Expand Up @@ -163,8 +163,9 @@ const (
type BackendStoreDriverType string

const (
BackendStoreDriverTypeV1 = BackendStoreDriverType("v1")
BackendStoreDriverTypeV2 = BackendStoreDriverType("v2")
BackendStoreDriverTypeV1 = BackendStoreDriverType("v1")
BackendStoreDriverTypeV2 = BackendStoreDriverType("v2")
BackendStoreDriverTypeAll = BackendStoreDriverType("all")
)

type OfflineReplicaRebuilding string
Expand Down

0 comments on commit bf7f801

Please sign in to comment.