enhance: Add config for auto upgrade segment index

Signed-off-by: Wei Liu <wei.liu@zilliz.com>

internal/datacoord/compaction_trigger.go

@@ -906,17 +906,19 @@ func (t *compactionTrigger) ShouldDoSingleCompaction(segment *SegmentInfo, isDis
 		return true
 	}
 
-	// index version of segment lower than current version and IndexFileKeys should have value, trigger compaction
-	for _, index := range segment.segmentIndexes {
-		if index.CurrentIndexVersion < t.indexEngineVersionManager.GetCurrentIndexEngineVersion() &&
-			len(index.IndexFileKeys) > 0 {
-			log.Info("index version is too old, trigger compaction",
-				zap.Int64("segmentID", segment.ID),
-				zap.Int64("indexID", index.IndexID),
-				zap.Strings("indexFileKeys", index.IndexFileKeys),
-				zap.Int32("currentIndexVersion", index.CurrentIndexVersion),
-				zap.Int32("currentEngineVersion", t.indexEngineVersionManager.GetCurrentIndexEngineVersion()))
-			return true
+	if Params.DataCoordCfg.AutoUpgradeSegmentIndex.GetAsBool() {
+		// index version of segment lower than current version and IndexFileKeys should have value, trigger compaction
+		for _, index := range segment.segmentIndexes {
+			if index.CurrentIndexVersion < t.indexEngineVersionManager.GetCurrentIndexEngineVersion() &&
+				len(index.IndexFileKeys) > 0 {
+				log.Info("index version is too old, trigger compaction",
+					zap.Int64("segmentID", segment.ID),
+					zap.Int64("indexID", index.IndexID),
+					zap.Strings("indexFileKeys", index.IndexFileKeys),
+					zap.Int32("currentIndexVersion", index.CurrentIndexVersion),
+					zap.Int32("currentEngineVersion", t.indexEngineVersionManager.GetCurrentIndexEngineVersion()))
+				return true
+			}
 		}
 	}
 

internal/datacoord/compaction_trigger_test.go

@@ -1890,6 +1890,7 @@ func Test_compactionTrigger_shouldDoSingleCompaction(t *testing.T) {
 	}
 
 	// expire time < Timestamp To, but index engine version is 2 which is larger than CurrentIndexVersion in segmentIndex
+	Params.Save(Params.DataCoordCfg.AutoUpgradeSegmentIndex.Key, "true")
 	couldDo = trigger.ShouldDoSingleCompaction(info4, false, &compactTime{expireTime: 300})
 	assert.True(t, couldDo)
 	// expire time < Timestamp To, and index engine version is 2 which is equal CurrentIndexVersion in segmentIndex

pkg/util/paramtable/component_param.go

@@ -2094,6 +2094,7 @@ type dataCoordConfig struct {
 	SegmentMaxIdleTime             ParamItem `refreshable:"false"`
 	SegmentMinSizeFromIdleToSealed ParamItem `refreshable:"false"`
 	SegmentMaxBinlogFileNumber     ParamItem `refreshable:"false"`
+	AutoUpgradeSegmentIndex        ParamItem `refreshable:"true"`
 
 	// compaction
 	EnableCompaction     ParamItem `refreshable:"false"`
@@ -2550,6 +2551,16 @@ During compaction, the size of segment # of rows is able to exceed segment max #
 		Export:       true,
 	}
 	p.CheckAutoBalanceConfigInterval.Init(base.mgr)
+
+	p.AutoUpgradeSegmentIndex = ParamItem{
+		Key:          "dataCoord.autoUpgradeSegmentIndex",
+		Version:      "2.3.4",
+		DefaultValue: "false",
+		PanicIfEmpty: true,
+		Doc:          "whether auto upgrade segment index to index engine's version",
+		Export:       true,
+	}
+	p.AutoUpgradeSegmentIndex.Init(base.mgr)
 }
 
 // /////////////////////////////////////////////////////////////////////////////

pkg/util/paramtable/component_param_test.go

@@ -356,6 +356,7 @@ func TestComponentParam(t *testing.T) {
 
 		assert.Equal(t, false, Params.AutoBalance.GetAsBool())
 		assert.Equal(t, 10, Params.CheckAutoBalanceConfigInterval.GetAsInt())
+		assert.Equal(t, false, Params.AutoUpgradeSegmentIndex.GetAsBool())
 	})
 
 	t.Run("test dataNodeConfig", func(t *testing.T) {