fix: fix bitmap supported type

Signed-off-by: luzhang <luzhang@zilliz.com>
milvus-io · Jul 2, 2024 · c9e42bc · c9e42bc
1 parent 96dcee5
commit c9e42bc
Show file tree

Hide file tree

Showing 27 changed files with 63 additions and 45 deletions.
diff --git a/internal/proxy/task_index.go b/internal/proxy/task_index.go
@@ -389,7 +389,7 @@ func checkTrain(field *schemapb.FieldSchema, indexParams map[string]string) erro
 		}
 	}
 
-	if err := checker.CheckValidDataType(field.GetDataType()); err != nil {
+	if err := checker.CheckValidDataType(field.GetDataType(), field.GetElementType()); err != nil {
 		log.Info("create index with invalid data type", zap.Error(err), zap.String("data_type", field.GetDataType().String()))
 		return err
 	}

diff --git a/pkg/util/indexparamcheck/auto_index_checker.go b/pkg/util/indexparamcheck/auto_index_checker.go
@@ -13,7 +13,7 @@ func (c *AUTOINDEXChecker) CheckTrain(params map[string]string) error {
 	return nil
 }
 
-func (c *AUTOINDEXChecker) CheckValidDataType(dType schemapb.DataType) error {
+func (c *AUTOINDEXChecker) CheckValidDataType(dType schemapb.DataType, elemType schemapb.DataType) error {
 	return nil
 }
 

diff --git a/pkg/util/indexparamcheck/base_checker.go b/pkg/util/indexparamcheck/base_checker.go
@@ -36,7 +36,7 @@ func (c baseChecker) CheckTrain(params map[string]string) error {
 }
 
 // CheckValidDataType check whether the field data type is supported for the index type
-func (c baseChecker) CheckValidDataType(dType schemapb.DataType) error {
+func (c baseChecker) CheckValidDataType(dType schemapb.DataType, elemType schemapb.DataType) error {
 	return nil
 }
 

diff --git a/pkg/util/indexparamcheck/base_checker_test.go b/pkg/util/indexparamcheck/base_checker_test.go
@@ -98,7 +98,7 @@ func Test_baseChecker_CheckValidDataType(t *testing.T) {
 
 	c := newBaseChecker()
 	for _, test := range cases {
-		err := c.CheckValidDataType(test.dType)
+		err := c.CheckValidDataType(test.dType, schemapb.DataType_None)
 		if test.errIsNil {
 			assert.NoError(t, err)
 		} else {

diff --git a/pkg/util/indexparamcheck/bin_flat_checker_test.go b/pkg/util/indexparamcheck/bin_flat_checker_test.go
@@ -136,7 +136,7 @@ func Test_binFlatChecker_CheckValidDataType(t *testing.T) {
 
 	c := newBinFlatChecker()
 	for _, test := range cases {
-		err := c.CheckValidDataType(test.dType)
+		err := c.CheckValidDataType(test.dType, schemapb.DataType_None)
 		if test.errIsNil {
 			assert.NoError(t, err)
 		} else {

diff --git a/pkg/util/indexparamcheck/bin_ivf_flat_checker_test.go b/pkg/util/indexparamcheck/bin_ivf_flat_checker_test.go
@@ -187,7 +187,7 @@ func Test_binIVFFlatChecker_CheckValidDataType(t *testing.T) {
 
 	c := newBinIVFFlatChecker()
 	for _, test := range cases {
-		err := c.CheckValidDataType(test.dType)
+		err := c.CheckValidDataType(test.dType, schemapb.DataType_None)
 		if test.errIsNil {
 			assert.NoError(t, err)
 		} else {

diff --git a/pkg/util/indexparamcheck/binary_vector_base_checker.go b/pkg/util/indexparamcheck/binary_vector_base_checker.go
@@ -27,7 +27,7 @@ func (c binaryVectorBaseChecker) CheckTrain(params map[string]string) error {
 	return c.staticCheck(params)
 }
 
-func (c binaryVectorBaseChecker) CheckValidDataType(dType schemapb.DataType) error {
+func (c binaryVectorBaseChecker) CheckValidDataType(dType schemapb.DataType, elemType schemapb.DataType) error {
 	if dType != schemapb.DataType_BinaryVector {
 		return fmt.Errorf("binary vector is only supported")
 	}

diff --git a/pkg/util/indexparamcheck/bitmap_checker_test.go b/pkg/util/indexparamcheck/bitmap_checker_test.go
@@ -13,12 +13,25 @@ func Test_BitmapIndexChecker(t *testing.T) {
 
 	assert.NoError(t, c.CheckTrain(map[string]string{"bitmap_cardinality_limit": "100"}))
 
-	assert.NoError(t, c.CheckValidDataType(schemapb.DataType_Int64))
-	assert.NoError(t, c.CheckValidDataType(schemapb.DataType_Float))
-	assert.NoError(t, c.CheckValidDataType(schemapb.DataType_String))
-	assert.NoError(t, c.CheckValidDataType(schemapb.DataType_Array))
+	assert.NoError(t, c.CheckValidDataType(schemapb.DataType_Bool, schemapb.DataType_None))
+	assert.NoError(t, c.CheckValidDataType(schemapb.DataType_Int8, schemapb.DataType_None))
+	assert.NoError(t, c.CheckValidDataType(schemapb.DataType_Int16, schemapb.DataType_None))
+	assert.NoError(t, c.CheckValidDataType(schemapb.DataType_Int32, schemapb.DataType_None))
+	assert.NoError(t, c.CheckValidDataType(schemapb.DataType_Int64, schemapb.DataType_None))
+	assert.NoError(t, c.CheckValidDataType(schemapb.DataType_Float, schemapb.DataType_None))
+	assert.NoError(t, c.CheckValidDataType(schemapb.DataType_String, schemapb.DataType_None))
+	assert.NoError(t, c.CheckValidDataType(schemapb.DataType_Array, schemapb.DataType_Bool))
+	assert.NoError(t, c.CheckValidDataType(schemapb.DataType_Array, schemapb.DataType_Int8))
+	assert.NoError(t, c.CheckValidDataType(schemapb.DataType_Array, schemapb.DataType_Int16))
+	assert.NoError(t, c.CheckValidDataType(schemapb.DataType_Array, schemapb.DataType_Int32))
+	assert.NoError(t, c.CheckValidDataType(schemapb.DataType_Array, schemapb.DataType_Int64))
+	assert.NoError(t, c.CheckValidDataType(schemapb.DataType_Array, schemapb.DataType_String))
 
-	assert.Error(t, c.CheckValidDataType(schemapb.DataType_JSON))
+	assert.Error(t, c.CheckValidDataType(schemapb.DataType_JSON, schemapb.DataType_None))
+	assert.Error(t, c.CheckValidDataType(schemapb.DataType_Float, schemapb.DataType_None))
+	assert.Error(t, c.CheckValidDataType(schemapb.DataType_Double, schemapb.DataType_None))
+	assert.Error(t, c.CheckValidDataType(schemapb.DataType_Array, schemapb.DataType_Float))
+	assert.Error(t, c.CheckValidDataType(schemapb.DataType_Array, schemapb.DataType_Double))
 	assert.Error(t, c.CheckTrain(map[string]string{}))
 	assert.Error(t, c.CheckTrain(map[string]string{"bitmap_cardinality_limit": "0"}))
 }
diff --git a/pkg/util/indexparamcheck/bitmap_index_checker.go b/pkg/util/indexparamcheck/bitmap_index_checker.go
@@ -20,9 +20,14 @@ func (c *BITMAPChecker) CheckTrain(params map[string]string) error {
 	return c.scalarIndexChecker.CheckTrain(params)
 }
 
-func (c *BITMAPChecker) CheckValidDataType(dType schemapb.DataType) error {
-	if !typeutil.IsArithmetic(dType) && !typeutil.IsStringType(dType) && !typeutil.IsArrayType(dType) {
-		return fmt.Errorf("bitmap index are only supported on numeric, string and array field")
+func (c *BITMAPChecker) CheckValidDataType(dType schemapb.DataType, elemType schemapb.DataType) error {
+	if !typeutil.IsBoolType(dType) && !typeutil.IsIntegerType(dType) && !typeutil.IsStringType(dType) && !typeutil.IsArrayType(dType) {
+		return fmt.Errorf("bitmap index are only supported on bool, int, string and array field")
+	}
+	if typeutil.IsArrayType(dType) {
+		if !typeutil.IsBoolType(elemType) && !typeutil.IsIntegerType(elemType) && !typeutil.IsStringType(elemType) {
+			return fmt.Errorf("bitmap index are only supported on bool, int, string for array field")
+		}
 	}
 	return nil
 }

diff --git a/pkg/util/indexparamcheck/diskann_checker_test.go b/pkg/util/indexparamcheck/diskann_checker_test.go
@@ -144,7 +144,7 @@ func Test_diskannChecker_CheckValidDataType(t *testing.T) {
 
 	c := newDiskannChecker()
 	for _, test := range cases {
-		err := c.CheckValidDataType(test.dType)
+		err := c.CheckValidDataType(test.dType, schemapb.DataType_None)
 		if test.errIsNil {
 			assert.NoError(t, err)
 		} else {

diff --git a/pkg/util/indexparamcheck/float_vector_base_checker.go b/pkg/util/indexparamcheck/float_vector_base_checker.go
@@ -28,7 +28,7 @@ func (c floatVectorBaseChecker) CheckTrain(params map[string]string) error {
 	return c.staticCheck(params)
 }
 
-func (c floatVectorBaseChecker) CheckValidDataType(dType schemapb.DataType) error {
+func (c floatVectorBaseChecker) CheckValidDataType(dType schemapb.DataType, elemType schemapb.DataType) error {
 	if !typeutil.IsDenseFloatVectorType(dType) {
 		return fmt.Errorf("data type should be FloatVector, Float16Vector or BFloat16Vector")
 	}

diff --git a/pkg/util/indexparamcheck/float_vector_base_checker_test.go b/pkg/util/indexparamcheck/float_vector_base_checker_test.go
@@ -69,7 +69,7 @@ func Test_floatVectorBaseChecker_CheckValidDataType(t *testing.T) {
 
 	c := newFloatVectorBaseChecker()
 	for _, test := range cases {
-		err := c.CheckValidDataType(test.dType)
+		err := c.CheckValidDataType(test.dType, schemapb.DataType_None)
 		if test.errIsNil {
 			assert.NoError(t, err)
 		} else {

diff --git a/pkg/util/indexparamcheck/hnsw_checker.go b/pkg/util/indexparamcheck/hnsw_checker.go
@@ -32,7 +32,7 @@ func (c hnswChecker) CheckTrain(params map[string]string) error {
 	return c.baseChecker.CheckTrain(params)
 }
 
-func (c hnswChecker) CheckValidDataType(dType schemapb.DataType) error {
+func (c hnswChecker) CheckValidDataType(dType schemapb.DataType, elemType schemapb.DataType) error {
 	if !typeutil.IsVectorType(dType) {
 		return fmt.Errorf("can't build hnsw in not vector type.")
 	}

diff --git a/pkg/util/indexparamcheck/hnsw_checker_test.go b/pkg/util/indexparamcheck/hnsw_checker_test.go
@@ -164,7 +164,7 @@ func Test_hnswChecker_CheckValidDataType(t *testing.T) {
 
 	c := newHnswChecker()
 	for _, test := range cases {
-		err := c.CheckValidDataType(test.dType)
+		err := c.CheckValidDataType(test.dType, schemapb.DataType_None)
 		if test.errIsNil {
 			assert.NoError(t, err)
 		} else {

diff --git a/pkg/util/indexparamcheck/index_checker.go b/pkg/util/indexparamcheck/index_checker.go
@@ -22,7 +22,7 @@ import (
 
 type IndexChecker interface {
 	CheckTrain(map[string]string) error
-	CheckValidDataType(dType schemapb.DataType) error
+	CheckValidDataType(dType schemapb.DataType, elementType schemapb.DataType) error
 	SetDefaultMetricTypeIfNotExist(map[string]string, schemapb.DataType)
 	StaticCheck(map[string]string) error
 }
diff --git a/pkg/util/indexparamcheck/inverted_checker.go b/pkg/util/indexparamcheck/inverted_checker.go
@@ -16,7 +16,7 @@ func (c *INVERTEDChecker) CheckTrain(params map[string]string) error {
 	return c.scalarIndexChecker.CheckTrain(params)
 }
 
-func (c *INVERTEDChecker) CheckValidDataType(dType schemapb.DataType) error {
+func (c *INVERTEDChecker) CheckValidDataType(dType schemapb.DataType, elemType schemapb.DataType) error {
 	if !typeutil.IsBoolType(dType) && !typeutil.IsArithmetic(dType) && !typeutil.IsStringType(dType) &&
 		!typeutil.IsArrayType(dType) {
 		return fmt.Errorf("INVERTED are not supported on %s field", dType.String())

diff --git a/pkg/util/indexparamcheck/inverted_checker_test.go b/pkg/util/indexparamcheck/inverted_checker_test.go
@@ -13,13 +13,13 @@ func Test_INVERTEDIndexChecker(t *testing.T) {
 
 	assert.NoError(t, c.CheckTrain(map[string]string{}))
 
-	assert.NoError(t, c.CheckValidDataType(schemapb.DataType_VarChar))
-	assert.NoError(t, c.CheckValidDataType(schemapb.DataType_String))
-	assert.NoError(t, c.CheckValidDataType(schemapb.DataType_Bool))
-	assert.NoError(t, c.CheckValidDataType(schemapb.DataType_Int64))
-	assert.NoError(t, c.CheckValidDataType(schemapb.DataType_Float))
-	assert.NoError(t, c.CheckValidDataType(schemapb.DataType_Array))
+	assert.NoError(t, c.CheckValidDataType(schemapb.DataType_VarChar, schemapb.DataType_None))
+	assert.NoError(t, c.CheckValidDataType(schemapb.DataType_String, schemapb.DataType_None))
+	assert.NoError(t, c.CheckValidDataType(schemapb.DataType_Bool, schemapb.DataType_None))
+	assert.NoError(t, c.CheckValidDataType(schemapb.DataType_Int64, schemapb.DataType_None))
+	assert.NoError(t, c.CheckValidDataType(schemapb.DataType_Float, schemapb.DataType_None))
+	assert.NoError(t, c.CheckValidDataType(schemapb.DataType_Array, schemapb.DataType_None))
 
-	assert.Error(t, c.CheckValidDataType(schemapb.DataType_JSON))
-	assert.Error(t, c.CheckValidDataType(schemapb.DataType_FloatVector))
+	assert.Error(t, c.CheckValidDataType(schemapb.DataType_JSON, schemapb.DataType_None))
+	assert.Error(t, c.CheckValidDataType(schemapb.DataType_FloatVector, schemapb.DataType_None))
 }
diff --git a/pkg/util/indexparamcheck/ivf_base_checker_test.go b/pkg/util/indexparamcheck/ivf_base_checker_test.go
@@ -142,7 +142,7 @@ func Test_ivfBaseChecker_CheckValidDataType(t *testing.T) {
 
 	c := newIVFBaseChecker()
 	for _, test := range cases {
-		err := c.CheckValidDataType(test.dType)
+		err := c.CheckValidDataType(test.dType, schemapb.DataType_None)
 		if test.errIsNil {
 			assert.NoError(t, err)
 		} else {

diff --git a/pkg/util/indexparamcheck/ivf_pq_checker_test.go b/pkg/util/indexparamcheck/ivf_pq_checker_test.go
@@ -207,7 +207,7 @@ func Test_ivfPQChecker_CheckValidDataType(t *testing.T) {
 
 	c := newIVFPQChecker()
 	for _, test := range cases {
-		err := c.CheckValidDataType(test.dType)
+		err := c.CheckValidDataType(test.dType, schemapb.DataType_None)
 		if test.errIsNil {
 			assert.NoError(t, err)
 		} else {

diff --git a/pkg/util/indexparamcheck/ivf_sq_checker_test.go b/pkg/util/indexparamcheck/ivf_sq_checker_test.go
@@ -162,7 +162,7 @@ func Test_ivfSQChecker_CheckValidDataType(t *testing.T) {
 
 	c := newIVFSQChecker()
 	for _, test := range cases {
-		err := c.CheckValidDataType(test.dType)
+		err := c.CheckValidDataType(test.dType, schemapb.DataType_None)
 		if test.errIsNil {
 			assert.NoError(t, err)
 		} else {

diff --git a/pkg/util/indexparamcheck/raft_ivf_flat_checker_test.go b/pkg/util/indexparamcheck/raft_ivf_flat_checker_test.go
@@ -156,7 +156,7 @@ func Test_raftIvfFlatChecker_CheckValidDataType(t *testing.T) {
 
 	c := newRaftIVFFlatChecker()
 	for _, test := range cases {
-		err := c.CheckValidDataType(test.dType)
+		err := c.CheckValidDataType(test.dType, schemapb.DataType_None)
 		if test.errIsNil {
 			assert.NoError(t, err)
 		} else {

diff --git a/pkg/util/indexparamcheck/raft_ivf_pq_checker_test.go b/pkg/util/indexparamcheck/raft_ivf_pq_checker_test.go
@@ -216,7 +216,7 @@ func Test_raftIVFPQChecker_CheckValidDataType(t *testing.T) {
 
 	c := newRaftIVFPQChecker()
 	for _, test := range cases {
-		err := c.CheckValidDataType(test.dType)
+		err := c.CheckValidDataType(test.dType, schemapb.DataType_None)
 		if test.errIsNil {
 			assert.NoError(t, err)
 		} else {

diff --git a/pkg/util/indexparamcheck/scann_checker_test.go b/pkg/util/indexparamcheck/scann_checker_test.go
@@ -159,7 +159,7 @@ func Test_scaNNChecker_CheckValidDataType(t *testing.T) {
 
 	c := newScaNNChecker()
 	for _, test := range cases {
-		err := c.CheckValidDataType(test.dType)
+		err := c.CheckValidDataType(test.dType, schemapb.DataType_None)
 		if test.errIsNil {
 			assert.NoError(t, err)
 		} else {

diff --git a/pkg/util/indexparamcheck/sparse_float_vector_base_checker.go b/pkg/util/indexparamcheck/sparse_float_vector_base_checker.go
@@ -32,7 +32,7 @@ func (c sparseFloatVectorBaseChecker) CheckTrain(params map[string]string) error
 	return nil
 }
 
-func (c sparseFloatVectorBaseChecker) CheckValidDataType(dType schemapb.DataType) error {
+func (c sparseFloatVectorBaseChecker) CheckValidDataType(dType schemapb.DataType, elemType schemapb.DataType) error {
 	if !typeutil.IsSparseFloatVectorType(dType) {
 		return fmt.Errorf("only sparse float vector is supported for the specified index tpye")
 	}

diff --git a/pkg/util/indexparamcheck/stl_sort_checker.go b/pkg/util/indexparamcheck/stl_sort_checker.go
@@ -16,7 +16,7 @@ func (c *STLSORTChecker) CheckTrain(params map[string]string) error {
 	return c.scalarIndexChecker.CheckTrain(params)
 }
 
-func (c *STLSORTChecker) CheckValidDataType(dType schemapb.DataType) error {
+func (c *STLSORTChecker) CheckValidDataType(dType schemapb.DataType, elemType schemapb.DataType) error {
 	if !typeutil.IsArithmetic(dType) {
 		return fmt.Errorf("STL_SORT are only supported on numeric field")
 	}

diff --git a/pkg/util/indexparamcheck/trie_checker.go b/pkg/util/indexparamcheck/trie_checker.go
@@ -16,7 +16,7 @@ func (c *TRIEChecker) CheckTrain(params map[string]string) error {
 	return c.scalarIndexChecker.CheckTrain(params)
 }
 
-func (c *TRIEChecker) CheckValidDataType(dType schemapb.DataType) error {
+func (c *TRIEChecker) CheckValidDataType(dType schemapb.DataType, elemType schemapb.DataType) error {
 	if !typeutil.IsStringType(dType) {
 		return fmt.Errorf("TRIE are only supported on varchar field")
 	}

diff --git a/pkg/util/indexparamcheck/trie_checker_test.go b/pkg/util/indexparamcheck/trie_checker_test.go
@@ -13,11 +13,11 @@ func Test_TrieIndexChecker(t *testing.T) {
 
 	assert.NoError(t, c.CheckTrain(map[string]string{}))
 
-	assert.NoError(t, c.CheckValidDataType(schemapb.DataType_VarChar))
-	assert.NoError(t, c.CheckValidDataType(schemapb.DataType_String))
+	assert.NoError(t, c.CheckValidDataType(schemapb.DataType_VarChar, schemapb.DataType_None))
+	assert.NoError(t, c.CheckValidDataType(schemapb.DataType_String, schemapb.DataType_None))
 
-	assert.Error(t, c.CheckValidDataType(schemapb.DataType_Bool))
-	assert.Error(t, c.CheckValidDataType(schemapb.DataType_Int64))
-	assert.Error(t, c.CheckValidDataType(schemapb.DataType_Float))
-	assert.Error(t, c.CheckValidDataType(schemapb.DataType_JSON))
+	assert.Error(t, c.CheckValidDataType(schemapb.DataType_Bool, schemapb.DataType_None))
+	assert.Error(t, c.CheckValidDataType(schemapb.DataType_Int64, schemapb.DataType_None))
+	assert.Error(t, c.CheckValidDataType(schemapb.DataType_Float, schemapb.DataType_None))
+	assert.Error(t, c.CheckValidDataType(schemapb.DataType_JSON, schemapb.DataType_None))
 }