Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

fix: fix bitmap supported type #34350

Merged
merged 1 commit into from
Jul 5, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion internal/proxy/task_index.go
Original file line number Diff line number Diff line change
Expand Up @@ -389,7 +389,7 @@ func checkTrain(field *schemapb.FieldSchema, indexParams map[string]string) erro
}
}

if err := checker.CheckValidDataType(field.GetDataType()); err != nil {
if err := checker.CheckValidDataType(field); err != nil {
log.Info("create index with invalid data type", zap.Error(err), zap.String("data_type", field.GetDataType().String()))
return err
}
Expand Down
2 changes: 1 addition & 1 deletion pkg/util/indexparamcheck/auto_index_checker.go
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@
return nil
}

func (c *AUTOINDEXChecker) CheckValidDataType(dType schemapb.DataType) error {
func (c *AUTOINDEXChecker) CheckValidDataType(field *schemapb.FieldSchema) error {

Check warning on line 16 in pkg/util/indexparamcheck/auto_index_checker.go

View check run for this annotation

Codecov / codecov/patch

pkg/util/indexparamcheck/auto_index_checker.go#L16

Added line #L16 was not covered by tests
return nil
}

Expand Down
2 changes: 1 addition & 1 deletion pkg/util/indexparamcheck/base_checker.go
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,7 @@ func (c baseChecker) CheckTrain(params map[string]string) error {
}

// CheckValidDataType check whether the field data type is supported for the index type
func (c baseChecker) CheckValidDataType(dType schemapb.DataType) error {
func (c baseChecker) CheckValidDataType(field *schemapb.FieldSchema) error {
return nil
}

Expand Down
3 changes: 2 additions & 1 deletion pkg/util/indexparamcheck/base_checker_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -98,7 +98,8 @@ func Test_baseChecker_CheckValidDataType(t *testing.T) {

c := newBaseChecker()
for _, test := range cases {
err := c.CheckValidDataType(test.dType)
field_schema := &schemapb.FieldSchema{DataType: test.dType}
err := c.CheckValidDataType(field_schema)
if test.errIsNil {
assert.NoError(t, err)
} else {
Expand Down
3 changes: 2 additions & 1 deletion pkg/util/indexparamcheck/bin_flat_checker_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -136,7 +136,8 @@ func Test_binFlatChecker_CheckValidDataType(t *testing.T) {

c := newBinFlatChecker()
for _, test := range cases {
err := c.CheckValidDataType(test.dType)
field_schema := &schemapb.FieldSchema{DataType: test.dType}
err := c.CheckValidDataType(field_schema)
if test.errIsNil {
assert.NoError(t, err)
} else {
Expand Down
3 changes: 2 additions & 1 deletion pkg/util/indexparamcheck/bin_ivf_flat_checker_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -187,7 +187,8 @@ func Test_binIVFFlatChecker_CheckValidDataType(t *testing.T) {

c := newBinIVFFlatChecker()
for _, test := range cases {
err := c.CheckValidDataType(test.dType)
field_schema := &schemapb.FieldSchema{DataType: test.dType}
err := c.CheckValidDataType(field_schema)
if test.errIsNil {
assert.NoError(t, err)
} else {
Expand Down
4 changes: 2 additions & 2 deletions pkg/util/indexparamcheck/binary_vector_base_checker.go
Original file line number Diff line number Diff line change
Expand Up @@ -27,8 +27,8 @@ func (c binaryVectorBaseChecker) CheckTrain(params map[string]string) error {
return c.staticCheck(params)
}

func (c binaryVectorBaseChecker) CheckValidDataType(dType schemapb.DataType) error {
if dType != schemapb.DataType_BinaryVector {
func (c binaryVectorBaseChecker) CheckValidDataType(field *schemapb.FieldSchema) error {
if field.GetDataType() != schemapb.DataType_BinaryVector {
return fmt.Errorf("binary vector is only supported")
}
return nil
Expand Down
3 changes: 2 additions & 1 deletion pkg/util/indexparamcheck/binary_vector_base_checker_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -69,7 +69,8 @@ func Test_binaryVectorBaseChecker_CheckValidDataType(t *testing.T) {

c := newBinaryVectorBaseChecker()
for _, test := range cases {
err := c.CheckValidDataType(test.dType)
field_schema := &schemapb.FieldSchema{DataType: test.dType}
err := c.CheckValidDataType(field_schema)
if test.errIsNil {
assert.NoError(t, err)
} else {
Expand Down
22 changes: 17 additions & 5 deletions pkg/util/indexparamcheck/bitmap_checker_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -13,12 +13,24 @@ func Test_BitmapIndexChecker(t *testing.T) {

assert.NoError(t, c.CheckTrain(map[string]string{"bitmap_cardinality_limit": "100"}))

assert.NoError(t, c.CheckValidDataType(schemapb.DataType_Int64))
assert.NoError(t, c.CheckValidDataType(schemapb.DataType_Float))
assert.NoError(t, c.CheckValidDataType(schemapb.DataType_String))
assert.NoError(t, c.CheckValidDataType(schemapb.DataType_Array))
assert.NoError(t, c.CheckValidDataType(&schemapb.FieldSchema{DataType: schemapb.DataType_Bool}))
assert.NoError(t, c.CheckValidDataType(&schemapb.FieldSchema{DataType: schemapb.DataType_Int8}))
assert.NoError(t, c.CheckValidDataType(&schemapb.FieldSchema{DataType: schemapb.DataType_Int16}))
assert.NoError(t, c.CheckValidDataType(&schemapb.FieldSchema{DataType: schemapb.DataType_Int32}))
assert.NoError(t, c.CheckValidDataType(&schemapb.FieldSchema{DataType: schemapb.DataType_Int64}))
assert.NoError(t, c.CheckValidDataType(&schemapb.FieldSchema{DataType: schemapb.DataType_String}))
assert.NoError(t, c.CheckValidDataType(&schemapb.FieldSchema{DataType: schemapb.DataType_Array, ElementType: schemapb.DataType_Bool}))
assert.NoError(t, c.CheckValidDataType(&schemapb.FieldSchema{DataType: schemapb.DataType_Array, ElementType: schemapb.DataType_Int8}))
assert.NoError(t, c.CheckValidDataType(&schemapb.FieldSchema{DataType: schemapb.DataType_Array, ElementType: schemapb.DataType_Int16}))
assert.NoError(t, c.CheckValidDataType(&schemapb.FieldSchema{DataType: schemapb.DataType_Array, ElementType: schemapb.DataType_Int32}))
assert.NoError(t, c.CheckValidDataType(&schemapb.FieldSchema{DataType: schemapb.DataType_Array, ElementType: schemapb.DataType_Int64}))
assert.NoError(t, c.CheckValidDataType(&schemapb.FieldSchema{DataType: schemapb.DataType_Array, ElementType: schemapb.DataType_String}))

assert.Error(t, c.CheckValidDataType(schemapb.DataType_JSON))
assert.Error(t, c.CheckValidDataType(&schemapb.FieldSchema{DataType: schemapb.DataType_JSON}))
assert.Error(t, c.CheckValidDataType(&schemapb.FieldSchema{DataType: schemapb.DataType_Float}))
assert.Error(t, c.CheckValidDataType(&schemapb.FieldSchema{DataType: schemapb.DataType_Double}))
assert.Error(t, c.CheckValidDataType(&schemapb.FieldSchema{DataType: schemapb.DataType_Array, ElementType: schemapb.DataType_Float}))
assert.Error(t, c.CheckValidDataType(&schemapb.FieldSchema{DataType: schemapb.DataType_Array, ElementType: schemapb.DataType_Double}))
assert.Error(t, c.CheckTrain(map[string]string{}))
assert.Error(t, c.CheckTrain(map[string]string{"bitmap_cardinality_limit": "0"}))
}
15 changes: 12 additions & 3 deletions pkg/util/indexparamcheck/bitmap_index_checker.go
Original file line number Diff line number Diff line change
Expand Up @@ -20,9 +20,18 @@ func (c *BITMAPChecker) CheckTrain(params map[string]string) error {
return c.scalarIndexChecker.CheckTrain(params)
}

func (c *BITMAPChecker) CheckValidDataType(dType schemapb.DataType) error {
if !typeutil.IsArithmetic(dType) && !typeutil.IsStringType(dType) && !typeutil.IsArrayType(dType) {
return fmt.Errorf("bitmap index are only supported on numeric, string and array field")
func (c *BITMAPChecker) CheckValidDataType(field *schemapb.FieldSchema) error {
main_type := field.GetDataType()
elem_type := field.GetElementType()
if !typeutil.IsBoolType(main_type) && !typeutil.IsIntegerType(main_type) &&
!typeutil.IsStringType(main_type) && !typeutil.IsArrayType(main_type) {
return fmt.Errorf("bitmap index are only supported on bool, int, string and array field")
}
if typeutil.IsArrayType(main_type) {
if !typeutil.IsBoolType(elem_type) && !typeutil.IsIntegerType(elem_type) &&
!typeutil.IsStringType(elem_type) {
return fmt.Errorf("bitmap index are only supported on bool, int, string for array field")
}
}
return nil
}
Expand Down
2 changes: 1 addition & 1 deletion pkg/util/indexparamcheck/diskann_checker_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -144,7 +144,7 @@ func Test_diskannChecker_CheckValidDataType(t *testing.T) {

c := newDiskannChecker()
for _, test := range cases {
err := c.CheckValidDataType(test.dType)
err := c.CheckValidDataType(&schemapb.FieldSchema{DataType: test.dType})
if test.errIsNil {
assert.NoError(t, err)
} else {
Expand Down
4 changes: 2 additions & 2 deletions pkg/util/indexparamcheck/float_vector_base_checker.go
Original file line number Diff line number Diff line change
Expand Up @@ -28,8 +28,8 @@ func (c floatVectorBaseChecker) CheckTrain(params map[string]string) error {
return c.staticCheck(params)
}

func (c floatVectorBaseChecker) CheckValidDataType(dType schemapb.DataType) error {
if !typeutil.IsDenseFloatVectorType(dType) {
func (c floatVectorBaseChecker) CheckValidDataType(field *schemapb.FieldSchema) error {
if !typeutil.IsDenseFloatVectorType(field.GetDataType()) {
return fmt.Errorf("data type should be FloatVector, Float16Vector or BFloat16Vector")
}
return nil
Expand Down
2 changes: 1 addition & 1 deletion pkg/util/indexparamcheck/float_vector_base_checker_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -69,7 +69,7 @@ func Test_floatVectorBaseChecker_CheckValidDataType(t *testing.T) {

c := newFloatVectorBaseChecker()
for _, test := range cases {
err := c.CheckValidDataType(test.dType)
err := c.CheckValidDataType(&schemapb.FieldSchema{DataType: test.dType})
if test.errIsNil {
assert.NoError(t, err)
} else {
Expand Down
6 changes: 3 additions & 3 deletions pkg/util/indexparamcheck/hnsw_checker.go
Original file line number Diff line number Diff line change
Expand Up @@ -32,9 +32,9 @@ func (c hnswChecker) CheckTrain(params map[string]string) error {
return c.baseChecker.CheckTrain(params)
}

func (c hnswChecker) CheckValidDataType(dType schemapb.DataType) error {
if !typeutil.IsVectorType(dType) {
return fmt.Errorf("can't build hnsw in not vector type.")
func (c hnswChecker) CheckValidDataType(field *schemapb.FieldSchema) error {
if !typeutil.IsVectorType(field.GetDataType()) {
return fmt.Errorf("can't build hnsw in not vector type")
}
return nil
}
Expand Down
2 changes: 1 addition & 1 deletion pkg/util/indexparamcheck/hnsw_checker_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -164,7 +164,7 @@ func Test_hnswChecker_CheckValidDataType(t *testing.T) {

c := newHnswChecker()
for _, test := range cases {
err := c.CheckValidDataType(test.dType)
err := c.CheckValidDataType(&schemapb.FieldSchema{DataType: test.dType})
if test.errIsNil {
assert.NoError(t, err)
} else {
Expand Down
2 changes: 1 addition & 1 deletion pkg/util/indexparamcheck/index_checker.go
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@ import (

type IndexChecker interface {
CheckTrain(map[string]string) error
CheckValidDataType(dType schemapb.DataType) error
CheckValidDataType(field *schemapb.FieldSchema) error
SetDefaultMetricTypeIfNotExist(map[string]string, schemapb.DataType)
StaticCheck(map[string]string) error
}
3 changes: 2 additions & 1 deletion pkg/util/indexparamcheck/inverted_checker.go
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,8 @@ func (c *INVERTEDChecker) CheckTrain(params map[string]string) error {
return c.scalarIndexChecker.CheckTrain(params)
}

func (c *INVERTEDChecker) CheckValidDataType(dType schemapb.DataType) error {
func (c *INVERTEDChecker) CheckValidDataType(field *schemapb.FieldSchema) error {
dType := field.GetDataType()
if !typeutil.IsBoolType(dType) && !typeutil.IsArithmetic(dType) && !typeutil.IsStringType(dType) &&
!typeutil.IsArrayType(dType) {
return fmt.Errorf("INVERTED are not supported on %s field", dType.String())
Expand Down
16 changes: 8 additions & 8 deletions pkg/util/indexparamcheck/inverted_checker_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -13,13 +13,13 @@ func Test_INVERTEDIndexChecker(t *testing.T) {

assert.NoError(t, c.CheckTrain(map[string]string{}))

assert.NoError(t, c.CheckValidDataType(schemapb.DataType_VarChar))
assert.NoError(t, c.CheckValidDataType(schemapb.DataType_String))
assert.NoError(t, c.CheckValidDataType(schemapb.DataType_Bool))
assert.NoError(t, c.CheckValidDataType(schemapb.DataType_Int64))
assert.NoError(t, c.CheckValidDataType(schemapb.DataType_Float))
assert.NoError(t, c.CheckValidDataType(schemapb.DataType_Array))
assert.NoError(t, c.CheckValidDataType(&schemapb.FieldSchema{DataType: schemapb.DataType_VarChar}))
assert.NoError(t, c.CheckValidDataType(&schemapb.FieldSchema{DataType: schemapb.DataType_String}))
assert.NoError(t, c.CheckValidDataType(&schemapb.FieldSchema{DataType: schemapb.DataType_Bool}))
assert.NoError(t, c.CheckValidDataType(&schemapb.FieldSchema{DataType: schemapb.DataType_Int64}))
assert.NoError(t, c.CheckValidDataType(&schemapb.FieldSchema{DataType: schemapb.DataType_Float}))
assert.NoError(t, c.CheckValidDataType(&schemapb.FieldSchema{DataType: schemapb.DataType_Array}))

assert.Error(t, c.CheckValidDataType(schemapb.DataType_JSON))
assert.Error(t, c.CheckValidDataType(schemapb.DataType_FloatVector))
assert.Error(t, c.CheckValidDataType(&schemapb.FieldSchema{DataType: schemapb.DataType_JSON}))
assert.Error(t, c.CheckValidDataType(&schemapb.FieldSchema{DataType: schemapb.DataType_FloatVector}))
}
2 changes: 1 addition & 1 deletion pkg/util/indexparamcheck/ivf_base_checker_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -142,7 +142,7 @@ func Test_ivfBaseChecker_CheckValidDataType(t *testing.T) {

c := newIVFBaseChecker()
for _, test := range cases {
err := c.CheckValidDataType(test.dType)
err := c.CheckValidDataType(&schemapb.FieldSchema{DataType: test.dType})
if test.errIsNil {
assert.NoError(t, err)
} else {
Expand Down
2 changes: 1 addition & 1 deletion pkg/util/indexparamcheck/ivf_pq_checker_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -207,7 +207,7 @@ func Test_ivfPQChecker_CheckValidDataType(t *testing.T) {

c := newIVFPQChecker()
for _, test := range cases {
err := c.CheckValidDataType(test.dType)
err := c.CheckValidDataType(&schemapb.FieldSchema{DataType: test.dType})
if test.errIsNil {
assert.NoError(t, err)
} else {
Expand Down
2 changes: 1 addition & 1 deletion pkg/util/indexparamcheck/ivf_sq_checker_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -162,7 +162,7 @@ func Test_ivfSQChecker_CheckValidDataType(t *testing.T) {

c := newIVFSQChecker()
for _, test := range cases {
err := c.CheckValidDataType(test.dType)
err := c.CheckValidDataType(&schemapb.FieldSchema{DataType: test.dType})
if test.errIsNil {
assert.NoError(t, err)
} else {
Expand Down
2 changes: 1 addition & 1 deletion pkg/util/indexparamcheck/raft_ivf_flat_checker_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -156,7 +156,7 @@ func Test_raftIvfFlatChecker_CheckValidDataType(t *testing.T) {

c := newRaftIVFFlatChecker()
for _, test := range cases {
err := c.CheckValidDataType(test.dType)
err := c.CheckValidDataType(&schemapb.FieldSchema{DataType: test.dType})
if test.errIsNil {
assert.NoError(t, err)
} else {
Expand Down
2 changes: 1 addition & 1 deletion pkg/util/indexparamcheck/raft_ivf_pq_checker_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -216,7 +216,7 @@ func Test_raftIVFPQChecker_CheckValidDataType(t *testing.T) {

c := newRaftIVFPQChecker()
for _, test := range cases {
err := c.CheckValidDataType(test.dType)
err := c.CheckValidDataType(&schemapb.FieldSchema{DataType: test.dType})
if test.errIsNil {
assert.NoError(t, err)
} else {
Expand Down
2 changes: 1 addition & 1 deletion pkg/util/indexparamcheck/scann_checker_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -159,7 +159,7 @@ func Test_scaNNChecker_CheckValidDataType(t *testing.T) {

c := newScaNNChecker()
for _, test := range cases {
err := c.CheckValidDataType(test.dType)
err := c.CheckValidDataType(&schemapb.FieldSchema{DataType: test.dType})
if test.errIsNil {
assert.NoError(t, err)
} else {
Expand Down
4 changes: 2 additions & 2 deletions pkg/util/indexparamcheck/sparse_float_vector_base_checker.go
Original file line number Diff line number Diff line change
Expand Up @@ -32,8 +32,8 @@ func (c sparseFloatVectorBaseChecker) CheckTrain(params map[string]string) error
return nil
}

func (c sparseFloatVectorBaseChecker) CheckValidDataType(dType schemapb.DataType) error {
if !typeutil.IsSparseFloatVectorType(dType) {
func (c sparseFloatVectorBaseChecker) CheckValidDataType(field *schemapb.FieldSchema) error {
if !typeutil.IsSparseFloatVectorType(field.GetDataType()) {
return fmt.Errorf("only sparse float vector is supported for the specified index tpye")
}
return nil
Expand Down
4 changes: 2 additions & 2 deletions pkg/util/indexparamcheck/stl_sort_checker.go
Original file line number Diff line number Diff line change
Expand Up @@ -16,8 +16,8 @@ func (c *STLSORTChecker) CheckTrain(params map[string]string) error {
return c.scalarIndexChecker.CheckTrain(params)
}

func (c *STLSORTChecker) CheckValidDataType(dType schemapb.DataType) error {
if !typeutil.IsArithmetic(dType) {
func (c *STLSORTChecker) CheckValidDataType(field *schemapb.FieldSchema) error {
if !typeutil.IsArithmetic(field.GetDataType()) {
return fmt.Errorf("STL_SORT are only supported on numeric field")
}
return nil
Expand Down
10 changes: 5 additions & 5 deletions pkg/util/indexparamcheck/stl_sort_checker_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -13,10 +13,10 @@ func Test_STLSORTIndexChecker(t *testing.T) {

assert.NoError(t, c.CheckTrain(map[string]string{}))

assert.NoError(t, c.CheckValidDataType(schemapb.DataType_Int64))
assert.NoError(t, c.CheckValidDataType(schemapb.DataType_Float))
assert.NoError(t, c.CheckValidDataType(&schemapb.FieldSchema{DataType: schemapb.DataType_Int64}))
assert.NoError(t, c.CheckValidDataType(&schemapb.FieldSchema{DataType: schemapb.DataType_Float}))

assert.Error(t, c.CheckValidDataType(schemapb.DataType_Bool))
assert.Error(t, c.CheckValidDataType(schemapb.DataType_VarChar))
assert.Error(t, c.CheckValidDataType(schemapb.DataType_JSON))
assert.Error(t, c.CheckValidDataType(&schemapb.FieldSchema{DataType: schemapb.DataType_VarChar}))
assert.Error(t, c.CheckValidDataType(&schemapb.FieldSchema{DataType: schemapb.DataType_Bool}))
assert.Error(t, c.CheckValidDataType(&schemapb.FieldSchema{DataType: schemapb.DataType_JSON}))
}
4 changes: 2 additions & 2 deletions pkg/util/indexparamcheck/trie_checker.go
Original file line number Diff line number Diff line change
Expand Up @@ -16,8 +16,8 @@ func (c *TRIEChecker) CheckTrain(params map[string]string) error {
return c.scalarIndexChecker.CheckTrain(params)
}

func (c *TRIEChecker) CheckValidDataType(dType schemapb.DataType) error {
if !typeutil.IsStringType(dType) {
func (c *TRIEChecker) CheckValidDataType(field *schemapb.FieldSchema) error {
if !typeutil.IsStringType(field.GetDataType()) {
return fmt.Errorf("TRIE are only supported on varchar field")
}
return nil
Expand Down
12 changes: 6 additions & 6 deletions pkg/util/indexparamcheck/trie_checker_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -13,11 +13,11 @@ func Test_TrieIndexChecker(t *testing.T) {

assert.NoError(t, c.CheckTrain(map[string]string{}))

assert.NoError(t, c.CheckValidDataType(schemapb.DataType_VarChar))
assert.NoError(t, c.CheckValidDataType(schemapb.DataType_String))
assert.NoError(t, c.CheckValidDataType(&schemapb.FieldSchema{DataType: schemapb.DataType_VarChar}))
assert.NoError(t, c.CheckValidDataType(&schemapb.FieldSchema{DataType: schemapb.DataType_String}))

assert.Error(t, c.CheckValidDataType(schemapb.DataType_Bool))
assert.Error(t, c.CheckValidDataType(schemapb.DataType_Int64))
assert.Error(t, c.CheckValidDataType(schemapb.DataType_Float))
assert.Error(t, c.CheckValidDataType(schemapb.DataType_JSON))
assert.Error(t, c.CheckValidDataType(&schemapb.FieldSchema{DataType: schemapb.DataType_Bool}))
assert.Error(t, c.CheckValidDataType(&schemapb.FieldSchema{DataType: schemapb.DataType_Int64}))
assert.Error(t, c.CheckValidDataType(&schemapb.FieldSchema{DataType: schemapb.DataType_Float}))
assert.Error(t, c.CheckValidDataType(&schemapb.FieldSchema{DataType: schemapb.DataType_JSON}))
}
Loading