diff --git a/internal/proxy/task_index.go b/internal/proxy/task_index.go index 17dde3ebc44c0..c8b90129e37a1 100644 --- a/internal/proxy/task_index.go +++ b/internal/proxy/task_index.go @@ -51,6 +51,7 @@ const ( AutoIndexName = "AUTOINDEX" DimKey = common.DimKey + IsSparseKey = common.IsSparseKey ) type createIndexTask struct { @@ -382,10 +383,15 @@ func checkTrain(field *schemapb.FieldSchema, indexParams map[string]string) erro return fmt.Errorf("invalid index type: %s", indexType) } - if !typeutil.IsSparseFloatVectorType(field.DataType) { + isSparse := typeutil.IsSparseFloatVectorType(field.DataType) + + if !isSparse { if err := fillDimension(field, indexParams); err != nil { return err } + } else { + // used only for checker, should be deleted after checking + indexParams[IsSparseKey] = "true" } if err := checker.CheckValidDataType(field.GetDataType()); err != nil { @@ -398,6 +404,10 @@ func checkTrain(field *schemapb.FieldSchema, indexParams map[string]string) erro return err } + if isSparse { + delete(indexParams, IsSparseKey) + } + return nil } diff --git a/pkg/common/common.go b/pkg/common/common.go index 5bd3aabc3c1d5..8b881c14a2de9 100644 --- a/pkg/common/common.go +++ b/pkg/common/common.go @@ -120,6 +120,7 @@ const ( DropRatioBuildKey = "drop_ratio_build" BitmapCardinalityLimitKey = "bitmap_cardinality_limit" + IsSparseKey = "is_sparse" ) // Collection properties key diff --git a/pkg/util/indexparamcheck/base_checker.go b/pkg/util/indexparamcheck/base_checker.go index 2566fc097591d..af7bac7fd7735 100644 --- a/pkg/util/indexparamcheck/base_checker.go +++ b/pkg/util/indexparamcheck/base_checker.go @@ -19,18 +19,37 @@ package indexparamcheck import ( "fmt" "math" + "strings" "github.com/cockroachdb/errors" "github.com/milvus-io/milvus-proto/go-api/v2/schemapb" + "github.com/milvus-io/milvus/pkg/common" ) type baseChecker struct{} func (c baseChecker) CheckTrain(params map[string]string) error { // vector dimension should be checked on collection creation. this is just some basic check - if !CheckIntByRange(params, DIM, 1, math.MaxInt) { - return fmt.Errorf("failed to check vector dimension, should be larger than 0 and smaller than math.MaxInt") + isSparse := false + if val, exist := params[common.IsSparseKey]; exist { + val = strings.ToLower(val) + if val != "true" && val != "false" { + return fmt.Errorf("invalid is_sparse value: %s, must be true or false", val) + } + if val == "true" { + isSparse = true + } + } + if isSparse { + if !CheckStrByValues(params, Metric, SparseMetrics) { + return fmt.Errorf("metric type not found or not supported for sparse float vectors, supported: %v", SparseMetrics) + } + } else { + // we do not check dim for sparse + if !CheckIntByRange(params, DIM, 1, math.MaxInt) { + return fmt.Errorf("failed to check vector dimension, should be larger than 0 and smaller than math.MaxInt") + } } return nil } diff --git a/pkg/util/indexparamcheck/base_checker_test.go b/pkg/util/indexparamcheck/base_checker_test.go index a016d4da88498..45eb4e22cc5b7 100644 --- a/pkg/util/indexparamcheck/base_checker_test.go +++ b/pkg/util/indexparamcheck/base_checker_test.go @@ -7,6 +7,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/milvus-io/milvus-proto/go-api/v2/schemapb" + "github.com/milvus-io/milvus/pkg/common" "github.com/milvus-io/milvus/pkg/util/metric" ) @@ -18,12 +19,27 @@ func Test_baseChecker_CheckTrain(t *testing.T) { paramsWithoutDim := map[string]string{ Metric: metric.L2, } + sparseParamsWithoutDim := map[string]string{ + Metric: metric.IP, + common.IsSparseKey: "tRue", + } + sparseParamsWrongMetric := map[string]string{ + Metric: metric.L2, + common.IsSparseKey: "True", + } + badSparseParams := map[string]string{ + Metric: metric.IP, + common.IsSparseKey: "ds", + } cases := []struct { params map[string]string errIsNil bool }{ {validParams, true}, {paramsWithoutDim, false}, + {sparseParamsWithoutDim, true}, + {sparseParamsWrongMetric, false}, + {badSparseParams, false}, } c := newBaseChecker()