diff --git a/client/column/columns.go b/client/column/columns.go index ff1bb7da6ace..29d39669200e 100644 --- a/client/column/columns.go +++ b/client/column/columns.go @@ -310,6 +310,25 @@ func FieldDataColumn(fd *schemapb.FieldData, begin, end int) (Column, error) { vector = append(vector, v) } return NewColumnBFloat16Vector(fd.GetFieldName(), dim, vector), nil + case schemapb.DataType_SparseFloatVector: + sparseVectors := fd.GetVectors().GetSparseFloatVector() + if sparseVectors == nil { + return nil, errFieldDataTypeNotMatch + } + data := sparseVectors.Contents + if end < 0 { + end = len(data) + } + data = data[begin:end] + vectors := make([]entity.SparseEmbedding, 0, len(data)) + for _, bs := range data { + vector, err := entity.DeserializeSliceSparseEmbedding(bs) + if err != nil { + return nil, err + } + vectors = append(vectors, vector) + } + return NewColumnSparseVectors(fd.GetFieldName(), vectors), nil default: return nil, fmt.Errorf("unsupported data type %s", fd.GetType()) } diff --git a/client/entity/sparse.go b/client/entity/sparse.go index 87edf58d152b..062a6195cd08 100644 --- a/client/entity/sparse.go +++ b/client/entity/sparse.go @@ -83,7 +83,7 @@ func (e sliceSparseEmbedding) Swap(i, j int) { e.values[i], e.values[j] = e.values[j], e.values[i] } -func deserializeSliceSparceEmbedding(bs []byte) (sliceSparseEmbedding, error) { +func DeserializeSliceSparseEmbedding(bs []byte) (sliceSparseEmbedding, error) { length := len(bs) if length%8 != 0 { return sliceSparseEmbedding{}, errors.New("not valid sparse embedding bytes") diff --git a/client/entity/sparse_test.go b/client/entity/sparse_test.go index 24f6b08a1975..649d332c45fc 100644 --- a/client/entity/sparse_test.go +++ b/client/entity/sparse_test.go @@ -40,7 +40,7 @@ func TestSliceSparseEmbedding(t *testing.T) { assert.EqualValues(t, length, se.Len()) bs := se.Serialize() - nv, err := deserializeSliceSparceEmbedding(bs) + nv, err := DeserializeSliceSparseEmbedding(bs) require.NoError(t, err) for i := 0; i < length; i++ { diff --git a/tests/go_client/common/response_checker.go b/tests/go_client/common/response_checker.go index eed14d4f5edf..08849b729468 100644 --- a/tests/go_client/common/response_checker.go +++ b/tests/go_client/common/response_checker.go @@ -2,6 +2,7 @@ package common import ( "fmt" + "reflect" "strings" "testing" @@ -32,7 +33,7 @@ func CheckErr(t *testing.T, actualErr error, expErrNil bool, expErrorMsg ...stri } } if !contains { - t.FailNow() + t.Fatalf("CheckErr failed, actualErr doesn contains any expErrorMsg, please check test cases!") } } } @@ -60,31 +61,90 @@ func EqualColumn(t *testing.T, columnA column.Column, columnB column.Column) { case entity.FieldTypeVarChar: require.ElementsMatch(t, columnA.(*column.ColumnVarChar).Data(), columnB.(*column.ColumnVarChar).Data()) case entity.FieldTypeJSON: - log.Debug("columnA", zap.Any("data", columnA.(*column.ColumnJSONBytes).Data())) - log.Debug("columnB", zap.Any("data", columnB.(*column.ColumnJSONBytes).Data())) - require.ElementsMatch(t, columnA.(*column.ColumnJSONBytes).Data(), columnB.(*column.ColumnJSONBytes).Data()) + log.Debug("data", zap.String("name", columnA.Name()), zap.Any("type", columnA.Type()), zap.Any("data", columnA.FieldData())) + log.Debug("data", zap.String("name", columnB.Name()), zap.Any("type", columnB.Type()), zap.Any("data", columnB.FieldData())) + require.Equal(t, reflect.TypeOf(columnA), reflect.TypeOf(columnB)) + switch columnA.(type) { + case *column.ColumnDynamic: + require.ElementsMatch(t, columnA.(*column.ColumnDynamic).Data(), columnB.(*column.ColumnDynamic).Data()) + case *column.ColumnJSONBytes: + require.ElementsMatch(t, columnA.(*column.ColumnJSONBytes).Data(), columnB.(*column.ColumnJSONBytes).Data()) + } case entity.FieldTypeFloatVector: require.ElementsMatch(t, columnA.(*column.ColumnFloatVector).Data(), columnB.(*column.ColumnFloatVector).Data()) case entity.FieldTypeBinaryVector: require.ElementsMatch(t, columnA.(*column.ColumnBinaryVector).Data(), columnB.(*column.ColumnBinaryVector).Data()) + case entity.FieldTypeFloat16Vector: + require.ElementsMatch(t, columnA.(*column.ColumnFloat16Vector).Data(), columnB.(*column.ColumnFloat16Vector).Data()) + case entity.FieldTypeBFloat16Vector: + require.ElementsMatch(t, columnA.(*column.ColumnBFloat16Vector).Data(), columnB.(*column.ColumnBFloat16Vector).Data()) + case entity.FieldTypeSparseVector: + require.ElementsMatch(t, columnA.(*column.ColumnSparseFloatVector).Data(), columnB.(*column.ColumnSparseFloatVector).Data()) case entity.FieldTypeArray: - log.Info("TODO support column element type") + EqualArrayColumn(t, columnA, columnB) default: log.Info("Support column type is:", zap.Any("FieldType", []entity.FieldType{ entity.FieldTypeBool, - entity.FieldTypeInt8, entity.FieldTypeInt16, entity.FieldTypeInt32, entity.FieldTypeInt64, - entity.FieldTypeFloat, entity.FieldTypeDouble, entity.FieldTypeString, entity.FieldTypeVarChar, - entity.FieldTypeArray, entity.FieldTypeFloatVector, entity.FieldTypeBinaryVector, + entity.FieldTypeInt8, entity.FieldTypeInt16, entity.FieldTypeInt32, + entity.FieldTypeInt64, entity.FieldTypeFloat, entity.FieldTypeDouble, entity.FieldTypeString, + entity.FieldTypeVarChar, entity.FieldTypeArray, entity.FieldTypeFloatVector, entity.FieldTypeBinaryVector, + })) + } +} + +// EqualColumn assert field data is equal of two columns +func EqualArrayColumn(t *testing.T, columnA column.Column, columnB column.Column) { + require.Equal(t, columnA.Name(), columnB.Name()) + require.IsType(t, columnA.Type(), entity.FieldTypeArray) + require.IsType(t, columnB.Type(), entity.FieldTypeArray) + switch columnA.(type) { + case *column.ColumnBoolArray: + require.ElementsMatch(t, columnA.(*column.ColumnBoolArray).Data(), columnB.(*column.ColumnBoolArray).Data()) + case *column.ColumnInt8Array: + require.ElementsMatch(t, columnA.(*column.ColumnInt8Array).Data(), columnB.(*column.ColumnInt8Array).Data()) + case *column.ColumnInt16Array: + require.ElementsMatch(t, columnA.(*column.ColumnInt16Array).Data(), columnB.(*column.ColumnInt16Array).Data()) + case *column.ColumnInt32Array: + require.ElementsMatch(t, columnA.(*column.ColumnInt32Array).Data(), columnB.(*column.ColumnInt32Array).Data()) + case *column.ColumnInt64Array: + require.ElementsMatch(t, columnA.(*column.ColumnInt64Array).Data(), columnB.(*column.ColumnInt64Array).Data()) + case *column.ColumnFloatArray: + require.ElementsMatch(t, columnA.(*column.ColumnFloatArray).Data(), columnB.(*column.ColumnFloatArray).Data()) + case *column.ColumnDoubleArray: + require.ElementsMatch(t, columnA.(*column.ColumnDoubleArray).Data(), columnB.(*column.ColumnDoubleArray).Data()) + case *column.ColumnVarCharArray: + require.ElementsMatch(t, columnA.(*column.ColumnVarCharArray).Data(), columnB.(*column.ColumnVarCharArray).Data()) + default: + log.Info("Support array element type is:", zap.Any("FieldType", []entity.FieldType{ + entity.FieldTypeBool, entity.FieldTypeInt8, entity.FieldTypeInt16, + entity.FieldTypeInt32, entity.FieldTypeInt64, entity.FieldTypeFloat, entity.FieldTypeDouble, entity.FieldTypeVarChar, })) } } +// CheckInsertResult check insert result, ids len (insert count), ids data (pks, but no auto ids) +func CheckInsertResult(t *testing.T, expIds column.Column, insertRes clientv2.InsertResult) { + require.Equal(t, expIds.Len(), insertRes.IDs.Len()) + require.Equal(t, expIds.Len(), int(insertRes.InsertCount)) + actualIds := insertRes.IDs + switch expIds.Type() { + // pk field support int64 and varchar type + case entity.FieldTypeInt64: + require.ElementsMatch(t, actualIds.(*column.ColumnInt64).Data(), expIds.(*column.ColumnInt64).Data()) + case entity.FieldTypeVarChar: + require.ElementsMatch(t, actualIds.(*column.ColumnVarChar).Data(), expIds.(*column.ColumnVarChar).Data()) + default: + log.Info("The primary field only support ", zap.Any("type", []entity.FieldType{entity.FieldTypeInt64, entity.FieldTypeVarChar})) + } +} + // CheckOutputFields check query output fields func CheckOutputFields(t *testing.T, expFields []string, actualColumns []column.Column) { actualFields := make([]string, 0) for _, actualColumn := range actualColumns { actualFields = append(actualFields, actualColumn.Name()) } + log.Debug("CheckOutputFields", zap.Any("expFields", expFields), zap.Any("actualFields", actualFields)) require.ElementsMatchf(t, expFields, actualFields, fmt.Sprintf("Expected search output fields: %v, actual: %v", expFields, actualFields)) } @@ -97,18 +157,20 @@ func CheckSearchResult(t *testing.T, actualSearchResults []clientv2.ResultSet, e } } -// CheckInsertResult check insert result, ids len (insert count), ids data (pks, but no auto ids) -func CheckInsertResult(t *testing.T, expIds column.Column, insertRes clientv2.InsertResult) { - require.Equal(t, expIds.Len(), insertRes.IDs.Len()) - require.Equal(t, expIds.Len(), int(insertRes.InsertCount)) - actualIds := insertRes.IDs - switch expIds.Type() { - // pk field support int64 and varchar type - case entity.FieldTypeInt64: - require.ElementsMatch(t, actualIds.(*column.ColumnInt64).Data(), expIds.(*column.ColumnInt64).Data()) - case entity.FieldTypeVarChar: - require.ElementsMatch(t, actualIds.(*column.ColumnVarChar).Data(), expIds.(*column.ColumnVarChar).Data()) - default: - log.Info("The primary field only support ", zap.Any("type", []entity.FieldType{entity.FieldTypeInt64, entity.FieldTypeVarChar})) +// CheckQueryResult check query result, column name, type and field +func CheckQueryResult(t *testing.T, expColumns []column.Column, actualColumns []column.Column) { + require.Equal(t, len(actualColumns), len(expColumns), + "The len of actual columns %d should greater or equal to the expected columns %d", len(actualColumns), len(expColumns)) + for _, expColumn := range expColumns { + exist := false + for _, actualColumn := range actualColumns { + if expColumn.Name() == actualColumn.Name() { + exist = true + EqualColumn(t, expColumn, actualColumn) + } + } + if !exist { + log.Error("CheckQueryResult actualColumns no column", zap.String("name", expColumn.Name())) + } } } diff --git a/tests/go_client/go.mod b/tests/go_client/go.mod index ba8105116425..8885a3d0d966 100644 --- a/tests/go_client/go.mod +++ b/tests/go_client/go.mod @@ -5,15 +5,16 @@ go 1.21 toolchain go1.21.10 require ( - github.com/milvus-io/milvus/client/v2 v2.0.0-20240612020955-8cb350598c9a + github.com/milvus-io/milvus/client/v2 v2.0.0-20240621033600-e653ad27e2d5 github.com/milvus-io/milvus/pkg v0.0.2-0.20240317152703-17b4938985f3 + github.com/quasilyte/go-ruleguard/dsl v0.3.22 github.com/stretchr/testify v1.9.0 github.com/x448/float16 v0.8.4 go.uber.org/zap v1.27.0 google.golang.org/grpc v1.64.0 ) -//replace github.com/milvus-io/milvus/client/v2 v2.0.0-20240521081339-017fd7bc25de => ../../../milvus/client +//replace github.com/milvus-io/milvus/client/v2 v2.0.0-20240621033600-e653ad27e2d5 => ../../../milvus/client require ( github.com/beorn7/perks v1.0.1 // indirect @@ -55,7 +56,7 @@ require ( github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 // indirect github.com/magiconair/properties v1.8.5 // indirect github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect - github.com/milvus-io/milvus-proto/go-api/v2 v2.3.4-0.20240430035521-259ae1d10016 // indirect + github.com/milvus-io/milvus-proto/go-api/v2 v2.4.3 // indirect github.com/mitchellh/mapstructure v1.4.1 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect @@ -69,7 +70,6 @@ require ( github.com/prometheus/client_model v0.3.0 // indirect github.com/prometheus/common v0.42.0 // indirect github.com/prometheus/procfs v0.9.0 // indirect - github.com/quasilyte/go-ruleguard/dsl v0.3.22 // indirect github.com/rogpeppe/go-internal v1.10.0 // indirect github.com/samber/lo v1.27.0 // indirect github.com/shirou/gopsutil/v3 v3.22.9 // indirect diff --git a/tests/go_client/go.sum b/tests/go_client/go.sum index a9f92bb07dd1..2426e346bc3c 100644 --- a/tests/go_client/go.sum +++ b/tests/go_client/go.sum @@ -401,12 +401,10 @@ github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfr github.com/mediocregopher/radix/v3 v3.4.2/go.mod h1:8FL3F6UQRXHXIBSPUs5h0RybMF8i4n7wVopoX3x7Bv8= github.com/microcosm-cc/bluemonday v1.0.2/go.mod h1:iVP4YcDBq+n/5fb23BhYFvIMq/leAFZyRl6bYmGDlGc= github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= -github.com/milvus-io/milvus-proto/go-api/v2 v2.3.4-0.20240430035521-259ae1d10016 h1:8WV4maXLeGEyJCCYIc1DmZ18H+VFAjMrwXJg5iI2nX4= -github.com/milvus-io/milvus-proto/go-api/v2 v2.3.4-0.20240430035521-259ae1d10016/go.mod h1:1OIl0v5PQeNxIJhCvY+K55CBUOYDZevw9g9380u1Wek= -github.com/milvus-io/milvus/client/v2 v2.0.0-20240521081339-017fd7bc25de h1:sZ1K+F+1RMlWzuAFlhF7m68e9+G6UlRDoL7Q4Bmsc08= -github.com/milvus-io/milvus/client/v2 v2.0.0-20240521081339-017fd7bc25de/go.mod h1:FU4rii9fMaMy1CGWgjROyKo5SCaJqcxZeIhdhuuEhrQ= -github.com/milvus-io/milvus/client/v2 v2.0.0-20240612020955-8cb350598c9a h1:iyXy8ee4pDutzO40VA9xem1O1h/o5gI5x86ZVqVzto4= -github.com/milvus-io/milvus/client/v2 v2.0.0-20240612020955-8cb350598c9a/go.mod h1:thfuEkUztRRmQ+qu4hCoO/6uxDJoUVNNx4vHqx9yh5I= +github.com/milvus-io/milvus-proto/go-api/v2 v2.4.3 h1:KUSaWVePVlHMIluAXf2qmNffI1CMlGFLLiP+4iy9014= +github.com/milvus-io/milvus-proto/go-api/v2 v2.4.3/go.mod h1:1OIl0v5PQeNxIJhCvY+K55CBUOYDZevw9g9380u1Wek= +github.com/milvus-io/milvus/client/v2 v2.0.0-20240621033600-e653ad27e2d5 h1:KNE4Smy6HxIpoJHLpds5BI2ZyvhrE5FcLhUcsjxAxAk= +github.com/milvus-io/milvus/client/v2 v2.0.0-20240621033600-e653ad27e2d5/go.mod h1:thfuEkUztRRmQ+qu4hCoO/6uxDJoUVNNx4vHqx9yh5I= github.com/milvus-io/milvus/pkg v0.0.2-0.20240317152703-17b4938985f3 h1:ZBpRWhBa7FTFxW4YYVv9AUESoW1Xyb3KNXTzTqfkZmw= github.com/milvus-io/milvus/pkg v0.0.2-0.20240317152703-17b4938985f3/go.mod h1:jQ2BUZny1COsgv1Qbcv8dmbppW+V9J/c4YQZNb3EOm8= github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc= diff --git a/tests/go_client/testcases/collection_test.go b/tests/go_client/testcases/collection_test.go index c106cb28af4f..645aebbd2a15 100644 --- a/tests/go_client/testcases/collection_test.go +++ b/tests/go_client/testcases/collection_test.go @@ -200,7 +200,7 @@ func TestCreateArrayCollections(t *testing.T) { // test create collection with partition key not supported field type func TestCreateCollectionPartitionKey(t *testing.T) { - ctx := hp.CreateContext(t, time.Second*common.DefaultTimeout) + ctx := hp.CreateContext(t, time.Second*common.DefaultTimeout*2) mc := createDefaultMilvusClient(ctx, t) int64Field := entity.NewField().WithName(common.DefaultInt64FieldName).WithDataType(entity.FieldTypeInt64).WithIsPrimaryKey(true) @@ -421,7 +421,7 @@ func TestCreateCollectionWithValidName(t *testing.T) { func TestCreateCollectionWithInvalidFieldName(t *testing.T) { t.Parallel() // connect - ctx := hp.CreateContext(t, time.Second*common.DefaultTimeout) + ctx := hp.CreateContext(t, time.Second*common.DefaultTimeout*2) mc := createDefaultMilvusClient(ctx, t) // create collection with invalid field name @@ -436,7 +436,8 @@ func TestCreateCollectionWithInvalidFieldName(t *testing.T) { common.CheckErr(t, err, false, "field name should not be empty", "The first character of a field name must be an underscore or letter", "Field name cannot only contain numbers, letters, and underscores", - "The length of a field name must be less than 255 characters") + "The length of a field name must be less than 255 characters", + "Field name can only contain numbers, letters, and underscores") } } @@ -515,7 +516,7 @@ func TestCreateCollectionInvalidFields(t *testing.T) { // create autoID or not collection with non-int64 and non-varchar field func TestCreateCollectionInvalidAutoPkField(t *testing.T) { - ctx := hp.CreateContext(t, time.Second*common.DefaultTimeout) + ctx := hp.CreateContext(t, time.Second*common.DefaultTimeout*2) mc := createDefaultMilvusClient(ctx, t) t.Parallel() // create collection with autoID true or not @@ -557,7 +558,7 @@ func TestCreateCollectionDuplicateField(t *testing.T) { // test create collection with partition key not supported field type func TestCreateCollectionInvalidPartitionKeyType(t *testing.T) { - ctx := hp.CreateContext(t, time.Second*common.DefaultTimeout) + ctx := hp.CreateContext(t, time.Second*common.DefaultTimeout*2) mc := createDefaultMilvusClient(ctx, t) int64Field := entity.NewField().WithName(common.DefaultInt64FieldName).WithDataType(entity.FieldTypeInt64).WithIsPrimaryKey(true) diff --git a/tests/go_client/testcases/helper/data_helper.go b/tests/go_client/testcases/helper/data_helper.go index 06397d3fb50b..a6ef4364d009 100644 --- a/tests/go_client/testcases/helper/data_helper.go +++ b/tests/go_client/testcases/helper/data_helper.go @@ -394,3 +394,22 @@ func GenDynamicColumnData(start int, nb int) []column.Column { } return data } + +func MergeColumnsToDynamic(nb int, columns []column.Column, columnName string) *column.ColumnJSONBytes { + values := make([][]byte, 0, nb) + for i := 0; i < nb; i++ { + m := make(map[string]interface{}) + for _, c := range columns { + // range guaranteed + m[c.Name()], _ = c.Get(i) + } + bs, err := json.Marshal(&m) + if err != nil { + log.Fatal("MergeColumnsToDynamic failed:", zap.Error(err)) + } + values = append(values, bs) + } + jsonColumn := column.NewColumnJSONBytes(columnName, values).WithIsDynamic(true) + + return jsonColumn +} diff --git a/tests/go_client/testcases/helper/helper.go b/tests/go_client/testcases/helper/helper.go index 7ce8e6f09b02..079467e813c8 100644 --- a/tests/go_client/testcases/helper/helper.go +++ b/tests/go_client/testcases/helper/helper.go @@ -136,16 +136,15 @@ func (chainTask *CollectionPrepare) InsertData(ctx context.Context, t *testing.T } fields := ip.Schema.Fields insertOpt := clientv2.NewColumnBasedInsertOption(ip.Schema.CollectionName) + if ip.Schema.EnableDynamicField { + insertOpt.WithColumns(GenDynamicColumnData(option.start, ip.Nb)...) + } for _, field := range fields { - if field.IsDynamic { - insertOpt.WithColumns(GenDynamicColumnData(option.start, ip.Nb)...) - } else { - if field.DataType == entity.FieldTypeArray { - option.TWithElementType(field.ElementType) - } - column := GenColumnData(ip.Nb, field.DataType, *option) - insertOpt.WithColumns(column) + if field.DataType == entity.FieldTypeArray { + option.TWithElementType(field.ElementType) } + column := GenColumnData(ip.Nb, field.DataType, *option) + insertOpt.WithColumns(column) } if ip.PartitionName != "" { insertOpt.WithPartition(ip.PartitionName) diff --git a/tests/go_client/testcases/query_test.go b/tests/go_client/testcases/query_test.go new file mode 100644 index 000000000000..da7c4a29b7d9 --- /dev/null +++ b/tests/go_client/testcases/query_test.go @@ -0,0 +1,612 @@ +package testcases + +import ( + "fmt" + "testing" + "time" + + "github.com/stretchr/testify/require" + "go.uber.org/zap" + + clientv2 "github.com/milvus-io/milvus/client/v2" + "github.com/milvus-io/milvus/client/v2/column" + "github.com/milvus-io/milvus/client/v2/entity" + "github.com/milvus-io/milvus/pkg/log" + "github.com/milvus-io/milvus/tests/go_client/common" + hp "github.com/milvus-io/milvus/tests/go_client/testcases/helper" +) + +// test query from default partition +func TestQueryDefault(t *testing.T) { + ctx := hp.CreateContext(t, time.Second*common.DefaultTimeout) + mc := createDefaultMilvusClient(ctx, t) + + // create and insert + prepare, schema := hp.CollPrepare.CreateCollection(ctx, t, mc, hp.NewCreateCollectionParams(hp.Int64Vec), hp.TNewFieldsOption(), hp.TNewSchemaOption()) + _, insertRes := prepare.InsertData(ctx, t, mc, hp.NewInsertParams(schema, common.DefaultNb), hp.TNewDataOption()) + + // flush -> index -> load + prepare.FlushData(ctx, t, mc, schema.CollectionName) + prepare.CreateIndex(ctx, t, mc, hp.NewIndexParams(schema)) + prepare.Load(ctx, t, mc, hp.NewLoadParams(schema.CollectionName)) + + // query + expr := fmt.Sprintf("%s < %d", common.DefaultInt64FieldName, 100) + queryRes, err := mc.Query(ctx, clientv2.NewQueryOption(schema.CollectionName).WithFilter(expr)) + common.CheckErr(t, err, true) + common.CheckQueryResult(t, queryRes.Fields, []column.Column{insertRes.IDs.Slice(0, 100)}) +} + +// test query with varchar field filter +func TestQueryVarcharPkDefault(t *testing.T) { + ctx := hp.CreateContext(t, time.Second*common.DefaultTimeout) + mc := createDefaultMilvusClient(ctx, t) + + // create and insert + prepare, schema := hp.CollPrepare.CreateCollection(ctx, t, mc, hp.NewCreateCollectionParams(hp.VarcharBinary), hp.TNewFieldsOption(), hp.TNewSchemaOption()) + _, insertRes := prepare.InsertData(ctx, t, mc, hp.NewInsertParams(schema, common.DefaultNb), hp.TNewDataOption()) + + // flush -> index -> load + prepare.CreateIndex(ctx, t, mc, hp.NewIndexParams(schema)) + prepare.Load(ctx, t, mc, hp.NewLoadParams(schema.CollectionName)) + + // query + expr := fmt.Sprintf("%s in ['0', '1', '2', '3', '4']", common.DefaultVarcharFieldName) + queryRes, err := mc.Query(ctx, clientv2.NewQueryOption(schema.CollectionName).WithFilter(expr)) + common.CheckErr(t, err, true) + common.CheckQueryResult(t, queryRes.Fields, []column.Column{insertRes.IDs.Slice(0, 5)}) +} + +// query from not existed collection name and partition name +func TestQueryNotExistName(t *testing.T) { + ctx := hp.CreateContext(t, time.Second*common.DefaultTimeout) + mc := createDefaultMilvusClient(ctx, t) + + // query with not existed collection + expr := fmt.Sprintf("%s < %d", common.DefaultInt64FieldName, 100) + _, errCol := mc.Query(ctx, clientv2.NewQueryOption("aaa").WithFilter(expr)) + common.CheckErr(t, errCol, false, "can't find collection") + + // create -> index -> load + prepare, schema := hp.CollPrepare.CreateCollection(ctx, t, mc, hp.NewCreateCollectionParams(hp.Int64Vec), hp.TNewFieldsOption(), hp.TNewSchemaOption()) + prepare.CreateIndex(ctx, t, mc, hp.NewIndexParams(schema)) + prepare.Load(ctx, t, mc, hp.NewLoadParams(schema.CollectionName)) + + // query with not existed partition + _, errPar := mc.Query(ctx, clientv2.NewQueryOption(schema.CollectionName).WithFilter(expr).WithPartitions([]string{"aaa"})) + common.CheckErr(t, errPar, false, "partition name aaa not found") +} + +// test query with invalid partition name +func TestQueryInvalidPartitionName(t *testing.T) { + ctx := hp.CreateContext(t, time.Second*common.DefaultTimeout) + mc := createDefaultMilvusClient(ctx, t) + + // create collection and partition + _, schema := hp.CollPrepare.CreateCollection(ctx, t, mc, hp.NewCreateCollectionParams(hp.Int64Vec), hp.TNewFieldsOption(), hp.TNewSchemaOption().TWithEnableDynamicField(true)) + + expr := fmt.Sprintf("%s >= %d", common.DefaultInt64FieldName, 0) + emptyPartitionName := "" + // query from "" partitions, expect to query from default partition + _, err := mc.Query(ctx, clientv2.NewQueryOption(schema.CollectionName).WithFilter(expr).WithPartitions([]string{emptyPartitionName})) + common.CheckErr(t, err, false, "Partition name should not be empty") +} + +// test query with empty partition name +func TestQueryPartition(t *testing.T) { + parName := "p1" + + ctx := hp.CreateContext(t, time.Second*common.DefaultTimeout) + mc := createDefaultMilvusClient(ctx, t) + + // create collection and partition + prepare, schema := hp.CollPrepare.CreateCollection(ctx, t, mc, hp.NewCreateCollectionParams(hp.Int64Vec), hp.TNewFieldsOption(), hp.TNewSchemaOption().TWithEnableDynamicField(true)) + err := mc.CreatePartition(ctx, clientv2.NewCreatePartitionOption(schema.CollectionName, parName)) + common.CheckErr(t, err, true) + + // insert [0, 3000) into default, insert [3000, 6000) into parName + _, i1Res := prepare.InsertData(ctx, t, mc, hp.NewInsertParams(schema, common.DefaultNb), hp.TNewDataOption()) + _, i2Res := prepare.InsertData(ctx, t, mc, hp.NewInsertParams(schema, common.DefaultNb).TWithPartitionName(parName), hp.TNewDataOption().TWithStart(common.DefaultNb)) + + // flush -> index -> load + prepare.FlushData(ctx, t, mc, schema.CollectionName) + prepare.CreateIndex(ctx, t, mc, hp.NewIndexParams(schema)) + prepare.Load(ctx, t, mc, hp.NewLoadParams(schema.CollectionName)) + + expr := fmt.Sprintf("%s >= %d", common.DefaultInt64FieldName, 0) + expColumn := hp.GenColumnData(common.DefaultNb*2, entity.FieldTypeInt64, *hp.TNewDataOption().TWithStart(0)) + + // query with default params, expect to query from all partitions + queryRes, err := mc.Query(ctx, clientv2.NewQueryOption(schema.CollectionName).WithFilter(expr).WithConsistencyLevel(entity.ClStrong)) + common.CheckErr(t, err, true) + common.CheckQueryResult(t, queryRes.Fields, []column.Column{expColumn}) + + // query with empty partition names + queryRes, err = mc.Query(ctx, clientv2.NewQueryOption(schema.CollectionName).WithFilter(expr).WithPartitions([]string{}).WithConsistencyLevel(entity.ClStrong)) + common.CheckErr(t, err, true) + common.CheckQueryResult(t, queryRes.Fields, []column.Column{expColumn}) + + // query with default partition + queryRes, err = mc.Query(ctx, clientv2.NewQueryOption(schema.CollectionName).WithFilter(expr).WithPartitions([]string{common.DefaultPartition}).WithConsistencyLevel(entity.ClStrong)) + common.CheckErr(t, err, true) + common.CheckQueryResult(t, queryRes.Fields, []column.Column{i1Res.IDs}) + + // query with specify partition + queryRes, err = mc.Query(ctx, clientv2.NewQueryOption(schema.CollectionName).WithFilter(expr).WithPartitions([]string{parName}).WithConsistencyLevel(entity.ClStrong)) + common.CheckErr(t, err, true) + common.CheckQueryResult(t, queryRes.Fields, []column.Column{i2Res.IDs}) + + // query with all partitions + queryRes, err = mc.Query(ctx, clientv2.NewQueryOption(schema.CollectionName).WithFilter(expr).WithPartitions([]string{common.DefaultPartition, parName}).WithConsistencyLevel(entity.ClStrong)) + common.CheckErr(t, err, true) + common.CheckQueryResult(t, queryRes.Fields, []column.Column{expColumn}) +} + +// test query with invalid partition name +func TestQueryWithoutExpr(t *testing.T) { + ctx := hp.CreateContext(t, time.Second*common.DefaultTimeout) + mc := createDefaultMilvusClient(ctx, t) + + // create collection and partition + _, schema := hp.CollPrepare.CreateCollection(ctx, t, mc, hp.NewCreateCollectionParams(hp.Int64Vec), hp.TNewFieldsOption(), hp.TNewSchemaOption()) + + // query without expr + _, err := mc.Query(ctx, clientv2.NewQueryOption(schema.CollectionName)) + common.CheckErr(t, err, false, "empty expression should be used with limit") + + // query with empty expr + _, err = mc.Query(ctx, clientv2.NewQueryOption(schema.CollectionName).WithFilter("")) + common.CheckErr(t, err, false, "empty expression should be used with limit") +} + +// test query empty output fields: []string{}, []string{""} +// test query with not existed field +func TestQueryEmptyOutputFields(t *testing.T) { + t.Parallel() + ctx := hp.CreateContext(t, time.Second*common.DefaultTimeout) + mc := createDefaultMilvusClient(ctx, t) + + for _, enableDynamic := range [2]bool{true, false} { + // create -> insert -> flush -> index -> load + prepare, schema := hp.CollPrepare.CreateCollection(ctx, t, mc, hp.NewCreateCollectionParams(hp.Int64Vec), hp.TNewFieldsOption(), hp.TNewSchemaOption().TWithEnableDynamicField(enableDynamic)) + prepare.InsertData(ctx, t, mc, hp.NewInsertParams(schema, common.DefaultNb), hp.TNewDataOption()) + prepare.FlushData(ctx, t, mc, schema.CollectionName) + prepare.CreateIndex(ctx, t, mc, hp.NewIndexParams(schema)) + prepare.Load(ctx, t, mc, hp.NewLoadParams(schema.CollectionName)) + + expr := fmt.Sprintf("%s < %d", common.DefaultInt64FieldName, 10) + + // query with empty output fields []string{}-> output "int64" + queryNilOutputs, err := mc.Query(ctx, clientv2.NewQueryOption(schema.CollectionName).WithConsistencyLevel(entity.ClStrong).WithFilter(expr).WithOutputFields([]string{})) + common.CheckErr(t, err, true) + common.CheckOutputFields(t, []string{common.DefaultInt64FieldName}, queryNilOutputs.Fields) + + // query with empty output fields []string{""}-> output "int64" and dynamic field + _, err1 := mc.Query(ctx, clientv2.NewQueryOption(schema.CollectionName).WithConsistencyLevel(entity.ClStrong).WithFilter(expr).WithOutputFields([]string{""})) + if enableDynamic { + common.CheckErr(t, err1, false, "parse output field name failed") + } else { + common.CheckErr(t, err1, false, "not exist") + } + + // query with empty not existed field -> output field as dynamic or error + fakeName := "aaa" + res, err2 := mc.Query(ctx, clientv2.NewQueryOption(schema.CollectionName).WithConsistencyLevel(entity.ClStrong).WithFilter(expr).WithOutputFields([]string{fakeName})) + if enableDynamic { + common.CheckErr(t, err2, true) + for _, c := range res.Fields { + log.Debug("data", zap.String("name", c.Name()), zap.Any("type", c.Type()), zap.Any("data", c.FieldData())) + } + common.CheckOutputFields(t, []string{common.DefaultInt64FieldName, fakeName}, res.Fields) + dynamicColumn := hp.MergeColumnsToDynamic(10, hp.GenDynamicColumnData(0, 10), common.DefaultDynamicFieldName) + expColumns := []column.Column{ + hp.GenColumnData(10, entity.FieldTypeInt64, *hp.TNewDataOption()), + column.NewColumnDynamic(dynamicColumn, fakeName), + } + common.CheckQueryResult(t, expColumns, res.Fields) + } else { + common.CheckErr(t, err2, false, fmt.Sprintf("%s not exist", fakeName)) + } + } +} + +// test query output all fields and verify data +func TestOutputAllFieldsColumn(t *testing.T) { + t.Skip("https://github.com/milvus-io/milvus/issues/33848") + ctx := hp.CreateContext(t, time.Second*common.DefaultTimeout) + mc := createDefaultMilvusClient(ctx, t) + + // create collection + for _, isDynamic := range [2]bool{true, false} { + prepare, schema := hp.CollPrepare.CreateCollection(ctx, t, mc, hp.NewCreateCollectionParams(hp.AllFields), hp.TNewFieldsOption(), hp.TNewSchemaOption().TWithEnableDynamicField(isDynamic)) + prepare.CreateIndex(ctx, t, mc, hp.NewIndexParams(schema)) + prepare.Load(ctx, t, mc, hp.NewLoadParams(schema.CollectionName)) + + // insert + columns := make([]column.Column, 0, len(schema.Fields)+1) + dynamicColumns := hp.GenDynamicColumnData(0, common.DefaultNb) + genDataOpt := hp.TNewDataOption().TWithMaxCapacity(common.TestCapacity) + insertOpt := clientv2.NewColumnBasedInsertOption(schema.CollectionName) + for _, field := range schema.Fields { + if field.DataType == entity.FieldTypeArray { + genDataOpt.TWithElementType(field.ElementType) + } + columns = append(columns, hp.GenColumnData(common.DefaultNb, field.DataType, *genDataOpt.TWithDim(common.DefaultDim))) + } + if isDynamic { + insertOpt.WithColumns(dynamicColumns...) + } + ids, err := mc.Insert(ctx, insertOpt.WithColumns(columns...)) + common.CheckErr(t, err, true) + require.Equal(t, int64(common.DefaultNb), ids.InsertCount) + prepare.FlushData(ctx, t, mc, schema.CollectionName) + + // query output all fields -> output all fields, includes vector and $meta field + pos := 10 + allFieldsName := make([]string, 0, len(schema.Fields)) + for _, field := range schema.Fields { + allFieldsName = append(allFieldsName, field.Name) + } + if isDynamic { + allFieldsName = append(allFieldsName, common.DefaultDynamicFieldName) + } + queryResultAll, errQuery := mc.Query(ctx, clientv2.NewQueryOption(schema.CollectionName).WithConsistencyLevel(entity.ClStrong). + WithFilter(fmt.Sprintf("%s < %d", common.DefaultInt64FieldName, pos)).WithOutputFields([]string{"*"})) + common.CheckErr(t, errQuery, true) + common.CheckOutputFields(t, allFieldsName, queryResultAll.Fields) + + expColumns := make([]column.Column, 0, len(columns)+1) + for _, _column := range columns { + expColumns = append(expColumns, _column.Slice(0, pos)) + } + if isDynamic { + expColumns = append(expColumns, hp.MergeColumnsToDynamic(pos, dynamicColumns, common.DefaultDynamicFieldName)) + } + common.CheckQueryResult(t, expColumns, queryResultAll.Fields) + } +} + +// test query output all fields +func TestOutputAllFieldsRows(t *testing.T) { + t.Skip("https://github.com/milvus-io/milvus/issues/33459") + ctx := hp.CreateContext(t, time.Second*common.DefaultTimeout) + mc := createDefaultMilvusClient(ctx, t) + + // create collection + prepare, schema := hp.CollPrepare.CreateCollection(ctx, t, mc, hp.NewCreateCollectionParams(hp.AllFields), hp.TNewFieldsOption(), + hp.TNewSchemaOption().TWithEnableDynamicField(true)) + + // prepare and insert data + rows := hp.GenAllFieldsRows(common.DefaultNb, false, *hp.TNewDataOption().TWithMaxCapacity(common.TestCapacity)) + ids, err := mc.Insert(ctx, clientv2.NewRowBasedInsertOption(schema.CollectionName, rows...)) + common.CheckErr(t, err, true) + require.Equal(t, int64(common.DefaultNb), ids.InsertCount) + + prepare.FlushData(ctx, t, mc, schema.CollectionName) + prepare.CreateIndex(ctx, t, mc, hp.NewIndexParams(schema)) + prepare.Load(ctx, t, mc, hp.NewLoadParams(schema.CollectionName)) + + // query output all fields -> output all fields, includes vector and $meta field + allFieldsName := []string{common.DefaultDynamicFieldName} + for _, field := range schema.Fields { + allFieldsName = append(allFieldsName, field.Name) + } + queryResultAll, errQuery := mc.Query(ctx, clientv2.NewQueryOption(schema.CollectionName).WithConsistencyLevel(entity.ClStrong). + WithFilter(fmt.Sprintf("%s < %d", common.DefaultInt64FieldName, 10)).WithOutputFields([]string{"*"})) + common.CheckErr(t, errQuery, true) + common.CheckOutputFields(t, allFieldsName, queryResultAll.Fields) +} + +// test query output varchar and binaryVector fields +func TestQueryOutputBinaryAndVarchar(t *testing.T) { + ctx := hp.CreateContext(t, time.Second*common.DefaultTimeout) + mc := createDefaultMilvusClient(ctx, t) + + prepare, schema := hp.CollPrepare.CreateCollection(ctx, t, mc, hp.NewCreateCollectionParams(hp.VarcharBinary), hp.TNewFieldsOption(), hp.TNewSchemaOption().TWithEnableDynamicField(true)) + prepare.CreateIndex(ctx, t, mc, hp.NewIndexParams(schema)) + prepare.Load(ctx, t, mc, hp.NewLoadParams(schema.CollectionName)) + + // insert + columns := make([]column.Column, 0, len(schema.Fields)+1) + dynamicColumns := hp.GenDynamicColumnData(0, common.DefaultNb) + + for _, field := range schema.Fields { + columns = append(columns, hp.GenColumnData(common.DefaultNb, field.DataType, *hp.TNewDataOption().TWithDim(common.DefaultDim))) + } + ids, err := mc.Insert(ctx, clientv2.NewColumnBasedInsertOption(schema.CollectionName, columns...).WithColumns(dynamicColumns...)) + common.CheckErr(t, err, true) + require.Equal(t, int64(common.DefaultNb), ids.InsertCount) + prepare.FlushData(ctx, t, mc, schema.CollectionName) + + // query output all fields -> output all fields, includes vector and $meta field + expr := fmt.Sprintf("%s in ['0', '1', '2', '3', '4', '5'] ", common.DefaultVarcharFieldName) + allFieldsName := []string{common.DefaultVarcharFieldName, common.DefaultBinaryVecFieldName, common.DefaultDynamicFieldName} + queryResultAll, errQuery := mc.Query(ctx, clientv2.NewQueryOption(schema.CollectionName).WithConsistencyLevel(entity.ClStrong). + WithFilter(expr).WithOutputFields([]string{"*"})) + common.CheckErr(t, errQuery, true) + common.CheckOutputFields(t, allFieldsName, queryResultAll.Fields) + + expColumns := []column.Column{hp.MergeColumnsToDynamic(6, dynamicColumns, common.DefaultDynamicFieldName)} + for _, _column := range columns { + expColumns = append(expColumns, _column.Slice(0, 6)) + } + common.CheckQueryResult(t, expColumns, queryResultAll.Fields) +} + +func TestQueryOutputSparse(t *testing.T) { + t.Skip("https://github.com/milvus-io/milvus-sdk-go/issues/769") + t.Parallel() + ctx := hp.CreateContext(t, time.Second*common.DefaultTimeout) + mc := createDefaultMilvusClient(ctx, t) + + prepare, schema := hp.CollPrepare.CreateCollection(ctx, t, mc, hp.NewCreateCollectionParams(hp.Int64VarcharSparseVec), hp.TNewFieldsOption(), hp.TNewSchemaOption()) + prepare.CreateIndex(ctx, t, mc, hp.NewIndexParams(schema)) + prepare.Load(ctx, t, mc, hp.NewLoadParams(schema.CollectionName)) + + // insert + columns := make([]column.Column, 0, len(schema.Fields)) + for _, field := range schema.Fields { + columns = append(columns, hp.GenColumnData(common.DefaultNb, field.DataType, *hp.TNewDataOption().TWithSparseMaxLen(10))) + } + + ids, err := mc.Insert(ctx, clientv2.NewColumnBasedInsertOption(schema.CollectionName, columns...)) + common.CheckErr(t, err, true) + require.Equal(t, int64(common.DefaultNb), ids.InsertCount) + prepare.FlushData(ctx, t, mc, schema.CollectionName) + + // query output all fields -> output all fields, includes vector and $meta field + expr := fmt.Sprintf("%s < 100 ", common.DefaultInt64FieldName) + expFieldsName := []string{common.DefaultInt64FieldName, common.DefaultVarcharFieldName, common.DefaultSparseVecFieldName} + queryResultAll, errQuery := mc.Query(ctx, clientv2.NewQueryOption(schema.CollectionName).WithConsistencyLevel(entity.ClStrong).WithFilter(expr).WithOutputFields([]string{"*"})) + common.CheckErr(t, errQuery, true) + common.CheckOutputFields(t, expFieldsName, queryResultAll.Fields) + + expColumns := make([]column.Column, 0, len(columns)) + for _, _column := range columns { + expColumns = append(expColumns, _column.Slice(0, 100)) + } + common.CheckQueryResult(t, expColumns, queryResultAll.Fields) +} + +// test query different array rows has different element length +func TestQueryArrayDifferentLenBetweenRows(t *testing.T) { + ctx := hp.CreateContext(t, time.Second*common.DefaultTimeout) + mc := createDefaultMilvusClient(ctx, t) + + prepare, schema := hp.CollPrepare.CreateCollection(ctx, t, mc, hp.NewCreateCollectionParams(hp.Int64VecAllScalar), + hp.TNewFieldsOption().TWithMaxCapacity(common.TestCapacity*2), hp.TNewSchemaOption()) + prepare.CreateIndex(ctx, t, mc, hp.NewIndexParams(schema)) + prepare.Load(ctx, t, mc, hp.NewLoadParams(schema.CollectionName)) + + // insert 2 batch with array capacity 100 and 200 + for i := 0; i < 2; i++ { + columns := make([]column.Column, 0, len(schema.Fields)) + // each batch has different array capacity + genDataOpt := hp.TNewDataOption().TWithMaxCapacity(common.TestCapacity * (i + 1)).TWithStart(common.DefaultNb * i) + for _, field := range schema.Fields { + if field.DataType == entity.FieldTypeArray { + genDataOpt.TWithElementType(field.ElementType) + } + columns = append(columns, hp.GenColumnData(common.DefaultNb, field.DataType, *genDataOpt)) + } + ids, err := mc.Insert(ctx, clientv2.NewColumnBasedInsertOption(schema.CollectionName, columns...)) + common.CheckErr(t, err, true) + require.Equal(t, int64(common.DefaultNb), ids.InsertCount) + } + prepare.FlushData(ctx, t, mc, schema.CollectionName) + + // query array idx exceeds max capacity, array[200] + expr := fmt.Sprintf("%s[%d] > 0", common.DefaultInt64ArrayField, common.TestCapacity*2) + countRes, err := mc.Query(ctx, clientv2.NewQueryOption(schema.CollectionName).WithConsistencyLevel(entity.ClStrong).WithFilter(expr).WithOutputFields([]string{common.QueryCountFieldName})) + common.CheckErr(t, err, true) + count, _ := countRes.Fields[0].GetAsInt64(0) + require.Equal(t, int64(0), count) + + countRes, err = mc.Query(ctx, clientv2.NewQueryOption(schema.CollectionName).WithConsistencyLevel(entity.ClStrong).WithFilter(expr).WithOutputFields([]string{"Count(*)"})) + common.CheckErr(t, err, true) + count, _ = countRes.Fields[0].GetAsInt64(0) + require.Equal(t, int64(0), count) + + // query: some rows has element greater than expr index array[100] + expr2 := fmt.Sprintf("%s[%d] > 0", common.DefaultInt64ArrayField, common.TestCapacity) + countRes2, err2 := mc.Query(ctx, clientv2.NewQueryOption(schema.CollectionName).WithConsistencyLevel(entity.ClStrong).WithFilter(expr2).WithOutputFields([]string{common.QueryCountFieldName})) + common.CheckErr(t, err2, true) + count2, _ := countRes2.Fields[0].GetAsInt64(0) + require.Equal(t, int64(common.DefaultNb), count2) +} + +// test query with expr and verify output dynamic field data +func TestQueryJsonDynamicExpr(t *testing.T) { + ctx := hp.CreateContext(t, time.Second*common.DefaultTimeout) + mc := createDefaultMilvusClient(ctx, t) + + prepare, schema := hp.CollPrepare.CreateCollection(ctx, t, mc, hp.NewCreateCollectionParams(hp.Int64VecJSON), + hp.TNewFieldsOption(), hp.TNewSchemaOption().TWithEnableDynamicField(true)) + prepare.InsertData(ctx, t, mc, hp.NewInsertParams(schema, common.DefaultNb), hp.TNewDataOption()) + prepare.CreateIndex(ctx, t, mc, hp.NewIndexParams(schema)) + prepare.Load(ctx, t, mc, hp.NewLoadParams(schema.CollectionName)) + + // query with different expr and count + expr := fmt.Sprintf("%s['number'] < 10 || %s < 10", common.DefaultJSONFieldName, common.DefaultDynamicNumberField) + + queryRes, err := mc.Query(ctx, clientv2.NewQueryOption(schema.CollectionName).WithFilter(expr).WithConsistencyLevel(entity.ClStrong). + WithOutputFields([]string{common.DefaultJSONFieldName, common.DefaultDynamicFieldName})) + + // verify output fields and count, dynamicNumber value + common.CheckErr(t, err, true) + common.CheckOutputFields(t, []string{common.DefaultInt64FieldName, common.DefaultJSONFieldName, common.DefaultDynamicFieldName}, queryRes.Fields) + require.Equal(t, 10, queryRes.ResultCount) + for _, _column := range queryRes.Fields { + if _column.Name() == common.DefaultDynamicNumberField { + var numberData []int64 + for i := 0; i < _column.Len(); i++ { + line, _ := _column.GetAsInt64(i) + numberData = append(numberData, line) + } + require.Equal(t, numberData, []int64{0, 1, 2, 3, 4, 5, 6, 7, 8, 9}) + } + } +} + +// test query with invalid expr +func TestQueryInvalidExpr(t *testing.T) { + ctx := hp.CreateContext(t, time.Second*common.DefaultTimeout) + mc := createDefaultMilvusClient(ctx, t) + + prepare, schema := hp.CollPrepare.CreateCollection(ctx, t, mc, hp.NewCreateCollectionParams(hp.Int64VecJSON), + hp.TNewFieldsOption(), hp.TNewSchemaOption().TWithEnableDynamicField(true)) + prepare.InsertData(ctx, t, mc, hp.NewInsertParams(schema, 100), hp.TNewDataOption()) + prepare.CreateIndex(ctx, t, mc, hp.NewIndexParams(schema)) + prepare.Load(ctx, t, mc, hp.NewLoadParams(schema.CollectionName)) + + for _, _invalidExpr := range common.InvalidExpressions { + _, err := mc.Query(ctx, clientv2.NewQueryOption(schema.CollectionName).WithConsistencyLevel(entity.ClStrong).WithFilter(_invalidExpr.Expr)) + common.CheckErr(t, err, _invalidExpr.ErrNil, _invalidExpr.ErrMsg) + } +} + +// Test query json and dynamic collection with string expr +func TestQueryCountJsonDynamicExpr(t *testing.T) { + ctx := hp.CreateContext(t, time.Second*common.DefaultTimeout) + mc := createDefaultMilvusClient(ctx, t) + + prepare, schema := hp.CollPrepare.CreateCollection(ctx, t, mc, hp.NewCreateCollectionParams(hp.AllFields), + hp.TNewFieldsOption(), hp.TNewSchemaOption().TWithEnableDynamicField(true)) + prepare.InsertData(ctx, t, mc, hp.NewInsertParams(schema, common.DefaultNb), hp.TNewDataOption()) + prepare.CreateIndex(ctx, t, mc, hp.NewIndexParams(schema)) + prepare.Load(ctx, t, mc, hp.NewLoadParams(schema.CollectionName)) + + // query with different expr and count + type exprCount struct { + expr string + count int64 + } + exprCounts := []exprCount{ + {expr: "", count: common.DefaultNb}, + // pk int64 field expr: < in && || + {expr: fmt.Sprintf("%s < 1000", common.DefaultInt64FieldName), count: 1000}, + {expr: fmt.Sprintf("%s in [0, 1, 2]", common.DefaultInt64FieldName), count: 3}, + {expr: fmt.Sprintf("%s >= 1000 && %s < 2000", common.DefaultInt64FieldName, common.DefaultInt64FieldName), count: 1000}, + {expr: fmt.Sprintf("%s >= 1000 || %s > 2000", common.DefaultInt64FieldName, common.DefaultInt64FieldName), count: 2000}, + {expr: fmt.Sprintf("%s < 1000", common.DefaultFloatFieldName), count: 1000}, + + // json and dynamic field filter expr: == < in bool/ list/ int + {expr: fmt.Sprintf("%s['number'] == 0", common.DefaultJSONFieldName), count: 0}, + {expr: fmt.Sprintf("%s['number'] < 100 and %s['number'] != 0", common.DefaultJSONFieldName, common.DefaultJSONFieldName), count: 50}, + {expr: fmt.Sprintf("%s < 100", common.DefaultDynamicNumberField), count: 100}, + {expr: "dynamicNumber % 2 == 0", count: 1500}, + {expr: fmt.Sprintf("%s['bool'] == true", common.DefaultJSONFieldName), count: 1500 / 2}, + {expr: fmt.Sprintf("%s == false", common.DefaultDynamicBoolField), count: 2000}, + {expr: fmt.Sprintf("%s in ['1', '2'] ", common.DefaultDynamicStringField), count: 2}, + {expr: fmt.Sprintf("%s['string'] in ['1', '2', '5'] ", common.DefaultJSONFieldName), count: 3}, + {expr: fmt.Sprintf("%s['list'] == [1, 2] ", common.DefaultJSONFieldName), count: 1}, + {expr: fmt.Sprintf("%s['list'] == [0, 1] ", common.DefaultJSONFieldName), count: 0}, + {expr: fmt.Sprintf("%s['list'][0] < 10 ", common.DefaultJSONFieldName), count: 5}, + {expr: fmt.Sprintf("%s[\"dynamicList\"] != [2, 3]", common.DefaultDynamicFieldName), count: 0}, + + // json contains + {expr: fmt.Sprintf("json_contains (%s['list'], 2)", common.DefaultJSONFieldName), count: 1}, + {expr: fmt.Sprintf("json_contains (%s['number'], 0)", common.DefaultJSONFieldName), count: 0}, + {expr: fmt.Sprintf("json_contains_all (%s['list'], [1, 2])", common.DefaultJSONFieldName), count: 1}, + {expr: fmt.Sprintf("JSON_CONTAINS_ANY (%s['list'], [1, 3])", common.DefaultJSONFieldName), count: 2}, + // string like + {expr: "dynamicString like '1%' ", count: 1111}, + + // key exist + {expr: fmt.Sprintf("exists %s['list']", common.DefaultJSONFieldName), count: common.DefaultNb / 4}, + {expr: "exists a ", count: 0}, + {expr: fmt.Sprintf("exists %s ", common.DefaultDynamicListField), count: common.DefaultNb}, + {expr: fmt.Sprintf("exists %s ", common.DefaultDynamicStringField), count: common.DefaultNb}, + // data type not match and no error + {expr: fmt.Sprintf("%s['number'] == '0' ", common.DefaultJSONFieldName), count: 0}, + + // json field + {expr: fmt.Sprintf("%s >= 1500", common.DefaultJSONFieldName), count: 1500 / 2}, // json >= 1500 + {expr: fmt.Sprintf("%s > 1499.5", common.DefaultJSONFieldName), count: 1500 / 2}, // json >= 1500.0 + {expr: fmt.Sprintf("%s like '21%%'", common.DefaultJSONFieldName), count: 100 / 4}, // json like '21%' + {expr: fmt.Sprintf("%s == [1503, 1504]", common.DefaultJSONFieldName), count: 1}, // json == [1,2] + {expr: fmt.Sprintf("%s[0] > 1", common.DefaultJSONFieldName), count: 1500 / 4}, // json[0] > 1 + {expr: fmt.Sprintf("%s[0][0] > 1", common.DefaultJSONFieldName), count: 0}, // json == [1,2] + } + + for _, _exprCount := range exprCounts { + log.Debug("TestQueryCountJsonDynamicExpr", zap.String("expr", _exprCount.expr)) + countRes, _ := mc.Query(ctx, clientv2.NewQueryOption(schema.CollectionName).WithConsistencyLevel(entity.ClStrong).WithFilter(_exprCount.expr).WithOutputFields([]string{common.QueryCountFieldName})) + count, _ := countRes.Fields[0].GetAsInt64(0) + require.Equal(t, _exprCount.count, count) + } +} + +// test query with all kinds of array expr +func TestQueryArrayFieldExpr(t *testing.T) { + ctx := hp.CreateContext(t, time.Second*common.DefaultTimeout) + mc := createDefaultMilvusClient(ctx, t) + + prepare, schema := hp.CollPrepare.CreateCollection(ctx, t, mc, hp.NewCreateCollectionParams(hp.AllFields), + hp.TNewFieldsOption(), hp.TNewSchemaOption().TWithEnableDynamicField(true)) + prepare.InsertData(ctx, t, mc, hp.NewInsertParams(schema, common.DefaultNb), hp.TNewDataOption()) + prepare.CreateIndex(ctx, t, mc, hp.NewIndexParams(schema)) + prepare.Load(ctx, t, mc, hp.NewLoadParams(schema.CollectionName)) + + // create collection + capacity := int64(common.TestCapacity) + type exprCount struct { + expr string + count int64 + } + exprCounts := []exprCount{ + {expr: fmt.Sprintf("%s[0] == false", common.DefaultBoolArrayField), count: common.DefaultNb / 2}, // array[0] == + {expr: fmt.Sprintf("%s[0] > 0", common.DefaultInt64ArrayField), count: common.DefaultNb - 1}, // array[0] > + {expr: fmt.Sprintf("%s[0] > 0", common.DefaultInt8ArrayField), count: 1524}, // array[0] > int8 range: [-128, 127] + {expr: fmt.Sprintf("json_contains (%s, %d)", common.DefaultInt16ArrayField, capacity), count: capacity}, // json_contains(array, 1) + {expr: fmt.Sprintf("array_contains (%s, %d)", common.DefaultInt16ArrayField, capacity), count: capacity}, // array_contains(array, 1) + {expr: fmt.Sprintf("array_contains (%s, 1)", common.DefaultInt32ArrayField), count: 2}, // array_contains(array, 1) + {expr: fmt.Sprintf("json_contains (%s, 1)", common.DefaultInt32ArrayField), count: 2}, // json_contains(array, 1) + {expr: fmt.Sprintf("array_contains (%s, 1000000)", common.DefaultInt32ArrayField), count: 0}, // array_contains(array, 1) + {expr: fmt.Sprintf("json_contains_all (%s, [90, 91])", common.DefaultInt64ArrayField), count: 91}, // json_contains_all(array, [x]) + {expr: fmt.Sprintf("array_contains_all (%s, [1, 2])", common.DefaultInt64ArrayField), count: 2}, // array_contains_all(array, [x]) + {expr: fmt.Sprintf("array_contains_any (%s, [0, 100, 10000])", common.DefaultFloatArrayField), count: 101}, // array_contains_any(array, [x]) + {expr: fmt.Sprintf("json_contains_any (%s, [0, 100, 10])", common.DefaultFloatArrayField), count: 101}, // json_contains_any (array, [x]) + {expr: fmt.Sprintf("%s == [0, 1]", common.DefaultDoubleArrayField), count: 0}, // array == + {expr: fmt.Sprintf("array_length(%s) == 10", common.DefaultVarcharArrayField), count: 0}, // array_length + {expr: fmt.Sprintf("array_length(%s) == %d", common.DefaultDoubleArrayField, capacity), count: common.DefaultNb}, // array_length + } + + for _, _exprCount := range exprCounts { + log.Debug("TestQueryCountJsonDynamicExpr", zap.String("expr", _exprCount.expr)) + countRes, _ := mc.Query(ctx, clientv2.NewQueryOption(schema.CollectionName).WithConsistencyLevel(entity.ClStrong).WithFilter(_exprCount.expr).WithOutputFields([]string{common.QueryCountFieldName})) + count, _ := countRes.Fields[0].GetAsInt64(0) + require.Equal(t, _exprCount.count, count) + } +} + +// test query output invalid count(*) fields +func TestQueryOutputInvalidOutputFieldCount(t *testing.T) { + type invalidCountStruct struct { + countField string + errMsg string + } + ctx := hp.CreateContext(t, time.Second*common.DefaultTimeout) + mc := createDefaultMilvusClient(ctx, t) + + prepare, schema := hp.CollPrepare.CreateCollection(ctx, t, mc, hp.NewCreateCollectionParams(hp.Int64Vec), + hp.TNewFieldsOption(), hp.TNewSchemaOption().TWithEnableDynamicField(false)) + prepare.InsertData(ctx, t, mc, hp.NewInsertParams(schema, common.DefaultNb), hp.TNewDataOption()) + prepare.CreateIndex(ctx, t, mc, hp.NewIndexParams(schema)) + prepare.Load(ctx, t, mc, hp.NewLoadParams(schema.CollectionName)) + + // invalid expr + invalidOutputFieldCount := []invalidCountStruct{ + {countField: "ccount(*)", errMsg: "field ccount(*) not exist"}, + {countField: "count[*]", errMsg: "field count[*] not exist"}, + {countField: "count", errMsg: "field count not exist"}, + {countField: "count(**)", errMsg: "field count(**) not exist"}, + } + for _, invalidCount := range invalidOutputFieldCount { + queryExpr := fmt.Sprintf("%s >= 0", common.DefaultInt64FieldName) + + // query with empty output fields []string{}-> output "int64" + _, err := mc.Query(ctx, clientv2.NewQueryOption(schema.CollectionName).WithConsistencyLevel(entity.ClStrong).WithFilter(queryExpr).WithOutputFields([]string{invalidCount.countField})) + common.CheckErr(t, err, false, invalidCount.errMsg) + } +}