test: add cases to test go milvusclient query

Signed-off-by: ThreadDao <yufen.zong@zilliz.com>

client/column/columns.go

@@ -310,6 +310,25 @@ func FieldDataColumn(fd *schemapb.FieldData, begin, end int) (Column, error) {
 			vector = append(vector, v)
 		}
 		return NewColumnBFloat16Vector(fd.GetFieldName(), dim, vector), nil
+	case schemapb.DataType_SparseFloatVector:
+		sparseVectors := fd.GetVectors().GetSparseFloatVector()
+		if sparseVectors == nil {
+			return nil, errFieldDataTypeNotMatch
+		}
+		data := sparseVectors.Contents
+		if end < 0 {
+			end = len(data)
+		}
+		data = data[begin:end]
+		vectors := make([]entity.SparseEmbedding, 0, len(data))
+		for _, bs := range data {
+			vector, err := entity.DeserializeSliceSparseEmbedding(bs)
+			if err != nil {
+				return nil, err
+			}
+			vectors = append(vectors, vector)
+		}
+		return NewColumnSparseVectors(fd.GetFieldName(), vectors), nil
 	default:
 		return nil, fmt.Errorf("unsupported data type %s", fd.GetType())
 	}

client/entity/sparse.go

@@ -83,7 +83,7 @@ func (e sliceSparseEmbedding) Swap(i, j int) {
 	e.values[i], e.values[j] = e.values[j], e.values[i]
 }
 
-func deserializeSliceSparceEmbedding(bs []byte) (sliceSparseEmbedding, error) {
+func DeserializeSliceSparseEmbedding(bs []byte) (sliceSparseEmbedding, error) {
 	length := len(bs)
 	if length%8 != 0 {
 		return sliceSparseEmbedding{}, errors.New("not valid sparse embedding bytes")

client/entity/sparse_test.go

@@ -40,7 +40,7 @@ func TestSliceSparseEmbedding(t *testing.T) {
 		assert.EqualValues(t, length, se.Len())
 
 		bs := se.Serialize()
-		nv, err := deserializeSliceSparceEmbedding(bs)
+		nv, err := DeserializeSliceSparseEmbedding(bs)
 		require.NoError(t, err)
 
 		for i := 0; i < length; i++ {

tests/go_client/common/response_checker.go

@@ -60,22 +60,74 @@ func EqualColumn(t *testing.T, columnA column.Column, columnB column.Column) {
 	case entity.FieldTypeVarChar:
 		require.ElementsMatch(t, columnA.(*column.ColumnVarChar).Data(), columnB.(*column.ColumnVarChar).Data())
 	case entity.FieldTypeJSON:
-		log.Debug("columnA", zap.Any("data", columnA.(*column.ColumnJSONBytes).Data()))
-		log.Debug("columnB", zap.Any("data", columnB.(*column.ColumnJSONBytes).Data()))
-		require.ElementsMatch(t, columnA.(*column.ColumnJSONBytes).Data(), columnB.(*column.ColumnJSONBytes).Data())
+		log.Debug("data", zap.String("name", columnA.Name()), zap.Any("type", columnA.Type()), zap.Any("data", columnA.FieldData()))
+		log.Debug("data", zap.String("name", columnB.Name()), zap.Any("type", columnB.Type()), zap.Any("data", columnB.FieldData()))
+		if columnA.FieldData().IsDynamic {
+			require.ElementsMatch(t, columnA.(*column.ColumnDynamic).Data(), columnB.(*column.ColumnDynamic).Data())
+		} else {
+			require.ElementsMatch(t, columnA.(*column.ColumnJSONBytes).Data(), columnB.(*column.ColumnJSONBytes).Data())
+		}
 	case entity.FieldTypeFloatVector:
 		require.ElementsMatch(t, columnA.(*column.ColumnFloatVector).Data(), columnB.(*column.ColumnFloatVector).Data())
 	case entity.FieldTypeBinaryVector:
 		require.ElementsMatch(t, columnA.(*column.ColumnBinaryVector).Data(), columnB.(*column.ColumnBinaryVector).Data())
+	case entity.FieldTypeFloat16Vector:
+		require.ElementsMatch(t, columnA.(*column.ColumnFloat16Vector).Data(), columnB.(*column.ColumnFloat16Vector).Data())
+	case entity.FieldTypeBFloat16Vector:
+		require.ElementsMatch(t, columnA.(*column.ColumnBFloat16Vector).Data(), columnB.(*column.ColumnBFloat16Vector).Data())
+	case entity.FieldTypeSparseVector:
+		require.ElementsMatch(t, columnA.(*column.ColumnSparseFloatVector).Data(), columnB.(*column.ColumnSparseFloatVector).Data())
 	case entity.FieldTypeArray:
-		log.Info("TODO support column element type")
+		EqualArrayColumn(t, columnA, columnB)
+	default:
+		log.Info("Support column type is:", zap.Any("FieldType", []entity.FieldType{entity.FieldTypeBool,
+			entity.FieldTypeInt8, entity.FieldTypeInt16, entity.FieldTypeInt32,
+			entity.FieldTypeInt64, entity.FieldTypeFloat, entity.FieldTypeDouble, entity.FieldTypeString,
+			entity.FieldTypeVarChar, entity.FieldTypeArray, entity.FieldTypeFloatVector, entity.FieldTypeBinaryVector}))
+	}
+}
+
+// EqualColumn assert field data is equal of two columns
+func EqualArrayColumn(t *testing.T, columnA column.Column, columnB column.Column) {
+	require.Equal(t, columnA.Name(), columnB.Name())
+	require.IsType(t, columnA.Type(), entity.FieldTypeArray)
+	require.IsType(t, columnB.Type(), entity.FieldTypeArray)
+	switch columnA.(type) {
+	case *column.ColumnBoolArray:
+		require.ElementsMatch(t, columnA.(*column.ColumnBoolArray).Data(), columnB.(*column.ColumnBoolArray).Data())
+	case *column.ColumnInt8Array:
+		require.ElementsMatch(t, columnA.(*column.ColumnInt8Array).Data(), columnB.(*column.ColumnInt8Array).Data())
+	case *column.ColumnInt16Array:
+		require.ElementsMatch(t, columnA.(*column.ColumnInt16Array).Data(), columnB.(*column.ColumnInt16Array).Data())
+	case *column.ColumnInt32Array:
+		require.ElementsMatch(t, columnA.(*column.ColumnInt32Array).Data(), columnB.(*column.ColumnInt32Array).Data())
+	case *column.ColumnInt64Array:
+		require.ElementsMatch(t, columnA.(*column.ColumnInt64Array).Data(), columnB.(*column.ColumnInt64Array).Data())
+	case *column.ColumnFloatArray:
+		require.ElementsMatch(t, columnA.(*column.ColumnFloatArray).Data(), columnB.(*column.ColumnFloatArray).Data())
+	case *column.ColumnDoubleArray:
+		require.ElementsMatch(t, columnA.(*column.ColumnDoubleArray).Data(), columnB.(*column.ColumnDoubleArray).Data())
+	case *column.ColumnVarCharArray:
+		require.ElementsMatch(t, columnA.(*column.ColumnVarCharArray).Data(), columnB.(*column.ColumnVarCharArray).Data())
 	default:
-		log.Info("Support column type is:", zap.Any("FieldType", []entity.FieldType{
-			entity.FieldTypeBool,
-			entity.FieldTypeInt8, entity.FieldTypeInt16, entity.FieldTypeInt32, entity.FieldTypeInt64,
-			entity.FieldTypeFloat, entity.FieldTypeDouble, entity.FieldTypeString, entity.FieldTypeVarChar,
-			entity.FieldTypeArray, entity.FieldTypeFloatVector, entity.FieldTypeBinaryVector,
-		}))
+		log.Info("Support array element type is:", zap.Any("FieldType", []entity.FieldType{entity.FieldTypeBool, entity.FieldTypeInt8, entity.FieldTypeInt16,
+			entity.FieldTypeInt32, entity.FieldTypeInt64, entity.FieldTypeFloat, entity.FieldTypeDouble, entity.FieldTypeVarChar}))
+	}
+}
+
+// CheckInsertResult check insert result, ids len (insert count), ids data (pks, but no auto ids)
+func CheckInsertResult(t *testing.T, expIds column.Column, insertRes clientv2.InsertResult) {
+	require.Equal(t, expIds.Len(), insertRes.IDs.Len())
+	require.Equal(t, expIds.Len(), int(insertRes.InsertCount))
+	actualIds := insertRes.IDs
+	switch expIds.Type() {
+	// pk field support int64 and varchar type
+	case entity.FieldTypeInt64:
+		require.ElementsMatch(t, actualIds.(*column.ColumnInt64).Data(), expIds.(*column.ColumnInt64).Data())
+	case entity.FieldTypeVarChar:
+		require.ElementsMatch(t, actualIds.(*column.ColumnVarChar).Data(), expIds.(*column.ColumnVarChar).Data())
+	default:
+		log.Info("The primary field only support ", zap.Any("type", []entity.FieldType{entity.FieldTypeInt64, entity.FieldTypeVarChar}))
 	}
 }
 
@@ -97,18 +149,20 @@ func CheckSearchResult(t *testing.T, actualSearchResults []clientv2.ResultSet, e
 	}
 }
 
-// CheckInsertResult check insert result, ids len (insert count), ids data (pks, but no auto ids)
-func CheckInsertResult(t *testing.T, expIds column.Column, insertRes clientv2.InsertResult) {
-	require.Equal(t, expIds.Len(), insertRes.IDs.Len())
-	require.Equal(t, expIds.Len(), int(insertRes.InsertCount))
-	actualIds := insertRes.IDs
-	switch expIds.Type() {
-	// pk field support int64 and varchar type
-	case entity.FieldTypeInt64:
-		require.ElementsMatch(t, actualIds.(*column.ColumnInt64).Data(), expIds.(*column.ColumnInt64).Data())
-	case entity.FieldTypeVarChar:
-		require.ElementsMatch(t, actualIds.(*column.ColumnVarChar).Data(), expIds.(*column.ColumnVarChar).Data())
-	default:
-		log.Info("The primary field only support ", zap.Any("type", []entity.FieldType{entity.FieldTypeInt64, entity.FieldTypeVarChar}))
+// CheckQueryResult check query result, column name, type and field
+func CheckQueryResult(t *testing.T, expColumns []column.Column,  actualColumns []column.Column) {
+	require.Equal(t, len(actualColumns), len(expColumns),
+		"The len of actual columns %d should greater or equal to the expected columns %d", len(actualColumns), len(expColumns))
+	for _, expColumn := range expColumns {
+		exist := false
+		for _, actualColumn := range actualColumns {
+			if expColumn.Name() == actualColumn.Name() {
+				exist = true
+				EqualColumn(t, expColumn, actualColumn)
+			}
+		}
+		if !exist{
+			log.Error("CheckQueryResult actualColumns no column", zap.String("name", expColumn.Name()))
+		}
 	}
 }

tests/go_client/go.mod

@@ -13,7 +13,7 @@ require (
 	google.golang.org/grpc v1.64.0
 )
 
-//replace github.com/milvus-io/milvus/client/v2 v2.0.0-20240521081339-017fd7bc25de => ../../../milvus/client
+//replace github.com/milvus-io/milvus/client/v2 v2.0.0-20240612020955-8cb350598c9a => ../../../milvus/client
 
 require (
 	github.com/beorn7/perks v1.0.1 // indirect

tests/go_client/go.sum

@@ -403,10 +403,6 @@ github.com/microcosm-cc/bluemonday v1.0.2/go.mod h1:iVP4YcDBq+n/5fb23BhYFvIMq/le
 github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg=
 github.com/milvus-io/milvus-proto/go-api/v2 v2.3.4-0.20240430035521-259ae1d10016 h1:8WV4maXLeGEyJCCYIc1DmZ18H+VFAjMrwXJg5iI2nX4=
 github.com/milvus-io/milvus-proto/go-api/v2 v2.3.4-0.20240430035521-259ae1d10016/go.mod h1:1OIl0v5PQeNxIJhCvY+K55CBUOYDZevw9g9380u1Wek=
-github.com/milvus-io/milvus/client/v2 v2.0.0-20240521081339-017fd7bc25de h1:sZ1K+F+1RMlWzuAFlhF7m68e9+G6UlRDoL7Q4Bmsc08=
-github.com/milvus-io/milvus/client/v2 v2.0.0-20240521081339-017fd7bc25de/go.mod h1:FU4rii9fMaMy1CGWgjROyKo5SCaJqcxZeIhdhuuEhrQ=
-github.com/milvus-io/milvus/client/v2 v2.0.0-20240612020955-8cb350598c9a h1:iyXy8ee4pDutzO40VA9xem1O1h/o5gI5x86ZVqVzto4=
-github.com/milvus-io/milvus/client/v2 v2.0.0-20240612020955-8cb350598c9a/go.mod h1:thfuEkUztRRmQ+qu4hCoO/6uxDJoUVNNx4vHqx9yh5I=
 github.com/milvus-io/milvus/pkg v0.0.2-0.20240317152703-17b4938985f3 h1:ZBpRWhBa7FTFxW4YYVv9AUESoW1Xyb3KNXTzTqfkZmw=
 github.com/milvus-io/milvus/pkg v0.0.2-0.20240317152703-17b4938985f3/go.mod h1:jQ2BUZny1COsgv1Qbcv8dmbppW+V9J/c4YQZNb3EOm8=
 github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc=

tests/go_client/testcases/helper/data_helper.go

@@ -394,3 +394,32 @@ func GenDynamicColumnData(start int, nb int) []column.Column {
 	}
 	return data
 }
+
+func MergeColumnsToDynamic(nb int, columns []column.Column, columnName string) *column.ColumnJSONBytes {
+	values := make([][]byte, 0, nb)
+	for i := 0; i < nb; i++ {
+		m := make(map[string]interface{})
+		for _, c := range columns {
+			// range guaranteed
+			m[c.Name()], _ = c.Get(i)
+		}
+		bs, err := json.Marshal(&m)
+		if err != nil {
+			log.Fatal("MergeColumnsToDynamic failed:", zap.Error(err))
+		}
+		values = append(values, bs)
+	}
+	jsonColumn := column.NewColumnJSONBytes(columnName, values)
+
+	var jsonData []string
+	for i := 0; i < jsonColumn.Len(); i++ {
+		line, err := jsonColumn.GetAsString(i)
+		if err != nil {
+			log.Error("MergeColumnsToDynamic", zap.Error(err))
+		}
+		jsonData = append(jsonData, line)
+	}
+	jsonColumn.WithIsDynamic(true)
+
+	return jsonColumn
+}

tests/go_client/testcases/helper/helper.go

@@ -136,16 +136,15 @@ func (chainTask *CollectionPrepare) InsertData(ctx context.Context, t *testing.T
 	}
 	fields := ip.Schema.Fields
 	insertOpt := clientv2.NewColumnBasedInsertOption(ip.Schema.CollectionName)
+	if ip.Schema.EnableDynamicField{
+		insertOpt.WithColumns(GenDynamicColumnData(option.start, ip.Nb)...)
+	}
 	for _, field := range fields {
-		if field.IsDynamic {
-			insertOpt.WithColumns(GenDynamicColumnData(option.start, ip.Nb)...)
-		} else {
-			if field.DataType == entity.FieldTypeArray {
-				option.TWithElementType(field.ElementType)
-			}
-			column := GenColumnData(ip.Nb, field.DataType, *option)
-			insertOpt.WithColumns(column)
+		if field.DataType == entity.FieldTypeArray{
+			option.TWithElementType(field.ElementType)
 		}
+		column := GenColumnData(ip.Nb, field.DataType, *option)
+		insertOpt.WithColumns(column)
 	}
 	if ip.PartitionName != "" {
 		insertOpt.WithPartition(ip.PartitionName)