Skip to content

Commit

Permalink
fix: add some check fieldata dim (milvus-io#31564)
Browse files Browse the repository at this point in the history 
Signed-off-by: bigsheeper <yihao.dai@zilliz.com>
  • Loading branch information
@cqy123456 @bigsheeper
cqy123456 authored and bigsheeper committed Apr 3, 2024
1 parent ba36f66 commit 70d5f35
Show file tree
Hide file tree
Showing 2 changed files with 58 additions and 38 deletions.
15 changes: 13 additions & 2 deletions internal/proxy/validate_util.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -115,7 +115,10 @@ func (v *validateUtil) checkAligned(data []*schemapb.FieldData, schema *typeutil
msg := fmt.Sprintf("the num_rows (%d) of field (%s) is not equal to passed num_rows (%d)", fieldNumRows, fieldName, passedNumRows)
return merr.WrapErrParameterInvalid(passedNumRows, numRows, msg)
}

errDimMismatch := func(fieldName string, dataDim int64, schemaDim int64) error {
msg := fmt.Sprintf("the dim (%d) of field data(%s) is not equal to schema dim (%d)", dataDim, fieldName, schemaDim)
return merr.WrapErrParameterInvalid(dataDim, schemaDim, msg)
}
for _, field := range data {
switch field.GetType() {
case schemapb.DataType_FloatVector:
Expand All @@ -133,6 +136,10 @@ func (v *validateUtil) checkAligned(data []*schemapb.FieldData, schema *typeutil
if err != nil {
return err
}
dataDim := field.GetVectors().Dim
if dataDim != dim {
return errDimMismatch(field.GetFieldName(), dataDim, dim)
}

if n != numRows {
return errNumRowsMismatch(field.GetFieldName(), n, numRows)
Expand All @@ -148,14 +155,18 @@ func (v *validateUtil) checkAligned(data []*schemapb.FieldData, schema *typeutil
if err != nil {
return err
}
dataDim := field.GetVectors().Dim
if dataDim != dim {
return errDimMismatch(field.GetFieldName(), dataDim, dim)
}

n, err := funcutil.GetNumRowsOfBinaryVectorField(field.GetVectors().GetBinaryVector(), dim)
if err != nil {
return err
}

if n != numRows {
return errNumRowsMismatch(field.GetFieldName(), n, numRows)
return errNumRowsMismatch(field.GetFieldName(), n)
}

default:
Expand Down
81 changes: 45 additions & 36 deletions internal/proxy/validate_util_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -316,7 +316,7 @@ func Test_validateUtil_checkAligned(t *testing.T) {
assert.Error(t, err)
})

t.Run("invalid num rows", func(t *testing.T) {
t.Run("field_data dim not match schema dim", func(t *testing.T) {
data := []*schemapb.FieldData{
{
FieldName: "test",
Expand All @@ -325,9 +325,10 @@ func Test_validateUtil_checkAligned(t *testing.T) {
Vectors: &schemapb.VectorField{
Data: &schemapb.VectorField_FloatVector{
FloatVector: &schemapb.FloatArray{
Data: []float32{1.1, 2.2},
Data: []float32{1.1, 2.2, 3.3, 4.4, 5.5, 6.6, 7.7, 8.8},
},
},
Dim: 16,
},
},
},
Expand All @@ -352,12 +353,12 @@ func Test_validateUtil_checkAligned(t *testing.T) {

v := newValidateUtil()

err = v.checkAligned(data, h, 100)
err = v.checkAligned(data, h, 1)

assert.Error(t, err)
})

t.Run("num rows mismatch", func(t *testing.T) {
t.Run("invalid num rows", func(t *testing.T) {
data := []*schemapb.FieldData{
{
FieldName: "test",
Expand All @@ -366,9 +367,10 @@ func Test_validateUtil_checkAligned(t *testing.T) {
Vectors: &schemapb.VectorField{
Data: &schemapb.VectorField_FloatVector{
FloatVector: &schemapb.FloatArray{
Data: []float32{1.1, 2.2, 3.3, 4.4, 5.5, 6.6, 7.7, 8.8},
Data: []float32{1.1, 2.2},
},
},
Dim: 8,
},
},
},
Expand Down Expand Up @@ -398,17 +400,38 @@ func Test_validateUtil_checkAligned(t *testing.T) {
assert.Error(t, err)
})

//////////////////////////////////////////////////////////////////////

t.Run("binary vector column not found", func(t *testing.T) {
t.Run("num rows mismatch", func(t *testing.T) {
data := []*schemapb.FieldData{
{
FieldName: "test",
Type: schemapb.DataType_BinaryVector,
Type: schemapb.DataType_FloatVector,
Field: &schemapb.FieldData_Vectors{
Vectors: &schemapb.VectorField{
Data: &schemapb.VectorField_FloatVector{
FloatVector: &schemapb.FloatArray{
Data: []float32{1.1, 2.2, 3.3, 4.4, 5.5, 6.6, 7.7, 8.8},
},
},
Dim: 8,
},
},
},
}

schema := &schemapb.CollectionSchema{}
schema := &schemapb.CollectionSchema{
Fields: []*schemapb.FieldSchema{
{
Name: "test",
DataType: schemapb.DataType_FloatVector,
TypeParams: []*commonpb.KeyValuePair{
{
Key: common.DimKey,
Value: "8",
},
},
},
},
}
h, err := typeutil.CreateSchemaHelper(schema)
assert.NoError(t, err)

Expand All @@ -419,22 +442,17 @@ func Test_validateUtil_checkAligned(t *testing.T) {
assert.Error(t, err)
})

t.Run("binary vector column dimension not found", func(t *testing.T) {
//////////////////////////////////////////////////////////////////////

t.Run("binary vector column not found", func(t *testing.T) {
data := []*schemapb.FieldData{
{
FieldName: "test",
Type: schemapb.DataType_BinaryVector,
},
}

schema := &schemapb.CollectionSchema{
Fields: []*schemapb.FieldSchema{
{
Name: "test",
DataType: schemapb.DataType_BinaryVector,
},
},
}
schema := &schemapb.CollectionSchema{}
h, err := typeutil.CreateSchemaHelper(schema)
assert.NoError(t, err)

Expand All @@ -445,18 +463,11 @@ func Test_validateUtil_checkAligned(t *testing.T) {
assert.Error(t, err)
})

t.Run("invalid num rows", func(t *testing.T) {
t.Run("binary vector column dimension not found", func(t *testing.T) {
data := []*schemapb.FieldData{
{
FieldName: "test",
Type: schemapb.DataType_BinaryVector,
Field: &schemapb.FieldData_Vectors{
Vectors: &schemapb.VectorField{
Data: &schemapb.VectorField_BinaryVector{
BinaryVector: []byte("not128"),
},
},
},
},
}

Expand All @@ -465,12 +476,6 @@ func Test_validateUtil_checkAligned(t *testing.T) {
{
Name: "test",
DataType: schemapb.DataType_BinaryVector,
TypeParams: []*commonpb.KeyValuePair{
{
Key: common.DimKey,
Value: "128",
},
},
},
},
}
Expand All @@ -484,16 +489,17 @@ func Test_validateUtil_checkAligned(t *testing.T) {
assert.Error(t, err)
})

t.Run("num rows mismatch", func(t *testing.T) {
t.Run("field data dim not match schema dim", func(t *testing.T) {
data := []*schemapb.FieldData{
{
FieldName: "test",
Type: schemapb.DataType_BinaryVector,
Field: &schemapb.FieldData_Vectors{
Vectors: &schemapb.VectorField{
Data: &schemapb.VectorField_BinaryVector{
BinaryVector: []byte{'1', '2'},
BinaryVector: []byte("66666666"),
},
Dim: 128,
},
},
},
Expand Down Expand Up @@ -522,7 +528,6 @@ func Test_validateUtil_checkAligned(t *testing.T) {

assert.Error(t, err)
})

//////////////////////////////////////////////////////////////////

t.Run("mismatch", func(t *testing.T) {
Expand Down Expand Up @@ -580,6 +585,7 @@ func Test_validateUtil_checkAligned(t *testing.T) {
Data: generateFloatVectors(10, 8),
},
},
Dim: 8,
},
},
},
Expand All @@ -591,6 +597,7 @@ func Test_validateUtil_checkAligned(t *testing.T) {
Data: &schemapb.VectorField_BinaryVector{
BinaryVector: generateBinaryVectors(10, 8),
},
Dim: 8,
},
},
},
Expand Down Expand Up @@ -1611,6 +1618,7 @@ func Test_validateUtil_Validate(t *testing.T) {
Type: schemapb.DataType_FloatVector,
Field: &schemapb.FieldData_Vectors{
Vectors: &schemapb.VectorField{
Dim: 8,
Data: &schemapb.VectorField_FloatVector{
FloatVector: &schemapb.FloatArray{
Data: generateFloatVectors(2, 8),
Expand All @@ -1624,6 +1632,7 @@ func Test_validateUtil_Validate(t *testing.T) {
Type: schemapb.DataType_BinaryVector,
Field: &schemapb.FieldData_Vectors{
Vectors: &schemapb.VectorField{
Dim: 8,
Data: &schemapb.VectorField_BinaryVector{
BinaryVector: generateBinaryVectors(2, 8),
},
Expand Down

0 comments on commit 70d5f35

Please sign in to comment.