Skip to content

Commit

Permalink
fix: add some check fieldata dim (#31564) (#31899)
Browse files Browse the repository at this point in the history 
issue: #30138
issue: #31724

pr: #31564

---------

Signed-off-by: bigsheeper <yihao.dai@zilliz.com>
Co-authored-by: cqy123456 <39671710+cqy123456@users.noreply.github.com>
  • Loading branch information
@bigsheeper @cqy123456
bigsheeper and cqy123456 committed Apr 5, 2024
1 parent ba36f66 commit fa44753
Show file tree
Hide file tree
Showing 2 changed files with 102 additions and 1 deletion.
13 changes: 12 additions & 1 deletion internal/proxy/validate_util.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -115,7 +115,10 @@ func (v *validateUtil) checkAligned(data []*schemapb.FieldData, schema *typeutil
msg := fmt.Sprintf("the num_rows (%d) of field (%s) is not equal to passed num_rows (%d)", fieldNumRows, fieldName, passedNumRows)
return merr.WrapErrParameterInvalid(passedNumRows, numRows, msg)
}

errDimMismatch := func(fieldName string, dataDim int64, schemaDim int64) error {
msg := fmt.Sprintf("the dim (%d) of field data(%s) is not equal to schema dim (%d)", dataDim, fieldName, schemaDim)
return merr.WrapErrParameterInvalid(dataDim, schemaDim, msg)
}
for _, field := range data {
switch field.GetType() {
case schemapb.DataType_FloatVector:
Expand All @@ -133,6 +136,10 @@ func (v *validateUtil) checkAligned(data []*schemapb.FieldData, schema *typeutil
if err != nil {
return err
}
dataDim := field.GetVectors().Dim
if dataDim != dim {
return errDimMismatch(field.GetFieldName(), dataDim, dim)
}

if n != numRows {
return errNumRowsMismatch(field.GetFieldName(), n, numRows)
Expand All @@ -148,6 +155,10 @@ func (v *validateUtil) checkAligned(data []*schemapb.FieldData, schema *typeutil
if err != nil {
return err
}
dataDim := field.GetVectors().Dim
if dataDim != dim {
return errDimMismatch(field.GetFieldName(), dataDim, dim)
}

n, err := funcutil.GetNumRowsOfBinaryVectorField(field.GetVectors().GetBinaryVector(), dim)
if err != nil {
Expand Down
90 changes: 90 additions & 0 deletions internal/proxy/validate_util_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -316,6 +316,48 @@ func Test_validateUtil_checkAligned(t *testing.T) {
assert.Error(t, err)
})

t.Run("field_data dim not match schema dim", func(t *testing.T) {
data := []*schemapb.FieldData{
{
FieldName: "test",
Type: schemapb.DataType_FloatVector,
Field: &schemapb.FieldData_Vectors{
Vectors: &schemapb.VectorField{
Data: &schemapb.VectorField_FloatVector{
FloatVector: &schemapb.FloatArray{
Data: []float32{1.1, 2.2, 3.3, 4.4, 5.5, 6.6, 7.7, 8.8},
},
},
Dim: 16,
},
},
},
}

schema := &schemapb.CollectionSchema{
Fields: []*schemapb.FieldSchema{
{
Name: "test",
DataType: schemapb.DataType_FloatVector,
TypeParams: []*commonpb.KeyValuePair{
{
Key: common.DimKey,
Value: "8",
},
},
},
},
}
h, err := typeutil.CreateSchemaHelper(schema)
assert.NoError(t, err)

v := newValidateUtil()

err = v.checkAligned(data, h, 1)

assert.Error(t, err)
})

t.Run("invalid num rows", func(t *testing.T) {
data := []*schemapb.FieldData{
{
Expand All @@ -328,6 +370,7 @@ func Test_validateUtil_checkAligned(t *testing.T) {
Data: []float32{1.1, 2.2},
},
},
Dim: 8,
},
},
},
Expand Down Expand Up @@ -369,6 +412,7 @@ func Test_validateUtil_checkAligned(t *testing.T) {
Data: []float32{1.1, 2.2, 3.3, 4.4, 5.5, 6.6, 7.7, 8.8},
},
},
Dim: 8,
},
},
},
Expand Down Expand Up @@ -445,6 +489,46 @@ func Test_validateUtil_checkAligned(t *testing.T) {
assert.Error(t, err)
})

t.Run("field data dim not match schema dim", func(t *testing.T) {
data := []*schemapb.FieldData{
{
FieldName: "test",
Type: schemapb.DataType_BinaryVector,
Field: &schemapb.FieldData_Vectors{
Vectors: &schemapb.VectorField{
Data: &schemapb.VectorField_BinaryVector{
BinaryVector: []byte("66666666"),
},
Dim: 128,
},
},
},
}

schema := &schemapb.CollectionSchema{
Fields: []*schemapb.FieldSchema{
{
Name: "test",
DataType: schemapb.DataType_BinaryVector,
TypeParams: []*commonpb.KeyValuePair{
{
Key: common.DimKey,
Value: "8",
},
},
},
},
}
h, err := typeutil.CreateSchemaHelper(schema)
assert.NoError(t, err)

v := newValidateUtil()

err = v.checkAligned(data, h, 100)

assert.Error(t, err)
})

t.Run("invalid num rows", func(t *testing.T) {
data := []*schemapb.FieldData{
{
Expand All @@ -455,6 +539,7 @@ func Test_validateUtil_checkAligned(t *testing.T) {
Data: &schemapb.VectorField_BinaryVector{
BinaryVector: []byte("not128"),
},
Dim: 128,
},
},
},
Expand Down Expand Up @@ -494,6 +579,7 @@ func Test_validateUtil_checkAligned(t *testing.T) {
Data: &schemapb.VectorField_BinaryVector{
BinaryVector: []byte{'1', '2'},
},
Dim: 8,
},
},
},
Expand Down Expand Up @@ -575,6 +661,7 @@ func Test_validateUtil_checkAligned(t *testing.T) {
Type: schemapb.DataType_FloatVector,
Field: &schemapb.FieldData_Vectors{
Vectors: &schemapb.VectorField{
Dim: 8,
Data: &schemapb.VectorField_FloatVector{
FloatVector: &schemapb.FloatArray{
Data: generateFloatVectors(10, 8),
Expand All @@ -588,6 +675,7 @@ func Test_validateUtil_checkAligned(t *testing.T) {
Type: schemapb.DataType_BinaryVector,
Field: &schemapb.FieldData_Vectors{
Vectors: &schemapb.VectorField{
Dim: 8,
Data: &schemapb.VectorField_BinaryVector{
BinaryVector: generateBinaryVectors(10, 8),
},
Expand Down Expand Up @@ -1616,6 +1704,7 @@ func Test_validateUtil_Validate(t *testing.T) {
Data: generateFloatVectors(2, 8),
},
},
Dim: 8,
},
},
},
Expand All @@ -1627,6 +1716,7 @@ func Test_validateUtil_Validate(t *testing.T) {
Data: &schemapb.VectorField_BinaryVector{
BinaryVector: generateBinaryVectors(2, 8),
},
Dim: 8,
},
},
},
Expand Down

0 comments on commit fa44753

Please sign in to comment.