Skip to content

Commit

Permalink
test: Update insert data tests and refactor some data gen functions (m…
Browse files Browse the repository at this point in the history
…ilvus-io#32924)

related issue: milvus-io#32653

Signed-off-by: yanliang567 <yanliang.qiao@zilliz.com>
  • Loading branch information
yanliang567 authored and David Pichler committed May 10, 2024
1 parent 6c4c104 commit 3820470
Show file tree
Hide file tree
Showing 3 changed files with 298 additions and 469 deletions.
21 changes: 17 additions & 4 deletions tests/python_client/common/common_func.py
Original file line number Diff line number Diff line change
Expand Up @@ -899,12 +899,12 @@ def get_column_data_by_schema(nb=ct.default_nb, schema=None, skip_vectors=False,
if field.dtype == DataType.FLOAT_VECTOR and skip_vectors is True:
tmp = []
else:
tmp = gen_data_by_type(field, nb=nb, start=start)
tmp = gen_data_by_collection_field(field, nb=nb, start=start)
data.append(tmp)
return data


def get_row_data_by_schema(nb=ct.default_nb, schema=None):
def gen_row_data_by_schema(nb=ct.default_nb, schema=None):
if schema is None:
schema = gen_default_collection_schema()
fields = schema.fields
Expand All @@ -916,7 +916,7 @@ def get_row_data_by_schema(nb=ct.default_nb, schema=None):
for i in range(nb):
tmp = {}
for field in fields_not_auto_id:
tmp[field.name] = gen_data_by_type(field)
tmp[field.name] = gen_data_by_collection_field(field)
data.append(tmp)
return data

Expand Down Expand Up @@ -1016,7 +1016,7 @@ def get_dim_by_schema(schema=None):
return None


def gen_data_by_type(field, nb=None, start=None):
def gen_data_by_collection_field(field, nb=None, start=None):
# if nb is None, return one data, else return a list of data
data_type = field.dtype
if data_type == DataType.BOOL:
Expand Down Expand Up @@ -1124,6 +1124,19 @@ def gen_data_by_type(field, nb=None, start=None):
return None


def gen_data_by_collection_schema(schema, nb, r=0):
"""
gen random data by collection schema, regardless of primary key or auto_id
vector type only support for DataType.FLOAT_VECTOR
"""
data = []
start_uid = r * nb
fields = schema.fields
for field in fields:
data.append(gen_data_by_collection_field(field, nb, start_uid))
return data


def gen_json_files_for_bulk_insert(data, schema, data_dir):
for d in data:
if len(d) > 0:
Expand Down
Loading

0 comments on commit 3820470

Please sign in to comment.