-
Notifications
You must be signed in to change notification settings - Fork 37
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Support more types of sparse vectors (#293)
* generate different types of sparse vector Signed-off-by: ryjiang <jiangruiyi@gmail.com> * fix sparse array in js Signed-off-by: ryjiang <jiangruiyi@gmail.com> * add sparse array test Signed-off-by: ryjiang <jiangruiyi@gmail.com> * add csr sparse vector test Signed-off-by: ryjiang <jiangruiyi@gmail.com> * add coo support Signed-off-by: ryjiang <jiangruiyi@gmail.com> * remove unused import Signed-off-by: ryjiang <jiangruiyi@gmail.com> * refine comments Signed-off-by: ryjiang <jiangruiyi@gmail.com> * refine comment Signed-off-by: ryjiang <jiangruiyi@gmail.com> --------- Signed-off-by: ryjiang <jiangruiyi@gmail.com>
- Loading branch information
1 parent
5034a82
commit 611a550
Showing
10 changed files
with
649 additions
and
28 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,131 @@ | ||
| import { | ||
| MilvusClient, | ||
| ErrorCode, | ||
| DataType, | ||
| IndexType, | ||
| MetricType, | ||
| } from '../../milvus'; | ||
| import { | ||
| IP, | ||
| genCollectionParams, | ||
| GENERATE_NAME, | ||
| generateInsertData, | ||
| } from '../tools'; | ||
|
|
||
| const milvusClient = new MilvusClient({ address: IP, logLevel: 'info' }); | ||
| const COLLECTION_NAME = GENERATE_NAME(); | ||
|
|
||
| const dbParam = { | ||
| db_name: 'sparse_array_vector_DB', | ||
| }; | ||
|
|
||
| const p = { | ||
| collectionName: COLLECTION_NAME, | ||
| vectorType: [DataType.SparseFloatVector], | ||
| dim: [24], // useless | ||
| }; | ||
| const collectionParams = genCollectionParams(p); | ||
| const data = generateInsertData(collectionParams.fields, 10, { | ||
| sparseType: 'array', | ||
| }); | ||
|
|
||
| describe(`Sparse vectors type:object API testing`, () => { | ||
| beforeAll(async () => { | ||
| await milvusClient.createDatabase(dbParam); | ||
| await milvusClient.use(dbParam); | ||
| }); | ||
|
|
||
| afterAll(async () => { | ||
| await milvusClient.dropCollection({ collection_name: COLLECTION_NAME }); | ||
| await milvusClient.dropDatabase(dbParam); | ||
| }); | ||
|
|
||
| it(`Create collection with sparse vectors should be successful`, async () => { | ||
| const create = await milvusClient.createCollection(collectionParams); | ||
| expect(create.error_code).toEqual(ErrorCode.SUCCESS); | ||
|
|
||
| const describe = await milvusClient.describeCollection({ | ||
| collection_name: COLLECTION_NAME, | ||
| }); | ||
|
|
||
| const sparseFloatVectorFields = describe.schema.fields.filter( | ||
| (field: any) => field.data_type === 'SparseFloatVector' | ||
| ); | ||
| expect(sparseFloatVectorFields.length).toBe(1); | ||
|
|
||
| // console.dir(describe.schema, { depth: null }); | ||
| }); | ||
|
|
||
| it(`insert sparse vector data should be successful`, async () => { | ||
| const insert = await milvusClient.insert({ | ||
| collection_name: COLLECTION_NAME, | ||
| data, | ||
| }); | ||
|
|
||
| // console.log('data to insert', data); | ||
|
|
||
| expect(insert.status.error_code).toEqual(ErrorCode.SUCCESS); | ||
| expect(insert.succ_index.length).toEqual(data.length); | ||
| }); | ||
|
|
||
| it(`create index should be successful`, async () => { | ||
| const indexes = await milvusClient.createIndex([ | ||
| { | ||
| collection_name: COLLECTION_NAME, | ||
| field_name: 'vector', | ||
| metric_type: MetricType.IP, | ||
| index_type: IndexType.SPARSE_WAND, | ||
| params: { | ||
| drop_ratio_build: 0.2, | ||
| }, | ||
| }, | ||
| ]); | ||
|
|
||
| expect(indexes.error_code).toEqual(ErrorCode.SUCCESS); | ||
| }); | ||
|
|
||
| it(`load collection should be successful`, async () => { | ||
| const load = await milvusClient.loadCollection({ | ||
| collection_name: COLLECTION_NAME, | ||
| }); | ||
|
|
||
| expect(load.error_code).toEqual(ErrorCode.SUCCESS); | ||
| }); | ||
|
|
||
| it(`query sparse vector should be successful`, async () => { | ||
| const query = await milvusClient.query({ | ||
| collection_name: COLLECTION_NAME, | ||
| filter: 'id > 0', | ||
| output_fields: ['vector', 'id'], | ||
| }); | ||
|
|
||
| // console.dir(query, { depth: null }); | ||
|
|
||
| const originKeys = Object.keys(query.data[0].vector); | ||
| const originValues = Object.values(query.data[0].vector); | ||
|
|
||
| const outputKeys: string[] = Object.keys(query.data[0].vector); | ||
| const outputValues: number[] = Object.values(query.data[0].vector); | ||
|
|
||
| expect(originKeys).toEqual(outputKeys); | ||
|
|
||
| // filter undefined in originValues | ||
| originValues.forEach((value, index) => { | ||
| if (value) { | ||
| expect(value).toBeCloseTo(outputValues[index]); | ||
| } | ||
| }); | ||
| }); | ||
|
|
||
| it(`search with sparse vector should be successful`, async () => { | ||
| const search = await milvusClient.search({ | ||
| vector: data[0].vector, | ||
| collection_name: COLLECTION_NAME, | ||
| output_fields: ['id', 'vector'], | ||
| limit: 5, | ||
| }); | ||
|
|
||
| expect(search.status.error_code).toEqual(ErrorCode.SUCCESS); | ||
| expect(search.results.length).toBeGreaterThan(0); | ||
| }); | ||
| }); |
Oops, something went wrong.