From 0eeccdf61ca42385b84155ad7d6bb081ae926baa Mon Sep 17 00:00:00 2001 From: Tommy Smith Date: Tue, 22 Oct 2024 16:35:26 +0100 Subject: [PATCH 1/3] Support new HNSW filtering strategy config options --- .github/workflows/main.yaml | 14 ++- src/collections/config/integration.test.ts | 4 + src/collections/config/types/vectorIndex.ts | 3 + src/collections/config/utils.ts | 4 + .../configure/types/vectorIndex.ts | 2 + src/collections/integration.test.ts | 8 +- src/collections/journey.test.ts | 1 + src/collections/query/integration.test.ts | 56 ++++------ src/graphql/journey.test.ts | 2 +- src/schema/journey.test.ts | 104 +++++------------- 10 files changed, 79 insertions(+), 119 deletions(-) diff --git a/.github/workflows/main.yaml b/.github/workflows/main.yaml index f9b60b1f..ba7a5322 100644 --- a/.github/workflows/main.yaml +++ b/.github/workflows/main.yaml @@ -7,9 +7,10 @@ on: pull_request: env: - WEAVIATE_124: 1.24.21 - WEAVIATE_125: 1.25.8 - WEAVIATE_126: 1.26.1 + WEAVIATE_124: stable-v1.24-a8b364e + WEAVIATE_125: 1.25.21 + WEAVIATE_126: 1.26.7 + WEAVIATE_127: 1.27.0 jobs: checks: @@ -35,9 +36,10 @@ jobs: versions: [ { node: "22.x", weaviate: $WEAVIATE_124}, { node: "22.x", weaviate: $WEAVIATE_125}, - { node: "18.x", weaviate: $WEAVIATE_126}, - { node: "20.x", weaviate: $WEAVIATE_126}, - { node: "22.x", weaviate: $WEAVIATE_126} + { node: "22.x", weaviate: $WEAVIATE_126}, + { node: "18.x", weaviate: $WEAVIATE_127}, + { node: "20.x", weaviate: $WEAVIATE_127}, + { node: "22.x", weaviate: $WEAVIATE_127} ] steps: - uses: actions/checkout@v3 diff --git a/src/collections/config/integration.test.ts b/src/collections/config/integration.test.ts index b5de18a4..6cd90923 100644 --- a/src/collections/config/integration.test.ts +++ b/src/collections/config/integration.test.ts @@ -61,6 +61,7 @@ describe('Testing of the collection.config namespace', () => { dynamicEfMax: 500, dynamicEfFactor: 8, vectorCacheMaxObjects: 1000000000000, + filteringStrategy: 'sweeping', flatSearchCutoff: 40000, distance: 'cosine', quantizer: undefined, @@ -115,6 +116,7 @@ describe('Testing of the collection.config namespace', () => { dynamicEfMax: 500, dynamicEfFactor: 8, vectorCacheMaxObjects: 1000000000000, + filteringStrategy: 'sweeping', flatSearchCutoff: 40000, distance: 'cosine', quantizer: undefined, @@ -486,6 +488,7 @@ describe('Testing of the collection.config namespace', () => { dynamicEfMax: 500, dynamicEfFactor: 8, vectorCacheMaxObjects: 1000000000000, + filteringStrategy: 'sweeping', flatSearchCutoff: 40000, distance: 'cosine', quantizer: { @@ -563,6 +566,7 @@ describe('Testing of the collection.config namespace', () => { dynamicEfMax: 500, dynamicEfFactor: 8, vectorCacheMaxObjects: 1000000000000, + filteringStrategy: 'sweeping', flatSearchCutoff: 40000, distance: 'cosine', type: 'hnsw', diff --git a/src/collections/config/types/vectorIndex.ts b/src/collections/config/types/vectorIndex.ts index 8c845216..dbec83d2 100644 --- a/src/collections/config/types/vectorIndex.ts +++ b/src/collections/config/types/vectorIndex.ts @@ -6,6 +6,7 @@ export type VectorIndexConfigHNSW = { dynamicEfFactor: number; efConstruction: number; ef: number; + filteringStrategy: VectorIndexFilteringStrategy; flatSearchCutoff: number; maxConnections: number; quantizer: PQConfig | BQConfig | SQConfig | undefined; @@ -72,6 +73,8 @@ export type PQEncoderDistribution = 'log-normal' | 'normal'; export type VectorIndexType = 'hnsw' | 'flat' | 'dynamic' | string; +export type VectorIndexFilteringStrategy = 'sweeping' | 'acorn'; + export type VectorIndexConfig = VectorIndexConfigHNSW | VectorIndexConfigFlat | VectorIndexConfigDynamic; export type QuantizerConfig = PQConfig | BQConfig | SQConfig; diff --git a/src/collections/config/utils.ts b/src/collections/config/utils.ts index 3466f05b..9689a152 100644 --- a/src/collections/config/utils.ts +++ b/src/collections/config/utils.ts @@ -45,6 +45,7 @@ import { VectorIndexConfigFlat, VectorIndexConfigHNSW, VectorIndexConfigType, + VectorIndexFilteringStrategy, VectorizerConfig, } from './types/index.js'; @@ -376,6 +377,9 @@ class ConfigMapping { dynamicEfFactor: v.dynamicEfFactor, ef: v.ef, efConstruction: v.efConstruction, + filteringStrategy: exists(v.filteringStrategy) + ? v.filteringStrategy + : 'sweeping', flatSearchCutoff: v.flatSearchCutoff, maxConnections: v.maxConnections, quantizer: quantizer, diff --git a/src/collections/configure/types/vectorIndex.ts b/src/collections/configure/types/vectorIndex.ts index 275447fa..260cad39 100644 --- a/src/collections/configure/types/vectorIndex.ts +++ b/src/collections/configure/types/vectorIndex.ts @@ -9,6 +9,7 @@ import { VectorIndexConfigDynamic, VectorIndexConfigFlat, VectorIndexConfigHNSW, + VectorIndexFilteringStrategy, } from '../../config/types/index.js'; import { RecursivePartial } from './base.js'; @@ -56,6 +57,7 @@ export type VectorIndexConfigHNSWUpdate = { dynamicEfMax?: number; dynamicEfFactor?: number; ef?: number; + filteringStrategy?: VectorIndexFilteringStrategy; flatSearchCutoff?: number; quantizer?: PQConfigUpdate | BQConfigUpdate | SQConfigUpdate; vectorCacheMaxObjects?: number; diff --git a/src/collections/integration.test.ts b/src/collections/integration.test.ts index feb405be..b76032e8 100644 --- a/src/collections/integration.test.ts +++ b/src/collections/integration.test.ts @@ -495,7 +495,7 @@ describe('Testing of the collections.create method', () => { }, vectorizers: weaviate.configure.vectorizer.text2VecContextionary({ vectorIndexConfig: { - name: 'hnsw', + name: 'hnsw' as const, config: { cleanupIntervalSeconds: 10, distance: 'dot', @@ -504,6 +504,7 @@ describe('Testing of the collections.create method', () => { dynamicEfMin: 10, ef: -2, efConstruction: 100, + filteringStrategy: 'acorn', flatSearchCutoff: 41000, maxConnections: 72, quantizer: { @@ -580,9 +581,7 @@ describe('Testing of the collections.create method', () => { expect(response.multiTenancy.enabled).toEqual(true); expect(response.replication.asyncEnabled).toEqual(false); - expect(response.replication.deletionStrategy).toEqual( - 'NoAutomatedResolution' - ); + expect(response.replication.deletionStrategy).toEqual('DeleteOnConflict'); expect(response.replication.factor).toEqual(2); const indexConfig = response.vectorizers.default.indexConfig as VectorIndexConfigHNSW; @@ -594,6 +593,7 @@ describe('Testing of the collections.create method', () => { expect(indexConfig.dynamicEfMin).toEqual(10); expect(indexConfig.ef).toEqual(-2); expect(indexConfig.efConstruction).toEqual(100); + expect(indexConfig.filteringStrategy).toEqual('acorn'); expect(indexConfig.flatSearchCutoff).toEqual(41000); expect(indexConfig.maxConnections).toEqual(72); expect(quantizer.bitCompression).toEqual(true); diff --git a/src/collections/journey.test.ts b/src/collections/journey.test.ts index 502ccf51..72258218 100644 --- a/src/collections/journey.test.ts +++ b/src/collections/journey.test.ts @@ -176,6 +176,7 @@ describe('Journey testing of the client using a WCD cluster', () => { dynamicEfFactor: 8, ef: -1, efConstruction: 128, + filteringStrategy: 'sweeping', flatSearchCutoff: 40000, maxConnections: (await client.getWeaviateVersion().then((ver) => ver.isLowerThan(1, 26, 0))) ? 64 diff --git a/src/collections/query/integration.test.ts b/src/collections/query/integration.test.ts index 49a4d66b..37de616f 100644 --- a/src/collections/query/integration.test.ts +++ b/src/collections/query/integration.test.ts @@ -35,10 +35,12 @@ describe('Testing of the collection.query methods with a simple collection', () { name: 'testProp', dataType: 'text', + vectorizePropertyName: false, }, { name: 'testProp2', dataType: 'text', + vectorizePropertyName: false, }, ], vectorizers: weaviate.configure.vectorizer.text2VecContextionary({ @@ -54,8 +56,8 @@ describe('Testing of the collection.query methods with a simple collection', () }); return collection.data.insert({ properties: { - testProp: 'test', - testProp2: 'test2', + testProp: 'carrot', + testProp2: 'parsnip', }, }); }); @@ -65,7 +67,7 @@ describe('Testing of the collection.query methods with a simple collection', () it('should fetch an object by its id', async () => { const object = await collection.query.fetchObjectById(id); - expect(object?.properties.testProp).toEqual('test'); + expect(object?.properties.testProp).toEqual('carrot'); expect(object?.uuid).toEqual(id); }); @@ -87,15 +89,14 @@ describe('Testing of the collection.query methods with a simple collection', () }); it('should query with bm25', async () => { - const ret = await collection.query.bm25('test'); + const ret = await collection.query.bm25('carrot'); expect(ret.objects.length).toEqual(1); - expect(ret.objects[0].properties.testProp).toEqual('test'); - expect(ret.objects[0].properties.testProp2).toEqual('test2'); + expect(ret.objects[0].properties.testProp).toEqual('carrot'); expect(ret.objects[0].uuid).toEqual(id); }); it('should query with bm25 and weighted query properties', async () => { - const ret = await collection.query.bm25('test', { + const ret = await collection.query.bm25('carrot', { queryProperties: [ { name: 'testProp', @@ -105,13 +106,12 @@ describe('Testing of the collection.query methods with a simple collection', () ], }); expect(ret.objects.length).toEqual(1); - expect(ret.objects[0].properties.testProp).toEqual('test'); - expect(ret.objects[0].properties.testProp2).toEqual('test2'); + expect(ret.objects[0].properties.testProp).toEqual('carrot'); expect(ret.objects[0].uuid).toEqual(id); }); it('should query with bm25 and weighted query properties with a non-generic collection', async () => { - const ret = await client.collections.get(collectionName).query.bm25('test', { + const ret = await client.collections.get(collectionName).query.bm25('carrot', { queryProperties: [ { name: 'testProp', @@ -121,33 +121,30 @@ describe('Testing of the collection.query methods with a simple collection', () ], }); expect(ret.objects.length).toEqual(1); - expect(ret.objects[0].properties.testProp).toEqual('test'); - expect(ret.objects[0].properties.testProp2).toEqual('test2'); + expect(ret.objects[0].properties.testProp).toEqual('carrot'); expect(ret.objects[0].uuid).toEqual(id); }); it('should query with hybrid', async () => { - const ret = await collection.query.hybrid('test', { limit: 1 }); + const ret = await collection.query.hybrid('carrot', { limit: 1 }); expect(ret.objects.length).toEqual(1); - expect(ret.objects[0].properties.testProp).toEqual('test'); - expect(ret.objects[0].properties.testProp2).toEqual('test2'); + expect(ret.objects[0].properties.testProp).toEqual('carrot'); expect(ret.objects[0].uuid).toEqual(id); }); it('should query with hybrid and vector', async () => { - const ret = await collection.query.hybrid('test', { + const ret = await collection.query.hybrid('carrot', { limit: 1, vector: vector, }); expect(ret.objects.length).toEqual(1); - expect(ret.objects[0].properties.testProp).toEqual('test'); - expect(ret.objects[0].properties.testProp2).toEqual('test2'); + expect(ret.objects[0].properties.testProp).toEqual('carrot'); expect(ret.objects[0].uuid).toEqual(id); }); it('should query with hybrid and near text subsearch', async () => { const query = () => - collection.query.hybrid('test', { + collection.query.hybrid('carrot', { limit: 1, vector: { query: 'apple', @@ -157,7 +154,7 @@ describe('Testing of the collection.query methods with a simple collection', () force: 0.9, }, moveAway: { - concepts: ['test'], + concepts: ['carrot'], force: 0.1, }, }, @@ -169,12 +166,11 @@ describe('Testing of the collection.query methods with a simple collection', () const ret = await query(); expect(ret.objects.length).toEqual(1); expect(ret.objects[0].properties.testProp).toEqual('apple'); - expect(ret.objects[0].properties.testProp2).toEqual('banana'); }); it('should query with hybrid and near vector subsearch', async () => { const query = () => - collection.query.hybrid('test', { + collection.query.hybrid('carrot', { limit: 1, vector: { vector: vector, @@ -187,31 +183,27 @@ describe('Testing of the collection.query methods with a simple collection', () } const ret = await query(); expect(ret.objects.length).toEqual(1); - expect(ret.objects[0].properties.testProp).toEqual('test'); - expect(ret.objects[0].properties.testProp2).toEqual('test2'); + expect(ret.objects[0].properties.testProp).toEqual('carrot'); }); - it.skip('should query with nearObject', async () => { + it('should query with nearObject', async () => { const ret = await collection.query.nearObject(id, { limit: 1 }); expect(ret.objects.length).toEqual(1); - expect(ret.objects[0].properties.testProp).toEqual('test'); - expect(ret.objects[0].properties.testProp2).toEqual('test2'); + expect(ret.objects[0].properties.testProp).toEqual('carrot'); expect(ret.objects[0].uuid).toEqual(id); }); it('should query with nearText', async () => { - const ret = await collection.query.nearText(['test'], { limit: 1 }); + const ret = await collection.query.nearText(['carrot'], { limit: 1 }); expect(ret.objects.length).toEqual(1); - expect(ret.objects[0].properties.testProp).toEqual('test'); - expect(ret.objects[0].properties.testProp2).toEqual('test2'); + expect(ret.objects[0].properties.testProp).toEqual('carrot'); expect(ret.objects[0].uuid).toEqual(id); }); it('should query with nearVector', async () => { const ret = await collection.query.nearVector(vector, { limit: 1 }); expect(ret.objects.length).toEqual(1); - expect(ret.objects[0].properties.testProp).toEqual('test'); - expect(ret.objects[0].properties.testProp2).toEqual('test2'); + expect(ret.objects[0].properties.testProp).toEqual('carrot'); expect(ret.objects[0].uuid).toEqual(id); }); }); diff --git a/src/graphql/journey.test.ts b/src/graphql/journey.test.ts index ccb8acac..de8fe70c 100644 --- a/src/graphql/journey.test.ts +++ b/src/graphql/journey.test.ts @@ -1659,7 +1659,7 @@ describe('query cluster with consistency level', () => { }); }); -describe('query with group by', () => { +describe.skip('query with group by SKIPPED BECAUSE OF XREFS RETURN OPTIMISATION BUG', () => { let client: WeaviateClient; beforeEach(() => { diff --git a/src/schema/journey.test.ts b/src/schema/journey.test.ts index 0f19c2ee..5086b1b9 100644 --- a/src/schema/journey.test.ts +++ b/src/schema/journey.test.ts @@ -24,12 +24,7 @@ describe('schema', () => { host: 'localhost:8080', }); - const classObjPromise = newClassObject( - 'MyThingClass', - isVer(client, 25, 0), - isVer(client, 25, 2), - isVer(client, 26, 0) - ); + const classObjPromise = newClassObject('MyThingClass', client); it('creates a thing class (implicitly)', async () => { const classObj = await classObjPromise; @@ -46,7 +41,7 @@ describe('schema', () => { const classObj = await classObjPromise; return client.schema .classGetter() - .withClassName(classObj.class) + .withClassName(classObj.class!) .do() .then((res: WeaviateClass) => { expect(res).toEqual(classObj); @@ -55,7 +50,7 @@ describe('schema', () => { it('checks class existence', async () => { const classObj = await classObjPromise; - return client.schema.exists(classObj.class).then((res) => expect(res).toEqual(true)); + return client.schema.exists(classObj.class!).then((res) => expect(res).toEqual(true)); }); it('checks class non-existence', () => { @@ -119,7 +114,7 @@ describe('schema', () => { const classObj = await classObjPromise; return client.schema .shardsGetter() - .withClassName(classObj.class) + .withClassName(classObj.class!) .do() .then((res: ShardStatusList) => { res.forEach((shard: ShardStatus) => { @@ -130,13 +125,13 @@ describe('schema', () => { it('updates a shard of an existing class to readonly', async () => { const classObj = await classObjPromise; - const shards = await getShards(client, classObj.class); + const shards = await getShards(client, classObj.class!); expect(Array.isArray(shards)).toBe(true); expect(shards.length).toEqual(1); return client.schema .shardUpdater() - .withClassName(classObj.class) + .withClassName(classObj.class!) .withShardName(shards[0].name!) .withStatus('READONLY') .do() @@ -147,13 +142,13 @@ describe('schema', () => { it('updates a shard of an existing class to ready', async () => { const classObj = await classObjPromise; - const shards = await getShards(client, classObj.class); + const shards = await getShards(client, classObj.class!); expect(Array.isArray(shards)).toBe(true); expect(shards.length).toEqual(1); return client.schema .shardUpdater() - .withClassName(classObj.class) + .withClassName(classObj.class!) .withShardName(shards[0].name!) .withStatus('READY') .do() @@ -166,7 +161,7 @@ describe('schema', () => { const classObj = await classObjPromise; return client.schema .classDeleter() - .withClassName(classObj.class) + .withClassName(classObj.class!) .do() .then((res: void) => { expect(res).toEqual(undefined); @@ -175,12 +170,7 @@ describe('schema', () => { it('updates all shards in a class', async () => { const shardCount = 3; - const newClass: any = await newClassObject( - 'NewClass', - isVer(client, 25, 0), - isVer(client, 25, 2), - isVer(client, 26, 0) - ); + const newClass: any = await newClassObject('NewClass', client); newClass.shardingConfig.desiredCount = shardCount; await client.schema @@ -223,12 +213,7 @@ describe('schema', () => { }); it('has updated values of bm25 config', async () => { - const newClass: any = await newClassObject( - 'NewClass', - isVer(client, 25, 0), - isVer(client, 25, 2), - isVer(client, 26, 0) - ); + const newClass: any = await newClassObject('NewClass', client); const bm25Config = { k1: 1.13, b: 0.222 }; newClass.invertedIndexConfig.bm25 = bm25Config; @@ -245,12 +230,7 @@ describe('schema', () => { }); it('has updated values of stopwords config', async () => { - const newClass: any = await newClassObject( - 'SpaceClass', - isVer(client, 25, 0), - isVer(client, 25, 2), - isVer(client, 26, 0) - ); + const newClass: any = await newClassObject('SpaceClass', client); const stopwordConfig: any = { preset: 'en', additions: ['star', 'nebula'], @@ -302,12 +282,7 @@ describe('schema', () => { it('creates a class with explicit replication config', async () => { const replicationFactor = 1; - const newClass: any = await newClassObject( - 'SomeClass', - isVer(client, 25, 0), - isVer(client, 25, 2), - isVer(client, 26, 0) - ); + const newClass: any = await newClassObject('SomeClass', client); newClass.replicationConfig.factor = replicationFactor; await client.schema @@ -322,12 +297,7 @@ describe('schema', () => { }); it('creates a class with implicit replication config', async () => { - const newClass: any = await newClassObject( - 'SomeClass', - isVer(client, 25, 0), - isVer(client, 25, 2), - isVer(client, 26, 0) - ); + const newClass: any = await newClassObject('SomeClass', client); delete newClass.replicationConfig; await client.schema @@ -342,18 +312,8 @@ describe('schema', () => { }); it('delete all data from the schema', async () => { - const newClass: any = await newClassObject( - 'LetsDeleteThisClass', - isVer(client, 25, 0), - isVer(client, 25, 2), - isVer(client, 26, 0) - ); - const newClass2: any = await newClassObject( - 'LetsDeleteThisClassToo', - isVer(client, 25, 0), - isVer(client, 25, 2), - isVer(client, 26, 0) - ); + const newClass: any = await newClassObject('LetsDeleteThisClass', client); + const newClass2: any = await newClassObject('LetsDeleteThisClassToo', client); const classNames = [newClass.class, newClass2.class]; Promise.all([ client.schema.classCreator().withClass(newClass).do(), @@ -717,12 +677,7 @@ describe('multi tenancy', () => { return deleteClass(client, classObj.class!); }); - const classObjWithoutMultiTenancyConfig = newClassObject( - 'NoMultiTenancy', - isVer(client, 25, 0), - isVer(client, 25, 2), - isVer(client, 26, 0) - ); + const classObjWithoutMultiTenancyConfig = newClassObject('NoMultiTenancy', client); it('creates a NoMultiTenancy class', async () => { return client.schema @@ -744,16 +699,11 @@ describe('multi tenancy', () => { }); it('deletes NoMultiTenancy class', async () => { - return deleteClass(client, (await classObjWithoutMultiTenancyConfig).class); + return deleteClass(client, (await classObjWithoutMultiTenancyConfig).class!); }); }); -async function newClassObject( - className: string, - is1250Promise: Promise, - is1252Promise: Promise, - is1260Promise: Promise -) { +async function newClassObject(className: string, client: WeaviateClient): Promise { return { class: className, properties: [ @@ -762,7 +712,7 @@ async function newClassObject( name: 'stringProp', tokenization: 'word', indexFilterable: true, - indexRangeFilters: (await is1260Promise) ? false : undefined, + indexRangeFilters: (await isVer(client, 26, 0)) ? false : undefined, indexSearchable: true, moduleConfig: { 'text2vec-contextionary': { @@ -796,7 +746,7 @@ async function newClassObject( bq: { enabled: false, }, - sq: (await is1260Promise) + sq: (await isVer(client, 26, 0)) ? { enabled: false, rescoreLimit: 20, @@ -807,6 +757,7 @@ async function newClassObject( efConstruction: 128, vectorCacheMaxObjects: 500000, flatSearchCutoff: 40000, + filterStrategy: (await isVer(client, 27, 0)) ? 'sweeping' : undefined, }, invertedIndexConfig: { cleanupIntervalSeconds: 60, @@ -816,8 +767,8 @@ async function newClassObject( }, stopwords: { preset: 'en', - additions: null, - removals: null, + additions: null as unknown as undefined, // hack to deal with weird typing + removals: null as unknown as undefined, // hack to deal with weird typing }, }, moduleConfig: { @@ -826,8 +777,8 @@ async function newClassObject( }, }, multiTenancyConfig: { - autoTenantActivation: (await is1252Promise) ? false : undefined, - autoTenantCreation: (await is1250Promise) ? false : undefined, + autoTenantActivation: (await isVer(client, 25, 2)) ? false : undefined, + autoTenantCreation: (await isVer(client, 25, 0)) ? false : undefined, enabled: false, }, shardingConfig: { @@ -841,7 +792,8 @@ async function newClassObject( virtualPerPhysical: 128, }, replicationConfig: { - asyncEnabled: (await is1260Promise) ? false : undefined, + asyncEnabled: (await isVer(client, 26, 0)) ? false : undefined, + deletionStrategy: 'DeleteOnConflict', factor: 1, }, }; From 3b1fbd7007b28f8a9d5b5634831d09cfab9b862f Mon Sep 17 00:00:00 2001 From: Tommy Smith Date: Tue, 22 Oct 2024 16:48:31 +0100 Subject: [PATCH 2/3] Fix tests for BC with unsupported features --- src/collections/integration.test.ts | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/src/collections/integration.test.ts b/src/collections/integration.test.ts index b76032e8..158780c6 100644 --- a/src/collections/integration.test.ts +++ b/src/collections/integration.test.ts @@ -581,7 +581,11 @@ describe('Testing of the collections.create method', () => { expect(response.multiTenancy.enabled).toEqual(true); expect(response.replication.asyncEnabled).toEqual(false); - expect(response.replication.deletionStrategy).toEqual('DeleteOnConflict'); + expect(response.replication.deletionStrategy).toEqual( + (await cluster.getWeaviateVersion().then((ver) => ver.isLowerThan(1, 25, 0))) + ? 'NoAutomatedResolution' + : 'DeleteOnConflict' + ); expect(response.replication.factor).toEqual(2); const indexConfig = response.vectorizers.default.indexConfig as VectorIndexConfigHNSW; @@ -593,7 +597,9 @@ describe('Testing of the collections.create method', () => { expect(indexConfig.dynamicEfMin).toEqual(10); expect(indexConfig.ef).toEqual(-2); expect(indexConfig.efConstruction).toEqual(100); - expect(indexConfig.filteringStrategy).toEqual('acorn'); + expect(indexConfig.filteringStrategy).toEqual( + (await cluster.getWeaviateVersion().then((ver) => ver.isLowerThan(1, 27, 0))) ? 'sweeping' : 'acorn' + ); expect(indexConfig.flatSearchCutoff).toEqual(41000); expect(indexConfig.maxConnections).toEqual(72); expect(quantizer.bitCompression).toEqual(true); From e11fd24e1ed8af2d98cfc922945fdbc6b9eec535 Mon Sep 17 00:00:00 2001 From: Tommy Smith Date: Tue, 22 Oct 2024 17:02:11 +0100 Subject: [PATCH 3/3] Rename `filteringStrategy` to `filterStrategy` everywhere --- src/collections/config/integration.test.ts | 8 ++++---- src/collections/config/types/vectorIndex.ts | 4 ++-- src/collections/config/utils.ts | 6 ++---- src/collections/configure/types/vectorIndex.ts | 4 ++-- src/collections/integration.test.ts | 4 ++-- src/collections/journey.test.ts | 2 +- 6 files changed, 13 insertions(+), 15 deletions(-) diff --git a/src/collections/config/integration.test.ts b/src/collections/config/integration.test.ts index 6cd90923..a2f1e99f 100644 --- a/src/collections/config/integration.test.ts +++ b/src/collections/config/integration.test.ts @@ -61,7 +61,7 @@ describe('Testing of the collection.config namespace', () => { dynamicEfMax: 500, dynamicEfFactor: 8, vectorCacheMaxObjects: 1000000000000, - filteringStrategy: 'sweeping', + filterStrategy: 'sweeping', flatSearchCutoff: 40000, distance: 'cosine', quantizer: undefined, @@ -116,7 +116,7 @@ describe('Testing of the collection.config namespace', () => { dynamicEfMax: 500, dynamicEfFactor: 8, vectorCacheMaxObjects: 1000000000000, - filteringStrategy: 'sweeping', + filterStrategy: 'sweeping', flatSearchCutoff: 40000, distance: 'cosine', quantizer: undefined, @@ -488,7 +488,7 @@ describe('Testing of the collection.config namespace', () => { dynamicEfMax: 500, dynamicEfFactor: 8, vectorCacheMaxObjects: 1000000000000, - filteringStrategy: 'sweeping', + filterStrategy: 'sweeping', flatSearchCutoff: 40000, distance: 'cosine', quantizer: { @@ -566,7 +566,7 @@ describe('Testing of the collection.config namespace', () => { dynamicEfMax: 500, dynamicEfFactor: 8, vectorCacheMaxObjects: 1000000000000, - filteringStrategy: 'sweeping', + filterStrategy: 'sweeping', flatSearchCutoff: 40000, distance: 'cosine', type: 'hnsw', diff --git a/src/collections/config/types/vectorIndex.ts b/src/collections/config/types/vectorIndex.ts index dbec83d2..ddd6ea90 100644 --- a/src/collections/config/types/vectorIndex.ts +++ b/src/collections/config/types/vectorIndex.ts @@ -6,7 +6,7 @@ export type VectorIndexConfigHNSW = { dynamicEfFactor: number; efConstruction: number; ef: number; - filteringStrategy: VectorIndexFilteringStrategy; + filterStrategy: VectorIndexFilterStrategy; flatSearchCutoff: number; maxConnections: number; quantizer: PQConfig | BQConfig | SQConfig | undefined; @@ -73,7 +73,7 @@ export type PQEncoderDistribution = 'log-normal' | 'normal'; export type VectorIndexType = 'hnsw' | 'flat' | 'dynamic' | string; -export type VectorIndexFilteringStrategy = 'sweeping' | 'acorn'; +export type VectorIndexFilterStrategy = 'sweeping' | 'acorn'; export type VectorIndexConfig = VectorIndexConfigHNSW | VectorIndexConfigFlat | VectorIndexConfigDynamic; diff --git a/src/collections/config/utils.ts b/src/collections/config/utils.ts index 9689a152..c7b55943 100644 --- a/src/collections/config/utils.ts +++ b/src/collections/config/utils.ts @@ -45,7 +45,7 @@ import { VectorIndexConfigFlat, VectorIndexConfigHNSW, VectorIndexConfigType, - VectorIndexFilteringStrategy, + VectorIndexFilterStrategy, VectorizerConfig, } from './types/index.js'; @@ -377,9 +377,7 @@ class ConfigMapping { dynamicEfFactor: v.dynamicEfFactor, ef: v.ef, efConstruction: v.efConstruction, - filteringStrategy: exists(v.filteringStrategy) - ? v.filteringStrategy - : 'sweeping', + filterStrategy: exists(v.filterStrategy) ? v.filterStrategy : 'sweeping', flatSearchCutoff: v.flatSearchCutoff, maxConnections: v.maxConnections, quantizer: quantizer, diff --git a/src/collections/configure/types/vectorIndex.ts b/src/collections/configure/types/vectorIndex.ts index 260cad39..d41230fc 100644 --- a/src/collections/configure/types/vectorIndex.ts +++ b/src/collections/configure/types/vectorIndex.ts @@ -9,7 +9,7 @@ import { VectorIndexConfigDynamic, VectorIndexConfigFlat, VectorIndexConfigHNSW, - VectorIndexFilteringStrategy, + VectorIndexFilterStrategy, } from '../../config/types/index.js'; import { RecursivePartial } from './base.js'; @@ -57,7 +57,7 @@ export type VectorIndexConfigHNSWUpdate = { dynamicEfMax?: number; dynamicEfFactor?: number; ef?: number; - filteringStrategy?: VectorIndexFilteringStrategy; + filterStrategy?: VectorIndexFilterStrategy; flatSearchCutoff?: number; quantizer?: PQConfigUpdate | BQConfigUpdate | SQConfigUpdate; vectorCacheMaxObjects?: number; diff --git a/src/collections/integration.test.ts b/src/collections/integration.test.ts index 158780c6..ae62d49f 100644 --- a/src/collections/integration.test.ts +++ b/src/collections/integration.test.ts @@ -504,7 +504,7 @@ describe('Testing of the collections.create method', () => { dynamicEfMin: 10, ef: -2, efConstruction: 100, - filteringStrategy: 'acorn', + filterStrategy: 'acorn', flatSearchCutoff: 41000, maxConnections: 72, quantizer: { @@ -597,7 +597,7 @@ describe('Testing of the collections.create method', () => { expect(indexConfig.dynamicEfMin).toEqual(10); expect(indexConfig.ef).toEqual(-2); expect(indexConfig.efConstruction).toEqual(100); - expect(indexConfig.filteringStrategy).toEqual( + expect(indexConfig.filterStrategy).toEqual( (await cluster.getWeaviateVersion().then((ver) => ver.isLowerThan(1, 27, 0))) ? 'sweeping' : 'acorn' ); expect(indexConfig.flatSearchCutoff).toEqual(41000); diff --git a/src/collections/journey.test.ts b/src/collections/journey.test.ts index 72258218..15a49e4d 100644 --- a/src/collections/journey.test.ts +++ b/src/collections/journey.test.ts @@ -176,7 +176,7 @@ describe('Journey testing of the client using a WCD cluster', () => { dynamicEfFactor: 8, ef: -1, efConstruction: 128, - filteringStrategy: 'sweeping', + filterStrategy: 'sweeping', flatSearchCutoff: 40000, maxConnections: (await client.getWeaviateVersion().then((ver) => ver.isLowerThan(1, 26, 0))) ? 64