From e7cf732a15f75a81b2957f894ce27da7362ab14c Mon Sep 17 00:00:00 2001 From: Tommy Smith Date: Tue, 15 Oct 2024 10:26:45 +0100 Subject: [PATCH 1/3] Add options for specifying `deletionStrategy` in replication config methods/returns --- src/collections/config/types/index.ts | 3 + src/collections/config/utils.ts | 1 + src/collections/configure/index.ts | 27 ++++++-- src/collections/configure/types/base.ts | 16 +++-- src/collections/configure/unit.test.ts | 2 + src/collections/integration.test.ts | 2 + src/openapi/schema.ts | 83 ++++++++++++++++++++++++- 7 files changed, 125 insertions(+), 9 deletions(-) diff --git a/src/collections/config/types/index.ts b/src/collections/config/types/index.ts index 2af0f07e..171cccf7 100644 --- a/src/collections/config/types/index.ts +++ b/src/collections/config/types/index.ts @@ -40,8 +40,11 @@ export type MultiTenancyConfig = { enabled: boolean; }; +export type ReplicationDeletionStrategy = 'DeleteOnConflict' | 'NoAutomatedResolution'; + export type ReplicationConfig = { asyncEnabled: boolean; + deletionStrategy: ReplicationDeletionStrategy; factor: number; }; diff --git a/src/collections/config/utils.ts b/src/collections/config/utils.ts index 79854a05..3466f05b 100644 --- a/src/collections/config/utils.ts +++ b/src/collections/config/utils.ts @@ -265,6 +265,7 @@ class ConfigMapping { return { factor: v.factor, asyncEnabled: v.asyncEnabled ? v.asyncEnabled : false, + deletionStrategy: v.deletionStrategy ? v.deletionStrategy : 'NoAutomatedResolution', }; } static sharding(v?: WeaviateShardingConfig): ShardingConfig { diff --git a/src/collections/configure/index.ts b/src/collections/configure/index.ts index 3ad323b8..26549248 100644 --- a/src/collections/configure/index.ts +++ b/src/collections/configure/index.ts @@ -3,6 +3,7 @@ import { InvertedIndexConfigUpdate, MultiTenancyConfigCreate, ReplicationConfigCreate, + ReplicationDeletionStrategy, ShardingConfigCreate, VectorConfigUpdate, VectorIndexType, @@ -131,10 +132,19 @@ const configure = { * See [the docs](https://weaviate.io/developers/weaviate/concepts/replication-architecture#replication-vs-sharding) for more details. * * @param {boolean} [options.asyncEnabled] Whether asynchronous replication is enabled. Default is false. + * @param {ReplicationDeletionStrategy} [options.deletionStrategy] The deletion strategy when replication conflicts are detected between deletes and reads. * @param {number} [options.factor] The replication factor. Default is 1. */ - replication: (options: { asyncEnabled?: boolean; factor?: number }): ReplicationConfigCreate => { - return { asyncEnabled: options.asyncEnabled, factor: options.factor }; + replication: (options: { + asyncEnabled?: boolean; + deletionStrategy?: ReplicationDeletionStrategy; + factor?: number; + }): ReplicationConfigCreate => { + return { + asyncEnabled: options.asyncEnabled, + deletionStrategy: options.deletionStrategy, + factor: options.factor, + }; }, /** * Create a `ShardingConfigCreate` object to be used when defining the sharding configuration of your collection. @@ -217,10 +227,19 @@ const reconfigure = { * See [the docs](https://weaviate.io/developers/weaviate/concepts/replication-architecture#replication-vs-sharding) for more details. * * @param {boolean} [options.asyncEnabled] Whether asynchronous replication is enabled. + * @param {ReplicationDeletionStrategy} [options.deletionStrategy] The deletion strategy when replication conflicts are detected between deletes and reads. * @param {number} [options.factor] The replication factor. */ - replication: (options: { asyncEnabled?: boolean; factor?: number }): ReplicationConfigCreate => { - return { asyncEnabled: options.asyncEnabled, factor: options.factor }; + replication: (options: { + asyncEnabled?: boolean; + deletionStrategy?: ReplicationDeletionStrategy; + factor?: number; + }): ReplicationConfigCreate => { + return { + asyncEnabled: options.asyncEnabled, + deletionStrategy: options.deletionStrategy, + factor: options.factor, + }; }, }; diff --git a/src/collections/configure/types/base.ts b/src/collections/configure/types/base.ts index 4e5d3bfd..d52050d0 100644 --- a/src/collections/configure/types/base.ts +++ b/src/collections/configure/types/base.ts @@ -1,11 +1,18 @@ import { WeaviateNestedProperty, WeaviateProperty } from '../../../openapi/types.js'; -import { InvertedIndexConfig, MultiTenancyConfig, ReplicationConfig } from '../../config/types/index.js'; +import { + InvertedIndexConfig, + MultiTenancyConfig, + ReplicationConfig, + ReplicationDeletionStrategy, +} from '../../config/types/index.js'; import { DataType } from '../../types/index.js'; import { NonRefKeys, RefKeys } from '../../types/internal.js'; -export type RecursivePartial = { - [P in keyof T]?: RecursivePartial; -}; +export type RecursivePartial = T extends object + ? { + [P in keyof T]?: RecursivePartial; + } + : T; export type InvertedIndexConfigCreate = RecursivePartial; @@ -133,6 +140,7 @@ export type ReplicationConfigCreate = RecursivePartial; export type ReplicationConfigUpdate = { asyncEnabled?: boolean; + deletionStragey?: ReplicationDeletionStrategy; factor?: number; }; diff --git a/src/collections/configure/unit.test.ts b/src/collections/configure/unit.test.ts index fbd28df3..3e68228e 100644 --- a/src/collections/configure/unit.test.ts +++ b/src/collections/configure/unit.test.ts @@ -79,10 +79,12 @@ describe('Unit testing of the configure factory class', () => { it('should create the correct ReplicationConfig type with all values', () => { const config = configure.replication({ asyncEnabled: true, + deletionStrategy: 'DeleteOnConflict', factor: 2, }); expect(config).toEqual({ asyncEnabled: true, + deletionStrategy: 'DeleteOnConflict', factor: 2, }); }); diff --git a/src/collections/integration.test.ts b/src/collections/integration.test.ts index a1e4c325..757b48cb 100644 --- a/src/collections/integration.test.ts +++ b/src/collections/integration.test.ts @@ -578,6 +578,8 @@ describe('Testing of the collections.create method', () => { expect(response.multiTenancy.enabled).toEqual(true); + expect(response.replication.asyncEnabled).toEqual(false); + expect(response.replication.deletionStrategy).toEqual('DeleteOnConflict'); expect(response.replication.factor).toEqual(2); const indexConfig = response.vectorizers.default.indexConfig as VectorIndexConfigHNSW; diff --git a/src/openapi/schema.ts b/src/openapi/schema.ts index d8ddf03b..fc08a81b 100644 --- a/src/openapi/schema.ts +++ b/src/openapi/schema.ts @@ -147,12 +147,16 @@ export interface paths { head: operations['tenant.exists']; }; '/backups/{backend}': { + /** [Coming soon] List all backups in progress not implemented yet. */ + get: operations['backups.list']; /** Starts a process of creating a backup for a set of classes */ post: operations['backups.create']; }; '/backups/{backend}/{id}': { /** Returns status of backup creation attempt for a set of classes */ get: operations['backups.create.status']; + /** Cancel created backup with specified ID */ + delete: operations['backups.cancel']; }; '/backups/{backend}/{id}/restore': { /** Returns status of a backup restoration attempt for a set of classes */ @@ -344,6 +348,11 @@ export interface definitions { factor?: number; /** @description Enable asynchronous replication */ asyncEnabled?: boolean; + /** + * @description Conflict resolution strategy for deleted objects + * @enum {string} + */ + deletionStrategy?: 'NoAutomatedResolution' | 'DeleteOnConflict'; }; /** @description tuning parameters for the BM25 algorithm */ BM25Config: { @@ -572,7 +581,7 @@ export interface definitions { * @default STARTED * @enum {string} */ - status?: 'STARTED' | 'TRANSFERRING' | 'TRANSFERRED' | 'SUCCESS' | 'FAILED'; + status?: 'STARTED' | 'TRANSFERRING' | 'TRANSFERRED' | 'SUCCESS' | 'FAILED' | 'CANCELED'; }; /** @description The definition of a backup restore metadata */ BackupRestoreStatusResponse: { @@ -648,6 +657,20 @@ export interface definitions { */ status?: 'STARTED' | 'TRANSFERRING' | 'TRANSFERRED' | 'SUCCESS' | 'FAILED'; }; + /** @description The definition of a backup create response body */ + BackupListResponse: { + /** @description The ID of the backup. Must be URL-safe and work as a filesystem path, only lowercase, numbers, underscore, minus characters allowed. */ + id?: string; + /** @description destination path of backup files proper to selected backend */ + path?: string; + /** @description The list of classes for which the existed backup process */ + classes?: string[]; + /** + * @description status of backup process + * @enum {string} + */ + status?: 'STARTED' | 'TRANSFERRING' | 'TRANSFERRED' | 'SUCCESS' | 'FAILED' | 'CANCELED'; + }[]; /** @description Request body for restoring a backup for a set of classes */ BackupRestoreRequest: { /** @description Custom configuration for the backup restoration process */ @@ -2685,6 +2708,35 @@ export interface operations { }; }; }; + /** [Coming soon] List all backups in progress not implemented yet. */ + 'backups.list': { + parameters: { + path: { + /** Backup backend name e.g. filesystem, gcs, s3. */ + backend: string; + }; + }; + responses: { + /** Existed backups */ + 200: { + schema: definitions['BackupListResponse']; + }; + /** Unauthorized or invalid credentials. */ + 401: unknown; + /** Forbidden */ + 403: { + schema: definitions['ErrorResponse']; + }; + /** Invalid backup list. */ + 422: { + schema: definitions['ErrorResponse']; + }; + /** An error has occurred while trying to fulfill the request. Most likely the ErrorResponse will contain more information about the error. */ + 500: { + schema: definitions['ErrorResponse']; + }; + }; + }; /** Starts a process of creating a backup for a set of classes */ 'backups.create': { parameters: { @@ -2752,6 +2804,35 @@ export interface operations { }; }; }; + /** Cancel created backup with specified ID */ + 'backups.cancel': { + parameters: { + path: { + /** Backup backend name e.g. filesystem, gcs, s3. */ + backend: string; + /** The ID of a backup. Must be URL-safe and work as a filesystem path, only lowercase, numbers, underscore, minus characters allowed. */ + id: string; + }; + }; + responses: { + /** Successfully deleted. */ + 204: never; + /** Unauthorized or invalid credentials. */ + 401: unknown; + /** Forbidden */ + 403: { + schema: definitions['ErrorResponse']; + }; + /** Invalid backup cancellation attempt. */ + 422: { + schema: definitions['ErrorResponse']; + }; + /** An error has occurred while trying to fulfill the request. Most likely the ErrorResponse will contain more information about the error. */ + 500: { + schema: definitions['ErrorResponse']; + }; + }; + }; /** Returns status of a backup restoration attempt for a set of classes */ 'backups.restore.status': { parameters: { From acec9f01321dbd004da06233242d706738c38d2e Mon Sep 17 00:00:00 2001 From: Tommy Smith Date: Wed, 16 Oct 2024 12:03:38 +0100 Subject: [PATCH 2/3] Fix broken tests --- src/collections/integration.test.ts | 5 ++++- src/collections/journey.test.ts | 1 + 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/src/collections/integration.test.ts b/src/collections/integration.test.ts index 757b48cb..feb405be 100644 --- a/src/collections/integration.test.ts +++ b/src/collections/integration.test.ts @@ -6,6 +6,7 @@ import { PQConfig, PhoneNumber, PropertyConfig, + ReplicationDeletionStrategy, Text2VecContextionaryConfig, Text2VecOpenAIConfig, VectorIndexConfigHNSW, @@ -579,7 +580,9 @@ describe('Testing of the collections.create method', () => { expect(response.multiTenancy.enabled).toEqual(true); expect(response.replication.asyncEnabled).toEqual(false); - expect(response.replication.deletionStrategy).toEqual('DeleteOnConflict'); + expect(response.replication.deletionStrategy).toEqual( + 'NoAutomatedResolution' + ); expect(response.replication.factor).toEqual(2); const indexConfig = response.vectorizers.default.indexConfig as VectorIndexConfigHNSW; diff --git a/src/collections/journey.test.ts b/src/collections/journey.test.ts index 5e50a070..502ccf51 100644 --- a/src/collections/journey.test.ts +++ b/src/collections/journey.test.ts @@ -145,6 +145,7 @@ describe('Journey testing of the client using a WCD cluster', () => { references: [], replication: { asyncEnabled: false, + deletionStrategy: 'NoAutomatedResolution', factor: 1, }, reranker: { From c5b5573504b3b006a352935ca3022a5fc0d4ff83 Mon Sep 17 00:00:00 2001 From: Tommy Smith Date: Wed, 16 Oct 2024 12:23:20 +0100 Subject: [PATCH 3/3] Fix reconfigure.replication and add unit test for its usage --- src/collections/configure/index.ts | 3 ++- src/collections/configure/types/base.ts | 2 +- src/collections/configure/unit.test.ts | 20 +++++++++++++++++--- 3 files changed, 20 insertions(+), 5 deletions(-) diff --git a/src/collections/configure/index.ts b/src/collections/configure/index.ts index 26549248..4c9d2ca4 100644 --- a/src/collections/configure/index.ts +++ b/src/collections/configure/index.ts @@ -3,6 +3,7 @@ import { InvertedIndexConfigUpdate, MultiTenancyConfigCreate, ReplicationConfigCreate, + ReplicationConfigUpdate, ReplicationDeletionStrategy, ShardingConfigCreate, VectorConfigUpdate, @@ -234,7 +235,7 @@ const reconfigure = { asyncEnabled?: boolean; deletionStrategy?: ReplicationDeletionStrategy; factor?: number; - }): ReplicationConfigCreate => { + }): ReplicationConfigUpdate => { return { asyncEnabled: options.asyncEnabled, deletionStrategy: options.deletionStrategy, diff --git a/src/collections/configure/types/base.ts b/src/collections/configure/types/base.ts index d52050d0..0ee1cde2 100644 --- a/src/collections/configure/types/base.ts +++ b/src/collections/configure/types/base.ts @@ -140,7 +140,7 @@ export type ReplicationConfigCreate = RecursivePartial; export type ReplicationConfigUpdate = { asyncEnabled?: boolean; - deletionStragey?: ReplicationDeletionStrategy; + deletionStrategy?: ReplicationDeletionStrategy; factor?: number; }; diff --git a/src/collections/configure/unit.test.ts b/src/collections/configure/unit.test.ts index 3e68228e..ef1d3215 100644 --- a/src/collections/configure/unit.test.ts +++ b/src/collections/configure/unit.test.ts @@ -14,17 +14,18 @@ import { ModuleConfig, VectorConfigCreate, } from '../types/index.js'; -import { configure } from './index.js'; +import { configure, reconfigure } from './index.js'; import { InvertedIndexConfigCreate, MultiTenancyConfigCreate, ReplicationConfigCreate, + ReplicationConfigUpdate, ShardingConfigCreate, VectorIndexConfigFlatCreate, VectorIndexConfigHNSWCreate, } from './types/index.js'; -describe('Unit testing of the configure factory class', () => { +describe('Unit testing of the configure & reconfigure factory classes', () => { it('should create the correct InvertedIndexConfig type with all values', () => { const config = configure.invertedIndex({ bm25b: 0.5, @@ -76,7 +77,7 @@ describe('Unit testing of the configure factory class', () => { }); }); - it('should create the correct ReplicationConfig type with all values', () => { + it('should create the correct ReplicationConfigCreate type with all values', () => { const config = configure.replication({ asyncEnabled: true, deletionStrategy: 'DeleteOnConflict', @@ -89,6 +90,19 @@ describe('Unit testing of the configure factory class', () => { }); }); + it('should create the correct ReplicationConfigUpdate type with all values', () => { + const config = reconfigure.replication({ + asyncEnabled: true, + deletionStrategy: 'DeleteOnConflict', + factor: 2, + }); + expect(config).toEqual({ + asyncEnabled: true, + deletionStrategy: 'DeleteOnConflict', + factor: 2, + }); + }); + it('should create the correct ShardingConfig type with all values', () => { const config = configure.sharding({ virtualPerPhysical: 256,