From 3ca240f206c2aa108e775b68966f1eff2db0158b Mon Sep 17 00:00:00 2001 From: Parker Duckworth Date: Tue, 11 Apr 2023 10:10:19 -0500 Subject: [PATCH 1/2] support gql get consistency level --- ci/compose.sh | 2 +- ci/docker-compose-cluster.yml | 56 +++++++++++++++++++++++++++ src/graphql/getter.test.ts | 38 ++++++++++++++++++ src/graphql/getter.ts | 11 ++++++ src/graphql/journey.test.ts | 72 ++++++++++++++++++++++++++++++++++- 5 files changed, 177 insertions(+), 2 deletions(-) create mode 100644 ci/docker-compose-cluster.yml diff --git a/ci/compose.sh b/ci/compose.sh index d3be4b33..02f96b18 100644 --- a/ci/compose.sh +++ b/ci/compose.sh @@ -19,5 +19,5 @@ function compose_down_all { } function all_weaviate_ports { - echo "8080 8081 8082 8083 8085 8086" + echo "8080 8081 8082 8083 8085 8086 8087 8088" } diff --git a/ci/docker-compose-cluster.yml b/ci/docker-compose-cluster.yml new file mode 100644 index 00000000..fceadaa7 --- /dev/null +++ b/ci/docker-compose-cluster.yml @@ -0,0 +1,56 @@ +--- +version: '3.4' +services: + weaviate-node-1: + image: semitechnologies/weaviate:preview-gql-handler-consistency-level-integration-4a12f55 + restart: on-failure:0 + ports: + - "8087:8080" + environment: + CONTEXTIONARY_URL: contextionary:9999 + QUERY_DEFAULTS_LIMIT: 20 + AUTHENTICATION_ANONYMOUS_ACCESS_ENABLED: 'true' + PERSISTENCE_DATA_PATH: "./weaviate-node-1" + DEFAULT_VECTORIZER_MODULE: text2vec-contextionary + ENABLE_MODULES: text2vec-contextionary + CLUSTER_GOSSIP_BIND_PORT: "7110" + CLUSTER_DATA_BIND_PORT: "7111" + + contextionary: + image: semitechnologies/contextionary:en0.16.0-v1.2.0 + ports: + - "9797:9999" + environment: + OCCURRENCE_WEIGHT_LINEAR_FACTOR: 0.75 + EXTENSIONS_STORAGE_MODE: weaviate + EXTENSIONS_STORAGE_ORIGIN: http://weaviate:8080 + NEIGHBOR_OCCURRENCE_IGNORE_PERCENTILE: 5 + ENABLE_COMPOUND_SPLITTING: 'false' + + weaviate-node-2: + init: true + command: + - --host + - 0.0.0.0 + - --port + - '8080' + - --scheme + - http + image: semitechnologies/weaviate:preview-gql-handler-consistency-level-integration-4a12f55 + ports: + - 8088:8080 + - 6061:6060 + restart: on-failure:0 + environment: + CONTEXTIONARY_URL: contextionary:9999 + LOG_LEVEL: 'debug' + QUERY_DEFAULTS_LIMIT: 20 + AUTHENTICATION_ANONYMOUS_ACCESS_ENABLED: 'true' + PERSISTENCE_DATA_PATH: './weaviate-node-2' + DEFAULT_VECTORIZER_MODULE: text2vec-contextionary + ENABLE_MODULES: text2vec-contextionary + CLUSTER_HOSTNAME: 'node2' + CLUSTER_GOSSIP_BIND_PORT: '7112' + CLUSTER_DATA_BIND_PORT: '7113' + CLUSTER_JOIN: 'weaviate-node-1:7110' +... diff --git a/src/graphql/getter.test.ts b/src/graphql/getter.test.ts index f620f4bf..64f5312c 100644 --- a/src/graphql/getter.test.ts +++ b/src/graphql/getter.test.ts @@ -75,6 +75,44 @@ test('a simple query with a group', () => { expect(mockClient.query).toHaveBeenCalledWith(expectedQuery); }); +describe('query with consistency level', () => { + test('One', () => { + const mockClient: any = { + query: jest.fn(), + }; + + const expectedQuery = `{Get{Person(consistencyLevel:ONE){name}}}`; + + new Getter(mockClient).withClassName('Person').withFields('name').withConsistencyLevel('ONE').do(); + + expect(mockClient.query).toHaveBeenCalledWith(expectedQuery); + }); + + test('Quorum', () => { + const mockClient: any = { + query: jest.fn(), + }; + + const expectedQuery = `{Get{Person(consistencyLevel:QUORUM){name}}}`; + + new Getter(mockClient).withClassName('Person').withFields('name').withConsistencyLevel('QUORUM').do(); + + expect(mockClient.query).toHaveBeenCalledWith(expectedQuery); + }); + + test('All', () => { + const mockClient: any = { + query: jest.fn(), + }; + + const expectedQuery = `{Get{Person(consistencyLevel:ALL){name}}}`; + + new Getter(mockClient).withClassName('Person').withFields('name').withConsistencyLevel('ALL').do(); + + expect(mockClient.query).toHaveBeenCalledWith(expectedQuery); + }); +}); + describe('where filters', () => { test('a query with a valid where filter', () => { const mockClient: any = { diff --git a/src/graphql/getter.ts b/src/graphql/getter.ts index 709fcba8..c1de0ebc 100644 --- a/src/graphql/getter.ts +++ b/src/graphql/getter.ts @@ -12,6 +12,7 @@ import Connection from '../connection'; import { CommandBase } from '../validation/commandBase'; import { WhereFilter } from '../openapi/types'; import { GenerateArgs, GraphQLGenerate } from './generate'; +import { ConsistencyLevel } from '../data'; export default class GraphQLGetter extends CommandBase { private after?: string; @@ -31,6 +32,7 @@ export default class GraphQLGetter extends CommandBase { private sortString?: string; private whereString?: string; private generateString?: string; + private consistencyLevel?: ConsistencyLevel; constructor(client: Connection) { super(client); @@ -176,6 +178,11 @@ export default class GraphQLGetter extends CommandBase { return this; }; + withConsistencyLevel = (level: ConsistencyLevel) => { + this.consistencyLevel = level; + return this; + }; + validateIsSet = (prop: string | undefined | null, name: string, setter: string) => { if (prop == undefined || prop == null || prop.length == 0) { this.addError(`${name} must be set - set with ${setter}`); @@ -256,6 +263,10 @@ export default class GraphQLGetter extends CommandBase { } } + if (this.consistencyLevel) { + args = [...args, `consistencyLevel:${this.consistencyLevel}`]; + } + if (args.length > 0) { params = `(${args.join(',')})`; } diff --git a/src/graphql/journey.test.ts b/src/graphql/journey.test.ts index b4173a76..270cb90c 100644 --- a/src/graphql/journey.test.ts +++ b/src/graphql/journey.test.ts @@ -1227,7 +1227,7 @@ describe('query with generative search', () => { .withClassName('Wine') .withFields('name review') .withGenerate({ - singlePrompt: `Describe the following as a Facebook Ad: + singlePrompt: `Describe the following as a Facebook Ad: Tastes like a fresh ocean breeze: {review}`, }) .do() @@ -1274,6 +1274,76 @@ Tastes like a fresh ocean breeze: {review}`, }); }); +describe('query cluster with consistency level', () => { + const client = weaviate.client({ + scheme: 'http', + host: 'localhost:8087', + }); + + it('sets up replicated class', () => { + return setupReplicated(client); + }); + + test('One', () => { + return client.graphql + .get() + .withClassName('Article') + .withFields('_additional { id isConsistent }') + .withConsistencyLevel('ONE') + .do() + .then((res: any) => { + expect(res.data.Get.Article.length).toBeGreaterThan(0); + res.data.Get.Article.forEach((article: any) => { + expect(article._additional.isConsistent).toBeTruthy(); + }); + return res; + }) + .catch((e: any) => { + throw new Error(`unexpected error: ${JSON.stringify(e)}`); + }); + }); + + test('Quorum', () => { + return client.graphql + .get() + .withClassName('Article') + .withFields('_additional { id isConsistent }') + .withConsistencyLevel('QUORUM') + .do() + .then((res: any) => { + expect(res.data.Get.Article.length).toBeGreaterThan(0); + res.data.Get.Article.forEach((article: any) => { + expect(article._additional.isConsistent).toBeTruthy(); + }); + }) + .catch((e: any) => { + throw new Error(`unexpected error: ${JSON.stringify(e)}`); + }); + }); + + test('All', () => { + return client.graphql + .get() + .withClassName('Article') + .withFields('_additional { id isConsistent }') + .withConsistencyLevel('ALL') + .do() + .then((res: any) => { + expect(res.data.Get.Article.length).toBeGreaterThan(0); + res.data.Get.Article.forEach((article: any) => { + expect(article._additional.isConsistent).toBeTruthy(); + }); + }) + .catch((e: any) => { + throw new Error(`unexpected error: ${JSON.stringify(e)}`); + }); + }); + + it('tears down cluster schema', () => { + return Promise.all([client.schema.classDeleter().withClassName('Article').do()]); + }); +}); + const setup = async (client: WeaviateClient) => { const thing = { class: 'Article', From 1b792fcd722cff57bef35f5699507e1b2b124abe Mon Sep 17 00:00:00 2001 From: Parker Duckworth Date: Tue, 11 Apr 2023 10:21:33 -0500 Subject: [PATCH 2/2] update docker compose cluster --- ci/docker-compose-cluster.yml | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/ci/docker-compose-cluster.yml b/ci/docker-compose-cluster.yml index fceadaa7..be2a29f0 100644 --- a/ci/docker-compose-cluster.yml +++ b/ci/docker-compose-cluster.yml @@ -16,17 +16,6 @@ services: CLUSTER_GOSSIP_BIND_PORT: "7110" CLUSTER_DATA_BIND_PORT: "7111" - contextionary: - image: semitechnologies/contextionary:en0.16.0-v1.2.0 - ports: - - "9797:9999" - environment: - OCCURRENCE_WEIGHT_LINEAR_FACTOR: 0.75 - EXTENSIONS_STORAGE_MODE: weaviate - EXTENSIONS_STORAGE_ORIGIN: http://weaviate:8080 - NEIGHBOR_OCCURRENCE_IGNORE_PERCENTILE: 5 - ENABLE_COMPOUND_SPLITTING: 'false' - weaviate-node-2: init: true command: