From f64d8761e02429af3d6fdf8b29c406af391cab53 Mon Sep 17 00:00:00 2001 From: etiennedi Date: Tue, 28 Feb 2023 08:59:11 +0100 Subject: [PATCH 1/2] support cursor API in REST and GraphQL --- data/getter.js | 8 ++++- data/journey.test.js | 65 ++++++++++++++++++++++++++++------------- data/path.js | 7 +++-- graphql/getter.js | 12 +++++++- graphql/getter.test.js | 17 +++++++++++ graphql/journey.test.js | 52 +++++++++++++++++++++------------ 6 files changed, 119 insertions(+), 42 deletions(-) diff --git a/data/getter.js b/data/getter.js index 6028c47..d702fc9 100644 --- a/data/getter.js +++ b/data/getter.js @@ -11,6 +11,11 @@ export default class Getter { return this; }; + withAfter = (id) => { + this.after = id; + return this; + }; + withLimit = (limit) => { this.limit = limit; return this; @@ -32,7 +37,8 @@ export default class Getter { ); } - return this.objectsPath.buildGet(this.className, this.limit, this.additionals) + return this.objectsPath.buildGet(this.className, this.limit, + this.additionals, this.after) .then(this.client.get); }; } diff --git a/data/journey.test.js b/data/journey.test.js index 6f98744..dfaf2a0 100644 --- a/data/journey.test.js +++ b/data/journey.test.js @@ -65,7 +65,12 @@ describe("data", () => { it("creates a new thing object with an explicit id", () => { const properties = { stringProp: "with-id" }; - const id = "1565c06c-463f-466c-9092-5930dbac3887"; + // explicitly make this an all-zero UUID. This way we can be sure that it's + // the first to come up when using the cursor API. Since this test suite + // also contains dynamicaly generated IDs, this is the only way to make + // sure that this ID is first. This way the tests returning objects after + // this ID won't be flaky. + const id = "00000000-0000-0000-0000-000000000000"; return client.data .creator() @@ -128,7 +133,7 @@ describe("data", () => { expect(res.objects).toEqual( expect.arrayContaining([ expect.objectContaining({ - id: "1565c06c-463f-466c-9092-5930dbac3887", + id: "00000000-0000-0000-0000-000000000000", properties: { stringProp: "with-id" }, }), expect.objectContaining({ @@ -150,7 +155,7 @@ describe("data", () => { expect(res.objects).toEqual( expect.arrayContaining([ expect.objectContaining({ - id: "1565c06c-463f-466c-9092-5930dbac3887", + id: "00000000-0000-0000-0000-000000000000", properties: { stringProp: "with-id" }, }), expect.objectContaining({ @@ -162,6 +167,26 @@ describe("data", () => { .catch((e) => fail("it should not have errord: " + e)); }); + it("gets all classes after a specfic object (Cursor API)", () => { + return client.data + .getter() + .withClassName(thingClassName) + .withLimit(100) + .withAfter("00000000-0000-0000-0000-000000000000") + .do() + .then((res) => { + expect(res.objects).toHaveLength(1); + expect(res.objects).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + properties: { stringProp: "without-id" }, + }), + ]) + ); + }) + .catch((e) => fail("it should not have errord: " + e)); + }); + it("gets all things with all optional _additional params", () => { return client.data .getter() @@ -207,12 +232,12 @@ describe("data", () => { it("gets one thing by id only", () => { return client.data .getterById() - .withId("1565c06c-463f-466c-9092-5930dbac3887") + .withId("00000000-0000-0000-0000-000000000000") .do() .then((res) => { expect(res).toEqual( expect.objectContaining({ - id: "1565c06c-463f-466c-9092-5930dbac3887", + id: "00000000-0000-0000-0000-000000000000", properties: { stringProp: "with-id" }, }) ); @@ -224,12 +249,12 @@ describe("data", () => { return client.data .getterById() .withClassName(thingClassName) - .withId("1565c06c-463f-466c-9092-5930dbac3887") + .withId("00000000-0000-0000-0000-000000000000") .do() .then((res) => { expect(res).toEqual( expect.objectContaining({ - id: "1565c06c-463f-466c-9092-5930dbac3887", + id: "00000000-0000-0000-0000-000000000000", properties: { stringProp: "with-id" }, }) ); @@ -241,7 +266,7 @@ describe("data", () => { return client.data .getterById() .withClassName("DoesNotExist") - .withId("1565c06c-463f-466c-9092-5930dbac3887") + .withId("00000000-0000-0000-0000-000000000000") .do() .catch(err => expect(err).toEqual("usage error (404): ") @@ -251,7 +276,7 @@ describe("data", () => { it("gets one thing by id with all optional additional props", () => { return client.data .getterById() - .withId("1565c06c-463f-466c-9092-5930dbac3887") + .withId("00000000-0000-0000-0000-000000000000") .withAdditional("classification") .withAdditional("interpretation") .withAdditional("nearestNeighbors") @@ -282,7 +307,7 @@ describe("data", () => { }); it("updates a thing by id only", () => { - const id = "1565c06c-463f-466c-9092-5930dbac3887"; + const id = "00000000-0000-0000-0000-000000000000"; return client.data .getterById() .withId(id) @@ -307,7 +332,7 @@ describe("data", () => { }); it("updates a thing by id and class name", () => { - const id = "1565c06c-463f-466c-9092-5930dbac3887"; + const id = "00000000-0000-0000-0000-000000000000"; return client.data .getterById() .withId(id) @@ -332,7 +357,7 @@ describe("data", () => { }); it("merges a thing", () => { - const id = "1565c06c-463f-466c-9092-5930dbac3887"; + const id = "00000000-0000-0000-0000-000000000000"; return client.data .getterById() .withId(id) @@ -353,7 +378,7 @@ describe("data", () => { it("adds a reference to a thing by id only", () => { const sourceId = "599a0c64-5ed5-4d30-978b-6c9c45516db1"; - const targetId = "1565c06c-463f-466c-9092-5930dbac3887"; + const targetId = "00000000-0000-0000-0000-000000000000"; return client.data .referenceCreator() @@ -398,7 +423,7 @@ describe("data", () => { it("adds a reference to a thing by id and class name", () => { const sourceId = "599a0c64-5ed5-4d30-978b-6c9c45516db1"; - const targetId = "1565c06c-463f-466c-9092-5930dbac3887"; + const targetId = "00000000-0000-0000-0000-000000000000"; return client.data .referenceCreator() @@ -447,7 +472,7 @@ describe("data", () => { it("checks that object exists by id only", () => { return client.data .checker() - .withId("1565c06c-463f-466c-9092-5930dbac3887") + .withId("00000000-0000-0000-0000-000000000000") .do() .then((exists) => { if (!exists) { @@ -460,7 +485,7 @@ describe("data", () => { it("checks that object exists by id and class name", () => { return client.data .checker() - .withId("1565c06c-463f-466c-9092-5930dbac3887") + .withId("00000000-0000-0000-0000-000000000000") .withClassName(thingClassName) .do() .then((exists) => { @@ -474,7 +499,7 @@ describe("data", () => { it("deletes a thing by id only", () => { return client.data .deleter() - .withId("1565c06c-463f-466c-9092-5930dbac3887") + .withId("00000000-0000-0000-0000-000000000000") .do() .catch((e) => fail("it should not have errord: " + e)); }); @@ -482,7 +507,7 @@ describe("data", () => { it("checks that object doesn't exist anymore with delete by id only", () => { return client.data .checker() - .withId("1565c06c-463f-466c-9092-5930dbac3887") + .withId("00000000-0000-0000-0000-000000000000") .do() .then((exists) => { if (exists) { @@ -585,7 +610,7 @@ describe("data", () => { }); it("forms a get by id query with node_name set", () => { - const id = "1565c06c-463f-466c-9092-5930dbac3887"; + const id = "00000000-0000-0000-0000-000000000000"; return client.data .getterById() @@ -602,7 +627,7 @@ describe("data", () => { }) it("forms a get by id query with consistency_level set", () => { - const id = "1565c06c-463f-466c-9092-5930dbac3887"; + const id = "00000000-0000-0000-0000-000000000000"; return client.data .getterById() diff --git a/data/path.js b/data/path.js index 6df560d..8684768 100644 --- a/data/path.js +++ b/data/path.js @@ -22,8 +22,8 @@ export class ObjectsPath { return this.build({id, className, additionals, consistencyLevel, nodeName}, [this.addClassNameDeprecatedNotSupportedCheck, this.addId, this.addQueryParams]); } - buildGet(className, limit, additionals) { - return this.build({className, limit, additionals}, [this.addQueryParamsForGet]); + buildGet(className, limit, additionals, after) { + return this.build({className, limit, additionals, after}, [this.addQueryParamsForGet]); } buildUpdate(id, className, consistencyLevel) { return this.build({id, className, consistencyLevel}, @@ -103,6 +103,9 @@ export class ObjectsPath { support.warns.notSupportedClassParameterInEndpointsForObjects(); } } + if (isValidStringProperty(params.after)) { + queryParams.push(`after=${params.after}`) + } if (queryParams.length > 0) { return `${path}?${queryParams.join("&")}`; } diff --git a/graphql/getter.js b/graphql/getter.js index 0e89036..71f2c43 100644 --- a/graphql/getter.js +++ b/graphql/getter.js @@ -26,6 +26,11 @@ export default class Getter { return this; }; + withAfter = (id) => { + this.after = id; + return this; + }; + withGroup = (groupObj) => { try { this.groupString = new Group(groupObj).toString(); @@ -202,7 +207,8 @@ export default class Getter { this.limit || this.offset || this.groupString || - this.sortString + this.sortString || + this.after ) { let args = []; @@ -254,6 +260,10 @@ export default class Getter { args = [...args, `sort:[${this.sortString}]`]; } + if (this.after) { + args = [...args, `after:"${this.after}"`]; + } + params = `(${args.join(",")})`; } diff --git a/graphql/getter.test.js b/graphql/getter.test.js index de85333..8ec94c3 100644 --- a/graphql/getter.test.js +++ b/graphql/getter.test.js @@ -46,6 +46,23 @@ test("a simple query with a limit and offset", () => { expect(mockClient.query).toHaveBeenCalledWith(expectedQuery); }); +test("a simple query with a limit and after", () => { + const mockClient = { + query: jest.fn(), + }; + + const expectedQuery = `{Get{Person(limit:7,after:"c6f379dd-94b7-4017-acd3-df769a320c92"){name}}}`; + + new Getter(mockClient) + .withClassName("Person") + .withFields("name") + .withAfter("c6f379dd-94b7-4017-acd3-df769a320c92") + .withLimit(7) + .do(); + + expect(mockClient.query).toHaveBeenCalledWith(expectedQuery); +}); + test("a simple query with a group", () => { const mockClient = { query: jest.fn(), diff --git a/graphql/journey.test.js b/graphql/journey.test.js index bbb47f6..1263b35 100644 --- a/graphql/journey.test.js +++ b/graphql/journey.test.js @@ -34,6 +34,20 @@ describe("the graphql journey", () => { }); }); + test("graphql get objects after id (Cursor API)", () => { + return client.graphql + .get() + .withClassName("Article") + .withLimit(10) + .withAfter("abefd256-8574-442b-9293-9205193737e0") + .withFields("title url wordCount") + .do() + .then(function (result) { + // one fewer than all + expect(result.data.Get.Article.length).toEqual(2); + }); + }); + test("graphql get method with optional fields (with certainty)", () => { return client.graphql .get() @@ -129,7 +143,7 @@ describe("the graphql journey", () => { return client.graphql .get() .withClassName("Article") - .withNearObject({ id: "abefd256-8574-442b-9293-9205193737ee", certainty: 0.7 }) + .withNearObject({ id: "abefd256-8574-442b-9293-9205193737e0", certainty: 0.7 }) .withFields("_additional { id }") .do() .then((res) => { @@ -142,7 +156,7 @@ describe("the graphql journey", () => { return client.graphql .get() .withClassName("Article") - .withNearObject({ id: "abefd256-8574-442b-9293-9205193737ee", distance: 0.3 }) + .withNearObject({ id: "abefd256-8574-442b-9293-9205193737e0", distance: 0.3 }) .withFields("_additional { id }") .do() .then((res) => { @@ -297,7 +311,7 @@ describe("the graphql journey", () => { .get() .withClassName("Article") .withNearText({ concepts: ["iphone"] }) - .withNearObject({ id: "abefd256-8574-442b-9293-9205193737ee", certainty: 0.65 }) + .withNearObject({ id: "abefd256-8574-442b-9293-9205193737e0", certainty: 0.65 }) .do() }) .toThrow("cannot use multiple near filters in a single query") @@ -309,7 +323,7 @@ describe("the graphql journey", () => { .get() .withClassName("Article") .withNearText({ concepts: ["iphone"] }) - .withNearObject({ id: "abefd256-8574-442b-9293-9205193737ee", distance: 0.35 }) + .withNearObject({ id: "abefd256-8574-442b-9293-9205193737e0", distance: 0.35 }) .do() }) .toThrow("cannot use multiple near filters in a single query") @@ -386,7 +400,7 @@ describe("the graphql journey", () => { return client.graphql .aggregate() .withClassName("Article") - .withNearObject({ id: "abefd256-8574-442b-9293-9205193737ee", certainty: 0.7 }) + .withNearObject({ id: "abefd256-8574-442b-9293-9205193737e0", certainty: 0.7 }) .withFields("meta { count }") .do() .then((res) => { @@ -400,7 +414,7 @@ describe("the graphql journey", () => { return client.graphql .aggregate() .withClassName("Article") - .withNearObject({ id: "abefd256-8574-442b-9293-9205193737ee", distance: 0.3 }) + .withNearObject({ id: "abefd256-8574-442b-9293-9205193737e0", distance: 0.3 }) .withFields("meta { count }") .do() .then((res) => { @@ -444,7 +458,7 @@ describe("the graphql journey", () => { .aggregate() .withClassName("Article") .withNearText({ concepts: ["iphone"] }) - .withNearObject({ id: "abefd256-8574-442b-9293-9205193737ee", certainty: 0.65 }) + .withNearObject({ id: "abefd256-8574-442b-9293-9205193737e0", certainty: 0.65 }) .do() }) .toThrow("cannot use multiple near filters in a single query") @@ -456,7 +470,7 @@ describe("the graphql journey", () => { .aggregate() .withClassName("Article") .withNearText({ concepts: ["iphone"] }) - .withNearObject({ id: "abefd256-8574-442b-9293-9205193737ee", distance: 0.35 }) + .withNearObject({ id: "abefd256-8574-442b-9293-9205193737e0", distance: 0.35 }) .do() }) .toThrow("cannot use multiple near filters in a single query") @@ -472,7 +486,7 @@ describe("the graphql journey", () => { .withWhere({ operator: weaviate.filters.Operator.EQUAL, path: ["_id"], - valueString: "abefd256-8574-442b-9293-9205193737ee", + valueString: "abefd256-8574-442b-9293-9205193737e0", }) .withFields("meta { count }") .do() @@ -493,7 +507,7 @@ describe("the graphql journey", () => { .withWhere({ operator: weaviate.filters.Operator.EQUAL, path: ["_id"], - valueString: "abefd256-8574-442b-9293-9205193737ee", + valueString: "abefd256-8574-442b-9293-9205193737e0", }) .withFields("meta { count }") .do() @@ -508,11 +522,11 @@ describe("the graphql journey", () => { return client.graphql .aggregate() .withClassName("Article") - .withNearObject({ id: "abefd256-8574-442b-9293-9205193737ee", certainty: 0.7 }) + .withNearObject({ id: "abefd256-8574-442b-9293-9205193737e0", certainty: 0.7 }) .withWhere({ operator: weaviate.filters.Operator.EQUAL, path: ["_id"], - valueString: "abefd256-8574-442b-9293-9205193737ee", + valueString: "abefd256-8574-442b-9293-9205193737e0", }) .withFields("meta { count }") .do() @@ -527,11 +541,11 @@ describe("the graphql journey", () => { return client.graphql .aggregate() .withClassName("Article") - .withNearObject({ id: "abefd256-8574-442b-9293-9205193737ee", distance: 0.3 }) + .withNearObject({ id: "abefd256-8574-442b-9293-9205193737e0", distance: 0.3 }) .withWhere({ operator: weaviate.filters.Operator.EQUAL, path: ["_id"], - valueString: "abefd256-8574-442b-9293-9205193737ee", + valueString: "abefd256-8574-442b-9293-9205193737e0", }) .withFields("meta { count }") .do() @@ -550,7 +564,7 @@ describe("the graphql journey", () => { .withWhere({ operator: weaviate.filters.Operator.EQUAL, path: ["_id"], - valueString: "abefd256-8574-442b-9293-9205193737ee", + valueString: "abefd256-8574-442b-9293-9205193737e0", }) .withFields("meta { count }") .do() @@ -569,7 +583,7 @@ describe("the graphql journey", () => { .withWhere({ operator: weaviate.filters.Operator.EQUAL, path: ["_id"], - valueString: "abefd256-8574-442b-9293-9205193737ee", + valueString: "abefd256-8574-442b-9293-9205193737e0", }) .withFields("meta { count }") .do() @@ -672,7 +686,7 @@ describe("the graphql journey", () => { test("graphql explore with nearObject field", () => { return client.graphql .explore() - .withNearObject({ id: "abefd256-8574-442b-9293-9205193737ee" }) + .withNearObject({ id: "abefd256-8574-442b-9293-9205193737e0" }) .withFields("beacon certainty distance className") .do() .then((res) => { @@ -846,9 +860,11 @@ const setup = async (client) => { await Promise.all([client.schema.classCreator().withClass(thing).do()]); + // Note that the UUIDs are in ascending order. This is on purpose as the + // Cursor API test relies on this fact. const toImport = [ { - id: "abefd256-8574-442b-9293-9205193737ee", + id: "abefd256-8574-442b-9293-9205193737e0", class: "Article", properties: { wordCount: 60, From daf944f25f059a3756e1cc168cc012d1bc2f3126 Mon Sep 17 00:00:00 2001 From: Marcin Antas Date: Wed, 1 Mar 2023 17:52:00 +0100 Subject: [PATCH 2/2] Update docker images --- ci/docker-compose-azure-cc.yml | 2 +- ci/docker-compose-okta-cc.yml | 2 +- ci/docker-compose-okta-users.yml | 2 +- ci/docker-compose-wcs-noscope.yml | 2 +- ci/docker-compose-wcs.yml | 2 +- ci/docker-compose.yml | 2 +- cluster/journey.test.js | 4 ++-- schema/journey.test.js | 20 ++++++++++++++++++++ 8 files changed, 28 insertions(+), 8 deletions(-) diff --git a/ci/docker-compose-azure-cc.yml b/ci/docker-compose-azure-cc.yml index bcb09c7..d2664d9 100644 --- a/ci/docker-compose-azure-cc.yml +++ b/ci/docker-compose-azure-cc.yml @@ -11,7 +11,7 @@ services: - --scheme - http - --write-timeout=600s - image: semitechnologies/weaviate:1.18.0-alpha.0-be532d2 + image: semitechnologies/weaviate:1.18.0-alpha.1-41f7cb9 ports: - 8081:8081 restart: on-failure:0 diff --git a/ci/docker-compose-okta-cc.yml b/ci/docker-compose-okta-cc.yml index dc1535e..50be022 100644 --- a/ci/docker-compose-okta-cc.yml +++ b/ci/docker-compose-okta-cc.yml @@ -10,7 +10,7 @@ services: - --scheme - http - --write-timeout=600s - image: semitechnologies/weaviate:1.18.0-alpha.0-be532d2 + image: semitechnologies/weaviate:1.18.0-alpha.1-41f7cb9 ports: - 8082:8082 restart: on-failure:0 diff --git a/ci/docker-compose-okta-users.yml b/ci/docker-compose-okta-users.yml index 44ad49d..b681998 100644 --- a/ci/docker-compose-okta-users.yml +++ b/ci/docker-compose-okta-users.yml @@ -10,7 +10,7 @@ services: - --scheme - http - --write-timeout=600s - image: semitechnologies/weaviate:1.18.0-alpha.0-be532d2 + image: semitechnologies/weaviate:1.18.0-alpha.1-41f7cb9 ports: - 8083:8083 restart: on-failure:0 diff --git a/ci/docker-compose-wcs-noscope.yml b/ci/docker-compose-wcs-noscope.yml index 60f7f28..7b84b44 100644 --- a/ci/docker-compose-wcs-noscope.yml +++ b/ci/docker-compose-wcs-noscope.yml @@ -10,7 +10,7 @@ services: - --scheme - http - --write-timeout=600s - image: semitechnologies/weaviate:1.18.0-alpha.0-be532d2 + image: semitechnologies/weaviate:1.18.0-alpha.1-41f7cb9 ports: - 8086:8086 restart: on-failure:0 diff --git a/ci/docker-compose-wcs.yml b/ci/docker-compose-wcs.yml index bccd1d1..e2c31c1 100644 --- a/ci/docker-compose-wcs.yml +++ b/ci/docker-compose-wcs.yml @@ -10,7 +10,7 @@ services: - --scheme - http - --write-timeout=600s - image: semitechnologies/weaviate:1.18.0-alpha.0-be532d2 + image: semitechnologies/weaviate:1.18.0-alpha.1-41f7cb9 ports: - 8085:8085 restart: on-failure:0 diff --git a/ci/docker-compose.yml b/ci/docker-compose.yml index fdc9281..48cc224 100644 --- a/ci/docker-compose.yml +++ b/ci/docker-compose.yml @@ -2,7 +2,7 @@ version: '3.4' services: weaviate: - image: semitechnologies/weaviate:1.18.0-alpha.0-be532d2 + image: semitechnologies/weaviate:1.18.0-alpha.1-41f7cb9 restart: on-failure:0 ports: - "8080:8080" diff --git a/cluster/journey.test.js b/cluster/journey.test.js index b35a9bd..7769b47 100644 --- a/cluster/journey.test.js +++ b/cluster/journey.test.js @@ -1,8 +1,8 @@ const weaviate = require("../index"); const { createTestFoodSchemaAndData, cleanupTestFood, PIZZA_CLASS_NAME, SOUP_CLASS_NAME } = require("../utils/testData"); -const EXPECTED_WEAVIATE_VERSION = "1.18.0-alpha.0" -const EXPECTED_WEAVIATE_GIT_HASH = "be532d2" +const EXPECTED_WEAVIATE_VERSION = "1.18.0-alpha.1" +const EXPECTED_WEAVIATE_GIT_HASH = "41f7cb9" describe("cluster nodes endpoint", () => { const client = weaviate.client({ diff --git a/schema/journey.test.js b/schema/journey.test.js index 9697b96..0edc0dc 100644 --- a/schema/journey.test.js +++ b/schema/journey.test.js @@ -156,6 +156,16 @@ describe("schema", () => { dynamicEfMin: 100, ef: -1, maxConnections: 64, + pq: { + bitCompression: false, + centroids: 256, + enabled: false, + encoder: { + distribution: "log-normal", + type: "kmeans" + }, + segments: 0, + }, skip: false, efConstruction: 128, vectorCacheMaxObjects: 500000, @@ -421,6 +431,16 @@ function newClassObject(className) { dynamicEfMin: 100, ef: -1, maxConnections: 64, + pq: { + bitCompression: false, + centroids: 256, + enabled: false, + encoder: { + distribution: "log-normal", + type: "kmeans" + }, + segments: 0, + }, skip: false, efConstruction: 128, vectorCacheMaxObjects: 500000,