feat: Add CQID and Parent CQID (#53)

package.json

@@ -46,6 +46,7 @@
     "@ava/typescript": "^4.1.0",
     "@grpc/grpc-js": "^1.9.0",
     "@tsconfig/node16": "^16.1.0",
+    "@types/uuid": "^9.0.2",
     "@types/yargs": "^17.0.24",
     "@typescript-eslint/eslint-plugin": "^6.2.1",
     "@typescript-eslint/parser": "^6.2.1",
@@ -89,6 +90,7 @@
     "matcher": "^5.0.0",
     "p-map": "^6.0.0",
     "p-timeout": "^6.1.2",
+    "uuid": "^9.0.0",
     "winston": "^3.10.0",
     "yargs": "^17.7.2"
   }

src/memdb/tables.ts

@@ -1,8 +1,8 @@
-import { Utf8 } from '@apache-arrow/esnext-esm';
+import { Int64 } from '@apache-arrow/esnext-esm';
 
 import { createColumn } from '../schema/column.js';
 import { addCQIDsColumns } from '../schema/meta.js';
-import { pathResolver } from '../schema/resolvers.js';
+import { pathResolver, parentColumnResolver } from '../schema/resolvers.js';
 import { createTable } from '../schema/table.js';
 
 export const createTables = () => {
@@ -12,14 +12,13 @@ export const createTables = () => {
       title: 'Table 1',
       description: 'Table 1 description',
       resolver: (clientMeta, parent, stream) => {
-        stream.write({ id: 'table1-name1' });
-        stream.write({ id: 'table1-name2' });
+        stream.write({ id: 'id-1' });
+        stream.write({ id: 'id-2' });
         return Promise.resolve();
       },
       columns: [
         createColumn({
           name: 'id',
-          type: new Utf8(),
           resolver: pathResolver('id'),
         }),
       ],
@@ -29,18 +28,61 @@ export const createTables = () => {
       title: 'Table 2',
       description: 'Table 2 description',
       resolver: (clientMeta, parent, stream) => {
-        stream.write({ name: 'table2-name1' });
-        stream.write({ name: 'table2-name2' });
+        stream.write({ name: 'name-1' });
+        stream.write({ name: 'name-2' });
         return Promise.resolve();
       },
       columns: [
         createColumn({
           name: 'name',
-          type: new Utf8(),
           resolver: pathResolver('name'),
         }),
       ],
     }),
+    createTable({
+      name: 'table3',
+      title: 'Table 3',
+      description: 'Table 3 description',
+      resolver: (clientMeta, parent, stream) => {
+        stream.write({ name: 'name-1' });
+        stream.write({ name: 'name-2' });
+        return Promise.resolve();
+      },
+      columns: [
+        createColumn({
+          name: 'name',
+          primaryKey: true,
+          resolver: pathResolver('name'),
+        }),
+      ],
+      relations: [
+        createTable({
+          name: 'table3_child1',
+          resolver: (clientMeta, parent, stream) => {
+            stream.write({ name: 'name-1', id: 1 });
+            stream.write({ name: 'name-2', id: 2 });
+            return Promise.resolve();
+          },
+          columns: [
+            createColumn({
+              name: 'name',
+              resolver: pathResolver('name'),
+            }),
+            createColumn({
+              name: 'id',
+              resolver: pathResolver('id'),
+              type: new Int64(),
+              primaryKey: true,
+            }),
+            createColumn({
+              name: 'parent_name',
+              resolver: parentColumnResolver('name'),
+              primaryKey: true,
+            }),
+          ],
+        }),
+      ],
+    }),
   ];
 
   const tableWithCQIDs = allTables.map((table) => addCQIDsColumns(table));

src/scalar/list.ts

@@ -1,11 +1,11 @@
 import { DataType, List as ArrowList } from '@apache-arrow/esnext-esm';
 
-import { Scalar } from './scalar.js';
+import { Scalar, Stringable } from './scalar.js';
 import { isInvalid, NULL_VALUE } from './util.js';
 
-type TVector<T extends Scalar<unknown>> = T[];
+type TVector<T extends Scalar<Stringable>> = T[];
 
-export class List<T extends Scalar<unknown>> implements Scalar<TVector<T>> {
+export class List<T extends Scalar<Stringable>> implements Scalar<TVector<T>> {
   private _type: new (value?: unknown) => T;
   private _valid = false;
   private _value: TVector<T> = [];

src/scalar/scalar.ts

@@ -6,17 +6,19 @@ import { Int64 } from './int64.js';
 import { Text } from './text.js';
 import { Timestamp } from './timestamp.js';
 
-export interface Scalar<T> {
+export type Stringable = { toString: () => string };
+
+export interface Scalar<T extends Stringable> {
   toString: () => string;
   get valid(): boolean;
   get value(): T;
   set value(value: unknown);
   get dataType(): DataType;
 }
 
-export type Vector = Scalar<unknown>[];
+export type Vector = Scalar<Stringable>[];
 
-export const newScalar = (dataType: DataType): Scalar<unknown> => {
+export const newScalar = (dataType: DataType): Scalar<Stringable> => {
   if (DataType.isBool(dataType)) {
     return new Bool();
   }

src/scalar/text.test.ts

@@ -4,7 +4,7 @@ import test from 'ava';
 import { Text } from './text.js';
 
 // eslint-disable-next-line unicorn/no-null
-[null, undefined].forEach((v) => {
+[null, undefined, new Text()].forEach((v) => {
   test(`should set values to empty string when ${v} is passed`, (t) => {
     const s = new Text(v);
     t.is(s.value, '');

src/scalar/text.ts

@@ -42,6 +42,12 @@ export class Text implements Scalar<string> {
       return;
     }
 
+    if (value instanceof Text) {
+      this._value = value.value;
+      this._valid = value.valid;
+      return;
+    }
+
     if (typeof value!.toString === 'function' && value!.toString !== Object.prototype.toString) {
       this._value = value!.toString();
       this._valid = true;

src/scheduler/cqid.test.ts

@@ -0,0 +1,64 @@
+import { createHash } from 'node:crypto';
+
+import test from 'ava';
+
+import { createColumn } from '../schema/column.js';
+import { addCQIDsColumns, cqIDColumn } from '../schema/meta.js';
+import { Resource } from '../schema/resource.js';
+import { createTable } from '../schema/table.js';
+
+import { setCQId } from './cqid.js';
+
+test('setCQId - should set to random value if deterministicCQId is false', (t): void => {
+  const resource = new Resource(addCQIDsColumns(createTable({ name: 'table1' })), null, null);
+
+  setCQId(resource, false, () => 'random');
+
+  t.is(resource.getColumnData(cqIDColumn.name).valid, true);
+  t.is(resource.getColumnData(cqIDColumn.name).value.toString(), 'random');
+});
+
+test('setCQId - should set to random value if deterministicCQId is true and table does not have non _cq_id PKs', (t): void => {
+  const resource = new Resource(addCQIDsColumns(createTable({ name: 'table1' })), null, null);
+
+  setCQId(resource, true, () => 'random');
+
+  t.is(resource.getColumnData(cqIDColumn.name).valid, true);
+  t.is(resource.getColumnData(cqIDColumn.name).value.toString(), 'random');
+});
+
+test('setCQId - should set to fixed value if deterministicCQId is true and table has non _cq_id PKs', (t): void => {
+  const resource = new Resource(
+    addCQIDsColumns(
+      createTable({
+        name: 'table1',
+        columns: [
+          createColumn({ name: 'pk1', primaryKey: true, unique: true, notNull: true }),
+          createColumn({ name: 'pk2', primaryKey: true, unique: true, notNull: true }),
+          createColumn({ name: 'pk3', primaryKey: true, unique: true, notNull: true }),
+          createColumn({ name: 'non_pk' }),
+        ],
+      }),
+    ),
+    null,
+    null,
+  );
+
+  resource.setColumData('pk1', 'pk1-value');
+  resource.setColumData('pk2', 'pk2-value');
+  resource.setColumData('pk3', 'pk3-value');
+  resource.setColumData('non_pk', 'non-pk-value');
+
+  const expectedSha256 = createHash('sha256');
+  expectedSha256.update('pk1');
+  expectedSha256.update('pk1-value');
+  expectedSha256.update('pk2');
+  expectedSha256.update('pk2-value');
+  expectedSha256.update('pk3');
+  expectedSha256.update('pk3-value');
+
+  setCQId(resource, true);
+
+  t.is(resource.getColumnData(cqIDColumn.name).valid, true);
+  t.is(resource.getColumnData(cqIDColumn.name).value.toString(), expectedSha256.digest('hex'));
+});

src/scheduler/cqid.ts

@@ -0,0 +1,28 @@
+import { createHash } from 'node:crypto';
+
+import { v4 as uuidv4 } from 'uuid';
+
+import { cqIDColumn } from '../schema/meta.js';
+import { Resource } from '../schema/resource.js';
+import { getPrimaryKeys } from '../schema/table.js';
+
+export const setCQId = (resource: Resource, deterministicCQId: boolean, generator: () => string = uuidv4) => {
+  const randomCQId = generator();
+  if (!deterministicCQId) {
+    resource.setCqId(randomCQId);
+  }
+
+  const primaryKeys = getPrimaryKeys(resource.table);
+  const hasNonCqPKs = primaryKeys.some((pk) => pk !== cqIDColumn.name);
+  if (hasNonCqPKs) {
+    const sha256 = createHash('sha256');
+    primaryKeys.sort();
+    for (const pk of primaryKeys) {
+      sha256.update(pk);
+      sha256.update(resource.getColumnData(pk).toString());
+    }
+    return resource.setCqId(sha256.digest('hex'));
+  }
+
+  return resource.setCqId(randomCQId);
+};