refactor(database)!: re-work SQL support (#492)

libraries/grpc-sdk/src/classes/ConduitActiveSchema.ts

@@ -55,20 +55,13 @@ export class ConduitActiveSchema<T> extends ConduitSchema {
   findByIdAndUpdate(
     id: string,
     document: Query<T>,
-    updateProvidedOnly?: boolean,
     populate?: string | string[],
   ): Promise<T | null> {
-    return this.dbInstance.findByIdAndUpdate<T>(
-      this.name,
-      id,
-      document,
-      updateProvidedOnly,
-      populate,
-    );
+    return this.dbInstance.findByIdAndUpdate<T>(this.name, id, document, populate);
   }
 
-  updateMany(filterQuery: Query<T>, query: Query<T>, updateProvidedOnly?: boolean) {
-    return this.dbInstance.updateMany(this.name, filterQuery, query, updateProvidedOnly);
+  updateMany(filterQuery: Query<T>, query: Query<T>, populate?: string | string[]) {
+    return this.dbInstance.updateMany(this.name, filterQuery, query, populate);
   }
 
   deleteOne(query: Query<T>) {

libraries/grpc-sdk/src/classes/ConduitSchema.ts

@@ -6,7 +6,7 @@ export class ConduitSchema {
   readonly collectionName: string | ''; // '' on implicit name, updated in createSchemaFromAdapter()
   readonly modelOptions: ConduitSchemaOptions;
   ownerModule: string = 'unknown';
-
+  parentSchema?: string;
   constructor(
     name: string,
     fields: ConduitModel,

libraries/grpc-sdk/src/modules/database/index.ts

@@ -178,7 +178,6 @@ export class DatabaseProvider extends ConduitModule<typeof DatabaseProviderDefin
     schemaName: string,
     id: string,
     document: Query<T>,
-    updateProvidedOnly: boolean = false,
     populate?: string | string[],
   ): Promise<T | any> {
     let populateArray = populate;
@@ -189,7 +188,6 @@ export class DatabaseProvider extends ConduitModule<typeof DatabaseProviderDefin
       schemaName,
       id,
       query: this.processQuery(document),
-      updateProvidedOnly,
       populate: (populateArray as string[]) ?? [],
     }).then(res => {
       return JSON.parse(res.result);
@@ -200,13 +198,17 @@ export class DatabaseProvider extends ConduitModule<typeof DatabaseProviderDefin
     schemaName: string,
     filterQuery: Query<T>,
     query: Query<T>,
-    updateProvidedOnly: boolean = false,
+    populate?: string | string[],
   ) {
+    let populateArray = populate;
+    if (populate && !Array.isArray(populate)) {
+      populateArray = [populate];
+    }
     return this.client!.updateMany({
       schemaName,
       filterQuery: this.processQuery(filterQuery),
       query: this.processQuery(query),
-      updateProvidedOnly,
+      populate: (populateArray as string[]) ?? [],
     }).then(res => {
       return JSON.parse(res.result);
     });
@@ -251,4 +253,8 @@ export class DatabaseProvider extends ConduitModule<typeof DatabaseProviderDefin
       return JSON.parse(res.result);
     });
   }
+
+  migrate(schemaName: string) {
+    return this.client!.migrate({ schemaName });
+  }
 }

libraries/grpc-sdk/src/types/db.ts

@@ -1,6 +1,16 @@
 type documentKeys<T> = keyof T;
 type documentValues<T> = T[keyof T];
-type operators = '$eq' | '$ne' | '$gt' | '$gte' | '$lt' | '$lte' | '$regex' | '$options';
+type operators =
+  | '$eq'
+  | '$ne'
+  | '$gt'
+  | '$gte'
+  | '$lt'
+  | '$lte'
+  | '$regex'
+  | '$options'
+  | '$like'
+  | '$ilike';
 type arrayOperators = '$in' | '$nin';
 type conditionOperators = '$or' | '$and';
 

libraries/hermes/src/GraphQl/utils/TypeUtils.ts

@@ -25,12 +25,23 @@ export function findPopulation(
         if (
           keys.length > 0 &&
           relations.indexOf(keys[0]) !== -1 &&
-          result.indexOf(key) === -1
+          result.indexOf(key) === -1 &&
+          !context.obj[key]
         ) {
-          result.push(_extractNestedPopulation(context._item.strPath));
+          let path = context._item.strPath;
+          path = path.split('.')[0];
+          if (context.obj[path]) {
+            result.push(
+              _extractNestedPopulation(
+                context._item.strPath.substring(context._item.strPath.indexOf('.') + 1),
+              ),
+            );
+          } else {
+            result.push(_extractNestedPopulation(context._item.strPath));
+          }
         }
       }
     },
   );
-  return result;
+  return result.length > 0 ? result : undefined;
 }

libraries/hermes/src/Rest/SwaggerParser.ts

@@ -1,4 +1,4 @@
-import { TYPE, ConduitModel, ConduitRouteOption } from '@conduitplatform/grpc-sdk';
+import { ConduitModel, ConduitRouteOption, TYPE } from '@conduitplatform/grpc-sdk';
 import { ConduitParser } from '../classes';
 
 export interface ParseResult {
@@ -182,17 +182,26 @@ export class SwaggerParser extends ConduitParser<ParseResult, ProcessingObject>
     isRequired: boolean = false,
     isArray: boolean,
   ): void {
-    // @ts-ignore
-    processingObject.properties[name] = {
-      oneOf: [
-        {
-          $ref: `#/components/schemas/${value}`,
-        },
-        {
-          $ref: `#/components/schemas/ModelId`,
-        },
-      ],
-    };
+    if (this.isInput) {
+      // @ts-ignore
+      processingObject.properties[name] = {
+        type: 'string',
+        format: 'uuid',
+      };
+    } else {
+      // @ts-ignore
+      processingObject.properties[name] = {
+        oneOf: [
+          {
+            $ref: `#/components/schemas/${value}`,
+          },
+          {
+            type: 'string',
+            format: 'uuid',
+          },
+        ],
+      };
+    }
     this.addFieldToRequired(processingObject, name, isRequired);
   }
 

modules/authentication/src/Authentication.ts

@@ -62,6 +62,15 @@ export default class Authentication extends ManagedModule<Config> {
     TokenProvider.getInstance(this.grpcSdk);
     await this.registerSchemas();
     await runMigrations(this.grpcSdk);
+    for (const model of Object.values(models)) {
+      const modelInstance = model.getInstance();
+      if (
+        Object.keys((modelInstance as ConduitActiveSchema<typeof modelInstance>).fields)
+          .length === 0
+      )
+        continue;
+      await this.database.migrate(modelInstance.name);
+    }
   }
 
   async preConfig(config: Config) {
@@ -266,13 +275,9 @@ export default class Authentication extends ManagedModule<Config> {
       }
 
       const hashedPassword = await AuthUtils.hashPassword(password);
-      await models.User.getInstance().findByIdAndUpdate(
-        user._id,
-        {
-          hashedPassword,
-        },
-        true,
-      );
+      await models.User.getInstance().findByIdAndUpdate(user._id, {
+        hashedPassword,
+      });
 
       return callback(null, { password });
     } catch (e) {

modules/authentication/src/admin/service.ts

@@ -54,7 +54,6 @@ export class ServiceAdmin {
     const service: Service | null = await Service.getInstance().findByIdAndUpdate(
       call.request.params.id,
       { hashedToken },
-      true,
     );
     if (isNil(service)) {
       throw new GrpcError(status.NOT_FOUND, 'Service does not exist');

modules/authentication/src/admin/user.ts

@@ -183,7 +183,7 @@ export class UserAdmin {
     if (users.length === 0) {
       throw new GrpcError(status.NOT_FOUND, 'Users do not exist');
     }
-    await User.getInstance().updateMany({ _id: { $in: ids } }, { active: block }, true);
+    await User.getInstance().updateMany({ _id: { $in: ids } }, { active: block });
     if (block) {
       this.grpcSdk.bus?.publish('authentication:block:user', JSON.stringify(users));
       return 'Users were blocked';

modules/authentication/src/handlers/local.ts

@@ -364,7 +364,7 @@ export class LocalHandlers implements IAuthenticationStrategy {
 
     user.hashedPassword = await AuthUtils.hashPassword(newPassword);
 
-    await User.getInstance().findByIdAndUpdate(user._id, user, true);
+    await User.getInstance().findByIdAndUpdate(user._id, user);
     await Token.getInstance().deleteOne(passwordResetTokenDoc);
 
     await TokenProvider.getInstance().deleteUserTokens({
@@ -459,7 +459,7 @@ export class LocalHandlers implements IAuthenticationStrategy {
       }
       return 'Verification required';
     }
-    await User.getInstance().findByIdAndUpdate(user._id, { email: newEmail }, true);
+    await User.getInstance().findByIdAndUpdate(user._id, { email: newEmail });
     return 'Email changed successfully';
   }
 

modules/authorization/src/controllers/resource.controller.ts

@@ -99,7 +99,6 @@ export class ResourceController {
     return await ResourceDefinition.getInstance().findByIdAndUpdate(
       resourceDefinition._id,
       resource,
-      true,
     );
   }
 
@@ -124,7 +123,6 @@ export class ResourceController {
     return (await ResourceDefinition.getInstance().findByIdAndUpdate(
       resourceDefinition._id,
       resource,
-      true,
     ))!;
   }
 

modules/database/package.json

@@ -25,15 +25,19 @@
     "deepdash-es": "^5.3.9",
     "escape-string-regexp": "^4.0.0",
     "lodash": "^4.17.21",
+    "mariadb": "^3.0.2",
     "mongodb-extended-json": "^1.11.0",
     "mongodb-schema": "^9.0.0",
     "mongoose": "6.4.6",
     "mongoose-deep-populate": "^3.2.0",
+    "mysql2": "^3.0.1",
     "object-hash": "^3.0.0",
     "pg": "^8.7.3",
     "pg-hstore": "^2.3.4",
     "sequelize": "^6.21.2",
-    "sequelize-auto": "^0.8.8"
+    "sequelize-auto": "^0.8.8",
+    "sqlite3": "^5.1.4",
+    "tedious": "^15.1.2"
   },
   "directories": {
     "lib": "src"

modules/database/src/Database.ts

@@ -1,36 +1,36 @@
 import ConduitGrpcSdk, {
+  ConduitModel,
   ConduitSchema,
   GrpcError,
-  HealthCheckStatus,
-  ManagedModule,
   GrpcRequest,
   GrpcResponse,
-  ConduitModel,
+  HealthCheckStatus,
+  ManagedModule,
 } from '@conduitplatform/grpc-sdk';
 import { AdminHandlers } from './admin';
 import { DatabaseRoutes } from './routes';
 import * as models from './models';
 import {
-  Schema as SchemaDto,
   DropCollectionRequest,
   DropCollectionResponse,
   FindOneRequest,
   FindRequest,
   GetSchemaRequest,
   GetSchemasRequest,
+  MigrateRequest,
   QueryRequest,
   QueryResponse,
+  RawQueryRequest,
+  Schema as SchemaDto,
   UpdateManyRequest,
   UpdateRequest,
-  RawQueryRequest,
 } from './protoTypes/database';
 import { CreateSchemaExtensionRequest, SchemaResponse, SchemasResponse } from './types';
 import { DatabaseAdapter } from './adapters/DatabaseAdapter';
 import { MongooseAdapter } from './adapters/mongoose-adapter';
-import { SequelizeAdapter } from './adapters/sequelize-adapter';
 import { MongooseSchema } from './adapters/mongoose-adapter/MongooseSchema';
 import { SequelizeSchema } from './adapters/sequelize-adapter/SequelizeSchema';
-import { Schema, ConduitDatabaseSchema } from './interfaces';
+import { ConduitDatabaseSchema, Schema } from './interfaces';
 import { canCreate, canDelete, canModify } from './permissions';
 import { runMigrations } from './migrations';
 import { SchemaController } from './controllers/cms/schema.controller';
@@ -40,6 +40,8 @@ import { status } from '@grpc/grpc-js';
 import path from 'path';
 import metricsSchema from './metrics';
 import { isNil } from 'lodash';
+import { PostgresAdapter } from './adapters/sequelize-adapter/postgres-adapter';
+import { SQLAdapter } from './adapters/sequelize-adapter/sql-adapter';
 
 export default class DatabaseModule extends ManagedModule<void> {
   configSchema = undefined;
@@ -63,6 +65,7 @@ export default class DatabaseModule extends ManagedModule<void> {
       deleteMany: this.deleteMany.bind(this),
       countDocuments: this.countDocuments.bind(this),
       rawQuery: this.rawQuery.bind(this),
+      migrate: this.migrate.bind(this),
     },
   };
   private adminRouter?: AdminHandlers;
@@ -74,9 +77,10 @@ export default class DatabaseModule extends ManagedModule<void> {
     this.updateHealth(HealthCheckStatus.UNKNOWN, true);
     if (dbType === 'mongodb') {
       this._activeAdapter = new MongooseAdapter(dbUri);
-    } else if (dbType === 'postgres' || dbType === 'sql') {
-      // Compat (<=0.12.2): sql
-      this._activeAdapter = new SequelizeAdapter(dbUri);
+    } else if (dbType === 'postgres') {
+      this._activeAdapter = new PostgresAdapter(dbUri);
+    } else if (['sql', 'mariadb', 'mysql', 'sqlite', 'mssql'].includes(dbType)) {
+      this._activeAdapter = new SQLAdapter(dbUri);
     } else {
       throw new Error('Database type not supported');
     }
@@ -89,8 +93,9 @@ export default class DatabaseModule extends ManagedModule<void> {
 
   async onServerStart() {
     await this._activeAdapter.registerSystemSchema(models.DeclaredSchema);
+    await this._activeAdapter.registerSystemSchema(models.MigratedSchemas);
     const modelPromises = Object.values(models).flatMap((model: ConduitSchema) => {
-      if (model.name === '_DeclaredSchema') return [];
+      if (['_DeclaredSchema', 'MigratedSchema'].includes(model.name)) return [];
       return this._activeAdapter.registerSystemSchema(model);
     });
     await Promise.all(modelPromises);
@@ -432,7 +437,6 @@ export default class DatabaseModule extends ManagedModule<void> {
       const result = await schemaAdapter.model.findByIdAndUpdate(
         call.request.id,
         call.request.query,
-        call.request.updateProvidedOnly,
         call.request.populate,
       );
       const resultString = JSON.stringify(result);
@@ -466,7 +470,7 @@ export default class DatabaseModule extends ManagedModule<void> {
       const result = await schemaAdapter.model.updateMany(
         call.request.filterQuery,
         call.request.query,
-        call.request.updateProvidedOnly,
+        call.request.populate,
       );
       const resultString = JSON.stringify(result);
 
@@ -563,7 +567,7 @@ export default class DatabaseModule extends ManagedModule<void> {
     const dbType = this._activeAdapter.getDatabaseType();
     if (
       (dbType === 'MongoDB' && isNil(query?.mongoQuery)) ||
-      (dbType === 'PostgreSQL' && isNil(query?.sqlQuery))
+      (dbType !== 'MongoDB' && isNil(query?.sqlQuery))
     ) {
       callback({
         code: status.INVALID_ARGUMENT,
@@ -597,4 +601,11 @@ export default class DatabaseModule extends ManagedModule<void> {
       callback({ code: status.INTERNAL, message: (e as Error).message });
     }
   }
+
+  async migrate(call: GrpcRequest<MigrateRequest>, callback: GrpcResponse<null>) {
+    if (this._activeAdapter.getDatabaseType() !== 'MongoDB') {
+      await this._activeAdapter.syncSchema(call.request.schemaName);
+    }
+    callback(null, null);
+  }
 }