feat(database): DB introspection (#128)

libraries/grpc-sdk/src/classes/ConduitSchema.ts

@@ -3,7 +3,7 @@ import { ConduitModel, ConduitModelOptions } from '../interfaces';
 export class ConduitSchema {
   readonly name: string;
   readonly fields: ConduitModel;
-  readonly collectionName: string;
+  readonly collectionName?: string;
   readonly schemaOptions: ConduitModelOptions;
   ownerModule: string = 'unknown';
 
@@ -18,8 +18,6 @@ export class ConduitSchema {
     this.schemaOptions = schemaOptions ?? {};
     if (collectionName && collectionName !== '') {
       this.collectionName = collectionName;
-    } else {
-      this.collectionName = this.name;
     }
   }
 

modules/database/package.json

@@ -15,22 +15,24 @@
   },
   "license": "ISC",
   "dependencies": {
+    "@conduitplatform/grpc-sdk": "^1.0.1",
     "@grpc/grpc-js": "^1.5.2",
     "@grpc/proto-loader": "^0.5.4",
-    "@conduitplatform/grpc-sdk": "^1.0.1",
     "bson": "^4.2.2",
     "deepdash": "^5.0.3",
     "deepdash-es": "^5.0.3",
+    "escape-string-regexp": "^4.0.0",
     "lodash": "^4.17.15",
     "mariadb": "^2.5.4",
     "mongodb-extended-json": "^1.11.0",
+    "mongodb-schema": "^9.0.0",
     "mongoose": "5.13.13",
-    "escape-string-regexp": "^4.0.0",
     "mongoose-deep-populate": "^3.2.0",
     "mysql2": "^2.3.0",
     "pg": "^8.6.0",
     "pg-hstore": "^2.3.4",
     "sequelize": "^6.6.2",
+    "sequelize-auto": "^0.8.8",
     "sqlite3": "^5.0.2",
     "tedious": "^14.3.0"
   },

modules/database/src/Database.ts

@@ -35,6 +35,7 @@ import { SchemaController } from './controllers/cms/schema.controller';
 import { CustomEndpointController } from './controllers/customEndpoints/customEndpoint.controller';
 import { status } from '@grpc/grpc-js';
 import path from 'path';
+import { isEmpty } from 'lodash';
 
 export default class DatabaseModule extends ManagedModule {
   config = undefined;
@@ -67,7 +68,8 @@ export default class DatabaseModule extends ManagedModule {
     this.updateHealth(HealthCheckStatus.UNKNOWN, true);
     if (dbType === 'mongodb') {
       this._activeAdapter = new MongooseAdapter(dbUri);
-    } else if (dbType === 'postgres' || dbType === 'sql') { // Compat (<=0.12.2): sql
+    } else if (dbType === 'postgres' || dbType === 'sql') {
+      // Compat (<=0.12.2): sql
       this._activeAdapter = new SequelizeAdapter(dbUri);
     } else {
       throw new Error('Database type not supported');
@@ -81,14 +83,19 @@ export default class DatabaseModule extends ManagedModule {
 
   async onServerStart() {
     await this._activeAdapter.createSchemaFromAdapter(models.DeclaredSchema);
-    // Introspection takes place
+    const isConduitDb = await this._activeAdapter.isConduitDb();
+    if (!isConduitDb) {
+      await this.introspectDb();
+    }
     this.updateHealth(HealthCheckStatus.SERVING);
     const modelPromises = Object.values(models).flatMap((model: any) => {
       if (model.name === '_DeclaredSchema') return [];
       return this._activeAdapter.createSchemaFromAdapter(model);
     });
+
     await Promise.all(modelPromises);
     await runMigrations(this._activeAdapter);
+
     await this._activeAdapter.recoverSchemasFromDatabase();
   }
 
@@ -108,16 +115,12 @@ export default class DatabaseModule extends ManagedModule {
             receivedSchema.name,
             receivedSchema.modelSchema,
             receivedSchema.modelOptions,
-            receivedSchema.collectionName,
+            receivedSchema.collectionName
           );
           schema.ownerModule = receivedSchema.ownerModule;
-          self._activeAdapter
-            .createSchemaFromAdapter(schema)
-            .then(() => {
-            })
-            .catch(() => {
-              console.log('Failed to create/update schema');
-            });
+          self._activeAdapter.createSchemaFromAdapter(schema).catch(() => {
+            console.log('Failed to create/update schema');
+          });
         }
       } catch (err) {
         console.error('Something was wrong with the message');
@@ -171,7 +174,7 @@ export default class DatabaseModule extends ManagedModule {
       call.request.schema.name,
       JSON.parse(call.request.schema.modelSchema),
       JSON.parse(call.request.schema.modelOptions),
-      call.request.schema.collectionName,
+      call.request.schema.collectionName
     );
     if (schema.name.indexOf('-') >= 0 || schema.name.indexOf(' ') >= 0) {
       return callback({
@@ -258,7 +261,7 @@ export default class DatabaseModule extends ManagedModule {
       const schemas = await this._activeAdapter.deleteSchema(
         call.request.schemaName,
         call.request.deleteData,
-        (call as any).metadata.get('module-name')[0],
+        (call as any).metadata.get('module-name')[0]
       );
       callback(null, { result: schemas });
     } catch (err) {
@@ -324,7 +327,7 @@ export default class DatabaseModule extends ManagedModule {
         call.request.query,
         call.request.select,
         call.request.populate,
-        schemaAdapter.relations,
+        schemaAdapter.relations
       );
       callback(null, { result: JSON.stringify(doc) });
     } catch (err) {
@@ -352,7 +355,7 @@ export default class DatabaseModule extends ManagedModule {
         select,
         sort,
         populate,
-        schemaAdapter.relations,
+        schemaAdapter.relations
       );
       callback(null, { result: JSON.stringify(docs) });
     } catch (err) {
@@ -432,7 +435,7 @@ export default class DatabaseModule extends ManagedModule {
         call.request.query,
         call.request.updateProvidedOnly,
         call.request.populate,
-        schemaAdapter.relations,
+        schemaAdapter.relations
       );
       const resultString = JSON.stringify(result);
 
@@ -462,7 +465,7 @@ export default class DatabaseModule extends ManagedModule {
       const result = await schemaAdapter.model.updateMany(
         call.request.filterQuery,
         call.request.query,
-        call.request.updateProvidedOnly,
+        call.request.updateProvidedOnly
       );
       const resultString = JSON.stringify(result);
 
@@ -541,4 +544,26 @@ export default class DatabaseModule extends ManagedModule {
       });
     }
   }
+
+  private async introspectDb() {
+    console.log(`Database is not a Conduit DB. Starting introspection...`);
+    let introspectedSchemas = await this._activeAdapter.introspectDatabase(false);
+    await this._activeAdapter.createSchemaFromAdapter(models.PendingSchemas);
+
+    await Promise.all(
+      introspectedSchemas.map(async (schema: ConduitSchema) => {
+        if(isEmpty(schema.fields))
+          return null;
+        await this._activeAdapter.getSchemaModel('_PendingSchemas').model.create(
+          JSON.stringify({
+            name: schema.name,
+            fields: schema.fields,
+            modelOptions: schema.schemaOptions,
+            ownerModule: schema.ownerModule,
+            extensions: (schema as any).extensions,
+          })
+        );
+      })
+    );
+  }
 }

modules/database/src/adapters/DatabaseAdapter.ts

@@ -1,5 +1,5 @@
 import { ConduitSchema, GrpcError } from '@conduitplatform/grpc-sdk';
-import { SchemaAdapter } from '../interfaces';
+import { MultiDocQuery, SchemaAdapter } from '../interfaces';
 import { validateExtensionFields } from './utils/extensions';
 import { status } from '@grpc/grpc-js';
 import { isNil } from 'lodash';
@@ -8,6 +8,16 @@ export abstract class DatabaseAdapter<T extends SchemaAdapter<any>> {
   registeredSchemas: Map<string, ConduitSchema>;
   models?: { [name: string]: T };
 
+  /**
+   * Checks if the database has already been connected with Conduit
+   */
+  abstract isConduitDb(): Promise<boolean>;
+
+  /**
+   * Introspects all schemas of current db connection, registers them to conduit
+   */
+  abstract introspectDatabase(isConduitDb : boolean): Promise<ConduitSchema[]>;
+
   /**
    * Should accept a JSON schema and output a .ts interface for the adapter
    * @param schema
@@ -63,7 +73,7 @@ export abstract class DatabaseAdapter<T extends SchemaAdapter<any>> {
   ): { model: SchemaAdapter<any>; relations: any };
 
   fixDatabaseSchemaOwnership(schema: ConduitSchema) {
-    const dbSchemas = ['CustomEndpoints'];
+    const dbSchemas = ['CustomEndpoints','_PendingSchemas'];
     if (dbSchemas.includes(schema.name)) {
       schema.ownerModule = 'database';
     }

modules/database/src/adapters/mongoose-adapter/MongooseSchema.ts

@@ -3,6 +3,7 @@ import { MultiDocQuery, Query, SchemaAdapter, SingleDocQuery } from '../../inter
 import { MongooseAdapter } from './index';
 import { ConduitSchema } from '@conduitplatform/grpc-sdk';
 import { createWithPopulations } from './utils';
+import { isNil } from 'lodash';
 
 const EJSON = require('mongodb-extended-json');
 
@@ -18,6 +19,13 @@ export class MongooseSchema implements SchemaAdapter<Model<any>> {
     private readonly adapter: MongooseAdapter,
   ) {
     this.originalSchema = originalSchema;
+
+    if (!isNil(schema.collectionName)) {
+      (schema as any).schemaOptions.collection = schema.collectionName;
+    }
+    else {
+      (schema as any).collectionName = schema.name //restore collectionName
+    }
     let mongooseSchema = new Schema(schema.modelSchema as any, schema.schemaOptions);
     mongooseSchema.plugin(deepPopulate, {});
     this.model = mongoose.model(schema.name, mongooseSchema);

modules/database/src/adapters/mongoose-adapter/SchemaConverter.ts

@@ -9,7 +9,6 @@ const deepdash = require('deepdash/standalone');
  */
 export function schemaConverter(jsonSchema: ConduitSchema) {
   let copy = cloneDeep(jsonSchema);
-
   if (copy.modelSchema.hasOwnProperty('_id')) {
     delete copy.modelSchema['_id'];
   }