feat: Add server-core options validate (#1089)

Author: RusovDmitriy

CONTRIBUTING.md

@@ -47,7 +47,7 @@ The rest will be done by `BaseDriver` class.
 6. If db requires connection pooling prefer use `generic-pool` implementation with settings similar to other db packages.
 7. Make sure your driver has `release()` method in case DB expects graceful shutdowns for connections.
 8. Please use yarn to add any dependencies and run `$ yarn` within the package before committing to ensure right `yarn.lock` is in place.
-9. Add this driver dependency to [cubejs-server-core/core/index.js](https://github.com/statsbotco/cube.js/blob/master/packages/cubejs-server-core/core/index.js#L8).
+9. Add this driver dependency to [cubejs-server-core/core/DriverDependencies.js](https://github.com/cube-js/cube.js/blob/master/packages/cubejs-server-core/core/DriverDependencies.js#L1).
 
 ### Implementing JDBC Driver
 

packages/cubejs-server-core/core/DriverDependencies.js

@@ -0,0 +1,19 @@
+module.exports = {
+  postgres: '@cubejs-backend/postgres-driver',
+  mysql: '@cubejs-backend/mysql-driver',
+  mssql: '@cubejs-backend/mssql-driver',
+  athena: '@cubejs-backend/athena-driver',
+  jdbc: '@cubejs-backend/jdbc-driver',
+  mongobi: '@cubejs-backend/mongobi-driver',
+  bigquery: '@cubejs-backend/bigquery-driver',
+  redshift: '@cubejs-backend/postgres-driver',
+  clickhouse: '@cubejs-backend/clickhouse-driver',
+  hive: '@cubejs-backend/hive-driver',
+  snowflake: '@cubejs-backend/snowflake-driver',
+  prestodb: '@cubejs-backend/prestodb-driver',
+  oracle: '@cubejs-backend/oracle-driver',
+  sqlite: '@cubejs-backend/sqlite-driver',
+  awselasticsearch: '@cubejs-backend/elasticsearch-driver',
+  elasticsearch: '@cubejs-backend/elasticsearch-driver',
+  dremio: '@cubejs-backend/dremio-driver',
+};

packages/cubejs-server-core/core/index.js

@@ -14,26 +14,8 @@ const DevServer = require('./DevServer');
 const track = require('./track');
 const agentCollect = require('./agentCollect');
 const { version } = require('../package.json');
-
-const DriverDependencies = {
-  postgres: '@cubejs-backend/postgres-driver',
-  mysql: '@cubejs-backend/mysql-driver',
-  mssql: '@cubejs-backend/mssql-driver',
-  athena: '@cubejs-backend/athena-driver',
-  jdbc: '@cubejs-backend/jdbc-driver',
-  mongobi: '@cubejs-backend/mongobi-driver',
-  bigquery: '@cubejs-backend/bigquery-driver',
-  redshift: '@cubejs-backend/postgres-driver',
-  clickhouse: '@cubejs-backend/clickhouse-driver',
-  hive: '@cubejs-backend/hive-driver',
-  snowflake: '@cubejs-backend/snowflake-driver',
-  prestodb: '@cubejs-backend/prestodb-driver',
-  oracle: '@cubejs-backend/oracle-driver',
-  sqlite: '@cubejs-backend/sqlite-driver',
-  awselasticsearch: '@cubejs-backend/elasticsearch-driver',
-  elasticsearch: '@cubejs-backend/elasticsearch-driver',
-  dremio: '@cubejs-backend/dremio-driver',
-};
+const DriverDependencies = require('./DriverDependencies');
+const optionsValidate = require('./optionsValidate');
 
 const checkEnvForPlaceholders = () => {
   const placeholderSubstr = '<YOUR_DB_';
@@ -170,6 +152,7 @@ const prodLogger = (level) => (msg, params) => {
 
 class CubejsServerCore {
   constructor(options) {
+    optionsValidate(options);
     options = options || {};
     options = {
       driverFactory: () => typeof options.dbType === 'string' && CubejsServerCore.createDriver(options.dbType),

packages/cubejs-server-core/core/index.test.js

@@ -0,0 +1,115 @@
+/* eslint-disable no-new */
+/* globals describe,test,expect */
+
+const CubejsServerCore = require('./index');
+
+process.env.CUBEJS_API_SECRET = 'api-secret';
+
+describe('index.test', () => {
+  test('Should create instance of CubejsServerCore, dbType as string', () => {
+    expect(new CubejsServerCore({
+      dbType: 'mysql'
+    })).toBeInstanceOf(CubejsServerCore);
+  });
+
+  test('Should create instance of CubejsServerCore, dbType as func', () => {
+    const options = { dbType: () => {} };
+
+    expect(new CubejsServerCore(options))
+      .toBeInstanceOf(CubejsServerCore);
+  });
+
+  test('Should throw error, unknown dbType', () => {
+    const options = { dbType: 'unknown-db' };
+
+    expect(() => { new CubejsServerCore(options); })
+      .toThrowError(/"dbType" must be one of/);
+  });
+
+  test('Should throw error, unknown options property', () => {
+    const options = { dbType: 'mysql', unknown: 'some-value' };
+
+    expect(() => { new CubejsServerCore(options); })
+      .toThrowError(/"unknown" is not allowed/);
+  });
+
+  test('Should throw error, invalid options', () => {
+    const options = {
+      dbType: 'mysql',
+      externalDbType: 'mysql',
+      schemaPath: '/test/path/test/',
+      basePath: '/basePath',
+      webSocketsBasePath: '/webSocketsBasePath',
+      devServer: true,
+      compilerCacheSize: -10,
+    };
+
+    expect(() => { new CubejsServerCore(options); })
+      .toThrowError(/"compilerCacheSize" must be larger than or equal to 0/);
+  });
+
+  test('Should create instance of CubejsServerCore, pass all options', () => {
+    const queueOptions = {
+      concurrency: 3,
+      continueWaitTimeout: 5,
+      executionTimeout: 600,
+      orphanedTimeout: 120,
+      heartBeatInterval: 500
+    };
+
+    const options = {
+      dbType: 'mysql',
+      externalDbType: 'mysql',
+      schemaPath: '/test/path/test/',
+      basePath: '/basePath',
+      webSocketsBasePath: '/webSocketsBasePath',
+      devServer: false,
+      logger: () => {},
+      driverFactory: () => {},
+      externalDriverFactory: () => {},
+      contextToAppId: () => {},
+      contextToDataSourceId: () => {},
+      repositoryFactory: () => {},
+      checkAuth: () => {},
+      checkAuthMiddleware: () => {},
+      queryTransformer: () => {},
+      preAggregationsSchema: () => {},
+      schemaVersion: () => {},
+      extendContext: () => {},
+      scheduledRefreshTimer: true,
+      compilerCacheSize: 1000,
+      maxCompilerCacheKeepAlive: 10,
+      updateCompilerCacheKeepAlive: true,
+      telemetry: false,
+      allowUngroupedWithoutPrimaryKey: true,
+      orchestratorOptions: {
+        redisPrefix: 'some-prefix',
+        queryCacheOptions: {
+          refreshKeyRenewalThreshold: 1000,
+          backgroundRenew: true,
+          queueOptions
+        },
+        preAggregationsOptions: {
+          queueOptions
+        }
+      }
+    };
+
+    expect(new CubejsServerCore(options))
+      .toBeInstanceOf(CubejsServerCore);
+  });
+
+  test('Should create instance of CubejsServerCore, dbType from process.env.CUBEJS_DB_TYPE', () => {
+    process.env.CUBEJS_DB_TYPE = 'mysql';
+
+    expect(new CubejsServerCore({}))
+      .toBeInstanceOf(CubejsServerCore);
+  });
+
+  test('Should throw error, dbType is required', () => {
+    delete process.env.CUBEJS_DB_TYPE;
+
+    expect(() => { new CubejsServerCore({}); })
+      .toThrowError(/driverFactory, apiSecret, dbType are required options/);
+  });
+});

packages/cubejs-server-core/core/optionsValidate.js

@@ -0,0 +1,66 @@
+const Joi = require('@hapi/joi');
+const DriverDependencies = require('./DriverDependencies');
+
+
+const schemaQueueOptions = Joi.object().keys({
+  concurrency: Joi.number().min(1).integer(),
+  continueWaitTimeout: Joi.number().min(0).integer(),
+  executionTimeout: Joi.number().min(0).integer(),
+  orphanedTimeout: Joi.number().min(0).integer(),
+  heartBeatInterval: Joi.number().min(0).integer()
+});
+
+const dbTypes = Joi.alternatives().try(
+  Joi.string().valid(...Object.keys(DriverDependencies)),
+  Joi.func()
+);
+
+const schemaOptions = Joi.object().keys({
+  dbType: dbTypes,
+  externalDbType: dbTypes,
+  schemaPath: Joi.string(),
+  basePath: Joi.string(),
+  webSocketsBasePath: Joi.string(),
+  devServer: Joi.boolean(),
+  logger: Joi.func(),
+  driverFactory: Joi.func(),
+  externalDriverFactory: Joi.func(),
+  contextToAppId: Joi.func(),
+  contextToDataSourceId: Joi.func(),
+  repositoryFactory: Joi.func(),
+  checkAuth: Joi.func(),
+  checkAuthMiddleware: Joi.func(),
+  queryTransformer: Joi.func(),
+  preAggregationsSchema: Joi.alternatives().try(
+    Joi.string(),
+    Joi.func()
+  ),
+  schemaVersion: Joi.func(),
+  extendContext: Joi.func(),
+  scheduledRefreshTimer: Joi.alternatives().try(
+    Joi.boolean(),
+    Joi.number().min(0).integer()
+  ),
+  compilerCacheSize: Joi.number().min(0).integer(),
+  maxCompilerCacheKeepAlive: Joi.number().min(0).integer(),
+  updateCompilerCacheKeepAlive: Joi.boolean(),
+  telemetry: Joi.boolean(),
+  allowUngroupedWithoutPrimaryKey: Joi.boolean(),
+  orchestratorOptions: Joi.object().keys({
+    redisPrefix: Joi.string().allow(''),
+    queryCacheOptions: Joi.object().keys({
+      refreshKeyRenewalThreshold: Joi.number().min(0).integer(),
+      backgroundRenew: Joi.boolean(),
+      queueOptions: schemaQueueOptions
+    }),
+    preAggregationsOptions: {
+      queueOptions: schemaQueueOptions
+    }
+  })
+});
+
+
+module.exports = (options) => {
+  const { error } = Joi.validate(options, schemaOptions, { abortEarly: false });
+  if (error) throw new Error(`Invalid cube-server-core options: ${error.message || error.toString()}`);
+};

packages/cubejs-server-core/package.json

@@ -19,7 +19,10 @@
     "playground"
   ],
   "scripts": {
-    "lint": "eslint core/**/*.js"
+    "lint": "eslint core/**/*.js",
+    "test": "npm run test:unit", 
+    "test:unit": "jest ./**/*.test.js"
+    
   },
   "dependencies": {
     "@cubejs-backend/api-gateway": "^0.20.7",
@@ -40,7 +43,8 @@
     "semver": "^6.3.0",
     "serve-static": "^1.13.2",
     "sqlstring": "^2.3.1",
-    "uuid": "^3.3.3"
+    "uuid": "^3.3.3",
+    "@hapi/joi": "^15.1.1"
   },
   "devDependencies": {
     "eslint": "^6.8.0",