feat: App multi-tenancy support in single ServerCore instance

Author: paveltiunov

packages/cubejs-api-gateway/index.js

@@ -221,8 +221,8 @@ class ApiGateway {
         });
         const normalizedQuery = normalizeQuery(query);
         const [compilerSqlResult, metaConfigResult] = await Promise.all([
-          this.compilerApi.getSql(coerceForSqlQuery(normalizedQuery, req)),
-          this.compilerApi.metaConfig()
+          this.getCompilerApi(req).getSql(coerceForSqlQuery(normalizedQuery, req)),
+          this.getCompilerApi(req).metaConfig()
         ]);
         const sqlQuery = compilerSqlResult;
         const metaConfig = metaConfigResult;
@@ -231,7 +231,7 @@ class ApiGateway {
         const toExecute = {
           ...sqlQuery, query: sqlQuery.sql[0], values: sqlQuery.sql[1], continueWait: true
         };
-        const response = await this.adapterApi.executeQuery(toExecute);
+        const response = await this.getAdapterApi(req).executeQuery(toExecute);
         this.log(req, {
           type: 'Load Request Success',
           query: req.query.query,
@@ -255,7 +255,7 @@ class ApiGateway {
       try {
         const query = JSON.parse(req.query.query);
         const normalizedQuery = normalizeQuery(query);
-        const sqlQuery = await this.compilerApi.getSql(coerceForSqlQuery(normalizedQuery, req));
+        const sqlQuery = await this.getCompilerApi(req).getSql(coerceForSqlQuery(normalizedQuery, req));
         res.json({
           sql: sqlQuery
         });
@@ -266,7 +266,7 @@ class ApiGateway {
 
     app.get(`${this.basePath}/v1/meta`, this.checkAuthMiddleware, (async (req, res) => {
       try {
-        const metaConfig = await this.compilerApi.metaConfig();
+        const metaConfig = await this.getCompilerApi(req).metaConfig();
         const cubes = metaConfig.map(c => c.config);
         res.json({ cubes });
       } catch (e) {
@@ -275,6 +275,24 @@ class ApiGateway {
     }));
   }
 
+  getCompilerApi(req) {
+    if (typeof this.compilerApi === 'function') {
+      return this.compilerApi(this.contextByReq(req));
+    }
+    return this.compilerApi;
+  }
+
+  getAdapterApi(req) {
+    if (typeof this.adapterApi === 'function') {
+      return this.adapterApi(this.contextByReq(req));
+    }
+    return this.adapterApi;
+  }
+
+  contextByReq(req) {
+    return { authInfo: req.authInfo };
+  }
+
   handleError(e, req, res) {
     if (e instanceof UserError) {
       this.log(req, {

packages/cubejs-playground/config-overrides.js

@@ -1,4 +1,45 @@
+const TerserPlugin = require('terser-webpack-plugin');
+const webpack = require('webpack');
+
 module.exports = function override(config, env) {
-  config.optimization = { minimize: false };
+  config.optimization = {
+    minimizer: [
+      new TerserPlugin({
+        cache: true,
+        parallel: true,
+        chunkFilter: (chunk) => chunk.name.indexOf('babel') === -1 // && chunk.name.indexOf('vendors') === -1
+      })
+    ],
+    splitChunks: {
+      chunks: 'all',
+      minSize: 30000,
+      maxSize: 0,
+      minChunks: 1,
+      maxAsyncRequests: 5,
+      maxInitialRequests: 3,
+      automaticNameDelimiter: '~',
+      name: true,
+      cacheGroups: {
+        babel: {
+          test: /babel/,
+          priority: -5
+        },
+        vendors: {
+          test: /[\\/]node_modules[\\/]/,
+          priority: -10
+        },
+        default: {
+          priority: -20
+        }
+      }
+    }
+  };
+  config.stats = 'verbose';
+  config.plugins = config.plugins.concat([
+    new webpack.ProgressPlugin()
+  ]);
+  if (env === 'production') {
+    config.devtool = false;
+  }
   return config;
 };

packages/cubejs-playground/package.json

@@ -34,7 +34,7 @@
   },
   "scripts": {
     "start": "react-app-rewired start",
-    "build": "react-app-rewired build && cp -R build/ ../cubejs-server-core/playground/",
+    "build": "react-app-rewired build && rm -Rf ../cubejs-server-core/playground/** && cp -R build/ ../cubejs-server-core/playground/",
     "test": "react-app-rewired test",
     "eject": "react-scripts eject"
   },
@@ -59,6 +59,7 @@
     "eslint-plugin-import": "^2.16.0",
     "eslint-plugin-jsx-a11y": "^6.2.1",
     "eslint-plugin-react": "^7.12.4",
-    "react-app-rewired": "^2.1.0"
+    "react-app-rewired": "^2.1.0",
+    "terser-webpack-plugin": "^1.2.3"
   }
 }

packages/cubejs-playground/yarn.lock

@@ -1063,10 +1063,10 @@
   resolved "https://registry.yarnpkg.com/@csstools/convert-colors/-/convert-colors-1.4.0.tgz#ad495dc41b12e75d588c6db8b9834f08fa131eb7"
   integrity sha512-5a6wqoJV/xEdbRNKVo6I4hO3VjyDq//8q2f9I6PBAvMesJHFauXDorcNCsr9RzvsZnaWi5NYCcfyqP1QeFHFbw==
 
-"@cubejs-client/core@^0.4.5":
-  version "0.4.5"
-  resolved "https://registry.yarnpkg.com/@cubejs-client/core/-/core-0.4.5.tgz#6df4b6c15fbf678323d14bbaae83607ca1ebe437"
-  integrity sha512-EGitKr6zPtsOcMyFSDe6fLKS7nzFhMIptnbMgEPbvfhlqw4CyvSACSBoThMC4Sa3NVOjpmGOTnzKrglzegE3sw==
+"@cubejs-client/core@^0.6.0":
+  version "0.6.0"
+  resolved "https://registry.yarnpkg.com/@cubejs-client/core/-/core-0.6.0.tgz#86b36526936a569e9de0b61bab4aecf9f9bb7584"
+  integrity sha512-z23Hghs6QH5E1/8nNAzqjWRtlmyqIsIFM8oSjj+/5npjwVl63RgHILex+nNzlTCanYZrHtkC+Fnta4ThmG9O5g==
   dependencies:
     "@babel/runtime" "^7.1.2"
     core-js "^2.5.3"
@@ -1075,10 +1075,10 @@
     ramda "^0.25.0"
     whatwg-fetch "^3.0.0"
 
-"@cubejs-client/react@^0.4.5":
-  version "0.4.5"
-  resolved "https://registry.yarnpkg.com/@cubejs-client/react/-/react-0.4.5.tgz#4cc4dbf86865a53a37d63f8ee09ff1f5653cf078"
-  integrity sha512-hryecjoLOqFs1fEHtu0ErZZidbDGQIw6CJiUVXzQbqt7jhh7/GaD3jVcbshm0UDfH9sAPDKoYT0+jZv54o2lbA==
+"@cubejs-client/react@^0.6.0":
+  version "0.6.0"
+  resolved "https://registry.yarnpkg.com/@cubejs-client/react/-/react-0.6.0.tgz#31516d96467a3d09fb018fb00b9f619cb465b7e1"
+  integrity sha512-9xwVp7ankCrF4wVdEbnYLNYNHMInavJCGnKObKhpzpAV0gqD3sGZHPAzmS2bf+WzxF9PWDkIJiooUCiTlJT9pQ==
   dependencies:
     "@babel/runtime" "^7.1.2"
     prop-types "^15.6.2"
@@ -10816,6 +10816,20 @@ terser-webpack-plugin@1.2.2, terser-webpack-plugin@^1.1.0:
     webpack-sources "^1.1.0"
     worker-farm "^1.5.2"
 
+terser-webpack-plugin@^1.2.3:
+  version "1.2.3"
+  resolved "https://registry.yarnpkg.com/terser-webpack-plugin/-/terser-webpack-plugin-1.2.3.tgz#3f98bc902fac3e5d0de730869f50668561262ec8"
+  integrity sha512-GOK7q85oAb/5kE12fMuLdn2btOS9OBZn4VsecpHDywoUC/jLhSAKOiYo0ezx7ss2EXPMzyEWFoE0s1WLE+4+oA==
+  dependencies:
+    cacache "^11.0.2"
+    find-cache-dir "^2.0.0"
+    schema-utils "^1.0.0"
+    serialize-javascript "^1.4.0"
+    source-map "^0.6.1"
+    terser "^3.16.1"
+    webpack-sources "^1.1.0"
+    worker-farm "^1.5.2"
+
 terser@^3.16.1:
   version "3.16.1"
   resolved "https://registry.yarnpkg.com/terser/-/terser-3.16.1.tgz#5b0dd4fa1ffd0b0b43c2493b2c364fd179160493"

packages/cubejs-server-core/core/index.js

@@ -66,6 +66,15 @@ class CubejsServerCore {
       }
     });
     this.repository = new FileRepository(this.schemaPath);
+    this.repositoryFactory = options.repositoryFactory || (() => this.repository);
+    this.contextToDbType = typeof options.dbType === 'function' ? options.dbType : () => options.dbType;
+    this.appIdToCompilerApi = {};
+    this.appIdToOrchestratorApi = {};
+    this.contextToAppId = options.contextToAppId || (() => process.env.CUBEJS_APP || 'STANDALONE');
+    this.orchestratorOptions =
+      typeof options.orchestratorOptions === 'function' ?
+        options.orchestratorOptions :
+        () => options.orchestratorOptions;
 
     const Analytics = require('analytics-node');
     const client = new Analytics('dSR8JiNYIGKyQHKid9OaLYugXLao18hA', { flushInterval: 100 });
@@ -148,12 +157,10 @@ class CubejsServerCore {
 
   async initApp(app) {
     checkEnvForPlaceholders();
-    this.compilerApi = this.createCompilerApi(this.repository);
-    this.orchestratorApi = this.createOrchestratorApi();
     const apiGateway = new ApiGateway(
       this.apiSecret,
-      this.compilerApi,
-      this.orchestratorApi,
+      this.getCompilerApi.bind(this),
+      this.getOrchestratorApi.bind(this),
       this.logger, {
         basePath: this.options.basePath,
         checkAuthMiddleware: this.options.checkAuthMiddleware
@@ -165,18 +172,55 @@ class CubejsServerCore {
     }
   }
 
-  createCompilerApi(repository) {
-    return new CompilerApi(repository, this.dbType, {
-      schemaVersion: this.options.schemaVersion,
+  getCompilerApi(context) {
+    const appId = this.contextToAppId(context);
+    if (!this.appIdToCompilerApi[appId]) {
+      this.appIdToCompilerApi[appId] = this.createCompilerApi(
+        this.repositoryFactory(context), {
+          dbType: this.contextToDbType(context),
+          schemaVersion: this.options.schemaVersion && (() => this.options.schemaVersion(context))
+        }
+      );
+    }
+    return this.appIdToCompilerApi[appId];
+  }
+
+  getOrchestratorApi(context) {
+    const appId = this.contextToAppId(context);
+    if (!this.appIdToOrchestratorApi[appId]) {
+      let driverPromise;
+      this.appIdToOrchestratorApi[appId] = this.createOrchestratorApi({
+        getDriver: () => {
+          if (!driverPromise) {
+            const driver = this.driverFactory(context);
+            driverPromise = driver.testConnection().then(() => driver).catch(e => {
+              driverPromise = null;
+              throw e;
+            });
+          }
+          return driverPromise;
+        },
+        redisPrefix: appId,
+        orchestratorOptions: this.orchestratorOptions(context)
+      });
+    }
+    return this.appIdToOrchestratorApi[appId];
+  }
+
+  createCompilerApi(repository, options) {
+    options = options || {};
+    return new CompilerApi(repository, options.dbType || this.dbType, {
+      schemaVersion: options.schemaVersion || this.options.schemaVersion,
       devServer: this.options.devServer,
       logger: this.logger
     });
   }
 
-  createOrchestratorApi() {
-    return new OrchestratorApi(() => this.getDriver(), this.logger, {
-      redisPrefix: process.env.CUBEJS_APP,
-      ...this.options.orchestratorOptions
+  createOrchestratorApi(options) {
+    options = options || {};
+    return new OrchestratorApi(options.getDriver || this.getDriver.bind(this), this.logger, {
+      redisPrefix: options.redisPrefix || process.env.CUBEJS_APP,
+      ...(options.orchestratorOptions || this.options.orchestratorOptions)
     });
   }