feat(testing): databricks test suite (#5311)

cube-js · Sep 24, 2022 · b77f33b · b77f33b
1 parent 3dcbd2e
commit b77f33b
Show file tree

Hide file tree

Showing 11 changed files with 10,064 additions and 9 deletions.
diff --git a/packages/cubejs-testing/.gitignore b/packages/cubejs-testing/.gitignore
@@ -26,3 +26,7 @@ birdbox-fixtures/questdb/schema/ECommerce.sql.js
 birdbox-fixtures/questdb/schema/Products.js
 birdbox-fixtures/questdb/schema/Products.sql.js
 birdbox-fixtures/postgresql.json
+
+birdbox-fixtures/databricks-jdbc.json
+birdbox-fixtures/databricks
+!birdbox-fixtures/databricks/cube.js
diff --git a/packages/cubejs-testing/birdbox-fixtures/databricks-jdbc.yml b/packages/cubejs-testing/birdbox-fixtures/databricks-jdbc.yml
@@ -0,0 +1,29 @@
+version: "2.2"
+
+services:
+  cube:
+    container_name: birdbox-cube
+    image: ${BIRDBOX_CUBEJS_REGISTRY_PATH}cubejs/cube:${BIRDBOX_CUBEJS_VERSION:-latest}
+    environment:
+      CUBEJS_DB_TYPE: databricks-jdbc
+
+      # from env
+      CUBEJS_DB_BQ_PROJECT_ID: ${CUBEJS_DB_BQ_PROJECT_ID}
+      CUBEJS_DB_BQ_CREDENTIALS: ${CUBEJS_DB_BQ_CREDENTIALS}
+      CUBEJS_DB_EXPORT_BUCKET: ${CUBEJS_DB_EXPORT_BUCKET}
+
+      # from test
+      CUBEJS_SCHEDULED_REFRESH_DEFAULT: ${CUBEJS_SCHEDULED_REFRESH_DEFAULT}
+      CUBEJS_REFRESH_WORKER: ${CUBEJS_REFRESH_WORKER}
+      CUBEJS_EXTERNAL_DEFAULT: ${CUBEJS_EXTERNAL_DEFAULT}
+      CUBEJS_ROLLUP_ONLY: ${CUBEJS_ROLLUP_ONLY}
+      CUBEJS_TELEMETRY: ${CUBEJS_TELEMETRY}
+
+      # defaults
+      CUBEJS_DEV_MODE: "true"
+      CUBEJS_API_SECRET: mysupersecret
+    volumes:
+      - ./databricks-jdbc/cube.js:/cube/conf/cube.js
+    ports:
+      - "4000:4000"
+    restart: always
diff --git a/packages/cubejs-testing/birdbox-fixtures/databricks-jdbc/cube.js b/packages/cubejs-testing/birdbox-fixtures/databricks-jdbc/cube.js
@@ -0,0 +1,9 @@
+// Cube.js configuration options: https://cube.dev/docs/config
+module.exports = {
+  queryRewrite: (query) => {
+    if (query.measures) {
+      query.measures = query.measures.filter(m => m !== 'Orders.toRemove');
+    }
+    return query;
+  }
+};
diff --git a/packages/cubejs-testing/birdbox-fixtures/driver-test-data/CAST.js b/packages/cubejs-testing/birdbox-fixtures/driver-test-data/CAST.js
@@ -15,6 +15,10 @@ export const DB_CAST = {
     SELECT_PREFIX: '',
     SELECT_SUFFIX: '',
   },
+  'databricks-jdbc': {
+    SELECT_PREFIX: '',
+    SELECT_SUFFIX: '',
+  },
   questdb: {
     SELECT_PREFIX: 'with tmp_tbl as (\n',
     SELECT_SUFFIX: ')\nselect * from tmp_tbl',
@@ -37,6 +41,10 @@ export const DATE_CAST = {
     DATE_PREFIX: 'to_date(',
     DATE_SUFFIX: ', \'YYYY-MM-DD\')',
   },
+  'databricks-jdbc': {
+    DATE_PREFIX: 'to_date(',
+    DATE_SUFFIX: ', \'y-M-d\')',
+  },
   questdb: {
     DATE_PREFIX: 'to_date(',
     DATE_SUFFIX: ', \'YYYY-MM-DD\')',

diff --git a/packages/cubejs-testing/package.json b/packages/cubejs-testing/package.json
@@ -44,6 +44,8 @@
     "driver:postgres:snap": "jest --verbose --updateSnapshot -i dist/test/driver-postgres.test.js",
     "driver:questdb": "jest --verbose -i dist/test/driver-questdb.test.js",
     "driver:questdb:snap": "jest --verbose --updateSnapshot -i dist/test/driver-questdb.test.js",
+    "driver:databricks": "jest --verbose -i dist/test/driver-databricks.test.js",
+    "driver:databricks:snap": "jest --verbose --updateSnapshot -i dist/test/driver-databricks.test.js",
     "smoke:athena": "jest --verbose -i dist/test/smoke-athena.test.js",
     "smoke:athena:snapshot": "jest --verbose --updateSnapshot -i dist/test/smoke-athena.test.js",
     "smoke:bigquery": "jest --verbose -i dist/test/smoke-bigquery.test.js",

diff --git a/packages/cubejs-testing/src/REQUIRED_ENV_VARS.ts b/packages/cubejs-testing/src/REQUIRED_ENV_VARS.ts
@@ -33,4 +33,15 @@ export const REQUIRED_ENV_VARS: {[key: string]: string[]} = {
     'CUBEJS_DB_USER',
     'CUBEJS_DB_PASS',
   ],
+  'databricks-jdbc': [
+    'CUBEJS_DB_TYPE',
+    'CUBEJS_DB_DATABRICKS_URL',
+    'CUBEJS_DB_DATABRICKS_ACCEPT_POLICY',
+    'CUBEJS_DB_NAME',
+    'CUBEJS_DB_EXPORT_BUCKET_TYPE',
+    'CUBEJS_DB_EXPORT_BUCKET',
+    'CUBEJS_DB_EXPORT_BUCKET_AWS_KEY',
+    'CUBEJS_DB_EXPORT_BUCKET_AWS_SECRET',
+    'CUBEJS_DB_EXPORT_BUCKET_AWS_REGION',
+  ],
 };
diff --git a/packages/cubejs-testing/src/birdbox.ts b/packages/cubejs-testing/src/birdbox.ts
@@ -48,7 +48,7 @@ interface Args {
   log: Log,
 }
 
-export type DriverType = 'postgresql' | 'postgres' | 'multidb' | 'materialize' | 'crate' | 'bigquery' | 'athena' | 'postgresql-cubestore' | 'firebolt' | 'questdb' | 'redshift';
+export type DriverType = 'postgresql' | 'postgres' | 'multidb' | 'materialize' | 'crate' | 'bigquery' | 'athena' | 'postgresql-cubestore' | 'firebolt' | 'questdb' | 'redshift' | 'databricks-jdbc';
 
 export type Schemas = string[];
 
@@ -102,6 +102,7 @@ const driverNameToFolderNameMapper: Record<DriverType, string> = {
   firebolt: 'postgresql',
   questdb: 'postgresql',
   redshift: 'postgresql',
+  'databricks-jdbc': 'databricks-jdbc'
 };
 
 /**
@@ -167,13 +168,17 @@ function clearTestData(type: DriverType) {
 function runSchemasGeneration(type: DriverType, schemas: Schemas) {
   const targetFolder = getTargetFolder(type);
 
+  if (!fs.existsSync(targetFolder)) {
+    fs.mkdirSync(targetFolder, { recursive: true });
+  }
   schemas.forEach((s) => {
     const originalContent = fs.readFileSync(
       path.join(SOURCE, s), 'utf8'
     );
 
     const { base } = path.parse(s);
     const updatedContent = originalContent.replace('_type_', type);
+
     fs.writeFileSync(
       path.join(targetFolder, base),
       updatedContent
@@ -321,6 +326,7 @@ export async function startBirdBoxFromContainer(
       'BIRDBOX_CUBESTORE_VERSION',
       process.env.BIRDBOX_CUBESTORE_VERSION
     )
+    .withEnv('CUBEJS_TELEMETRY', 'false')
     .up();
 
   const host = '127.0.0.1';
@@ -419,12 +425,17 @@ export async function startBirdBoxFromCli(
   let db: StartedTestContainer;
   let cli: ChildProcess;
 
-  if (!options.schemaDir) {
-    options.schemaDir = 'postgresql/schema';
-  }
+  if (options.schemas) {
+    options.schemaDir = `${options.type}/schema`;
+    options.cubejsConfig = `${options.type}/cube.js`;
+  } else {
+    if (!options.schemaDir) {
+      options.schemaDir = 'postgresql/schema';
+    }
 
-  if (!options.cubejsConfig) {
-    options.cubejsConfig = 'postgresql/single/cube.js';
+    if (!options.cubejsConfig) {
+      options.cubejsConfig = 'postgresql/single/cube.js';
+    }
   }
 
   if (options.loadScript) {
@@ -509,6 +520,8 @@ export async function startBirdBoxFromCli(
     );
   }
 
+  fs.writeFileSync(path.join(testDir, 'package.json'), '{}', { encoding: 'utf-8' });
+
   const env = {
     ...process.env,
     CUBEJS_DB_TYPE: options.type === 'postgresql'
@@ -518,6 +531,7 @@ export async function startBirdBoxFromCli(
     CUBEJS_API_SECRET: 'mysupersecret',
     CUBEJS_WEB_SOCKETS: 'true',
     CUBEJS_PLAYGROUND_AUTH_SECRET: 'mysupersecret',
+    CUBEJS_TELEMETRY: 'false',
     ...options.env
       ? options.env
       : {
@@ -672,12 +686,13 @@ export async function getBirdbox(
       case Mode.CLI:
       case Mode.LOCAL: {
         birdbox = await startBirdBoxFromCli({
-          type,
+          type: type === 'postgres' ? 'postgresql' : type,
           env,
           log,
           cubejsConfig: options.cubejsConfig,
           schemaDir: options.schemaDir,
           useCubejsServerBinary: mode === Mode.LOCAL,
+          schemas: options?.schemas,
         });
         break;
       }