From 95e39260f4b9ef0120c2d31e07268e424a483eab Mon Sep 17 00:00:00 2001 From: Mikhail Nitsenko Date: Mon, 25 Sep 2023 16:01:59 +0600 Subject: [PATCH 01/10] fix --- packages/cubejs-databricks-jdbc-driver/src/DatabricksDriver.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/cubejs-databricks-jdbc-driver/src/DatabricksDriver.ts b/packages/cubejs-databricks-jdbc-driver/src/DatabricksDriver.ts index b64aed783af68..cc5279759476f 100644 --- a/packages/cubejs-databricks-jdbc-driver/src/DatabricksDriver.ts +++ b/packages/cubejs-databricks-jdbc-driver/src/DatabricksDriver.ts @@ -202,6 +202,7 @@ export class DatabricksDriver extends JDBCDriver { drivername: 'com.databricks.client.jdbc.Driver', customClassPath: undefined, properties: { + UID: 'token', // PWD-parameter passed to the connection string has higher priority, // so we can set this one to an empty string to avoid a Java error. PWD: From f0bf1181b232d83f2e74b0e9bd50604509204411 Mon Sep 17 00:00:00 2001 From: Mikhail Nitsenko Date: Thu, 28 Sep 2023 00:09:19 +0600 Subject: [PATCH 02/10] dev --- .../src/DatabricksDriver.ts | 35 +----------- .../src/helpers.ts | 56 +++++++++++++++++++ 2 files changed, 58 insertions(+), 33 deletions(-) create mode 100644 packages/cubejs-databricks-jdbc-driver/src/helpers.ts diff --git a/packages/cubejs-databricks-jdbc-driver/src/DatabricksDriver.ts b/packages/cubejs-databricks-jdbc-driver/src/DatabricksDriver.ts index cc5279759476f..d071a41be4ed3 100644 --- a/packages/cubejs-databricks-jdbc-driver/src/DatabricksDriver.ts +++ b/packages/cubejs-databricks-jdbc-driver/src/DatabricksDriver.ts @@ -8,8 +8,6 @@ import { getEnv, assertDataSource, } from '@cubejs-backend/shared'; -import fs from 'fs'; -import path from 'path'; import { S3, GetObjectCommand } from '@aws-sdk/client-s3'; import { getSignedUrl } from '@aws-sdk/s3-request-presigner'; import { @@ -25,7 +23,7 @@ import { JDBCDriverConfiguration, } from '@cubejs-backend/jdbc-driver'; import { DatabricksQuery } from './DatabricksQuery'; -import { downloadJDBCDriver } from './installer'; +import { resolveJDBCDriver, extractUidFromJdbcUrl } from './helpers'; export type DatabricksDriverConfiguration = JDBCDriverConfiguration & { @@ -91,16 +89,6 @@ export type DatabricksDriverConfiguration = JDBCDriverConfiguration & token?: string, }; -async function fileExistsOr( - fsPath: string, - fn: () => Promise, -): Promise { - if (fs.existsSync(fsPath)) { - return fsPath; - } - return fn(); -} - type ShowTableRow = { database: string, tableName: string, @@ -115,25 +103,6 @@ const DatabricksToGenericType: Record = { 'decimal(10,0)': 'bigint', }; -async function resolveJDBCDriver(): Promise { - return fileExistsOr( - path.join(process.cwd(), 'DatabricksJDBC42.jar'), - async () => fileExistsOr( - path.join(__dirname, '..', 'download', 'DatabricksJDBC42.jar'), - async () => { - const pathOrNull = await downloadJDBCDriver(); - if (pathOrNull) { - return pathOrNull; - } - throw new Error( - 'Please download and place DatabricksJDBC42.jar inside your ' + - 'project directory' - ); - } - ) - ); -} - /** * Databricks driver class. */ @@ -202,7 +171,7 @@ export class DatabricksDriver extends JDBCDriver { drivername: 'com.databricks.client.jdbc.Driver', customClassPath: undefined, properties: { - UID: 'token', + UID: extractUidFromJdbcUrl(url), // PWD-parameter passed to the connection string has higher priority, // so we can set this one to an empty string to avoid a Java error. PWD: diff --git a/packages/cubejs-databricks-jdbc-driver/src/helpers.ts b/packages/cubejs-databricks-jdbc-driver/src/helpers.ts new file mode 100644 index 0000000000000..3fdf4e5d2f3d8 --- /dev/null +++ b/packages/cubejs-databricks-jdbc-driver/src/helpers.ts @@ -0,0 +1,56 @@ +import fs from 'fs'; +import path from 'path'; + +import { downloadJDBCDriver } from './installer'; + +async function fileExistsOr( + fsPath: string, + fn: () => Promise, +): Promise { + if (fs.existsSync(fsPath)) { + return fsPath; + } + return fn(); +} + +export async function resolveJDBCDriver(): Promise { + return fileExistsOr( + path.join(process.cwd(), 'DatabricksJDBC42.jar'), + async () => fileExistsOr( + path.join(__dirname, '..', 'download', 'DatabricksJDBC42.jar'), + async () => { + const pathOrNull = await downloadJDBCDriver(); + if (pathOrNull) { + return pathOrNull; + } + throw new Error( + 'Please download and place DatabricksJDBC42.jar inside your ' + + 'project directory' + ); + } + ) + ); +} + +export function extractUidFromJdbcUrl(url: string): string { + const regex = /^jdbc:([^:]+):\/\/([^/]+)(\/[^;]+);(.+)$/; + const match = url.match(regex); + + if (!match) { + throw new Error(`Invalid JDBC URL: ${url}`); + } + + const paramsString = match[4]; + const paramsArray = paramsString.split(';'); + const params: any = {}; + + paramsArray.forEach(param => { + const [key, value] = param.split('='); + if (key && value) { + params[key] = value; + } + }); + + // Return the value of UID if present, otherwise return 'token' + return params.UID || 'token'; +} From 51462c583f328eadbce10c02825db21874db55a8 Mon Sep 17 00:00:00 2001 From: Mikhail Nitsenko Date: Thu, 28 Sep 2023 00:13:48 +0600 Subject: [PATCH 03/10] dev --- packages/cubejs-databricks-jdbc-driver/src/helpers.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/packages/cubejs-databricks-jdbc-driver/src/helpers.ts b/packages/cubejs-databricks-jdbc-driver/src/helpers.ts index 3fdf4e5d2f3d8..904391aec98b3 100644 --- a/packages/cubejs-databricks-jdbc-driver/src/helpers.ts +++ b/packages/cubejs-databricks-jdbc-driver/src/helpers.ts @@ -51,6 +51,5 @@ export function extractUidFromJdbcUrl(url: string): string { } }); - // Return the value of UID if present, otherwise return 'token' return params.UID || 'token'; } From ea0081322e49fd94266424ac512b63158ed51f77 Mon Sep 17 00:00:00 2001 From: Mikhail Nitsenko Date: Thu, 28 Sep 2023 00:50:45 +0600 Subject: [PATCH 04/10] upd --- .../cubejs-databricks-jdbc-driver/src/helpers.ts | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/packages/cubejs-databricks-jdbc-driver/src/helpers.ts b/packages/cubejs-databricks-jdbc-driver/src/helpers.ts index 904391aec98b3..966ccd1833d42 100644 --- a/packages/cubejs-databricks-jdbc-driver/src/helpers.ts +++ b/packages/cubejs-databricks-jdbc-driver/src/helpers.ts @@ -33,23 +33,27 @@ export async function resolveJDBCDriver(): Promise { } export function extractUidFromJdbcUrl(url: string): string { - const regex = /^jdbc:([^:]+):\/\/([^/]+)(\/[^;]+);(.+)$/; + const regex = /^jdbc:([^:]+):\/\/([^/]+)(\/[^;]*)?(?:;(.*))?$/; const match = url.match(regex); - + if (!match) { throw new Error(`Invalid JDBC URL: ${url}`); } const paramsString = match[4]; + if (!paramsString) { + return 'token'; + } + + const params: Record = {}; const paramsArray = paramsString.split(';'); - const params: any = {}; paramsArray.forEach(param => { const [key, value] = param.split('='); if (key && value) { - params[key] = value; + params[key] = decodeURIComponent(value); } }); - return params.UID || 'token'; + return params.UID || 'token'; // Return null if UID is not found } From ed22090920cfb5e96a23c6552b8577331fc878a3 Mon Sep 17 00:00:00 2001 From: Mikhail Nitsenko Date: Thu, 28 Sep 2023 01:04:06 +0600 Subject: [PATCH 05/10] Revert "arrange tables creation in incrementalSchemaLoading test" This reverts commit 990045a5831fbe29025856b7002b5961d0878dd4. --- .../src/tests/testIncrementalSchemaLoading.ts | 39 +++++++++---------- 1 file changed, 18 insertions(+), 21 deletions(-) diff --git a/packages/cubejs-testing-drivers/src/tests/testIncrementalSchemaLoading.ts b/packages/cubejs-testing-drivers/src/tests/testIncrementalSchemaLoading.ts index b701134caaefa..78211d7813bd2 100644 --- a/packages/cubejs-testing-drivers/src/tests/testIncrementalSchemaLoading.ts +++ b/packages/cubejs-testing-drivers/src/tests/testIncrementalSchemaLoading.ts @@ -22,6 +22,7 @@ export function testIncrementalSchemaLoading(type: string): void { options: { highWaterMark: number }, ) => Promise }; + let query: string[]; let env: Environment; let inputSchemas: QuerySchemasResult[]; let inputTables: QueryTablesResult[]; @@ -46,35 +47,23 @@ export function testIncrementalSchemaLoading(type: string): void { process.env.CUBEJS_DB_PORT = `${env.data.port}`; } driver = (await getDriver(type)).source; - const queries = getCreateQueries(type, suffix); - console.log(`Creating ${queries.length} fixture tables`); - try { - for (const q of queries) { - await driver.query(q); - } - console.log(`Creating ${queries.length} fixture tables completed`); - } catch (e: any) { - console.log('Error creating fixtures', e.stack); - throw e; - } }); afterAll(async () => { - try { - console.log(`Dropping ${tables.length} fixture tables`); - for (const t of tables) { - await driver.dropTable(t); - } - console.log(`Dropping ${tables.length} fixture tables completed`); - } finally { - await driver.release(); - await env.stop(); - } + await driver.release(); + await env.stop(); }); execute('should establish a connection', async () => { await driver.testConnection(); }); + + execute('should create the data source', async () => { + query = getCreateQueries(type, suffix); + await Promise.all(query.map(async (q) => { + await driver.query(q); + })); + }); execute('should load and check driver capabilities', async () => { const capabilities = driver.capabilities(); @@ -115,5 +104,13 @@ export function testIncrementalSchemaLoading(type: string): void { data_type: expect.any(String), }); }); + + execute('should delete the data source', async () => { + await Promise.all( + tables.map(async (t) => { + await driver.dropTable(t); + }) + ); + }); }); } From 8cb565e00dfc52e5caaa3a1b9a08a142fdabaa92 Mon Sep 17 00:00:00 2001 From: Mikhail Nitsenko Date: Thu, 28 Sep 2023 09:12:40 +0600 Subject: [PATCH 06/10] upd --- .../src/helpers.ts | 30 ++++--------------- 1 file changed, 6 insertions(+), 24 deletions(-) diff --git a/packages/cubejs-databricks-jdbc-driver/src/helpers.ts b/packages/cubejs-databricks-jdbc-driver/src/helpers.ts index 966ccd1833d42..f6f71a9eaf6fd 100644 --- a/packages/cubejs-databricks-jdbc-driver/src/helpers.ts +++ b/packages/cubejs-databricks-jdbc-driver/src/helpers.ts @@ -32,28 +32,10 @@ export async function resolveJDBCDriver(): Promise { ); } -export function extractUidFromJdbcUrl(url: string): string { - const regex = /^jdbc:([^:]+):\/\/([^/]+)(\/[^;]*)?(?:;(.*))?$/; - const match = url.match(regex); - - if (!match) { - throw new Error(`Invalid JDBC URL: ${url}`); - } - - const paramsString = match[4]; - if (!paramsString) { - return 'token'; - } - - const params: Record = {}; - const paramsArray = paramsString.split(';'); - - paramsArray.forEach(param => { - const [key, value] = param.split('='); - if (key && value) { - params[key] = decodeURIComponent(value); - } - }); - - return params.UID || 'token'; // Return null if UID is not found +export function extractUidFromJdbcUrl(jdbcUrl: string): string { + const [baseUrl, ...params] = jdbcUrl.split(';'); + const queryString = params.join('&'); + const standardUrl = `${baseUrl}?${queryString}`; + const url = new URL(standardUrl); + return url.searchParams.get('UID') || 'token'; } From 5e99e147498123742320dacabe0858450951f1b5 Mon Sep 17 00:00:00 2001 From: Mikhail Nitsenko Date: Thu, 28 Sep 2023 10:39:17 +0600 Subject: [PATCH 07/10] upd --- .../src/tests/testIncrementalSchemaLoading.ts | 39 ++++++++++--------- 1 file changed, 21 insertions(+), 18 deletions(-) diff --git a/packages/cubejs-testing-drivers/src/tests/testIncrementalSchemaLoading.ts b/packages/cubejs-testing-drivers/src/tests/testIncrementalSchemaLoading.ts index 78211d7813bd2..b701134caaefa 100644 --- a/packages/cubejs-testing-drivers/src/tests/testIncrementalSchemaLoading.ts +++ b/packages/cubejs-testing-drivers/src/tests/testIncrementalSchemaLoading.ts @@ -22,7 +22,6 @@ export function testIncrementalSchemaLoading(type: string): void { options: { highWaterMark: number }, ) => Promise }; - let query: string[]; let env: Environment; let inputSchemas: QuerySchemasResult[]; let inputTables: QueryTablesResult[]; @@ -47,23 +46,35 @@ export function testIncrementalSchemaLoading(type: string): void { process.env.CUBEJS_DB_PORT = `${env.data.port}`; } driver = (await getDriver(type)).source; + const queries = getCreateQueries(type, suffix); + console.log(`Creating ${queries.length} fixture tables`); + try { + for (const q of queries) { + await driver.query(q); + } + console.log(`Creating ${queries.length} fixture tables completed`); + } catch (e: any) { + console.log('Error creating fixtures', e.stack); + throw e; + } }); afterAll(async () => { - await driver.release(); - await env.stop(); + try { + console.log(`Dropping ${tables.length} fixture tables`); + for (const t of tables) { + await driver.dropTable(t); + } + console.log(`Dropping ${tables.length} fixture tables completed`); + } finally { + await driver.release(); + await env.stop(); + } }); execute('should establish a connection', async () => { await driver.testConnection(); }); - - execute('should create the data source', async () => { - query = getCreateQueries(type, suffix); - await Promise.all(query.map(async (q) => { - await driver.query(q); - })); - }); execute('should load and check driver capabilities', async () => { const capabilities = driver.capabilities(); @@ -104,13 +115,5 @@ export function testIncrementalSchemaLoading(type: string): void { data_type: expect.any(String), }); }); - - execute('should delete the data source', async () => { - await Promise.all( - tables.map(async (t) => { - await driver.dropTable(t); - }) - ); - }); }); } From e44b7859be8ad383bb0c4e72db37ff605cd1c11f Mon Sep 17 00:00:00 2001 From: Mikhail Nitsenko Date: Thu, 28 Sep 2023 17:38:23 +0600 Subject: [PATCH 08/10] upd --- packages/cubejs-databricks-jdbc-driver/src/helpers.ts | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/packages/cubejs-databricks-jdbc-driver/src/helpers.ts b/packages/cubejs-databricks-jdbc-driver/src/helpers.ts index f6f71a9eaf6fd..b864d07b4c007 100644 --- a/packages/cubejs-databricks-jdbc-driver/src/helpers.ts +++ b/packages/cubejs-databricks-jdbc-driver/src/helpers.ts @@ -33,9 +33,8 @@ export async function resolveJDBCDriver(): Promise { } export function extractUidFromJdbcUrl(jdbcUrl: string): string { - const [baseUrl, ...params] = jdbcUrl.split(';'); - const queryString = params.join('&'); - const standardUrl = `${baseUrl}?${queryString}`; - const url = new URL(standardUrl); - return url.searchParams.get('UID') || 'token'; + const { pathname } = new URL(jdbcUrl); + const [_, ...params] = pathname.split(';'); + const searchParams = new URLSearchParams(params.join('&')); + return searchParams.get('UID') || 'token'; } From 68fb2d27f53159d53e8fbd077e8fbb66d72fdf4e Mon Sep 17 00:00:00 2001 From: Mikhail Nitsenko Date: Thu, 28 Sep 2023 20:33:50 +0600 Subject: [PATCH 09/10] dev --- .../src/tests/testIncrementalSchemaLoading.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/packages/cubejs-testing-drivers/src/tests/testIncrementalSchemaLoading.ts b/packages/cubejs-testing-drivers/src/tests/testIncrementalSchemaLoading.ts index b701134caaefa..120e87d63cf81 100644 --- a/packages/cubejs-testing-drivers/src/tests/testIncrementalSchemaLoading.ts +++ b/packages/cubejs-testing-drivers/src/tests/testIncrementalSchemaLoading.ts @@ -66,6 +66,8 @@ export function testIncrementalSchemaLoading(type: string): void { await driver.dropTable(t); } console.log(`Dropping ${tables.length} fixture tables completed`); + } catch (e: any) { + console.log('Error dropping fixtures', e.stack, e); } finally { await driver.release(); await env.stop(); From 0c4c94e7bd4befd23eceb9a2bbd5aaffca0ad587 Mon Sep 17 00:00:00 2001 From: Mikhail Nitsenko Date: Thu, 28 Sep 2023 21:36:19 +0600 Subject: [PATCH 10/10] dev --- .../src/tests/testIncrementalSchemaLoading.ts | 2 -- 1 file changed, 2 deletions(-) diff --git a/packages/cubejs-testing-drivers/src/tests/testIncrementalSchemaLoading.ts b/packages/cubejs-testing-drivers/src/tests/testIncrementalSchemaLoading.ts index 120e87d63cf81..b701134caaefa 100644 --- a/packages/cubejs-testing-drivers/src/tests/testIncrementalSchemaLoading.ts +++ b/packages/cubejs-testing-drivers/src/tests/testIncrementalSchemaLoading.ts @@ -66,8 +66,6 @@ export function testIncrementalSchemaLoading(type: string): void { await driver.dropTable(t); } console.log(`Dropping ${tables.length} fixture tables completed`); - } catch (e: any) { - console.log('Error dropping fixtures', e.stack, e); } finally { await driver.release(); await env.stop();