From 4cc7a14dab9bff8ceecf87ec039555c0368d9860 Mon Sep 17 00:00:00 2001 From: Levko Kravets Date: Mon, 18 Sep 2023 11:53:09 +0300 Subject: [PATCH 01/10] Update library version Signed-off-by: Levko Kravets --- package-lock.json | 4 ++-- package.json | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/package-lock.json b/package-lock.json index 5d8169dd..d84ee13d 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "@databricks/sql", - "version": "1.4.0", + "version": "1.5.0", "lockfileVersion": 2, "requires": true, "packages": { "": { "name": "@databricks/sql", - "version": "1.4.0", + "version": "1.5.0", "hasInstallScript": true, "license": "Apache 2.0", "dependencies": { diff --git a/package.json b/package.json index 25ca6459..4253e7bf 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@databricks/sql", - "version": "1.4.0", + "version": "1.5.0", "description": "Driver for connection to Databricks SQL via Thrift API.", "main": "dist/index.js", "types": "dist/index.d.ts", From 6a3ed37dfccc8131157bbfa951d17a3a32c3ace7 Mon Sep 17 00:00:00 2001 From: Levko Kravets Date: Mon, 18 Sep 2023 11:53:54 +0300 Subject: [PATCH 02/10] Update changelog Signed-off-by: Levko Kravets --- CHANGELOG.md | 94 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 94 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index e7aa2961..8426d456 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,99 @@ # Release History +## 1.5.0 + +### Highlights + +- Added OAuth M2M support (databricks/databricks-sql-nodejs#168, databricks/databricks-sql-nodejs#177) +- Added named query parameters support (databricks/databricks-sql-nodejs#162, databricks/databricks-sql-nodejs#175) +- `runAsync` options is now deprecated (databricks/databricks-sql-nodejs#176) +- Added staging ingestion support (databricks/databricks-sql-nodejs#164) + +### Databricks OAuth support + +Databricks OAuth support added in v1.4.0 is now extended with M2M flow. To use OAuth instead of PAT, pass +a corresponding auth provider type and options to `DBSQL.connect`: + +```ts +// instantiate DBSQLClient as usual + +client.connect({ + // other mandatory options - e.g. host, path, etc. + authType: 'databricks-oauth', + oauthClientId: '...', // optional - overwrite default OAuth client ID + azureTenantId: '...', // optional - provide custom Azure tenant ID + persistence: ..., // optional; user-provided storage for OAuth tokens, should implement OAuthPersistence interface +}) +``` + +U2M flow involves user interaction - the library will open a browser tab asking user to log in. To use this flow, +no other options are required except of selecting auth provider type. + +M2M flow does not require any user interaction, and therefore may be a good option, say, for scripting. To use this +flow, two extra options are required for `DBSQLClient.connect`: `oauthClientId` and `oauthClientSecret`. + +Also see [Databricks docs](https://docs.databricks.com/en/dev-tools/auth.html#oauth-machine-to-machine-m2m-authentication) +for more details about Databricks OAuth. + +### Named query parameters + +v1.5.0 adds a support of [query parameters](https://docs.databricks.com/en/sql/language-manual/sql-ref-parameter-marker.html). +Currently only named parameters are supported. + +Basic usage example: + +```ts +// obtain session object as usual + +const operation = session.executeStatement('SELECT :p1 AS "str_param", :p2 AS "number_param"', { + namedParameters: { + p1: 'Hello, World', + p2: 3.14, + }, +}); +``` + +The library will infer parameter types from passed primitive objects. Supported data types include booleans, various +numeric types (including native `BigInt` and `Int64` from `node-int64`), native `Date` type, and string. + +It's also possible to explicitly specify parameter type by passing a `DBSQLParameter` instances instead of primitive +values. It also allows to use values that don't have a corresponding primitive representation: + +```ts +import { ..., DBSQLParameter, DBSQLParameterType } from '@databricks/sql'; + +// obtain session object as usual + +const operation = session.executeStatement('SELECT :p1 AS "date_param", :p2 AS "interval_type"', { + namedParameters: { + p1: new DBSQLParameter({ + value: new DBSQLParameter({ + value: new Date('2023-09-06T03:14:27.843Z'), + type: DBSQLParameterType.DATE, // by default, Date objects are inferred as TIMESTAMP, this allows to override the type + }), + }), + p2: new DBSQLParameter({ + value: new DBSQLParameter({ + value: 5, // INTERVAL '5' DAY + type: DBSQLParameterType.INTERVALDAY + }), + }), + }, +}); +``` + +Of course, you can mix primitive values and `DBSQLParameter` instances. + +### `runAsync` deprecation + +The `runAsync` is going to become unsupported soon, and we're deprecating it. It will remain available for the next +few releases, but from now it will be ignored and behave like it's always `true`. From user's point, the library +behaviour won't change, so if you used `runAsync` anywhere in your code - you can now just remove it. + +### Data ingestion support + +This feature allows to upload, retrieve and remove unity catalog volume files using SQL `PUT`, `GET` and `REMOVE` commands. + ## 1.4.0 - Added Cloud Fetch support (databricks/databricks-sql-nodejs#158) From fd04cde00fb7a5d2eb67010ef928220de05eecc1 Mon Sep 17 00:00:00 2001 From: Levko Kravets Date: Mon, 18 Sep 2023 11:56:14 +0300 Subject: [PATCH 03/10] Minor code fixes Signed-off-by: Levko Kravets --- lib/index.ts | 4 ++-- tests/e2e/query_parameters.test.js | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/lib/index.ts b/lib/index.ts index d39c9930..bf3b3d81 100644 --- a/lib/index.ts +++ b/lib/index.ts @@ -3,7 +3,7 @@ import TCLIService from '../thrift/TCLIService'; import TCLIService_types from '../thrift/TCLIService_types'; import DBSQLClient from './DBSQLClient'; import DBSQLSession from './DBSQLSession'; -import { DBSQLParameter } from './DBSQLParameter'; +import { DBSQLParameter, DBSQLParameterType } from './DBSQLParameter'; import DBSQLLogger from './DBSQLLogger'; import PlainHttpAuthentication from './connection/auth/PlainHttpAuthentication'; import HttpConnection from './connection/connections/HttpConnection'; @@ -32,4 +32,4 @@ export const utils = { formatProgress, }; -export { DBSQLClient, DBSQLSession, DBSQLParameter, DBSQLLogger, LogLevel }; +export { DBSQLClient, DBSQLSession, DBSQLParameter, DBSQLParameterType, DBSQLLogger, LogLevel }; diff --git a/tests/e2e/query_parameters.test.js b/tests/e2e/query_parameters.test.js index d599390c..b705419e 100644 --- a/tests/e2e/query_parameters.test.js +++ b/tests/e2e/query_parameters.test.js @@ -1,7 +1,7 @@ const { expect } = require('chai'); const Int64 = require('node-int64'); const config = require('./utils/config'); -const { DBSQLClient, DBSQLParameter } = require('../..'); +const { DBSQLClient, DBSQLParameter, DBSQLParameterType } = require('../..'); const openSession = async () => { const client = new DBSQLClient(); @@ -40,7 +40,7 @@ describe('Query parameters', () => { p_double: new DBSQLParameter({ value: 3.14 }), p_bigint_1: new DBSQLParameter({ value: BigInt(1234) }), p_bigint_2: new DBSQLParameter({ value: new Int64(1234) }), - p_date: new DBSQLParameter({ value: new Date('2023-09-06T03:14:27.843Z'), type: 'DATE' }), + p_date: new DBSQLParameter({ value: new Date('2023-09-06T03:14:27.843Z'), type: DBSQLParameterType.DATE }), p_timestamp: new DBSQLParameter({ value: new Date('2023-09-06T03:14:27.843Z') }), p_str: new DBSQLParameter({ value: 'Hello' }), }, From 5c1cf88d4ec9b6b5b67f7c2cf124a553325c3aff Mon Sep 17 00:00:00 2001 From: Levko Kravets Date: Mon, 18 Sep 2023 15:28:17 +0300 Subject: [PATCH 04/10] Update changelog Signed-off-by: Levko Kravets --- CHANGELOG.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8426d456..927746b7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,24 +12,24 @@ ### Databricks OAuth support Databricks OAuth support added in v1.4.0 is now extended with M2M flow. To use OAuth instead of PAT, pass -a corresponding auth provider type and options to `DBSQL.connect`: +a corresponding auth provider type and options to `DBSQLClient.connect`: ```ts // instantiate DBSQLClient as usual client.connect({ - // other mandatory options - e.g. host, path, etc. + // provide other mandatory options as usual - e.g. host, path, etc. authType: 'databricks-oauth', oauthClientId: '...', // optional - overwrite default OAuth client ID azureTenantId: '...', // optional - provide custom Azure tenant ID - persistence: ..., // optional; user-provided storage for OAuth tokens, should implement OAuthPersistence interface + persistence: ..., // optional - user-provided storage for OAuth tokens, should implement OAuthPersistence interface }) ``` U2M flow involves user interaction - the library will open a browser tab asking user to log in. To use this flow, -no other options are required except of selecting auth provider type. +no other options are required except of setting the auth provider type. -M2M flow does not require any user interaction, and therefore may be a good option, say, for scripting. To use this +M2M flow does not require any user interaction, and therefore is a good option, say, for scripting. To use this flow, two extra options are required for `DBSQLClient.connect`: `oauthClientId` and `oauthClientSecret`. Also see [Databricks docs](https://docs.databricks.com/en/dev-tools/auth.html#oauth-machine-to-machine-m2m-authentication) From b5abcc43ba8338c1e345f2df4888513c9b25bb50 Mon Sep 17 00:00:00 2001 From: Levko Kravets Date: Mon, 18 Sep 2023 18:49:35 +0300 Subject: [PATCH 05/10] Apply suggestions from code review Signed-off-by: Levko Kravets Co-authored-by: Jesse --- CHANGELOG.md | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 927746b7..47be8dc3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -27,7 +27,7 @@ client.connect({ ``` U2M flow involves user interaction - the library will open a browser tab asking user to log in. To use this flow, -no other options are required except of setting the auth provider type. +no other options are required except for `authType`. M2M flow does not require any user interaction, and therefore is a good option, say, for scripting. To use this flow, two extra options are required for `DBSQLClient.connect`: `oauthClientId` and `oauthClientSecret`. @@ -37,7 +37,7 @@ for more details about Databricks OAuth. ### Named query parameters -v1.5.0 adds a support of [query parameters](https://docs.databricks.com/en/sql/language-manual/sql-ref-parameter-marker.html). +v1.5.0 adds a support for [query parameters](https://docs.databricks.com/en/sql/language-manual/sql-ref-parameter-marker.html). Currently only named parameters are supported. Basic usage example: @@ -56,8 +56,8 @@ const operation = session.executeStatement('SELECT :p1 AS "str_param", :p2 AS "n The library will infer parameter types from passed primitive objects. Supported data types include booleans, various numeric types (including native `BigInt` and `Int64` from `node-int64`), native `Date` type, and string. -It's also possible to explicitly specify parameter type by passing a `DBSQLParameter` instances instead of primitive -values. It also allows to use values that don't have a corresponding primitive representation: +It's also possible to explicitly specify the parameter type by passing `DBSQLParameter` instances instead of primitive +values. It also allows one to use values that don't have a corresponding primitive representation: ```ts import { ..., DBSQLParameter, DBSQLParameterType } from '@databricks/sql'; @@ -87,12 +87,12 @@ Of course, you can mix primitive values and `DBSQLParameter` instances. ### `runAsync` deprecation The `runAsync` is going to become unsupported soon, and we're deprecating it. It will remain available for the next -few releases, but from now it will be ignored and behave like it's always `true`. From user's point, the library -behaviour won't change, so if you used `runAsync` anywhere in your code - you can now just remove it. +few releases, but from now it will be ignored and behave like it's always `true`. From user's perspective, the library +behaviour won't change. So if you used `runAsync` anywhere in your code - you can now just remove it. ### Data ingestion support -This feature allows to upload, retrieve and remove unity catalog volume files using SQL `PUT`, `GET` and `REMOVE` commands. +This feature allows you to upload, retrieve, and remove unity catalog volume files using SQL `PUT`, `GET` and `REMOVE` commands. ## 1.4.0 From 251becd591a4ef6e57689daea51303e646de501b Mon Sep 17 00:00:00 2001 From: Levko Kravets Date: Mon, 18 Sep 2023 18:53:16 +0300 Subject: [PATCH 06/10] Fix example in changelog Signed-off-by: Levko Kravets --- CHANGELOG.md | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 47be8dc3..fb11a1da 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -67,16 +67,12 @@ import { ..., DBSQLParameter, DBSQLParameterType } from '@databricks/sql'; const operation = session.executeStatement('SELECT :p1 AS "date_param", :p2 AS "interval_type"', { namedParameters: { p1: new DBSQLParameter({ - value: new DBSQLParameter({ - value: new Date('2023-09-06T03:14:27.843Z'), - type: DBSQLParameterType.DATE, // by default, Date objects are inferred as TIMESTAMP, this allows to override the type - }), + value: new Date('2023-09-06T03:14:27.843Z'), + type: DBSQLParameterType.DATE, // by default, Date objects are inferred as TIMESTAMP, this allows to override the type }), p2: new DBSQLParameter({ - value: new DBSQLParameter({ - value: 5, // INTERVAL '5' DAY - type: DBSQLParameterType.INTERVALDAY - }), + value: 5, // INTERVAL '5' DAY + type: DBSQLParameterType.INTERVALDAY }), }, }); From a3426d1f5730792eaee774822404602cb2138ac2 Mon Sep 17 00:00:00 2001 From: Levko Kravets Date: Wed, 20 Sep 2023 13:18:18 +0300 Subject: [PATCH 07/10] Update changelog Signed-off-by: Levko Kravets --- CHANGELOG.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index fb11a1da..93d538e2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -82,9 +82,9 @@ Of course, you can mix primitive values and `DBSQLParameter` instances. ### `runAsync` deprecation -The `runAsync` is going to become unsupported soon, and we're deprecating it. It will remain available for the next -few releases, but from now it will be ignored and behave like it's always `true`. From user's perspective, the library -behaviour won't change. So if you used `runAsync` anywhere in your code - you can now just remove it. +Since this release, the library will execute all queries asynchronously, so we deprecate `runAsync` option. +It will be completely removed in v2, but we encourage to stop using it since now and remove all the usages +from your code. From user's perspective, the library behaviour won't change. ### Data ingestion support From a507a8cbd66e88056ddc0dd2061dcdff38cb001d Mon Sep 17 00:00:00 2001 From: Levko Kravets Date: Fri, 22 Sep 2023 19:16:36 +0300 Subject: [PATCH 08/10] Apply suggestions from code review Signed-off-by: Levko Kravets Co-authored-by: Jesse --- CHANGELOG.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 93d538e2..fe127cd2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -82,9 +82,9 @@ Of course, you can mix primitive values and `DBSQLParameter` instances. ### `runAsync` deprecation -Since this release, the library will execute all queries asynchronously, so we deprecate `runAsync` option. -It will be completely removed in v2, but we encourage to stop using it since now and remove all the usages -from your code. From user's perspective, the library behaviour won't change. +Starting with this release, the library will execute all queries asynchronously, so we have deprecated the `runAsync` option. +It will be completely removed in v2. So you should not use it going forward and remove all the usages +from your code before version 2 is released. From user's perspective the library behaviour won't change. ### Data ingestion support From e2656687ae2fe068e400c878c3c8969728653d11 Mon Sep 17 00:00:00 2001 From: Levko Kravets Date: Fri, 22 Sep 2023 19:18:25 +0300 Subject: [PATCH 09/10] Code style Signed-off-by: Levko Kravets --- CHANGELOG.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index fe127cd2..5967770b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -82,9 +82,9 @@ Of course, you can mix primitive values and `DBSQLParameter` instances. ### `runAsync` deprecation -Starting with this release, the library will execute all queries asynchronously, so we have deprecated the `runAsync` option. -It will be completely removed in v2. So you should not use it going forward and remove all the usages -from your code before version 2 is released. From user's perspective the library behaviour won't change. +Starting with this release, the library will execute all queries asynchronously, so we have deprecated +the `runAsync` option. It will be completely removed in v2. So you should not use it going forward and remove all +the usages from your code before version 2 is released. From user's perspective the library behaviour won't change. ### Data ingestion support From 3faa0898da360b17967dfa519f37afa351c31078 Mon Sep 17 00:00:00 2001 From: Levko Kravets Date: Fri, 22 Sep 2023 19:23:24 +0300 Subject: [PATCH 10/10] Temporarily disable tests until we fix e2e env Signed-off-by: Levko Kravets --- tests/e2e/query_parameters.test.js | 3 ++- tests/e2e/staging_ingestion.test.js | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/e2e/query_parameters.test.js b/tests/e2e/query_parameters.test.js index b705419e..48df1ef8 100644 --- a/tests/e2e/query_parameters.test.js +++ b/tests/e2e/query_parameters.test.js @@ -18,7 +18,8 @@ const openSession = async () => { }); }; -describe('Query parameters', () => { +// TODO: Temporarily disable those tests until we figure out issues with E2E test env +describe.skip('Query parameters', () => { it('should use named parameters', async () => { const session = await openSession(); const operation = await session.executeStatement( diff --git a/tests/e2e/staging_ingestion.test.js b/tests/e2e/staging_ingestion.test.js index 9459687a..77bd82ca 100644 --- a/tests/e2e/staging_ingestion.test.js +++ b/tests/e2e/staging_ingestion.test.js @@ -3,7 +3,7 @@ const config = require('./utils/config'); const { DBSQLClient } = require('../..'); const fs = require('fs'); -// TODO Temporarily disable those tests until we figure out issues with E2E test env +// TODO: Temporarily disable those tests until we figure out issues with E2E test env describe.skip('Staging Test', () => { it('put staging data and receive it', async () => { const client = new DBSQLClient();