From d934f61c3025f5f95b2a281b028bd5226316394c Mon Sep 17 00:00:00 2001 From: Igor Lukanin Date: Tue, 11 Feb 2025 14:55:27 +0100 Subject: [PATCH 01/10] feat(pinot-driver): Add enableNullHandling=true to query options --- docs/pages/product/configuration/data-sources/pinot.mdx | 6 ++++-- packages/cubejs-pinot-driver/src/PinotDriver.ts | 2 +- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/docs/pages/product/configuration/data-sources/pinot.mdx b/docs/pages/product/configuration/data-sources/pinot.mdx index 4d888e776f372..552345201b2fe 100644 --- a/docs/pages/product/configuration/data-sources/pinot.mdx +++ b/docs/pages/product/configuration/data-sources/pinot.mdx @@ -14,8 +14,9 @@ workloads. [StarTree][link-startree] is a fully-managed platform for Pinot. - The hostname for the [Pinot][pinot] broker - The port for the [Pinot][pinot] broker -With the current implementation of the Pinot driver, you have to enable the -[multi-stage query engine][link-pinot-msqe] in your Pinot cluster. +Note that the following features should be enabled in your Pinot cluster: +- [Multi-stage query engine][link-pinot-msqe]. +- [Advanced null value support][link-pinot-nvs]. ## Setup @@ -97,6 +98,7 @@ Cube does not require any additional configuration to enable SSL as Pinot connec [link-pinot]: https://pinot.apache.org/ [pinot]: https://docs.pinot.apache.org/ [link-pinot-msqe]: https://docs.pinot.apache.org/reference/multi-stage-engine +[link-pinot-nvs]: https://docs.pinot.apache.org/developers/advanced/null-value-support#advanced-null-handling-support [pinot-docs-approx-agg-fns]: https://docs.pinot.apache.org/users/user-guide-query/query-syntax/how-to-handle-unique-counting [ref-recipe-enable-ssl]: diff --git a/packages/cubejs-pinot-driver/src/PinotDriver.ts b/packages/cubejs-pinot-driver/src/PinotDriver.ts index be02426001acf..2cf0dbb1f8646 100644 --- a/packages/cubejs-pinot-driver/src/PinotDriver.ts +++ b/packages/cubejs-pinot-driver/src/PinotDriver.ts @@ -167,7 +167,7 @@ export class PinotDriver extends BaseDriver implements DriverInterface { }), body: JSON.stringify({ sql: query, - queryOptions: `useMultistageEngine=true;timeoutMs=${this.config.queryTimeout}` + queryOptions: `useMultistageEngine=true;enableNullHandling=true;timeoutMs=${this.config.queryTimeout}` }) }); From 0d66050c48953b5ccf372f1e97fef95fad9ae308 Mon Sep 17 00:00:00 2001 From: sergisulca Date: Tue, 11 Feb 2025 17:39:40 -0300 Subject: [PATCH 02/10] Add enableNullHandling=true to query options using env var CUBEJS_DB_NULL_HANDLING --- packages/cubejs-backend-shared/src/env.ts | 29 +++++++++++++++++++ .../cubejs-pinot-driver/src/PinotDriver.ts | 4 ++- 2 files changed, 32 insertions(+), 1 deletion(-) diff --git a/packages/cubejs-backend-shared/src/env.ts b/packages/cubejs-backend-shared/src/env.ts index 2f8760f412c35..094a166fee6f5 100644 --- a/packages/cubejs-backend-shared/src/env.ts +++ b/packages/cubejs-backend-shared/src/env.ts @@ -1780,6 +1780,35 @@ const variables: Record any> = { ] ), + /** + * Pinot / Startree Null value support + */ + + pinotNullHandling: ({ dataSource }: { dataSource: string }) => { + const val = process.env[ + keyByDataSource('CUBEJS_DB_NULL_HANDLING', dataSource) + ]; + + if (val) { + if (val.toLocaleLowerCase() === 'true') { + return true; + } else if (val.toLowerCase() === 'false') { + return false; + } else { + throw new TypeError( + `The ${ + keyByDataSource( + 'CUBEJS_DB_NULL_HANDLING', + dataSource, + ) + } must be either 'true' or 'false'.` + ); + } + } else { + return false; + } + }, + /** **************************************************************** * Dremio Driver * ***************************************************************** */ diff --git a/packages/cubejs-pinot-driver/src/PinotDriver.ts b/packages/cubejs-pinot-driver/src/PinotDriver.ts index 2cf0dbb1f8646..fa7ed2e3d7c9b 100644 --- a/packages/cubejs-pinot-driver/src/PinotDriver.ts +++ b/packages/cubejs-pinot-driver/src/PinotDriver.ts @@ -33,6 +33,7 @@ export type PinotDriverConfiguration = { ssl?: string | TLSConnectionOptions; dataSource?: string; queryTimeout?: number; + nullHandling?: boolean; }; type AuthorizationHeaders = { @@ -108,6 +109,7 @@ export class PinotDriver extends BaseDriver implements DriverInterface { : undefined, authToken: getEnv('pinotAuthToken', { dataSource }), ssl: this.getSslOptions(dataSource), + nullHandling: getEnv('pinotNullHandling', { dataSource }), queryTimeout: getEnv('dbQueryTimeout', { dataSource }), ...config }; @@ -167,7 +169,7 @@ export class PinotDriver extends BaseDriver implements DriverInterface { }), body: JSON.stringify({ sql: query, - queryOptions: `useMultistageEngine=true;enableNullHandling=true;timeoutMs=${this.config.queryTimeout}` + queryOptions: `useMultistageEngine=true;enableNullHandling=${this.config.nullHandling};timeoutMs=${this.config.queryTimeout}` }) }); From a07831c42a0a962031390859aa7321fa4ccdda55 Mon Sep 17 00:00:00 2001 From: sergisulca Date: Tue, 11 Feb 2025 18:03:30 -0300 Subject: [PATCH 03/10] Add 'The Startree/Pinot null value support' to docs --- .../configuration/data-sources/pinot.mdx | 17 +++++++++-------- .../configuration/environment-variables.mdx | 9 +++++++++ 2 files changed, 18 insertions(+), 8 deletions(-) diff --git a/docs/pages/product/configuration/data-sources/pinot.mdx b/docs/pages/product/configuration/data-sources/pinot.mdx index 552345201b2fe..c9359f9e22a2d 100644 --- a/docs/pages/product/configuration/data-sources/pinot.mdx +++ b/docs/pages/product/configuration/data-sources/pinot.mdx @@ -42,14 +42,15 @@ CUBEJS_DB_PASS=********** ## Environment Variables -| Environment Variable | Description | Possible Values | Required | -|----------------------|--------------------------------------------|---------------------|:--------:| -| `CUBEJS_DB_HOST` | The host URL for your Pinot broker | A valid host URL | ✅ | -| `CUBEJS_DB_PORT` | The port for the database connection | A valid port number | ✅ | -| `CUBEJS_DB_USER` | The username used to connect to the broker | A valid username | ❌ | -| `CUBEJS_DB_PASS` | The password used to connect to the broker | A valid password | ❌ | -| `CUBEJS_DB_NAME` | The database name for StarTree | A valid name | ❌ | -| `CUBEJS_DB_PINOT_AUTH_TOKEN` | The authentication token for StarTree | A valid token | ❌ | +| Environment Variable | Description | Possible Values | Required | +|-------------------------------|-------------------------------------------------------|---------------------|:--------:| +| `CUBEJS_DB_HOST` | The host URL for your Pinot broker | A valid host URL | ✅ | +| `CUBEJS_DB_PORT` | The port for the database connection | A valid port number | ✅ | +| `CUBEJS_DB_USER` | The username used to connect to the broker | A valid username | ❌ | +| `CUBEJS_DB_PASS` | The password used to connect to the broker | A valid password | ❌ | +| `CUBEJS_DB_NAME` | The database name for StarTree | A valid name | ❌ | +| `CUBEJS_DB_NULL_HANDLING` | If `true`, enables null handling. Default is `false` | `true`, `false` | ❌ | +| `CUBEJS_DB_PINOT_AUTH_TOKEN` | The authentication token for StarTree | A valid token | ❌ | ## Pre-Aggregation Feature Support diff --git a/docs/pages/reference/configuration/environment-variables.mdx b/docs/pages/reference/configuration/environment-variables.mdx index f227eb192cba5..99ed72c2e5a99 100644 --- a/docs/pages/reference/configuration/environment-variables.mdx +++ b/docs/pages/reference/configuration/environment-variables.mdx @@ -740,6 +740,15 @@ The Snowflake warehouse to use when connecting to the database. | ---------------------------------------------------------------------- | ---------------------- | --------------------- | | [A valid Snowflake warehouse][snowflake-docs-warehouse] in the account | N/A | N/A | +## `CUBEJS_DB_NULL_HANDLING` + +The Startree/Pinot null value support. If `true`, enables null handling. + +| Possible Values | Default in Development | Default in Production | +| --------------- | ---------------------- | --------------------- | +| `true`, `false` | `false` | `false` | + + ## `CUBEJS_DB_SSL` If `true`, enables SSL encryption for database connections from Cube. From 3f9b6cd7921e401d2c2e59762cfccaf83df046b2 Mon Sep 17 00:00:00 2001 From: sergisulca Date: Tue, 11 Feb 2025 18:51:36 -0300 Subject: [PATCH 04/10] fix reference env var --- docs/pages/reference/configuration/environment-variables.mdx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/pages/reference/configuration/environment-variables.mdx b/docs/pages/reference/configuration/environment-variables.mdx index 99ed72c2e5a99..b1a3984fdd891 100644 --- a/docs/pages/reference/configuration/environment-variables.mdx +++ b/docs/pages/reference/configuration/environment-variables.mdx @@ -746,7 +746,7 @@ The Startree/Pinot null value support. If `true`, enables null handling. | Possible Values | Default in Development | Default in Production | | --------------- | ---------------------- | --------------------- | -| `true`, `false` | `false` | `false` | +| `true`, `false` | N/A | N/A | ## `CUBEJS_DB_SSL` From a84ce2c71e2b43d98ed82ec6c3c5524225d0ccfb Mon Sep 17 00:00:00 2001 From: Sergio Sulca <104989816+sergisulca@users.noreply.github.com> Date: Wed, 12 Feb 2025 09:14:22 -0300 Subject: [PATCH 05/10] Update docs/pages/product/configuration/data-sources/pinot.mdx Co-authored-by: Konstantin Burkalev --- docs/pages/product/configuration/data-sources/pinot.mdx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/pages/product/configuration/data-sources/pinot.mdx b/docs/pages/product/configuration/data-sources/pinot.mdx index c9359f9e22a2d..a736ff8e5d1e3 100644 --- a/docs/pages/product/configuration/data-sources/pinot.mdx +++ b/docs/pages/product/configuration/data-sources/pinot.mdx @@ -49,7 +49,7 @@ CUBEJS_DB_PASS=********** | `CUBEJS_DB_USER` | The username used to connect to the broker | A valid username | ❌ | | `CUBEJS_DB_PASS` | The password used to connect to the broker | A valid password | ❌ | | `CUBEJS_DB_NAME` | The database name for StarTree | A valid name | ❌ | -| `CUBEJS_DB_NULL_HANDLING` | If `true`, enables null handling. Default is `false` | `true`, `false` | ❌ | +| `CUBEJS_DB_PINOT_NULL_HANDLING` | If `true`, enables null handling. Default is `false` | `true`, `false` | ❌ | | `CUBEJS_DB_PINOT_AUTH_TOKEN` | The authentication token for StarTree | A valid token | ❌ | ## Pre-Aggregation Feature Support From 372dfd25d7e7e08486327e7db77d1e6303b7471c Mon Sep 17 00:00:00 2001 From: Sergio Sulca <104989816+sergisulca@users.noreply.github.com> Date: Wed, 12 Feb 2025 09:14:28 -0300 Subject: [PATCH 06/10] Update docs/pages/reference/configuration/environment-variables.mdx Co-authored-by: ron-damon <48366185+ron-damon@users.noreply.github.com> --- docs/pages/reference/configuration/environment-variables.mdx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/pages/reference/configuration/environment-variables.mdx b/docs/pages/reference/configuration/environment-variables.mdx index b1a3984fdd891..8be716abbfc3a 100644 --- a/docs/pages/reference/configuration/environment-variables.mdx +++ b/docs/pages/reference/configuration/environment-variables.mdx @@ -746,7 +746,7 @@ The Startree/Pinot null value support. If `true`, enables null handling. | Possible Values | Default in Development | Default in Production | | --------------- | ---------------------- | --------------------- | -| `true`, `false` | N/A | N/A | +| `true`, `false` | `false` | `false` | ## `CUBEJS_DB_SSL` From 5d67e25bfcc59ffb612a737a82f6252267aae219 Mon Sep 17 00:00:00 2001 From: Sergio Sulca <104989816+sergisulca@users.noreply.github.com> Date: Wed, 12 Feb 2025 09:14:35 -0300 Subject: [PATCH 07/10] Update docs/pages/reference/configuration/environment-variables.mdx Co-authored-by: ron-damon <48366185+ron-damon@users.noreply.github.com> --- docs/pages/reference/configuration/environment-variables.mdx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/pages/reference/configuration/environment-variables.mdx b/docs/pages/reference/configuration/environment-variables.mdx index 8be716abbfc3a..ce0c922f12570 100644 --- a/docs/pages/reference/configuration/environment-variables.mdx +++ b/docs/pages/reference/configuration/environment-variables.mdx @@ -740,7 +740,7 @@ The Snowflake warehouse to use when connecting to the database. | ---------------------------------------------------------------------- | ---------------------- | --------------------- | | [A valid Snowflake warehouse][snowflake-docs-warehouse] in the account | N/A | N/A | -## `CUBEJS_DB_NULL_HANDLING` +## `CUBEJS_DB_PINOT_NULL_HANDLING` The Startree/Pinot null value support. If `true`, enables null handling. From 27a7bc58c99dd01c43f47f114229289f07b414b8 Mon Sep 17 00:00:00 2001 From: Sergio Sulca <104989816+sergisulca@users.noreply.github.com> Date: Wed, 12 Feb 2025 09:14:42 -0300 Subject: [PATCH 08/10] Update packages/cubejs-backend-shared/src/env.ts Co-authored-by: Konstantin Burkalev --- packages/cubejs-backend-shared/src/env.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/cubejs-backend-shared/src/env.ts b/packages/cubejs-backend-shared/src/env.ts index 094a166fee6f5..6ce626422567a 100644 --- a/packages/cubejs-backend-shared/src/env.ts +++ b/packages/cubejs-backend-shared/src/env.ts @@ -1786,7 +1786,7 @@ const variables: Record any> = { pinotNullHandling: ({ dataSource }: { dataSource: string }) => { const val = process.env[ - keyByDataSource('CUBEJS_DB_NULL_HANDLING', dataSource) + keyByDataSource('CUBEJS_DB_PINOT_NULL_HANDLING', dataSource) ]; if (val) { From 54f0f3e934b9b7ad31e9e3c77503f15d1bdefc56 Mon Sep 17 00:00:00 2001 From: sergisulca Date: Wed, 12 Feb 2025 09:17:12 -0300 Subject: [PATCH 09/10] fix env var name --- .../configuration/data-sources/pinot.mdx | 18 +++++++++--------- packages/cubejs-backend-shared/src/env.ts | 2 +- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/docs/pages/product/configuration/data-sources/pinot.mdx b/docs/pages/product/configuration/data-sources/pinot.mdx index a736ff8e5d1e3..6b8eb052cf488 100644 --- a/docs/pages/product/configuration/data-sources/pinot.mdx +++ b/docs/pages/product/configuration/data-sources/pinot.mdx @@ -42,15 +42,15 @@ CUBEJS_DB_PASS=********** ## Environment Variables -| Environment Variable | Description | Possible Values | Required | -|-------------------------------|-------------------------------------------------------|---------------------|:--------:| -| `CUBEJS_DB_HOST` | The host URL for your Pinot broker | A valid host URL | ✅ | -| `CUBEJS_DB_PORT` | The port for the database connection | A valid port number | ✅ | -| `CUBEJS_DB_USER` | The username used to connect to the broker | A valid username | ❌ | -| `CUBEJS_DB_PASS` | The password used to connect to the broker | A valid password | ❌ | -| `CUBEJS_DB_NAME` | The database name for StarTree | A valid name | ❌ | -| `CUBEJS_DB_PINOT_NULL_HANDLING` | If `true`, enables null handling. Default is `false` | `true`, `false` | ❌ | -| `CUBEJS_DB_PINOT_AUTH_TOKEN` | The authentication token for StarTree | A valid token | ❌ | +| Environment Variable | Description | Possible Values | Required | +|---------------------------------|-------------------------------------------------------|---------------------|:--------:| +| `CUBEJS_DB_HOST` | The host URL for your Pinot broker | A valid host URL | ✅ | +| `CUBEJS_DB_PORT` | The port for the database connection | A valid port number | ✅ | +| `CUBEJS_DB_USER` | The username used to connect to the broker | A valid username | ❌ | +| `CUBEJS_DB_PASS` | The password used to connect to the broker | A valid password | ❌ | +| `CUBEJS_DB_NAME` | The database name for StarTree | A valid name | ❌ | +| `CUBEJS_DB_PINOT_NULL_HANDLING` | If `true`, enables null handling. Default is `false` | `true`, `false` | ❌ | +| `CUBEJS_DB_PINOT_AUTH_TOKEN` | The authentication token for StarTree | A valid token | ❌ | ## Pre-Aggregation Feature Support diff --git a/packages/cubejs-backend-shared/src/env.ts b/packages/cubejs-backend-shared/src/env.ts index 6ce626422567a..3634190f6d5f4 100644 --- a/packages/cubejs-backend-shared/src/env.ts +++ b/packages/cubejs-backend-shared/src/env.ts @@ -1798,7 +1798,7 @@ const variables: Record any> = { throw new TypeError( `The ${ keyByDataSource( - 'CUBEJS_DB_NULL_HANDLING', + 'CUBEJS_DB_PINOT_NULL_HANDLING', dataSource, ) } must be either 'true' or 'false'.` From 29c1dfbafe1631ef1f6978350733dbfb5a25a635 Mon Sep 17 00:00:00 2001 From: marianore-muttdata Date: Thu, 6 Mar 2025 14:12:56 -0300 Subject: [PATCH 10/10] Add docs for auth token config --- .../reference/configuration/environment-variables.mdx | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/docs/pages/reference/configuration/environment-variables.mdx b/docs/pages/reference/configuration/environment-variables.mdx index ce0c922f12570..114ee641cfde2 100644 --- a/docs/pages/reference/configuration/environment-variables.mdx +++ b/docs/pages/reference/configuration/environment-variables.mdx @@ -748,6 +748,13 @@ The Startree/Pinot null value support. If `true`, enables null handling. | --------------- | ---------------------- | --------------------- | | `true`, `false` | `false` | `false` | +## `CUBEJS_DB_PINOT_AUTH_TOKEN` + +The authentication token for StarTree to be passed as HTTP headers. + +| Possible Values | Default in Development | Default in Production | +| ----------------------------------- | ---------------------- | --------------------- | +| A valid string containing the token | N/A | N/A | ## `CUBEJS_DB_SSL`