From 53f5f227e6ddbb79558692662cced046b7098f83 Mon Sep 17 00:00:00 2001 From: ramalingamt <182550672+ramalingamt@users.noreply.github.com> Date: Tue, 24 Sep 2024 15:56:44 +0400 Subject: [PATCH 1/2] Added support for installing and loading DuckDB Extensions. --- .../configuration/data-sources/duckdb.mdx | 1 + .../configuration/environment-variables.mdx | 8 +++++ packages/cubejs-backend-shared/src/env.ts | 14 +++++++++ .../cubejs-duckdb-driver/src/DuckDBDriver.ts | 30 +++++++++++++++++++ 4 files changed, 53 insertions(+) diff --git a/docs/pages/product/configuration/data-sources/duckdb.mdx b/docs/pages/product/configuration/data-sources/duckdb.mdx index 2f363e61c770f..1478383721718 100644 --- a/docs/pages/product/configuration/data-sources/duckdb.mdx +++ b/docs/pages/product/configuration/data-sources/duckdb.mdx @@ -73,6 +73,7 @@ deployment][ref-demo-deployment] in Cube Cloud. | `CUBEJS_DB_DUCKDB_S3_USE_SSL` | Use SSL for connection | A boolean | ❌ | ❌ | | `CUBEJS_DB_DUCKDB_S3_URL_STYLE` | To choose the S3 URL style(vhost or path) | 'vhost' or 'path' | ❌ | ❌ | | `CUBEJS_DB_DUCKDB_S3_SESSION_TOKEN` | The token for the S3 session | A valid Session Token | ❌ | ✅ | +| `CUBEJS_DB_DUCKDB_EXTENSIONS` | A comma-separated list of DuckDB extensions to install and load | A comma-separated list of DuckDB extensions | ❌ | ✅ | ## Pre-Aggregation Feature Support diff --git a/docs/pages/reference/configuration/environment-variables.mdx b/docs/pages/reference/configuration/environment-variables.mdx index da0b9c3266a74..c2b4876020f89 100644 --- a/docs/pages/reference/configuration/environment-variables.mdx +++ b/docs/pages/reference/configuration/environment-variables.mdx @@ -315,6 +315,14 @@ The S3 session token. | -------------------------------------- | ---------------------- | --------------------- | | A valid S3 session token | N/A | N/A | +## `CUBEJS_DB_DUCKDB_EXTENSIONS` + +A comma-separated list of DuckDB extensions to install and load. + +| Possible Values | Default in Development | Default in Production | +| ------------------------------------------- | ---------------------- | --------------------- | +| A comma-separated list of DuckDB extensions | N/A | N/A | + ## `CUBEJS_DB_ELASTIC_APIKEY_ID` The [ID of the API key from elastic.co][elastic-docs-api-keys]. Required when diff --git a/packages/cubejs-backend-shared/src/env.ts b/packages/cubejs-backend-shared/src/env.ts index 3b5f7d9bba259..d6dd298c4b48f 100644 --- a/packages/cubejs-backend-shared/src/env.ts +++ b/packages/cubejs-backend-shared/src/env.ts @@ -1535,6 +1535,20 @@ const variables: Record any> = { ] ), + duckdbExtensions: ({ + dataSource + }: { + dataSource: string, + }) => { + const extensions = process.env[ + keyByDataSource('CUBEJS_DB_DUCKDB_EXTENSIONS', dataSource) + ] + if (extensions) { + return extensions.split(',').map(e => e.trim()); + } + return []; + }, + /** * Presto catalog. */ diff --git a/packages/cubejs-duckdb-driver/src/DuckDBDriver.ts b/packages/cubejs-duckdb-driver/src/DuckDBDriver.ts index 9eeba91b27ac1..18d95bd563e13 100644 --- a/packages/cubejs-duckdb-driver/src/DuckDBDriver.ts +++ b/packages/cubejs-duckdb-driver/src/DuckDBDriver.ts @@ -160,6 +160,36 @@ export class DuckDBDriver extends BaseDriver implements DriverInterface { } } + // Install & load extensions if configured in env variable. + const extensions = getEnv('duckdbExtensions', this.config); + for (const extension of extensions) { + try { + await execAsync('INSTALL ' + extension); + } catch (e) { + if (this.logger) { + console.error('DuckDB - error on installing ' + extension, { + e + }); + } + + // DuckDB will lose connection_ref on connection on error, this will lead to broken connection object + throw e; + } + + try { + await execAsync('LOAD ' + extension); + } catch (e) { + if (this.logger) { + console.error('DuckDB - error on loading ' + extension, { + e + }); + } + + // DuckDB will lose connection_ref on connection on error, this will lead to broken connection object + throw e; + } + } + if (this.config.initSql) { try { await execAsync(this.config.initSql); From 352df730190c020a35475973b639c7ded892c6af Mon Sep 17 00:00:00 2001 From: ramalingamt Date: Tue, 17 Dec 2024 12:09:34 +0400 Subject: [PATCH 2/2] Lint fixes - checked-in. --- packages/cubejs-backend-shared/src/env.ts | 4 ++-- packages/cubejs-duckdb-driver/src/DuckDBDriver.ts | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/packages/cubejs-backend-shared/src/env.ts b/packages/cubejs-backend-shared/src/env.ts index efe35eda55caa..3d5a2c0e2e81c 100644 --- a/packages/cubejs-backend-shared/src/env.ts +++ b/packages/cubejs-backend-shared/src/env.ts @@ -1613,14 +1613,14 @@ const variables: Record any> = { ] ), -duckdbExtensions: ({ + duckdbExtensions: ({ dataSource }: { dataSource: string, }) => { const extensions = process.env[ keyByDataSource('CUBEJS_DB_DUCKDB_EXTENSIONS', dataSource) - ] + ]; if (extensions) { return extensions.split(',').map(e => e.trim()); } diff --git a/packages/cubejs-duckdb-driver/src/DuckDBDriver.ts b/packages/cubejs-duckdb-driver/src/DuckDBDriver.ts index ba18706da9fa3..8fd887f29c9e4 100644 --- a/packages/cubejs-duckdb-driver/src/DuckDBDriver.ts +++ b/packages/cubejs-duckdb-driver/src/DuckDBDriver.ts @@ -138,10 +138,10 @@ export class DuckDBDriver extends BaseDriver implements DriverInterface { const extensions = getEnv('duckdbExtensions', this.config); for (const extension of extensions) { try { - await execAsync('INSTALL ' + extension); + await execAsync(`INSTALL ${extension}`); } catch (e) { if (this.logger) { - console.error('DuckDB - error on installing ' + extension, { + console.error(`DuckDB - error on installing ${extension}`, { e }); } @@ -151,10 +151,10 @@ export class DuckDBDriver extends BaseDriver implements DriverInterface { } try { - await execAsync('LOAD ' + extension); + await execAsync(`LOAD ${extension}`); } catch (e) { if (this.logger) { - console.error('DuckDB - error on loading ' + extension, { + console.error(`DuckDB - error on loading ${extension}`, { e }); }