feat(materialize-driver): Add cluster option to Materialize driver (#…

…7773) Thanks @bobbyiliev!

* Add cluster option to Materialize driver

* Fix lint error

* Set connection application_name

* Update materialized docker image

* Disable SSL for smoke tests

* Change the name of the MZ cluster env var

* Add link to Materialize Slack

* Fix failing tests due to SSL error

docs/pages/product/configuration/data-sources/materialize.mdx

@@ -29,19 +29,21 @@ CUBEJS_DB_PORT=6875
 CUBEJS_DB_NAME=materialize
 CUBEJS_DB_USER=materialize
 CUBEJS_DB_PASS=materialize
+CUBEJS_DB_MATERIALIZE_CLUSTER=quickstart
 ```
 
 ## Environment Variables
 
-| Environment Variable | Description                                                                         | Possible Values           | Required |
-| -------------------- | ----------------------------------------------------------------------------------- | ------------------------- | :------: |
-| `CUBEJS_DB_HOST`     | The host URL for a database                                                         | A valid database host URL |    ✅    |
-| `CUBEJS_DB_PORT`     | The port for the database connection                                                | A valid port number       |    ✅    |
-| `CUBEJS_DB_NAME`     | The name of the database to connect to                                              | A valid database name     |    ✅    |
-| `CUBEJS_DB_USER`     | The username used to connect to the database                                        | A valid database username |    ✅    |
-| `CUBEJS_DB_PASS`     | The password used to connect to the database                                        | A valid database password |    ✅    |
-| `CUBEJS_CONCURRENCY` | The number of concurrent connections each queue has to the database. Default is `2` | A valid number            |    ❌    |
-| `CUBEJS_DB_MAX_POOL` | The maximum number of concurrent database connections to pool. Default is `8`       | A valid number            |    ❌    |
+| Environment Variable             | Description                                                                         | Possible Values           | Required |
+| -------------------------------- | ----------------------------------------------------------------------------------- | ------------------------- | :------: |
+| `CUBEJS_DB_HOST`                 | The host URL for a database                                                         | A valid database host URL |    ✅    |
+| `CUBEJS_DB_PORT`                 | The port for the database connection                                                | A valid port number       |    ✅    |
+| `CUBEJS_DB_NAME`                 | The name of the database to connect to                                              | A valid database name     |    ✅    |
+| `CUBEJS_DB_USER`                 | The username used to connect to the database                                        | A valid database username |    ✅    |
+| `CUBEJS_DB_PASS`                 | The password used to connect to the database                                        | A valid database password |    ✅    |
+| `CUBEJS_CONCURRENCY`             | The number of concurrent connections each queue has to the database. Default is `2` | A valid number            |    ❌    |
+| `CUBEJS_DB_MAX_POOL`             | The maximum number of concurrent database connections to pool. Default is `8`       | A valid number            |    ❌    |
+| `CUBEJS_DB_MATERIALIZE_CLUSTER`  | The name of the Materialize cluster to connect to                                   | A valid cluster name      |    ❌    |
 
 ## SSL
 

packages/cubejs-backend-shared/src/env.ts

@@ -1231,6 +1231,23 @@ const variables: Record<string, (...args: any) => any> = {
     ]
   ),
 
+  /** ****************************************************************
+   * Materialize Driver                                              *
+   ***************************************************************** */
+
+  /**
+   * Materialize cluster.
+   */
+  materializeCluster: ({
+    dataSource
+  }: {
+    dataSource: string,
+  }) => (
+    process.env[
+      keyByDataSource('CUBEJS_DB_MATERIALIZE_CLUSTER', dataSource)
+    ]
+  ),
+
   /** ****************************************************************
    * Snowflake Driver                                                *
    ***************************************************************** */

packages/cubejs-materialize-driver/README.md

@@ -11,7 +11,7 @@ Pure Javascript Materialize driver. Based on pg driver for PostgreSQL.
 
 ## Support
 
-This driver has been contributed by [Joaquin Colacci](https://github.com/joacoc). This package is **community supported** and should be used at your own risk. 
+This driver has been contributed by [Joaquin Colacci](https://github.com/joacoc) and is currently maintained by [Materialize](https://www.materialize.com/s/chat). This package is **community supported** and should be used at your own risk.
 
 While the Cube Dev team is happy to review and accept future community contributions, we don't have active plans for further development. This includes bug fixes unless they affect different parts of Cube.js. **We're looking for maintainers for this package.** If you'd like to become a maintainer, please contact us in Cube.js Slack. 
 

packages/cubejs-materialize-driver/src/MaterializeDriver.ts

@@ -59,8 +59,35 @@ export class MaterializeDriver extends PostgresDriver {
        * request before determining it as not valid. Default - 10000 ms.
        */
       testConnectionTimeout?: number,
+
+      /**
+       * Optional cluster name to set for the connection.
+       */
+      cluster?: string,
+
+      /**
+       * SSL is enabled by default. Set to false to disable.
+       */
+      ssl?: boolean | { rejectUnauthorized: boolean },
+
+      /**
+       * Application name to set for the connection.
+       */
+      application_name?: string,
     } = {},
   ) {
+    // Enable SSL by default if not set explicitly to false
+    const sslEnv = process.env.CUBEJS_DB_SSL;
+    if (sslEnv === 'false') {
+      options.ssl = false;
+    } else if (sslEnv === 'true') {
+      options.ssl = { rejectUnauthorized: true };
+    } else if (options.ssl === undefined) {
+      options.ssl = true;
+    }
+    // Set application name to 'cubejs-materialize-driver' by default
+    options.application_name = options.application_name || 'cubejs-materialize-driver';
+
     super(options);
   }
 
@@ -69,6 +96,11 @@ export class MaterializeDriver extends PostgresDriver {
   ) {
     await conn.query(`SET TIME ZONE '${this.config.storeTimezone || 'UTC'}'`);
     // Support for statement_timeout is still pending. https://github.com/MaterializeInc/materialize/issues/10390
+
+    // Set cluster to the CUBEJS_DB_MATERIALIZE_CLUSTER env variable if it exists
+    if (process.env.CUBEJS_DB_MATERIALIZE_CLUSTER) {
+      await conn.query(`SET CLUSTER TO ${process.env.CUBEJS_DB_MATERIALIZE_CLUSTER}`);
+    }
   }
 
   protected async loadUserDefinedTypes(): Promise<void> {

packages/cubejs-materialize-driver/test/MaterializeDriver.test.ts

@@ -31,6 +31,8 @@ describe('MaterializeDriver', () => {
       user: 'materialize',
       password: 'materialize',
       database: 'materialize',
+      cluster: 'quickstart',
+      ssl: false,
     });
     await driver.query('CREATE SCHEMA IF NOT EXISTS test;', []);
   });
@@ -71,8 +73,8 @@ describe('MaterializeDriver', () => {
   test('schema detection', async () => {
     await Promise.all([
       driver.query('CREATE TABLE A (a INT, b BIGINT, c TEXT, d DOUBLE, e FLOAT);', []),
-      driver.query('CREATE VIEW V AS SELECT * FROM mz_views;', []),
-      driver.query('CREATE MATERIALIZED VIEW MV AS SELECT * FROM mz_views;', []),
+      driver.query('CREATE VIEW V AS SELECT * FROM A;', []),
+      driver.query('CREATE MATERIALIZED VIEW MV AS SELECT * FROM A;', []),
     ]);
 
     const tablesSchemaData = await driver.tablesSchema();
@@ -149,4 +151,13 @@ describe('MaterializeDriver', () => {
       );
     }
   });
+
+  test('cluster', async () => {
+    const data = await driver.query(`SHOW CLUSTER;`, []);
+    expect(data).toEqual([
+      {
+        'cluster': 'quickstart',
+      }]);
+  });
+
 });

packages/cubejs-playground/src/shared/env-vars-db-map.tsx

@@ -38,7 +38,6 @@ const envVarsDbMap = [
       { title: 'Hive/SparkSQL', driver: 'hive', logo: logoHive },
       { title: 'Oracle', driver: 'oracle', logo: logoOracle },
       { title: 'QuestDB', driver: 'questdb', logo: logoQuestdb },
-      { title: 'Materialize', driver: 'materialize', logo: logoMaterialize },
       { title: 'Crate', driver: 'crate', logo: logoCrate },
     ],
     settings: [...BASE_SERVER, DB_NAME, ...BASE_CRED],
@@ -119,6 +118,15 @@ Upload a service account JSON keyfile to connect to BigQuery.<br/>Alternatively,
       // { env: 'CUBEJS_DB_SSL_PASSPHRASE' }
     ],
   },
+  {
+    databases: [{ title: 'Materialize', driver: 'materialize', logo: logoMaterialize },],
+    settings: [
+      ...BASE_SERVER,
+      ...BASE_CRED,
+      DB_NAME,
+      { env: 'CUBEJS_DB_MATERIALIZE_CLUSTER', title: 'Cluster' },
+    ],
+  },
   {
     databases: [
       { title: 'Snowflake', driver: 'snowflake', logo: logoSnowflake },

packages/cubejs-testing-shared/src/db/materialize.ts

@@ -8,10 +8,7 @@ type MaterializeStartOptions = DBRunnerContainerOptions & {
 
 export class MaterializeDBRunner extends DbRunnerAbstract {
   public static startContainer(options: MaterializeStartOptions) {
-    /**
-     * Version v0.26.0 is latest Materialize release with Long Term Support
-     */
-    const version = process.env.TEST_MZSQL_VERSION || options.version || 'v0.26.0';
+    const version = process.env.TEST_MZSQL_VERSION || options.version || 'v0.88.0';
 
     const container = new GenericContainer(`materialize/materialized:${version}`)
       .withExposedPorts(6875)

packages/cubejs-testing/test/smoke-materialize.test.ts

@@ -31,6 +31,7 @@ describe('materialize', () => {
         CUBEJS_DB_NAME: 'materialize',
         CUBEJS_DB_USER: 'materialize',
         CUBEJS_DB_PASS: 'materialize',
+        CUBEJS_DB_SSL: 'false',
 
         ...DEFAULT_CONFIG,
       },