fix(duckdb-driver): Compatibility issue with Cube Store for DATE type (…

…#7394)
cube-js · Nov 9, 2023 · 3e1389b · 3e1389b
1 parent 3c9ffd4
commit 3e1389b
Show file tree

Hide file tree

Showing 2 changed files with 57 additions and 35 deletions.
diff --git a/packages/cubejs-duckdb-driver/src/DuckDBDriver.ts b/packages/cubejs-duckdb-driver/src/DuckDBDriver.ts
@@ -2,7 +2,9 @@ import {
   BaseDriver,
   DriverInterface,
   StreamOptions,
-  QueryOptions, StreamTableData,
+  QueryOptions,
+  StreamTableData,
+  GenericDataBaseType,
 } from '@cubejs-backend/base-driver';
 import { getEnv } from '@cubejs-backend/shared';
 import { promisify } from 'util';
@@ -24,6 +26,13 @@ type InitPromise = {
   db: Database;
 };
 
+const DuckDBToGenericType: Record<string, GenericDataBaseType> = {
+  // DATE_TRUNC returns DATE, but Cube Store still doesn't support DATE type
+  // DuckDB's driver transform date/timestamp to Date object, but HydrationStream converts any Date object to ISO timestamp
+  // That's why It's safe to use timestamp here
+  date: 'timestamp',
+};
+
 export class DuckDBDriver extends BaseDriver implements DriverInterface {
   protected initPromise: Promise<InitPromise> | null = null;
 
@@ -37,6 +46,14 @@ export class DuckDBDriver extends BaseDriver implements DriverInterface {
     this.schema = this.config.schema || getEnv('duckdbSchema', this.config);
   }
 
+  public toGenericType(columnType: string): GenericDataBaseType {
+    if (columnType.toLowerCase() in DuckDBToGenericType) {
+      return DuckDBToGenericType[columnType.toLowerCase()];
+    }
+
+    return super.toGenericType(columnType.toLowerCase());
+  }
+
   protected async init(): Promise<InitPromise> {
     const token = getEnv('duckdbMotherDuckToken', this.config);
 

diff --git a/packages/cubejs-duckdb-driver/test/DuckDBDriver.test.ts b/packages/cubejs-duckdb-driver/test/DuckDBDriver.test.ts
@@ -9,63 +9,68 @@ describe('DuckDBDriver', () => {
   beforeAll(async () => {
     driver = new DuckDBDriver({});
     await driver.query('CREATE SCHEMA IF NOT EXISTS test;', []);
-  });
-
-  afterAll(async () => {
-    await driver.release();
-  });
-
-  test('query', async () => {
     await driver.uploadTable(
-      'test.querying_test',
+      'test.select_test',
       [
         { name: 'id', type: 'bigint' },
-        { name: 'created', type: 'date' },
+        { name: 'created', type: 'timestamp' },
+        { name: 'created_date', type: 'date' },
         { name: 'price', type: 'decimal' },
       ],
       {
         rows: [
-          { id: 1, created: '2020-01-01', price: '100' },
-          { id: 2, created: '2020-01-02', price: '200' },
-          { id: 3, created: '2020-01-03', price: '300' }
+          { id: 1, created: '2020-01-01 01:01:01.11111', created_date: '2020-01-01', price: '100' },
+          { id: 2, created: '2020-02-02 02:02:02.22222', created_date: '2020-02-02', price: '200' },
+          { id: 3, created: '2020-03-03 03:03:03.33333', created_date: '2020-03-03', price: '300' }
         ]
       }
     );
+  });
 
-    const result = await driver.query('select * from test.querying_test', []);
+  afterAll(async () => {
+    await driver.release();
+  });
+
+  test('query', async () => {
+    const result = await driver.query('select * from test.select_test ORDER BY id ASC', []);
     expect(result).toEqual([
-      { id: '1', created: '2020-01-01T00:00:00.000Z', price: '100' },
-      { id: '2', created: '2020-01-02T00:00:00.000Z', price: '200' },
-      { id: '3', created: '2020-01-03T00:00:00.000Z', price: '300' }
+      { id: '1', created: '2020-01-01T01:01:01.111Z', created_date: '2020-01-01T00:00:00.000Z', price: '100' },
+      { id: '2', created: '2020-02-02T02:02:02.222Z', created_date: '2020-02-02T00:00:00.000Z', price: '200' },
+      { id: '3', created: '2020-03-03T03:03:03.333Z', created_date: '2020-03-03T00:00:00.000Z', price: '300' }
     ]);
   });
 
-  test('stream', async () => {
-    await driver.uploadTable(
-      'test.streaming_test',
-      [
-        { name: 'id', type: 'bigint' },
-        { name: 'created', type: 'date' },
-        { name: 'price', type: 'decimal' },
-      ],
+  test('column types', async () => {
+    expect(await driver.tableColumnTypes('test.select_test')).toEqual([
       {
-        rows: [
-          { id: 1, created: '2020-01-01', price: '100' },
-          { id: 2, created: '2020-01-02', price: '200' },
-          { id: 3, created: '2020-01-03', price: '300' }
-        ]
+        name: 'id',
+        type: 'bigint',
+      },
+      {
+        name: 'created',
+        type: 'timestamp',
+      },
+      {
+        name: 'created_date',
+        type: 'timestamp',
+      },
+      {
+        name: 'price',
+        type: 'decimal(18,3)',
       }
-    );
+    ]);
+  });
 
-    const tableData = await driver.stream('select * from test.streaming_test', [], {
+  test('stream', async () => {
+    const tableData = await driver.stream('select * from test.select_test ORDER BY id ASC', [], {
       highWaterMark: 1000,
     });
 
     expect(await tableData.types).toEqual(undefined);
     expect(await streamToArray(tableData.rowStream as any)).toEqual([
-      { id: '1', created: '2020-01-01T00:00:00.000Z', price: '100' },
-      { id: '2', created: '2020-01-02T00:00:00.000Z', price: '200' },
-      { id: '3', created: '2020-01-03T00:00:00.000Z', price: '300' }
+      { id: '1', created: '2020-01-01T01:01:01.111Z', created_date: '2020-01-01T00:00:00.000Z', price: '100' },
+      { id: '2', created: '2020-02-02T02:02:02.222Z', created_date: '2020-02-02T00:00:00.000Z', price: '200' },
+      { id: '3', created: '2020-03-03T03:03:03.333Z', created_date: '2020-03-03T00:00:00.000Z', price: '300' }
     ]);
   });
 });