fix(mysql-driver): use utf8mb4 charset for columns to fix ER_TRUNCATE…

…D_WRONG_VALUE_FOR_FIELD

packages/cubejs-mysql-driver/driver/MySqlDriver.js

@@ -4,8 +4,8 @@ const { promisify } = require('util');
 const BaseDriver = require('@cubejs-backend/query-orchestrator/driver/BaseDriver');
 
 const GenericTypeToMySql = {
-  string: 'varchar(255)',
-  text: 'varchar(255)'
+  string: 'varchar(255) CHARACTER SET utf8mb4',
+  text: 'varchar(255) CHARACTER SET utf8mb4'
 };
 
 class MySqlDriver extends BaseDriver {
@@ -17,25 +17,24 @@ class MySqlDriver extends BaseDriver {
       port: process.env.CUBEJS_DB_PORT,
       user: process.env.CUBEJS_DB_USER,
       password: process.env.CUBEJS_DB_PASS,
-      charset: 'utf8mb4',
       ...config
     };
     this.pool = genericPool.createPool({
       create: async () => {
         const conn = mysql.createConnection(this.config);
         const connect = promisify(conn.connect.bind(conn));
 
-        conn.on && conn.on('error', (err) => {
-          conn.destroy();
-        });
+        if (conn.on) {
+          conn.on('error', () => {
+            conn.destroy();
+          });
+        }
         conn.execute = promisify(conn.query.bind(conn));
 
         await connect();
         return conn;
       },
-      destroy: (connection) => {
-        return promisify(connection.end.bind(connection))();
-      },
+      destroy: (connection) => promisify(connection.end.bind(connection))(),
       validate: async (connection) => {
         try {
           await connection.execute('SELECT 1');
@@ -64,52 +63,46 @@ class MySqlDriver extends BaseDriver {
     const promise = connectionPromise.then(conn => {
       cancelObj.cancel = async () => {
         cancelled = true;
-        await self.withConnection(async conn => {
-          const processRows = await conn.execute('SHOW PROCESSLIST');
-          await Promise.all(processRows.filter(row => row.Time >= 599).map(row => {
-            return conn.execute(`KILL ${row.Id}`);
-          }));
+        await self.withConnection(async processConnection => {
+          const processRows = await processConnection.execute('SHOW PROCESSLIST');
+          await Promise.all(processRows.filter(row => row.Time >= 599)
+            .map(row => processConnection.execute(`KILL ${row.Id}`)));
         });
       };
       return fn(conn)
-        .then(res => {
-          return this.pool.release(conn).then(() => {
-            if (cancelled) {
-              throw new Error('Query cancelled');
-            }
-            return res;
-          });
-        })
-        .catch((err) => {
-          return this.pool.release(conn).then(() => {
-            if (cancelled) {
-              throw new Error('Query cancelled');
-            }
-            throw err;
-          });
-        })
+        .then(res => this.pool.release(conn).then(() => {
+          if (cancelled) {
+            throw new Error('Query cancelled');
+          }
+          return res;
+        }))
+        .catch((err) => this.pool.release(conn).then(() => {
+          if (cancelled) {
+            throw new Error('Query cancelled');
+          }
+          throw err;
+        }));
     });
     promise.cancel = () => cancelObj.cancel();
     return promise;
   }
 
   async testConnection() {
+    // eslint-disable-next-line no-underscore-dangle
     const conn = await this.pool._factory.create();
     try {
       return await conn.execute('SELECT 1');
     } finally {
+      // eslint-disable-next-line no-underscore-dangle
       await this.pool._factory.destroy(conn);
     }
   }
 
   query(query, values) {
     const self = this;
-    return this.withConnection(db => {
-
-      return db.execute(`SET time_zone = '${self.config.storeTimezone || '+00:00'}'`, [])
-        .then(() => db.execute(query, values))
-        .then(res => res);
-    });
+    return this.withConnection(db => db.execute(`SET time_zone = '${self.config.storeTimezone || '+00:00'}'`, [])
+      .then(() => db.execute(query, values))
+      .then(res => res));
   }
 
   async release() {
@@ -118,7 +111,7 @@ class MySqlDriver extends BaseDriver {
   }
 
   informationSchemaQuery() {
-    return `${super.informationSchemaQuery()} AND columns.table_schema = '${this.config.database}'`
+    return `${super.informationSchemaQuery()} AND columns.table_schema = '${this.config.database}'`;
   }
 
   quoteIdentifier(identifier) {
@@ -166,4 +159,4 @@ class MySqlDriver extends BaseDriver {
   }
 }
 
-module.exports = MySqlDriver;
+module.exports = MySqlDriver;

packages/cubejs-mysql-driver/package.json

@@ -12,11 +12,23 @@
     "node": ">=8.11.1"
   },
   "main": "driver/MySqlDriver.js",
+  "scripts": {
+    "test": "jest",
+    "lint": "eslint driver/*.js test/*.js"
+  },
   "dependencies": {
     "@cubejs-backend/query-orchestrator": "^0.18.7",
     "generic-pool": "^3.6.0",
     "mysql": "^2.16.0"
   },
+  "devDependencies": {
+    "eslint": "^6.8.0",
+    "eslint-config-airbnb-base": "^13.1.0",
+    "eslint-plugin-import": "^2.16.0",
+    "eslint-plugin-node": "^5.2.1",
+    "jest": "^25.1.0",
+    "testcontainers": "^2.4.0"
+  },
   "license": "Apache-2.0",
   "publishConfig": {
     "access": "public"

packages/cubejs-mysql-driver/test/MySqlDriver.test.js

@@ -0,0 +1,44 @@
+/* globals describe, afterAll, beforeAll, test, expect, jest */
+const { GenericContainer } = require("testcontainers");
+const MySqlDriver = require('../driver/MySqlDriver');
+
+describe('MySqlDriver', () => {
+  let container;
+  let mySqlDriver;
+
+  jest.setTimeout(50000);
+
+  beforeAll(async () => {
+    if (!process.env.TEST_LOCAL) {
+      container = await new GenericContainer("mysql", '5.7')
+        .withEnv("MYSQL_ROOT_PASSWORD", process.env.TEST_DB_PASSWORD || "Test1test")
+        .withExposedPorts(3306)
+        .start();
+    }
+
+    mySqlDriver = new MySqlDriver({
+      host: 'localhost',
+      user: 'root',
+      password: process.env.TEST_DB_PASSWORD || "Test1test",
+      port: container && container.getMappedPort(3306) || 3306
+    });
+    await mySqlDriver.createSchemaIfNotExists('test');
+    await mySqlDriver.query('DROP SCHEMA test');
+    await mySqlDriver.createSchemaIfNotExists('test');
+  });
+
+  afterAll(async () => {
+    if (container) {
+      await container.stop();
+    }
+  });
+
+  test('truncated wrong value', async () => {
+    await mySqlDriver.uploadTable(`test.wrong_value`, [{ name: 'value', type: 'string' }], {
+      rows: [{ value: "Tekirdağ" }]
+    });
+    expect(JSON.parse(JSON.stringify(await mySqlDriver.query('select * from test.wrong_value'))))
+      .toStrictEqual([{ value: "Tekirdağ" }]);
+    await mySqlDriver.release();
+  });
+});