From 7b880d211e4030a5aed9181f84c03dc20989038c Mon Sep 17 00:00:00 2001 From: Cameron Braid Date: Tue, 30 Apr 2019 10:44:59 +1000 Subject: [PATCH 1/2] clickhouse dialect implementation --- .../driver/ClickHouseDriver.js | 35 +- .../test/ClickHouseDriverTest.js | 68 +- .../adapter/ClickHouseQuery.js | 209 +++ .../adapter/QueryBuilder.js | 5 +- .../cubejs-schema-compiler/docker-compose.yml | 17 + packages/cubejs-schema-compiler/package.json | 5 +- .../test/ClickHouseDataSchemaCompilerTest.js | 460 ++++++ .../test/ClickHouseDbRunner.js | 120 ++ .../test/ClickHouseGraphBuilderTest.js | 1329 +++++++++++++++++ .../cubejs-schema-compiler/test/TestUtil.js | 15 + packages/cubejs-schema-compiler/yarn.lock | 81 +- 11 files changed, 2310 insertions(+), 34 deletions(-) create mode 100644 packages/cubejs-schema-compiler/adapter/ClickHouseQuery.js create mode 100644 packages/cubejs-schema-compiler/docker-compose.yml create mode 100644 packages/cubejs-schema-compiler/test/ClickHouseDataSchemaCompilerTest.js create mode 100644 packages/cubejs-schema-compiler/test/ClickHouseDbRunner.js create mode 100644 packages/cubejs-schema-compiler/test/ClickHouseGraphBuilderTest.js create mode 100644 packages/cubejs-schema-compiler/test/TestUtil.js diff --git a/packages/cubejs-clickhouse-driver/driver/ClickHouseDriver.js b/packages/cubejs-clickhouse-driver/driver/ClickHouseDriver.js index 54899f1d9b61a..da71432c7657f 100644 --- a/packages/cubejs-clickhouse-driver/driver/ClickHouseDriver.js +++ b/packages/cubejs-clickhouse-driver/driver/ClickHouseDriver.js @@ -90,10 +90,43 @@ class ClickHouseDriver extends BaseDriver { return this.withConnection((connection, queryId) => { return connection.querying(formattedQuery, { dataObjects: true, queryOptions: { query_id: queryId, join_use_nulls: 1 } }) - .then(res => res.data); + .then(res => this._normaliseResponse(res)); }); } + _normaliseResponse(res) { + // + // + // ClickHouse returns DateTime as strings in format "YYYY-DD-MM HH:MM:SS" in UTC timezone + // cube.js expects them in format "YYYY-DD-MMTHH:MM:SS.000Z", so translate them based on the metadata returned + // + // ClickHouse returns some number types as js numbers, others as js string, normalise them all to strings + // + // + if (res.data) { + res.data.forEach(row=>{ + for (let field in row) { + let value = row[field] + if (value !== null) { + let meta = res.meta.find(m=>m.name == field) + if (meta.type.includes("DateTime")) { + row[field] = value.substring(0, 10) + "T" + value.substring(11, 22) + ".000" + } + else if (meta.type.includes("Date")) { + row[field] = value + "T00:00:00.000" + } + else if (meta.type.includes("Int") || meta.type.includes("Float")) { + // convert all numbers into strings + row[field] = `${value}` + } + } + } + }) + } + return res.data + return res.data + } + async release() { await this.pool.drain(); await this.pool.clear(); diff --git a/packages/cubejs-clickhouse-driver/test/ClickHouseDriverTest.js b/packages/cubejs-clickhouse-driver/test/ClickHouseDriverTest.js index 52b88361e4f08..68c170d4a7191 100644 --- a/packages/cubejs-clickhouse-driver/test/ClickHouseDriverTest.js +++ b/packages/cubejs-clickhouse-driver/test/ClickHouseDriverTest.js @@ -104,6 +104,64 @@ describe('ClickHouseDriver', () => { }) }); + // Int8 + // Int16 + // Int32 + // Int64 + // UInt8 + // UInt16 + // UInt32 + // UInt64 + // Float32 + // Float64 + it('should normalise all numbers as strings', async () => { + await doWithDriver(async (driver) => { + let name = `temp_${Date.now()}` + try { + await driver.createSchemaIfNotExists(name); + await driver.query(`CREATE TABLE ${name}.a (int8 Int8, int16 Int16, int32 Int32, int64 Int64, uint8 UInt8, uint16 UInt16, uint32 UInt32, uint64 UInt64, float32 Float32, float64 Float64) ENGINE Log`); + await driver.query(`INSERT INTO ${name}.a VALUES (1,1,1,1,1,1,1,1,1,1)`); + + const values = await driver.query(`SELECT * FROM ${name}.a`); + values.should.deepEqual([{ + int8: '1', + int16: '1', + int32: '1', + int64: '1', + uint8: '1', + uint16: '1', + uint32: '1', + uint64: '1', + float32: '1', + float64: '1', + }]) + } + finally { + await driver.query(`DROP DATABASE ${name}`) + } + }) + }); + + it('should normalise all dates as ISO8601', async () => { + await doWithDriver(async (driver) => { + let name = `temp_${Date.now()}` + try { + await driver.createSchemaIfNotExists(name); + await driver.query(`CREATE TABLE ${name}.a (dateTime DateTime, date Date) ENGINE Log`); + await driver.query(`INSERT INTO ${name}.a VALUES ('2019-04-30 11:55:00', '2019-04-30')`); + + const values = await driver.query(`SELECT * FROM ${name}.a`); + values.should.deepEqual([{ + dateTime: '2019-04-30T11:55:00.000Z', + date: '2019-04-30T00:00:00.000Z', + }]) + } + finally { + await driver.query(`DROP DATABASE ${name}`) + } + }) + }); + it('should substitute parameters', async () => { await doWithDriver(async (driver) => { let name = `temp_${Date.now()}` @@ -112,7 +170,7 @@ describe('ClickHouseDriver', () => { await driver.query(`CREATE TABLE ${name}.test (x Int32, s String) ENGINE Log`); await driver.query(`INSERT INTO ${name}.test VALUES (?, ?), (?, ?), (?, ?)`, [1, "str1", 2, "str2", 3, "str3"]); const values = await driver.query(`SELECT * FROM ${name}.test WHERE x = ?`, 2); - values.should.deepEqual([{x: 2, s: "str2"}]) + values.should.deepEqual([{x: '2', s: "str2"}]) } finally { await driver.query(`DROP DATABASE ${name}`) @@ -133,9 +191,9 @@ describe('ClickHouseDriver', () => { const values = await driver.query(`SELECT * FROM ${name}.a LEFT OUTER JOIN ${name}.b ON a.x = b.x`); values.should.deepEqual([ - { x: 1, s: 'str1', 'b.x': 0, 'b.s': null }, - { x: 2, s: 'str2', 'b.x': 2, 'b.s': 'str2' }, - { x: 3, s: 'str3', 'b.x': 3, 'b.s': 'str3' } + { x: '1', s: 'str1', 'b.x': '0', 'b.s': null }, + { x: '2', s: 'str2', 'b.x': '2', 'b.s': 'str2' }, + { x: '3', s: 'str3', 'b.x': '3', 'b.s': 'str3' } ]) } finally { @@ -143,4 +201,6 @@ describe('ClickHouseDriver', () => { } }) }); + + }); diff --git a/packages/cubejs-schema-compiler/adapter/ClickHouseQuery.js b/packages/cubejs-schema-compiler/adapter/ClickHouseQuery.js new file mode 100644 index 0000000000000..e6540e191ce3b --- /dev/null +++ b/packages/cubejs-schema-compiler/adapter/ClickHouseQuery.js @@ -0,0 +1,209 @@ +const moment = require('moment-timezone'); +const R = require('ramda'); + +const BaseQuery = require('./BaseQuery'); +const BaseFilter = require('./BaseFilter'); + +const GRANULARITY_TO_INTERVAL = { + date: 'Day', + hour: 'Hour', + month: 'Month', + quarter: 'Quarter', + year: 'Year', +}; + +class ClickHouseFilter extends BaseFilter { + likeIgnoreCase(column, not) { + return `lower(${column}) ${not ? 'NOT':''} LIKE CONCAT('%', lower(?), '%')`; + } +} + + +class ClickHouseQuery extends BaseQuery { + + newFilter(filter) { + return new ClickHouseFilter(this, filter); + } + + escapeColumnName(name) { + return `\`${name}\``; + } + + + convertTz(field) { + // + // field yields a Date or a DateTime so add in the extra toDateTime to support the Date case + // + // https://clickhouse.yandex/docs/en/data_types/datetime/ + // https://clickhouse.yandex/docs/en/query_language/functions/date_time_functions/ + // + // + return `toTimeZone(toDateTime(${field}), '${this.timezone}')`; + } + + timeGroupedColumn(granularity, dimension) { + if (granularity == 'week') { + return `toDateTime(toMonday(${dimension}, '${this.timezone}'), '${this.timezone}')` + } + else { + let interval = GRANULARITY_TO_INTERVAL[granularity] + return `toDateTime(toStartOf${interval}(${dimension}, '${this.timezone}'), '${this.timezone}')`; + } + } + + _calcInterval(operation, date, interval) { + const [intervalValue, intervalUnit] = interval.split(" "); + let fn = operation + intervalUnit[0].toUpperCase() + intervalUnit.substring(1) + "s" + return `${fn}(${date}, ${intervalValue})`; + } + + subtractInterval(date, interval) { + return this._calcInterval("subtract", date, interval) + } + + addInterval(date, interval) { + return this._calcInterval("add", date, interval) + } + + timeStampCast(value) { + // value yields a string formatted in ISO8601, so this function returns a expression to parse a string to a DateTime + + // + // ClickHouse provides toDateTime which expects dates in UTC in format YYYY-MM-DD HH:MM:SS + // + // However parseDateTimeBestEffort works with ISO8601 + // + return `parseDateTimeBestEffort(${value})` + } + + dateTimeCast(value) { + + // value yields a string formatted in ISO8601, so this function returns a expression to parse a string to a DateTime + + // + // ClickHouse provides toDateTime which expects dates in UTC in format YYYY-MM-DD HH:MM:SS + // + // However parseDateTimeBestEffort works with ISO8601 + // + return `parseDateTimeBestEffort(${value})` + } + + getFieldAlias(id) { + const equalIgnoreCase = (a, b) => ( + typeof a === 'string' && typeof b === 'string' + && a.toUpperCase() === b.toUpperCase() + ); + + let field; + + field = this.dimensionsForSelect().find(d => + equalIgnoreCase(d.dimension, id) + ); + + if (field) { + return field.aliasName(); + } + + field = this.measures.find(d => + equalIgnoreCase(d.measure, id) || equalIgnoreCase(d.expressionName, id) + ); + + if (field) { + return field.aliasName(); + } + + return null; + } + + orderHashToString(hash) { + + // + // ClickHouse doesn't support order by index column, so map these to the alias names + // + + if (!hash || !hash.id) { + return null; + } + + const fieldAlias = this.getFieldAlias(hash.id); + + if (fieldAlias === null) { + return null; + } + + const direction = hash.desc ? 'DESC' : 'ASC'; + return `${fieldAlias} ${direction}`; + } + + groupByClause() { + + // + // ClickHouse doesn't support group by index column, so map these to the alias names + // + + const names = this.dimensionAliasNames(); + return names.length ? ` GROUP BY ${names.join(', ')}` : ''; + } + + + primaryKeyCount(cubeName, distinct) { + const primaryKeySql = this.primaryKeySql(this.cubeEvaluator.primaryKeys[cubeName], cubeName); + if (distinct) { + return `uniqExact(${primaryKeySql})` + } + else { + return `count(${primaryKeySql})` + } + } + + + seriesSql(timeDimension) { + /* + postgres uses : + + SELECT parseDateTimeBestEffort(date_from), parseDateTimeBestEffort(date_to) FROM + ( + VALUES + ('2017-01-01T00:00:00.000', '2017-01-01T23:59:59.999'), + ('2017-01-02T00:00:00.000', '2017-01-02T23:59:59.999'), + ('2017-01-03T00:00:00.000', '2017-01-03T23:59:59.999'), + ('2017-01-04T00:00:00.000', '2017-01-04T23:59:59.999'), + ('2017-01-05T00:00:00.000', '2017-01-05T23:59:59.999'), + ('2017-01-06T00:00:00.000', '2017-01-06T23:59:59.999'), + ('2017-01-07T00:00:00.000', '2017-01-07T23:59:59.999'), + ('2017-01-08T00:00:00.000', '2017-01-08T23:59:59.999'), + ('2017-01-09T00:00:00.000', '2017-01-09T23:59:59.999'), + ('2017-01-10T00:00:00.000', '2017-01-10T23:59:59.999') + ) AS dates (date_from, date_to) + ) AS `visitors.created_at_series` + + */ + /* + + ClickHouse uses : + + select + parseDateTimeBestEffort(arrayJoin(['2017-01-01T00:00:00.000','2017-01-02T00:00:00.000'])) as date_from, + parseDateTimeBestEffort(arrayJoin(['2017-01-01T23:59:59.999','2017-01-02T23:59:59.999'])) as date_to + ... + ) + */ + + let dates_from = [] + let dates_to = [] + timeDimension.timeSeries().forEach(([from, to]) => { + dates_from.push(from) + dates_to.push(to) + } + + ); + return `SELECT parseDateTimeBestEffort(arrayJoin(['${dates_from.join("','")}'])) as date_from, parseDateTimeBestEffort(arrayJoin(['${dates_to.join("','")}'])) as date_to`; + } + + concatStringsSql(strings) { + return "toString(" + strings.join(") || toString(") + ")"; + } + +} + +module.exports = ClickHouseQuery; diff --git a/packages/cubejs-schema-compiler/adapter/QueryBuilder.js b/packages/cubejs-schema-compiler/adapter/QueryBuilder.js index b0b10ac1cb746..5ae12cc0b8d24 100644 --- a/packages/cubejs-schema-compiler/adapter/QueryBuilder.js +++ b/packages/cubejs-schema-compiler/adapter/QueryBuilder.js @@ -7,6 +7,7 @@ const redshift = require('./RedshiftQuery'); const prestodb = require('./PrestodbQuery'); const vertica = require('./VerticaQuery'); const snowflake = require('./SnowflakeQuery'); +const clickhouse = require('./ClickHouseQuery'); const ADAPTERS = { postgres, @@ -19,9 +20,9 @@ const ADAPTERS = { qubole_prestodb: prestodb, athena: prestodb, vertica, - snowflake + snowflake, + clickhouse, }; - exports.query = (compilers, adapter, queryOptions) => { if (!ADAPTERS[adapter]) { return null; diff --git a/packages/cubejs-schema-compiler/docker-compose.yml b/packages/cubejs-schema-compiler/docker-compose.yml new file mode 100644 index 0000000000000..cc11c63fe21b5 --- /dev/null +++ b/packages/cubejs-schema-compiler/docker-compose.yml @@ -0,0 +1,17 @@ +version: "3" + +services: + + clickhouse-server: + image: yandex/clickhouse-server + ports: + - "18123:8123" + + postgresql-server: + image: postgres:11.2 + ports: + - "5432:5432" + environment: + POSTGRES_DB: model_test + POSTGRES_PASSWORD: test + POSTGRES_USER: test diff --git a/packages/cubejs-schema-compiler/package.json b/packages/cubejs-schema-compiler/package.json index 3da757eab8401..5e795187b12fe 100644 --- a/packages/cubejs-schema-compiler/package.json +++ b/packages/cubejs-schema-compiler/package.json @@ -21,9 +21,12 @@ "moment-timezone": "^0.5.13", "node-dijkstra": "^2.5.0", "ramda": "^0.24.1", - "syntax-error": "^1.3.0" + "sqlstring": "^2.3.1", + "syntax-error": "^1.3.0", + "uuid": "^3.3.2" }, "devDependencies": { + "@apla/clickhouse": "^1.5.5", "mocha": "^3.4.2", "pg-promise": "^7.3.2", "should": "^11.2.1" diff --git a/packages/cubejs-schema-compiler/test/ClickHouseDataSchemaCompilerTest.js b/packages/cubejs-schema-compiler/test/ClickHouseDataSchemaCompilerTest.js new file mode 100644 index 0000000000000..c8ddb79307ed8 --- /dev/null +++ b/packages/cubejs-schema-compiler/test/ClickHouseDataSchemaCompilerTest.js @@ -0,0 +1,460 @@ +const CompileError = require('../compiler/CompileError'); +const PrepareCompiler = require('./PrepareCompiler'); +const MainPrepareCompiler = require('../compiler/PrepareCompiler'); +require('should'); + +const prepareCompiler = PrepareCompiler.prepareCompiler; +const dbRunner = require('./ClickHouseDbRunner'); + +const { debugLog, logSqlAndParams } = require('./TestUtil'); + +describe('ClickHouse DataSchemaCompiler', async () => { + + it('gutter', () => { + const { compiler } = prepareCompiler(` + cube('visitors', { + sql: \` + select * from visitors + \`, + + measures: { + visitor_count: { + type: 'count', + sql: 'id', + drillMembers: [source, created_at] + }, + visitor_revenue: { + type: 'sum', + sql: 'amount', + drillMemberReferences: [source, created_at] + } + }, + + dimensions: { + source: { + type: 'string', + sql: 'source' + }, + created_at: { + type: 'time', + sql: 'created_at' + } + } + }) + `); + return compiler.compile(); + }); + + it('error', () => { + const { compiler } = prepareCompiler(` + cube({}, { + measures: {} + }) + `); + return compiler.compile() + .then(() => { + compiler.throwIfAnyErrors(); + throw new Error(); + }) + .catch((error) => { + error.should.be.instanceof(CompileError); + }); + }); + + it('duplicate member', () => { + const { compiler } = prepareCompiler(` + cube('visitors', { + sql: \` + select * from visitors + \`, + + measures: { + count: { + type: 'count', + sql: 'id' + }, + id: { + type: 'sum', + sql: 'id' + } + }, + + dimensions: { + id: { + type: 'number', + sql: 'id', + primaryKey: true + } + } + }) + `); + return compiler.compile().then(() => { + compiler.throwIfAnyErrors(); + throw new Error(); + }).catch((error) => { + error.should.be.instanceof(CompileError); + }); + }); + + it('calculated metrics', () => { + const { compiler, transformer, cubeEvaluator, joinGraph } = prepareCompiler(` + cube('visitors', { + sql: \` + select * from visitors + \`, + + measures: { + visitor_count: { + type: 'count', + sql: 'id' + }, + visitor_revenue: { + type: 'sum', + sql: 'amount' + }, + per_visitor_revenue: { + type: 'number', + sql: visitor_revenue + "/" + visitor_count + } + }, + + dimensions: { + source: { + type: 'string', + sql: 'source' + }, + created_at: { + type: 'time', + sql: 'created_at' + }, + updated_at: { + type: 'time', + sql: 'updated_at' + } + } + }) + `); + const result = compiler.compile().then(() => { + const query = dbRunner.newQuery({ joinGraph, cubeEvaluator, compiler }, { + measures: ['visitors.visitor_count'], + timeDimensions: [{ + dimension: 'visitors.created_at', + granularity: 'date', + dateRange: ['2017-01-01', '2017-01-30'] + }], + filters: [{ + dimension: 'visitors.updated_at', + operator: 'in_date_range', + values: ['2017-01-01', '2017-01-30'] + }], + order: [{ + id: 'visitors.created_at' + }], + timezone: 'America/Los_Angeles' + }); + + logSqlAndParams(query); + + return dbRunner.testQuery(query.buildSqlAndParams()).then(res => { + res.should.be.deepEqual( + [ + { "visitors.created_at_date": "2017-01-02T00:00:00.000", "visitors.visitor_count": "1" }, + { "visitors.created_at_date": "2017-01-04T00:00:00.000", "visitors.visitor_count": "1" }, + { "visitors.created_at_date": "2017-01-05T00:00:00.000", "visitors.visitor_count": "1" }, + { "visitors.created_at_date": "2017-01-06T00:00:00.000", "visitors.visitor_count": "2" } + ] + ); + }); + }); + + return result; + }); + + it('dimension case', () => { + const { compiler, transformer, cubeEvaluator, joinGraph } = prepareCompiler(` + cube('visitors', { + sql: \` + select * from visitors + \`, + + measures: { + visitor_count: { + type: 'count', + sql: 'id' + } + }, + + dimensions: { + status: { + type: 'string', + case: { + when: [{ + sql: \`\${CUBE}.status = 1\`, + label: 'Approved' + }, { + sql: \`\${CUBE}.status = 2\`, + label: 'Canceled' + }], + else: { label: 'Unknown' } + } + }, + created_at: { + type: 'time', + sql: 'created_at' + } + } + }) + `); + const result = compiler.compile().then(() => { + const query = dbRunner.newQuery({ joinGraph, cubeEvaluator, compiler }, { + measures: ['visitors.visitor_count'], + dimensions: ['visitors.status'], + timezone: 'America/Los_Angeles', + order: [{ + id: 'visitors.status' + }] + }); + + logSqlAndParams(query); + + return dbRunner.testQuery(query.buildSqlAndParams()).then(res => { + res.should.be.deepEqual( + [ + { "visitors.status": "Approved", "visitors.visitor_count": "2" }, + { "visitors.status": "Canceled", "visitors.visitor_count": "4" } + ] + ); + }); + }); + + return result; + }); + + { + const { compiler, cubeEvaluator, joinGraph } = prepareCompiler(` + cube('visitors', { + sql: \` + select * from visitors + \`, + + dimensions: { + source: { + type: 'string', + sql: 'source' + }, + created_at: { + type: 'time', + sql: 'created_at' + }, + updated_at: { + type: 'time', + sql: 'updated_at' + } + } + }) + `); + const responses = [ + [{ 'visitors.created_at': '2017-01-02T16:00:00.000' }], + [ + { 'visitors.created_at': '2016-09-06T16:00:00.000' }, + { 'visitors.created_at': '2017-01-04T16:00:00.000' }, + { 'visitors.created_at': '2017-01-05T16:00:00.000' }, + { 'visitors.created_at': '2017-01-06T16:00:00.000' } + ], + [{ 'visitors.created_at': '2017-01-06T16:00:00.000' }], + [ + { 'visitors.created_at': '2016-09-06T16:00:00.000' }, + { 'visitors.created_at': '2017-01-02T16:00:00.000' }, + { 'visitors.created_at': '2017-01-04T16:00:00.000' }, + { 'visitors.created_at': '2017-01-05T16:00:00.000' } + ], + [{ 'visitors.created_at': '2017-01-06T16:00:00.000' }] + ]; + ['in_date_range', 'not_in_date_range', 'on_the_date', 'before_date', 'after_date'].map((operator, index) => { + const filterValues = index < 2 ? ['2017-01-01', '2017-01-03'] : ['2017-01-06', '2017-01-06']; + it(`filtered dates ${operator}`, async () => { + + await compiler.compile() + + let query = dbRunner.newQuery({ joinGraph, cubeEvaluator, compiler }, { + measures: [], + dimensions: ['visitors.created_at'], + timeDimensions: [], + filters: [{ + operator, + dimension: 'visitors.created_at', + values: filterValues + }], + order: [{ + id: 'visitors.created_at', + desc: false + }], + timezone: 'America/Los_Angeles' + }); + logSqlAndParams(query); + const res = await dbRunner.testQuery(query.buildSqlAndParams()); + + res.should.be.deepEqual(responses[index]); + + }); + + }); + } + + it('export import', () => { + const { compiler, cubeEvaluator, joinGraph } = MainPrepareCompiler.prepareCompiler({ + dataSchemaFiles: () => Promise.resolve([ + { + fileName: "main.js", + content: ` + const fooTable = require('./some.js').foo; + cube('Main', { + sql: \`select * from \${fooTable}\`, + measures: { + count: { + sql: 'count(*)', + type: 'number' + } + } + }) + ` + }, { + fileName: "some.js", + content: ` + export const foo = 'bar'; + ` + } + ]) + }, { adapter: dbRunner.adapter }); + return compiler.compile().then(() => { + const query = dbRunner.newQuery({ joinGraph, cubeEvaluator, compiler }, { + measures: ['Main.count'], + dimensions: [], + timeDimensions: [], + order: [], + timezone: 'America/Los_Angeles' + }); + logSqlAndParams(query); + query.buildSqlAndParams()[0].should.match(/bar/); + }); + }); + + it('contexts', () => { + const { compiler, contextEvaluator } = prepareCompiler(` + cube('Visitors', { + sql: \` + select * from visitors + \`, + + measures: { + visitor_count: { + type: 'count', + sql: 'id' + }, + }, + + dimensions: { + source: { + type: 'string', + sql: 'source' + }, + } + }) + + context('Marketing', { + contextMembers: [Visitors] + }); + `); + return compiler.compile().then(() => { + contextEvaluator.contextList.should.be.deepEqual( + ['Marketing'] + ) + }); + }); + + it('dashboard templates', () => { + const { compiler, contextEvaluator, dashboardTemplateEvaluator } = prepareCompiler(` + cube('Visitors', { + sql: \` + select * from visitors + \`, + + measures: { + count: { + type: 'count', + sql: 'id' + }, + }, + + dimensions: { + source: { + type: 'string', + sql: 'source' + }, + + createdAt: { + sql: 'created_at', + type: 'time' + } + } + }) + + dashboardTemplate('VisitorsMarketing', { + title: 'Marketing', + items: [{ + measures: [Visitors.count], + dimensions: [Visitors.source], + visualization: { + type: 'pie' + }, + timeDimension: { + dimension: Visitors.createdAt, + dateRange: 'last week' + }, + filters: [{ + member: Visitors.source, + operator: 'equals', + params: ['google'] + }], + order: [{ + member: Visitors.source, + direction: 'asc' + }], + layout: { + w: 24, + h: 4, + x: 0, + y: 0 + } + }] + }); + `); + return compiler.compile().then(() => { + JSON.parse(JSON.stringify(dashboardTemplateEvaluator.compiledTemplates)).should.be.deepEqual( + [{ + name: 'VisitorsMarketing', + title: 'Marketing', + fileName: 'main.js', + items: [{ + config: { + visualization_type: 'pie', + metrics: ['Visitors.count'], + dimension: ['Visitors.source'], + daterange: 'last week', + time_dimension_field: 'Visitors.createdAt', + order: [{ desc: false, id: 'Visitors.source' }], + filters: [ + { + value: ['google'], + operator: 'equals', + dimension: 'Visitors.source' + } + ] + }, + layout: { w: 24, h: 4, x: 0, y: 0 } + }] + }] + ) + }); + }); +}); diff --git a/packages/cubejs-schema-compiler/test/ClickHouseDbRunner.js b/packages/cubejs-schema-compiler/test/ClickHouseDbRunner.js new file mode 100644 index 0000000000000..d3238e91ccf92 --- /dev/null +++ b/packages/cubejs-schema-compiler/test/ClickHouseDbRunner.js @@ -0,0 +1,120 @@ +const ClickHouse = require('@apla/clickhouse'); +const ClickHouseQuery = require('../adapter/ClickHouseQuery'); +const sqlstring = require('sqlstring'); +const uuidv4 = require('uuid/v4'); + +process.env.TZ = 'GMT'; + +exports.newQuery = function(a,b) { + return new ClickHouseQuery(a,b) +} + +// let engine = 'MergeTree PARTITION BY id ORDER BY (id) SETTINGS index_granularity = 8192' +let engine = 'Memory' +exports.gutterDataSet = async function (clickHouse) { + await clickHouse.querying(` + CREATE TEMPORARY TABLE visitors (id UInt64, amount UInt64, created_at DateTime, updated_at DateTime, status UInt64, source Nullable(String), latitude Float64, longitude Float64) + ENGINE = ${engine} + `,{ queryOptions: { session_id : clickHouse.sessionId, join_use_nulls: "1" } }), + await clickHouse.querying(` + CREATE TEMPORARY TABLE visitor_checkins (id UInt64, visitor_id UInt64, created_at DateTime, source Nullable(String)) + ENGINE = ${engine} + `,{ queryOptions: { session_id : clickHouse.sessionId, join_use_nulls: "1" } }), + await clickHouse.querying(` + CREATE TEMPORARY TABLE cards (id UInt64, visitor_id UInt64, visitor_checkin_id UInt64) + ENGINE = ${engine} + `,{ queryOptions: { session_id : clickHouse.sessionId, join_use_nulls: "1" } }), + + await clickHouse.querying(` + INSERT INTO + visitors + (id, amount, created_at, updated_at, status, source, latitude, longitude) VALUES + (1, 100, '2017-01-02 16:00:00', '2017-01-29 16:00:00', 1, 'some', 120.120, 40.60), + (2, 200, '2017-01-04 16:00:00', '2017-01-14 16:00:00', 1, 'some', 120.120, 58.60), + (3, 300, '2017-01-05 16:00:00', '2017-01-19 16:00:00', 2, 'google', 120.120, 70.60), + (4, 400, '2017-01-06 16:00:00', '2017-01-24 16:00:00', 2, null, 120.120, 10.60), + (5, 500, '2017-01-06 16:00:00', '2017-01-24 16:00:00', 2, null, 120.120, 58.10), + (6, 500, '2016-09-06 16:00:00', '2016-09-06 16:00:00', 2, null, 120.120, 58.10) + `,{ queryOptions: { session_id : clickHouse.sessionId, join_use_nulls: "1" } }), + await clickHouse.querying(` + INSERT INTO + visitor_checkins + (id, visitor_id, created_at, source) VALUES + (1, 1, '2017-01-02 16:00:00', null), + (2, 1, '2017-01-03 16:00:00', null), + (3, 1, '2017-01-04 16:00:00', 'google'), + (4, 2, '2017-01-04 16:00:00', null), + (5, 2, '2017-01-04 16:00:00', null), + (6, 3, '2017-01-05 16:00:00', null) + `,{ queryOptions: { session_id : clickHouse.sessionId, join_use_nulls: "1" } }), + await clickHouse.querying(` + INSERT INTO + cards + (id, visitor_id, visitor_checkin_id) VALUES + (1, 1, 1), + (2, 1, 2), + (3, 3, 6) + `,{ queryOptions: { session_id : clickHouse.sessionId, join_use_nulls: "1" } }) + +}; + +exports.testQuery = function (queryAndParams, prepareDataSet) { + return exports.testQueries([queryAndParams], prepareDataSet).then(res=>res[0]); +}; + +exports.testQueries = async function (queries, prepareDataSet) { + let clickHouse = new ClickHouse({ + host: 'localhost', + port: 18123, + }) + + clickHouse.sessionId = uuidv4() // needed for tests to use temporary tables + + + prepareDataSet = prepareDataSet || exports.gutterDataSet; + await prepareDataSet(clickHouse) + let results = [] + for ([query, params] of queries) { + results.push(_normaliseResponse((await clickHouse.querying(sqlstring.format(query, params), { + dataObjects:true, + queryOptions: { session_id : clickHouse.sessionId, join_use_nulls: "1" } + })))) + } + return results +}; + + + +// +// +// ClickHouse returns DateTime as strings in format "YYYY-DD-MM HH:MM:SS" +// cube.js expects them in format "YYYY-DD-MMTHH:MM:SS.000", so translate them based on the metadata returned +// +// https://github.com/statsbotco/cube.js/pull/98#discussion_r279698399 +// +function _normaliseResponse(res) { + if (process.env.DEBUG_LOG === "true") console.log(res) + if (res.data) { + res.data.forEach(row=>{ + for (let field in row) { + let value = row[field] + if (value !== null) { + let meta = res.meta.find(m=>m.name == field) + if (meta.type.includes("DateTime")) { + row[field] = value.substring(0, 10) + "T" + value.substring(11, 22) + ".000" + } + else if (meta.type.includes("Date")) { + row[field] = value + "T00:00:00.000" + } + else if (meta.type.includes("Int") || meta.type.includes("Float")) { + // convert all numbers into strings + row[field] = `${value}` + } + } + } + }) + } + return res.data +} + +exports.adapter = 'clickhouse' \ No newline at end of file diff --git a/packages/cubejs-schema-compiler/test/ClickHouseGraphBuilderTest.js b/packages/cubejs-schema-compiler/test/ClickHouseGraphBuilderTest.js new file mode 100644 index 0000000000000..e3608ad9ac2b1 --- /dev/null +++ b/packages/cubejs-schema-compiler/test/ClickHouseGraphBuilderTest.js @@ -0,0 +1,1329 @@ +const CompileError = require('../compiler/CompileError'); +const UserError = require('../compiler/UserError'); +const BigqueryQuery = require('../adapter/BigqueryQuery'); +const PrepareCompiler = require('./PrepareCompiler'); +require('should'); + +const prepareCompiler = PrepareCompiler.prepareCompiler; +const dbRunner = require('./ClickHouseDbRunner'); + +const { debugLog, logSqlAndParams } = require('./TestUtil'); + +describe('ClickHouse JoinGraph', () => { + const { compiler, joinGraph, cubeEvaluator, transformer } = prepareCompiler(` + const perVisitorRevenueMeasure = { + type: 'number', + sql: new Function('visitor_revenue', 'visitor_count', 'return visitor_revenue + "/" + visitor_count') + } + + cube(\`visitors\`, { + sql: \` + select * from visitors WHERE \${USER_CONTEXT.source.filter('source')} AND + \${USER_CONTEXT.sourceArray.filter(sourceArray => \`source in (\${sourceArray.join(',')})\`)} + \`, + + sqlAlias: 'visitors_table', + + refreshKey: { + sql: 'SELECT 1', + }, + + joins: { + visitor_checkins: { + relationship: 'hasMany', + sql: \`\${CUBE}.id = \${visitor_checkins}.visitor_id\` + } + }, + + measures: { + visitor_count: { + type: 'number', + sql: \`count(*)\`, + aliases: ['users count'] + }, + visitor_revenue: { + type: 'sum', + sql: 'amount', + filters: [{ + sql: \`\${CUBE}.source = 'some'\` + }] + }, + per_visitor_revenue: perVisitorRevenueMeasure, + revenueRunning: { + type: 'runningTotal', + sql: 'amount' + }, + revenueRolling: { + type: 'sum', + sql: 'amount', + rollingWindow: { + trailing: '2 day', + offset: 'start' + } + }, + revenueRolling3day: { + type: 'sum', + sql: 'amount', + rollingWindow: { + trailing: '3 day', + offset: 'start' + } + }, + countRolling: { + type: 'count', + rollingWindow: { + trailing: '2 day', + offset: 'start' + } + }, + countDistinctApproxRolling: { + type: 'countDistinctApprox', + sql: 'id', + rollingWindow: { + trailing: '2 day', + offset: 'start' + } + }, + runningCount: { + type: 'runningTotal', + sql: '1' + }, + runningRevenuePerCount: { + type: 'number', + sql: \`round(\${revenueRunning} / \${runningCount})\` + }, + averageCheckins: { + type: 'avg', + sql: \`\${doubledCheckings}\` + } + }, + + dimensions: { + id: { + type: 'number', + sql: 'id', + primaryKey: true + }, + source: { + type: 'string', + sql: 'source' + }, + created_at: { + type: 'time', + sql: 'created_at' + }, + + createdAtSqlUtils: { + type: 'time', + sql: SQL_UTILS.convertTz('created_at') + }, + + checkins: { + sql: \`\${visitor_checkins.visitor_checkins_count}\`, + type: \`number\`, + subQuery: true + }, + + subQueryFail: { + sql: '2', + type: \`number\`, + subQuery: true + }, + + doubledCheckings: { + sql: \`\${checkins} * 2\`, + type: 'number' + }, + minVisitorCheckinDate: { + sql: \`\${visitor_checkins.minDate}\`, + type: 'time', + subQuery: true + }, + minVisitorCheckinDate1: { + sql: \`\${visitor_checkins.minDate1}\`, + type: 'time', + subQuery: true + }, + location: { + type: \`geo\`, + latitude: { sql: \`latitude\` }, + longitude: { sql: \`longitude\` } + } + } + }) + + cube('visitor_checkins', { + sql: \` + select * from visitor_checkins + \`, + + joins: { + cards: { + relationship: 'hasMany', + sql: \`\${CUBE}.id = \${cards}.visitor_checkin_id\` + } + }, + + measures: { + visitor_checkins_count: { + type: 'count' + }, + revenue_per_checkin: { + type: 'number', + sql: \`\${visitors.visitor_revenue} / \${visitor_checkins_count}\` + }, + google_sourced_checkins: { + type: 'count', + sql: 'id', + filters: [{ + sql: \`\${visitors}.source = 'google'\` + }] + }, + minDate: { + type: 'min', + sql: 'created_at' + } + }, + + dimensions: { + id: { + type: 'number', + sql: 'id', + primaryKey: true + }, + visitor_id: { + type: 'number', + sql: 'visitor_id' + }, + source: { + type: 'string', + sql: 'source' + }, + created_at: { + type: 'time', + sql: 'created_at' + }, + cardsCount: { + sql: \`\${cards.count}\`, + type: \`number\`, + subQuery: true + }, + }, + + // preAggregations: { + // checkinSource: { + // type: 'rollup', + // measureReferences: [visitors.per_visitor_revenue], + // dimensionReferences: [visitor_checkins.source], + // timeDimensionReference: visitors.created_at, + // granularity: 'day' + // }, + // visitorCountCheckinSource: { + // type: 'rollup', + // measureReferences: [visitors.visitor_revenue], + // dimensionReferences: [visitor_checkins.source], + // timeDimensionReference: visitors.created_at, + // granularity: 'day' + // } + // } + }) + + cube('cards', { + sql: \` + select * from cards + \`, + + joins: { + visitors: { + relationship: 'belongsTo', + sql: \`\${visitors}.id = \${cards}.visitor_id\` + } + }, + + measures: { + count: { + type: 'count' + } + }, + + dimensions: { + id: { + type: 'number', + sql: 'id', + primaryKey: true + } + } + }) + + cube('ReferenceVisitors', { + sql: \` + select * from \${visitors.sql()} as t + WHERE \${FILTER_PARAMS.ReferenceVisitors.createdAt.filter(\`addDays(t.created_at, 28)\`)} AND + \${FILTER_PARAMS.ReferenceVisitors.createdAt.filter((from, to) => \`(addDays(t.created_at,28)) >= parseDateTimeBestEffort(\${from}) AND (addDays(t.created_at, 28)) <= parseDateTimeBestEffort(\${to})\`)} + \`, + + measures: { + count: { + type: 'count' + }, + + googleSourcedCount: { + type: 'count', + filters: [{ + sql: \`\${CUBE}.source = 'google'\` + }] + }, + }, + + dimensions: { + createdAt: { + type: 'time', + sql: 'created_at' + } + } + }) + `); + + // FAILS - ClickHouse doesn't support OR in JOIN expressions + it.skip('simple join', () => { + const result = compiler.compile().then(() => { + debugLog(joinGraph.buildJoin(['visitor_checkins', 'visitors'])); + + const query = dbRunner.newQuery({ joinGraph, cubeEvaluator, compiler }, { + measures: [ + 'visitors.visitor_revenue', + 'visitors.visitor_count', + 'visitor_checkins.visitor_checkins_count', + 'visitors.per_visitor_revenue' + ], + timeDimensions: [{ + dimension: 'visitors.created_at', + granularity: 'date', + dateRange: ['2017-01-01', '2017-01-30'] + }], + timezone: 'America/Los_Angeles', + order: [{ + id: 'visitors.created_at' + }] + }); + + logSqlAndParams(query) + + return dbRunner.testQuery(query.buildSqlAndParams()).then(res => { + res.should.be.deepEqual( + [ + { + "visitors.created_at_date": "2017-01-02T00:00:00.000Z", + "visitors.visitor_revenue": "100", + "visitors.visitor_count": "1", + "visitor_checkins.visitor_checkins_count": "3", + "visitors.per_visitor_revenue": "100" + }, + { + "visitors.created_at_date": "2017-01-04T00:00:00.000Z", + "visitors.visitor_revenue": "200", + "visitors.visitor_count": "1", + "visitor_checkins.visitor_checkins_count": "2", + "visitors.per_visitor_revenue": "200" + }, + { + "visitors.created_at_date": "2017-01-05T00:00:00.000Z", + "visitors.visitor_revenue": null, + "visitors.visitor_count": "1", + "visitor_checkins.visitor_checkins_count": "1", + "visitors.per_visitor_revenue": null + }, + { + "visitors.created_at_date": "2017-01-06T00:00:00.000Z", + "visitors.visitor_revenue": null, + "visitors.visitor_count": "2", + "visitor_checkins.visitor_checkins_count": "0", + "visitors.per_visitor_revenue": null + } + ] + ); + }); + }); + + return result; + }); + + it('simple join total', () => + runQueryTest({ + measures: [ + 'visitors.visitor_revenue', + 'visitors.visitor_count', + 'visitor_checkins.visitor_checkins_count', + 'visitors.per_visitor_revenue' + ], + timeDimensions: [{ + dimension: 'visitors.created_at', + dateRange: ['2017-01-01', '2017-01-30'] + }], + timezone: 'America/Los_Angeles', + order: [] + }, [{ + "visitors.visitor_revenue": "300", + "visitors.visitor_count": "5", + "visitor_checkins.visitor_checkins_count": "6", + "visitors.per_visitor_revenue": "60" + }]) + ); + + // FAILS - need to finish query to override ::timestamptz + it.skip('running total', () => { + const result = compiler.compile().then(() => { + let query = dbRunner.newQuery({ joinGraph, cubeEvaluator, compiler }, { + measures: [ + 'visitors.revenueRunning' + ], + timeDimensions: [{ + dimension: 'visitors.created_at', + granularity: 'date', + dateRange: ['2017-01-01', '2017-01-10'] + }], + order: [{ + id: 'visitors.created_at' + }], + timezone: 'America/Los_Angeles' + }); + + logSqlAndParams(query) + + // TODO ordering doesn't work for running total + return dbRunner.testQuery(query.buildSqlAndParams()).then(res => { + debugLog(JSON.stringify(res)); + res.should.be.deepEqual( + [{ + "visitors.created_at_date": "2017-01-01T00:00:00.000Z", + "visitors.revenue_running": null + }, { + "visitors.created_at_date": "2017-01-02T00:00:00.000Z", + "visitors.revenue_running": "100" + }, { + "visitors.created_at_date": "2017-01-03T00:00:00.000Z", + "visitors.revenue_running": "100" + }, { + "visitors.created_at_date": "2017-01-04T00:00:00.000Z", + "visitors.revenue_running": "300" + }, { + "visitors.created_at_date": "2017-01-05T00:00:00.000Z", + "visitors.revenue_running": "600" + }, { + "visitors.created_at_date": "2017-01-06T00:00:00.000Z", + "visitors.revenue_running": "1500" + }, { + "visitors.created_at_date": "2017-01-07T00:00:00.000Z", + "visitors.revenue_running": "1500" + }, { + "visitors.created_at_date": "2017-01-08T00:00:00.000Z", + "visitors.revenue_running": "1500" + }, { + "visitors.created_at_date": "2017-01-09T00:00:00.000Z", + "visitors.revenue_running": "1500" + }, { + "visitors.created_at_date": "2017-01-10T00:00:00.000Z", + "visitors.revenue_running": "1500" + }] + ); + }); + }); + + return result; + }); + + // FAILS - need to finish query to override ::timestamptz + it.skip('rolling', () => + runQueryTest({ + measures: [ + 'visitors.revenueRolling' + ], + timeDimensions: [{ + dimension: 'visitors.created_at', + granularity: 'date', + dateRange: ['2017-01-01', '2017-01-10'] + }], + order: [{ + id: 'visitors.created_at' + }], + timezone: 'America/Los_Angeles' + }, [ + { "visitors.created_at_date": "2017-01-01T00:00:00.000Z", "visitors.revenue_rolling": null }, + { "visitors.created_at_date": "2017-01-02T00:00:00.000Z", "visitors.revenue_rolling": null }, + { "visitors.created_at_date": "2017-01-03T00:00:00.000Z", "visitors.revenue_rolling": "100" }, + { "visitors.created_at_date": "2017-01-04T00:00:00.000Z", "visitors.revenue_rolling": "100" }, + { "visitors.created_at_date": "2017-01-05T00:00:00.000Z", "visitors.revenue_rolling": "200" }, + { "visitors.created_at_date": "2017-01-06T00:00:00.000Z", "visitors.revenue_rolling": "500" }, + { "visitors.created_at_date": "2017-01-07T00:00:00.000Z", "visitors.revenue_rolling": "1200" }, + { "visitors.created_at_date": "2017-01-08T00:00:00.000Z", "visitors.revenue_rolling": "900" }, + { "visitors.created_at_date": "2017-01-09T00:00:00.000Z", "visitors.revenue_rolling": null }, + { "visitors.created_at_date": "2017-01-10T00:00:00.000Z", "visitors.revenue_rolling": null } + ]) + ); + + // FAILS - need to finish query to override ::timestamptz + it.skip('rolling multiplied', () => + runQueryTest({ + measures: [ + 'visitors.revenueRolling', + 'visitor_checkins.visitor_checkins_count' + ], + timeDimensions: [{ + dimension: 'visitors.created_at', + granularity: 'date', + dateRange: ['2017-01-01', '2017-01-10'] + }], + order: [{ + id: 'visitors.created_at' + }], + timezone: 'America/Los_Angeles' + }, [ + { + "visitors.created_at_date": "2017-01-02T00:00:00.000Z", "visitors.revenue_rolling": null, + "visitor_checkins.visitor_checkins_count": "3" + }, + { + "visitors.created_at_date": "2017-01-04T00:00:00.000Z", "visitors.revenue_rolling": "100", + 'visitor_checkins.visitor_checkins_count': '2' + }, + { + "visitors.created_at_date": "2017-01-05T00:00:00.000Z", "visitors.revenue_rolling": "200", + 'visitor_checkins.visitor_checkins_count': '1' + }, + { + "visitors.created_at_date": "2017-01-06T00:00:00.000Z", "visitors.revenue_rolling": "500", + 'visitor_checkins.visitor_checkins_count': '0' + } + ]) + ); + + // FAILS - Syntax error: failed at position 107 + it.skip('rolling month', () => + runQueryTest({ + measures: [ + 'visitors.revenueRolling3day' + ], + timeDimensions: [{ + dimension: 'visitors.created_at', + granularity: 'week', + dateRange: ['2017-01-09', '2017-01-10'] + }], + order: [{ + id: 'visitors.created_at' + }], + timezone: 'America/Los_Angeles' + }, [ + { "visitors.created_at_week": "2017-01-09T00:00:00.000Z", "visitors.revenue_rolling3day": "900" } + ]) + ); + + // FAILS - Syntax error: failed at position 249 + it.skip('rolling count', () => + runQueryTest({ + measures: [ + 'visitors.countRolling' + ], + timeDimensions: [{ + dimension: 'visitors.created_at', + granularity: 'date', + dateRange: ['2017-01-01', '2017-01-10'] + }], + order: [{ + id: 'visitors.created_at' + }], + timezone: 'America/Los_Angeles' + }, [ + { "visitors.created_at_date": "2017-01-01T00:00:00.000Z", "visitors.count_rolling": null }, + { "visitors.created_at_date": "2017-01-02T00:00:00.000Z", "visitors.count_rolling": null }, + { "visitors.created_at_date": "2017-01-03T00:00:00.000Z", "visitors.count_rolling": "1" }, + { "visitors.created_at_date": "2017-01-04T00:00:00.000Z", "visitors.count_rolling": "1" }, + { "visitors.created_at_date": "2017-01-05T00:00:00.000Z", "visitors.count_rolling": "1" }, + { "visitors.created_at_date": "2017-01-06T00:00:00.000Z", "visitors.count_rolling": "2" }, + { "visitors.created_at_date": "2017-01-07T00:00:00.000Z", "visitors.count_rolling": "3" }, + { "visitors.created_at_date": "2017-01-08T00:00:00.000Z", "visitors.count_rolling": "2" }, + { "visitors.created_at_date": "2017-01-09T00:00:00.000Z", "visitors.count_rolling": null }, + { "visitors.created_at_date": "2017-01-10T00:00:00.000Z", "visitors.count_rolling": null } + ]) + ); + + it('sql utils', () => + runQueryTest({ + measures: [ + 'visitors.visitor_count' + ], + timeDimensions: [{ + dimension: 'visitors.createdAtSqlUtils', + granularity: 'date', + dateRange: ['2017-01-01', '2017-01-10'] + }], + order: [{ + id: 'visitors.createdAtSqlUtils' + }], + timezone: 'America/Los_Angeles' + }, [ + {"visitors.created_at_sql_utils_date":"2017-01-02T00:00:00.000","visitors.visitor_count":"1"}, + {"visitors.created_at_sql_utils_date":"2017-01-04T00:00:00.000","visitors.visitor_count":"1"}, + {"visitors.created_at_sql_utils_date":"2017-01-05T00:00:00.000","visitors.visitor_count":"1"}, + {"visitors.created_at_sql_utils_date":"2017-01-06T00:00:00.000","visitors.visitor_count":"2"} + ]) + ); + + it('running total total', () => + runQueryTest({ + measures: [ + 'visitors.revenueRunning' + ], + timeDimensions: [{ + dimension: 'visitors.created_at', + dateRange: ['2017-01-01', '2017-01-10'] + }], + order: [{ + id: 'visitors.created_at' + }], + timezone: 'America/Los_Angeles' + }, [ + { + "visitors.revenue_running": "1500" + } + ]) + ); + + // FAILS Unmatched parentheses + it.skip('running total ratio', () => + runQueryTest({ + measures: [ + 'visitors.runningRevenuePerCount' + ], + timeDimensions: [{ + dimension: 'visitors.created_at', + granularity: 'date', + dateRange: ['2017-01-01', '2017-01-10'] + }], + order: [{ + id: 'visitors.created_at' + }], + timezone: 'America/Los_Angeles' + }, [ + { "visitors.created_at_date": "2017-01-01T00:00:00.000Z", "visitors.running_revenue_per_count": null }, + { "visitors.created_at_date": "2017-01-02T00:00:00.000Z", "visitors.running_revenue_per_count": "100" }, + { "visitors.created_at_date": "2017-01-03T00:00:00.000Z", "visitors.running_revenue_per_count": "100" }, + { "visitors.created_at_date": "2017-01-04T00:00:00.000Z", "visitors.running_revenue_per_count": "150" }, + { "visitors.created_at_date": "2017-01-05T00:00:00.000Z", "visitors.running_revenue_per_count": "200" }, + { "visitors.created_at_date": "2017-01-06T00:00:00.000Z", "visitors.running_revenue_per_count": "300" }, + { "visitors.created_at_date": "2017-01-07T00:00:00.000Z", "visitors.running_revenue_per_count": "300" }, + { "visitors.created_at_date": "2017-01-08T00:00:00.000Z", "visitors.running_revenue_per_count": "300" }, + { "visitors.created_at_date": "2017-01-09T00:00:00.000Z", "visitors.running_revenue_per_count": "300" }, + { "visitors.created_at_date": "2017-01-10T00:00:00.000Z", "visitors.running_revenue_per_count": "300" } + ]) + ); + + // FAILS ClickHouse supports multiple approximate aggregators : uniq, uniqCombined, uniqHLL12, need to pick one to use and implement it in query + it.skip('hll rolling', () => { + const result = compiler.compile().then(() => { + let query = dbRunner.newQuery({ joinGraph, cubeEvaluator, compiler }, { + measures: [ + 'visitors.countDistinctApproxRolling' + ], + timeDimensions: [{ + dimension: 'visitors.created_at', + granularity: 'date', + dateRange: ['2017-01-01', '2017-01-10'] + }], + order: [{ + id: 'visitors.created_at' + }], + timezone: 'America/Los_Angeles' + }); + + logSqlAndParams(query) + + query.buildSqlAndParams()[0].should.match(/HLL_COUNT\.MERGE/); + query.buildSqlAndParams()[0].should.match(/HLL_COUNT\.INIT/); + }); + + return result; + }); + + it('calculated join', () => { + const result = compiler.compile().then(() => { + let query = dbRunner.newQuery({ joinGraph, cubeEvaluator, compiler }, { + measures: [ + 'visitor_checkins.revenue_per_checkin' + ], + timeDimensions: [], + timezone: 'America/Los_Angeles' + }); + + logSqlAndParams(query) + + return dbRunner.testQuery(query.buildSqlAndParams()).then(res => { + debugLog(JSON.stringify(res)); + res.should.be.deepEqual( + [{ "visitor_checkins.revenue_per_checkin": "50" }] + ); + }); + }); + + return result; + }); + + it('filter join', () => { + const result = compiler.compile().then(() => { + let query = dbRunner.newQuery({ joinGraph, cubeEvaluator, compiler }, { + measures: [ + 'visitor_checkins.google_sourced_checkins' + ], + timeDimensions: [], + timezone: 'America/Los_Angeles' + }); + + logSqlAndParams(query) + + return dbRunner.testQuery(query.buildSqlAndParams()).then(res => { + debugLog(JSON.stringify(res)); + res.should.be.deepEqual( + [{ "visitor_checkins.google_sourced_checkins": "1" }] + ); + }); + }); + + return result; + }); + + it('filter join not multiplied', () => { + const result = compiler.compile().then(() => { + let query = dbRunner.newQuery({ joinGraph, cubeEvaluator, compiler }, { + measures: [ + 'visitor_checkins.google_sourced_checkins' + ], + timeDimensions: [], + filters: [ + { dimension: 'cards.id', operator: 'equals', values: [3] } // must be number + ], + timezone: 'America/Los_Angeles' + }); + + logSqlAndParams(query) + + return dbRunner.testQuery(query.buildSqlAndParams()).then(res => { + debugLog(JSON.stringify(res)); + res.should.be.deepEqual( + [{ "visitor_checkins.google_sourced_checkins": "1" }] + ); + }); + }); + + return result; + }); + + it('having filter', () => { + return compiler.compile().then(() => { + const query = dbRunner.newQuery({ joinGraph, cubeEvaluator, compiler }, { + measures: [ + 'visitors.visitor_count' + ], + dimensions: [ + 'visitors.source' + ], + timeDimensions: [], + timezone: 'America/Los_Angeles', + filters: [{ + dimension: 'visitors.visitor_count', + operator: 'gt', + values: [1] // must be a number + }], + order: [{ + id: 'visitors.source' + }] + }); + + logSqlAndParams(query) + + return dbRunner.testQuery(query.buildSqlAndParams()).then(res => { + debugLog(JSON.stringify(res)); + res.should.be.deepEqual( + [{ + "visitors.source": "some", + "visitors.visitor_count": "2" + },{ + "visitors.source": null, + "visitors.visitor_count": "3" + }] + ); + }); + }); + }); + + it('having filter without measure', () => + compiler.compile().then(() => { + const query = dbRunner.newQuery({ joinGraph, cubeEvaluator, compiler }, { + measures: [], + dimensions: [ + 'visitors.source' + ], + timeDimensions: [], + timezone: 'America/Los_Angeles', + filters: [{ + dimension: 'visitors.visitor_count', + operator: 'gt', + values: [1] // must be a number + }], + order: [{ + id: 'visitors.source' + }] + }); + + logSqlAndParams(query); + + return dbRunner.testQuery(query.buildSqlAndParams()).then(res => { + debugLog(JSON.stringify(res)); + res.should.be.deepEqual( + [ + { + "visitors.source": "some" + }, + { + "visitors.source": null + }, + ] + ); + }); + }) + ); + + // FAILS - doesnt support OR in JOIN + it.skip('having filter without measure with join', () => + compiler.compile().then(() => { + const query = dbRunner.newQuery({ joinGraph, cubeEvaluator, compiler }, { + measures: [], + dimensions: [ + 'visitors.source' + ], + timeDimensions: [], + timezone: 'America/Los_Angeles', + filters: [{ + dimension: 'visitor_checkins.revenue_per_checkin', + operator: 'gte', + values: [60] // must be a number + }], + order: [{ + id: 'visitors.source' + }] + }); + + logSqlAndParams(query) + + return dbRunner.testQuery(query.buildSqlAndParams()).then(res => { + debugLog(JSON.stringify(res)); + res.should.be.deepEqual( + [{ + "visitors.source": "some" + }] + ); + }); + }) + ); + + it('having filter without measure single multiplied', () => + compiler.compile().then(() => { + const query = dbRunner.newQuery({ joinGraph, cubeEvaluator, compiler }, { + measures: [], + dimensions: [ + 'visitors.source' + ], + timeDimensions: [], + timezone: 'America/Los_Angeles', + filters: [{ + dimension: 'visitors.visitor_revenue', + operator: 'gte', + values: [1] // must be a number + }, { + dimension: 'visitor_checkins.source', + operator: 'equals', + values: ['google'] + }], + order: [{ + id: 'visitors.source' + }] + }); + + logSqlAndParams(query) + + return dbRunner.testQuery(query.buildSqlAndParams()).then(res => { + debugLog(JSON.stringify(res)); + res.should.be.deepEqual( + [{ + "visitors.source": "some" + }] + ); + }); + }) + ); + + it('subquery', () => { + const result = compiler.compile().then(() => { + let query = dbRunner.newQuery({ joinGraph, cubeEvaluator, compiler }, { + measures: [ + 'visitors.visitor_count' + ], + dimensions: [ + 'visitors.checkins' + ], + timeDimensions: [{ + dimension: 'visitors.created_at', + granularity: 'date', + dateRange: ['2017-01-01', '2017-01-30'] + }], + timezone: 'America/Los_Angeles', + filters: [], + order: [{ + id: 'visitors.checkins' + }] + }); + + logSqlAndParams(query) + + return dbRunner.testQuery(query.buildSqlAndParams()).then(res => { + debugLog(JSON.stringify(res)); + res.should.be.deepEqual( + [{ + "visitors.checkins": "0", + "visitors.created_at_date": "2017-01-06T00:00:00.000", + "visitors.visitor_count": "2" + }, { + "visitors.checkins": "1", + "visitors.created_at_date": "2017-01-05T00:00:00.000", + "visitors.visitor_count": "1" + }, { + "visitors.checkins": "2", + "visitors.created_at_date": "2017-01-04T00:00:00.000", + "visitors.visitor_count": "1" + }, { + "visitors.checkins": "3", + "visitors.created_at_date": "2017-01-02T00:00:00.000", + "visitors.visitor_count": "1" + }] + ); + }); + }); + + return result; + }); + + // ClickHouse does NOT support correlated subqueries + // the SQL will have to be re-written to use array functions + // FAILS Error: Unknown identifier: visitors.created_at_date + it.skip('average subquery', () => { + const result = compiler.compile().then(() => { + let query = dbRunner.newQuery({ joinGraph, cubeEvaluator, compiler }, { + measures: [ + 'visitors.averageCheckins' + ], + timeDimensions: [{ + dimension: 'visitors.created_at', + granularity: 'date', + dateRange: ['2017-01-01', '2017-01-30'] + }], + timezone: 'America/Los_Angeles', + filters: [{ + dimension: 'visitor_checkins.source', + operator: 'equals', + values: ['google'] + }], + order: [{ + id: 'visitors.averageCheckins' + }] + }); + + logSqlAndParams(query) + + return dbRunner.testQuery(query.buildSqlAndParams()).then(res => { + debugLog(JSON.stringify(res)); + res.should.be.deepEqual( + [{ "visitors.created_at_date": "2017-01-02T00:00:00.000Z", "visitors.average_checkins": "6.0000000000000000" }] + ); + }); + }); + + return result; + }); + + function runQueryTest(q, expectedResult) { + return compiler.compile().then(() => { + const query = dbRunner.newQuery({ joinGraph, cubeEvaluator, compiler }, q); + + logSqlAndParams(query) + + return dbRunner.testQuery(query.buildSqlAndParams()).then(res => { + debugLog(JSON.stringify(res)); + res.should.be.deepEqual( + expectedResult + ); + }); + }); + } + + it('subquery without measure', () => + runQueryTest({ + dimensions: [ + 'visitors.subQueryFail' + ], + timeDimensions: [{ + dimension: 'visitors.created_at', + granularity: 'date', + dateRange: ['2017-01-01', '2017-01-30'] + }], + timezone: 'America/Los_Angeles', + order: [{ + id: 'visitors.created_at' + }] + }, [ + { + "visitors.min_visitor_checkin_date_date": "2017-01-02T00:00:00.000Z", + "visitors.visitor_count": "1" + }, + { + "visitors.min_visitor_checkin_date_date": "2017-01-04T00:00:00.000Z", + "visitors.visitor_count": "1" + }, + { + "visitors.min_visitor_checkin_date_date": "2017-01-05T00:00:00.000Z", + "visitors.visitor_count": "1" + } + ]).then(() => { + throw new Error(); + }).catch((error) => { + error.should.be.instanceof(UserError); + }) + ); + + it('min date subquery', () => + runQueryTest({ + measures: [ + 'visitors.visitor_count' + ], + timeDimensions: [{ + dimension: 'visitors.minVisitorCheckinDate', + granularity: 'date', + dateRange: ['2017-01-01', '2017-01-30'] + }], + timezone: 'America/Los_Angeles', + order: [{ + id: 'visitors.minVisitorCheckinDate' + }] + }, [ + { + "visitors.min_visitor_checkin_date_date": "2017-01-02T00:00:00.000", + "visitors.visitor_count": "1" + }, + { + "visitors.min_visitor_checkin_date_date": "2017-01-04T00:00:00.000", + "visitors.visitor_count": "1" + }, + { + "visitors.min_visitor_checkin_date_date": "2017-01-05T00:00:00.000", + "visitors.visitor_count": "1" + } + ]) + ); + + it('min date subquery with error', () => + runQueryTest({ + measures: [ + 'visitors.visitor_count' + ], + timeDimensions: [{ + dimension: 'visitors.minVisitorCheckinDate1', + granularity: 'date', + dateRange: ['2017-01-01', '2017-01-30'] + }], + timezone: 'America/Los_Angeles', + order: [{ + id: 'visitors.minVisitorCheckinDate1' + }] + }, []).catch((error) => { + error.should.be.instanceof(UserError); + }) + ); + + it('subquery dimension with join', () => + runQueryTest({ + measures: [ + 'visitors.visitor_revenue' + ], + dimensions: ['visitor_checkins.cardsCount'], + timezone: 'America/Los_Angeles', + order: [{ + id: 'visitor_checkins.cardsCount' + }] + }, [ + { + "visitor_checkins.cards_count": "0", + "visitors.visitor_revenue": "300" + }, + { + "visitor_checkins.cards_count": "1", + "visitors.visitor_revenue": "100" + }, + { + "visitor_checkins.cards_count": null, + "visitors.visitor_revenue": null + } + ]) + ); + + // TODO + it.skip('join rollup pre-aggregation', () => { + return compiler.compile().then(() => { + const query = dbRunner.newQuery({ joinGraph, cubeEvaluator, compiler }, { + measures: [ + 'visitors.per_visitor_revenue' + ], + dimensions: ['visitor_checkins.source'], + timeDimensions: [{ + dimension: 'visitors.created_at', + granularity: 'date', + dateRange: ['2017-01-01', '2017-01-30'] + }], + timezone: 'America/Los_Angeles', + order: [{ + id: 'visitors.created_at' + }], + filters: [{ + dimension: 'visitors.per_visitor_revenue', + operator: 'gt', + values: ['50'] + }, { + dimension: 'visitor_checkins.source', + operator: 'equals', + values: ['google'] + }], + preAggregationsSchema: '' + }); + + logSqlAndParams(query) + + const preAggregationsDescription = query.preAggregations.preAggregationsDescription()[0]; + debugLog(preAggregationsDescription); + + return dbRunner.testQueries(preAggregationsDescription.invalidateKeyQueries.concat([ + [preAggregationsDescription.loadSql[0].replace('CREATE TABLE', 'CREATE TEMP TABLE'), preAggregationsDescription.loadSql[1]], + query.buildSqlAndParams() + ])).then(res => { + debugLog(JSON.stringify(res)); + res.should.be.deepEqual( + [ + { + "visitor_checkins.source": "google", + "visitors.created_at_date": "2017-01-02T00:00:00.000Z", + "visitors.per_visitor_revenue": "100" + } + ] + ); + }); + }); + }); + + // TODO + it.skip('join rollup total pre-aggregation', () => { + return compiler.compile().then(() => { + const query = dbRunner.newQuery({ joinGraph, cubeEvaluator, compiler }, { + measures: [ + 'visitors.visitor_revenue' + ], + timeDimensions: [{ + dimension: 'visitors.created_at', + dateRange: ['2017-01-01', '2017-01-30'] + }], + timezone: 'America/Los_Angeles', + order: [], + filters: [{ + dimension: 'visitor_checkins.source', + operator: 'equals', + values: ['google'] + }], + preAggregationsSchema: '' + }); + + logSqlAndParams(query) + + const preAggregationsDescription = query.preAggregations.preAggregationsDescription()[0]; + debugLog(preAggregationsDescription); + + return dbRunner.testQueries(preAggregationsDescription.invalidateKeyQueries.concat([ + [ + preAggregationsDescription.loadSql[0].replace('CREATE TABLE', 'CREATE TEMP TABLE'), + preAggregationsDescription.loadSql[1] + ], + query.buildSqlAndParams() + ])).then(res => { + debugLog(JSON.stringify(res)); + res.should.be.deepEqual( + [ + { + "visitors.visitor_revenue": "100" + } + ] + ); + }); + }); + }); + + it('user context', () => { + return compiler.compile().then(() => { + let query = dbRunner.newQuery({ joinGraph, cubeEvaluator, compiler }, { + measures: [ + 'visitor_checkins.revenue_per_checkin' + ], + timeDimensions: [], + timezone: 'America/Los_Angeles', + contextSymbols: { + userContext: { source: 'some' } + } + }); + + logSqlAndParams(query) + + return dbRunner.testQuery(query.buildSqlAndParams()).then(res => { + debugLog(JSON.stringify(res)); + res.should.be.deepEqual( + [{ "visitor_checkins.revenue_per_checkin": "60" }] + ); + }); + }); + }); + + it('user context array', () => compiler.compile().then(() => { + const query = dbRunner.newQuery({ joinGraph, cubeEvaluator, compiler }, { + measures: [ + 'visitor_checkins.revenue_per_checkin' + ], + timeDimensions: [], + timezone: 'America/Los_Angeles', + contextSymbols: { + userContext: { + sourceArray: ['some', 'google'] + } + } + }); + + logSqlAndParams(query) + + return dbRunner.testQuery(query.buildSqlAndParams()).then(res => { + debugLog(JSON.stringify(res)); + res.should.be.deepEqual( + [{ "visitor_checkins.revenue_per_checkin": "50" }] + ); + }); + })); + + it('reference cube sql', () => + runQueryTest({ + measures: [ + 'ReferenceVisitors.count' + ], + timezone: 'America/Los_Angeles', + order: [], + timeDimensions: [{ + dimension: 'ReferenceVisitors.createdAt', + dateRange: ['2017-01-01', '2017-01-30'] + }], + }, [{ "reference_visitors.count": "1" }]) + ); + + it('Filtered count without primaryKey', () => + runQueryTest({ + measures: [ + 'ReferenceVisitors.googleSourcedCount' + ], + timezone: 'America/Los_Angeles', + order: [], + timeDimensions: [{ + dimension: 'ReferenceVisitors.createdAt', + dateRange: ['2016-12-01', '2017-03-30'] + }], + }, [{"reference_visitors.google_sourced_count":"1"}]) + ); + + it('builds geo dimension', () => + runQueryTest({ + dimensions: [ + 'visitors.location' + ], + timezone: 'America/Los_Angeles', + order: [{ id: 'visitors.location' }], + }, [ + // in ClickHouse float to string omits any trailing zeros after the decimal point + { 'visitors.location': '120.12,10.6' }, + { 'visitors.location': '120.12,40.6' }, + { 'visitors.location': '120.12,58.1' }, + { 'visitors.location': '120.12,58.6' }, + { 'visitors.location': '120.12,70.6' } + ]) + ); + + it('applies measure_filter type filter', () => + runQueryTest({ + dimensions: [ + 'visitors.id' + ], + filters: [{ + dimension: 'visitors.visitor_revenue', + operator: 'measure_filter' + }], + timezone: 'America/Los_Angeles', + order: [{ id: 'visitors.id' }], // was visitors.location which is odd since its not in the select list + }, [ + { 'visitors.id': '1' }, // all numbers are transformed to strings. ClickHouse returns large number types as strings so we normalise that to all numbers as strings + { 'visitors.id': '2' } + ]) + ); + + it( + 'contains filter', + () => runQueryTest({ + measures: [], + dimensions: [ + 'visitors.source' + ], + timeDimensions: [], + timezone: 'America/Los_Angeles', + filters: [{ + dimension: 'visitor_checkins.source', + operator: 'contains', + values: ['goo'] + }], + order: [{ + id: 'visitors.source' + }] + }, [ + { 'visitors.source': 'some' } + ]) + ); + + it('year granularity', () => + runQueryTest({ + measures: [ + 'visitors.visitor_count' + ], + timeDimensions: [{ + dimension: 'visitors.created_at', + granularity: 'year', + dateRange: ['2016-01-09', '2017-01-10'] + }], + order: [{ + id: 'visitors.created_at' + }], + timezone: 'America/Los_Angeles' + }, [ + { + "visitors.created_at_year": "2016-01-01T00:00:00.000", + "visitors.visitor_count": "1" + }, + { + "visitors.created_at_year": "2017-01-01T00:00:00.000", + "visitors.visitor_count": "5" + } + ]) + ); +}); diff --git a/packages/cubejs-schema-compiler/test/TestUtil.js b/packages/cubejs-schema-compiler/test/TestUtil.js new file mode 100644 index 0000000000000..57b177288175a --- /dev/null +++ b/packages/cubejs-schema-compiler/test/TestUtil.js @@ -0,0 +1,15 @@ +const sqlstring = require('sqlstring'); + +exports.logSqlAndParams = function(query) { + let parts = query.buildSqlAndParams() + // debugLog(parts[0]); + // debugLog(parts[1]); + exports.debugLog(sqlstring.format(parts[0], parts[1])); +} + +exports.debugLog = function() { + if (process.env.DEBUG_LOG === 'true') { + console.log.apply(console, [...arguments]) + } +} + \ No newline at end of file diff --git a/packages/cubejs-schema-compiler/yarn.lock b/packages/cubejs-schema-compiler/yarn.lock index c57da48076e0a..76cdcdc991091 100644 --- a/packages/cubejs-schema-compiler/yarn.lock +++ b/packages/cubejs-schema-compiler/yarn.lock @@ -2,6 +2,14 @@ # yarn lockfile v1 +"@apla/clickhouse@^1.5.5": + version "1.5.5" + resolved "https://registry.yarnpkg.com/@apla/clickhouse/-/clickhouse-1.5.5.tgz#45cf5f9b662f123954ed804b5638f203ee8fb67e" + integrity sha512-Px7KczgL6REH3/WiBLiIxmDs3v8evpT/gFaWo/yoTaNs3ch2D70HtbfA8lkZUHO6gKndxLfgpWNFnJ9H+W/wEg== + dependencies: + buffer-indexof-polyfill "^1.0.1" + object-assign "^4.1.0" + acorn-dynamic-import@^4.0.0: version "4.0.0" resolved "https://registry.yarnpkg.com/acorn-dynamic-import/-/acorn-dynamic-import-4.0.0.tgz#482210140582a36b83c3e342e1cfebcaa9240948" @@ -123,6 +131,11 @@ browser-stdout@1.3.0: resolved "https://registry.yarnpkg.com/browser-stdout/-/browser-stdout-1.3.0.tgz#f351d32969d32fa5d7a5567154263d928ae3bd1f" integrity sha1-81HTKWnTL6XXpVZxVCY9korjvR8= +buffer-indexof-polyfill@^1.0.1: + version "1.0.1" + resolved "https://registry.yarnpkg.com/buffer-indexof-polyfill/-/buffer-indexof-polyfill-1.0.1.tgz#a9fb806ce8145d5428510ce72f278bb363a638bf" + integrity sha1-qfuAbOgUXVQoUQznLyeLs2OmOL8= + buffer-writer@1.0.1: version "1.0.1" resolved "https://registry.yarnpkg.com/buffer-writer/-/buffer-writer-1.0.1.tgz#22a936901e3029afcd7547eb4487ceb697a3bf08" @@ -274,10 +287,10 @@ he@1.1.1: resolved "https://registry.yarnpkg.com/he/-/he-1.1.1.tgz#93410fd21b009735151f8868c2f271f3427e23fd" integrity sha1-k0EP0hsAlzUVH4howvJx80J+I/0= -hoek@4.x.x: - version "4.2.1" - resolved "https://registry.yarnpkg.com/hoek/-/hoek-4.2.1.tgz#9634502aa12c445dd5a7c5734b572bb8738aacbb" - integrity sha512-QLg82fGkfnJ/4iy1xZ81/9SIJiq1NGFUMGs6ParyjBZr6jW2Ufj/snDqTHixNlHdPNwN2RLVD0Pi3igeK9+JfA== +hoek@6.x.x: + version "6.1.3" + resolved "https://registry.yarnpkg.com/hoek/-/hoek-6.1.3.tgz#73b7d33952e01fe27a38b0457294b79dd8da242c" + integrity sha512-YXXAAhmF9zpQbC7LEcREFtXfGq5K1fmd+4PHkBq8NUqmzW3G+Dq10bI/i0KucLRwss3YYFQ0fSfoxBZYiGUqtQ== humps@^2.0.1: version "2.0.1" @@ -316,25 +329,21 @@ is-finite@^1.0.0: dependencies: number-is-nan "^1.0.0" -isemail@2.x.x: - version "2.2.1" - resolved "https://registry.yarnpkg.com/isemail/-/isemail-2.2.1.tgz#0353d3d9a62951080c262c2aa0a42b8ea8e9e2a6" - integrity sha1-A1PT2aYpUQgMJiwqoKQrjqjp4qY= - -items@2.x.x: - version "2.1.2" - resolved "https://registry.yarnpkg.com/items/-/items-2.1.2.tgz#0849354595805d586dac98e7e6e85556ea838558" - integrity sha512-kezcEqgB97BGeZZYtX/MA8AG410ptURstvnz5RAgyFZ8wQFPMxHY8GpTq+/ZHKT3frSlIthUq7EvLt9xn3TvXg== +isemail@3.x.x: + version "3.2.0" + resolved "https://registry.yarnpkg.com/isemail/-/isemail-3.2.0.tgz#59310a021931a9fb06bbb51e155ce0b3f236832c" + integrity sha512-zKqkK+O+dGqevc93KNsbZ/TqTUFd46MwWjYOoMrjIMZ51eU7DtQG3Wmd9SQQT7i7RVnuTPEiYEWHU3MSbxC1Tg== + dependencies: + punycode "2.x.x" -joi@^10.6.0: - version "10.6.0" - resolved "https://registry.yarnpkg.com/joi/-/joi-10.6.0.tgz#52587f02d52b8b75cdb0c74f0b164a191a0e1fc2" - integrity sha512-hBF3LcqyAid+9X/pwg+eXjD2QBZI5eXnBFJYaAkH4SK3mp9QSRiiQnDYlmlz5pccMvnLcJRS4whhDOTCkmsAdQ== +joi@^14.3.1: + version "14.3.1" + resolved "https://registry.yarnpkg.com/joi/-/joi-14.3.1.tgz#164a262ec0b855466e0c35eea2a885ae8b6c703c" + integrity sha512-LQDdM+pkOrpAn4Lp+neNIFV3axv1Vna3j38bisbQhETPMANYRbFJFUyOZcOClYvM/hppMhGWuKSFEK9vjrB+bQ== dependencies: - hoek "4.x.x" - isemail "2.x.x" - items "2.x.x" - topo "2.x.x" + hoek "6.x.x" + isemail "3.x.x" + topo "3.x.x" "js-tokens@^3.0.0 || ^4.0.0": version "4.0.0" @@ -505,6 +514,11 @@ number-is-nan@^1.0.0: resolved "https://registry.yarnpkg.com/number-is-nan/-/number-is-nan-1.0.1.tgz#097b602b53422a522c1afb8790318336941a011d" integrity sha1-CXtgK1NCKlIsGvuHkDGDNpQaAR0= +object-assign@^4.1.0: + version "4.1.1" + resolved "https://registry.yarnpkg.com/object-assign/-/object-assign-4.1.1.tgz#2109adc7965887cfc05cbbd442cac8bfbb360863" + integrity sha1-IQmtx5ZYh8/AXLvUQsrIv7s2CGM= + once@^1.3.0: version "1.4.0" resolved "https://registry.yarnpkg.com/once/-/once-1.4.0.tgz#583b1aa775961d4b113ac17d9c50baef9dd76bd1" @@ -599,6 +613,11 @@ postgres-interval@^1.1.0: dependencies: xtend "^4.0.0" +punycode@2.x.x: + version "2.1.1" + resolved "https://registry.yarnpkg.com/punycode/-/punycode-2.1.1.tgz#b58b010ac40c22c5657616c8d2c2c02c7bf479ec" + integrity sha512-XRsRjdf+j5ml+y/6GKHPZbrF/8p2Yga0JPtdqTIY2Xe5ohJPD9saDJJLPvp9+NSBprVvevdXZybnj2cv8OEd0A== + ramda@^0.24.1: version "0.24.1" resolved "https://registry.yarnpkg.com/ramda/-/ramda-0.24.1.tgz#c3b7755197f35b8dc3502228262c4c91ddb6b857" @@ -682,6 +701,11 @@ split@^1.0.0: dependencies: through "2" +sqlstring@^2.3.1: + version "2.3.1" + resolved "https://registry.yarnpkg.com/sqlstring/-/sqlstring-2.3.1.tgz#475393ff9e91479aea62dcaf0ca3d14983a7fb40" + integrity sha1-R1OT/56RR5rqYtyvDKPRSYOn+0A= + strip-ansi@^3.0.0: version "3.0.1" resolved "https://registry.yarnpkg.com/strip-ansi/-/strip-ansi-3.0.1.tgz#6a385fb8853d952d5ff05d0e8aaf94278dc63dcf" @@ -718,18 +742,23 @@ to-fast-properties@^1.0.3: resolved "https://registry.yarnpkg.com/to-fast-properties/-/to-fast-properties-1.0.3.tgz#b83571fa4d8c25b82e231b06e3a3055de4ca1a47" integrity sha1-uDVx+k2MJbguIxsG46MFXeTKGkc= -topo@2.x.x: - version "2.0.2" - resolved "https://registry.yarnpkg.com/topo/-/topo-2.0.2.tgz#cd5615752539057c0dc0491a621c3bc6fbe1d182" - integrity sha1-zVYVdSU5BXwNwEkaYhw7xvvh0YI= +topo@3.x.x: + version "3.0.3" + resolved "https://registry.yarnpkg.com/topo/-/topo-3.0.3.tgz#d5a67fb2e69307ebeeb08402ec2a2a6f5f7ad95c" + integrity sha512-IgpPtvD4kjrJ7CRA3ov2FhWQADwv+Tdqbsf1ZnPUSAtCJ9e1Z44MmoSGDXGk4IppoZA7jd/QRkNddlLJWlUZsQ== dependencies: - hoek "4.x.x" + hoek "6.x.x" trim-right@^1.0.1: version "1.0.1" resolved "https://registry.yarnpkg.com/trim-right/-/trim-right-1.0.1.tgz#cb2e1203067e0c8de1f614094b9fe45704ea6003" integrity sha1-yy4SAwZ+DI3h9hQJS5/kVwTqYAM= +uuid@^3.3.2: + version "3.3.2" + resolved "https://registry.yarnpkg.com/uuid/-/uuid-3.3.2.tgz#1b4af4955eb3077c501c23872fc6513811587131" + integrity sha512-yXJmeNaw3DnnKAOKJE51sL/ZaYfWJRl1pK9dr19YFCu0ObS231AB1/LbqTKRAQ5kw8A90rA6fr4riOUpTZvQZA== + wrappy@1: version "1.0.2" resolved "https://registry.yarnpkg.com/wrappy/-/wrappy-1.0.2.tgz#b5243d8f3ec1aa35f1364605bc0d1036e30ab69f" From efb4410ccd33e8e0bb08b851abefb171669d074e Mon Sep 17 00:00:00 2001 From: Cameron Braid Date: Wed, 1 May 2019 00:24:42 +1000 Subject: [PATCH 2/2] fixed comment --- packages/cubejs-clickhouse-driver/driver/ClickHouseDriver.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/cubejs-clickhouse-driver/driver/ClickHouseDriver.js b/packages/cubejs-clickhouse-driver/driver/ClickHouseDriver.js index da71432c7657f..a9d653ce976b7 100644 --- a/packages/cubejs-clickhouse-driver/driver/ClickHouseDriver.js +++ b/packages/cubejs-clickhouse-driver/driver/ClickHouseDriver.js @@ -97,8 +97,8 @@ class ClickHouseDriver extends BaseDriver { _normaliseResponse(res) { // // - // ClickHouse returns DateTime as strings in format "YYYY-DD-MM HH:MM:SS" in UTC timezone - // cube.js expects them in format "YYYY-DD-MMTHH:MM:SS.000Z", so translate them based on the metadata returned + // ClickHouse returns DateTime as strings in format "YYYY-DD-MM HH:MM:SS" + // cube.js expects them in format "YYYY-DD-MMTHH:MM:SS.000", so translate them based on the metadata returned // // ClickHouse returns some number types as js numbers, others as js string, normalise them all to strings //