From 7f0d2c8c03c3d045ce913543ab33c06c700d9b78 Mon Sep 17 00:00:00 2001 From: Igor Nikolaev Date: Fri, 26 Apr 2019 21:26:21 +0300 Subject: [PATCH 01/11] Initial work on ClickHouse driver --- .../cubejs-clickhouse-driver/CHANGELOG.md | 10 + packages/cubejs-clickhouse-driver/LICENSE | 202 ++++++++++++++++++ packages/cubejs-clickhouse-driver/README.md | 9 + .../driver/ClickHouseDriver.js | 114 ++++++++++ .../cubejs-clickhouse-driver/package.json | 19 ++ packages/cubejs-clickhouse-driver/yarn.lock | 121 +++++++++++ 6 files changed, 475 insertions(+) create mode 100644 packages/cubejs-clickhouse-driver/CHANGELOG.md create mode 100644 packages/cubejs-clickhouse-driver/LICENSE create mode 100644 packages/cubejs-clickhouse-driver/README.md create mode 100644 packages/cubejs-clickhouse-driver/driver/ClickHouseDriver.js create mode 100644 packages/cubejs-clickhouse-driver/package.json create mode 100644 packages/cubejs-clickhouse-driver/yarn.lock diff --git a/packages/cubejs-clickhouse-driver/CHANGELOG.md b/packages/cubejs-clickhouse-driver/CHANGELOG.md new file mode 100644 index 0000000000000..f21b2ef280215 --- /dev/null +++ b/packages/cubejs-clickhouse-driver/CHANGELOG.md @@ -0,0 +1,10 @@ +# Change Log + +All notable changes to this project will be documented in this file. +See [Conventional Commits](https://conventionalcommits.org) for commit guidelines. + +# [0.0.1](https://github.com/statsbotco/cubejs-client/compare/v0.6.2...v0.7.0) (2019-04-15) + +**Note:** Version bump only for package @cubejs-backend/clickhouse-driver + + diff --git a/packages/cubejs-clickhouse-driver/LICENSE b/packages/cubejs-clickhouse-driver/LICENSE new file mode 100644 index 0000000000000..11e5c539bbc34 --- /dev/null +++ b/packages/cubejs-clickhouse-driver/LICENSE @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright 2018-2019 Statsbot, Inc. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. \ No newline at end of file diff --git a/packages/cubejs-clickhouse-driver/README.md b/packages/cubejs-clickhouse-driver/README.md new file mode 100644 index 0000000000000..9df649095f0f5 --- /dev/null +++ b/packages/cubejs-clickhouse-driver/README.md @@ -0,0 +1,9 @@ +# Cube.js ClickHouse Database Driver + +Pure Javascript ClickHouse driver. + +[Learn more](https://github.com/statsbotco/cube.js#getting-started) + +### License + +Cube.js ClickHouse Database Driver is [Apache 2.0 licensed](./LICENSE). \ No newline at end of file diff --git a/packages/cubejs-clickhouse-driver/driver/ClickHouseDriver.js b/packages/cubejs-clickhouse-driver/driver/ClickHouseDriver.js new file mode 100644 index 0000000000000..5598090a9b679 --- /dev/null +++ b/packages/cubejs-clickhouse-driver/driver/ClickHouseDriver.js @@ -0,0 +1,114 @@ +const ClickHouse = require('clickhouse'); +const genericPool = require('generic-pool'); +const BaseDriver = require('@cubejs-backend/query-orchestrator/driver/BaseDriver'); + +class ClickHouseDriver extends BaseDriver { + constructor(config) { + super(); + this.config = { + host: process.env.CUBEJS_DB_HOST, + port: process.env.CUBEJS_DB_PORT, + auth: process.env.CUBEJS_DB_USER + ":" + process.env.CUBEJS_DB_PASS, + queryOptions: { + database: process.env.CUBEJS_DB_NAME + }, + ...config + }; + this.pool = genericPool.createPool({ + create: () => new ClickHouse(this.config), + destroy: (connection) => { + return Promise.resolve(); + }, + validate: async (connection) => { + try { + await connection.querying('SELECT 1'); + } catch (e) { + return false; + } + return true; + } + }, { + min: 0, + max: 8, + evictionRunIntervalMillis: 10000, + softIdleTimeoutMillis: 30000, + idleTimeoutMillis: 30000, + testOnBorrow: true, + acquireTimeoutMillis: 20000 + }); + } + + withConnection(fn) { + const self = this; + const connectionPromise = this.pool.acquire(); + + let cancelled = false; + const cancelObj = {}; + const promise = connectionPromise.then(conn => { + cancelObj.cancel = async () => { + cancelled = true; + await self.withConnection(async conn => { + const processRows = await conn.querying('SHOW PROCESSLIST'); + await Promise.all(processRows.filter(row => row.elapsed >= 599).map(row => { + return conn.execute(`KILL QUERY WHERE query_id = '${row.query_id}'`); + })); + }); + }; + return fn(conn) + .then(res => { + return this.pool.release(conn).then(() => { + if (cancelled) { + throw new Error('Query cancelled'); + } + return res; + }); + }) + .catch((err) => { + return this.pool.release(conn).then(() => { + if (cancelled) { + throw new Error('Query cancelled'); + } + throw err; + }); + }) + }); + promise.cancel = () => cancelObj.cancel(); + return promise; + } + + async testConnection() { + const conn = await this.pool._factory.create(); + try { + return await conn.querying('SELECT 1'); + } finally { + await this.pool._factory.destroy(conn); + } + } + + query(query, values) { + // TODO: handle values + const self = this; + return this.withConnection(connection => { + return connection.querying(query) + .then(res => res); + }); + } + + async release() { + await this.pool.drain(); + await this.pool.clear(); + } + + informationSchemaQuery() { + return ` + SELECT name as column_name, + table as table_name, + database as table_schema, + type as data_type + FROM system.columns + WHERE database = '${this.config.database}' + `; + } +} + +module.exports = ClickHouseDriver; \ No newline at end of file diff --git a/packages/cubejs-clickhouse-driver/package.json b/packages/cubejs-clickhouse-driver/package.json new file mode 100644 index 0000000000000..ca66f914c20aa --- /dev/null +++ b/packages/cubejs-clickhouse-driver/package.json @@ -0,0 +1,19 @@ +{ + "name": "@cubejs-backend/clickhouse-driver", + "description": "Cube.js ClickHouse database driver", + "author": "Statsbot, Inc.", + "version": "0.0.1", + "engines": { + "node": ">=8.11.1" + }, + "main": "driver/ClickHouseDriver.js", + "scripts": { + "test": "mocha" + }, + "dependencies": { + "@cubejs-backend/query-orchestrator": "^0.0.21", + "generic-pool": "^3.6.0", + "@apla/clickhouse": "^1.5.5" + }, + "license": "Apache-2.0" +} diff --git a/packages/cubejs-clickhouse-driver/yarn.lock b/packages/cubejs-clickhouse-driver/yarn.lock new file mode 100644 index 0000000000000..a35aa53a2456a --- /dev/null +++ b/packages/cubejs-clickhouse-driver/yarn.lock @@ -0,0 +1,121 @@ +# THIS IS AN AUTOGENERATED FILE. DO NOT EDIT THIS FILE DIRECTLY. +# yarn lockfile v1 + + +"@cubejs-backend/query-orchestrator@^0.0.21": + version "0.0.21" + resolved "https://registry.yarnpkg.com/@cubejs-backend/query-orchestrator/-/query-orchestrator-0.0.21.tgz#b5a0422a34931a834f4429280679460d08dc6112" + integrity sha512-jQS5F2D9PzKEJ4guzba6S2PymruA5pTIFu4DnOnM7Y3sFQkjVDcl7KYEr5VmpX6+Hu1u+naApuCQo2l6PtOXKg== + dependencies: + ramda "^0.24.1" + redis "^2.8.0" + util-promisifyall "^1.0.4" + +bignumber.js@4.1.0: + version "4.1.0" + resolved "https://registry.yarnpkg.com/bignumber.js/-/bignumber.js-4.1.0.tgz#db6f14067c140bd46624815a7916c92d9b6c24b1" + integrity sha512-eJzYkFYy9L4JzXsbymsFn3p54D+llV27oTQ+ziJG7WFRheJcNZilgVXMG0LoZtlQSKBsJdWtLFqOD0u+U0jZKA== + +core-util-is@~1.0.0: + version "1.0.2" + resolved "https://registry.yarnpkg.com/core-util-is/-/core-util-is-1.0.2.tgz#b5fd54220aa2bc5ab57aab7140c940754503c1a7" + integrity sha1-tf1UIgqivFq1eqtxQMlAdUUDwac= + +double-ended-queue@^2.1.0-0: + version "2.1.0-0" + resolved "https://registry.yarnpkg.com/double-ended-queue/-/double-ended-queue-2.1.0-0.tgz#103d3527fd31528f40188130c841efdd78264e5c" + integrity sha1-ED01J/0xUo9AGIEwyEHv3XgmTlw= + +generic-pool@^3.6.0: + version "3.6.0" + resolved "https://registry.yarnpkg.com/generic-pool/-/generic-pool-3.6.0.tgz#34e3124d9af754c8bfee7c15906e7347dc543afc" + integrity sha512-9tRjIJzgW+ZyYeFBhfLRUefnQwF+IFfEsUkdZB5o75/gb4OZWnYb4ZjPcZc1bl9ofhWCN2rGhXa0SQwF64jTLw== + +inherits@~2.0.3: + version "2.0.3" + resolved "https://registry.yarnpkg.com/inherits/-/inherits-2.0.3.tgz#633c2c83e3da42a502f52466022480f4208261de" + integrity sha1-Yzwsg+PaQqUC9SRmAiSA9CCCYd4= + +isarray@~1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/isarray/-/isarray-1.0.0.tgz#bb935d48582cba168c06834957a54a3e07124f11" + integrity sha1-u5NdSFgsuhaMBoNJV6VKPgcSTxE= + +mysql@^2.16.0: + version "2.16.0" + resolved "https://registry.yarnpkg.com/mysql/-/mysql-2.16.0.tgz#b23b22ab5de44fc2d5d32bd4f5af6653fc45e2ba" + integrity sha512-dPbN2LHonQp7D5ja5DJXNbCLe/HRdu+f3v61aguzNRQIrmZLOeRoymBYyeThrR6ug+FqzDL95Gc9maqZUJS+Gw== + dependencies: + bignumber.js "4.1.0" + readable-stream "2.3.6" + safe-buffer "5.1.2" + sqlstring "2.3.1" + +process-nextick-args@~2.0.0: + version "2.0.0" + resolved "https://registry.yarnpkg.com/process-nextick-args/-/process-nextick-args-2.0.0.tgz#a37d732f4271b4ab1ad070d35508e8290788ffaa" + integrity sha512-MtEC1TqN0EU5nephaJ4rAtThHtC86dNN9qCuEhtshvpVBkAW5ZO7BASN9REnF9eoXGcRub+pFuKEpOHE+HbEMw== + +ramda@^0.24.1: + version "0.24.1" + resolved "https://registry.yarnpkg.com/ramda/-/ramda-0.24.1.tgz#c3b7755197f35b8dc3502228262c4c91ddb6b857" + integrity sha1-w7d1UZfzW43DUCIoJixMkd22uFc= + +readable-stream@2.3.6: + version "2.3.6" + resolved "https://registry.yarnpkg.com/readable-stream/-/readable-stream-2.3.6.tgz#b11c27d88b8ff1fbe070643cf94b0c79ae1b0aaf" + integrity sha512-tQtKA9WIAhBF3+VLAseyMqZeBjW0AHJoxOtYqSUZNJxauErmLbVm2FW1y+J/YA9dUrAC39ITejlZWhVIwawkKw== + dependencies: + core-util-is "~1.0.0" + inherits "~2.0.3" + isarray "~1.0.0" + process-nextick-args "~2.0.0" + safe-buffer "~5.1.1" + string_decoder "~1.1.1" + util-deprecate "~1.0.1" + +redis-commands@^1.2.0: + version "1.4.0" + resolved "https://registry.yarnpkg.com/redis-commands/-/redis-commands-1.4.0.tgz#52f9cf99153efcce56a8f86af986bd04e988602f" + integrity sha512-cu8EF+MtkwI4DLIT0x9P8qNTLFhQD4jLfxLR0cCNkeGzs87FN6879JOJwNQR/1zD7aSYNbU0hgsV9zGY71Itvw== + +redis-parser@^2.6.0: + version "2.6.0" + resolved "https://registry.yarnpkg.com/redis-parser/-/redis-parser-2.6.0.tgz#52ed09dacac108f1a631c07e9b69941e7a19504b" + integrity sha1-Uu0J2srBCPGmMcB+m2mUHnoZUEs= + +redis@^2.8.0: + version "2.8.0" + resolved "https://registry.yarnpkg.com/redis/-/redis-2.8.0.tgz#202288e3f58c49f6079d97af7a10e1303ae14b02" + integrity sha512-M1OkonEQwtRmZv4tEWF2VgpG0JWJ8Fv1PhlgT5+B+uNq2cA3Rt1Yt/ryoR+vQNOQcIEgdCdfH0jr3bDpihAw1A== + dependencies: + double-ended-queue "^2.1.0-0" + redis-commands "^1.2.0" + redis-parser "^2.6.0" + +safe-buffer@5.1.2, safe-buffer@~5.1.0, safe-buffer@~5.1.1: + version "5.1.2" + resolved "https://registry.yarnpkg.com/safe-buffer/-/safe-buffer-5.1.2.tgz#991ec69d296e0313747d59bdfd2b745c35f8828d" + integrity sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g== + +sqlstring@2.3.1: + version "2.3.1" + resolved "https://registry.yarnpkg.com/sqlstring/-/sqlstring-2.3.1.tgz#475393ff9e91479aea62dcaf0ca3d14983a7fb40" + integrity sha1-R1OT/56RR5rqYtyvDKPRSYOn+0A= + +string_decoder@~1.1.1: + version "1.1.1" + resolved "https://registry.yarnpkg.com/string_decoder/-/string_decoder-1.1.1.tgz#9cf1611ba62685d7030ae9e4ba34149c3af03fc8" + integrity sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg== + dependencies: + safe-buffer "~5.1.0" + +util-deprecate@~1.0.1: + version "1.0.2" + resolved "https://registry.yarnpkg.com/util-deprecate/-/util-deprecate-1.0.2.tgz#450d4dc9fa70de732762fbd2d4a28981419a0ccf" + integrity sha1-RQ1Nyfpw3nMnYvvS1KKJgUGaDM8= + +util-promisifyall@^1.0.4: + version "1.0.4" + resolved "https://registry.yarnpkg.com/util-promisifyall/-/util-promisifyall-1.0.4.tgz#82047e87e0637ca799b3480cd6ede3ddeedadf09" + integrity sha512-dgR84x56iVKsSMNo5GGED+xciS1bLS84Wsi5sYHW59Wf/P0mVdJckx2o7vLHrChG94J4c/T9EXoRJSoNy6qSEQ== From 3746e966b97e4130bf84d94ea2161ec2c7078eb0 Mon Sep 17 00:00:00 2001 From: Igor Nikolaev Date: Fri, 26 Apr 2019 22:30:43 +0300 Subject: [PATCH 02/11] Update dependencies --- .../cubejs-clickhouse-driver/package.json | 4 +- packages/cubejs-clickhouse-driver/yarn.lock | 93 ++++--------------- 2 files changed, 19 insertions(+), 78 deletions(-) diff --git a/packages/cubejs-clickhouse-driver/package.json b/packages/cubejs-clickhouse-driver/package.json index ca66f914c20aa..3f48edf980f38 100644 --- a/packages/cubejs-clickhouse-driver/package.json +++ b/packages/cubejs-clickhouse-driver/package.json @@ -11,9 +11,9 @@ "test": "mocha" }, "dependencies": { + "@apla/clickhouse": "^1.5.5", "@cubejs-backend/query-orchestrator": "^0.0.21", - "generic-pool": "^3.6.0", - "@apla/clickhouse": "^1.5.5" + "generic-pool": "^3.6.0" }, "license": "Apache-2.0" } diff --git a/packages/cubejs-clickhouse-driver/yarn.lock b/packages/cubejs-clickhouse-driver/yarn.lock index a35aa53a2456a..c4f78a491adfb 100644 --- a/packages/cubejs-clickhouse-driver/yarn.lock +++ b/packages/cubejs-clickhouse-driver/yarn.lock @@ -2,120 +2,61 @@ # yarn lockfile v1 +"@apla/clickhouse@^1.5.5": + version "1.5.5" + resolved "https://registry.yarnpkg.com/@apla/clickhouse/-/clickhouse-1.5.5.tgz#45cf5f9b662f123954ed804b5638f203ee8fb67e" + dependencies: + buffer-indexof-polyfill "^1.0.1" + object-assign "^4.1.0" + "@cubejs-backend/query-orchestrator@^0.0.21": version "0.0.21" resolved "https://registry.yarnpkg.com/@cubejs-backend/query-orchestrator/-/query-orchestrator-0.0.21.tgz#b5a0422a34931a834f4429280679460d08dc6112" - integrity sha512-jQS5F2D9PzKEJ4guzba6S2PymruA5pTIFu4DnOnM7Y3sFQkjVDcl7KYEr5VmpX6+Hu1u+naApuCQo2l6PtOXKg== dependencies: ramda "^0.24.1" redis "^2.8.0" util-promisifyall "^1.0.4" -bignumber.js@4.1.0: - version "4.1.0" - resolved "https://registry.yarnpkg.com/bignumber.js/-/bignumber.js-4.1.0.tgz#db6f14067c140bd46624815a7916c92d9b6c24b1" - integrity sha512-eJzYkFYy9L4JzXsbymsFn3p54D+llV27oTQ+ziJG7WFRheJcNZilgVXMG0LoZtlQSKBsJdWtLFqOD0u+U0jZKA== - -core-util-is@~1.0.0: - version "1.0.2" - resolved "https://registry.yarnpkg.com/core-util-is/-/core-util-is-1.0.2.tgz#b5fd54220aa2bc5ab57aab7140c940754503c1a7" - integrity sha1-tf1UIgqivFq1eqtxQMlAdUUDwac= +buffer-indexof-polyfill@^1.0.1: + version "1.0.1" + resolved "https://registry.yarnpkg.com/buffer-indexof-polyfill/-/buffer-indexof-polyfill-1.0.1.tgz#a9fb806ce8145d5428510ce72f278bb363a638bf" double-ended-queue@^2.1.0-0: version "2.1.0-0" resolved "https://registry.yarnpkg.com/double-ended-queue/-/double-ended-queue-2.1.0-0.tgz#103d3527fd31528f40188130c841efdd78264e5c" - integrity sha1-ED01J/0xUo9AGIEwyEHv3XgmTlw= generic-pool@^3.6.0: version "3.6.0" resolved "https://registry.yarnpkg.com/generic-pool/-/generic-pool-3.6.0.tgz#34e3124d9af754c8bfee7c15906e7347dc543afc" - integrity sha512-9tRjIJzgW+ZyYeFBhfLRUefnQwF+IFfEsUkdZB5o75/gb4OZWnYb4ZjPcZc1bl9ofhWCN2rGhXa0SQwF64jTLw== - -inherits@~2.0.3: - version "2.0.3" - resolved "https://registry.yarnpkg.com/inherits/-/inherits-2.0.3.tgz#633c2c83e3da42a502f52466022480f4208261de" - integrity sha1-Yzwsg+PaQqUC9SRmAiSA9CCCYd4= -isarray@~1.0.0: - version "1.0.0" - resolved "https://registry.yarnpkg.com/isarray/-/isarray-1.0.0.tgz#bb935d48582cba168c06834957a54a3e07124f11" - integrity sha1-u5NdSFgsuhaMBoNJV6VKPgcSTxE= - -mysql@^2.16.0: - version "2.16.0" - resolved "https://registry.yarnpkg.com/mysql/-/mysql-2.16.0.tgz#b23b22ab5de44fc2d5d32bd4f5af6653fc45e2ba" - integrity sha512-dPbN2LHonQp7D5ja5DJXNbCLe/HRdu+f3v61aguzNRQIrmZLOeRoymBYyeThrR6ug+FqzDL95Gc9maqZUJS+Gw== - dependencies: - bignumber.js "4.1.0" - readable-stream "2.3.6" - safe-buffer "5.1.2" - sqlstring "2.3.1" - -process-nextick-args@~2.0.0: - version "2.0.0" - resolved "https://registry.yarnpkg.com/process-nextick-args/-/process-nextick-args-2.0.0.tgz#a37d732f4271b4ab1ad070d35508e8290788ffaa" - integrity sha512-MtEC1TqN0EU5nephaJ4rAtThHtC86dNN9qCuEhtshvpVBkAW5ZO7BASN9REnF9eoXGcRub+pFuKEpOHE+HbEMw== +object-assign@^4.1.0: + version "4.1.1" + resolved "https://registry.yarnpkg.com/object-assign/-/object-assign-4.1.1.tgz#2109adc7965887cfc05cbbd442cac8bfbb360863" ramda@^0.24.1: version "0.24.1" resolved "https://registry.yarnpkg.com/ramda/-/ramda-0.24.1.tgz#c3b7755197f35b8dc3502228262c4c91ddb6b857" - integrity sha1-w7d1UZfzW43DUCIoJixMkd22uFc= - -readable-stream@2.3.6: - version "2.3.6" - resolved "https://registry.yarnpkg.com/readable-stream/-/readable-stream-2.3.6.tgz#b11c27d88b8ff1fbe070643cf94b0c79ae1b0aaf" - integrity sha512-tQtKA9WIAhBF3+VLAseyMqZeBjW0AHJoxOtYqSUZNJxauErmLbVm2FW1y+J/YA9dUrAC39ITejlZWhVIwawkKw== - dependencies: - core-util-is "~1.0.0" - inherits "~2.0.3" - isarray "~1.0.0" - process-nextick-args "~2.0.0" - safe-buffer "~5.1.1" - string_decoder "~1.1.1" - util-deprecate "~1.0.1" redis-commands@^1.2.0: version "1.4.0" resolved "https://registry.yarnpkg.com/redis-commands/-/redis-commands-1.4.0.tgz#52f9cf99153efcce56a8f86af986bd04e988602f" - integrity sha512-cu8EF+MtkwI4DLIT0x9P8qNTLFhQD4jLfxLR0cCNkeGzs87FN6879JOJwNQR/1zD7aSYNbU0hgsV9zGY71Itvw== redis-parser@^2.6.0: version "2.6.0" resolved "https://registry.yarnpkg.com/redis-parser/-/redis-parser-2.6.0.tgz#52ed09dacac108f1a631c07e9b69941e7a19504b" - integrity sha1-Uu0J2srBCPGmMcB+m2mUHnoZUEs= redis@^2.8.0: version "2.8.0" resolved "https://registry.yarnpkg.com/redis/-/redis-2.8.0.tgz#202288e3f58c49f6079d97af7a10e1303ae14b02" - integrity sha512-M1OkonEQwtRmZv4tEWF2VgpG0JWJ8Fv1PhlgT5+B+uNq2cA3Rt1Yt/ryoR+vQNOQcIEgdCdfH0jr3bDpihAw1A== dependencies: double-ended-queue "^2.1.0-0" redis-commands "^1.2.0" redis-parser "^2.6.0" -safe-buffer@5.1.2, safe-buffer@~5.1.0, safe-buffer@~5.1.1: - version "5.1.2" - resolved "https://registry.yarnpkg.com/safe-buffer/-/safe-buffer-5.1.2.tgz#991ec69d296e0313747d59bdfd2b745c35f8828d" - integrity sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g== - -sqlstring@2.3.1: - version "2.3.1" - resolved "https://registry.yarnpkg.com/sqlstring/-/sqlstring-2.3.1.tgz#475393ff9e91479aea62dcaf0ca3d14983a7fb40" - integrity sha1-R1OT/56RR5rqYtyvDKPRSYOn+0A= - -string_decoder@~1.1.1: - version "1.1.1" - resolved "https://registry.yarnpkg.com/string_decoder/-/string_decoder-1.1.1.tgz#9cf1611ba62685d7030ae9e4ba34149c3af03fc8" - integrity sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg== - dependencies: - safe-buffer "~5.1.0" - -util-deprecate@~1.0.1: - version "1.0.2" - resolved "https://registry.yarnpkg.com/util-deprecate/-/util-deprecate-1.0.2.tgz#450d4dc9fa70de732762fbd2d4a28981419a0ccf" - integrity sha1-RQ1Nyfpw3nMnYvvS1KKJgUGaDM8= - util-promisifyall@^1.0.4: version "1.0.4" resolved "https://registry.yarnpkg.com/util-promisifyall/-/util-promisifyall-1.0.4.tgz#82047e87e0637ca799b3480cd6ede3ddeedadf09" - integrity sha512-dgR84x56iVKsSMNo5GGED+xciS1bLS84Wsi5sYHW59Wf/P0mVdJckx2o7vLHrChG94J4c/T9EXoRJSoNy6qSEQ== + +yarn@^1.15.2: + version "1.15.2" + resolved "https://registry.yarnpkg.com/yarn/-/yarn-1.15.2.tgz#7a064ca81ca34235f16376ad2f796ed432f9e285" From 6f9f823e8aec946453a4689818b73fcdff979fd4 Mon Sep 17 00:00:00 2001 From: Cameron Braid Date: Sat, 27 Apr 2019 20:11:24 +1000 Subject: [PATCH 03/11] added basic test including some fixes (#1) * Added tests and made some fixes --- packages/cubejs-clickhouse-driver/.gitignore | 1 + packages/cubejs-clickhouse-driver/README.md | 11 + .../driver/ClickHouseDriver.js | 31 +- .../cubejs-clickhouse-driver/package.json | 7 +- .../test/ClickHouseDriverTest.js | 107 +++++ packages/cubejs-clickhouse-driver/yarn.lock | 395 +++++++++++++++++- 6 files changed, 536 insertions(+), 16 deletions(-) create mode 100644 packages/cubejs-clickhouse-driver/.gitignore create mode 100644 packages/cubejs-clickhouse-driver/test/ClickHouseDriverTest.js diff --git a/packages/cubejs-clickhouse-driver/.gitignore b/packages/cubejs-clickhouse-driver/.gitignore new file mode 100644 index 0000000000000..1269488f7fb1f --- /dev/null +++ b/packages/cubejs-clickhouse-driver/.gitignore @@ -0,0 +1 @@ +data diff --git a/packages/cubejs-clickhouse-driver/README.md b/packages/cubejs-clickhouse-driver/README.md index 9df649095f0f5..c937c537551b7 100644 --- a/packages/cubejs-clickhouse-driver/README.md +++ b/packages/cubejs-clickhouse-driver/README.md @@ -4,6 +4,17 @@ Pure Javascript ClickHouse driver. [Learn more](https://github.com/statsbotco/cube.js#getting-started) +### Testing + +To test follow the following steps + +``` +$ yarn +$ docker-compose up -d +$ yarn test +$ docker-compose stop +``` + ### License Cube.js ClickHouse Database Driver is [Apache 2.0 licensed](./LICENSE). \ No newline at end of file diff --git a/packages/cubejs-clickhouse-driver/driver/ClickHouseDriver.js b/packages/cubejs-clickhouse-driver/driver/ClickHouseDriver.js index 5598090a9b679..9196e20760d92 100644 --- a/packages/cubejs-clickhouse-driver/driver/ClickHouseDriver.js +++ b/packages/cubejs-clickhouse-driver/driver/ClickHouseDriver.js @@ -1,4 +1,4 @@ -const ClickHouse = require('clickhouse'); +const ClickHouse = require('@apla/clickhouse'); const genericPool = require('generic-pool'); const BaseDriver = require('@cubejs-backend/query-orchestrator/driver/BaseDriver'); @@ -8,9 +8,9 @@ class ClickHouseDriver extends BaseDriver { this.config = { host: process.env.CUBEJS_DB_HOST, port: process.env.CUBEJS_DB_PORT, - auth: process.env.CUBEJS_DB_USER + ":" + process.env.CUBEJS_DB_PASS, + auth: process.env.CUBEJS_DB_USER || process.env.CUBEJS_DB_PASS ? process.env.CUBEJS_DB_USER + ":" + process.env.CUBEJS_DB_PASS : '', queryOptions: { - database: process.env.CUBEJS_DB_NAME + database: process.env.CUBEJS_DB_NAME || config.database }, ...config }; @@ -76,21 +76,16 @@ class ClickHouseDriver extends BaseDriver { return promise; } - async testConnection() { - const conn = await this.pool._factory.create(); - try { - return await conn.querying('SELECT 1'); - } finally { - await this.pool._factory.destroy(conn); - } + testConnection() { + return this.query("SELECT 1") } query(query, values) { // TODO: handle values const self = this; return this.withConnection(connection => { - return connection.querying(query) - .then(res => res); + return connection.querying(query, {dataObjects:true}) + .then(res => res.data); }); } @@ -109,6 +104,18 @@ class ClickHouseDriver extends BaseDriver { WHERE database = '${this.config.database}' `; } + + async createSchemaIfNotExists(schemaName) { + let schemas = await this.query(`SELECT name FROM system.databases WHERE name = '${schemaName}'`) + if (schemas.length === 0) { + await this.query(`CREATE DATABASE ${schemaName}`); + } + } + + getTablesQuery(schemaName) { + return this.query(`SELECT name as table_name FROM system.tables WHERE database = '${schemaName}'`) + } + } module.exports = ClickHouseDriver; \ No newline at end of file diff --git a/packages/cubejs-clickhouse-driver/package.json b/packages/cubejs-clickhouse-driver/package.json index 3f48edf980f38..924d26b6fe50d 100644 --- a/packages/cubejs-clickhouse-driver/package.json +++ b/packages/cubejs-clickhouse-driver/package.json @@ -15,5 +15,10 @@ "@cubejs-backend/query-orchestrator": "^0.0.21", "generic-pool": "^3.6.0" }, - "license": "Apache-2.0" + "license": "Apache-2.0", + "devDependencies": { + "should": "^13.2.3", + "mocha": "^5.2.0", + "testcontainers": "^1.1.10" + } } diff --git a/packages/cubejs-clickhouse-driver/test/ClickHouseDriverTest.js b/packages/cubejs-clickhouse-driver/test/ClickHouseDriverTest.js new file mode 100644 index 0000000000000..ebe044f5ac599 --- /dev/null +++ b/packages/cubejs-clickhouse-driver/test/ClickHouseDriverTest.js @@ -0,0 +1,107 @@ +const ClickHouseDriver = require('../driver/ClickHouseDriver'); +require('should'); +const { GenericContainer } = require("testcontainers"); + + +describe('ClickHouseDriver', () => { + + + let container, config + before(async function() { + this.timeout(20000); + + container = await new GenericContainer("yandex/clickhouse-server") + .withExposedPorts(8123) + .start(); + + config = { + host:'localhost', + port:container.getMappedPort(8123), + } + + }) + + after(async ()=>{ + if (container) { + await container.stop() + } + }) + + async function doWithDriver(callback) { + let driver = new ClickHouseDriver(config) + try { + await callback(driver) + } + finally { + await driver.release() + } + } + + it('should construct', async () => { + await doWithDriver(driver=>{}) + }) + it('should test connection', async () => { + await doWithDriver(async (driver) => { + await driver.testConnection() + }) + }) + it('should select raw sql', async () => { + await doWithDriver(async (driver) => { + let numbers = await driver.query("SELECT number FROM system.numbers LIMIT 10") + numbers.should.be.deepEqual([ + { number: '0' }, + { number: '1' }, + { number: '2' }, + { number: '3' }, + { number: '4' }, + { number: '5' }, + { number: '6' }, + { number: '7' }, + { number: '8' }, + { number: '9' }, + ]) + }) + }) + it('should select raw sql multiple times', async () => { + await doWithDriver(async (driver) => { + let numbers = await driver.query("SELECT number FROM system.numbers LIMIT 5") + numbers.should.be.deepEqual([ + { number: '0' }, + { number: '1' }, + { number: '2' }, + { number: '3' }, + { number: '4' }, + ]) + numbers = await driver.query("SELECT number FROM system.numbers LIMIT 5") + numbers.should.be.deepEqual([ + { number: '0' }, + { number: '1' }, + { number: '2' }, + { number: '3' }, + { number: '4' }, + ]) + }) + }) + it('should get tables', async () => { + await doWithDriver(async (driver) => { + let tables = await driver.getTablesQuery("system") + tables.should.containDeep([ + {table_name:"numbers"}, + ]) + }) + }) + + it('should create schema if not exists', async () => { + await doWithDriver(async (driver) => { + let name = `temp_${Date.now()}` + try { + await driver.createSchemaIfNotExists(name) + } + finally { + await driver.query(`drop database ${name}`) + } + }) + }) + + + }) \ No newline at end of file diff --git a/packages/cubejs-clickhouse-driver/yarn.lock b/packages/cubejs-clickhouse-driver/yarn.lock index c4f78a491adfb..46cae0791721a 100644 --- a/packages/cubejs-clickhouse-driver/yarn.lock +++ b/packages/cubejs-clickhouse-driver/yarn.lock @@ -17,26 +17,296 @@ redis "^2.8.0" util-promisifyall "^1.0.4" +JSONStream@1.3.2: + version "1.3.2" + resolved "https://nexus.drivenow.com.au/repository/npm/JSONStream/-/JSONStream-1.3.2.tgz#c102371b6ec3a7cf3b847ca00c20bb0fce4c6dea" + dependencies: + jsonparse "^1.2.0" + through ">=2.2.7 <3" + +any-promise@^1.1.0: + version "1.3.0" + resolved "https://nexus.drivenow.com.au/repository/npm/any-promise/-/any-promise-1.3.0.tgz#abc6afeedcea52e809cdc0376aed3ce39635d17f" + +balanced-match@^1.0.0: + version "1.0.0" + resolved "https://nexus.drivenow.com.au/repository/npm/balanced-match/-/balanced-match-1.0.0.tgz#89b4d199ab2bee49de164ea02b89ce462d71b767" + +bl@^1.0.0: + version "1.2.2" + resolved "https://nexus.drivenow.com.au/repository/npm/bl/-/bl-1.2.2.tgz#a160911717103c07410cef63ef51b397c025af9c" + dependencies: + readable-stream "^2.3.5" + safe-buffer "^5.1.1" + +brace-expansion@^1.1.7: + version "1.1.11" + resolved "https://nexus.drivenow.com.au/repository/npm/brace-expansion/-/brace-expansion-1.1.11.tgz#3c7fcbf529d87226f3d2f52b966ff5271eb441dd" + dependencies: + balanced-match "^1.0.0" + concat-map "0.0.1" + +browser-stdout@1.3.1: + version "1.3.1" + resolved "https://nexus.drivenow.com.au/repository/npm/browser-stdout/-/browser-stdout-1.3.1.tgz#baa559ee14ced73452229bad7326467c61fabd60" + +buffer-alloc-unsafe@^1.1.0: + version "1.1.0" + resolved "https://nexus.drivenow.com.au/repository/npm/buffer-alloc-unsafe/-/buffer-alloc-unsafe-1.1.0.tgz#bd7dc26ae2972d0eda253be061dba992349c19f0" + +buffer-alloc@^1.2.0: + version "1.2.0" + resolved "https://nexus.drivenow.com.au/repository/npm/buffer-alloc/-/buffer-alloc-1.2.0.tgz#890dd90d923a873e08e10e5fd51a57e5b7cce0ec" + dependencies: + buffer-alloc-unsafe "^1.1.0" + buffer-fill "^1.0.0" + +buffer-fill@^1.0.0: + version "1.0.0" + resolved "https://nexus.drivenow.com.au/repository/npm/buffer-fill/-/buffer-fill-1.0.0.tgz#f8f78b76789888ef39f205cd637f68e702122b2c" + +buffer-from@^1.0.0: + version "1.1.1" + resolved "https://nexus.drivenow.com.au/repository/npm/buffer-from/-/buffer-from-1.1.1.tgz#32713bc028f75c02fdb710d7c7bcec1f2c6070ef" + buffer-indexof-polyfill@^1.0.1: version "1.0.1" resolved "https://registry.yarnpkg.com/buffer-indexof-polyfill/-/buffer-indexof-polyfill-1.0.1.tgz#a9fb806ce8145d5428510ce72f278bb363a638bf" +chownr@^1.0.1: + version "1.1.1" + resolved "https://nexus.drivenow.com.au/repository/npm/chownr/-/chownr-1.1.1.tgz#54726b8b8fff4df053c42187e801fb4412df1494" + +commander@2.15.1: + version "2.15.1" + resolved "https://nexus.drivenow.com.au/repository/npm/commander/-/commander-2.15.1.tgz#df46e867d0fc2aec66a34662b406a9ccafff5b0f" + +concat-map@0.0.1: + version "0.0.1" + resolved "https://nexus.drivenow.com.au/repository/npm/concat-map/-/concat-map-0.0.1.tgz#d8a96bd77fd68df7793a73036a3ba0d5405d477b" + +concat-stream@~1.6.2: + version "1.6.2" + resolved "https://nexus.drivenow.com.au/repository/npm/concat-stream/-/concat-stream-1.6.2.tgz#904bdf194cd3122fc675c77fc4ac3d4ff0fd1a34" + dependencies: + buffer-from "^1.0.0" + inherits "^2.0.3" + readable-stream "^2.2.2" + typedarray "^0.0.6" + +core-util-is@~1.0.0: + version "1.0.2" + resolved "https://nexus.drivenow.com.au/repository/npm/core-util-is/-/core-util-is-1.0.2.tgz#b5fd54220aa2bc5ab57aab7140c940754503c1a7" + +debug@3.1.0: + version "3.1.0" + resolved "https://nexus.drivenow.com.au/repository/npm/debug/-/debug-3.1.0.tgz#5bb5a0672628b64149566ba16819e61518c67261" + dependencies: + ms "2.0.0" + +debug@^3.2.6: + version "3.2.6" + resolved "https://nexus.drivenow.com.au/repository/npm/debug/-/debug-3.2.6.tgz#e83d17de16d8a7efb7717edbe5fb10135eee629b" + dependencies: + ms "^2.1.1" + +debug@^4.1.1: + version "4.1.1" + resolved "https://nexus.drivenow.com.au/repository/npm/debug/-/debug-4.1.1.tgz#3b72260255109c6b589cee050f1d516139664791" + dependencies: + ms "^2.1.1" + +diff@3.5.0: + version "3.5.0" + resolved "https://nexus.drivenow.com.au/repository/npm/diff/-/diff-3.5.0.tgz#800c0dd1e0a8bfbc95835c202ad220fe317e5a12" + +docker-modem@^1.0.8: + version "1.0.9" + resolved "https://nexus.drivenow.com.au/repository/npm/docker-modem/-/docker-modem-1.0.9.tgz#a1f13e50e6afb6cf3431b2d5e7aac589db6aaba8" + dependencies: + JSONStream "1.3.2" + debug "^3.2.6" + readable-stream "~1.0.26-4" + split-ca "^1.0.0" + +dockerode@^2.5.8: + version "2.5.8" + resolved "https://nexus.drivenow.com.au/repository/npm/dockerode/-/dockerode-2.5.8.tgz#1b661e36e1e4f860e25f56e0deabe9f87f1d0acc" + dependencies: + concat-stream "~1.6.2" + docker-modem "^1.0.8" + tar-fs "~1.16.3" + double-ended-queue@^2.1.0-0: version "2.1.0-0" resolved "https://registry.yarnpkg.com/double-ended-queue/-/double-ended-queue-2.1.0-0.tgz#103d3527fd31528f40188130c841efdd78264e5c" +end-of-stream@^1.0.0, end-of-stream@^1.1.0: + version "1.4.1" + resolved "https://nexus.drivenow.com.au/repository/npm/end-of-stream/-/end-of-stream-1.4.1.tgz#ed29634d19baba463b6ce6b80a37213eab71ec43" + dependencies: + once "^1.4.0" + +escape-string-regexp@1.0.5: + version "1.0.5" + resolved "https://nexus.drivenow.com.au/repository/npm/escape-string-regexp/-/escape-string-regexp-1.0.5.tgz#1b61c0562190a8dff6ae3bb2cf0200ca130b86d4" + +fs-constants@^1.0.0: + version "1.0.0" + resolved "https://nexus.drivenow.com.au/repository/npm/fs-constants/-/fs-constants-1.0.0.tgz#6be0de9be998ce16af8afc24497b9ee9b7ccd9ad" + +fs.realpath@^1.0.0: + version "1.0.0" + resolved "https://nexus.drivenow.com.au/repository/npm/fs.realpath/-/fs.realpath-1.0.0.tgz#1504ad2523158caa40db4a2787cb01411994ea4f" + generic-pool@^3.6.0: version "3.6.0" resolved "https://registry.yarnpkg.com/generic-pool/-/generic-pool-3.6.0.tgz#34e3124d9af754c8bfee7c15906e7347dc543afc" +get-port@^4.1.0: + version "4.2.0" + resolved "https://nexus.drivenow.com.au/repository/npm/get-port/-/get-port-4.2.0.tgz#e37368b1e863b7629c43c5a323625f95cf24b119" + +glob@7.1.2: + version "7.1.2" + resolved "https://nexus.drivenow.com.au/repository/npm/glob/-/glob-7.1.2.tgz#c19c9df9a028702d678612384a6552404c636d15" + dependencies: + fs.realpath "^1.0.0" + inflight "^1.0.4" + inherits "2" + minimatch "^3.0.4" + once "^1.3.0" + path-is-absolute "^1.0.0" + +growl@1.10.5: + version "1.10.5" + resolved "https://nexus.drivenow.com.au/repository/npm/growl/-/growl-1.10.5.tgz#f2735dc2283674fa67478b10181059355c369e5e" + +has-flag@^3.0.0: + version "3.0.0" + resolved "https://nexus.drivenow.com.au/repository/npm/has-flag/-/has-flag-3.0.0.tgz#b5d454dc2199ae225699f3467e5a07f3b955bafd" + +he@1.1.1: + version "1.1.1" + resolved "https://nexus.drivenow.com.au/repository/npm/he/-/he-1.1.1.tgz#93410fd21b009735151f8868c2f271f3427e23fd" + +inflight@^1.0.4: + version "1.0.6" + resolved "https://nexus.drivenow.com.au/repository/npm/inflight/-/inflight-1.0.6.tgz#49bd6331d7d02d0c09bc910a1075ba8165b56df9" + dependencies: + once "^1.3.0" + wrappy "1" + +inherits@2, inherits@^2.0.3, inherits@~2.0.1, inherits@~2.0.3: + version "2.0.3" + resolved "https://nexus.drivenow.com.au/repository/npm/inherits/-/inherits-2.0.3.tgz#633c2c83e3da42a502f52466022480f4208261de" + +isarray@0.0.1: + version "0.0.1" + resolved "https://nexus.drivenow.com.au/repository/npm/isarray/-/isarray-0.0.1.tgz#8a18acfca9a8f4177e09abfc6038939b05d1eedf" + +isarray@~1.0.0: + version "1.0.0" + resolved "https://nexus.drivenow.com.au/repository/npm/isarray/-/isarray-1.0.0.tgz#bb935d48582cba168c06834957a54a3e07124f11" + +jsonparse@^1.2.0: + version "1.3.1" + resolved "https://nexus.drivenow.com.au/repository/npm/jsonparse/-/jsonparse-1.3.1.tgz#3f4dae4a91fac315f71062f8521cc239f1366280" + +minimatch@3.0.4, minimatch@^3.0.4: + version "3.0.4" + resolved "https://nexus.drivenow.com.au/repository/npm/minimatch/-/minimatch-3.0.4.tgz#5166e286457f03306064be5497e8dbb0c3d32083" + dependencies: + brace-expansion "^1.1.7" + +minimist@0.0.8: + version "0.0.8" + resolved "https://nexus.drivenow.com.au/repository/npm/minimist/-/minimist-0.0.8.tgz#857fcabfc3397d2625b8228262e86aa7a011b05d" + +mkdirp@0.5.1, mkdirp@^0.5.1: + version "0.5.1" + resolved "https://nexus.drivenow.com.au/repository/npm/mkdirp/-/mkdirp-0.5.1.tgz#30057438eac6cf7f8c4767f38648d6697d75c903" + dependencies: + minimist "0.0.8" + +mocha@^5.2.0: + version "5.2.0" + resolved "https://nexus.drivenow.com.au/repository/npm/mocha/-/mocha-5.2.0.tgz#6d8ae508f59167f940f2b5b3c4a612ae50c90ae6" + dependencies: + browser-stdout "1.3.1" + commander "2.15.1" + debug "3.1.0" + diff "3.5.0" + escape-string-regexp "1.0.5" + glob "7.1.2" + growl "1.10.5" + he "1.1.1" + minimatch "3.0.4" + mkdirp "0.5.1" + supports-color "5.4.0" + +ms@2.0.0: + version "2.0.0" + resolved "https://nexus.drivenow.com.au/repository/npm/ms/-/ms-2.0.0.tgz#5608aeadfc00be6c2901df5f9861788de0d597c8" + +ms@^2.1.1: + version "2.1.1" + resolved "https://nexus.drivenow.com.au/repository/npm/ms/-/ms-2.1.1.tgz#30a5864eb3ebb0a66f2ebe6d727af06a09d86e0a" + +node-duration@^1.0.2: + version "1.0.4" + resolved "https://nexus.drivenow.com.au/repository/npm/node-duration/-/node-duration-1.0.4.tgz#3e94ecc0e473691c89c4560074503362071cecac" + object-assign@^4.1.0: version "4.1.1" resolved "https://registry.yarnpkg.com/object-assign/-/object-assign-4.1.1.tgz#2109adc7965887cfc05cbbd442cac8bfbb360863" +once@^1.3.0, once@^1.3.1, once@^1.4.0: + version "1.4.0" + resolved "https://nexus.drivenow.com.au/repository/npm/once/-/once-1.4.0.tgz#583b1aa775961d4b113ac17d9c50baef9dd76bd1" + dependencies: + wrappy "1" + +path-is-absolute@^1.0.0: + version "1.0.1" + resolved "https://nexus.drivenow.com.au/repository/npm/path-is-absolute/-/path-is-absolute-1.0.1.tgz#174b9268735534ffbc7ace6bf53a5a9e1b5c5f5f" + +process-nextick-args@~2.0.0: + version "2.0.0" + resolved "https://nexus.drivenow.com.au/repository/npm/process-nextick-args/-/process-nextick-args-2.0.0.tgz#a37d732f4271b4ab1ad070d35508e8290788ffaa" + +pump@^1.0.0: + version "1.0.3" + resolved "https://nexus.drivenow.com.au/repository/npm/pump/-/pump-1.0.3.tgz#5dfe8311c33bbf6fc18261f9f34702c47c08a954" + dependencies: + end-of-stream "^1.1.0" + once "^1.3.1" + ramda@^0.24.1: version "0.24.1" resolved "https://registry.yarnpkg.com/ramda/-/ramda-0.24.1.tgz#c3b7755197f35b8dc3502228262c4c91ddb6b857" +readable-stream@^2.2.2, readable-stream@^2.3.0, readable-stream@^2.3.5: + version "2.3.6" + resolved "https://nexus.drivenow.com.au/repository/npm/readable-stream/-/readable-stream-2.3.6.tgz#b11c27d88b8ff1fbe070643cf94b0c79ae1b0aaf" + dependencies: + core-util-is "~1.0.0" + inherits "~2.0.3" + isarray "~1.0.0" + process-nextick-args "~2.0.0" + safe-buffer "~5.1.1" + string_decoder "~1.1.1" + util-deprecate "~1.0.1" + +readable-stream@~1.0.26-4: + version "1.0.34" + resolved "https://nexus.drivenow.com.au/repository/npm/readable-stream/-/readable-stream-1.0.34.tgz#125820e34bc842d2f2aaafafe4c2916ee32c157c" + dependencies: + core-util-is "~1.0.0" + inherits "~2.0.1" + isarray "0.0.1" + string_decoder "~0.10.x" + redis-commands@^1.2.0: version "1.4.0" resolved "https://registry.yarnpkg.com/redis-commands/-/redis-commands-1.4.0.tgz#52f9cf99153efcce56a8f86af986bd04e988602f" @@ -53,10 +323,129 @@ redis@^2.8.0: redis-commands "^1.2.0" redis-parser "^2.6.0" +safe-buffer@^5.1.1, safe-buffer@~5.1.0, safe-buffer@~5.1.1: + version "5.1.2" + resolved "https://nexus.drivenow.com.au/repository/npm/safe-buffer/-/safe-buffer-5.1.2.tgz#991ec69d296e0313747d59bdfd2b745c35f8828d" + +should-equal@^2.0.0: + version "2.0.0" + resolved "https://nexus.drivenow.com.au/repository/npm/should-equal/-/should-equal-2.0.0.tgz#6072cf83047360867e68e98b09d71143d04ee0c3" + dependencies: + should-type "^1.4.0" + +should-format@^3.0.3: + version "3.0.3" + resolved "https://nexus.drivenow.com.au/repository/npm/should-format/-/should-format-3.0.3.tgz#9bfc8f74fa39205c53d38c34d717303e277124f1" + dependencies: + should-type "^1.3.0" + should-type-adaptors "^1.0.1" + +should-type-adaptors@^1.0.1: + version "1.1.0" + resolved "https://nexus.drivenow.com.au/repository/npm/should-type-adaptors/-/should-type-adaptors-1.1.0.tgz#401e7f33b5533033944d5cd8bf2b65027792e27a" + dependencies: + should-type "^1.3.0" + should-util "^1.0.0" + +should-type@^1.3.0, should-type@^1.4.0: + version "1.4.0" + resolved "https://nexus.drivenow.com.au/repository/npm/should-type/-/should-type-1.4.0.tgz#0756d8ce846dfd09843a6947719dfa0d4cff5cf3" + +should-util@^1.0.0: + version "1.0.0" + resolved "https://nexus.drivenow.com.au/repository/npm/should-util/-/should-util-1.0.0.tgz#c98cda374aa6b190df8ba87c9889c2b4db620063" + +should@^13.2.3: + version "13.2.3" + resolved "https://nexus.drivenow.com.au/repository/npm/should/-/should-13.2.3.tgz#96d8e5acf3e97b49d89b51feaa5ae8d07ef58f10" + dependencies: + should-equal "^2.0.0" + should-format "^3.0.3" + should-type "^1.4.0" + should-type-adaptors "^1.0.1" + should-util "^1.0.0" + +split-ca@^1.0.0: + version "1.0.1" + resolved "https://nexus.drivenow.com.au/repository/npm/split-ca/-/split-ca-1.0.1.tgz#6c83aff3692fa61256e0cd197e05e9de157691a6" + +stream-to-array@^2.3.0: + version "2.3.0" + resolved "https://nexus.drivenow.com.au/repository/npm/stream-to-array/-/stream-to-array-2.3.0.tgz#bbf6b39f5f43ec30bc71babcb37557acecf34353" + dependencies: + any-promise "^1.1.0" + +string_decoder@~0.10.x: + version "0.10.31" + resolved "https://nexus.drivenow.com.au/repository/npm/string_decoder/-/string_decoder-0.10.31.tgz#62e203bc41766c6c28c9fc84301dab1c5310fa94" + +string_decoder@~1.1.1: + version "1.1.1" + resolved "https://nexus.drivenow.com.au/repository/npm/string_decoder/-/string_decoder-1.1.1.tgz#9cf1611ba62685d7030ae9e4ba34149c3af03fc8" + dependencies: + safe-buffer "~5.1.0" + +supports-color@5.4.0: + version "5.4.0" + resolved "https://nexus.drivenow.com.au/repository/npm/supports-color/-/supports-color-5.4.0.tgz#1c6b337402c2137605efe19f10fec390f6faab54" + dependencies: + has-flag "^3.0.0" + +tar-fs@~1.16.3: + version "1.16.3" + resolved "https://nexus.drivenow.com.au/repository/npm/tar-fs/-/tar-fs-1.16.3.tgz#966a628841da2c4010406a82167cbd5e0c72d509" + dependencies: + chownr "^1.0.1" + mkdirp "^0.5.1" + pump "^1.0.0" + tar-stream "^1.1.2" + +tar-stream@^1.1.2: + version "1.6.2" + resolved "https://nexus.drivenow.com.au/repository/npm/tar-stream/-/tar-stream-1.6.2.tgz#8ea55dab37972253d9a9af90fdcd559ae435c555" + dependencies: + bl "^1.0.0" + buffer-alloc "^1.2.0" + end-of-stream "^1.0.0" + fs-constants "^1.0.0" + readable-stream "^2.3.0" + to-buffer "^1.1.1" + xtend "^4.0.0" + +testcontainers@^1.1.10: + version "1.1.10" + resolved "https://nexus.drivenow.com.au/repository/npm/testcontainers/-/testcontainers-1.1.10.tgz#9507a5ec4fddbff506721a7e2a523e356f4fe1b0" + dependencies: + debug "^4.1.1" + dockerode "^2.5.8" + get-port "^4.1.0" + node-duration "^1.0.2" + stream-to-array "^2.3.0" + +"through@>=2.2.7 <3": + version "2.3.8" + resolved "https://nexus.drivenow.com.au/repository/npm/through/-/through-2.3.8.tgz#0dd4c9ffaabc357960b1b724115d7e0e86a2e1f5" + +to-buffer@^1.1.1: + version "1.1.1" + resolved "https://nexus.drivenow.com.au/repository/npm/to-buffer/-/to-buffer-1.1.1.tgz#493bd48f62d7c43fcded313a03dcadb2e1213a80" + +typedarray@^0.0.6: + version "0.0.6" + resolved "https://nexus.drivenow.com.au/repository/npm/typedarray/-/typedarray-0.0.6.tgz#867ac74e3864187b1d3d47d996a78ec5c8830777" + +util-deprecate@~1.0.1: + version "1.0.2" + resolved "https://nexus.drivenow.com.au/repository/npm/util-deprecate/-/util-deprecate-1.0.2.tgz#450d4dc9fa70de732762fbd2d4a28981419a0ccf" + util-promisifyall@^1.0.4: version "1.0.4" resolved "https://registry.yarnpkg.com/util-promisifyall/-/util-promisifyall-1.0.4.tgz#82047e87e0637ca799b3480cd6ede3ddeedadf09" -yarn@^1.15.2: - version "1.15.2" - resolved "https://registry.yarnpkg.com/yarn/-/yarn-1.15.2.tgz#7a064ca81ca34235f16376ad2f796ed432f9e285" +wrappy@1: + version "1.0.2" + resolved "https://nexus.drivenow.com.au/repository/npm/wrappy/-/wrappy-1.0.2.tgz#b5243d8f3ec1aa35f1364605bc0d1036e30ab69f" + +xtend@^4.0.0: + version "4.0.1" + resolved "https://nexus.drivenow.com.au/repository/npm/xtend/-/xtend-4.0.1.tgz#a5c6d532be656e23db820efb943a1f04998d63af" From 2b56f99987a795ec2a760e686aee88421d3beca5 Mon Sep 17 00:00:00 2001 From: Igor Nikolaev Date: Sat, 27 Apr 2019 13:49:33 +0300 Subject: [PATCH 04/11] Generate custom query id --- .../driver/ClickHouseDriver.js | 21 ++- .../cubejs-clickhouse-driver/package.json | 5 +- packages/cubejs-clickhouse-driver/yarn.lock | 156 ++++++++++-------- 3 files changed, 96 insertions(+), 86 deletions(-) diff --git a/packages/cubejs-clickhouse-driver/driver/ClickHouseDriver.js b/packages/cubejs-clickhouse-driver/driver/ClickHouseDriver.js index 9196e20760d92..6eead350490aa 100644 --- a/packages/cubejs-clickhouse-driver/driver/ClickHouseDriver.js +++ b/packages/cubejs-clickhouse-driver/driver/ClickHouseDriver.js @@ -1,6 +1,7 @@ const ClickHouse = require('@apla/clickhouse'); const genericPool = require('generic-pool'); const BaseDriver = require('@cubejs-backend/query-orchestrator/driver/BaseDriver'); +const uuid = require('uuidv4'); class ClickHouseDriver extends BaseDriver { constructor(config) { @@ -41,22 +42,20 @@ class ClickHouseDriver extends BaseDriver { withConnection(fn) { const self = this; const connectionPromise = this.pool.acquire(); + const queryId = uuid(); let cancelled = false; const cancelObj = {}; - const promise = connectionPromise.then(conn => { + const promise = connectionPromise.then(connection => { cancelObj.cancel = async () => { cancelled = true; - await self.withConnection(async conn => { - const processRows = await conn.querying('SHOW PROCESSLIST'); - await Promise.all(processRows.filter(row => row.elapsed >= 599).map(row => { - return conn.execute(`KILL QUERY WHERE query_id = '${row.query_id}'`); - })); + await self.withConnection(async connection => { + await connection.querying(`KILL QUERY WHERE query_id = '${queryId}'`); }); }; - return fn(conn) + return fn(connection, queryId) .then(res => { - return this.pool.release(conn).then(() => { + return this.pool.release(connection).then(() => { if (cancelled) { throw new Error('Query cancelled'); } @@ -64,7 +63,7 @@ class ClickHouseDriver extends BaseDriver { }); }) .catch((err) => { - return this.pool.release(conn).then(() => { + return this.pool.release(connection).then(() => { if (cancelled) { throw new Error('Query cancelled'); } @@ -83,8 +82,8 @@ class ClickHouseDriver extends BaseDriver { query(query, values) { // TODO: handle values const self = this; - return this.withConnection(connection => { - return connection.querying(query, {dataObjects:true}) + return this.withConnection((connection, queryId) => { + return connection.querying(query, { dataObjects: true, queryOptions: { query_id: queryId } }) .then(res => res.data); }); } diff --git a/packages/cubejs-clickhouse-driver/package.json b/packages/cubejs-clickhouse-driver/package.json index 924d26b6fe50d..8cda9c7e4bfb2 100644 --- a/packages/cubejs-clickhouse-driver/package.json +++ b/packages/cubejs-clickhouse-driver/package.json @@ -13,12 +13,13 @@ "dependencies": { "@apla/clickhouse": "^1.5.5", "@cubejs-backend/query-orchestrator": "^0.0.21", - "generic-pool": "^3.6.0" + "generic-pool": "^3.6.0", + "uuidv4": "^4.0.0" }, "license": "Apache-2.0", "devDependencies": { - "should": "^13.2.3", "mocha": "^5.2.0", + "should": "^13.2.3", "testcontainers": "^1.1.10" } } diff --git a/packages/cubejs-clickhouse-driver/yarn.lock b/packages/cubejs-clickhouse-driver/yarn.lock index 46cae0791721a..f36e51d5fea9a 100644 --- a/packages/cubejs-clickhouse-driver/yarn.lock +++ b/packages/cubejs-clickhouse-driver/yarn.lock @@ -19,55 +19,55 @@ JSONStream@1.3.2: version "1.3.2" - resolved "https://nexus.drivenow.com.au/repository/npm/JSONStream/-/JSONStream-1.3.2.tgz#c102371b6ec3a7cf3b847ca00c20bb0fce4c6dea" + resolved "https://registry.yarnpkg.com/JSONStream/-/JSONStream-1.3.2.tgz#c102371b6ec3a7cf3b847ca00c20bb0fce4c6dea" dependencies: jsonparse "^1.2.0" through ">=2.2.7 <3" any-promise@^1.1.0: version "1.3.0" - resolved "https://nexus.drivenow.com.au/repository/npm/any-promise/-/any-promise-1.3.0.tgz#abc6afeedcea52e809cdc0376aed3ce39635d17f" + resolved "https://registry.yarnpkg.com/any-promise/-/any-promise-1.3.0.tgz#abc6afeedcea52e809cdc0376aed3ce39635d17f" balanced-match@^1.0.0: version "1.0.0" - resolved "https://nexus.drivenow.com.au/repository/npm/balanced-match/-/balanced-match-1.0.0.tgz#89b4d199ab2bee49de164ea02b89ce462d71b767" + resolved "https://registry.yarnpkg.com/balanced-match/-/balanced-match-1.0.0.tgz#89b4d199ab2bee49de164ea02b89ce462d71b767" bl@^1.0.0: version "1.2.2" - resolved "https://nexus.drivenow.com.au/repository/npm/bl/-/bl-1.2.2.tgz#a160911717103c07410cef63ef51b397c025af9c" + resolved "https://registry.yarnpkg.com/bl/-/bl-1.2.2.tgz#a160911717103c07410cef63ef51b397c025af9c" dependencies: readable-stream "^2.3.5" safe-buffer "^5.1.1" brace-expansion@^1.1.7: version "1.1.11" - resolved "https://nexus.drivenow.com.au/repository/npm/brace-expansion/-/brace-expansion-1.1.11.tgz#3c7fcbf529d87226f3d2f52b966ff5271eb441dd" + resolved "https://registry.yarnpkg.com/brace-expansion/-/brace-expansion-1.1.11.tgz#3c7fcbf529d87226f3d2f52b966ff5271eb441dd" dependencies: balanced-match "^1.0.0" concat-map "0.0.1" browser-stdout@1.3.1: version "1.3.1" - resolved "https://nexus.drivenow.com.au/repository/npm/browser-stdout/-/browser-stdout-1.3.1.tgz#baa559ee14ced73452229bad7326467c61fabd60" + resolved "https://registry.yarnpkg.com/browser-stdout/-/browser-stdout-1.3.1.tgz#baa559ee14ced73452229bad7326467c61fabd60" buffer-alloc-unsafe@^1.1.0: version "1.1.0" - resolved "https://nexus.drivenow.com.au/repository/npm/buffer-alloc-unsafe/-/buffer-alloc-unsafe-1.1.0.tgz#bd7dc26ae2972d0eda253be061dba992349c19f0" + resolved "https://registry.yarnpkg.com/buffer-alloc-unsafe/-/buffer-alloc-unsafe-1.1.0.tgz#bd7dc26ae2972d0eda253be061dba992349c19f0" buffer-alloc@^1.2.0: version "1.2.0" - resolved "https://nexus.drivenow.com.au/repository/npm/buffer-alloc/-/buffer-alloc-1.2.0.tgz#890dd90d923a873e08e10e5fd51a57e5b7cce0ec" + resolved "https://registry.yarnpkg.com/buffer-alloc/-/buffer-alloc-1.2.0.tgz#890dd90d923a873e08e10e5fd51a57e5b7cce0ec" dependencies: buffer-alloc-unsafe "^1.1.0" buffer-fill "^1.0.0" buffer-fill@^1.0.0: version "1.0.0" - resolved "https://nexus.drivenow.com.au/repository/npm/buffer-fill/-/buffer-fill-1.0.0.tgz#f8f78b76789888ef39f205cd637f68e702122b2c" + resolved "https://registry.yarnpkg.com/buffer-fill/-/buffer-fill-1.0.0.tgz#f8f78b76789888ef39f205cd637f68e702122b2c" buffer-from@^1.0.0: version "1.1.1" - resolved "https://nexus.drivenow.com.au/repository/npm/buffer-from/-/buffer-from-1.1.1.tgz#32713bc028f75c02fdb710d7c7bcec1f2c6070ef" + resolved "https://registry.yarnpkg.com/buffer-from/-/buffer-from-1.1.1.tgz#32713bc028f75c02fdb710d7c7bcec1f2c6070ef" buffer-indexof-polyfill@^1.0.1: version "1.0.1" @@ -75,19 +75,19 @@ buffer-indexof-polyfill@^1.0.1: chownr@^1.0.1: version "1.1.1" - resolved "https://nexus.drivenow.com.au/repository/npm/chownr/-/chownr-1.1.1.tgz#54726b8b8fff4df053c42187e801fb4412df1494" + resolved "https://registry.yarnpkg.com/chownr/-/chownr-1.1.1.tgz#54726b8b8fff4df053c42187e801fb4412df1494" commander@2.15.1: version "2.15.1" - resolved "https://nexus.drivenow.com.au/repository/npm/commander/-/commander-2.15.1.tgz#df46e867d0fc2aec66a34662b406a9ccafff5b0f" + resolved "https://registry.yarnpkg.com/commander/-/commander-2.15.1.tgz#df46e867d0fc2aec66a34662b406a9ccafff5b0f" concat-map@0.0.1: version "0.0.1" - resolved "https://nexus.drivenow.com.au/repository/npm/concat-map/-/concat-map-0.0.1.tgz#d8a96bd77fd68df7793a73036a3ba0d5405d477b" + resolved "https://registry.yarnpkg.com/concat-map/-/concat-map-0.0.1.tgz#d8a96bd77fd68df7793a73036a3ba0d5405d477b" concat-stream@~1.6.2: version "1.6.2" - resolved "https://nexus.drivenow.com.au/repository/npm/concat-stream/-/concat-stream-1.6.2.tgz#904bdf194cd3122fc675c77fc4ac3d4ff0fd1a34" + resolved "https://registry.yarnpkg.com/concat-stream/-/concat-stream-1.6.2.tgz#904bdf194cd3122fc675c77fc4ac3d4ff0fd1a34" dependencies: buffer-from "^1.0.0" inherits "^2.0.3" @@ -96,33 +96,33 @@ concat-stream@~1.6.2: core-util-is@~1.0.0: version "1.0.2" - resolved "https://nexus.drivenow.com.au/repository/npm/core-util-is/-/core-util-is-1.0.2.tgz#b5fd54220aa2bc5ab57aab7140c940754503c1a7" + resolved "https://registry.yarnpkg.com/core-util-is/-/core-util-is-1.0.2.tgz#b5fd54220aa2bc5ab57aab7140c940754503c1a7" debug@3.1.0: version "3.1.0" - resolved "https://nexus.drivenow.com.au/repository/npm/debug/-/debug-3.1.0.tgz#5bb5a0672628b64149566ba16819e61518c67261" + resolved "https://registry.yarnpkg.com/debug/-/debug-3.1.0.tgz#5bb5a0672628b64149566ba16819e61518c67261" dependencies: ms "2.0.0" debug@^3.2.6: version "3.2.6" - resolved "https://nexus.drivenow.com.au/repository/npm/debug/-/debug-3.2.6.tgz#e83d17de16d8a7efb7717edbe5fb10135eee629b" + resolved "https://registry.yarnpkg.com/debug/-/debug-3.2.6.tgz#e83d17de16d8a7efb7717edbe5fb10135eee629b" dependencies: ms "^2.1.1" debug@^4.1.1: version "4.1.1" - resolved "https://nexus.drivenow.com.au/repository/npm/debug/-/debug-4.1.1.tgz#3b72260255109c6b589cee050f1d516139664791" + resolved "https://registry.yarnpkg.com/debug/-/debug-4.1.1.tgz#3b72260255109c6b589cee050f1d516139664791" dependencies: ms "^2.1.1" diff@3.5.0: version "3.5.0" - resolved "https://nexus.drivenow.com.au/repository/npm/diff/-/diff-3.5.0.tgz#800c0dd1e0a8bfbc95835c202ad220fe317e5a12" + resolved "https://registry.yarnpkg.com/diff/-/diff-3.5.0.tgz#800c0dd1e0a8bfbc95835c202ad220fe317e5a12" docker-modem@^1.0.8: version "1.0.9" - resolved "https://nexus.drivenow.com.au/repository/npm/docker-modem/-/docker-modem-1.0.9.tgz#a1f13e50e6afb6cf3431b2d5e7aac589db6aaba8" + resolved "https://registry.yarnpkg.com/docker-modem/-/docker-modem-1.0.9.tgz#a1f13e50e6afb6cf3431b2d5e7aac589db6aaba8" dependencies: JSONStream "1.3.2" debug "^3.2.6" @@ -131,7 +131,7 @@ docker-modem@^1.0.8: dockerode@^2.5.8: version "2.5.8" - resolved "https://nexus.drivenow.com.au/repository/npm/dockerode/-/dockerode-2.5.8.tgz#1b661e36e1e4f860e25f56e0deabe9f87f1d0acc" + resolved "https://registry.yarnpkg.com/dockerode/-/dockerode-2.5.8.tgz#1b661e36e1e4f860e25f56e0deabe9f87f1d0acc" dependencies: concat-stream "~1.6.2" docker-modem "^1.0.8" @@ -143,33 +143,33 @@ double-ended-queue@^2.1.0-0: end-of-stream@^1.0.0, end-of-stream@^1.1.0: version "1.4.1" - resolved "https://nexus.drivenow.com.au/repository/npm/end-of-stream/-/end-of-stream-1.4.1.tgz#ed29634d19baba463b6ce6b80a37213eab71ec43" + resolved "https://registry.yarnpkg.com/end-of-stream/-/end-of-stream-1.4.1.tgz#ed29634d19baba463b6ce6b80a37213eab71ec43" dependencies: once "^1.4.0" escape-string-regexp@1.0.5: version "1.0.5" - resolved "https://nexus.drivenow.com.au/repository/npm/escape-string-regexp/-/escape-string-regexp-1.0.5.tgz#1b61c0562190a8dff6ae3bb2cf0200ca130b86d4" + resolved "https://registry.yarnpkg.com/escape-string-regexp/-/escape-string-regexp-1.0.5.tgz#1b61c0562190a8dff6ae3bb2cf0200ca130b86d4" fs-constants@^1.0.0: version "1.0.0" - resolved "https://nexus.drivenow.com.au/repository/npm/fs-constants/-/fs-constants-1.0.0.tgz#6be0de9be998ce16af8afc24497b9ee9b7ccd9ad" + resolved "https://registry.yarnpkg.com/fs-constants/-/fs-constants-1.0.0.tgz#6be0de9be998ce16af8afc24497b9ee9b7ccd9ad" fs.realpath@^1.0.0: version "1.0.0" - resolved "https://nexus.drivenow.com.au/repository/npm/fs.realpath/-/fs.realpath-1.0.0.tgz#1504ad2523158caa40db4a2787cb01411994ea4f" + resolved "https://registry.yarnpkg.com/fs.realpath/-/fs.realpath-1.0.0.tgz#1504ad2523158caa40db4a2787cb01411994ea4f" generic-pool@^3.6.0: - version "3.6.0" - resolved "https://registry.yarnpkg.com/generic-pool/-/generic-pool-3.6.0.tgz#34e3124d9af754c8bfee7c15906e7347dc543afc" + version "3.7.1" + resolved "https://registry.yarnpkg.com/generic-pool/-/generic-pool-3.7.1.tgz#36fe5bb83e7e0e032e5d32cd05dc00f5ff119aa8" get-port@^4.1.0: version "4.2.0" - resolved "https://nexus.drivenow.com.au/repository/npm/get-port/-/get-port-4.2.0.tgz#e37368b1e863b7629c43c5a323625f95cf24b119" + resolved "https://registry.yarnpkg.com/get-port/-/get-port-4.2.0.tgz#e37368b1e863b7629c43c5a323625f95cf24b119" glob@7.1.2: version "7.1.2" - resolved "https://nexus.drivenow.com.au/repository/npm/glob/-/glob-7.1.2.tgz#c19c9df9a028702d678612384a6552404c636d15" + resolved "https://registry.yarnpkg.com/glob/-/glob-7.1.2.tgz#c19c9df9a028702d678612384a6552404c636d15" dependencies: fs.realpath "^1.0.0" inflight "^1.0.4" @@ -180,58 +180,58 @@ glob@7.1.2: growl@1.10.5: version "1.10.5" - resolved "https://nexus.drivenow.com.au/repository/npm/growl/-/growl-1.10.5.tgz#f2735dc2283674fa67478b10181059355c369e5e" + resolved "https://registry.yarnpkg.com/growl/-/growl-1.10.5.tgz#f2735dc2283674fa67478b10181059355c369e5e" has-flag@^3.0.0: version "3.0.0" - resolved "https://nexus.drivenow.com.au/repository/npm/has-flag/-/has-flag-3.0.0.tgz#b5d454dc2199ae225699f3467e5a07f3b955bafd" + resolved "https://registry.yarnpkg.com/has-flag/-/has-flag-3.0.0.tgz#b5d454dc2199ae225699f3467e5a07f3b955bafd" he@1.1.1: version "1.1.1" - resolved "https://nexus.drivenow.com.au/repository/npm/he/-/he-1.1.1.tgz#93410fd21b009735151f8868c2f271f3427e23fd" + resolved "https://registry.yarnpkg.com/he/-/he-1.1.1.tgz#93410fd21b009735151f8868c2f271f3427e23fd" inflight@^1.0.4: version "1.0.6" - resolved "https://nexus.drivenow.com.au/repository/npm/inflight/-/inflight-1.0.6.tgz#49bd6331d7d02d0c09bc910a1075ba8165b56df9" + resolved "https://registry.yarnpkg.com/inflight/-/inflight-1.0.6.tgz#49bd6331d7d02d0c09bc910a1075ba8165b56df9" dependencies: once "^1.3.0" wrappy "1" inherits@2, inherits@^2.0.3, inherits@~2.0.1, inherits@~2.0.3: version "2.0.3" - resolved "https://nexus.drivenow.com.au/repository/npm/inherits/-/inherits-2.0.3.tgz#633c2c83e3da42a502f52466022480f4208261de" + resolved "https://registry.yarnpkg.com/inherits/-/inherits-2.0.3.tgz#633c2c83e3da42a502f52466022480f4208261de" isarray@0.0.1: version "0.0.1" - resolved "https://nexus.drivenow.com.au/repository/npm/isarray/-/isarray-0.0.1.tgz#8a18acfca9a8f4177e09abfc6038939b05d1eedf" + resolved "https://registry.yarnpkg.com/isarray/-/isarray-0.0.1.tgz#8a18acfca9a8f4177e09abfc6038939b05d1eedf" isarray@~1.0.0: version "1.0.0" - resolved "https://nexus.drivenow.com.au/repository/npm/isarray/-/isarray-1.0.0.tgz#bb935d48582cba168c06834957a54a3e07124f11" + resolved "https://registry.yarnpkg.com/isarray/-/isarray-1.0.0.tgz#bb935d48582cba168c06834957a54a3e07124f11" jsonparse@^1.2.0: version "1.3.1" - resolved "https://nexus.drivenow.com.au/repository/npm/jsonparse/-/jsonparse-1.3.1.tgz#3f4dae4a91fac315f71062f8521cc239f1366280" + resolved "https://registry.yarnpkg.com/jsonparse/-/jsonparse-1.3.1.tgz#3f4dae4a91fac315f71062f8521cc239f1366280" minimatch@3.0.4, minimatch@^3.0.4: version "3.0.4" - resolved "https://nexus.drivenow.com.au/repository/npm/minimatch/-/minimatch-3.0.4.tgz#5166e286457f03306064be5497e8dbb0c3d32083" + resolved "https://registry.yarnpkg.com/minimatch/-/minimatch-3.0.4.tgz#5166e286457f03306064be5497e8dbb0c3d32083" dependencies: brace-expansion "^1.1.7" minimist@0.0.8: version "0.0.8" - resolved "https://nexus.drivenow.com.au/repository/npm/minimist/-/minimist-0.0.8.tgz#857fcabfc3397d2625b8228262e86aa7a011b05d" + resolved "https://registry.yarnpkg.com/minimist/-/minimist-0.0.8.tgz#857fcabfc3397d2625b8228262e86aa7a011b05d" mkdirp@0.5.1, mkdirp@^0.5.1: version "0.5.1" - resolved "https://nexus.drivenow.com.au/repository/npm/mkdirp/-/mkdirp-0.5.1.tgz#30057438eac6cf7f8c4767f38648d6697d75c903" + resolved "https://registry.yarnpkg.com/mkdirp/-/mkdirp-0.5.1.tgz#30057438eac6cf7f8c4767f38648d6697d75c903" dependencies: minimist "0.0.8" mocha@^5.2.0: version "5.2.0" - resolved "https://nexus.drivenow.com.au/repository/npm/mocha/-/mocha-5.2.0.tgz#6d8ae508f59167f940f2b5b3c4a612ae50c90ae6" + resolved "https://registry.yarnpkg.com/mocha/-/mocha-5.2.0.tgz#6d8ae508f59167f940f2b5b3c4a612ae50c90ae6" dependencies: browser-stdout "1.3.1" commander "2.15.1" @@ -247,15 +247,15 @@ mocha@^5.2.0: ms@2.0.0: version "2.0.0" - resolved "https://nexus.drivenow.com.au/repository/npm/ms/-/ms-2.0.0.tgz#5608aeadfc00be6c2901df5f9861788de0d597c8" + resolved "https://registry.yarnpkg.com/ms/-/ms-2.0.0.tgz#5608aeadfc00be6c2901df5f9861788de0d597c8" ms@^2.1.1: version "2.1.1" - resolved "https://nexus.drivenow.com.au/repository/npm/ms/-/ms-2.1.1.tgz#30a5864eb3ebb0a66f2ebe6d727af06a09d86e0a" + resolved "https://registry.yarnpkg.com/ms/-/ms-2.1.1.tgz#30a5864eb3ebb0a66f2ebe6d727af06a09d86e0a" node-duration@^1.0.2: version "1.0.4" - resolved "https://nexus.drivenow.com.au/repository/npm/node-duration/-/node-duration-1.0.4.tgz#3e94ecc0e473691c89c4560074503362071cecac" + resolved "https://registry.yarnpkg.com/node-duration/-/node-duration-1.0.4.tgz#3e94ecc0e473691c89c4560074503362071cecac" object-assign@^4.1.0: version "4.1.1" @@ -263,21 +263,21 @@ object-assign@^4.1.0: once@^1.3.0, once@^1.3.1, once@^1.4.0: version "1.4.0" - resolved "https://nexus.drivenow.com.au/repository/npm/once/-/once-1.4.0.tgz#583b1aa775961d4b113ac17d9c50baef9dd76bd1" + resolved "https://registry.yarnpkg.com/once/-/once-1.4.0.tgz#583b1aa775961d4b113ac17d9c50baef9dd76bd1" dependencies: wrappy "1" path-is-absolute@^1.0.0: version "1.0.1" - resolved "https://nexus.drivenow.com.au/repository/npm/path-is-absolute/-/path-is-absolute-1.0.1.tgz#174b9268735534ffbc7ace6bf53a5a9e1b5c5f5f" + resolved "https://registry.yarnpkg.com/path-is-absolute/-/path-is-absolute-1.0.1.tgz#174b9268735534ffbc7ace6bf53a5a9e1b5c5f5f" process-nextick-args@~2.0.0: version "2.0.0" - resolved "https://nexus.drivenow.com.au/repository/npm/process-nextick-args/-/process-nextick-args-2.0.0.tgz#a37d732f4271b4ab1ad070d35508e8290788ffaa" + resolved "https://registry.yarnpkg.com/process-nextick-args/-/process-nextick-args-2.0.0.tgz#a37d732f4271b4ab1ad070d35508e8290788ffaa" pump@^1.0.0: version "1.0.3" - resolved "https://nexus.drivenow.com.au/repository/npm/pump/-/pump-1.0.3.tgz#5dfe8311c33bbf6fc18261f9f34702c47c08a954" + resolved "https://registry.yarnpkg.com/pump/-/pump-1.0.3.tgz#5dfe8311c33bbf6fc18261f9f34702c47c08a954" dependencies: end-of-stream "^1.1.0" once "^1.3.1" @@ -288,7 +288,7 @@ ramda@^0.24.1: readable-stream@^2.2.2, readable-stream@^2.3.0, readable-stream@^2.3.5: version "2.3.6" - resolved "https://nexus.drivenow.com.au/repository/npm/readable-stream/-/readable-stream-2.3.6.tgz#b11c27d88b8ff1fbe070643cf94b0c79ae1b0aaf" + resolved "https://registry.yarnpkg.com/readable-stream/-/readable-stream-2.3.6.tgz#b11c27d88b8ff1fbe070643cf94b0c79ae1b0aaf" dependencies: core-util-is "~1.0.0" inherits "~2.0.3" @@ -300,7 +300,7 @@ readable-stream@^2.2.2, readable-stream@^2.3.0, readable-stream@^2.3.5: readable-stream@~1.0.26-4: version "1.0.34" - resolved "https://nexus.drivenow.com.au/repository/npm/readable-stream/-/readable-stream-1.0.34.tgz#125820e34bc842d2f2aaafafe4c2916ee32c157c" + resolved "https://registry.yarnpkg.com/readable-stream/-/readable-stream-1.0.34.tgz#125820e34bc842d2f2aaafafe4c2916ee32c157c" dependencies: core-util-is "~1.0.0" inherits "~2.0.1" @@ -325,39 +325,39 @@ redis@^2.8.0: safe-buffer@^5.1.1, safe-buffer@~5.1.0, safe-buffer@~5.1.1: version "5.1.2" - resolved "https://nexus.drivenow.com.au/repository/npm/safe-buffer/-/safe-buffer-5.1.2.tgz#991ec69d296e0313747d59bdfd2b745c35f8828d" + resolved "https://registry.yarnpkg.com/safe-buffer/-/safe-buffer-5.1.2.tgz#991ec69d296e0313747d59bdfd2b745c35f8828d" should-equal@^2.0.0: version "2.0.0" - resolved "https://nexus.drivenow.com.au/repository/npm/should-equal/-/should-equal-2.0.0.tgz#6072cf83047360867e68e98b09d71143d04ee0c3" + resolved "https://registry.yarnpkg.com/should-equal/-/should-equal-2.0.0.tgz#6072cf83047360867e68e98b09d71143d04ee0c3" dependencies: should-type "^1.4.0" should-format@^3.0.3: version "3.0.3" - resolved "https://nexus.drivenow.com.au/repository/npm/should-format/-/should-format-3.0.3.tgz#9bfc8f74fa39205c53d38c34d717303e277124f1" + resolved "https://registry.yarnpkg.com/should-format/-/should-format-3.0.3.tgz#9bfc8f74fa39205c53d38c34d717303e277124f1" dependencies: should-type "^1.3.0" should-type-adaptors "^1.0.1" should-type-adaptors@^1.0.1: version "1.1.0" - resolved "https://nexus.drivenow.com.au/repository/npm/should-type-adaptors/-/should-type-adaptors-1.1.0.tgz#401e7f33b5533033944d5cd8bf2b65027792e27a" + resolved "https://registry.yarnpkg.com/should-type-adaptors/-/should-type-adaptors-1.1.0.tgz#401e7f33b5533033944d5cd8bf2b65027792e27a" dependencies: should-type "^1.3.0" should-util "^1.0.0" should-type@^1.3.0, should-type@^1.4.0: version "1.4.0" - resolved "https://nexus.drivenow.com.au/repository/npm/should-type/-/should-type-1.4.0.tgz#0756d8ce846dfd09843a6947719dfa0d4cff5cf3" + resolved "https://registry.yarnpkg.com/should-type/-/should-type-1.4.0.tgz#0756d8ce846dfd09843a6947719dfa0d4cff5cf3" should-util@^1.0.0: version "1.0.0" - resolved "https://nexus.drivenow.com.au/repository/npm/should-util/-/should-util-1.0.0.tgz#c98cda374aa6b190df8ba87c9889c2b4db620063" + resolved "https://registry.yarnpkg.com/should-util/-/should-util-1.0.0.tgz#c98cda374aa6b190df8ba87c9889c2b4db620063" should@^13.2.3: version "13.2.3" - resolved "https://nexus.drivenow.com.au/repository/npm/should/-/should-13.2.3.tgz#96d8e5acf3e97b49d89b51feaa5ae8d07ef58f10" + resolved "https://registry.yarnpkg.com/should/-/should-13.2.3.tgz#96d8e5acf3e97b49d89b51feaa5ae8d07ef58f10" dependencies: should-equal "^2.0.0" should-format "^3.0.3" @@ -367,33 +367,33 @@ should@^13.2.3: split-ca@^1.0.0: version "1.0.1" - resolved "https://nexus.drivenow.com.au/repository/npm/split-ca/-/split-ca-1.0.1.tgz#6c83aff3692fa61256e0cd197e05e9de157691a6" + resolved "https://registry.yarnpkg.com/split-ca/-/split-ca-1.0.1.tgz#6c83aff3692fa61256e0cd197e05e9de157691a6" stream-to-array@^2.3.0: version "2.3.0" - resolved "https://nexus.drivenow.com.au/repository/npm/stream-to-array/-/stream-to-array-2.3.0.tgz#bbf6b39f5f43ec30bc71babcb37557acecf34353" + resolved "https://registry.yarnpkg.com/stream-to-array/-/stream-to-array-2.3.0.tgz#bbf6b39f5f43ec30bc71babcb37557acecf34353" dependencies: any-promise "^1.1.0" string_decoder@~0.10.x: version "0.10.31" - resolved "https://nexus.drivenow.com.au/repository/npm/string_decoder/-/string_decoder-0.10.31.tgz#62e203bc41766c6c28c9fc84301dab1c5310fa94" + resolved "https://registry.yarnpkg.com/string_decoder/-/string_decoder-0.10.31.tgz#62e203bc41766c6c28c9fc84301dab1c5310fa94" string_decoder@~1.1.1: version "1.1.1" - resolved "https://nexus.drivenow.com.au/repository/npm/string_decoder/-/string_decoder-1.1.1.tgz#9cf1611ba62685d7030ae9e4ba34149c3af03fc8" + resolved "https://registry.yarnpkg.com/string_decoder/-/string_decoder-1.1.1.tgz#9cf1611ba62685d7030ae9e4ba34149c3af03fc8" dependencies: safe-buffer "~5.1.0" supports-color@5.4.0: version "5.4.0" - resolved "https://nexus.drivenow.com.au/repository/npm/supports-color/-/supports-color-5.4.0.tgz#1c6b337402c2137605efe19f10fec390f6faab54" + resolved "https://registry.yarnpkg.com/supports-color/-/supports-color-5.4.0.tgz#1c6b337402c2137605efe19f10fec390f6faab54" dependencies: has-flag "^3.0.0" tar-fs@~1.16.3: version "1.16.3" - resolved "https://nexus.drivenow.com.au/repository/npm/tar-fs/-/tar-fs-1.16.3.tgz#966a628841da2c4010406a82167cbd5e0c72d509" + resolved "https://registry.yarnpkg.com/tar-fs/-/tar-fs-1.16.3.tgz#966a628841da2c4010406a82167cbd5e0c72d509" dependencies: chownr "^1.0.1" mkdirp "^0.5.1" @@ -402,7 +402,7 @@ tar-fs@~1.16.3: tar-stream@^1.1.2: version "1.6.2" - resolved "https://nexus.drivenow.com.au/repository/npm/tar-stream/-/tar-stream-1.6.2.tgz#8ea55dab37972253d9a9af90fdcd559ae435c555" + resolved "https://registry.yarnpkg.com/tar-stream/-/tar-stream-1.6.2.tgz#8ea55dab37972253d9a9af90fdcd559ae435c555" dependencies: bl "^1.0.0" buffer-alloc "^1.2.0" @@ -414,7 +414,7 @@ tar-stream@^1.1.2: testcontainers@^1.1.10: version "1.1.10" - resolved "https://nexus.drivenow.com.au/repository/npm/testcontainers/-/testcontainers-1.1.10.tgz#9507a5ec4fddbff506721a7e2a523e356f4fe1b0" + resolved "https://registry.yarnpkg.com/testcontainers/-/testcontainers-1.1.10.tgz#9507a5ec4fddbff506721a7e2a523e356f4fe1b0" dependencies: debug "^4.1.1" dockerode "^2.5.8" @@ -424,28 +424,38 @@ testcontainers@^1.1.10: "through@>=2.2.7 <3": version "2.3.8" - resolved "https://nexus.drivenow.com.au/repository/npm/through/-/through-2.3.8.tgz#0dd4c9ffaabc357960b1b724115d7e0e86a2e1f5" + resolved "https://registry.yarnpkg.com/through/-/through-2.3.8.tgz#0dd4c9ffaabc357960b1b724115d7e0e86a2e1f5" to-buffer@^1.1.1: version "1.1.1" - resolved "https://nexus.drivenow.com.au/repository/npm/to-buffer/-/to-buffer-1.1.1.tgz#493bd48f62d7c43fcded313a03dcadb2e1213a80" + resolved "https://registry.yarnpkg.com/to-buffer/-/to-buffer-1.1.1.tgz#493bd48f62d7c43fcded313a03dcadb2e1213a80" typedarray@^0.0.6: version "0.0.6" - resolved "https://nexus.drivenow.com.au/repository/npm/typedarray/-/typedarray-0.0.6.tgz#867ac74e3864187b1d3d47d996a78ec5c8830777" + resolved "https://registry.yarnpkg.com/typedarray/-/typedarray-0.0.6.tgz#867ac74e3864187b1d3d47d996a78ec5c8830777" util-deprecate@~1.0.1: version "1.0.2" - resolved "https://nexus.drivenow.com.au/repository/npm/util-deprecate/-/util-deprecate-1.0.2.tgz#450d4dc9fa70de732762fbd2d4a28981419a0ccf" + resolved "https://registry.yarnpkg.com/util-deprecate/-/util-deprecate-1.0.2.tgz#450d4dc9fa70de732762fbd2d4a28981419a0ccf" util-promisifyall@^1.0.4: - version "1.0.4" - resolved "https://registry.yarnpkg.com/util-promisifyall/-/util-promisifyall-1.0.4.tgz#82047e87e0637ca799b3480cd6ede3ddeedadf09" + version "1.0.6" + resolved "https://registry.yarnpkg.com/util-promisifyall/-/util-promisifyall-1.0.6.tgz#03439a5a76461e1f71f76954d945cffc8bd6aad8" + +uuid@3.3.2: + version "3.3.2" + resolved "https://registry.yarnpkg.com/uuid/-/uuid-3.3.2.tgz#1b4af4955eb3077c501c23872fc6513811587131" + +uuidv4@^4.0.0: + version "4.0.0" + resolved "https://registry.yarnpkg.com/uuidv4/-/uuidv4-4.0.0.tgz#ac4b05ecf0efa62dda532d55b7a8de8f726f8f7c" + dependencies: + uuid "3.3.2" wrappy@1: version "1.0.2" - resolved "https://nexus.drivenow.com.au/repository/npm/wrappy/-/wrappy-1.0.2.tgz#b5243d8f3ec1aa35f1364605bc0d1036e30ab69f" + resolved "https://registry.yarnpkg.com/wrappy/-/wrappy-1.0.2.tgz#b5243d8f3ec1aa35f1364605bc0d1036e30ab69f" xtend@^4.0.0: version "4.0.1" - resolved "https://nexus.drivenow.com.au/repository/npm/xtend/-/xtend-4.0.1.tgz#a5c6d532be656e23db820efb943a1f04998d63af" + resolved "https://registry.yarnpkg.com/xtend/-/xtend-4.0.1.tgz#a5c6d532be656e23db820efb943a1f04998d63af" From 2778a766171b31b6280ae9504dda242c38391149 Mon Sep 17 00:00:00 2001 From: Igor Nikolaev Date: Sat, 27 Apr 2019 20:12:24 +0300 Subject: [PATCH 05/11] Add support for parameter placeholders --- .../driver/ClickHouseDriver.js | 14 ++++++-------- packages/cubejs-clickhouse-driver/package.json | 1 + .../test/ClickHouseDriverTest.js | 18 ++++++++++++++++-- packages/cubejs-clickhouse-driver/yarn.lock | 4 ++++ 4 files changed, 27 insertions(+), 10 deletions(-) diff --git a/packages/cubejs-clickhouse-driver/driver/ClickHouseDriver.js b/packages/cubejs-clickhouse-driver/driver/ClickHouseDriver.js index 6eead350490aa..891ece99f59df 100644 --- a/packages/cubejs-clickhouse-driver/driver/ClickHouseDriver.js +++ b/packages/cubejs-clickhouse-driver/driver/ClickHouseDriver.js @@ -2,6 +2,7 @@ const ClickHouse = require('@apla/clickhouse'); const genericPool = require('generic-pool'); const BaseDriver = require('@cubejs-backend/query-orchestrator/driver/BaseDriver'); const uuid = require('uuidv4'); +const sqlstring = require('sqlstring'); class ClickHouseDriver extends BaseDriver { constructor(config) { @@ -80,10 +81,10 @@ class ClickHouseDriver extends BaseDriver { } query(query, values) { - // TODO: handle values - const self = this; + const formattedQuery = sqlstring.format(query, values); + return this.withConnection((connection, queryId) => { - return connection.querying(query, { dataObjects: true, queryOptions: { query_id: queryId } }) + return connection.querying(formattedQuery, { dataObjects: true, queryOptions: { query_id: queryId } }) .then(res => res.data); }); } @@ -105,14 +106,11 @@ class ClickHouseDriver extends BaseDriver { } async createSchemaIfNotExists(schemaName) { - let schemas = await this.query(`SELECT name FROM system.databases WHERE name = '${schemaName}'`) - if (schemas.length === 0) { - await this.query(`CREATE DATABASE ${schemaName}`); - } + await this.query(`CREATE DATABASE IF NOT EXISTS ${schemaName}`); } getTablesQuery(schemaName) { - return this.query(`SELECT name as table_name FROM system.tables WHERE database = '${schemaName}'`) + return this.query('SELECT name as table_name FROM system.tables WHERE database = ?', [schemaName]) } } diff --git a/packages/cubejs-clickhouse-driver/package.json b/packages/cubejs-clickhouse-driver/package.json index 8cda9c7e4bfb2..0a404c37e4206 100644 --- a/packages/cubejs-clickhouse-driver/package.json +++ b/packages/cubejs-clickhouse-driver/package.json @@ -14,6 +14,7 @@ "@apla/clickhouse": "^1.5.5", "@cubejs-backend/query-orchestrator": "^0.0.21", "generic-pool": "^3.6.0", + "sqlstring": "^2.3.1", "uuidv4": "^4.0.0" }, "license": "Apache-2.0", diff --git a/packages/cubejs-clickhouse-driver/test/ClickHouseDriverTest.js b/packages/cubejs-clickhouse-driver/test/ClickHouseDriverTest.js index ebe044f5ac599..c683fe7589223 100644 --- a/packages/cubejs-clickhouse-driver/test/ClickHouseDriverTest.js +++ b/packages/cubejs-clickhouse-driver/test/ClickHouseDriverTest.js @@ -98,10 +98,24 @@ describe('ClickHouseDriver', () => { await driver.createSchemaIfNotExists(name) } finally { - await driver.query(`drop database ${name}`) + await driver.query(`DROP DATABASE ${name}`) } }) }) - + it('should substitute parameters', async () => { + await doWithDriver(async (driver) => { + let name = `temp_${Date.now()}` + try { + await driver.createSchemaIfNotExists(name); + await driver.query(`CREATE TABLE ${name}.test (x Int32, s String) ENGINE Log`); + await driver.query(`INSERT INTO ${name}.test VALUES (?, ?), (?, ?), (?, ?)`, [1, "str1", 2, "str2", 3, "str3"]); + const values = await driver.query(`SELECT * FROM ${name}.test WHERE x = ?`, 2); + values.should.deepEqual([{x: 2, s: "str2"}]) + } + finally { + await driver.query(`DROP DATABASE ${name}`) + } + }) + }) }) \ No newline at end of file diff --git a/packages/cubejs-clickhouse-driver/yarn.lock b/packages/cubejs-clickhouse-driver/yarn.lock index f36e51d5fea9a..a6d468a5501de 100644 --- a/packages/cubejs-clickhouse-driver/yarn.lock +++ b/packages/cubejs-clickhouse-driver/yarn.lock @@ -369,6 +369,10 @@ split-ca@^1.0.0: version "1.0.1" resolved "https://registry.yarnpkg.com/split-ca/-/split-ca-1.0.1.tgz#6c83aff3692fa61256e0cd197e05e9de157691a6" +sqlstring@^2.3.1: + version "2.3.1" + resolved "https://registry.yarnpkg.com/sqlstring/-/sqlstring-2.3.1.tgz#475393ff9e91479aea62dcaf0ca3d14983a7fb40" + stream-to-array@^2.3.0: version "2.3.0" resolved "https://registry.yarnpkg.com/stream-to-array/-/stream-to-array-2.3.0.tgz#bbf6b39f5f43ec30bc71babcb37557acecf34353" From 690a61111bce058d3dc4c153c53d9bc6ceb6cf01 Mon Sep 17 00:00:00 2001 From: Igor Nikolaev Date: Sun, 28 Apr 2019 21:36:22 +0300 Subject: [PATCH 06/11] Enable join_use_nulls option --- .../driver/ClickHouseDriver.js | 8 +++- .../test/ClickHouseDriverTest.js | 46 +++++++++++++++---- 2 files changed, 43 insertions(+), 11 deletions(-) diff --git a/packages/cubejs-clickhouse-driver/driver/ClickHouseDriver.js b/packages/cubejs-clickhouse-driver/driver/ClickHouseDriver.js index 891ece99f59df..a6499ce5a092d 100644 --- a/packages/cubejs-clickhouse-driver/driver/ClickHouseDriver.js +++ b/packages/cubejs-clickhouse-driver/driver/ClickHouseDriver.js @@ -17,7 +17,11 @@ class ClickHouseDriver extends BaseDriver { ...config }; this.pool = genericPool.createPool({ - create: () => new ClickHouse(this.config), + create: () => new ClickHouse(Object.assign({}, this.config, { + queryOptions: { + join_use_nulls: 1 + } + })), destroy: (connection) => { return Promise.resolve(); }, @@ -84,7 +88,7 @@ class ClickHouseDriver extends BaseDriver { const formattedQuery = sqlstring.format(query, values); return this.withConnection((connection, queryId) => { - return connection.querying(formattedQuery, { dataObjects: true, queryOptions: { query_id: queryId } }) + return connection.querying(formattedQuery, { dataObjects: true, queryOptions: { query_id: queryId, join_use_nulls: 1 } }) .then(res => res.data); }); } diff --git a/packages/cubejs-clickhouse-driver/test/ClickHouseDriverTest.js b/packages/cubejs-clickhouse-driver/test/ClickHouseDriverTest.js index c683fe7589223..b0c0f63d95242 100644 --- a/packages/cubejs-clickhouse-driver/test/ClickHouseDriverTest.js +++ b/packages/cubejs-clickhouse-driver/test/ClickHouseDriverTest.js @@ -19,13 +19,13 @@ describe('ClickHouseDriver', () => { port:container.getMappedPort(8123), } - }) + }); after(async ()=>{ if (container) { await container.stop() } - }) + }); async function doWithDriver(callback) { let driver = new ClickHouseDriver(config) @@ -39,12 +39,14 @@ describe('ClickHouseDriver', () => { it('should construct', async () => { await doWithDriver(driver=>{}) - }) + }); + it('should test connection', async () => { await doWithDriver(async (driver) => { await driver.testConnection() }) - }) + }); + it('should select raw sql', async () => { await doWithDriver(async (driver) => { let numbers = await driver.query("SELECT number FROM system.numbers LIMIT 10") @@ -61,7 +63,8 @@ describe('ClickHouseDriver', () => { { number: '9' }, ]) }) - }) + }); + it('should select raw sql multiple times', async () => { await doWithDriver(async (driver) => { let numbers = await driver.query("SELECT number FROM system.numbers LIMIT 5") @@ -81,7 +84,8 @@ describe('ClickHouseDriver', () => { { number: '4' }, ]) }) - }) + }); + it('should get tables', async () => { await doWithDriver(async (driver) => { let tables = await driver.getTablesQuery("system") @@ -89,7 +93,7 @@ describe('ClickHouseDriver', () => { {table_name:"numbers"}, ]) }) - }) + }); it('should create schema if not exists', async () => { await doWithDriver(async (driver) => { @@ -101,7 +105,7 @@ describe('ClickHouseDriver', () => { await driver.query(`DROP DATABASE ${name}`) } }) - }) + }); it('should substitute parameters', async () => { await doWithDriver(async (driver) => { @@ -117,5 +121,29 @@ describe('ClickHouseDriver', () => { await driver.query(`DROP DATABASE ${name}`) } }) - }) + }); + + it('should return null for missing values on left outer join', async () => { + await doWithDriver(async (driver) => { + let name = `temp_${Date.now()}` + try { + await driver.createSchemaIfNotExists(name); + await driver.query(`CREATE TABLE ${name}.a (x Int32, s String) ENGINE Log`); + await driver.query(`INSERT INTO ${name}.a VALUES (?, ?), (?, ?), (?, ?)`, [1, 'str1', 2, 'str2', 3, 'str3']); + + await driver.query(`CREATE TABLE ${name}.b (x Int32, s String) ENGINE Log`); + await driver.query(`INSERT INTO ${name}.b VALUES (?, ?), (?, ?), (?, ?)`, [2, 'str2', 3, 'str3', 4, 'str4']); + + const values = await driver.query(`SELECT * FROM ${name}.a LEFT OUTER JOIN ${name}.b ON a.x = b.x`); + values.should.deepEqual([ + { x: 1, s: 'str1', 'b.x': 0, 'b.s': null }, + { x: 2, s: 'str2', 'b.x': 2, 'b.s': 'str2' }, + { x: 3, s: 'str3', 'b.x': 3, 'b.s': 'str3' } + ]) + } + finally { + await driver.query(`DROP DATABASE ${name}`) + } + }) + }); }) \ No newline at end of file From 1bc03d0af9726c78bea245b4338b2c7b653413f9 Mon Sep 17 00:00:00 2001 From: Igor Nikolaev Date: Sun, 28 Apr 2019 22:20:53 +0300 Subject: [PATCH 07/11] Add reference to ClickHouse driver --- packages/cubejs-server-core/core/index.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/packages/cubejs-server-core/core/index.js b/packages/cubejs-server-core/core/index.js index 5181fc943c05a..4c63e4fbad749 100644 --- a/packages/cubejs-server-core/core/index.js +++ b/packages/cubejs-server-core/core/index.js @@ -16,7 +16,8 @@ const DriverDependencies = { jdbc: '@cubejs-backend/jdbc-driver', mongobi: '@cubejs-backend/mongobi-driver', bigquery: '@cubejs-backend/bigquery-driver', - redshift: '@cubejs-backend/postgres-driver' + redshift: '@cubejs-backend/postgres-driver', + clickhouse: '@cubejs-backend/clickhouse-driver' }; const checkEnvForPlaceholders = () => { From b6788988ebbdbaf517ac4f85b42720f9fa83f01e Mon Sep 17 00:00:00 2001 From: Igor Nikolaev Date: Sun, 28 Apr 2019 22:23:46 +0300 Subject: [PATCH 08/11] Add clickhouse option to the usage --- packages/cubejs-cli/cubejsCli.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/cubejs-cli/cubejsCli.js b/packages/cubejs-cli/cubejsCli.js index e30a177151dcd..4e4703b7d944b 100644 --- a/packages/cubejs-cli/cubejsCli.js +++ b/packages/cubejs-cli/cubejsCli.js @@ -234,7 +234,7 @@ program .option( '-d, --db-type ', 'Preconfigure for selected database.\n\t\t\t ' + - 'Options: postgres, mysql, mongobi, athena, bigquery, redshift' + 'Options: postgres, mysql, mongobi, athena, bigquery, redshift, clickhouse' ) .option('-t, --template