Skip to content

Commit

Permalink
Merge pull request #222 from Altinity/fix_github_issues
Browse files Browse the repository at this point in the history 
- add adhoc values support, fix #198
- fix AST parsing corner case in `WHERE [test, 'test']` "," was skipped, fix ahdoc ast FROM recursive parsing
- fix corner cases for table functions parsing when adhoc filter applied, fix #99, fix #130
- actualize clickhouse_dashboard.json

Signed-off-by: Eugene Klimov <eklimov@altinity.com>
  • Loading branch information
@Slach
Slach committed Jul 6, 2020
2 parents 26d04a4 + fe4e282 commit 3c5e0b5
Show file tree
Hide file tree
Showing 6 changed files with 101 additions and 31 deletions.
4 changes: 2 additions & 2 deletions dist/module.js
View file

Large diffs are not rendered by default.

2 changes: 1 addition & 1 deletion dist/module.js.map
View file

Large diffs are not rendered by default.

52 changes: 38 additions & 14 deletions docker/grafana/dashboards/clickhouse_dashboard.json
View file
Expand Up @@ -11,7 +11,7 @@
"limit": 100,
"name": "Annotations & Alerts",
"query": "SELECT \n toUInt64(event_time)*1000 AS time,\n if(rand() % 2 = 1, '[begin] description', '[end] description') AS text,\n arrayStringConcat(['test1', 'test2', service_name],',') AS tags\nFROM default.test_grafana\nWHERE\n rand() % 100 IN (1,2) AND\n toUInt64(event_time) >= $from AND toUInt64(event_time) < $to\n",
"rawQuery": "SELECT toUInt64(event_time)*1000 AS time, if(rand() % 2 = 1, '[begin] description', '[end] description') AS text, arrayStringConcat(['test1', 'test2', service_name],',') AS tags FROM default.test_grafana WHERE rand() % 100 IN (1,2) AND toUInt64(event_time) >= 1593728631 AND toUInt64(event_time) < 1593750231",
"rawQuery": "SELECT toUInt64(event_time)*1000 AS time, if(rand() % 2 = 1, '[begin] description', '[end] description') AS text, arrayStringConcat(['test1', 'test2', service_name],',') AS tags FROM default.test_grafana WHERE rand() % 100 IN (1,2) AND toUInt64(event_time) >= 1593992152 AND toUInt64(event_time) < 1594013752",
"showIn": 0,
"type": "dashboard"
}
Expand All @@ -20,7 +20,7 @@
"editable": true,
"gnetId": null,
"graphTooltip": 0,
"iteration": 1593745562480,
"iteration": 1594013092510,
"links": [],
"panels": [
{
Expand Down Expand Up @@ -82,7 +82,7 @@
"interval": "",
"intervalFactor": 1,
"query": "$columns(service_name,\r\n count() c\r\n)\r\nFROM $table WHERE [service_name, ' test array'] IN (${array_var}) AND $timeFilter\r\n",
"rawQuery": "SELECT t, groupArray((service_name, c)) AS groupArr FROM ( SELECT (intDiv(toUInt32(event_time), 20) * 20) * 1000 AS t, service_name, count() c FROM default.test_grafana WHERE event_time >= toDateTime(1593725067) AND [service_name, ' test array'] IN (['postgresql', ' test array'],['mysql', ' test array']) AND event_time >= toDateTime(1593725067) GROUP BY t, service_name ORDER BY t, service_name) GROUP BY t ORDER BY t",
"rawQuery": "SELECT t, groupArray((service_name, c)) AS groupArr FROM ( SELECT (intDiv(toUInt32(event_time), 20) * 20) * 1000 AS t, service_name, count() c FROM default.test_grafana WHERE event_time >= toDateTime(1593992068) AND [service_name , ' test array'] IN (['mysql',' test array'],['postgresql',' test array']) AND event_time >= toDateTime(1593992068) AND service_name LIKE '%sql%' GROUP BY t, service_name ORDER BY t, service_name) GROUP BY t ORDER BY t",
"refId": "A",
"round": "0s",
"table": "test_grafana",
Expand All @@ -109,6 +109,7 @@
},
"yaxes": [
{
"$$hashKey": "object:37",
"format": "short",
"label": null,
"logBase": 1,
Expand All @@ -117,6 +118,7 @@
"show": true
},
{
"$$hashKey": "object:38",
"format": "short",
"label": null,
"logBase": 1,
Expand Down Expand Up @@ -189,8 +191,8 @@
"formattedQuery": "SELECT $timeSeries as t, count() FROM $table WHERE $timeFilter GROUP BY t ORDER BY t",
"interval": "",
"intervalFactor": 1,
"query": "$columns(service_name,\r\n count() c\r\n)\r\nFROM remote('${remote_host}',default.test_grafana) WHERE [service_name, ' test array'] IN (${array_var}) AND $timeFilter\r\n",
"rawQuery": "SELECT t, groupArray((service_name, c)) AS groupArr FROM ( SELECT (intDiv(toUInt32(event_time), 20) * 20) * 1000 AS t, service_name, count() c FROM remote('localhost:9000',default.test_grafana) WHERE event_time >= toDateTime(1593725770) AND [service_name, ' test array'] IN (['postgresql', ' test array'],['mysql', ' test array']) AND event_time >= toDateTime(1593725770) GROUP BY t, service_name ORDER BY t, service_name) GROUP BY t ORDER BY t",
"query": "$columns(service_name,\r\n count() c\r\n)\r\nFROM remote('${remote_host}',default.test_grafana) WHERE [service_name, ' test array'] IN (${array_var}) AND $timeFilter",
"rawQuery": "SELECT t, groupArray((service_name, c)) AS groupArr FROM ( SELECT (intDiv(toUInt32(event_time), 20) * 20) * 1000 AS t, service_name, count() c FROM remote('localhost:9000',default.test_grafana) WHERE event_time >= toDateTime(1593992191) AND [service_name , ' test array'] IN (['mysql',' test array'],['postgresql',' test array']) AND event_time >= toDateTime(1593992191) GROUP BY t, service_name ORDER BY t, service_name) GROUP BY t ORDER BY t",
"refId": "A",
"round": "0s",
"table": "test_grafana",
Expand All @@ -217,6 +219,7 @@
},
"yaxes": [
{
"$$hashKey": "object:357",
"format": "short",
"label": null,
"logBase": 1,
Expand All @@ -225,6 +228,7 @@
"show": true
},
{
"$$hashKey": "object:358",
"format": "short",
"label": null,
"logBase": 1,
Expand Down Expand Up @@ -413,8 +417,8 @@
"formattedQuery": "SELECT $timeSeries as t, count() FROM $table WHERE $timeFilter GROUP BY t ORDER BY t",
"interval": "",
"intervalFactor": 1,
"query": "$columns(service_name,\r\n count() c\r\n)\r\nFROM $table WHERE service_name IN (${repeated_service}) AND $timeFilter\r\n",
"rawQuery": "SELECT t, groupArray((service_name, c)) AS groupArr FROM ( SELECT (intDiv(toUInt32(event_time), 20) * 20) * 1000 AS t, service_name, count() c FROM default.test_grafana WHERE event_time >= toDateTime(1593726153) AND service_name IN (['mysql','postgresql']) AND event_time >= toDateTime(1593726153) GROUP BY t, service_name ORDER BY t, service_name) GROUP BY t ORDER BY t",
"query": "$columns(service_name,\r\n count() c\r\n)\r\nFROM $table WHERE service_name IN (${repeated_service}) AND $timeFilter",
"rawQuery": "SELECT t, groupArray((service_name, c)) AS groupArr FROM ( SELECT (intDiv(toUInt32(event_time), 20) * 20) * 1000 AS t, service_name, count() c FROM default.test_grafana WHERE event_time >= toDateTime(1593992104) AND service_name IN ('mysql') AND event_time >= toDateTime(1593992104) AND service_name LIKE '%sql%' GROUP BY t, service_name ORDER BY t, service_name) GROUP BY t ORDER BY t",
"refId": "A",
"round": "0s",
"table": "test_grafana",
Expand All @@ -441,6 +445,7 @@
},
"yaxes": [
{
"$$hashKey": "object:277",
"format": "short",
"label": null,
"logBase": 1,
Expand All @@ -449,6 +454,7 @@
"show": true
},
{
"$$hashKey": "object:278",
"format": "short",
"label": null,
"logBase": 1,
Expand Down Expand Up @@ -505,7 +511,7 @@
"points": false,
"renderer": "flot",
"repeat": null,
"repeatIteration": 1593745562480,
"repeatIteration": 1594013092510,
"repeatPanelId": 5,
"scopedVars": {
"repeated_service": {
Expand All @@ -531,8 +537,8 @@
"formattedQuery": "SELECT $timeSeries as t, count() FROM $table WHERE $timeFilter GROUP BY t ORDER BY t",
"interval": "",
"intervalFactor": 1,
"query": "$columns(service_name,\r\n count() c\r\n)\r\nFROM $table WHERE service_name IN (${repeated_service}) AND $timeFilter\r\n",
"rawQuery": "SELECT t, groupArray((service_name, c)) AS groupArr FROM ( SELECT (intDiv(toUInt32(event_time), 20) * 20) * 1000 AS t, service_name, count() c FROM default.test_grafana WHERE event_time >= toDateTime(1593726153) AND service_name IN (['mysql','postgresql']) AND event_time >= toDateTime(1593726153) GROUP BY t, service_name ORDER BY t, service_name) GROUP BY t ORDER BY t",
"query": "$columns(service_name,\r\n count() c\r\n)\r\nFROM $table WHERE service_name IN (${repeated_service}) AND $timeFilter",
"rawQuery": "SELECT t, groupArray((service_name, c)) AS groupArr FROM ( SELECT (intDiv(toUInt32(event_time), 20) * 20) * 1000 AS t, service_name, count() c FROM default.test_grafana WHERE event_time >= toDateTime(1593992104) AND service_name IN ('mysql') AND event_time >= toDateTime(1593992104) AND service_name LIKE '%sql%' GROUP BY t, service_name ORDER BY t, service_name) GROUP BY t ORDER BY t",
"refId": "A",
"round": "0s",
"table": "test_grafana",
Expand All @@ -559,6 +565,7 @@
},
"yaxes": [
{
"$$hashKey": "object:277",
"format": "short",
"label": null,
"logBase": 1,
Expand All @@ -567,6 +574,7 @@
"show": true
},
{
"$$hashKey": "object:278",
"format": "short",
"label": null,
"logBase": 1,
Expand Down Expand Up @@ -639,7 +647,7 @@
"interval": "",
"intervalFactor": 1,
"query": "SELECT\r\n $timeSeries AS t,\r\n service_name,\r\n _table,\r\n count() c\r\nFROM merge('default','^test_grafana$')\r\nWHERE service_name IN (${repeated_service}) AND $timeFilter\r\nGROUP BY t, service_name, _table\r\nORDER BY t\r\n",
"rawQuery": "SELECT (intDiv(toUInt32(event_time), 20) * 20) * 1000 AS t, service_name, _table, count() c FROM merge('default','^test_grafana$') WHERE service_name IN (['mysql','postgresql']) AND event_time >= toDateTime(1593729034) GROUP BY t, service_name, _table ORDER BY t",
"rawQuery": "SELECT (intDiv(toUInt32(event_time), 20) * 20) * 1000 AS t, service_name, _table, count() c FROM merge('default','^test_grafana$') WHERE service_name IN (['mysql','postgresql']) AND event_time >= toDateTime(1593991951) GROUP BY t, service_name, _table ORDER BY t",
"refId": "A",
"round": "0s",
"table": "test_grafana",
Expand All @@ -666,6 +674,7 @@
},
"yaxes": [
{
"$$hashKey": "object:37",
"format": "short",
"label": null,
"logBase": 1,
Expand All @@ -674,6 +683,7 @@
"show": true
},
{
"$$hashKey": "object:38",
"format": "short",
"label": null,
"logBase": 1,
Expand All @@ -697,7 +707,6 @@
"allValue": null,
"current": {
"selected": true,
"tags": [],
"text": "All",
"value": [
"$__all"
Expand Down Expand Up @@ -835,7 +844,6 @@
"allValue": null,
"current": {
"selected": true,
"tags": [],
"text": "mysql + postgresql",
"value": [
"mysql",
Expand All @@ -860,6 +868,22 @@
"tagsQuery": "",
"type": "query",
"useTags": false
},
{
"datasource": "clickhouse",
"filters": [
{
"condition": "",
"key": "default.test_grafana.service_name",
"operator": "=~",
"value": "%sql%"
}
],
"hide": 0,
"label": null,
"name": "adhoc",
"skipUrlSync": false,
"type": "adhoc"
}
]
},
Expand All @@ -883,5 +907,5 @@
"timezone": "",
"title": "clickhouse dashboard",
"uid": "DnOcQSGGz",
"version": 1
"version": 3
}
56 changes: 44 additions & 12 deletions src/adhoc.ts
View file
@@ -1,5 +1,6 @@
const queryFilter = "database != 'system'";
const columnsQuery = "SELECT database, table, name, type FROM system.columns WHERE {filter} ORDER BY database, table";
const valuesQuery = "SELECT DISTINCT {field} AS value FROM {database}.{table} LIMIT 300";
const regexEnum = /'(?:[^']+|'')+'/gmi;

export default class AdhocCtrl {
Expand Down Expand Up @@ -35,21 +36,12 @@ export default class AdhocCtrl {
}
return this.datasource.metricFindQuery(q)
.then(function (response) {
return self.processResponse(response);
return self.processTagKeysResponse(response);
});
}

// GetTagValues returns column values according to passed options
// It supposed that values were already fetched in GetTagKeys func and stored in `tagValues`
GetTagValues(options) {
if (this.tagValues.hasOwnProperty(options.key)) {
return Promise.resolve(this.tagValues[options.key]);
}
return Promise.resolve([]);
}

processResponse(response) {
var self = this;
processTagKeysResponse(response) {
let self = this;
let columnNames = {};
response.forEach(function (item) {
let text = item.table + '.' + item.name;
Expand All @@ -76,4 +68,44 @@ export default class AdhocCtrl {
});
return Promise.resolve(self.tagKeys);
}

// GetTagValues returns column values according to passed options
// Values for fields with Enum type were already fetched in GetTagKeys func and stored in `tagValues`
// Values for fields which not represented on `tagValues` get from ClickHouse and cached on `tagValues`
GetTagValues(options) {
let self = this;
if (this.tagValues.hasOwnProperty(options.key)) {
return Promise.resolve(this.tagValues[options.key]);
}
let key_items = options.key.split('.');
if (key_items.length < 2 || (key_items.length == 2 && this.datasource.defaultDatabase.length == 0) || key_items.length > 3) {
return Promise.resolve([]);
}
let field, database, table;
if (key_items.length == 3) {
[database, table, field] = key_items;
}
if (key_items.length == 2) {
database = self.datasource.defaultDatabase;
[table, field] = key_items;
}
let q = valuesQuery
.replace('{field}', field)
.replace('{database}', database)
.replace('{table}', table);

return this.datasource.metricFindQuery(q)
.then(function (response) {
self.tagValues[options.key] = self.processTagValuesResponse(response);
return self.tagValues[options.key];
});
}

processTagValuesResponse(response) {
let tagValues = [];
response.forEach(function (item) {
tagValues.push({text: item.text, value: item.text});
});
return Promise.resolve(tagValues);
}
}
16 changes: 15 additions & 1 deletion src/scanner.ts
View file
Expand Up @@ -119,10 +119,18 @@ export default class Scanner {
} else if (this.token === ',' && isClosured(argument)) {
this.push(argument);
argument = '';
if (this.rootToken == 'where') {
this.push(this.token);
}
this.expectedNext = true;
} else if (isClosureChars(this.token) && this.rootToken === 'from') {
subQuery = betweenBraces(this._s);
this.tree[this.rootToken] = toAST(subQuery);
if (!isTableFunc(argument)) {
this.tree[this.rootToken] = toAST(subQuery);
} else {
this.push(argument + '(' + subQuery + ')');
argument = '';
}
this._s = this._s.substring(subQuery.length + 1);
} else if (isMacroFunc(this.token)) {
let func = this.token;
Expand Down Expand Up @@ -387,6 +395,7 @@ const wsRe = "\\s+",
macroRe = "\\$[A-Za-z0-9_$]+",
skipSpaceRe = "[\\(\\.! \\[]",

tableFuncRe = "\\b(remote|remoteSecure|cluster|clusterAllReplicas|merge|numbers|url|mysql|jdbc|odbc|hdfs|input|generateRandom)\\b",
builtInFuncRe = "\\b(avg|countIf|first|last|max|min|sum|sumIf|ucase|lcase|mid|round|rank|now|" +
"coalesce|ifnull|isnull|nvl|count|timeSlot|yesterday|today|now|toRelativeSecondNum|" +
"toRelativeMinuteNum|toRelativeHourNum|toRelativeDayNum|toRelativeWeekNum|toRelativeMonthNum|" +
Expand Down Expand Up @@ -441,6 +450,7 @@ const wsRe = "\\s+",
operatorOnlyRe = new RegExp("^(?:" + operatorRe + ")$", 'i'),
dataTypeOnlyRe = new RegExp("^(?:" + dataTypeRe + ")$"),
builtInFuncOnlyRe = new RegExp("^(?:" + builtInFuncRe + ")$"),
tableFuncOnlyRe = new RegExp("^(?:" + tableFuncRe + ")$", 'i'),
macroOnlyRe = new RegExp("^(?:" + macroRe + ")$", 'i'),
inOnlyRe = new RegExp("^(?:" + inRe + ")$", 'i'),
condOnlyRe = new RegExp("^(?:" + condRe + ")$", 'i'),
Expand Down Expand Up @@ -508,6 +518,10 @@ function isBuiltInFunc(token) {
return builtInFuncOnlyRe.test(token);
}

function isTableFunc(token) {
return tableFuncOnlyRe.test(token);
}

function isClosureChars(token) {
return closureOnlyRe.test(token);
}
Expand Down
2 changes: 1 addition & 1 deletion src/sql_query.ts
View file
Expand Up @@ -46,7 +46,7 @@ export default class SqlQuery {
let topQuery = ast;
if (adhocFilters.length > 0) {
/* Check subqueries for ad-hoc filters */
while (!isArray(ast.from)) {
while (ast.hasOwnProperty('from') && !isArray(ast.from)) {
ast = ast.from;
}
if (!ast.hasOwnProperty('where')) {
Expand Down

0 comments on commit 3c5e0b5

Please sign in to comment.