diff --git a/deploy/grafana/grafana-with-grafana-operator/install-grafana-with-operator.sh b/deploy/grafana/grafana-with-grafana-operator/install-grafana-with-operator.sh index 4acc9b92e..969eea8b8 100755 --- a/deploy/grafana/grafana-with-grafana-operator/install-grafana-with-operator.sh +++ b/deploy/grafana/grafana-with-grafana-operator/install-grafana-with-operator.sh @@ -179,8 +179,8 @@ if ! command -v yq &> /dev/null; then echo "Install 'yq', see installation instruction in https://github.com/mikefarah/yq/#install" exit 1 else - export OPERATOR_CH_USER=$(yq eval .clickhouse.access.username ${CUR_DIR}/../../../config/config.yaml) - export OPERATOR_CH_PASS=$(yq eval .clickhouse.access.password ${CUR_DIR}/../../../config/config.yaml) + export OPERATOR_CH_USER=$(yq 'select(.kind == "Secret") | .stringData.username' "${CUR_DIR}/../../operator/clickhouse-operator-install-bundle.yaml") + export OPERATOR_CH_PASS=$(yq 'select(.kind == "Secret") | .stringData.password' "${CUR_DIR}/../../operator/clickhouse-operator-install-bundle.yaml") fi echo "Create ClickHouse DataSource for each ClickHouseInstallation" @@ -199,6 +199,23 @@ for LINE in $(kubectl get --all-namespaces chi -o custom-columns=NAMESPACE:.meta clickhouse-client --echo -mn -q 'SELECT hostName(), dummy FROM system.one SETTINGS log_queries=1; SYSTEM FLUSH LOGS' done + echo "Ensure clickhouse-operator will connect to any chi from grafana" + POD_NETWORK_CIDR=$(kubectl get cm -n kube-system kube-proxy -o yaml | grep clusterCIDR | cut -d ":" -f 2) + kubectl apply --validate=${VALIDATE_YAML} --namespace="${NAMESPACE}" -f <( + { + echo "apiVersion: clickhouse.altinity.com/v1" + echo "kind: ClickHouseInstallationTemplate" + echo "metadata:" + echo " name: clickhouse-operator-from-cluster-network" + echo "spec:" + echo " templating:" + echo " policy: auto" + echo " configuration:" + echo " users:" + echo " ${OPERATOR_CH_USER}/networks/ip: ${POD_NETWORK_CIDR}" + } + ) + GRAFANA_CLICKHOUSE_DATASOURCE_NAME="k8s-${NAMESPACE}-${CHI}" CLICKHOUSE_URL="http://${ENDPOINT}:${PORT}" echo "Create ClickHouse DataSource for ClickHouseInstallation ${CHI} '${GRAFANA_NAMESPACE}/${GRAFANA_CLICKHOUSE_DATASOURCE_NAME}'" diff --git a/deploy/prometheus/create-prometheus.sh b/deploy/prometheus/create-prometheus.sh index 0d3ba4fbb..274db7999 100755 --- a/deploy/prometheus/create-prometheus.sh +++ b/deploy/prometheus/create-prometheus.sh @@ -6,7 +6,7 @@ echo "External value for \$VALIDATE_YAML=$VALIDATE_YAML" export PROMETHEUS_NAMESPACE="${PROMETHEUS_NAMESPACE:-prometheus}" export OPERATOR_NAMESPACE="${OPERATOR_NAMESPACE:-kube-system}" -export PROMETHEUS_OPERATOR_BRANCH="${PROMETHEUS_OPERATOR_BRANCH:-v0.62.0}" +export PROMETHEUS_OPERATOR_BRANCH="${PROMETHEUS_OPERATOR_BRANCH:-v0.68.0}" export ALERT_MANAGER_EXTERNAL_URL="${ALERT_MANAGER_EXTERNAL_URL:-http://localhost:9093}" # Possible values for "validate yaml" are values from --validate=XXX kubectl option. They are true/false ATM export VALIDATE_YAML="${VALIDATE_YAML:-true}" diff --git a/grafana-dashboard/ClickHouse_Queries_dashboard.json b/grafana-dashboard/ClickHouse_Queries_dashboard.json index 69bb0656c..ef7fc6158 100644 --- a/grafana-dashboard/ClickHouse_Queries_dashboard.json +++ b/grafana-dashboard/ClickHouse_Queries_dashboard.json @@ -14,7 +14,7 @@ "type": "grafana", "id": "grafana", "name": "Grafana", - "version": "7.5.16" + "version": "7.5.17" }, { "type": "panel", @@ -43,7 +43,7 @@ { "type": "datasource", "id": "vertamedia-clickhouse-datasource", - "name": "ClickHouse", + "name": "Altinity plugin for ClickHouse", "version": "2.5.3" } ], @@ -65,7 +65,7 @@ "gnetId": 2515, "graphTooltip": 1, "id": null, - "iteration": 1596171309294, + "iteration": 1694526851642, "links": [], "panels": [ { @@ -89,11 +89,17 @@ "dashLength": 10, "dashes": false, "datasource": "$db", + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 7, - "w": 20, + "w": 24, "x": 0, "y": 1 }, @@ -117,9 +123,10 @@ "links": [], "nullPointMode": "null", "options": { - "dataLinks": [] + "alertThreshold": true }, "percentage": false, + "pluginVersion": "7.5.17", "pointradius": 5, "points": false, "renderer": "flot", @@ -136,14 +143,16 @@ "dateTimeType": "DATETIME", "datetimeLoading": false, "expr": "", + "extrapolate": false, "format": "time_series", - "formattedQuery": "SELECT * FROM ($rateColumns(\n substring(query, 1, 45) AS query,\n count() c)\nFROM cluster('all-sharded',system.query_log)\nWHERE\n cityHash64(query) global in (\n SELECT cityHash64(substring(query, 1, 45)) AS h\n FROM cluster('all-sharded',system.query_log)\n WHERE\n $timeFilter\n AND type in ($type)\n AND initial_user in ($user)\n AND('$query_type' = 'all' or(positionCaseInsensitive(query, '$query_type') = 1))\n GROUP BY h\n ORDER BY count() desc\n LIMIT $top)\n AND type in ($type)\n AND initial_user in ($user)\n AND('$query_type' = 'all' or(positionCaseInsensitive(query, '$query_type') = 1))) SETTINGS skip_unavailable_shards=1", + "interval": "", "intervalFactor": 2, - "query": "SELECT\r\n t,\r\n arrayMap(a -> (a.1, a.2 / runningDifference(t / 1000)), groupArr)\r\nFROM (\r\n SELECT t, groupArray((query, c)) AS groupArr\r\n FROM (\r\n SELECT\r\n (intDiv(toUInt32(event_time), 2) * 2) * 1000 AS t,\r\n substring(query, 1, 45) AS query,\r\n count() c\r\n FROM cluster('all-sharded',system.query_log)\r\n WHERE $timeFilter\r\n AND cityHash64(query) GLOBAL IN (\r\n SELECT cityHash64(substring(query, 1, 45)) AS h\r\n FROM cluster('all-sharded',system.query_log)\r\n WHERE $timeFilter\r\n AND type IN ($type)\r\n AND initial_user IN ($user)\r\n AND ('$query_type' = 'all' OR (positionCaseInsensitive(query, '$query_type') = 1))\r\n GROUP BY h\r\n ORDER BY count() DESC\r\n LIMIT $top\r\n SETTINGS skip_unavailable_shards=1\r\n )\r\n AND type IN ($type)\r\n AND initial_user IN ($user)\r\n AND ('$query_type' = 'all' OR (positionCaseInsensitive(query, '$query_type') = 1))\r\n GROUP BY t, query\r\n ORDER BY t\r\n )\r\n GROUP BY t\r\n ORDER BY t\r\n) SETTINGS skip_unavailable_shards=1", - "rawQuery": "SELECT t, arrayMap(a -> (a.1, a.2 / runningDifference(t / 1000)), groupArr) FROM ( SELECT t, groupArray((query, c)) AS groupArr FROM ( SELECT (intDiv(toUInt32(event_time), 2) * 2) * 1000 AS t, substring(query, 1, 45) AS query, count() c FROM cluster('all-sharded',system.query_log) WHERE event_date >= toDate(1586192379) AND event_time >= toDateTime(1586192379) AND cityHash64(query) GLOBAL IN ( SELECT cityHash64(substring(query, 1, 45)) AS h FROM cluster('all-sharded',system.query_log) WHERE event_date >= toDate(1586192379) AND event_time >= toDateTime(1586192379) AND type IN (1,2,3,4) AND initial_user IN ('default') AND ('all' = 'all' OR (positionCaseInsensitive(query, 'all') = 1)) GROUP BY h ORDER BY count() DESC LIMIT 5 SETTINGS skip_unavailable_shards=1 ) AND type IN (1,2,3,4) AND initial_user IN ('default') AND ('all' = 'all' OR (positionCaseInsensitive(query, 'all') = 1)) GROUP BY t, query ORDER BY t ) GROUP BY t ORDER BY t ) SETTINGS skip_unavailable_shards=1", + "query": "SELECT\r\n t,\r\n arrayMap(a -> (a.1, a.2 / runningDifference(t / 1000)), groupArr)\r\nFROM (\r\n SELECT t, groupArray((q, c)) AS groupArr\r\n FROM (\r\n SELECT\r\n (intDiv(toUInt32(event_time), 2) * 2) * 1000 AS t,\r\n normalizeQuery(query) AS q,\r\n count() c\r\n FROM cluster('all-sharded',system.query_log)\r\n WHERE $timeFilter\r\n AND( ('$type' = '1,2,3,4' AND type != 'QueryStart') OR ('$type' != '1,2,3,4' AND type IN ($type)))\r\n $conditionalTest(AND query_kind IN ($query_kind), $query_kind)\r\n $conditionalTest(AND initial_user IN ($user), $user)\r\n $conditionalTest(AND query_duration_ms >= $min_duration_ms, $min_duration_ms)\r\n $conditionalTest(AND query_duration_ms <= $max_duration_ms, $max_duration_ms)\r\n AND normalized_query_hash GLOBAL IN (\r\n SELECT normalized_query_hash AS h\r\n FROM cluster('all-sharded',system.query_log)\r\n WHERE $timeFilter\r\n AND( ('$type' = '1,2,3,4' AND type != 'QueryStart') OR ('$type' != '1,2,3,4' AND type IN ($type)))\r\n $conditionalTest(AND query_kind IN ($query_kind), $query_kind)\r\n $conditionalTest(AND type IN ($type), $type)\r\n $conditionalTest(AND initial_user IN ($user), $user)\r\n $conditionalTest(AND query_duration_ms >= $min_duration_ms, $min_duration_ms)\r\n $conditionalTest(AND query_duration_ms <= $max_duration_ms, $max_duration_ms)\r\n GROUP BY h\r\n ORDER BY count() DESC\r\n LIMIT $top\r\n SETTINGS skip_unavailable_shards=1\r\n )\r\n GROUP BY t, query\r\n ORDER BY t\r\n )\r\n GROUP BY t\r\n ORDER BY t\r\n) SETTINGS skip_unavailable_shards=1", + "rawQuery": "SELECT\r\n t,\r\n arrayMap(a -> (a.1, a.2 / runningDifference(t / 1000)), groupArr)\r\nFROM (\r\n SELECT t, groupArray((q, c)) AS groupArr\r\n FROM (\r\n SELECT\r\n (intDiv(toUInt32(event_time), 2) * 2) * 1000 AS t,\r\n normalizeQuery(query) AS q,\r\n count() c\r\n FROM cluster('all-sharded',system.query_log)\r\n WHERE event_date >= toDate(1694531137) AND event_date <= toDate(1694534737) AND event_time >= toDateTime(1694531137) AND event_time <= toDateTime(1694534737)\r\n AND( ('1,2,3,4' = '1,2,3,4' AND type != 'QueryStart') OR ('1,2,3,4' != '1,2,3,4' AND type IN (1,2,3,4)))\r\n \r\n \r\n \r\n \r\n AND normalized_query_hash GLOBAL IN (\r\n SELECT normalized_query_hash AS h\r\n FROM cluster('all-sharded',system.query_log)\r\n WHERE event_date >= toDate(1694531137) AND event_date <= toDate(1694534737) AND event_time >= toDateTime(1694531137) AND event_time <= toDateTime(1694534737)\r\n AND( ('1,2,3,4' = '1,2,3,4' AND type != 'QueryStart') OR ('1,2,3,4' != '1,2,3,4' AND type IN (1,2,3,4)))\r\n \r\n \r\n \r\n \r\n \r\n GROUP BY h\r\n ORDER BY count() DESC\r\n LIMIT 30\r\n SETTINGS skip_unavailable_shards=1\r\n )\r\n GROUP BY t, query\r\n ORDER BY t\r\n )\r\n GROUP BY t\r\n ORDER BY t\r\n) SETTINGS skip_unavailable_shards=1", "refId": "A", "resultFormat": "time_series", "round": "0s", + "skip_comments": true, "table": "query_log", "tableLoading": false } @@ -152,7 +161,7 @@ "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Top $top request's rate by type: $type; user: $user; query type: $query_type", + "title": "Top $top request's rate by type: $type; user: $user; query kind: $query_kind", "tooltip": { "shared": true, "sort": 2, @@ -189,26 +198,57 @@ "alignLevel": null } }, - { - "content": "1 - successful start of query execution\n\n2 - successful end of query execution\n\n3 - exception before start of query execution\n\n4 - exception while query execution", - "datasource": "$db", - "gridPos": { - "h": 7, - "w": 4, - "x": 20, - "y": 1 - }, - "id": 17, - "links": [], - "mode": "markdown", - "options": {}, - "title": "Types", - "transparent": true, - "type": "text" - }, { "columns": [], "datasource": "$db", + "fieldConfig": { + "defaults": { + "custom": { + "align": null, + "filterable": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "duration_ms" + }, + "properties": [ + { + "id": "custom.width", + "value": 100 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "count" + }, + "properties": [ + { + "id": "custom.width", + "value": 100 + } + ] + } + ] + }, "fontSize": "100%", "gridPos": { "h": 10, @@ -219,8 +259,12 @@ "height": "400px", "id": 18, "links": [], - "options": {}, + "options": { + "showHeader": true, + "sortBy": [] + }, "pageSize": null, + "pluginVersion": "7.5.17", "scroll": true, "showHeader": true, "sort": { @@ -230,12 +274,14 @@ "styles": [ { "alias": "Time", + "align": "auto", "dateFormat": "YYYY-MM-DD HH:mm:ss", "pattern": "t", "type": "hidden" }, { "alias": "duration", + "align": "auto", "colorMode": null, "colors": [ "rgba(245, 54, 54, 0.9)", @@ -251,6 +297,7 @@ }, { "alias": "", + "align": "auto", "colorMode": null, "colors": [ "rgba(245, 54, 54, 0.9)", @@ -266,6 +313,7 @@ }, { "alias": "", + "align": "auto", "colorMode": null, "colors": [ "rgba(245, 54, 54, 0.9)", @@ -288,25 +336,79 @@ "dateTimeType": "DATETIME", "datetimeLoading": false, "expr": "", + "extrapolate": false, "format": "table", - "formattedQuery": "SELECT
    rand() as t,
    substring(query,  1,  70) AS query,
    avg(query_duration_ms) duration,
    count() count
FROM cluster('all-sharded',system.query_log)
WHERE
    $timeFilter
    AND type in ($type)
    and initial_user in ($user)
    and('$query_type' = 'all' or(positionCaseInsensitive(query,  '$query_type') = 1))
GROUP BY query
ORDER BY duration desc
LIMIT $top", + "interval": "", "intervalFactor": 2, - "query": "SELECT\n rand() as t,\n substring(query, 1, 70) AS query,\n avg(query_duration_ms) duration,\n count() count\nFROM cluster('all-sharded',system.query_log)\nWHERE\n $timeFilter\n AND type in ($type)\n and initial_user in ($user)\n and('$query_type' = 'all' or(positionCaseInsensitive(query, '$query_type') = 1))\nGROUP BY query\nORDER BY duration desc\nLIMIT $top", - "rawQuery": "SELECT t, arrayMap(a -> (a.1, a.2 / runningDifference(t / 1000)), groupArr) FROM ( SELECT t, groupArray((query, c)) AS groupArr FROM ( SELECT (intDiv(toUInt32(event_time), 2) * 2) * 1000 AS t, substring(query, 1, 45) AS query, count() c FROM cluster('all-sharded',system.query_log) WHERE event_date >= toDate(1586234275) AND event_time >= toDateTime(1586234275) AND cityHash64(query) GLOBAL IN ( SELECT cityHash64(substring(query, 1, 45)) AS h FROM cluster('all-sharded',system.query_log) WHERE event_date >= toDate(1586234275) AND event_time >= toDateTime(1586234275) AND type IN (1,2,3,4) AND initial_user IN ('default') AND ('all' = 'all' OR (positionCaseInsensitive(query, 'all') = 1)) GROUP BY h ORDER BY count() DESC LIMIT 5 SETTINGS skip_unavailable_shards=1 ) AND type IN (1,2,3,4) AND initial_user IN ('default') AND ('all' = 'all' OR (positionCaseInsensitive(query, 'all') = 1)) GROUP BY t, query ORDER BY t ) GROUP BY t ORDER BY t ) SETTINGS skip_unavailable_shards=1", + "query": "SELECT\n normalizeQuery(query) AS query,\n avg(query_duration_ms) duration_ms,\n count() count\nFROM cluster('all-sharded',system.query_log)\nWHERE\n $timeFilter\n AND( ('$type' = '1,2,3,4' AND type != 'QueryStart') OR ('$type' != '1,2,3,4' AND type IN ($type)))\n $conditionalTest(AND query_kind IN ($query_kind), $query_kind)\n $conditionalTest(AND initial_user IN ($user), $user)\n $conditionalTest(AND query_duration_ms >= $min_duration_ms,$min_duration_ms)\n $conditionalTest(AND query_duration_ms <= $max_duration_ms,$max_duration_ms)\nGROUP BY query\nORDER BY duration_ms DESC\nLIMIT $top", + "rawQuery": "SELECT\n normalizeQuery(query) AS query,\n avg(query_duration_ms) duration_ms,\n count() count\nFROM cluster('all-sharded',system.query_log)\nWHERE\n event_date >= toDate(1694531150) AND event_date <= toDate(1694534750) AND event_time >= toDateTime(1694531150) AND event_time <= toDateTime(1694534750)\n AND( ('1,2,3,4' = '1,2,3,4' AND type != 'QueryStart') OR ('1,2,3,4' != '1,2,3,4' AND type IN (1,2,3,4)))\n \n \n \n \nGROUP BY query\nORDER BY duration_ms DESC\nLIMIT 30", "refId": "A", "resultFormat": "time_series", "round": "0s", + "skip_comments": true, "table": "query_log", "tableLoading": false } ], - "title": "Top slow queries by type: $type; user: $user; query type: $query_type", + "title": "Top slow queries by type: $type; user: $user; query kind: $query_kind", "transform": "table", "type": "table" }, { "columns": [], "datasource": "$db", + "fieldConfig": { + "defaults": { + "custom": { + "align": null, + "filterable": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "usage" + }, + "properties": [ + { + "id": "unit", + "value": "bytes" + }, + { + "id": "custom.width", + "value": 100 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "count" + }, + "properties": [ + { + "id": "custom.width", + "value": 100 + } + ] + } + ] + }, "fontSize": "100%", "gridPos": { "h": 10, @@ -317,8 +419,11 @@ "height": "400px", "id": 19, "links": [], - "options": {}, + "options": { + "showHeader": true + }, "pageSize": null, + "pluginVersion": "7.5.17", "scroll": true, "showHeader": true, "sort": { @@ -328,12 +433,14 @@ "styles": [ { "alias": "Time", + "align": "auto", "dateFormat": "YYYY-MM-DD HH:mm:ss", "pattern": "t", "type": "hidden" }, { "alias": "usage", + "align": "auto", "colorMode": null, "colors": [ "rgba(245, 54, 54, 0.9)", @@ -349,6 +456,7 @@ }, { "alias": "", + "align": "auto", "colorMode": null, "colors": [ "rgba(245, 54, 54, 0.9)", @@ -364,6 +472,7 @@ }, { "alias": "", + "align": "auto", "colorMode": null, "colors": [ "rgba(245, 54, 54, 0.9)", @@ -386,25 +495,74 @@ "dateTimeType": "DATETIME", "datetimeLoading": false, "expr": "", + "extrapolate": false, "format": "table", - "formattedQuery": "SELECT
    rand() as t,
    substring(query,  1,  70) AS query,
    avg(memory_usage) usage,
    count() count
FROM cluster('all-sharded',system.query_log)
WHERE
    $timeFilter
    AND type in ($type)
    and initial_user in ($user)
    and('$query_type' = 'all' or(positionCaseInsensitive(query,  '$query_type') = 1))
GROUP BY query
ORDER BY usage desc
LIMIT $top", "intervalFactor": 2, - "query": "SELECT\n rand() as t,\n substring(query, 1, 70) AS query,\n avg(memory_usage) usage,\n count() count\nFROM cluster('all-sharded',system.query_log)\nWHERE\n $timeFilter\n AND type IN ($type)\n AND initial_user IN ($user)\n AND ('$query_type' = 'all' OR (positionCaseInsensitive(query, '$query_type') = 1))\nGROUP BY query\nORDER BY usage DESC\nLIMIT $top", - "rawQuery": "SELECT t, arrayMap(a -> (a.1, a.2 / runningDifference(t / 1000)), groupArr) FROM ( SELECT t, groupArray((query, c)) AS groupArr FROM ( SELECT (intDiv(toUInt32(event_time), 2) * 2) * 1000 AS t, substring(query, 1, 45) AS query, count() c FROM cluster('all-sharded',system.query_log) WHERE event_date >= toDate(1586234300) AND event_time >= toDateTime(1586234300) AND cityHash64(query) GLOBAL IN ( SELECT cityHash64(substring(query, 1, 45)) AS h FROM cluster('all-sharded',system.query_log) WHERE event_date >= toDate(1586234300) AND event_time >= toDateTime(1586234300) AND type IN (1,2,3,4) AND initial_user IN ('default') AND ('all' = 'all' OR (positionCaseInsensitive(query, 'all') = 1)) GROUP BY h ORDER BY count() DESC LIMIT 5 SETTINGS skip_unavailable_shards=1 ) AND type IN (1,2,3,4) AND initial_user IN ('default') AND ('all' = 'all' OR (positionCaseInsensitive(query, 'all') = 1)) GROUP BY t, query ORDER BY t ) GROUP BY t ORDER BY t ) SETTINGS skip_unavailable_shards=1", + "query": "SELECT\n normalizeQuery(query) AS query,\n avg(memory_usage) usage,\n count() count\nFROM cluster('all-sharded',system.query_log)\nWHERE $timeFilter\n AND( ('$type' = '1,2,3,4' AND type != 'QueryStart') OR ('$type' != '1,2,3,4' AND type IN ($type)))\n $conditionalTest(AND query_kind IN ($query_kind), $query_kind)\n $conditionalTest(AND initial_user IN ($user), $user)\n $conditionalTest(AND query_duration_ms >= $min_duration_ms, $min_duration_ms)\n $conditionalTest(AND query_duration_ms <= $max_duration_ms, $max_duration_ms)\nGROUP BY query\nORDER BY usage DESC\nLIMIT $top", + "rawQuery": "SELECT\n normalizeQuery(query) AS query,\n avg(memory_usage) usage,\n count() count\nFROM cluster('all-sharded',system.query_log)\nWHERE event_date >= toDate(1694531173) AND event_date <= toDate(1694534773) AND event_time >= toDateTime(1694531173) AND event_time <= toDateTime(1694534773)\n AND( ('1,2,3,4' = '1,2,3,4' AND type != 'QueryStart') OR ('1,2,3,4' != '1,2,3,4' AND type IN (1,2,3,4)))\n \n \n \n \nGROUP BY query\nORDER BY usage DESC\nLIMIT 30", "refId": "A", "resultFormat": "time_series", "round": "0s", + "skip_comments": true, "table": "query_log", "tableLoading": false } ], - "title": "Top memory consumers by type: $type; user: $user; query type: $query_type", + "title": "Top memory consumers by type: $type; user: $user; query kind: $query_kind", "transform": "table", "type": "table" }, { "columns": [], "datasource": "$db", + "fieldConfig": { + "defaults": { + "custom": { + "align": null, + "filterable": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "type" + }, + "properties": [ + { + "id": "custom.width", + "value": 150 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "count" + }, + "properties": [ + { + "id": "custom.width", + "value": 100 + } + ] + } + ] + }, "fontSize": "100%", "gridPos": { "h": 10, @@ -415,8 +573,11 @@ "height": "400px", "id": 20, "links": [], - "options": {}, + "options": { + "showHeader": true + }, "pageSize": null, + "pluginVersion": "7.5.17", "scroll": true, "showHeader": true, "sort": { @@ -426,12 +587,14 @@ "styles": [ { "alias": "Time", + "align": "auto", "dateFormat": "YYYY-MM-DD HH:mm:ss", "pattern": "t", "type": "hidden" }, { "alias": "type", + "align": "auto", "colorMode": null, "colors": [ "rgba(245, 54, 54, 0.9)", @@ -447,6 +610,7 @@ }, { "alias": "", + "align": "auto", "colorMode": null, "colors": [ "rgba(245, 54, 54, 0.9)", @@ -462,6 +626,7 @@ }, { "alias": "", + "align": "auto", "colorMode": null, "colors": [ "rgba(245, 54, 54, 0.9)", @@ -484,19 +649,20 @@ "dateTimeType": "DATETIME", "datetimeLoading": false, "expr": "", + "extrapolate": false, "format": "table", - "formattedQuery": "SELECT
    rand() as t,
    substring(query,  1,  70) AS query,
    type,
    count() count
FROM cluster('all-sharded',system.query_log)
WHERE
    $timeFilter
    AND type in (3,4)
    and initial_user in ($user)
    and('$query_type' = 'all' or(positionCaseInsensitive(query,  '$query_type') = 1))
GROUP BY
    query,
    type
ORDER BY count desc
LIMIT $top", "intervalFactor": 2, - "query": "SELECT\n rand() as t,\n substring(query, 1, 70) AS query,\n type,\n count() count\nFROM cluster('all-sharded',system.query_log)\nWHERE\n $timeFilter\n AND type IN (3,4)\n AND initial_user IN ($user)\n AND ('$query_type' = 'all' OR (positionCaseInsensitive(query, '$query_type') = 1))\nGROUP BY\n query,\n type\nORDER BY count desc\nLIMIT $top", - "rawQuery": "SELECT t, arrayMap(a -> (a.1, a.2 / runningDifference(t / 1000)), groupArr) FROM ( SELECT t, groupArray((query, c)) AS groupArr FROM ( SELECT (intDiv(toUInt32(event_time), 2) * 2) * 1000 AS t, substring(query, 1, 45) AS query, count() c FROM cluster('all-sharded',system.query_log) WHERE event_date >= toDate(1586234335) AND event_time >= toDateTime(1586234335) AND cityHash64(query) GLOBAL IN ( SELECT cityHash64(substring(query, 1, 45)) AS h FROM cluster('all-sharded',system.query_log) WHERE event_date >= toDate(1586234335) AND event_time >= toDateTime(1586234335) AND type IN (1,2,3,4) AND initial_user IN ('default') AND ('all' = 'all' OR (positionCaseInsensitive(query, 'all') = 1)) GROUP BY h ORDER BY count() DESC LIMIT 5 SETTINGS skip_unavailable_shards=1 ) AND type IN (1,2,3,4) AND initial_user IN ('default') AND ('all' = 'all' OR (positionCaseInsensitive(query, 'all') = 1)) GROUP BY t, query ORDER BY t ) GROUP BY t ORDER BY t ) SETTINGS skip_unavailable_shards=1", + "query": "SELECT\n normalizeQuery(query) AS query,\n type,\n count() count\nFROM cluster('all-sharded',system.query_log)\nWHERE $timeFilter\n AND( ('$type' = '1,2,3,4' AND type != 'QueryStart') OR ('$type' != '1,2,3,4' AND type IN ($type)))\n AND type NOT IN ('QueryStart','QueryFinish')\n $conditionalTest(AND query_kind IN ($query_kind), $query_kind)\n $conditionalTest(AND initial_user IN ($user), $user)\n $conditionalTest(AND query_duration_ms >= $min_duration_ms, $min_duration_ms)\n $conditionalTest(AND query_duration_ms <= $max_duration_ms, $max_duration_ms)\nGROUP BY\n query,\n type\nORDER BY count DESC\nLIMIT $top\nSETTINGS skip_unavailable_shards=1", + "rawQuery": "SELECT\n normalizeQuery(query) AS query,\n type,\n count() count\nFROM cluster('all-sharded',system.query_log)\nWHERE event_date >= toDate(1694531196) AND event_date <= toDate(1694534796) AND event_time >= toDateTime(1694531196) AND event_time <= toDateTime(1694534796)\n AND( ('1,2,3,4' = '1,2,3,4' AND type != 'QueryStart') OR ('1,2,3,4' != '1,2,3,4' AND type IN (1,2,3,4)))\n AND type NOT IN ('QueryStart','QueryFinish')\n \n \n \n \nGROUP BY\n query,\n type\nORDER BY count DESC\nLIMIT 30\nSETTINGS skip_unavailable_shards=1", "refId": "A", "resultFormat": "time_series", "round": "0s", + "skip_comments": true, "table": "query_log", "tableLoading": false } ], - "title": "Top failed queries by user: $user; query type: $query_type", + "title": "Top failed queries by user: $user; query kind: $query_kind", "transform": "table", "type": "table" }, @@ -521,6 +687,12 @@ "dashLength": 10, "dashes": false, "datasource": "$db", + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, "fill": 1, "fillGradient": 0, "gridPos": { @@ -545,9 +717,10 @@ "links": [], "nullPointMode": "null", "options": { - "dataLinks": [] + "alertThreshold": true }, "percentage": false, + "pluginVersion": "7.5.17", "pointradius": 5, "points": false, "renderer": "flot", @@ -564,15 +737,17 @@ "dateTimeType": "DATETIME", "datetimeLoading": false, "expr": "", + "extrapolate": false, "format": "time_series", - "formattedQuery": "$rate(count() c)
FROM cluster('all-sharded',system.query_log)
where  type in ($type)
    and initial_user in ($user)
    and('$query_type' = 'all' or(positionCaseInsensitive(query,  '$query_type') = 1))", + "formattedQuery": "$rate(count() c)
FROM cluster('all-sharded',system.query_log)
where  type in ($type)
    and initial_user in ($user)
    and('$query_kind' = 'all' or(positionCaseInsensitive(query,  '$query_kind') = 1))", "interval": "", "intervalFactor": 2, - "query": "$rate(count() c)\nFROM cluster('all-sharded',system.query_log)\nWHERE type IN($type)\n AND initial_user IN ($user)\n AND ('$query_type' = 'all' OR (positionCaseInsensitive(query, '$query_type') = 1))", - "rawQuery": "SELECT t, c/runningDifference(t/1000) cRate FROM ( SELECT (intDiv(toUInt32(event_time), 120) * 120) * 1000 AS t, count() c FROM cluster('all-sharded',system.query_log) WHERE event_date >= toDate(1586192459) AND event_time >= toDateTime(1586192459) AND type IN (1,2,3,4) AND initial_user IN ('default') AND ('all' = 'all' OR (positionCaseInsensitive(query, 'all') = 1)) GROUP BY t ORDER BY t)", + "query": "$rate(count() c)\nFROM cluster('all-sharded',system.query_log)\nWHERE $timeFilter\n AND( ('$type' = '1,2,3,4' AND type != 'QueryStart') OR ('$type' != '1,2,3,4' AND type IN ($type)))\n $conditionalTest(AND query_kind IN ($query_kind), $query_kind)\n $conditionalTest(AND initial_user IN ($user), $user)\n $conditionalTest(AND query_duration_ms >= $min_duration_ms,$min_duration_ms)\n $conditionalTest(AND query_duration_ms <= $max_duration_ms,$max_duration_ms)\n", + "rawQuery": "SELECT t, c/runningDifference(t/1000) cRate FROM ( SELECT (intDiv(toUInt32(event_time), 4) * 4) * 1000 AS t, count() c FROM cluster('all-sharded',system.query_log)\nWHERE event_date >= toDate(1694531229) AND event_date <= toDate(1694534829) AND event_time >= toDateTime(1694531229) AND event_time <= toDateTime(1694534829) AND event_date >= toDate(1694531229) AND event_date <= toDate(1694534829) AND event_time >= toDateTime(1694531229) AND event_time <= toDateTime(1694534829)\n AND( ('1,2,3,4' = '1,2,3,4' AND type != 'QueryStart') OR ('1,2,3,4' != '1,2,3,4' AND type IN (1,2,3,4)))\n \n \n \n GROUP BY t ORDER BY t)", "refId": "A", "resultFormat": "time_series", "round": "0s", + "skip_comments": true, "table": "query_log", "tableLoading": false } @@ -581,7 +756,7 @@ "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Reqs/s by type: $type; user: $user; query type: $query_type", + "title": "Reqs/s type: $type; user: $user; query kind: $query_kind", "tooltip": { "shared": true, "sort": 0, @@ -624,6 +799,10 @@ "dashLength": 10, "dashes": false, "datasource": "$db", + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, "fill": 1, "fillGradient": 1, "gridPos": { @@ -648,9 +827,10 @@ "links": [], "nullPointMode": "null as zero", "options": { - "dataLinks": [] + "alertThreshold": true }, "percentage": false, + "pluginVersion": "7.5.17", "pointradius": 5, "points": false, "renderer": "flot", @@ -688,14 +868,17 @@ "dateTimeType": "DATETIME", "datetimeLoading": false, "expr": "", + "extrapolate": false, "format": "time_series", "formattedQuery": "SELECT
    $timeSeries as t,
    avg(query_duration_ms)  select_duration
FROM cluster('all-sharded',system.query_log)
WHERE
    $timeFilter
    AND type = 2
    and positionCaseInsensitive(query,  'select') = 1
    and initial_user in ($user)
GROUP BY t
ORDER BY t", + "interval": "", "intervalFactor": 2, - "query": "SELECT\n $timeSeries as t,\n quantile(0.5)(query_duration_ms) select_quantile_50,\n quantile(0.9)(query_duration_ms) select_quantile_90\nFROM cluster('all-sharded',system.query_log)\nWHERE\n $timeFilter\n AND type = 2\n AND positionCaseInsensitive(query, 'select') = 1\n AND initial_user IN ($user)\nGROUP BY t\nORDER BY t", - "rawQuery": "SELECT (intDiv(toUInt32(event_time), 10) * 10) * 1000 as t, quantile(0.5)(query_duration_ms) select_quantile_50, quantile(0.9)(query_duration_ms) select_quantile_90 FROM cluster('all-sharded',system.query_log) WHERE event_date >= toDate(1595573984) AND event_time >= toDateTime(1595573984) AND type = 2 AND positionCaseInsensitive(query, 'select') = 1 AND initial_user IN ('default') GROUP BY t ORDER BY t", + "query": "SELECT\n $timeSeries as t,\n quantile(0.5)(query_duration_ms) select_quantile_50,\n quantile(0.9)(query_duration_ms) select_quantile_90\nFROM cluster('all-sharded',system.query_log)\nWHERE\n $timeFilter\n AND type != 1\n AND query_kind ='Select'\n AND query_duration_ms > 0\n $conditionalTest(AND initial_user IN ($user), $user)\n $conditionalTest(AND query_duration_ms <= $max_duration_ms,$max_duration_ms)\nGROUP BY t\nORDER BY t", + "rawQuery": "SELECT\n (intDiv(toUInt32(event_time), 4) * 4) * 1000 as t,\n quantile(0.5)(query_duration_ms) select_quantile_50,\n quantile(0.9)(query_duration_ms) select_quantile_90\nFROM cluster('all-sharded',system.query_log)\nWHERE\n event_date >= toDate(1694526375) AND event_date <= toDate(1694529975) AND event_time >= toDateTime(1694526375) AND event_time <= toDateTime(1694529975)\n AND type != 1\n AND query_kind ='Select'\n AND query_duration_ms > 0\n \n \nGROUP BY t\nORDER BY t", "refId": "A", "resultFormat": "time_series", "round": "0s", + "skip_comments": true, "table": "query_log", "tableLoading": false }, @@ -707,14 +890,16 @@ "dateTimeType": "DATETIME", "datetimeLoading": false, "expr": "", + "extrapolate": false, "format": "time_series", "formattedQuery": "SELECT
    $timeSeries as t,
    avg(query_duration_ms) insert_duration
FROM cluster('all-sharded',system.query_log)
WHERE
    $timeFilter
    AND type = 2
and positionCaseInsensitive(query,  'insert into') = 1
    and initial_user in ($user)
GROUP BY t
ORDER BY t", "intervalFactor": 2, - "query": "SELECT\n $timeSeries as t,\n quantile(0.5)(query_duration_ms) insert_quantile_50,\n quantile(0.9)(query_duration_ms) insert_quantile_90\nFROM cluster('all-sharded',system.query_log)\nWHERE\n $timeFilter\n AND type = 2\nand positionCaseInsensitive(query, 'insert into') = 1\n and initial_user in ($user)\nGROUP BY t\nORDER BY t", - "rawQuery": "SELECT (intDiv(toUInt32(event_time), 10) * 10) * 1000 as t, quantile(0.5)(query_duration_ms) insert_quantile_50, quantile(0.9)(query_duration_ms) insert_quantile_90 FROM cluster('all-sharded',system.query_log) WHERE event_date >= toDate(1595573984) AND event_time >= toDateTime(1595573984) AND type = 2 and positionCaseInsensitive(query, 'insert into') = 1 and initial_user in ('default') GROUP BY t ORDER BY t", + "query": "SELECT\n $timeSeries as t,\n quantile(0.5)(query_duration_ms) insert_quantile_50,\n quantile(0.9)(query_duration_ms) insert_quantile_90\nFROM cluster('all-sharded',system.query_log)\nWHERE\n $timeFilter\n AND type = 2\n AND query_kind = 'Insert'\n AND query_duration_ms > 0\n $conditionalTest(AND initial_user IN ($user), $user)\n $conditionalTest(AND query_duration_ms <= $max_duration_ms,$max_duration_ms)\nGROUP BY t\nORDER BY t", + "rawQuery": "SELECT\n (intDiv(toUInt32(event_time), 4) * 4) * 1000 as t,\n quantile(0.5)(query_duration_ms) insert_quantile_50,\n quantile(0.9)(query_duration_ms) insert_quantile_90\nFROM cluster('all-sharded',system.query_log)\nWHERE\n event_date >= toDate(1694526375) AND event_date <= toDate(1694529975) AND event_time >= toDateTime(1694526375) AND event_time <= toDateTime(1694529975)\n AND type = 2\n AND query_kind = 'Insert'\n AND query_duration_ms > 0\n \n \nGROUP BY t\nORDER BY t", "refId": "B", "resultFormat": "time_series", "round": "0s", + "skip_comments": true, "table": "query_log", "tableLoading": false } @@ -778,6 +963,102 @@ { "columns": [], "datasource": "$db", + "fieldConfig": { + "defaults": { + "custom": { + "align": null, + "filterable": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "result" + }, + "properties": [ + { + "id": "custom.width", + "value": 106 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "memory" + }, + "properties": [ + { + "id": "custom.width", + "value": 108 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "duration" + }, + "properties": [ + { + "id": "custom.width", + "value": 117 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "user" + }, + "properties": [ + { + "id": "custom.width", + "value": 114 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Time" + }, + "properties": [ + { + "id": "custom.width", + "value": 149 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "host" + }, + "properties": [ + { + "id": "custom.width", + "value": 223 + } + ] + } + ] + }, "fontSize": "100%", "gridPos": { "h": 7, @@ -787,8 +1068,12 @@ }, "id": 21, "links": [], - "options": {}, + "options": { + "showHeader": true, + "sortBy": [] + }, "pageSize": null, + "pluginVersion": "7.5.17", "scroll": true, "showHeader": true, "sort": { @@ -798,12 +1083,14 @@ "styles": [ { "alias": "Time", + "align": "auto", "dateFormat": "YYYY-MM-DD HH:mm:ss", "pattern": "Time", "type": "date" }, { "alias": "", + "align": "auto", "colorMode": null, "colors": [ "rgba(245, 54, 54, 0.9)", @@ -819,6 +1106,7 @@ }, { "alias": "", + "align": "auto", "colorMode": null, "colors": [ "rgba(245, 54, 54, 0.9)", @@ -834,6 +1122,7 @@ }, { "alias": "", + "align": "auto", "colorMode": null, "colors": [ "rgba(245, 54, 54, 0.9)", @@ -855,25 +1144,33 @@ "dateTimeColDataType": "event_time", "dateTimeType": "DATETIME", "datetimeLoading": false, + "extrapolate": false, "format": "time_series", - "formattedQuery": "SELECT
    event_time,
    user,
    query_duration_ms duration,
    memory_usage memory,
    if(exception!='', 'fail', 'success') result,
    concat(substring(query,1,120), '...') query
FROM cluster('all-sharded',system.query_log)
WHERE $timeFilter
    AND type in ($type)
    and initial_user in ($user)
    and('$query_type' = 'all' or(positionCaseInsensitive(query,  '$query_type') = 1))
ORDER BY event_time DESC
LIMIT 1000", + "interval": "", "intervalFactor": 1, - "query": "SELECT\n event_time,\n hostName() host,\n user,\n query_duration_ms duration,\n memory_usage memory,\n if(exception!='', 'fail', 'success') result,\n concat(substring(query,1,120), '...') query\nFROM cluster('all-sharded',system.query_log)\nWHERE $timeFilter\n AND type in ($type)\n and initial_user in ($user)\n and('$query_type' = 'all' OR (positionCaseInsensitive(query, '$query_type') = 1))\nORDER BY event_time DESC\nLIMIT 1000", - "rawQuery": "SELECT (intDiv(toUInt32(event_time), 10) * 10) * 1000 as t, avg(query_duration_ms) select_duration FROM cluster('all-sharded',system.query_log) WHERE event_date >= toDate(1586232299) AND event_time >= toDateTime(1586232299) AND type = 2 and positionCaseInsensitive(query, 'select') = 1 and initial_user in ('default') GROUP BY t ORDER BY t", + "query": "SELECT\n event_time,\n hostName() host,\n user,\n query_duration_ms duration,\n memory_usage memory,\n if(exception!='', 'fail', 'success') result,\n normalizeQuery(query) query\nFROM cluster('all-sharded',system.query_log)\nWHERE $timeFilter\n AND type in ($type)\n $conditionalTest(AND query_kind IN ($query_kind), $query_kind)\n $conditionalTest(AND initial_user IN ($user), $user)\n $conditionalTest(AND query_duration_ms >= $min_duration_ms,$min_duration_ms)\n $conditionalTest(AND query_duration_ms <= $max_duration_ms,$max_duration_ms)\nORDER BY event_time DESC\nLIMIT 1000", + "rawQuery": "SELECT\n event_time,\n hostName() host,\n user,\n query_duration_ms duration,\n memory_usage memory,\n if(exception!='', 'fail', 'success') result,\n normalizeQuery(query) query\nFROM cluster('all-sharded',system.query_log)\nWHERE event_date >= toDate(1694531300) AND event_date <= toDate(1694534900) AND event_time >= toDateTime(1694531300) AND event_time <= toDateTime(1694534900)\n AND type in (1,2,3,4)\n \n \n \n \nORDER BY event_time DESC\nLIMIT 1000", "refId": "A", "resultFormat": "time_series", "round": "0s", + "skip_comments": true, "table": "query_log", "tableLoading": false } ], - "title": "Query log by type: $type; user: $user; query type: $query_type", + "title": "Query log by type: $type; user: $user; query kind: $query_kind", "transform": "timeseries_to_columns", + "transformations": [ + { + "id": "prepareTimeSeries", + "options": {} + } + ], "type": "table" } ], - "refresh": "5s", - "schemaVersion": 21, + "refresh": "1m", + "schemaVersion": 27, "style": "dark", "tags": [ "Altinity", @@ -885,9 +1182,12 @@ "list": [ { "current": { - "text": "k8s-test-test-002-tpl", - "value": "k8s-test-test-002-tpl" + "selected": false, + "text": "k8s-test-test-001", + "value": "k8s-test-test-001" }, + "description": null, + "error": null, "hide": 2, "includeAll": false, "label": "K8S ClickHouse installation", @@ -905,13 +1205,18 @@ "current": {}, "datasource": "${DS_PROMETHEUS}", "definition": "label_values({__name__ =~ \"chi_clickhouse_metric_Uptime|chi_clickhouse_metric_fetch_errors\"}, exported_namespace)", + "description": null, + "error": null, "hide": 0, "includeAll": false, "label": "K8S Namespace", "multi": false, "name": "exported_namespace", "options": [], - "query": "label_values({__name__ =~ \"chi_clickhouse_metric_Uptime|chi_clickhouse_metric_fetch_errors\"}, exported_namespace)", + "query": { + "query": "label_values({__name__ =~ \"chi_clickhouse_metric_Uptime|chi_clickhouse_metric_fetch_errors\"}, exported_namespace)", + "refId": "clickhouse-operator-prometheus-exported_namespace-Variable-Query" + }, "refresh": 2, "regex": "", "skipUrlSync": false, @@ -927,13 +1232,18 @@ "current": {}, "datasource": "${DS_PROMETHEUS}", "definition": "label_values({__name__ =~ \"chi_clickhouse_metric_Uptime|chi_clickhouse_metric_fetch_errors\", exported_namespace=\"$exported_namespace\"}, chi)", + "description": null, + "error": null, "hide": 0, "includeAll": false, "label": "K8S Clickhouse Installation", "multi": false, "name": "chi", "options": [], - "query": "label_values({__name__ =~ \"chi_clickhouse_metric_Uptime|chi_clickhouse_metric_fetch_errors\",exported_namespace=\"$exported_namespace\"}, chi)", + "query": { + "query": "label_values({__name__ =~ \"chi_clickhouse_metric_Uptime|chi_clickhouse_metric_fetch_errors\",exported_namespace=\"$exported_namespace\"}, chi)", + "refId": "clickhouse-operator-prometheus-chi-Variable-Query" + }, "refresh": 1, "regex": "", "skipUrlSync": false, @@ -945,79 +1255,28 @@ "useTags": false }, { - "auto": true, - "auto_count": 100, - "auto_min": "1m", - "current": { - "selected": false, - "text": "5m", - "value": "5m" - }, - "hide": 2, - "label": null, - "name": "interval", - "options": [ - { - "selected": false, - "text": "auto", - "value": "$__auto_interval_interval" - }, - { - "selected": true, - "text": "5m", - "value": "5m" - } - ], - "query": "5m", - "refresh": 2, - "skipUrlSync": false, - "type": "interval" - }, - { - "allValue": "1,2,3,4", - "current": { - "selected": true, - "tags": [], - "text": "All", - "value": [ - "$__all" - ] - }, + "allValue": "", + "current": {}, + "datasource": "$db", + "definition": "SELECT\n values[2] value,\n keys[1] text\nFROM (\n SELECT\n arrayMap(x -> splitByString('\\' = ', x), splitByString(', \\'',extract(type,'Enum8\\(\\'(.+)\\)'))) AS enum_arr\n FROM system.columns\n WHERE database='system' and table='query_log' AND name='type'\n)\nARRAY JOIN enum_arr as keys, enum_arr as values", + "description": null, + "error": null, "hide": 0, "includeAll": true, "label": "type", "multi": true, "name": "type", - "options": [ - { - "selected": true, - "text": "All", - "value": "$__all" - }, - { - "selected": false, - "text": "1", - "value": "1" - }, - { - "selected": false, - "text": "2", - "value": "2" - }, - { - "selected": false, - "text": "3", - "value": "3" - }, - { - "selected": false, - "text": "4", - "value": "4" - } - ], - "query": "1,2,3,4", + "options": [], + "query": "SELECT\n values[2] value,\n keys[1] text\nFROM (\n SELECT\n arrayMap(x -> splitByString('\\' = ', x), splitByString(', \\'',extract(type,'Enum8\\(\\'(.+)\\)'))) AS enum_arr\n FROM system.columns\n WHERE database='system' and table='query_log' AND name='type'\n)\nARRAY JOIN enum_arr as keys, enum_arr as values", + "refresh": 2, + "regex": "", "skipUrlSync": false, - "type": "custom" + "sort": 0, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false }, { "allValue": null, @@ -1027,6 +1286,8 @@ "text": "5", "value": "5" }, + "description": null, + "error": null, "hide": 0, "includeAll": false, "label": "top elements", @@ -1065,6 +1326,7 @@ } ], "query": "5,10,15,20,25,30", + "queryValue": "", "skipUrlSync": false, "type": "custom" }, @@ -1072,14 +1334,16 @@ "allValue": "", "current": {}, "datasource": "$db", - "definition": "SELECT DISTINCT initial_user FROM cluster('all-sharded',system.query_log) WHERE event_date >= today()-3 AND event_time > toDateTime(today()-3) SETTINGS skip_unavailable_shards=1", + "definition": "SELECT DISTINCT initial_user FROM cluster('all-sharded',system.query_log) WHERE event_date BETWEEN toDate($__from / 1000) AND toDate($__to / 1000) AND event_time BETWEEN toDateTime($__from / 1000) AND toDateTime($__to / 1000) SETTINGS skip_unavailable_shards=1", + "description": null, + "error": null, "hide": 0, "includeAll": true, "label": "initial user", "multi": true, "name": "user", "options": [], - "query": "SELECT DISTINCT initial_user FROM cluster('all-sharded',system.query_log) WHERE event_date >= today()-3 AND event_time > toDateTime(today()-3) SETTINGS skip_unavailable_shards=1", + "query": "SELECT DISTINCT initial_user FROM cluster('all-sharded',system.query_log) WHERE event_date BETWEEN toDate($__from / 1000) AND toDate($__to / 1000) AND event_time BETWEEN toDateTime($__from / 1000) AND toDateTime($__to / 1000) SETTINGS skip_unavailable_shards=1", "refresh": 1, "regex": "", "skipUrlSync": false, @@ -1092,36 +1356,71 @@ }, { "allValue": "", + "current": {}, + "datasource": "$db", + "definition": "SELECT DISTINCT query_kind FROM cluster('all-sharded',system.query_log) WHERE event_date BETWEEN toDate($__from / 1000) AND toDate($__to / 1000) AND event_time BETWEEN toDateTime($__from / 1000) AND toDateTime($__to / 1000) SETTINGS skip_unavailable_shards=1", + "description": null, + "error": null, + "hide": 0, + "includeAll": true, + "label": "query_kind", + "multi": true, + "name": "query_kind", + "options": [], + "query": "SELECT DISTINCT query_kind FROM cluster('all-sharded',system.query_log) WHERE event_date BETWEEN toDate($__from / 1000) AND toDate($__to / 1000) AND event_time BETWEEN toDateTime($__from / 1000) AND toDateTime($__to / 1000) SETTINGS skip_unavailable_shards=1", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { "current": { - "selected": true, - "text": "all", - "value": "all" + "selected": false, + "text": "", + "value": "" }, + "description": null, + "error": null, "hide": 0, - "includeAll": false, - "label": "query type", - "multi": false, - "name": "query_type", + "label": null, + "name": "min_duration_ms", "options": [ - { - "selected": true, - "text": "all", - "value": "all" - }, { "selected": false, - "text": "select", - "value": "select" - }, + "text": "0", + "value": "0" + } + ], + "query": "", + "skipUrlSync": false, + "type": "textbox" + }, + { + "current": { + "selected": true, + "text": "", + "value": "" + }, + "description": null, + "error": null, + "hide": 0, + "label": null, + "name": "max_duration_ms", + "options": [ { - "selected": false, - "text": "insert", - "value": "insert" + "selected": true, + "text": "", + "value": "" } ], - "query": "all,select,insert", + "query": "", "skipUrlSync": false, - "type": "custom" + "type": "textbox" } ] }, @@ -1157,5 +1456,5 @@ "timezone": "browser", "title": "ClickHouse in Kubernetes Queries", "uid": "clickhouse-queries", - "version": 20220214 -} + "version": 20230912 +} \ No newline at end of file