From f1b972bf11f5b4eae18c4f486a94c4440d98dead Mon Sep 17 00:00:00 2001 From: Slach Date: Mon, 11 Sep 2023 17:59:51 +0400 Subject: [PATCH 1/5] change dashboard refresh rate and add min_duration_ms, max_duration_ms dashboard variables Signed-off-by: Slach --- .../install-grafana-with-operator.sh | 21 ++- deploy/prometheus/create-prometheus.sh | 2 +- .../ClickHouse_Queries_dashboard.json | 150 ++++++++++++++---- 3 files changed, 142 insertions(+), 31 deletions(-) diff --git a/deploy/grafana/grafana-with-grafana-operator/install-grafana-with-operator.sh b/deploy/grafana/grafana-with-grafana-operator/install-grafana-with-operator.sh index 4acc9b92e..969eea8b8 100755 --- a/deploy/grafana/grafana-with-grafana-operator/install-grafana-with-operator.sh +++ b/deploy/grafana/grafana-with-grafana-operator/install-grafana-with-operator.sh @@ -179,8 +179,8 @@ if ! command -v yq &> /dev/null; then echo "Install 'yq', see installation instruction in https://github.com/mikefarah/yq/#install" exit 1 else - export OPERATOR_CH_USER=$(yq eval .clickhouse.access.username ${CUR_DIR}/../../../config/config.yaml) - export OPERATOR_CH_PASS=$(yq eval .clickhouse.access.password ${CUR_DIR}/../../../config/config.yaml) + export OPERATOR_CH_USER=$(yq 'select(.kind == "Secret") | .stringData.username' "${CUR_DIR}/../../operator/clickhouse-operator-install-bundle.yaml") + export OPERATOR_CH_PASS=$(yq 'select(.kind == "Secret") | .stringData.password' "${CUR_DIR}/../../operator/clickhouse-operator-install-bundle.yaml") fi echo "Create ClickHouse DataSource for each ClickHouseInstallation" @@ -199,6 +199,23 @@ for LINE in $(kubectl get --all-namespaces chi -o custom-columns=NAMESPACE:.meta clickhouse-client --echo -mn -q 'SELECT hostName(), dummy FROM system.one SETTINGS log_queries=1; SYSTEM FLUSH LOGS' done + echo "Ensure clickhouse-operator will connect to any chi from grafana" + POD_NETWORK_CIDR=$(kubectl get cm -n kube-system kube-proxy -o yaml | grep clusterCIDR | cut -d ":" -f 2) + kubectl apply --validate=${VALIDATE_YAML} --namespace="${NAMESPACE}" -f <( + { + echo "apiVersion: clickhouse.altinity.com/v1" + echo "kind: ClickHouseInstallationTemplate" + echo "metadata:" + echo " name: clickhouse-operator-from-cluster-network" + echo "spec:" + echo " templating:" + echo " policy: auto" + echo " configuration:" + echo " users:" + echo " ${OPERATOR_CH_USER}/networks/ip: ${POD_NETWORK_CIDR}" + } + ) + GRAFANA_CLICKHOUSE_DATASOURCE_NAME="k8s-${NAMESPACE}-${CHI}" CLICKHOUSE_URL="http://${ENDPOINT}:${PORT}" echo "Create ClickHouse DataSource for ClickHouseInstallation ${CHI} '${GRAFANA_NAMESPACE}/${GRAFANA_CLICKHOUSE_DATASOURCE_NAME}'" diff --git a/deploy/prometheus/create-prometheus.sh b/deploy/prometheus/create-prometheus.sh index 0d3ba4fbb..274db7999 100755 --- a/deploy/prometheus/create-prometheus.sh +++ b/deploy/prometheus/create-prometheus.sh @@ -6,7 +6,7 @@ echo "External value for \$VALIDATE_YAML=$VALIDATE_YAML" export PROMETHEUS_NAMESPACE="${PROMETHEUS_NAMESPACE:-prometheus}" export OPERATOR_NAMESPACE="${OPERATOR_NAMESPACE:-kube-system}" -export PROMETHEUS_OPERATOR_BRANCH="${PROMETHEUS_OPERATOR_BRANCH:-v0.62.0}" +export PROMETHEUS_OPERATOR_BRANCH="${PROMETHEUS_OPERATOR_BRANCH:-v0.68.0}" export ALERT_MANAGER_EXTERNAL_URL="${ALERT_MANAGER_EXTERNAL_URL:-http://localhost:9093}" # Possible values for "validate yaml" are values from --validate=XXX kubectl option. They are true/false ATM export VALIDATE_YAML="${VALIDATE_YAML:-true}" diff --git a/grafana-dashboard/ClickHouse_Queries_dashboard.json b/grafana-dashboard/ClickHouse_Queries_dashboard.json index 69bb0656c..bf670e2a2 100644 --- a/grafana-dashboard/ClickHouse_Queries_dashboard.json +++ b/grafana-dashboard/ClickHouse_Queries_dashboard.json @@ -14,7 +14,7 @@ "type": "grafana", "id": "grafana", "name": "Grafana", - "version": "7.5.16" + "version": "7.5.17" }, { "type": "panel", @@ -43,7 +43,7 @@ { "type": "datasource", "id": "vertamedia-clickhouse-datasource", - "name": "ClickHouse", + "name": "Altinity plugin for ClickHouse", "version": "2.5.3" } ], @@ -65,7 +65,7 @@ "gnetId": 2515, "graphTooltip": 1, "id": null, - "iteration": 1596171309294, + "iteration": 1694437448827, "links": [], "panels": [ { @@ -139,11 +139,12 @@ "format": "time_series", "formattedQuery": "SELECT * FROM ($rateColumns(\n substring(query, 1, 45) AS query,\n count() c)\nFROM cluster('all-sharded',system.query_log)\nWHERE\n cityHash64(query) global in (\n SELECT cityHash64(substring(query, 1, 45)) AS h\n FROM cluster('all-sharded',system.query_log)\n WHERE\n $timeFilter\n AND type in ($type)\n AND initial_user in ($user)\n AND('$query_type' = 'all' or(positionCaseInsensitive(query, '$query_type') = 1))\n GROUP BY h\n ORDER BY count() desc\n LIMIT $top)\n AND type in ($type)\n AND initial_user in ($user)\n AND('$query_type' = 'all' or(positionCaseInsensitive(query, '$query_type') = 1))) SETTINGS skip_unavailable_shards=1", "intervalFactor": 2, - "query": "SELECT\r\n t,\r\n arrayMap(a -> (a.1, a.2 / runningDifference(t / 1000)), groupArr)\r\nFROM (\r\n SELECT t, groupArray((query, c)) AS groupArr\r\n FROM (\r\n SELECT\r\n (intDiv(toUInt32(event_time), 2) * 2) * 1000 AS t,\r\n substring(query, 1, 45) AS query,\r\n count() c\r\n FROM cluster('all-sharded',system.query_log)\r\n WHERE $timeFilter\r\n AND cityHash64(query) GLOBAL IN (\r\n SELECT cityHash64(substring(query, 1, 45)) AS h\r\n FROM cluster('all-sharded',system.query_log)\r\n WHERE $timeFilter\r\n AND type IN ($type)\r\n AND initial_user IN ($user)\r\n AND ('$query_type' = 'all' OR (positionCaseInsensitive(query, '$query_type') = 1))\r\n GROUP BY h\r\n ORDER BY count() DESC\r\n LIMIT $top\r\n SETTINGS skip_unavailable_shards=1\r\n )\r\n AND type IN ($type)\r\n AND initial_user IN ($user)\r\n AND ('$query_type' = 'all' OR (positionCaseInsensitive(query, '$query_type') = 1))\r\n GROUP BY t, query\r\n ORDER BY t\r\n )\r\n GROUP BY t\r\n ORDER BY t\r\n) SETTINGS skip_unavailable_shards=1", - "rawQuery": "SELECT t, arrayMap(a -> (a.1, a.2 / runningDifference(t / 1000)), groupArr) FROM ( SELECT t, groupArray((query, c)) AS groupArr FROM ( SELECT (intDiv(toUInt32(event_time), 2) * 2) * 1000 AS t, substring(query, 1, 45) AS query, count() c FROM cluster('all-sharded',system.query_log) WHERE event_date >= toDate(1586192379) AND event_time >= toDateTime(1586192379) AND cityHash64(query) GLOBAL IN ( SELECT cityHash64(substring(query, 1, 45)) AS h FROM cluster('all-sharded',system.query_log) WHERE event_date >= toDate(1586192379) AND event_time >= toDateTime(1586192379) AND type IN (1,2,3,4) AND initial_user IN ('default') AND ('all' = 'all' OR (positionCaseInsensitive(query, 'all') = 1)) GROUP BY h ORDER BY count() DESC LIMIT 5 SETTINGS skip_unavailable_shards=1 ) AND type IN (1,2,3,4) AND initial_user IN ('default') AND ('all' = 'all' OR (positionCaseInsensitive(query, 'all') = 1)) GROUP BY t, query ORDER BY t ) GROUP BY t ORDER BY t ) SETTINGS skip_unavailable_shards=1", + "query": "SELECT\r\n t,\r\n arrayMap(a -> (a.1, a.2 / runningDifference(t / 1000)), groupArr)\r\nFROM (\r\n SELECT t, groupArray((q, c)) AS groupArr\r\n FROM (\r\n SELECT\r\n (intDiv(toUInt32(event_time), 2) * 2) * 1000 AS t,\r\n normalizeQuery(query) AS q,\r\n count() c\r\n FROM cluster('all-sharded',system.query_log)\r\n WHERE $timeFilter\r\n AND normalized_query_hash GLOBAL IN (\r\n SELECT normalized_query_hash AS h\r\n FROM cluster('all-sharded',system.query_log)\r\n WHERE $timeFilter\r\n AND type IN ($type)\r\n AND initial_user IN ($user)\r\n AND ('$query_type' = 'all' OR (positionCaseInsensitive(query, '$query_type') = 1))\r\n $conditionalTest(AND query_duration_ms >= $min_duration_ms,$min_duration_ms)\r\n $conditionalTest(AND query_duration_ms <= $max_duration_ms,$max_duration_ms)\r\n GROUP BY h\r\n ORDER BY count() DESC\r\n LIMIT $top\r\n SETTINGS skip_unavailable_shards=1\r\n )\r\n AND type IN ($type)\r\n AND initial_user IN ($user)\r\n AND ('$query_type' = 'all' OR (positionCaseInsensitive(query, '$query_type') = 1))\r\n $conditionalTest(AND query_duration_ms >= $min_duration_ms,$min_duration_ms)\r\n $conditionalTest(AND query_duration_ms <= $max_duration_ms,$max_duration_ms)\r\n GROUP BY t, query\r\n ORDER BY t\r\n )\r\n GROUP BY t\r\n ORDER BY t\r\n) SETTINGS skip_unavailable_shards=1", + "rawQuery": "SELECT\r\n t,\r\n arrayMap(a -> (a.1, a.2 / runningDifference(t / 1000)), groupArr)\r\nFROM (\r\n SELECT t, groupArray((q, c)) AS groupArr\r\n FROM (\r\n SELECT\r\n (intDiv(toUInt32(event_time), 2) * 2) * 1000 AS t,\r\n normalizeQuery(query) AS q,\r\n count() c\r\n FROM cluster('all-sharded',system.query_log)\r\n WHERE event_date >= toDate(1694434397) AND event_date <= toDate(1694437997) AND event_time >= toDateTime(1694434397) AND event_time <= toDateTime(1694437997)\r\n AND normalized_query_hash GLOBAL IN (\r\n SELECT normalized_query_hash AS h\r\n FROM cluster('all-sharded',system.query_log)\r\n WHERE event_date >= toDate(1694434397) AND event_date <= toDate(1694437997) AND event_time >= toDateTime(1694434397) AND event_time <= toDateTime(1694437997)\r\n AND type IN (1,2,3,4)\r\n AND initial_user IN ('default')\r\n AND ('all' = 'all' OR (positionCaseInsensitive(query, 'all') = 1))\r\n AND query_duration_ms >= 0 \r\n \r\n GROUP BY h\r\n ORDER BY count() DESC\r\n LIMIT 5\r\n SETTINGS skip_unavailable_shards=1\r\n )\r\n AND type IN (1,2,3,4)\r\n AND initial_user IN ('default')\r\n AND ('all' = 'all' OR (positionCaseInsensitive(query, 'all') = 1))\r\n AND query_duration_ms >= 0 \r\n \r\n GROUP BY t, query\r\n ORDER BY t\r\n )\r\n GROUP BY t\r\n ORDER BY t\r\n) SETTINGS skip_unavailable_shards=1", "refId": "A", "resultFormat": "time_series", "round": "0s", + "skip_comments": true, "table": "query_log", "tableLoading": false } @@ -230,12 +231,14 @@ "styles": [ { "alias": "Time", + "align": "auto", "dateFormat": "YYYY-MM-DD HH:mm:ss", "pattern": "t", "type": "hidden" }, { "alias": "duration", + "align": "auto", "colorMode": null, "colors": [ "rgba(245, 54, 54, 0.9)", @@ -251,6 +254,7 @@ }, { "alias": "", + "align": "auto", "colorMode": null, "colors": [ "rgba(245, 54, 54, 0.9)", @@ -266,6 +270,7 @@ }, { "alias": "", + "align": "auto", "colorMode": null, "colors": [ "rgba(245, 54, 54, 0.9)", @@ -291,11 +296,12 @@ "format": "table", "formattedQuery": "SELECT
    rand() as t,
    substring(query,  1,  70) AS query,
    avg(query_duration_ms) duration,
    count() count
FROM cluster('all-sharded',system.query_log)
WHERE
    $timeFilter
    AND type in ($type)
    and initial_user in ($user)
    and('$query_type' = 'all' or(positionCaseInsensitive(query,  '$query_type') = 1))
GROUP BY query
ORDER BY duration desc
LIMIT $top", "intervalFactor": 2, - "query": "SELECT\n rand() as t,\n substring(query, 1, 70) AS query,\n avg(query_duration_ms) duration,\n count() count\nFROM cluster('all-sharded',system.query_log)\nWHERE\n $timeFilter\n AND type in ($type)\n and initial_user in ($user)\n and('$query_type' = 'all' or(positionCaseInsensitive(query, '$query_type') = 1))\nGROUP BY query\nORDER BY duration desc\nLIMIT $top", - "rawQuery": "SELECT t, arrayMap(a -> (a.1, a.2 / runningDifference(t / 1000)), groupArr) FROM ( SELECT t, groupArray((query, c)) AS groupArr FROM ( SELECT (intDiv(toUInt32(event_time), 2) * 2) * 1000 AS t, substring(query, 1, 45) AS query, count() c FROM cluster('all-sharded',system.query_log) WHERE event_date >= toDate(1586234275) AND event_time >= toDateTime(1586234275) AND cityHash64(query) GLOBAL IN ( SELECT cityHash64(substring(query, 1, 45)) AS h FROM cluster('all-sharded',system.query_log) WHERE event_date >= toDate(1586234275) AND event_time >= toDateTime(1586234275) AND type IN (1,2,3,4) AND initial_user IN ('default') AND ('all' = 'all' OR (positionCaseInsensitive(query, 'all') = 1)) GROUP BY h ORDER BY count() DESC LIMIT 5 SETTINGS skip_unavailable_shards=1 ) AND type IN (1,2,3,4) AND initial_user IN ('default') AND ('all' = 'all' OR (positionCaseInsensitive(query, 'all') = 1)) GROUP BY t, query ORDER BY t ) GROUP BY t ORDER BY t ) SETTINGS skip_unavailable_shards=1", + "query": "SELECT\n rand() as t,\n normalizeQuery(query) AS query,\n avg(query_duration_ms) duration,\n count() count\nFROM cluster('all-sharded',system.query_log)\nWHERE\n $timeFilter\n AND type in ($type)\n AND initial_user in ($user)\n AND('$query_type' = 'all' or(positionCaseInsensitive(query, '$query_type') = 1))\n $conditionalTest(AND query_duration_ms >= $min_duration_ms,$min_duration_ms)\n $conditionalTest(AND query_duration_ms <= $max_duration_ms,$max_duration_ms)\nGROUP BY query\nORDER BY duration desc\nLIMIT $top", + "rawQuery": "SELECT\n rand() as t,\n normalizeQuery(query) AS query,\n avg(query_duration_ms) duration,\n count() count\nFROM cluster('all-sharded',system.query_log)\nWHERE\n event_date >= toDate(1694434499) AND event_date <= toDate(1694438099) AND event_time >= toDateTime(1694434499) AND event_time <= toDateTime(1694438099)\n AND type in (1,2,3,4)\n AND initial_user in ('default')\n AND('all' = 'all' or(positionCaseInsensitive(query, 'all') = 1))\n AND query_duration_ms >= 0 \n \nGROUP BY query\nORDER BY duration desc\nLIMIT 5", "refId": "A", "resultFormat": "time_series", "round": "0s", + "skip_comments": true, "table": "query_log", "tableLoading": false } @@ -328,12 +334,14 @@ "styles": [ { "alias": "Time", + "align": "auto", "dateFormat": "YYYY-MM-DD HH:mm:ss", "pattern": "t", "type": "hidden" }, { "alias": "usage", + "align": "auto", "colorMode": null, "colors": [ "rgba(245, 54, 54, 0.9)", @@ -349,6 +357,7 @@ }, { "alias": "", + "align": "auto", "colorMode": null, "colors": [ "rgba(245, 54, 54, 0.9)", @@ -364,6 +373,7 @@ }, { "alias": "", + "align": "auto", "colorMode": null, "colors": [ "rgba(245, 54, 54, 0.9)", @@ -389,11 +399,12 @@ "format": "table", "formattedQuery": "SELECT
    rand() as t,
    substring(query,  1,  70) AS query,
    avg(memory_usage) usage,
    count() count
FROM cluster('all-sharded',system.query_log)
WHERE
    $timeFilter
    AND type in ($type)
    and initial_user in ($user)
    and('$query_type' = 'all' or(positionCaseInsensitive(query,  '$query_type') = 1))
GROUP BY query
ORDER BY usage desc
LIMIT $top", "intervalFactor": 2, - "query": "SELECT\n rand() as t,\n substring(query, 1, 70) AS query,\n avg(memory_usage) usage,\n count() count\nFROM cluster('all-sharded',system.query_log)\nWHERE\n $timeFilter\n AND type IN ($type)\n AND initial_user IN ($user)\n AND ('$query_type' = 'all' OR (positionCaseInsensitive(query, '$query_type') = 1))\nGROUP BY query\nORDER BY usage DESC\nLIMIT $top", - "rawQuery": "SELECT t, arrayMap(a -> (a.1, a.2 / runningDifference(t / 1000)), groupArr) FROM ( SELECT t, groupArray((query, c)) AS groupArr FROM ( SELECT (intDiv(toUInt32(event_time), 2) * 2) * 1000 AS t, substring(query, 1, 45) AS query, count() c FROM cluster('all-sharded',system.query_log) WHERE event_date >= toDate(1586234300) AND event_time >= toDateTime(1586234300) AND cityHash64(query) GLOBAL IN ( SELECT cityHash64(substring(query, 1, 45)) AS h FROM cluster('all-sharded',system.query_log) WHERE event_date >= toDate(1586234300) AND event_time >= toDateTime(1586234300) AND type IN (1,2,3,4) AND initial_user IN ('default') AND ('all' = 'all' OR (positionCaseInsensitive(query, 'all') = 1)) GROUP BY h ORDER BY count() DESC LIMIT 5 SETTINGS skip_unavailable_shards=1 ) AND type IN (1,2,3,4) AND initial_user IN ('default') AND ('all' = 'all' OR (positionCaseInsensitive(query, 'all') = 1)) GROUP BY t, query ORDER BY t ) GROUP BY t ORDER BY t ) SETTINGS skip_unavailable_shards=1", + "query": "SELECT\n rand() as t,\n substring(query, 1, 70) AS query,\n avg(memory_usage) usage,\n count() count\nFROM cluster('all-sharded',system.query_log)\nWHERE\n $timeFilter\n AND type IN ($type)\n AND initial_user IN ($user)\n AND ('$query_type' = 'all' OR (positionCaseInsensitive(query, '$query_type') = 1))\n $conditionalTest(AND query_duration_ms >= $min_duration_ms,$min_duration_ms)\n $conditionalTest(AND query_duration_ms <= $max_duration_ms,$max_duration_ms)\nGROUP BY query\nORDER BY usage DESC\nLIMIT $top", + "rawQuery": "SELECT\n rand() as t,\n substring(query, 1, 70) AS query,\n avg(memory_usage) usage,\n count() count\nFROM cluster('all-sharded',system.query_log)\nWHERE\n event_date >= toDate(1694434541) AND event_date <= toDate(1694438141) AND event_time >= toDateTime(1694434541) AND event_time <= toDateTime(1694438141)\n AND type IN (1,2,3,4)\n AND initial_user IN ('default')\n AND ('all' = 'all' OR (positionCaseInsensitive(query, 'all') = 1))\n AND query_duration_ms >= 0 \n \nGROUP BY query\nORDER BY usage DESC\nLIMIT 5", "refId": "A", "resultFormat": "time_series", "round": "0s", + "skip_comments": true, "table": "query_log", "tableLoading": false } @@ -426,12 +437,14 @@ "styles": [ { "alias": "Time", + "align": "auto", "dateFormat": "YYYY-MM-DD HH:mm:ss", "pattern": "t", "type": "hidden" }, { "alias": "type", + "align": "auto", "colorMode": null, "colors": [ "rgba(245, 54, 54, 0.9)", @@ -447,6 +460,7 @@ }, { "alias": "", + "align": "auto", "colorMode": null, "colors": [ "rgba(245, 54, 54, 0.9)", @@ -462,6 +476,7 @@ }, { "alias": "", + "align": "auto", "colorMode": null, "colors": [ "rgba(245, 54, 54, 0.9)", @@ -487,11 +502,12 @@ "format": "table", "formattedQuery": "SELECT
    rand() as t,
    substring(query,  1,  70) AS query,
    type,
    count() count
FROM cluster('all-sharded',system.query_log)
WHERE
    $timeFilter
    AND type in (3,4)
    and initial_user in ($user)
    and('$query_type' = 'all' or(positionCaseInsensitive(query,  '$query_type') = 1))
GROUP BY
    query,
    type
ORDER BY count desc
LIMIT $top", "intervalFactor": 2, - "query": "SELECT\n rand() as t,\n substring(query, 1, 70) AS query,\n type,\n count() count\nFROM cluster('all-sharded',system.query_log)\nWHERE\n $timeFilter\n AND type IN (3,4)\n AND initial_user IN ($user)\n AND ('$query_type' = 'all' OR (positionCaseInsensitive(query, '$query_type') = 1))\nGROUP BY\n query,\n type\nORDER BY count desc\nLIMIT $top", - "rawQuery": "SELECT t, arrayMap(a -> (a.1, a.2 / runningDifference(t / 1000)), groupArr) FROM ( SELECT t, groupArray((query, c)) AS groupArr FROM ( SELECT (intDiv(toUInt32(event_time), 2) * 2) * 1000 AS t, substring(query, 1, 45) AS query, count() c FROM cluster('all-sharded',system.query_log) WHERE event_date >= toDate(1586234335) AND event_time >= toDateTime(1586234335) AND cityHash64(query) GLOBAL IN ( SELECT cityHash64(substring(query, 1, 45)) AS h FROM cluster('all-sharded',system.query_log) WHERE event_date >= toDate(1586234335) AND event_time >= toDateTime(1586234335) AND type IN (1,2,3,4) AND initial_user IN ('default') AND ('all' = 'all' OR (positionCaseInsensitive(query, 'all') = 1)) GROUP BY h ORDER BY count() DESC LIMIT 5 SETTINGS skip_unavailable_shards=1 ) AND type IN (1,2,3,4) AND initial_user IN ('default') AND ('all' = 'all' OR (positionCaseInsensitive(query, 'all') = 1)) GROUP BY t, query ORDER BY t ) GROUP BY t ORDER BY t ) SETTINGS skip_unavailable_shards=1", + "query": "SELECT\n rand() as t,\n normalizeQuery(query) AS query,\n type,\n count() count\nFROM cluster('all-sharded',system.query_log)\nWHERE\n $timeFilter\n AND type IN (3,4)\n AND initial_user IN ($user)\n AND ('$query_type' = 'all' OR (positionCaseInsensitive(query, '$query_type') = 1))\n $conditionalTest(AND query_duration_ms >= $min_duration_ms,$min_duration_ms)\n $conditionalTest(AND query_duration_ms <= $max_duration_ms,$max_duration_ms)\nGROUP BY\n query,\n type\nORDER BY count desc\nLIMIT $top", + "rawQuery": "SELECT\n rand() as t,\n normalizeQuery(query) AS query,\n type,\n count() count\nFROM cluster('all-sharded',system.query_log)\nWHERE\n event_date >= toDate(1694434583) AND event_date <= toDate(1694438183) AND event_time >= toDateTime(1694434583) AND event_time <= toDateTime(1694438183)\n AND type IN (3,4)\n AND initial_user IN ('default')\n AND ('all' = 'all' OR (positionCaseInsensitive(query, 'all') = 1))\n AND query_duration_ms >= 0 \n \nGROUP BY\n query,\n type\nORDER BY count desc\nLIMIT 5", "refId": "A", "resultFormat": "time_series", "round": "0s", + "skip_comments": true, "table": "query_log", "tableLoading": false } @@ -568,11 +584,12 @@ "formattedQuery": "$rate(count() c)
FROM cluster('all-sharded',system.query_log)
where  type in ($type)
    and initial_user in ($user)
    and('$query_type' = 'all' or(positionCaseInsensitive(query,  '$query_type') = 1))", "interval": "", "intervalFactor": 2, - "query": "$rate(count() c)\nFROM cluster('all-sharded',system.query_log)\nWHERE type IN($type)\n AND initial_user IN ($user)\n AND ('$query_type' = 'all' OR (positionCaseInsensitive(query, '$query_type') = 1))", - "rawQuery": "SELECT t, c/runningDifference(t/1000) cRate FROM ( SELECT (intDiv(toUInt32(event_time), 120) * 120) * 1000 AS t, count() c FROM cluster('all-sharded',system.query_log) WHERE event_date >= toDate(1586192459) AND event_time >= toDateTime(1586192459) AND type IN (1,2,3,4) AND initial_user IN ('default') AND ('all' = 'all' OR (positionCaseInsensitive(query, 'all') = 1)) GROUP BY t ORDER BY t)", + "query": "$rate(count() c)\nFROM cluster('all-sharded',system.query_log)\nWHERE type IN($type)\n AND initial_user IN ($user)\n AND ('$query_type' = 'all' OR (positionCaseInsensitive(query, '$query_type') = 1))\n $conditionalTest(AND query_duration_ms >= $min_duration_ms,$min_duration_ms)\n $conditionalTest(AND query_duration_ms <= $max_duration_ms,$max_duration_ms)\n ", + "rawQuery": "SELECT t, c/runningDifference(t/1000) cRate FROM ( SELECT (intDiv(toUInt32(event_time), 4) * 4) * 1000 AS t, count() c FROM cluster('all-sharded',system.query_log)\nWHERE event_date >= toDate(1694434633) AND event_date <= toDate(1694438233) AND event_time >= toDateTime(1694434633) AND event_time <= toDateTime(1694438233) AND type IN(1,2,3,4)\n AND initial_user IN ('default')\n AND ('all' = 'all' OR (positionCaseInsensitive(query, 'all') = 1))\n AND query_duration_ms >= 0 \n GROUP BY t ORDER BY t)", "refId": "A", "resultFormat": "time_series", "round": "0s", + "skip_comments": true, "table": "query_log", "tableLoading": false } @@ -648,7 +665,7 @@ "links": [], "nullPointMode": "null as zero", "options": { - "dataLinks": [] + "alertThreshold": true }, "percentage": false, "pointradius": 5, @@ -691,11 +708,12 @@ "format": "time_series", "formattedQuery": "SELECT
    $timeSeries as t,
    avg(query_duration_ms)  select_duration
FROM cluster('all-sharded',system.query_log)
WHERE
    $timeFilter
    AND type = 2
    and positionCaseInsensitive(query,  'select') = 1
    and initial_user in ($user)
GROUP BY t
ORDER BY t", "intervalFactor": 2, - "query": "SELECT\n $timeSeries as t,\n quantile(0.5)(query_duration_ms) select_quantile_50,\n quantile(0.9)(query_duration_ms) select_quantile_90\nFROM cluster('all-sharded',system.query_log)\nWHERE\n $timeFilter\n AND type = 2\n AND positionCaseInsensitive(query, 'select') = 1\n AND initial_user IN ($user)\nGROUP BY t\nORDER BY t", - "rawQuery": "SELECT (intDiv(toUInt32(event_time), 10) * 10) * 1000 as t, quantile(0.5)(query_duration_ms) select_quantile_50, quantile(0.9)(query_duration_ms) select_quantile_90 FROM cluster('all-sharded',system.query_log) WHERE event_date >= toDate(1595573984) AND event_time >= toDateTime(1595573984) AND type = 2 AND positionCaseInsensitive(query, 'select') = 1 AND initial_user IN ('default') GROUP BY t ORDER BY t", + "query": "SELECT\n $timeSeries as t,\n quantile(0.5)(query_duration_ms) select_quantile_50,\n quantile(0.9)(query_duration_ms) select_quantile_90\nFROM cluster('all-sharded',system.query_log)\nWHERE\n $timeFilter\n AND type = 2\n AND query_kind ='Select'\n AND initial_user IN ($user)\n AND query_duration_ms > 0\n $conditionalTest(AND query_duration_ms <= $max_duration_ms,$max_duration_ms)\nGROUP BY t\nORDER BY t", + "rawQuery": "SELECT\n (intDiv(toUInt32(event_time), 4) * 4) * 1000 as t,\n quantile(0.5)(query_duration_ms) select_quantile_50,\n quantile(0.9)(query_duration_ms) select_quantile_90\nFROM cluster('all-sharded',system.query_log)\nWHERE\n event_date >= toDate(1694434806) AND event_date <= toDate(1694438406) AND event_time >= toDateTime(1694434806) AND event_time <= toDateTime(1694438406)\n AND type = 2\n AND query_kind ='Select'\n AND initial_user IN ('default')\n AND query_duration_ms > 0\n \nGROUP BY t\nORDER BY t", "refId": "A", "resultFormat": "time_series", "round": "0s", + "skip_comments": true, "table": "query_log", "tableLoading": false }, @@ -710,11 +728,12 @@ "format": "time_series", "formattedQuery": "SELECT
    $timeSeries as t,
    avg(query_duration_ms) insert_duration
FROM cluster('all-sharded',system.query_log)
WHERE
    $timeFilter
    AND type = 2
and positionCaseInsensitive(query,  'insert into') = 1
    and initial_user in ($user)
GROUP BY t
ORDER BY t", "intervalFactor": 2, - "query": "SELECT\n $timeSeries as t,\n quantile(0.5)(query_duration_ms) insert_quantile_50,\n quantile(0.9)(query_duration_ms) insert_quantile_90\nFROM cluster('all-sharded',system.query_log)\nWHERE\n $timeFilter\n AND type = 2\nand positionCaseInsensitive(query, 'insert into') = 1\n and initial_user in ($user)\nGROUP BY t\nORDER BY t", - "rawQuery": "SELECT (intDiv(toUInt32(event_time), 10) * 10) * 1000 as t, quantile(0.5)(query_duration_ms) insert_quantile_50, quantile(0.9)(query_duration_ms) insert_quantile_90 FROM cluster('all-sharded',system.query_log) WHERE event_date >= toDate(1595573984) AND event_time >= toDateTime(1595573984) AND type = 2 and positionCaseInsensitive(query, 'insert into') = 1 and initial_user in ('default') GROUP BY t ORDER BY t", + "query": "SELECT\n $timeSeries as t,\n quantile(0.5)(query_duration_ms) insert_quantile_50,\n quantile(0.9)(query_duration_ms) insert_quantile_90\nFROM cluster('all-sharded',system.query_log)\nWHERE\n $timeFilter\n AND type = 2\n AND query_kind = 'Insert'\n AND initial_user in ($user)\n AND query_duration_ms > 0\n $conditionalTest(AND query_duration_ms <= $max_duration_ms,$max_duration_ms)\nGROUP BY t\nORDER BY t", + "rawQuery": "SELECT\n (intDiv(toUInt32(event_time), 4) * 4) * 1000 as t,\n quantile(0.5)(query_duration_ms) insert_quantile_50,\n quantile(0.9)(query_duration_ms) insert_quantile_90\nFROM cluster('all-sharded',system.query_log)\nWHERE\n event_date >= toDate(1694434806) AND event_date <= toDate(1694438406) AND event_time >= toDateTime(1694434806) AND event_time <= toDateTime(1694438406)\n AND type = 2\n AND query_kind = 'Insert'\n AND initial_user in ('default')\n AND query_duration_ms > 0\n \nGROUP BY t\nORDER BY t", "refId": "B", "resultFormat": "time_series", "round": "0s", + "skip_comments": true, "table": "query_log", "tableLoading": false } @@ -798,12 +817,14 @@ "styles": [ { "alias": "Time", + "align": "auto", "dateFormat": "YYYY-MM-DD HH:mm:ss", "pattern": "Time", "type": "date" }, { "alias": "", + "align": "auto", "colorMode": null, "colors": [ "rgba(245, 54, 54, 0.9)", @@ -819,6 +840,7 @@ }, { "alias": "", + "align": "auto", "colorMode": null, "colors": [ "rgba(245, 54, 54, 0.9)", @@ -834,6 +856,7 @@ }, { "alias": "", + "align": "auto", "colorMode": null, "colors": [ "rgba(245, 54, 54, 0.9)", @@ -858,11 +881,12 @@ "format": "time_series", "formattedQuery": "SELECT
    event_time,
    user,
    query_duration_ms duration,
    memory_usage memory,
    if(exception!='', 'fail', 'success') result,
    concat(substring(query,1,120), '...') query
FROM cluster('all-sharded',system.query_log)
WHERE $timeFilter
    AND type in ($type)
    and initial_user in ($user)
    and('$query_type' = 'all' or(positionCaseInsensitive(query,  '$query_type') = 1))
ORDER BY event_time DESC
LIMIT 1000", "intervalFactor": 1, - "query": "SELECT\n event_time,\n hostName() host,\n user,\n query_duration_ms duration,\n memory_usage memory,\n if(exception!='', 'fail', 'success') result,\n concat(substring(query,1,120), '...') query\nFROM cluster('all-sharded',system.query_log)\nWHERE $timeFilter\n AND type in ($type)\n and initial_user in ($user)\n and('$query_type' = 'all' OR (positionCaseInsensitive(query, '$query_type') = 1))\nORDER BY event_time DESC\nLIMIT 1000", - "rawQuery": "SELECT (intDiv(toUInt32(event_time), 10) * 10) * 1000 as t, avg(query_duration_ms) select_duration FROM cluster('all-sharded',system.query_log) WHERE event_date >= toDate(1586232299) AND event_time >= toDateTime(1586232299) AND type = 2 and positionCaseInsensitive(query, 'select') = 1 and initial_user in ('default') GROUP BY t ORDER BY t", + "query": "SELECT\n event_time,\n hostName() host,\n user,\n query_duration_ms duration,\n memory_usage memory,\n if(exception!='', 'fail', 'success') result,\n concat(substring(normalizeQuery(query),1,120), '...') query\nFROM cluster('all-sharded',system.query_log)\nWHERE $timeFilter\n AND type in ($type)\n and initial_user in ($user)\n and('$query_type' = 'all' OR (positionCaseInsensitive(query, '$query_type') = 1))\n $conditionalTest(AND query_duration_ms >= $min_duration_ms,$min_duration_ms)\n $conditionalTest(AND query_duration_ms <= $max_duration_ms,$max_duration_ms)\nORDER BY event_time DESC\nLIMIT 1000", + "rawQuery": "SELECT\n event_time,\n hostName() host,\n user,\n query_duration_ms duration,\n memory_usage memory,\n if(exception!='', 'fail', 'success') result,\n concat(substring(normalizeQuery(query),1,120), '...') query\nFROM cluster('all-sharded',system.query_log)\nWHERE event_date >= toDate(1694434937) AND event_date <= toDate(1694438537) AND event_time >= toDateTime(1694434937) AND event_time <= toDateTime(1694438537)\n AND type in (1,2,3,4)\n and initial_user in ('default')\n and('all' = 'all' OR (positionCaseInsensitive(query, 'all') = 1))\n AND query_duration_ms >= 0 \n \nORDER BY event_time DESC\nLIMIT 1000", "refId": "A", "resultFormat": "time_series", "round": "0s", + "skip_comments": true, "table": "query_log", "tableLoading": false } @@ -872,8 +896,8 @@ "type": "table" } ], - "refresh": "5s", - "schemaVersion": 21, + "refresh": "30s", + "schemaVersion": 27, "style": "dark", "tags": [ "Altinity", @@ -885,9 +909,12 @@ "list": [ { "current": { - "text": "k8s-test-test-002-tpl", - "value": "k8s-test-test-002-tpl" + "selected": false, + "text": "k8s-test-test-001", + "value": "k8s-test-test-001" }, + "description": null, + "error": null, "hide": 2, "includeAll": false, "label": "K8S ClickHouse installation", @@ -905,13 +932,18 @@ "current": {}, "datasource": "${DS_PROMETHEUS}", "definition": "label_values({__name__ =~ \"chi_clickhouse_metric_Uptime|chi_clickhouse_metric_fetch_errors\"}, exported_namespace)", + "description": null, + "error": null, "hide": 0, "includeAll": false, "label": "K8S Namespace", "multi": false, "name": "exported_namespace", "options": [], - "query": "label_values({__name__ =~ \"chi_clickhouse_metric_Uptime|chi_clickhouse_metric_fetch_errors\"}, exported_namespace)", + "query": { + "query": "label_values({__name__ =~ \"chi_clickhouse_metric_Uptime|chi_clickhouse_metric_fetch_errors\"}, exported_namespace)", + "refId": "clickhouse-operator-prometheus-exported_namespace-Variable-Query" + }, "refresh": 2, "regex": "", "skipUrlSync": false, @@ -927,13 +959,18 @@ "current": {}, "datasource": "${DS_PROMETHEUS}", "definition": "label_values({__name__ =~ \"chi_clickhouse_metric_Uptime|chi_clickhouse_metric_fetch_errors\", exported_namespace=\"$exported_namespace\"}, chi)", + "description": null, + "error": null, "hide": 0, "includeAll": false, "label": "K8S Clickhouse Installation", "multi": false, "name": "chi", "options": [], - "query": "label_values({__name__ =~ \"chi_clickhouse_metric_Uptime|chi_clickhouse_metric_fetch_errors\",exported_namespace=\"$exported_namespace\"}, chi)", + "query": { + "query": "label_values({__name__ =~ \"chi_clickhouse_metric_Uptime|chi_clickhouse_metric_fetch_errors\",exported_namespace=\"$exported_namespace\"}, chi)", + "refId": "clickhouse-operator-prometheus-chi-Variable-Query" + }, "refresh": 1, "regex": "", "skipUrlSync": false, @@ -953,6 +990,8 @@ "text": "5m", "value": "5m" }, + "description": null, + "error": null, "hide": 2, "label": null, "name": "interval", @@ -978,11 +1017,15 @@ "current": { "selected": true, "tags": [], - "text": "All", + "text": [ + "All" + ], "value": [ "$__all" ] }, + "description": null, + "error": null, "hide": 0, "includeAll": true, "label": "type", @@ -1016,6 +1059,7 @@ } ], "query": "1,2,3,4", + "queryValue": "", "skipUrlSync": false, "type": "custom" }, @@ -1027,6 +1071,8 @@ "text": "5", "value": "5" }, + "description": null, + "error": null, "hide": 0, "includeAll": false, "label": "top elements", @@ -1073,6 +1119,8 @@ "current": {}, "datasource": "$db", "definition": "SELECT DISTINCT initial_user FROM cluster('all-sharded',system.query_log) WHERE event_date >= today()-3 AND event_time > toDateTime(today()-3) SETTINGS skip_unavailable_shards=1", + "description": null, + "error": null, "hide": 0, "includeAll": true, "label": "initial user", @@ -1097,6 +1145,8 @@ "text": "all", "value": "all" }, + "description": null, + "error": null, "hide": 0, "includeAll": false, "label": "query type", @@ -1122,6 +1172,50 @@ "query": "all,select,insert", "skipUrlSync": false, "type": "custom" + }, + { + "current": { + "selected": true, + "text": "0", + "value": "0" + }, + "description": null, + "error": null, + "hide": 0, + "label": null, + "name": "min_duration_ms", + "options": [ + { + "selected": true, + "text": "0", + "value": "0" + } + ], + "query": "0", + "skipUrlSync": false, + "type": "textbox" + }, + { + "current": { + "selected": true, + "text": "", + "value": "" + }, + "description": null, + "error": null, + "hide": 0, + "label": null, + "name": "max_duration_ms", + "options": [ + { + "selected": true, + "text": "", + "value": "" + } + ], + "query": "", + "skipUrlSync": false, + "type": "textbox" } ] }, @@ -1157,5 +1251,5 @@ "timezone": "browser", "title": "ClickHouse in Kubernetes Queries", "uid": "clickhouse-queries", - "version": 20220214 + "version": 20230911 } From 2d0636c893cb24916affc902e276c0ce402d7bc6 Mon Sep 17 00:00:00 2001 From: Slach Date: Mon, 11 Sep 2023 19:34:30 +0400 Subject: [PATCH 2/5] change refresh rate to 1m Signed-off-by: Slach --- grafana-dashboard/ClickHouse_Queries_dashboard.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/grafana-dashboard/ClickHouse_Queries_dashboard.json b/grafana-dashboard/ClickHouse_Queries_dashboard.json index bf670e2a2..912204dae 100644 --- a/grafana-dashboard/ClickHouse_Queries_dashboard.json +++ b/grafana-dashboard/ClickHouse_Queries_dashboard.json @@ -896,7 +896,7 @@ "type": "table" } ], - "refresh": "30s", + "refresh": "1m", "schemaVersion": 27, "style": "dark", "tags": [ From d629ad06206e48de95ed6d897842674d02af49a7 Mon Sep 17 00:00:00 2001 From: Slach Date: Tue, 12 Sep 2023 20:20:54 +0400 Subject: [PATCH 3/5] rename query_type to query_kind, check all dropdown for work for All and Multi select, change some variables type and queries Signed-off-by: Slach --- .../ClickHouse_Queries_dashboard.json | 512 ++++++++++++------ 1 file changed, 361 insertions(+), 151 deletions(-) diff --git a/grafana-dashboard/ClickHouse_Queries_dashboard.json b/grafana-dashboard/ClickHouse_Queries_dashboard.json index 912204dae..b10d86a7a 100644 --- a/grafana-dashboard/ClickHouse_Queries_dashboard.json +++ b/grafana-dashboard/ClickHouse_Queries_dashboard.json @@ -65,7 +65,7 @@ "gnetId": 2515, "graphTooltip": 1, "id": null, - "iteration": 1694437448827, + "iteration": 1694526851642, "links": [], "panels": [ { @@ -89,11 +89,17 @@ "dashLength": 10, "dashes": false, "datasource": "$db", + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 7, - "w": 20, + "w": 24, "x": 0, "y": 1 }, @@ -117,9 +123,10 @@ "links": [], "nullPointMode": "null", "options": { - "dataLinks": [] + "alertThreshold": true }, "percentage": false, + "pluginVersion": "7.5.17", "pointradius": 5, "points": false, "renderer": "flot", @@ -136,11 +143,13 @@ "dateTimeType": "DATETIME", "datetimeLoading": false, "expr": "", + "extrapolate": false, "format": "time_series", "formattedQuery": "SELECT * FROM ($rateColumns(\n substring(query, 1, 45) AS query,\n count() c)\nFROM cluster('all-sharded',system.query_log)\nWHERE\n cityHash64(query) global in (\n SELECT cityHash64(substring(query, 1, 45)) AS h\n FROM cluster('all-sharded',system.query_log)\n WHERE\n $timeFilter\n AND type in ($type)\n AND initial_user in ($user)\n AND('$query_type' = 'all' or(positionCaseInsensitive(query, '$query_type') = 1))\n GROUP BY h\n ORDER BY count() desc\n LIMIT $top)\n AND type in ($type)\n AND initial_user in ($user)\n AND('$query_type' = 'all' or(positionCaseInsensitive(query, '$query_type') = 1))) SETTINGS skip_unavailable_shards=1", + "interval": "", "intervalFactor": 2, - "query": "SELECT\r\n t,\r\n arrayMap(a -> (a.1, a.2 / runningDifference(t / 1000)), groupArr)\r\nFROM (\r\n SELECT t, groupArray((q, c)) AS groupArr\r\n FROM (\r\n SELECT\r\n (intDiv(toUInt32(event_time), 2) * 2) * 1000 AS t,\r\n normalizeQuery(query) AS q,\r\n count() c\r\n FROM cluster('all-sharded',system.query_log)\r\n WHERE $timeFilter\r\n AND normalized_query_hash GLOBAL IN (\r\n SELECT normalized_query_hash AS h\r\n FROM cluster('all-sharded',system.query_log)\r\n WHERE $timeFilter\r\n AND type IN ($type)\r\n AND initial_user IN ($user)\r\n AND ('$query_type' = 'all' OR (positionCaseInsensitive(query, '$query_type') = 1))\r\n $conditionalTest(AND query_duration_ms >= $min_duration_ms,$min_duration_ms)\r\n $conditionalTest(AND query_duration_ms <= $max_duration_ms,$max_duration_ms)\r\n GROUP BY h\r\n ORDER BY count() DESC\r\n LIMIT $top\r\n SETTINGS skip_unavailable_shards=1\r\n )\r\n AND type IN ($type)\r\n AND initial_user IN ($user)\r\n AND ('$query_type' = 'all' OR (positionCaseInsensitive(query, '$query_type') = 1))\r\n $conditionalTest(AND query_duration_ms >= $min_duration_ms,$min_duration_ms)\r\n $conditionalTest(AND query_duration_ms <= $max_duration_ms,$max_duration_ms)\r\n GROUP BY t, query\r\n ORDER BY t\r\n )\r\n GROUP BY t\r\n ORDER BY t\r\n) SETTINGS skip_unavailable_shards=1", - "rawQuery": "SELECT\r\n t,\r\n arrayMap(a -> (a.1, a.2 / runningDifference(t / 1000)), groupArr)\r\nFROM (\r\n SELECT t, groupArray((q, c)) AS groupArr\r\n FROM (\r\n SELECT\r\n (intDiv(toUInt32(event_time), 2) * 2) * 1000 AS t,\r\n normalizeQuery(query) AS q,\r\n count() c\r\n FROM cluster('all-sharded',system.query_log)\r\n WHERE event_date >= toDate(1694434397) AND event_date <= toDate(1694437997) AND event_time >= toDateTime(1694434397) AND event_time <= toDateTime(1694437997)\r\n AND normalized_query_hash GLOBAL IN (\r\n SELECT normalized_query_hash AS h\r\n FROM cluster('all-sharded',system.query_log)\r\n WHERE event_date >= toDate(1694434397) AND event_date <= toDate(1694437997) AND event_time >= toDateTime(1694434397) AND event_time <= toDateTime(1694437997)\r\n AND type IN (1,2,3,4)\r\n AND initial_user IN ('default')\r\n AND ('all' = 'all' OR (positionCaseInsensitive(query, 'all') = 1))\r\n AND query_duration_ms >= 0 \r\n \r\n GROUP BY h\r\n ORDER BY count() DESC\r\n LIMIT 5\r\n SETTINGS skip_unavailable_shards=1\r\n )\r\n AND type IN (1,2,3,4)\r\n AND initial_user IN ('default')\r\n AND ('all' = 'all' OR (positionCaseInsensitive(query, 'all') = 1))\r\n AND query_duration_ms >= 0 \r\n \r\n GROUP BY t, query\r\n ORDER BY t\r\n )\r\n GROUP BY t\r\n ORDER BY t\r\n) SETTINGS skip_unavailable_shards=1", + "query": "SELECT\r\n t,\r\n arrayMap(a -> (a.1, a.2 / runningDifference(t / 1000)), groupArr)\r\nFROM (\r\n SELECT t, groupArray((q, c)) AS groupArr\r\n FROM (\r\n SELECT\r\n (intDiv(toUInt32(event_time), 2) * 2) * 1000 AS t,\r\n normalizeQuery(query) AS q,\r\n count() c\r\n FROM cluster('all-sharded',system.query_log)\r\n WHERE $timeFilter\r\n AND( ('$type' = '1,2,3,4' AND type != 'QueryStart') OR ('$type' != '1,2,3,4' AND type IN ($type)))\r\n $conditionalTest(AND query_kind IN ($query_kind), $query_kind)\r\n $conditionalTest(AND initial_user IN ($user), $user)\r\n $conditionalTest(AND query_duration_ms >= $min_duration_ms, $min_duration_ms)\r\n $conditionalTest(AND query_duration_ms <= $max_duration_ms, $max_duration_ms)\r\n AND normalized_query_hash GLOBAL IN (\r\n SELECT normalized_query_hash AS h\r\n FROM cluster('all-sharded',system.query_log)\r\n WHERE $timeFilter\r\n AND( ('$type' = '1,2,3,4' AND type != 'QueryStart') OR ('$type' != '1,2,3,4' AND type IN ($type)))\r\n $conditionalTest(AND query_kind IN ($query_kind), $query_kind)\r\n $conditionalTest(AND type IN ($type), $type)\r\n $conditionalTest(AND initial_user IN ($user), $user)\r\n $conditionalTest(AND query_duration_ms >= $min_duration_ms, $min_duration_ms)\r\n $conditionalTest(AND query_duration_ms <= $max_duration_ms, $max_duration_ms)\r\n GROUP BY h\r\n ORDER BY count() DESC\r\n LIMIT $top\r\n SETTINGS skip_unavailable_shards=1\r\n )\r\n GROUP BY t, query\r\n ORDER BY t\r\n )\r\n GROUP BY t\r\n ORDER BY t\r\n) SETTINGS skip_unavailable_shards=1", + "rawQuery": "SELECT\r\n t,\r\n arrayMap(a -> (a.1, a.2 / runningDifference(t / 1000)), groupArr)\r\nFROM (\r\n SELECT t, groupArray((q, c)) AS groupArr\r\n FROM (\r\n SELECT\r\n (intDiv(toUInt32(event_time), 2) * 2) * 1000 AS t,\r\n normalizeQuery(query) AS q,\r\n count() c\r\n FROM cluster('all-sharded',system.query_log)\r\n WHERE event_date >= toDate(1694531137) AND event_date <= toDate(1694534737) AND event_time >= toDateTime(1694531137) AND event_time <= toDateTime(1694534737)\r\n AND( ('1,2,3,4' = '1,2,3,4' AND type != 'QueryStart') OR ('1,2,3,4' != '1,2,3,4' AND type IN (1,2,3,4)))\r\n \r\n \r\n \r\n \r\n AND normalized_query_hash GLOBAL IN (\r\n SELECT normalized_query_hash AS h\r\n FROM cluster('all-sharded',system.query_log)\r\n WHERE event_date >= toDate(1694531137) AND event_date <= toDate(1694534737) AND event_time >= toDateTime(1694531137) AND event_time <= toDateTime(1694534737)\r\n AND( ('1,2,3,4' = '1,2,3,4' AND type != 'QueryStart') OR ('1,2,3,4' != '1,2,3,4' AND type IN (1,2,3,4)))\r\n \r\n \r\n \r\n \r\n \r\n GROUP BY h\r\n ORDER BY count() DESC\r\n LIMIT 30\r\n SETTINGS skip_unavailable_shards=1\r\n )\r\n GROUP BY t, query\r\n ORDER BY t\r\n )\r\n GROUP BY t\r\n ORDER BY t\r\n) SETTINGS skip_unavailable_shards=1", "refId": "A", "resultFormat": "time_series", "round": "0s", @@ -190,26 +199,57 @@ "alignLevel": null } }, - { - "content": "1 - successful start of query execution\n\n2 - successful end of query execution\n\n3 - exception before start of query execution\n\n4 - exception while query execution", - "datasource": "$db", - "gridPos": { - "h": 7, - "w": 4, - "x": 20, - "y": 1 - }, - "id": 17, - "links": [], - "mode": "markdown", - "options": {}, - "title": "Types", - "transparent": true, - "type": "text" - }, { "columns": [], "datasource": "$db", + "fieldConfig": { + "defaults": { + "custom": { + "align": null, + "filterable": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "duration_ms" + }, + "properties": [ + { + "id": "custom.width", + "value": 100 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "count" + }, + "properties": [ + { + "id": "custom.width", + "value": 100 + } + ] + } + ] + }, "fontSize": "100%", "gridPos": { "h": 10, @@ -220,8 +260,12 @@ "height": "400px", "id": 18, "links": [], - "options": {}, + "options": { + "showHeader": true, + "sortBy": [] + }, "pageSize": null, + "pluginVersion": "7.5.17", "scroll": true, "showHeader": true, "sort": { @@ -293,11 +337,13 @@ "dateTimeType": "DATETIME", "datetimeLoading": false, "expr": "", + "extrapolate": false, "format": "table", "formattedQuery": "SELECT
    rand() as t,
    substring(query,  1,  70) AS query,
    avg(query_duration_ms) duration,
    count() count
FROM cluster('all-sharded',system.query_log)
WHERE
    $timeFilter
    AND type in ($type)
    and initial_user in ($user)
    and('$query_type' = 'all' or(positionCaseInsensitive(query,  '$query_type') = 1))
GROUP BY query
ORDER BY duration desc
LIMIT $top", + "interval": "", "intervalFactor": 2, - "query": "SELECT\n rand() as t,\n normalizeQuery(query) AS query,\n avg(query_duration_ms) duration,\n count() count\nFROM cluster('all-sharded',system.query_log)\nWHERE\n $timeFilter\n AND type in ($type)\n AND initial_user in ($user)\n AND('$query_type' = 'all' or(positionCaseInsensitive(query, '$query_type') = 1))\n $conditionalTest(AND query_duration_ms >= $min_duration_ms,$min_duration_ms)\n $conditionalTest(AND query_duration_ms <= $max_duration_ms,$max_duration_ms)\nGROUP BY query\nORDER BY duration desc\nLIMIT $top", - "rawQuery": "SELECT\n rand() as t,\n normalizeQuery(query) AS query,\n avg(query_duration_ms) duration,\n count() count\nFROM cluster('all-sharded',system.query_log)\nWHERE\n event_date >= toDate(1694434499) AND event_date <= toDate(1694438099) AND event_time >= toDateTime(1694434499) AND event_time <= toDateTime(1694438099)\n AND type in (1,2,3,4)\n AND initial_user in ('default')\n AND('all' = 'all' or(positionCaseInsensitive(query, 'all') = 1))\n AND query_duration_ms >= 0 \n \nGROUP BY query\nORDER BY duration desc\nLIMIT 5", + "query": "SELECT\n normalizeQuery(query) AS query,\n avg(query_duration_ms) duration_ms,\n count() count\nFROM cluster('all-sharded',system.query_log)\nWHERE\n $timeFilter\n AND( ('$type' = '1,2,3,4' AND type != 'QueryStart') OR ('$type' != '1,2,3,4' AND type IN ($type)))\n $conditionalTest(AND query_kind IN ($query_kind), $query_kind)\n $conditionalTest(AND initial_user IN ($user), $user)\n $conditionalTest(AND query_duration_ms >= $min_duration_ms,$min_duration_ms)\n $conditionalTest(AND query_duration_ms <= $max_duration_ms,$max_duration_ms)\nGROUP BY query\nORDER BY duration_ms DESC\nLIMIT $top", + "rawQuery": "SELECT\n normalizeQuery(query) AS query,\n avg(query_duration_ms) duration_ms,\n count() count\nFROM cluster('all-sharded',system.query_log)\nWHERE\n event_date >= toDate(1694531150) AND event_date <= toDate(1694534750) AND event_time >= toDateTime(1694531150) AND event_time <= toDateTime(1694534750)\n AND( ('1,2,3,4' = '1,2,3,4' AND type != 'QueryStart') OR ('1,2,3,4' != '1,2,3,4' AND type IN (1,2,3,4)))\n \n \n \n \nGROUP BY query\nORDER BY duration_ms DESC\nLIMIT 30", "refId": "A", "resultFormat": "time_series", "round": "0s", @@ -313,6 +359,58 @@ { "columns": [], "datasource": "$db", + "fieldConfig": { + "defaults": { + "custom": { + "align": null, + "filterable": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "usage" + }, + "properties": [ + { + "id": "unit", + "value": "bytes" + }, + { + "id": "custom.width", + "value": 100 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "count" + }, + "properties": [ + { + "id": "custom.width", + "value": 100 + } + ] + } + ] + }, "fontSize": "100%", "gridPos": { "h": 10, @@ -323,8 +421,11 @@ "height": "400px", "id": 19, "links": [], - "options": {}, + "options": { + "showHeader": true + }, "pageSize": null, + "pluginVersion": "7.5.17", "scroll": true, "showHeader": true, "sort": { @@ -396,11 +497,12 @@ "dateTimeType": "DATETIME", "datetimeLoading": false, "expr": "", + "extrapolate": false, "format": "table", "formattedQuery": "SELECT
    rand() as t,
    substring(query,  1,  70) AS query,
    avg(memory_usage) usage,
    count() count
FROM cluster('all-sharded',system.query_log)
WHERE
    $timeFilter
    AND type in ($type)
    and initial_user in ($user)
    and('$query_type' = 'all' or(positionCaseInsensitive(query,  '$query_type') = 1))
GROUP BY query
ORDER BY usage desc
LIMIT $top", "intervalFactor": 2, - "query": "SELECT\n rand() as t,\n substring(query, 1, 70) AS query,\n avg(memory_usage) usage,\n count() count\nFROM cluster('all-sharded',system.query_log)\nWHERE\n $timeFilter\n AND type IN ($type)\n AND initial_user IN ($user)\n AND ('$query_type' = 'all' OR (positionCaseInsensitive(query, '$query_type') = 1))\n $conditionalTest(AND query_duration_ms >= $min_duration_ms,$min_duration_ms)\n $conditionalTest(AND query_duration_ms <= $max_duration_ms,$max_duration_ms)\nGROUP BY query\nORDER BY usage DESC\nLIMIT $top", - "rawQuery": "SELECT\n rand() as t,\n substring(query, 1, 70) AS query,\n avg(memory_usage) usage,\n count() count\nFROM cluster('all-sharded',system.query_log)\nWHERE\n event_date >= toDate(1694434541) AND event_date <= toDate(1694438141) AND event_time >= toDateTime(1694434541) AND event_time <= toDateTime(1694438141)\n AND type IN (1,2,3,4)\n AND initial_user IN ('default')\n AND ('all' = 'all' OR (positionCaseInsensitive(query, 'all') = 1))\n AND query_duration_ms >= 0 \n \nGROUP BY query\nORDER BY usage DESC\nLIMIT 5", + "query": "SELECT\n normalizeQuery(query) AS query,\n avg(memory_usage) usage,\n count() count\nFROM cluster('all-sharded',system.query_log)\nWHERE $timeFilter\n AND( ('$type' = '1,2,3,4' AND type != 'QueryStart') OR ('$type' != '1,2,3,4' AND type IN ($type)))\n $conditionalTest(AND query_kind IN ($query_kind), $query_kind)\n $conditionalTest(AND initial_user IN ($user), $user)\n $conditionalTest(AND query_duration_ms >= $min_duration_ms, $min_duration_ms)\n $conditionalTest(AND query_duration_ms <= $max_duration_ms, $max_duration_ms)\nGROUP BY query\nORDER BY usage DESC\nLIMIT $top", + "rawQuery": "SELECT\n normalizeQuery(query) AS query,\n avg(memory_usage) usage,\n count() count\nFROM cluster('all-sharded',system.query_log)\nWHERE event_date >= toDate(1694531173) AND event_date <= toDate(1694534773) AND event_time >= toDateTime(1694531173) AND event_time <= toDateTime(1694534773)\n AND( ('1,2,3,4' = '1,2,3,4' AND type != 'QueryStart') OR ('1,2,3,4' != '1,2,3,4' AND type IN (1,2,3,4)))\n \n \n \n \nGROUP BY query\nORDER BY usage DESC\nLIMIT 30", "refId": "A", "resultFormat": "time_series", "round": "0s", @@ -416,6 +518,54 @@ { "columns": [], "datasource": "$db", + "fieldConfig": { + "defaults": { + "custom": { + "align": null, + "filterable": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "type" + }, + "properties": [ + { + "id": "custom.width", + "value": 150 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "count" + }, + "properties": [ + { + "id": "custom.width", + "value": 100 + } + ] + } + ] + }, "fontSize": "100%", "gridPos": { "h": 10, @@ -426,8 +576,11 @@ "height": "400px", "id": 20, "links": [], - "options": {}, + "options": { + "showHeader": true + }, "pageSize": null, + "pluginVersion": "7.5.17", "scroll": true, "showHeader": true, "sort": { @@ -499,11 +652,12 @@ "dateTimeType": "DATETIME", "datetimeLoading": false, "expr": "", + "extrapolate": false, "format": "table", "formattedQuery": "SELECT
    rand() as t,
    substring(query,  1,  70) AS query,
    type,
    count() count
FROM cluster('all-sharded',system.query_log)
WHERE
    $timeFilter
    AND type in (3,4)
    and initial_user in ($user)
    and('$query_type' = 'all' or(positionCaseInsensitive(query,  '$query_type') = 1))
GROUP BY
    query,
    type
ORDER BY count desc
LIMIT $top", "intervalFactor": 2, - "query": "SELECT\n rand() as t,\n normalizeQuery(query) AS query,\n type,\n count() count\nFROM cluster('all-sharded',system.query_log)\nWHERE\n $timeFilter\n AND type IN (3,4)\n AND initial_user IN ($user)\n AND ('$query_type' = 'all' OR (positionCaseInsensitive(query, '$query_type') = 1))\n $conditionalTest(AND query_duration_ms >= $min_duration_ms,$min_duration_ms)\n $conditionalTest(AND query_duration_ms <= $max_duration_ms,$max_duration_ms)\nGROUP BY\n query,\n type\nORDER BY count desc\nLIMIT $top", - "rawQuery": "SELECT\n rand() as t,\n normalizeQuery(query) AS query,\n type,\n count() count\nFROM cluster('all-sharded',system.query_log)\nWHERE\n event_date >= toDate(1694434583) AND event_date <= toDate(1694438183) AND event_time >= toDateTime(1694434583) AND event_time <= toDateTime(1694438183)\n AND type IN (3,4)\n AND initial_user IN ('default')\n AND ('all' = 'all' OR (positionCaseInsensitive(query, 'all') = 1))\n AND query_duration_ms >= 0 \n \nGROUP BY\n query,\n type\nORDER BY count desc\nLIMIT 5", + "query": "SELECT\n normalizeQuery(query) AS query,\n type,\n count() count\nFROM cluster('all-sharded',system.query_log)\nWHERE $timeFilter\n AND( ('$type' = '1,2,3,4' AND type != 'QueryStart') OR ('$type' != '1,2,3,4' AND type IN ($type)))\n AND type NOT IN ('QueryStart','QueryFinish')\n $conditionalTest(AND query_kind IN ($query_kind), $query_kind)\n $conditionalTest(AND initial_user IN ($user), $user)\n $conditionalTest(AND query_duration_ms >= $min_duration_ms, $min_duration_ms)\n $conditionalTest(AND query_duration_ms <= $max_duration_ms, $max_duration_ms)\nGROUP BY\n query,\n type\nORDER BY count DESC\nLIMIT $top\nSETTINGS skip_unavailable_shards=1", + "rawQuery": "SELECT\n normalizeQuery(query) AS query,\n type,\n count() count\nFROM cluster('all-sharded',system.query_log)\nWHERE event_date >= toDate(1694531196) AND event_date <= toDate(1694534796) AND event_time >= toDateTime(1694531196) AND event_time <= toDateTime(1694534796)\n AND( ('1,2,3,4' = '1,2,3,4' AND type != 'QueryStart') OR ('1,2,3,4' != '1,2,3,4' AND type IN (1,2,3,4)))\n AND type NOT IN ('QueryStart','QueryFinish')\n \n \n \n \nGROUP BY\n query,\n type\nORDER BY count DESC\nLIMIT 30\nSETTINGS skip_unavailable_shards=1", "refId": "A", "resultFormat": "time_series", "round": "0s", @@ -537,6 +691,12 @@ "dashLength": 10, "dashes": false, "datasource": "$db", + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, "fill": 1, "fillGradient": 0, "gridPos": { @@ -561,9 +721,10 @@ "links": [], "nullPointMode": "null", "options": { - "dataLinks": [] + "alertThreshold": true }, "percentage": false, + "pluginVersion": "7.5.17", "pointradius": 5, "points": false, "renderer": "flot", @@ -580,12 +741,13 @@ "dateTimeType": "DATETIME", "datetimeLoading": false, "expr": "", + "extrapolate": false, "format": "time_series", "formattedQuery": "$rate(count() c)
FROM cluster('all-sharded',system.query_log)
where  type in ($type)
    and initial_user in ($user)
    and('$query_type' = 'all' or(positionCaseInsensitive(query,  '$query_type') = 1))", "interval": "", "intervalFactor": 2, - "query": "$rate(count() c)\nFROM cluster('all-sharded',system.query_log)\nWHERE type IN($type)\n AND initial_user IN ($user)\n AND ('$query_type' = 'all' OR (positionCaseInsensitive(query, '$query_type') = 1))\n $conditionalTest(AND query_duration_ms >= $min_duration_ms,$min_duration_ms)\n $conditionalTest(AND query_duration_ms <= $max_duration_ms,$max_duration_ms)\n ", - "rawQuery": "SELECT t, c/runningDifference(t/1000) cRate FROM ( SELECT (intDiv(toUInt32(event_time), 4) * 4) * 1000 AS t, count() c FROM cluster('all-sharded',system.query_log)\nWHERE event_date >= toDate(1694434633) AND event_date <= toDate(1694438233) AND event_time >= toDateTime(1694434633) AND event_time <= toDateTime(1694438233) AND type IN(1,2,3,4)\n AND initial_user IN ('default')\n AND ('all' = 'all' OR (positionCaseInsensitive(query, 'all') = 1))\n AND query_duration_ms >= 0 \n GROUP BY t ORDER BY t)", + "query": "$rate(count() c)\nFROM cluster('all-sharded',system.query_log)\nWHERE $timeFilter\n AND( ('$type' = '1,2,3,4' AND type != 'QueryStart') OR ('$type' != '1,2,3,4' AND type IN ($type)))\n $conditionalTest(AND query_kind IN ($query_kind), $query_kind)\n $conditionalTest(AND initial_user IN ($user), $user)\n $conditionalTest(AND query_duration_ms >= $min_duration_ms,$min_duration_ms)\n $conditionalTest(AND query_duration_ms <= $max_duration_ms,$max_duration_ms)\n", + "rawQuery": "SELECT t, c/runningDifference(t/1000) cRate FROM ( SELECT (intDiv(toUInt32(event_time), 4) * 4) * 1000 AS t, count() c FROM cluster('all-sharded',system.query_log)\nWHERE event_date >= toDate(1694531229) AND event_date <= toDate(1694534829) AND event_time >= toDateTime(1694531229) AND event_time <= toDateTime(1694534829) AND event_date >= toDate(1694531229) AND event_date <= toDate(1694534829) AND event_time >= toDateTime(1694531229) AND event_time <= toDateTime(1694534829)\n AND( ('1,2,3,4' = '1,2,3,4' AND type != 'QueryStart') OR ('1,2,3,4' != '1,2,3,4' AND type IN (1,2,3,4)))\n \n \n \n GROUP BY t ORDER BY t)", "refId": "A", "resultFormat": "time_series", "round": "0s", @@ -598,7 +760,7 @@ "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Reqs/s by type: $type; user: $user; query type: $query_type", + "title": "Reqs/s type: $type; user: $user; query type: $query_type", "tooltip": { "shared": true, "sort": 0, @@ -641,6 +803,10 @@ "dashLength": 10, "dashes": false, "datasource": "$db", + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, "fill": 1, "fillGradient": 1, "gridPos": { @@ -668,6 +834,7 @@ "alertThreshold": true }, "percentage": false, + "pluginVersion": "7.5.17", "pointradius": 5, "points": false, "renderer": "flot", @@ -705,11 +872,13 @@ "dateTimeType": "DATETIME", "datetimeLoading": false, "expr": "", + "extrapolate": false, "format": "time_series", "formattedQuery": "SELECT
    $timeSeries as t,
    avg(query_duration_ms)  select_duration
FROM cluster('all-sharded',system.query_log)
WHERE
    $timeFilter
    AND type = 2
    and positionCaseInsensitive(query,  'select') = 1
    and initial_user in ($user)
GROUP BY t
ORDER BY t", + "interval": "", "intervalFactor": 2, - "query": "SELECT\n $timeSeries as t,\n quantile(0.5)(query_duration_ms) select_quantile_50,\n quantile(0.9)(query_duration_ms) select_quantile_90\nFROM cluster('all-sharded',system.query_log)\nWHERE\n $timeFilter\n AND type = 2\n AND query_kind ='Select'\n AND initial_user IN ($user)\n AND query_duration_ms > 0\n $conditionalTest(AND query_duration_ms <= $max_duration_ms,$max_duration_ms)\nGROUP BY t\nORDER BY t", - "rawQuery": "SELECT\n (intDiv(toUInt32(event_time), 4) * 4) * 1000 as t,\n quantile(0.5)(query_duration_ms) select_quantile_50,\n quantile(0.9)(query_duration_ms) select_quantile_90\nFROM cluster('all-sharded',system.query_log)\nWHERE\n event_date >= toDate(1694434806) AND event_date <= toDate(1694438406) AND event_time >= toDateTime(1694434806) AND event_time <= toDateTime(1694438406)\n AND type = 2\n AND query_kind ='Select'\n AND initial_user IN ('default')\n AND query_duration_ms > 0\n \nGROUP BY t\nORDER BY t", + "query": "SELECT\n $timeSeries as t,\n quantile(0.5)(query_duration_ms) select_quantile_50,\n quantile(0.9)(query_duration_ms) select_quantile_90\nFROM cluster('all-sharded',system.query_log)\nWHERE\n $timeFilter\n AND type != 1\n AND query_kind ='Select'\n AND query_duration_ms > 0\n $conditionalTest(AND initial_user IN ($user), $user)\n $conditionalTest(AND query_duration_ms <= $max_duration_ms,$max_duration_ms)\nGROUP BY t\nORDER BY t", + "rawQuery": "SELECT\n (intDiv(toUInt32(event_time), 4) * 4) * 1000 as t,\n quantile(0.5)(query_duration_ms) select_quantile_50,\n quantile(0.9)(query_duration_ms) select_quantile_90\nFROM cluster('all-sharded',system.query_log)\nWHERE\n event_date >= toDate(1694526375) AND event_date <= toDate(1694529975) AND event_time >= toDateTime(1694526375) AND event_time <= toDateTime(1694529975)\n AND type != 1\n AND query_kind ='Select'\n AND query_duration_ms > 0\n \n \nGROUP BY t\nORDER BY t", "refId": "A", "resultFormat": "time_series", "round": "0s", @@ -725,11 +894,12 @@ "dateTimeType": "DATETIME", "datetimeLoading": false, "expr": "", + "extrapolate": false, "format": "time_series", "formattedQuery": "SELECT
    $timeSeries as t,
    avg(query_duration_ms) insert_duration
FROM cluster('all-sharded',system.query_log)
WHERE
    $timeFilter
    AND type = 2
and positionCaseInsensitive(query,  'insert into') = 1
    and initial_user in ($user)
GROUP BY t
ORDER BY t", "intervalFactor": 2, - "query": "SELECT\n $timeSeries as t,\n quantile(0.5)(query_duration_ms) insert_quantile_50,\n quantile(0.9)(query_duration_ms) insert_quantile_90\nFROM cluster('all-sharded',system.query_log)\nWHERE\n $timeFilter\n AND type = 2\n AND query_kind = 'Insert'\n AND initial_user in ($user)\n AND query_duration_ms > 0\n $conditionalTest(AND query_duration_ms <= $max_duration_ms,$max_duration_ms)\nGROUP BY t\nORDER BY t", - "rawQuery": "SELECT\n (intDiv(toUInt32(event_time), 4) * 4) * 1000 as t,\n quantile(0.5)(query_duration_ms) insert_quantile_50,\n quantile(0.9)(query_duration_ms) insert_quantile_90\nFROM cluster('all-sharded',system.query_log)\nWHERE\n event_date >= toDate(1694434806) AND event_date <= toDate(1694438406) AND event_time >= toDateTime(1694434806) AND event_time <= toDateTime(1694438406)\n AND type = 2\n AND query_kind = 'Insert'\n AND initial_user in ('default')\n AND query_duration_ms > 0\n \nGROUP BY t\nORDER BY t", + "query": "SELECT\n $timeSeries as t,\n quantile(0.5)(query_duration_ms) insert_quantile_50,\n quantile(0.9)(query_duration_ms) insert_quantile_90\nFROM cluster('all-sharded',system.query_log)\nWHERE\n $timeFilter\n AND type = 2\n AND query_kind = 'Insert'\n AND query_duration_ms > 0\n $conditionalTest(AND initial_user IN ($user), $user)\n $conditionalTest(AND query_duration_ms <= $max_duration_ms,$max_duration_ms)\nGROUP BY t\nORDER BY t", + "rawQuery": "SELECT\n (intDiv(toUInt32(event_time), 4) * 4) * 1000 as t,\n quantile(0.5)(query_duration_ms) insert_quantile_50,\n quantile(0.9)(query_duration_ms) insert_quantile_90\nFROM cluster('all-sharded',system.query_log)\nWHERE\n event_date >= toDate(1694526375) AND event_date <= toDate(1694529975) AND event_time >= toDateTime(1694526375) AND event_time <= toDateTime(1694529975)\n AND type = 2\n AND query_kind = 'Insert'\n AND query_duration_ms > 0\n \n \nGROUP BY t\nORDER BY t", "refId": "B", "resultFormat": "time_series", "round": "0s", @@ -797,6 +967,102 @@ { "columns": [], "datasource": "$db", + "fieldConfig": { + "defaults": { + "custom": { + "align": null, + "filterable": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "result" + }, + "properties": [ + { + "id": "custom.width", + "value": 106 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "memory" + }, + "properties": [ + { + "id": "custom.width", + "value": 108 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "duration" + }, + "properties": [ + { + "id": "custom.width", + "value": 117 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "user" + }, + "properties": [ + { + "id": "custom.width", + "value": 114 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Time" + }, + "properties": [ + { + "id": "custom.width", + "value": 149 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "host" + }, + "properties": [ + { + "id": "custom.width", + "value": 223 + } + ] + } + ] + }, "fontSize": "100%", "gridPos": { "h": 7, @@ -806,8 +1072,12 @@ }, "id": 21, "links": [], - "options": {}, + "options": { + "showHeader": true, + "sortBy": [] + }, "pageSize": null, + "pluginVersion": "7.5.17", "scroll": true, "showHeader": true, "sort": { @@ -878,11 +1148,13 @@ "dateTimeColDataType": "event_time", "dateTimeType": "DATETIME", "datetimeLoading": false, + "extrapolate": false, "format": "time_series", "formattedQuery": "SELECT
    event_time,
    user,
    query_duration_ms duration,
    memory_usage memory,
    if(exception!='', 'fail', 'success') result,
    concat(substring(query,1,120), '...') query
FROM cluster('all-sharded',system.query_log)
WHERE $timeFilter
    AND type in ($type)
    and initial_user in ($user)
    and('$query_type' = 'all' or(positionCaseInsensitive(query,  '$query_type') = 1))
ORDER BY event_time DESC
LIMIT 1000", + "interval": "", "intervalFactor": 1, - "query": "SELECT\n event_time,\n hostName() host,\n user,\n query_duration_ms duration,\n memory_usage memory,\n if(exception!='', 'fail', 'success') result,\n concat(substring(normalizeQuery(query),1,120), '...') query\nFROM cluster('all-sharded',system.query_log)\nWHERE $timeFilter\n AND type in ($type)\n and initial_user in ($user)\n and('$query_type' = 'all' OR (positionCaseInsensitive(query, '$query_type') = 1))\n $conditionalTest(AND query_duration_ms >= $min_duration_ms,$min_duration_ms)\n $conditionalTest(AND query_duration_ms <= $max_duration_ms,$max_duration_ms)\nORDER BY event_time DESC\nLIMIT 1000", - "rawQuery": "SELECT\n event_time,\n hostName() host,\n user,\n query_duration_ms duration,\n memory_usage memory,\n if(exception!='', 'fail', 'success') result,\n concat(substring(normalizeQuery(query),1,120), '...') query\nFROM cluster('all-sharded',system.query_log)\nWHERE event_date >= toDate(1694434937) AND event_date <= toDate(1694438537) AND event_time >= toDateTime(1694434937) AND event_time <= toDateTime(1694438537)\n AND type in (1,2,3,4)\n and initial_user in ('default')\n and('all' = 'all' OR (positionCaseInsensitive(query, 'all') = 1))\n AND query_duration_ms >= 0 \n \nORDER BY event_time DESC\nLIMIT 1000", + "query": "SELECT\n event_time,\n hostName() host,\n user,\n query_duration_ms duration,\n memory_usage memory,\n if(exception!='', 'fail', 'success') result,\n concat(substring(normalizeQuery(query),1,120), '...') query\nFROM cluster('all-sharded',system.query_log)\nWHERE $timeFilter\n AND type in ($type)\n $conditionalTest(AND query_kind IN ($query_kind), $query_kind)\n $conditionalTest(AND initial_user IN ($user), $user)\n $conditionalTest(AND query_duration_ms >= $min_duration_ms,$min_duration_ms)\n $conditionalTest(AND query_duration_ms <= $max_duration_ms,$max_duration_ms)\nORDER BY event_time DESC\nLIMIT 1000", + "rawQuery": "SELECT\n event_time,\n hostName() host,\n user,\n query_duration_ms duration,\n memory_usage memory,\n if(exception!='', 'fail', 'success') result,\n concat(substring(normalizeQuery(query),1,120), '...') query\nFROM cluster('all-sharded',system.query_log)\nWHERE event_date >= toDate(1694531300) AND event_date <= toDate(1694534900) AND event_time >= toDateTime(1694531300) AND event_time <= toDateTime(1694534900)\n AND type in (1,2,3,4)\n \n \n \n \nORDER BY event_time DESC\nLIMIT 1000", "refId": "A", "resultFormat": "time_series", "round": "0s", @@ -893,6 +1165,12 @@ ], "title": "Query log by type: $type; user: $user; query type: $query_type", "transform": "timeseries_to_columns", + "transformations": [ + { + "id": "prepareTimeSeries", + "options": {} + } + ], "type": "table" } ], @@ -982,48 +1260,10 @@ "useTags": false }, { - "auto": true, - "auto_count": 100, - "auto_min": "1m", - "current": { - "selected": false, - "text": "5m", - "value": "5m" - }, - "description": null, - "error": null, - "hide": 2, - "label": null, - "name": "interval", - "options": [ - { - "selected": false, - "text": "auto", - "value": "$__auto_interval_interval" - }, - { - "selected": true, - "text": "5m", - "value": "5m" - } - ], - "query": "5m", - "refresh": 2, - "skipUrlSync": false, - "type": "interval" - }, - { - "allValue": "1,2,3,4", - "current": { - "selected": true, - "tags": [], - "text": [ - "All" - ], - "value": [ - "$__all" - ] - }, + "allValue": "", + "current": {}, + "datasource": "$db", + "definition": "SELECT\n values[2] value,\n keys[1] text\nFROM (\n SELECT\n arrayMap(x -> splitByString('\\' = ', x), splitByString(', \\'',extract(type,'Enum8\\(\\'(.+)\\)'))) AS enum_arr\n FROM system.columns\n WHERE database='system' and table='query_log' AND name='type'\n)\nARRAY JOIN enum_arr as keys, enum_arr as values", "description": null, "error": null, "hide": 0, @@ -1031,37 +1271,17 @@ "label": "type", "multi": true, "name": "type", - "options": [ - { - "selected": true, - "text": "All", - "value": "$__all" - }, - { - "selected": false, - "text": "1", - "value": "1" - }, - { - "selected": false, - "text": "2", - "value": "2" - }, - { - "selected": false, - "text": "3", - "value": "3" - }, - { - "selected": false, - "text": "4", - "value": "4" - } - ], - "query": "1,2,3,4", - "queryValue": "", + "options": [], + "query": "SELECT\n values[2] value,\n keys[1] text\nFROM (\n SELECT\n arrayMap(x -> splitByString('\\' = ', x), splitByString(', \\'',extract(type,'Enum8\\(\\'(.+)\\)'))) AS enum_arr\n FROM system.columns\n WHERE database='system' and table='query_log' AND name='type'\n)\nARRAY JOIN enum_arr as keys, enum_arr as values", + "refresh": 2, + "regex": "", "skipUrlSync": false, - "type": "custom" + "sort": 0, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false }, { "allValue": null, @@ -1111,6 +1331,7 @@ } ], "query": "5,10,15,20,25,30", + "queryValue": "", "skipUrlSync": false, "type": "custom" }, @@ -1118,7 +1339,7 @@ "allValue": "", "current": {}, "datasource": "$db", - "definition": "SELECT DISTINCT initial_user FROM cluster('all-sharded',system.query_log) WHERE event_date >= today()-3 AND event_time > toDateTime(today()-3) SETTINGS skip_unavailable_shards=1", + "definition": "SELECT DISTINCT initial_user FROM cluster('all-sharded',system.query_log) WHERE event_date BETWEEN toDate($__from / 1000) AND toDate($__to / 1000) AND event_time BETWEEN toDateTime($__from / 1000) AND toDateTime($__to / 1000) SETTINGS skip_unavailable_shards=1", "description": null, "error": null, "hide": 0, @@ -1127,7 +1348,7 @@ "multi": true, "name": "user", "options": [], - "query": "SELECT DISTINCT initial_user FROM cluster('all-sharded',system.query_log) WHERE event_date >= today()-3 AND event_time > toDateTime(today()-3) SETTINGS skip_unavailable_shards=1", + "query": "SELECT DISTINCT initial_user FROM cluster('all-sharded',system.query_log) WHERE event_date BETWEEN toDate($__from / 1000) AND toDate($__to / 1000) AND event_time BETWEEN toDateTime($__from / 1000) AND toDateTime($__to / 1000) SETTINGS skip_unavailable_shards=1", "refresh": 1, "regex": "", "skipUrlSync": false, @@ -1140,44 +1361,33 @@ }, { "allValue": "", - "current": { - "selected": true, - "text": "all", - "value": "all" - }, + "current": {}, + "datasource": "$db", + "definition": "SELECT DISTINCT query_kind FROM cluster('all-sharded',system.query_log) WHERE event_date BETWEEN toDate($__from / 1000) AND toDate($__to / 1000) AND event_time BETWEEN toDateTime($__from / 1000) AND toDateTime($__to / 1000) SETTINGS skip_unavailable_shards=1", "description": null, "error": null, "hide": 0, - "includeAll": false, - "label": "query type", - "multi": false, - "name": "query_type", - "options": [ - { - "selected": true, - "text": "all", - "value": "all" - }, - { - "selected": false, - "text": "select", - "value": "select" - }, - { - "selected": false, - "text": "insert", - "value": "insert" - } - ], - "query": "all,select,insert", + "includeAll": true, + "label": "query_kind", + "multi": true, + "name": "query_kind", + "options": [], + "query": "SELECT DISTINCT query_kind FROM cluster('all-sharded',system.query_log) WHERE event_date BETWEEN toDate($__from / 1000) AND toDate($__to / 1000) AND event_time BETWEEN toDateTime($__from / 1000) AND toDateTime($__to / 1000) SETTINGS skip_unavailable_shards=1", + "refresh": 1, + "regex": "", "skipUrlSync": false, - "type": "custom" + "sort": 0, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false }, { "current": { - "selected": true, - "text": "0", - "value": "0" + "selected": false, + "text": "", + "value": "" }, "description": null, "error": null, @@ -1186,12 +1396,12 @@ "name": "min_duration_ms", "options": [ { - "selected": true, + "selected": false, "text": "0", "value": "0" } ], - "query": "0", + "query": "", "skipUrlSync": false, "type": "textbox" }, @@ -1251,5 +1461,5 @@ "timezone": "browser", "title": "ClickHouse in Kubernetes Queries", "uid": "clickhouse-queries", - "version": 20230911 -} + "version": 20230912 +} \ No newline at end of file From 82f5f1ac2334fc72cf8305306310d3b384c30f1b Mon Sep 17 00:00:00 2001 From: Slach Date: Tue, 12 Sep 2023 20:25:03 +0400 Subject: [PATCH 4/5] rename query_type to query_kind Signed-off-by: Slach --- .../ClickHouse_Queries_dashboard.json | 24 +++++++++---------- 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/grafana-dashboard/ClickHouse_Queries_dashboard.json b/grafana-dashboard/ClickHouse_Queries_dashboard.json index b10d86a7a..bcfecf4ed 100644 --- a/grafana-dashboard/ClickHouse_Queries_dashboard.json +++ b/grafana-dashboard/ClickHouse_Queries_dashboard.json @@ -145,7 +145,7 @@ "expr": "", "extrapolate": false, "format": "time_series", - "formattedQuery": "SELECT * FROM ($rateColumns(\n substring(query, 1, 45) AS query,\n count() c)\nFROM cluster('all-sharded',system.query_log)\nWHERE\n cityHash64(query) global in (\n SELECT cityHash64(substring(query, 1, 45)) AS h\n FROM cluster('all-sharded',system.query_log)\n WHERE\n $timeFilter\n AND type in ($type)\n AND initial_user in ($user)\n AND('$query_type' = 'all' or(positionCaseInsensitive(query, '$query_type') = 1))\n GROUP BY h\n ORDER BY count() desc\n LIMIT $top)\n AND type in ($type)\n AND initial_user in ($user)\n AND('$query_type' = 'all' or(positionCaseInsensitive(query, '$query_type') = 1))) SETTINGS skip_unavailable_shards=1", + "formattedQuery": "SELECT * FROM ($rateColumns(\n substring(query, 1, 45) AS query,\n count() c)\nFROM cluster('all-sharded',system.query_log)\nWHERE\n cityHash64(query) global in (\n SELECT cityHash64(substring(query, 1, 45)) AS h\n FROM cluster('all-sharded',system.query_log)\n WHERE\n $timeFilter\n AND type in ($type)\n AND initial_user in ($user)\n AND('$query_kind' = 'all' or(positionCaseInsensitive(query, '$query_kind') = 1))\n GROUP BY h\n ORDER BY count() desc\n LIMIT $top)\n AND type in ($type)\n AND initial_user in ($user)\n AND('$query_kind' = 'all' or(positionCaseInsensitive(query, '$query_kind') = 1))) SETTINGS skip_unavailable_shards=1", "interval": "", "intervalFactor": 2, "query": "SELECT\r\n t,\r\n arrayMap(a -> (a.1, a.2 / runningDifference(t / 1000)), groupArr)\r\nFROM (\r\n SELECT t, groupArray((q, c)) AS groupArr\r\n FROM (\r\n SELECT\r\n (intDiv(toUInt32(event_time), 2) * 2) * 1000 AS t,\r\n normalizeQuery(query) AS q,\r\n count() c\r\n FROM cluster('all-sharded',system.query_log)\r\n WHERE $timeFilter\r\n AND( ('$type' = '1,2,3,4' AND type != 'QueryStart') OR ('$type' != '1,2,3,4' AND type IN ($type)))\r\n $conditionalTest(AND query_kind IN ($query_kind), $query_kind)\r\n $conditionalTest(AND initial_user IN ($user), $user)\r\n $conditionalTest(AND query_duration_ms >= $min_duration_ms, $min_duration_ms)\r\n $conditionalTest(AND query_duration_ms <= $max_duration_ms, $max_duration_ms)\r\n AND normalized_query_hash GLOBAL IN (\r\n SELECT normalized_query_hash AS h\r\n FROM cluster('all-sharded',system.query_log)\r\n WHERE $timeFilter\r\n AND( ('$type' = '1,2,3,4' AND type != 'QueryStart') OR ('$type' != '1,2,3,4' AND type IN ($type)))\r\n $conditionalTest(AND query_kind IN ($query_kind), $query_kind)\r\n $conditionalTest(AND type IN ($type), $type)\r\n $conditionalTest(AND initial_user IN ($user), $user)\r\n $conditionalTest(AND query_duration_ms >= $min_duration_ms, $min_duration_ms)\r\n $conditionalTest(AND query_duration_ms <= $max_duration_ms, $max_duration_ms)\r\n GROUP BY h\r\n ORDER BY count() DESC\r\n LIMIT $top\r\n SETTINGS skip_unavailable_shards=1\r\n )\r\n GROUP BY t, query\r\n ORDER BY t\r\n )\r\n GROUP BY t\r\n ORDER BY t\r\n) SETTINGS skip_unavailable_shards=1", @@ -162,7 +162,7 @@ "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Top $top request's rate by type: $type; user: $user; query type: $query_type", + "title": "Top $top request's rate by type: $type; user: $user; query kind: $query_kind", "tooltip": { "shared": true, "sort": 2, @@ -339,7 +339,7 @@ "expr": "", "extrapolate": false, "format": "table", - "formattedQuery": "SELECT
    rand() as t,
    substring(query,  1,  70) AS query,
    avg(query_duration_ms) duration,
    count() count
FROM cluster('all-sharded',system.query_log)
WHERE
    $timeFilter
    AND type in ($type)
    and initial_user in ($user)
    and('$query_type' = 'all' or(positionCaseInsensitive(query,  '$query_type') = 1))
GROUP BY query
ORDER BY duration desc
LIMIT $top", + "formattedQuery": "SELECT
    rand() as t,
    substring(query,  1,  70) AS query,
    avg(query_duration_ms) duration,
    count() count
FROM cluster('all-sharded',system.query_log)
WHERE
    $timeFilter
    AND type in ($type)
    and initial_user in ($user)
    and('$query_kind' = 'all' or(positionCaseInsensitive(query,  '$query_kind') = 1))
GROUP BY query
ORDER BY duration desc
LIMIT $top", "interval": "", "intervalFactor": 2, "query": "SELECT\n normalizeQuery(query) AS query,\n avg(query_duration_ms) duration_ms,\n count() count\nFROM cluster('all-sharded',system.query_log)\nWHERE\n $timeFilter\n AND( ('$type' = '1,2,3,4' AND type != 'QueryStart') OR ('$type' != '1,2,3,4' AND type IN ($type)))\n $conditionalTest(AND query_kind IN ($query_kind), $query_kind)\n $conditionalTest(AND initial_user IN ($user), $user)\n $conditionalTest(AND query_duration_ms >= $min_duration_ms,$min_duration_ms)\n $conditionalTest(AND query_duration_ms <= $max_duration_ms,$max_duration_ms)\nGROUP BY query\nORDER BY duration_ms DESC\nLIMIT $top", @@ -352,7 +352,7 @@ "tableLoading": false } ], - "title": "Top slow queries by type: $type; user: $user; query type: $query_type", + "title": "Top slow queries by type: $type; user: $user; query kind: $query_kind", "transform": "table", "type": "table" }, @@ -499,7 +499,7 @@ "expr": "", "extrapolate": false, "format": "table", - "formattedQuery": "SELECT
    rand() as t,
    substring(query,  1,  70) AS query,
    avg(memory_usage) usage,
    count() count
FROM cluster('all-sharded',system.query_log)
WHERE
    $timeFilter
    AND type in ($type)
    and initial_user in ($user)
    and('$query_type' = 'all' or(positionCaseInsensitive(query,  '$query_type') = 1))
GROUP BY query
ORDER BY usage desc
LIMIT $top", + "formattedQuery": "SELECT
    rand() as t,
    substring(query,  1,  70) AS query,
    avg(memory_usage) usage,
    count() count
FROM cluster('all-sharded',system.query_log)
WHERE
    $timeFilter
    AND type in ($type)
    and initial_user in ($user)
    and('$query_kind' = 'all' or(positionCaseInsensitive(query,  '$query_kind') = 1))
GROUP BY query
ORDER BY usage desc
LIMIT $top", "intervalFactor": 2, "query": "SELECT\n normalizeQuery(query) AS query,\n avg(memory_usage) usage,\n count() count\nFROM cluster('all-sharded',system.query_log)\nWHERE $timeFilter\n AND( ('$type' = '1,2,3,4' AND type != 'QueryStart') OR ('$type' != '1,2,3,4' AND type IN ($type)))\n $conditionalTest(AND query_kind IN ($query_kind), $query_kind)\n $conditionalTest(AND initial_user IN ($user), $user)\n $conditionalTest(AND query_duration_ms >= $min_duration_ms, $min_duration_ms)\n $conditionalTest(AND query_duration_ms <= $max_duration_ms, $max_duration_ms)\nGROUP BY query\nORDER BY usage DESC\nLIMIT $top", "rawQuery": "SELECT\n normalizeQuery(query) AS query,\n avg(memory_usage) usage,\n count() count\nFROM cluster('all-sharded',system.query_log)\nWHERE event_date >= toDate(1694531173) AND event_date <= toDate(1694534773) AND event_time >= toDateTime(1694531173) AND event_time <= toDateTime(1694534773)\n AND( ('1,2,3,4' = '1,2,3,4' AND type != 'QueryStart') OR ('1,2,3,4' != '1,2,3,4' AND type IN (1,2,3,4)))\n \n \n \n \nGROUP BY query\nORDER BY usage DESC\nLIMIT 30", @@ -511,7 +511,7 @@ "tableLoading": false } ], - "title": "Top memory consumers by type: $type; user: $user; query type: $query_type", + "title": "Top memory consumers by type: $type; user: $user; query kind: $query_kind", "transform": "table", "type": "table" }, @@ -654,7 +654,7 @@ "expr": "", "extrapolate": false, "format": "table", - "formattedQuery": "SELECT
    rand() as t,
    substring(query,  1,  70) AS query,
    type,
    count() count
FROM cluster('all-sharded',system.query_log)
WHERE
    $timeFilter
    AND type in (3,4)
    and initial_user in ($user)
    and('$query_type' = 'all' or(positionCaseInsensitive(query,  '$query_type') = 1))
GROUP BY
    query,
    type
ORDER BY count desc
LIMIT $top", + "formattedQuery": "SELECT
    rand() as t,
    substring(query,  1,  70) AS query,
    type,
    count() count
FROM cluster('all-sharded',system.query_log)
WHERE
    $timeFilter
    AND type in (3,4)
    and initial_user in ($user)
    and('$query_kind' = 'all' or(positionCaseInsensitive(query,  '$query_kind') = 1))
GROUP BY
    query,
    type
ORDER BY count desc
LIMIT $top", "intervalFactor": 2, "query": "SELECT\n normalizeQuery(query) AS query,\n type,\n count() count\nFROM cluster('all-sharded',system.query_log)\nWHERE $timeFilter\n AND( ('$type' = '1,2,3,4' AND type != 'QueryStart') OR ('$type' != '1,2,3,4' AND type IN ($type)))\n AND type NOT IN ('QueryStart','QueryFinish')\n $conditionalTest(AND query_kind IN ($query_kind), $query_kind)\n $conditionalTest(AND initial_user IN ($user), $user)\n $conditionalTest(AND query_duration_ms >= $min_duration_ms, $min_duration_ms)\n $conditionalTest(AND query_duration_ms <= $max_duration_ms, $max_duration_ms)\nGROUP BY\n query,\n type\nORDER BY count DESC\nLIMIT $top\nSETTINGS skip_unavailable_shards=1", "rawQuery": "SELECT\n normalizeQuery(query) AS query,\n type,\n count() count\nFROM cluster('all-sharded',system.query_log)\nWHERE event_date >= toDate(1694531196) AND event_date <= toDate(1694534796) AND event_time >= toDateTime(1694531196) AND event_time <= toDateTime(1694534796)\n AND( ('1,2,3,4' = '1,2,3,4' AND type != 'QueryStart') OR ('1,2,3,4' != '1,2,3,4' AND type IN (1,2,3,4)))\n AND type NOT IN ('QueryStart','QueryFinish')\n \n \n \n \nGROUP BY\n query,\n type\nORDER BY count DESC\nLIMIT 30\nSETTINGS skip_unavailable_shards=1", @@ -666,7 +666,7 @@ "tableLoading": false } ], - "title": "Top failed queries by user: $user; query type: $query_type", + "title": "Top failed queries by user: $user; query kind: $query_kind", "transform": "table", "type": "table" }, @@ -743,7 +743,7 @@ "expr": "", "extrapolate": false, "format": "time_series", - "formattedQuery": "$rate(count() c)
FROM cluster('all-sharded',system.query_log)
where  type in ($type)
    and initial_user in ($user)
    and('$query_type' = 'all' or(positionCaseInsensitive(query,  '$query_type') = 1))", + "formattedQuery": "$rate(count() c)
FROM cluster('all-sharded',system.query_log)
where  type in ($type)
    and initial_user in ($user)
    and('$query_kind' = 'all' or(positionCaseInsensitive(query,  '$query_kind') = 1))", "interval": "", "intervalFactor": 2, "query": "$rate(count() c)\nFROM cluster('all-sharded',system.query_log)\nWHERE $timeFilter\n AND( ('$type' = '1,2,3,4' AND type != 'QueryStart') OR ('$type' != '1,2,3,4' AND type IN ($type)))\n $conditionalTest(AND query_kind IN ($query_kind), $query_kind)\n $conditionalTest(AND initial_user IN ($user), $user)\n $conditionalTest(AND query_duration_ms >= $min_duration_ms,$min_duration_ms)\n $conditionalTest(AND query_duration_ms <= $max_duration_ms,$max_duration_ms)\n", @@ -760,7 +760,7 @@ "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Reqs/s type: $type; user: $user; query type: $query_type", + "title": "Reqs/s type: $type; user: $user; query kind: $query_kind", "tooltip": { "shared": true, "sort": 0, @@ -1150,7 +1150,7 @@ "datetimeLoading": false, "extrapolate": false, "format": "time_series", - "formattedQuery": "SELECT
    event_time,
    user,
    query_duration_ms duration,
    memory_usage memory,
    if(exception!='', 'fail', 'success') result,
    concat(substring(query,1,120), '...') query
FROM cluster('all-sharded',system.query_log)
WHERE $timeFilter
    AND type in ($type)
    and initial_user in ($user)
    and('$query_type' = 'all' or(positionCaseInsensitive(query,  '$query_type') = 1))
ORDER BY event_time DESC
LIMIT 1000", + "formattedQuery": "SELECT
    event_time,
    user,
    query_duration_ms duration,
    memory_usage memory,
    if(exception!='', 'fail', 'success') result,
    concat(substring(query,1,120), '...') query
FROM cluster('all-sharded',system.query_log)
WHERE $timeFilter
    AND type in ($type)
    and initial_user in ($user)
    and('$query_kind' = 'all' or(positionCaseInsensitive(query,  '$query_kind') = 1))
ORDER BY event_time DESC
LIMIT 1000", "interval": "", "intervalFactor": 1, "query": "SELECT\n event_time,\n hostName() host,\n user,\n query_duration_ms duration,\n memory_usage memory,\n if(exception!='', 'fail', 'success') result,\n concat(substring(normalizeQuery(query),1,120), '...') query\nFROM cluster('all-sharded',system.query_log)\nWHERE $timeFilter\n AND type in ($type)\n $conditionalTest(AND query_kind IN ($query_kind), $query_kind)\n $conditionalTest(AND initial_user IN ($user), $user)\n $conditionalTest(AND query_duration_ms >= $min_duration_ms,$min_duration_ms)\n $conditionalTest(AND query_duration_ms <= $max_duration_ms,$max_duration_ms)\nORDER BY event_time DESC\nLIMIT 1000", @@ -1163,7 +1163,7 @@ "tableLoading": false } ], - "title": "Query log by type: $type; user: $user; query type: $query_type", + "title": "Query log by type: $type; user: $user; query kind: $query_kind", "transform": "timeseries_to_columns", "transformations": [ { From 119e5ffb371b39d1d7cdb86b4bd99a6debfe2ec0 Mon Sep 17 00:00:00 2001 From: Slach Date: Tue, 12 Sep 2023 20:45:15 +0400 Subject: [PATCH 5/5] remove query truncate Signed-off-by: Slach --- grafana-dashboard/ClickHouse_Queries_dashboard.json | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/grafana-dashboard/ClickHouse_Queries_dashboard.json b/grafana-dashboard/ClickHouse_Queries_dashboard.json index bcfecf4ed..ef7fc6158 100644 --- a/grafana-dashboard/ClickHouse_Queries_dashboard.json +++ b/grafana-dashboard/ClickHouse_Queries_dashboard.json @@ -145,7 +145,6 @@ "expr": "", "extrapolate": false, "format": "time_series", - "formattedQuery": "SELECT * FROM ($rateColumns(\n substring(query, 1, 45) AS query,\n count() c)\nFROM cluster('all-sharded',system.query_log)\nWHERE\n cityHash64(query) global in (\n SELECT cityHash64(substring(query, 1, 45)) AS h\n FROM cluster('all-sharded',system.query_log)\n WHERE\n $timeFilter\n AND type in ($type)\n AND initial_user in ($user)\n AND('$query_kind' = 'all' or(positionCaseInsensitive(query, '$query_kind') = 1))\n GROUP BY h\n ORDER BY count() desc\n LIMIT $top)\n AND type in ($type)\n AND initial_user in ($user)\n AND('$query_kind' = 'all' or(positionCaseInsensitive(query, '$query_kind') = 1))) SETTINGS skip_unavailable_shards=1", "interval": "", "intervalFactor": 2, "query": "SELECT\r\n t,\r\n arrayMap(a -> (a.1, a.2 / runningDifference(t / 1000)), groupArr)\r\nFROM (\r\n SELECT t, groupArray((q, c)) AS groupArr\r\n FROM (\r\n SELECT\r\n (intDiv(toUInt32(event_time), 2) * 2) * 1000 AS t,\r\n normalizeQuery(query) AS q,\r\n count() c\r\n FROM cluster('all-sharded',system.query_log)\r\n WHERE $timeFilter\r\n AND( ('$type' = '1,2,3,4' AND type != 'QueryStart') OR ('$type' != '1,2,3,4' AND type IN ($type)))\r\n $conditionalTest(AND query_kind IN ($query_kind), $query_kind)\r\n $conditionalTest(AND initial_user IN ($user), $user)\r\n $conditionalTest(AND query_duration_ms >= $min_duration_ms, $min_duration_ms)\r\n $conditionalTest(AND query_duration_ms <= $max_duration_ms, $max_duration_ms)\r\n AND normalized_query_hash GLOBAL IN (\r\n SELECT normalized_query_hash AS h\r\n FROM cluster('all-sharded',system.query_log)\r\n WHERE $timeFilter\r\n AND( ('$type' = '1,2,3,4' AND type != 'QueryStart') OR ('$type' != '1,2,3,4' AND type IN ($type)))\r\n $conditionalTest(AND query_kind IN ($query_kind), $query_kind)\r\n $conditionalTest(AND type IN ($type), $type)\r\n $conditionalTest(AND initial_user IN ($user), $user)\r\n $conditionalTest(AND query_duration_ms >= $min_duration_ms, $min_duration_ms)\r\n $conditionalTest(AND query_duration_ms <= $max_duration_ms, $max_duration_ms)\r\n GROUP BY h\r\n ORDER BY count() DESC\r\n LIMIT $top\r\n SETTINGS skip_unavailable_shards=1\r\n )\r\n GROUP BY t, query\r\n ORDER BY t\r\n )\r\n GROUP BY t\r\n ORDER BY t\r\n) SETTINGS skip_unavailable_shards=1", @@ -339,7 +338,6 @@ "expr": "", "extrapolate": false, "format": "table", - "formattedQuery": "SELECT
    rand() as t,
    substring(query,  1,  70) AS query,
    avg(query_duration_ms) duration,
    count() count
FROM cluster('all-sharded',system.query_log)
WHERE
    $timeFilter
    AND type in ($type)
    and initial_user in ($user)
    and('$query_kind' = 'all' or(positionCaseInsensitive(query,  '$query_kind') = 1))
GROUP BY query
ORDER BY duration desc
LIMIT $top", "interval": "", "intervalFactor": 2, "query": "SELECT\n normalizeQuery(query) AS query,\n avg(query_duration_ms) duration_ms,\n count() count\nFROM cluster('all-sharded',system.query_log)\nWHERE\n $timeFilter\n AND( ('$type' = '1,2,3,4' AND type != 'QueryStart') OR ('$type' != '1,2,3,4' AND type IN ($type)))\n $conditionalTest(AND query_kind IN ($query_kind), $query_kind)\n $conditionalTest(AND initial_user IN ($user), $user)\n $conditionalTest(AND query_duration_ms >= $min_duration_ms,$min_duration_ms)\n $conditionalTest(AND query_duration_ms <= $max_duration_ms,$max_duration_ms)\nGROUP BY query\nORDER BY duration_ms DESC\nLIMIT $top", @@ -499,7 +497,6 @@ "expr": "", "extrapolate": false, "format": "table", - "formattedQuery": "SELECT
    rand() as t,
    substring(query,  1,  70) AS query,
    avg(memory_usage) usage,
    count() count
FROM cluster('all-sharded',system.query_log)
WHERE
    $timeFilter
    AND type in ($type)
    and initial_user in ($user)
    and('$query_kind' = 'all' or(positionCaseInsensitive(query,  '$query_kind') = 1))
GROUP BY query
ORDER BY usage desc
LIMIT $top", "intervalFactor": 2, "query": "SELECT\n normalizeQuery(query) AS query,\n avg(memory_usage) usage,\n count() count\nFROM cluster('all-sharded',system.query_log)\nWHERE $timeFilter\n AND( ('$type' = '1,2,3,4' AND type != 'QueryStart') OR ('$type' != '1,2,3,4' AND type IN ($type)))\n $conditionalTest(AND query_kind IN ($query_kind), $query_kind)\n $conditionalTest(AND initial_user IN ($user), $user)\n $conditionalTest(AND query_duration_ms >= $min_duration_ms, $min_duration_ms)\n $conditionalTest(AND query_duration_ms <= $max_duration_ms, $max_duration_ms)\nGROUP BY query\nORDER BY usage DESC\nLIMIT $top", "rawQuery": "SELECT\n normalizeQuery(query) AS query,\n avg(memory_usage) usage,\n count() count\nFROM cluster('all-sharded',system.query_log)\nWHERE event_date >= toDate(1694531173) AND event_date <= toDate(1694534773) AND event_time >= toDateTime(1694531173) AND event_time <= toDateTime(1694534773)\n AND( ('1,2,3,4' = '1,2,3,4' AND type != 'QueryStart') OR ('1,2,3,4' != '1,2,3,4' AND type IN (1,2,3,4)))\n \n \n \n \nGROUP BY query\nORDER BY usage DESC\nLIMIT 30", @@ -654,7 +651,6 @@ "expr": "", "extrapolate": false, "format": "table", - "formattedQuery": "SELECT
    rand() as t,
    substring(query,  1,  70) AS query,
    type,
    count() count
FROM cluster('all-sharded',system.query_log)
WHERE
    $timeFilter
    AND type in (3,4)
    and initial_user in ($user)
    and('$query_kind' = 'all' or(positionCaseInsensitive(query,  '$query_kind') = 1))
GROUP BY
    query,
    type
ORDER BY count desc
LIMIT $top", "intervalFactor": 2, "query": "SELECT\n normalizeQuery(query) AS query,\n type,\n count() count\nFROM cluster('all-sharded',system.query_log)\nWHERE $timeFilter\n AND( ('$type' = '1,2,3,4' AND type != 'QueryStart') OR ('$type' != '1,2,3,4' AND type IN ($type)))\n AND type NOT IN ('QueryStart','QueryFinish')\n $conditionalTest(AND query_kind IN ($query_kind), $query_kind)\n $conditionalTest(AND initial_user IN ($user), $user)\n $conditionalTest(AND query_duration_ms >= $min_duration_ms, $min_duration_ms)\n $conditionalTest(AND query_duration_ms <= $max_duration_ms, $max_duration_ms)\nGROUP BY\n query,\n type\nORDER BY count DESC\nLIMIT $top\nSETTINGS skip_unavailable_shards=1", "rawQuery": "SELECT\n normalizeQuery(query) AS query,\n type,\n count() count\nFROM cluster('all-sharded',system.query_log)\nWHERE event_date >= toDate(1694531196) AND event_date <= toDate(1694534796) AND event_time >= toDateTime(1694531196) AND event_time <= toDateTime(1694534796)\n AND( ('1,2,3,4' = '1,2,3,4' AND type != 'QueryStart') OR ('1,2,3,4' != '1,2,3,4' AND type IN (1,2,3,4)))\n AND type NOT IN ('QueryStart','QueryFinish')\n \n \n \n \nGROUP BY\n query,\n type\nORDER BY count DESC\nLIMIT 30\nSETTINGS skip_unavailable_shards=1", @@ -1150,11 +1146,10 @@ "datetimeLoading": false, "extrapolate": false, "format": "time_series", - "formattedQuery": "SELECT
    event_time,
    user,
    query_duration_ms duration,
    memory_usage memory,
    if(exception!='', 'fail', 'success') result,
    concat(substring(query,1,120), '...') query
FROM cluster('all-sharded',system.query_log)
WHERE $timeFilter
    AND type in ($type)
    and initial_user in ($user)
    and('$query_kind' = 'all' or(positionCaseInsensitive(query,  '$query_kind') = 1))
ORDER BY event_time DESC
LIMIT 1000", "interval": "", "intervalFactor": 1, - "query": "SELECT\n event_time,\n hostName() host,\n user,\n query_duration_ms duration,\n memory_usage memory,\n if(exception!='', 'fail', 'success') result,\n concat(substring(normalizeQuery(query),1,120), '...') query\nFROM cluster('all-sharded',system.query_log)\nWHERE $timeFilter\n AND type in ($type)\n $conditionalTest(AND query_kind IN ($query_kind), $query_kind)\n $conditionalTest(AND initial_user IN ($user), $user)\n $conditionalTest(AND query_duration_ms >= $min_duration_ms,$min_duration_ms)\n $conditionalTest(AND query_duration_ms <= $max_duration_ms,$max_duration_ms)\nORDER BY event_time DESC\nLIMIT 1000", - "rawQuery": "SELECT\n event_time,\n hostName() host,\n user,\n query_duration_ms duration,\n memory_usage memory,\n if(exception!='', 'fail', 'success') result,\n concat(substring(normalizeQuery(query),1,120), '...') query\nFROM cluster('all-sharded',system.query_log)\nWHERE event_date >= toDate(1694531300) AND event_date <= toDate(1694534900) AND event_time >= toDateTime(1694531300) AND event_time <= toDateTime(1694534900)\n AND type in (1,2,3,4)\n \n \n \n \nORDER BY event_time DESC\nLIMIT 1000", + "query": "SELECT\n event_time,\n hostName() host,\n user,\n query_duration_ms duration,\n memory_usage memory,\n if(exception!='', 'fail', 'success') result,\n normalizeQuery(query) query\nFROM cluster('all-sharded',system.query_log)\nWHERE $timeFilter\n AND type in ($type)\n $conditionalTest(AND query_kind IN ($query_kind), $query_kind)\n $conditionalTest(AND initial_user IN ($user), $user)\n $conditionalTest(AND query_duration_ms >= $min_duration_ms,$min_duration_ms)\n $conditionalTest(AND query_duration_ms <= $max_duration_ms,$max_duration_ms)\nORDER BY event_time DESC\nLIMIT 1000", + "rawQuery": "SELECT\n event_time,\n hostName() host,\n user,\n query_duration_ms duration,\n memory_usage memory,\n if(exception!='', 'fail', 'success') result,\n normalizeQuery(query) query\nFROM cluster('all-sharded',system.query_log)\nWHERE event_date >= toDate(1694531300) AND event_date <= toDate(1694534900) AND event_time >= toDateTime(1694531300) AND event_time <= toDateTime(1694534900)\n AND type in (1,2,3,4)\n \n \n \n \nORDER BY event_time DESC\nLIMIT 1000", "refId": "A", "resultFormat": "time_series", "round": "0s",