From 7313833a3dcf19041b8ff3e43dae16e1b298ff18 Mon Sep 17 00:00:00 2001 From: Tsvetan Stoychev Date: Sat, 27 Jan 2024 19:39:52 +0100 Subject: [PATCH] We fill with 0s the buckets that have missing values for the histograms. This helps with displaying the data in a better way. We also added a missing setup clickhouse script that adds a table with hostnames. We also changed the hostnames variable to use values from querying the hostnames table. --- build.test.js | 4 +- docker-compose.yaml | 5 ++- fill-test-data.js | 4 +- setup/ch/init-schema/3.sql | 8 ++++ .../ae3e-plotly-panel/default/layout.json | 7 +++- templates/ae3e-plotly-panel/default/query.sql | 4 +- templates/ae3e-plotly-panel/default/script.js | 13 +++++- templates/ae3e-plotly-panel/fcp/query.sql | 3 +- templates/ae3e-plotly-panel/fid/query.sql | 3 +- templates/ae3e-plotly-panel/lcp/query.sql | 3 +- templates/ae3e-plotly-panel/metric/query.sql | 1 + .../ae3e-plotly-panel/screen-width/query.sql | 1 + templates/dashboards/General.yaml | 2 +- templates/filters/hosts.json | 41 ++++++------------- templates/filters/testHosts.json | 39 ------------------ .../default/layout.json | 9 +++- .../default/query.sql | 3 +- .../default/script.js | 13 +++++- .../landing-ae3e-plotly-panel/fcp/query.sql | 3 +- .../landing-ae3e-plotly-panel/fid/query.sql | 3 +- .../landing-ae3e-plotly-panel/lcp/query.sql | 3 +- .../ttfb/layout.merge.json | 10 ++--- .../landing-ae3e-plotly-panel/ttfb/query.sql | 3 +- 23 files changed, 88 insertions(+), 97 deletions(-) create mode 100644 setup/ch/init-schema/3.sql delete mode 100644 templates/filters/testHosts.json diff --git a/build.test.js b/build.test.js index 3892c1d..be528f2 100644 --- a/build.test.js +++ b/build.test.js @@ -6,9 +6,7 @@ function getDashboard(dashboard) { const options = { table: "basicrum_friends_webperf_rum_events", datasourceUid: "A0Wl5Mc4z", - filterMap: { - "hosts":"testHosts", - }, + filterMap: {}, } builder.build(dashboard, options); diff --git a/docker-compose.yaml b/docker-compose.yaml index 1fbf5af..5d00515 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -12,13 +12,16 @@ services: - ./build/dashboards:/etc/grafana/provisioning/dashboards basicrum_clickhouse_server_build: - image: clickhouse/clickhouse-server:22.8 + image: clickhouse/clickhouse-server:23.6.2.18-alpine env_file: - .env volumes: - ./clickhouse/data:/var/lib/clickhouse - ./setup/ch/users.d:/etc/clickhouse-server/users.d - ./setup/ch/init-schema:/docker-entrypoint-initdb.d + ports: + - 8153:8123 + - 9005:9000 ulimits: nproc: 65535 diff --git a/fill-test-data.js b/fill-test-data.js index 3760995..6dd6f51 100644 --- a/fill-test-data.js +++ b/fill-test-data.js @@ -47,7 +47,7 @@ const params = { "nt_first_paint": "1700515101401", "nt_red_cnt": "1", "nt_nav_type": "0", - "u": "https://calendar.perfplanet.com/", + "u": "https://www.example.com/", "v": "1.737.60", "sv": "14", "vsm": "p", @@ -114,7 +114,7 @@ formBody = formBody.join("&"); console.log(formBody); -fetch('https://ocetinssl.basicrum.com/beacon/catcher', { +fetch('http://localhost:8087/beacon/catcher', { method: 'POST', headers: { 'Content-Type': 'application/x-www-form-urlencoded;charset=UTF-8' diff --git a/setup/ch/init-schema/3.sql b/setup/ch/init-schema/3.sql new file mode 100644 index 0000000..f2980c9 --- /dev/null +++ b/setup/ch/init-schema/3.sql @@ -0,0 +1,8 @@ +CREATE TABLE IF NOT EXISTS default.webperf_rum_hostnames ( + hostname LowCardinality(String), + updated_at DateTime64(3) DEFAULT now() +) +ENGINE = ReplacingMergeTree +PARTITION BY hostname +ORDER BY hostname +SETTINGS index_granularity = 8192 \ No newline at end of file diff --git a/templates/ae3e-plotly-panel/default/layout.json b/templates/ae3e-plotly-panel/default/layout.json index 998de8f..5413812 100644 --- a/templates/ae3e-plotly-panel/default/layout.json +++ b/templates/ae3e-plotly-panel/default/layout.json @@ -15,6 +15,11 @@ "plot_bgcolor": "rgba(0, 0, 0, 0)", "xaxis": { "autorange": false, - "tickmode": "array" + "tickmode": "array", + "fixedrange": true + }, + "yaxis": { + "fixedrange": true, + "tickformat": "d" } } diff --git a/templates/ae3e-plotly-panel/default/query.sql b/templates/ae3e-plotly-panel/default/query.sql index 1f039b7..29f32b7 100644 --- a/templates/ae3e-plotly-panel/default/query.sql +++ b/templates/ae3e-plotly-panel/default/query.sql @@ -8,7 +8,9 @@ WHERE $timeFilter AND event_type = 'visit_page' AND browser_name = '$$browser_name' + AND $$column_name < 12000 $conditionalTest(AND hostname in($hostname), $hostname) GROUP BY value -ORDER BY value \ No newline at end of file +ORDER BY value +WITH FILL FROM 0 TO 12000 STEP $bucket_size \ No newline at end of file diff --git a/templates/ae3e-plotly-panel/default/script.js b/templates/ae3e-plotly-panel/default/script.js index 52d3345..1d21260 100644 --- a/templates/ae3e-plotly-panel/default/script.js +++ b/templates/ae3e-plotly-panel/default/script.js @@ -4,8 +4,17 @@ var yValue = []; var dataFound = true; try { - var xValue = data.series[0].fields[0].values.buffer; - var yValue = data.series[0].fields[1].values.buffer; + if (data.series[0].fields[0].values.buffer !== undefined + && data.series[0].fields[1].values.buffer !== undefined) + { + xValue = data.series[0].fields[0].values.buffer; + yValue = data.series[0].fields[1].values.buffer; + } + else + { + xValue = data.series[0].fields[0].values; + yValue = data.series[0].fields[1].values; + } } catch (e) { dataFound = false diff --git a/templates/ae3e-plotly-panel/fcp/query.sql b/templates/ae3e-plotly-panel/fcp/query.sql index 54f4b8b..abd690c 100644 --- a/templates/ae3e-plotly-panel/fcp/query.sql +++ b/templates/ae3e-plotly-panel/fcp/query.sql @@ -13,4 +13,5 @@ WHERE $conditionalTest(AND hostname in($hostname), $hostname) GROUP BY value -ORDER BY value \ No newline at end of file +ORDER BY value +WITH FILL FROM 0 TO 10000 STEP $bucket_size \ No newline at end of file diff --git a/templates/ae3e-plotly-panel/fid/query.sql b/templates/ae3e-plotly-panel/fid/query.sql index e7ad079..75d182b 100644 --- a/templates/ae3e-plotly-panel/fid/query.sql +++ b/templates/ae3e-plotly-panel/fid/query.sql @@ -12,4 +12,5 @@ WHERE $conditionalTest(AND hostname in($hostname), $hostname) GROUP BY value -ORDER BY value \ No newline at end of file +ORDER BY value +WITH FILL FROM 0 TO 500 STEP 20 \ No newline at end of file diff --git a/templates/ae3e-plotly-panel/lcp/query.sql b/templates/ae3e-plotly-panel/lcp/query.sql index a27a68c..dbebea9 100644 --- a/templates/ae3e-plotly-panel/lcp/query.sql +++ b/templates/ae3e-plotly-panel/lcp/query.sql @@ -13,4 +13,5 @@ WHERE $conditionalTest(AND hostname in($hostname), $hostname) GROUP BY value -ORDER BY value \ No newline at end of file +ORDER BY value +WITH FILL FROM 0 TO 10000 STEP $bucket_size \ No newline at end of file diff --git a/templates/ae3e-plotly-panel/metric/query.sql b/templates/ae3e-plotly-panel/metric/query.sql index 42116ae..c6fdef1 100644 --- a/templates/ae3e-plotly-panel/metric/query.sql +++ b/templates/ae3e-plotly-panel/metric/query.sql @@ -14,3 +14,4 @@ WHERE GROUP BY metric_value ORDER BY metric_value +WITH FILL FROM 0 TO 10000 STEP $bucket_size \ No newline at end of file diff --git a/templates/ae3e-plotly-panel/screen-width/query.sql b/templates/ae3e-plotly-panel/screen-width/query.sql index 314b1fe..3a5e9fb 100644 --- a/templates/ae3e-plotly-panel/screen-width/query.sql +++ b/templates/ae3e-plotly-panel/screen-width/query.sql @@ -12,3 +12,4 @@ WHERE GROUP BY screen_width ORDER BY screen_width +WITH FILL FROM 0 TO 2600 STEP 100 \ No newline at end of file diff --git a/templates/dashboards/General.yaml b/templates/dashboards/General.yaml index f140dcd..feb24bd 100644 --- a/templates/dashboards/General.yaml +++ b/templates/dashboards/General.yaml @@ -2,7 +2,7 @@ apiVersion: 1 providers: - name: Basic RUM # A uniquely identifiable name for the provider - folder: Custom # The folder where to place the dashboards + folder: WebPerformance # The folder where to place the dashboards type: file options: path: /etc/grafana/provisioning/dashboards \ No newline at end of file diff --git a/templates/filters/hosts.json b/templates/filters/hosts.json index 80e5370..6dcfb34 100644 --- a/templates/filters/hosts.json +++ b/templates/filters/hosts.json @@ -1,34 +1,19 @@ { - "allValue": "", - "current": { - "selected": true, - "text": ["All"], - "value": ["$__all"] + "datasource": { + "type": "vertamedia-clickhouse-datasource", + "uid": "BRUMMUCSTARBUCS" }, + "definition": "select hostname from webperf_rum_hostnames group by hostname\n", "hide": 1, - "includeAll": true, - "label": "Hostname", + "includeAll": false, + "label": "Host", "multi": true, "name": "hostname", - "options": [ - { - "selected": true, - "text": "All", - "value": "$__all" - }, - { - "selected": false, - "text": "example1.com", - "value": "example1.com" - }, - { - "selected": false, - "text": "example2.com", - "value": "example2.com" - } - ], - "query": "example1.com : example1.com, example2.com : example2.com", - "queryValue": "", + "options": [], + "query": "select hostname from webperf_rum_hostnames group by hostname\n", + "refresh": 1, + "regex": "", "skipUrlSync": false, - "type": "custom" -} \ No newline at end of file + "sort": 1, + "type": "query" + } \ No newline at end of file diff --git a/templates/filters/testHosts.json b/templates/filters/testHosts.json deleted file mode 100644 index b71c28f..0000000 --- a/templates/filters/testHosts.json +++ /dev/null @@ -1,39 +0,0 @@ -{ - "allValue": "", - "current": { - "selected": true, - "text": ["All"], - "value": ["$__all"] - }, - "hide": 1, - "includeAll": true, - "label": "Hostname", - "multi": true, - "name": "hostname", - "options": [ - { - "selected": true, - "text": "All", - "value": "$__all" - }, - { - "selected": false, - "text": "example1.com", - "value": "example1.com" - }, - { - "selected": false, - "text": "example2.com", - "value": "example2.com" - }, - { - "selected": false, - "text": "calendar.perfplanet.com", - "value": "calendar.perfplanet.com" - } - ], - "query": "example1.com : example1.com, example2.com : example2.com, calendar.perfplanet.com : calendar.perfplanet.com", - "queryValue": "", - "skipUrlSync": false, - "type": "custom" -} diff --git a/templates/landing-ae3e-plotly-panel/default/layout.json b/templates/landing-ae3e-plotly-panel/default/layout.json index 4a3dbfd..5413812 100644 --- a/templates/landing-ae3e-plotly-panel/default/layout.json +++ b/templates/landing-ae3e-plotly-panel/default/layout.json @@ -14,7 +14,12 @@ "paper_bgcolor": "rgba(0, 0, 0, 0)", "plot_bgcolor": "rgba(0, 0, 0, 0)", "xaxis": { - "autorange": true, - "tickmode": "array" + "autorange": false, + "tickmode": "array", + "fixedrange": true + }, + "yaxis": { + "fixedrange": true, + "tickformat": "d" } } diff --git a/templates/landing-ae3e-plotly-panel/default/query.sql b/templates/landing-ae3e-plotly-panel/default/query.sql index 6792916..9ffef0f 100644 --- a/templates/landing-ae3e-plotly-panel/default/query.sql +++ b/templates/landing-ae3e-plotly-panel/default/query.sql @@ -13,4 +13,5 @@ WHERE $conditionalTest(AND hostname in($hostname), $hostname) GROUP BY value -ORDER BY value \ No newline at end of file +ORDER BY value +WITH FILL FROM 0 TO 12000 STEP $bucket_size \ No newline at end of file diff --git a/templates/landing-ae3e-plotly-panel/default/script.js b/templates/landing-ae3e-plotly-panel/default/script.js index 52d3345..1d21260 100644 --- a/templates/landing-ae3e-plotly-panel/default/script.js +++ b/templates/landing-ae3e-plotly-panel/default/script.js @@ -4,8 +4,17 @@ var yValue = []; var dataFound = true; try { - var xValue = data.series[0].fields[0].values.buffer; - var yValue = data.series[0].fields[1].values.buffer; + if (data.series[0].fields[0].values.buffer !== undefined + && data.series[0].fields[1].values.buffer !== undefined) + { + xValue = data.series[0].fields[0].values.buffer; + yValue = data.series[0].fields[1].values.buffer; + } + else + { + xValue = data.series[0].fields[0].values; + yValue = data.series[0].fields[1].values; + } } catch (e) { dataFound = false diff --git a/templates/landing-ae3e-plotly-panel/fcp/query.sql b/templates/landing-ae3e-plotly-panel/fcp/query.sql index 5e77ad5..38cf89c 100644 --- a/templates/landing-ae3e-plotly-panel/fcp/query.sql +++ b/templates/landing-ae3e-plotly-panel/fcp/query.sql @@ -14,4 +14,5 @@ WHERE $conditionalTest(AND hostname in($hostname), $hostname) GROUP BY value -ORDER BY value \ No newline at end of file +ORDER BY value +WITH FILL FROM 0 TO 10000 STEP $bucket_size \ No newline at end of file diff --git a/templates/landing-ae3e-plotly-panel/fid/query.sql b/templates/landing-ae3e-plotly-panel/fid/query.sql index 782ae7b..6eb93cf 100644 --- a/templates/landing-ae3e-plotly-panel/fid/query.sql +++ b/templates/landing-ae3e-plotly-panel/fid/query.sql @@ -13,4 +13,5 @@ WHERE $conditionalTest(AND hostname in($hostname), $hostname) GROUP BY value -ORDER BY value \ No newline at end of file +ORDER BY value +WITH FILL FROM 0 TO 500 STEP 20 \ No newline at end of file diff --git a/templates/landing-ae3e-plotly-panel/lcp/query.sql b/templates/landing-ae3e-plotly-panel/lcp/query.sql index d6a5ddb..0ee30c0 100644 --- a/templates/landing-ae3e-plotly-panel/lcp/query.sql +++ b/templates/landing-ae3e-plotly-panel/lcp/query.sql @@ -14,4 +14,5 @@ WHERE $conditionalTest(AND hostname in($hostname), $hostname) GROUP BY value -ORDER BY value \ No newline at end of file +ORDER BY value +WITH FILL FROM 0 TO 10000 STEP $bucket_size \ No newline at end of file diff --git a/templates/landing-ae3e-plotly-panel/ttfb/layout.merge.json b/templates/landing-ae3e-plotly-panel/ttfb/layout.merge.json index 11a831d..2b56d97 100644 --- a/templates/landing-ae3e-plotly-panel/ttfb/layout.merge.json +++ b/templates/landing-ae3e-plotly-panel/ttfb/layout.merge.json @@ -1,6 +1,6 @@ { "xaxis": { - "range": [-1, 6], + "range": [-100, 8100], "ticktext": [ "1 sec", "2 sec", @@ -9,12 +9,8 @@ "5 sec", "6 sec", "7 sec", - "8 sec", - "9 sec", - "10 sec", - "11 sec", - "12 sec" + "8 sec" ], - "tickvals": [1000, 2000, 3000, 4000, 5000, 6000, 7000, 8000, 9000, 10000, 11000, 12000] + "tickvals": [1000, 2000, 3000, 4000, 5000, 6000, 7000, 8000] } } diff --git a/templates/landing-ae3e-plotly-panel/ttfb/query.sql b/templates/landing-ae3e-plotly-panel/ttfb/query.sql index 6792916..9ffef0f 100644 --- a/templates/landing-ae3e-plotly-panel/ttfb/query.sql +++ b/templates/landing-ae3e-plotly-panel/ttfb/query.sql @@ -13,4 +13,5 @@ WHERE $conditionalTest(AND hostname in($hostname), $hostname) GROUP BY value -ORDER BY value \ No newline at end of file +ORDER BY value +WITH FILL FROM 0 TO 12000 STEP $bucket_size \ No newline at end of file