Simplified local setup

Dockerfile

@@ -38,19 +38,16 @@ RUN find /usr/share/grafana/public/build/ -name *.js -exec sed -i 's|..createEle
 
 RUN /bin/bash -c 'grafana cli plugins install ae3e-plotly-panel 0.5.0'
 RUN /bin/bash -c 'grafana cli plugins install vertamedia-clickhouse-datasource 2.5.1'
-RUN /bin/bash -c 'grafana cli plugins install volkovlabs-form-panel 3.2.1'
 
 ## Set Home Dashboard
 ENV GF_DASHBOARDS_DEFAULT_HOME_DASHBOARD_PATH=/etc/grafana/provisioning/dashboards/General.json
 
 # Adding datasources
-ADD templates/datasources /etc/grafana/provisioning/datasources
+ADD templates/datasources/default.yaml /etc/grafana/provisioning/datasources/default.yaml
 
 # Adding dashboards yaml
-ADD templates/dashboards /etc/grafana/provisioning/dashboards
+ADD templates/dashboards/General.yaml /etc/grafana/provisioning/dashboards/General.yaml
 
 # Adding dashboards
 ADD build/dashboards /etc/grafana/provisioning/dashboards
 
-# Replace grafana.ini
-ADD grafana/grafana.ini /etc/grafana/grafana.ini

Makefile

@@ -5,9 +5,8 @@ SHELL=bash
 UID := $(shell id -u)
 
 dc_path=./docker-compose.yaml
-dc_path_quick_test=./docker-compose.quick-test.yaml
 clickhouse_container=basicrum_clickhouse_server_build
-grafana_container=basicrum_grafana
+grafana_container=basicrum_grafana_build
 
 help:
 	@grep -E '^[a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | sort | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}'
@@ -19,14 +18,6 @@ up: ## Starts the environment
 down: ## Stops the environment
 	env UID=${UID}  docker-compose -f ${dc_path} down
 
-
-up_quick_test: ## Starts quick test environment
-	env UID=${UID} docker-compose -f ${dc_path_quick_test} build
-	env UID=${UID} docker-compose -f ${dc_path_quick_test} up
-
-down_quick_test: ## Stops quick test  environment
-	env UID=${UID}  docker-compose -f ${dc_path_quick_test} down
-
 restart: down up # Restart the environment
 
 rebuild: ## Rebuilds the environment from scratch

build.js

@@ -1,6 +1,7 @@
-var fs = require('fs');
+var fs = require("fs");
 
-const DashboardBuilder = require('./lib/DashboardBuilder');
+const DashboardBuilder = require("./lib/DashboardBuilder");
+const { exit } = require("process");
 const builder = new DashboardBuilder()
 
 const options = {
@@ -9,17 +10,52 @@ const options = {
     filterMap: {},
 }
 
-var dir = './build';
-if (!fs.existsSync(dir)){
-    fs.mkdirSync(dir);
+const buildDir = "./build";
+
+try {
+    if(fs.existsSync(buildDir)) {
+        fs.rmSync(buildDir, { recursive: true, force: true });
+        console.log("Removed the build folder.");
+    }
+} catch(e) {
+    console.error(e);
+    exit(1);
+}
+
+if (!fs.existsSync(buildDir)){
+    fs.mkdirSync(buildDir);
+    console.log("Created build folder.");
+}
+
+const dashboardsDir = "./build/dashboards";
+if (!fs.existsSync(dashboardsDir)){
+    fs.mkdirSync(dashboardsDir);
+    console.log("Created build/dashboards folder.");
+}
+
+const datasourcesDir = "./build/datasources";
+if (!fs.existsSync(datasourcesDir)){
+    fs.mkdirSync(datasourcesDir);
+    console.log("Created build/datasources folder.");
+}
+
+try {
+    fs.copyFileSync("./templates/dashboards/General.yaml", "./build/dashboards/General.yaml");
+    console.log("Added build/dashboards/General.yaml");
+} catch(e) {
+    console.error(e);
+    exit(1);
 }
 
-dir = './build/dashboards';
-if (!fs.existsSync(dir)){
-    fs.mkdirSync(dir);
+try {
+    fs.copyFileSync("./templates/datasources/default.yaml", "./build/datasources/default.yaml");
+    console.log("Added build/datasources/default.yaml");
+} catch(e) {
+    console.error(e);
+    exit(1);
 }
 
-const dashboards = ['General', 'Metrics', 'Summary'];
+const dashboards = ["General", "Metrics", "Summary"];
 
 var summaryList = [];
 for (const dashboard of dashboards) {

build.test.js

@@ -1,31 +1,35 @@
-const fs = require('fs');
-const DashboardBuilder = require('./lib/DashboardBuilder');
+const fs = require("fs");
+const DashboardBuilder = require("./lib/DashboardBuilder");
 const builder = new DashboardBuilder()
 
-function runTest(dashboard) {
+function getDashboard(dashboard) {
     const options = {
         table: "basicrum_friends_webperf_rum_events",
         datasourceUid: "A0Wl5Mc4z",
         filterMap: {
             "hosts":"testHosts",
         },
     }
-    
+
     builder.build(dashboard, options);
 
-    const expected = JSON.parse(fs.readFileSync(`./testdata/dashboards/${dashboard}.json`, { encoding: 'utf8', flag: 'r' }));
-    const actual = JSON.parse(fs.readFileSync(`./build/dashboards/${dashboard}.json`, { encoding: 'utf8', flag: 'r' }));
-    expect(actual).toMatchObject(expected);
+    return JSON.parse(fs.readFileSync(`./build/dashboards/${dashboard}.json`, { encoding: "utf8", flag: "r" }));    
 }
 
-test('build General dashboard should be as expected', () => {
-    runTest('General')
+test("Test General dashboard should be as expected", () => {
+    const generalDashboard = getDashboard("General");
+
+    expect(34).toBe(generalDashboard.panels.length);
 });
 
-test('build Metrics dashboard should be as expected', () => {
-    runTest('Metrics')
+test("Test Metrics dashboard should be as expected", () => {
+    const metricsDashboard = getDashboard("Metrics");
+
+    expect(4).toBe(metricsDashboard.panels.length);
 });
 
-test('build Summary dashboard should be as expected', () => {
-    runTest('Summary')
+test("Test Summary dashboard should be as expected", () => {
+    const summaryDashboard = getDashboard("Summary");
+
+    expect(14).toBe(summaryDashboard.panels.length);
 });

docker-compose.yaml

@@ -4,20 +4,21 @@ services:
   basicrum_grafana_build:
     build: .
     ports:
-      - 3200:3000
+      - 3300:3000
     env_file:
       - .env
     volumes:
-      # Mount provisioning related stuff
       - ./build/datasources:/etc/grafana/provisioning/datasources
       - ./build/dashboards:/etc/grafana/provisioning/dashboards
 
   basicrum_clickhouse_server_build:
     image: clickhouse/clickhouse-server:22.8
+    env_file:
+      - .env
     volumes:
       - ./clickhouse/data:/var/lib/clickhouse
-      - ./clickhouse/users.d/:/etc/clickhouse-server/users.d/
-      - ./init-schema/:/docker-entrypoint-initdb.d/
+      - ./setup/ch/users.d:/etc/clickhouse-server/users.d
+      - ./setup/ch/init-schema:/docker-entrypoint-initdb.d
 
     ulimits:
       nproc: 65535

setup/ch/init-schema/2.sql

@@ -0,0 +1,4 @@
+ALTER TABLE default.webperf_rum_events
+    ADD COLUMN IF NOT EXISTS mob_dl Nullable(UInt16),
+    ADD COLUMN IF NOT EXISTS mob_rtt Nullable(UInt16),
+    ADD COLUMN IF NOT EXISTS mob_etype LowCardinality(Nullable(String))

setup/ch/users.d/default-user.xml

@@ -0,0 +1,18 @@
+    <clickhouse>
+      <!-- Docs: <https://clickhouse.com/docs/en/operations/settings/settings_users/> -->
+      <users>
+        <!-- Remove default user -->
+        <default remove="remove">
+        </default>
+
+        <basicrumreader>
+          <profile>default</profile>
+          <networks>
+            <ip>::/0</ip>
+          </networks>
+          <password>testpassword</password>
+          <quota>default</quota>
+          <access_management>0</access_management>
+        </basicrumreader>
+      </users>
+    </clickhouse>

templates/datasources/default.yaml

@@ -13,10 +13,3 @@ datasources:
       basicAuthPassword: "$CLICKHOUSE_PASSWORD"
     secureJsonFields:
       basicAuthPassword: false
-  - name: Basic RUM API Proxy
-    uid: APIUID
-    type: basicrum-api-datasource
-    jsonData:
-      url: "$BASICRUM_API_DATASOURCE_URL"
-    secureJsonData:
-      apiToken: "$BASICRUM_API_DATASOURCE_TOKEN"