Added basic tests

Signed-off-by: Aravinda Vishwanathapura <aravinda@kadalu.io>

.github/workflows/on-pr-submit.yml

@@ -0,0 +1,41 @@
+name: Run on every PR
+
+on:
+  pull_request:
+    branches:
+      - main
+    paths-ignore:
+      - 'docs/**'
+      - 'extras/**'
+      - '**.md'
+      - '**.adoc'
+
+jobs:
+  # TODO: Add unit test as well here
+  tests:
+    name: Gluster Metrics Exporter Tests
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - name: Install Crystal
+        run: |
+          curl -fsSL https://crystal-lang.org/install.sh | sudo bash
+      - name: Format Check
+        run: make fmt-check
+      - name: Lint
+        run: make lint
+      - name: Install Binnacle
+        run: |
+          curl -L https://github.com/kadalu/binnacle/releases/latest/download/binnacle -o binnacle
+          chmod +x ./binnacle
+          sudo mv ./binnacle /usr/local/bin/binnacle
+          binnacle --version
+      - name: Build Storage node container
+        run: |
+          cd tests && ./build-container.sh
+      - name: Setup Test environment
+        run: binnacle -v tests/setup.t
+      - name: Build and Install Gluster Metrics Exporter to nodes/containers
+        run: binnacle -v tests/install.t
+      - name: Run all Tests
+        run: binnacle -v tests/all

extra/gluster-metrics-exporter.service

@@ -4,7 +4,7 @@ After=network.target
 
 [Service]
 PIDFile=/var/run/gluster/metrics-exporter.pid
-ExecStart=/usr/bin/gluster-metrics-exporter
+ExecStart=/usr/sbin/gluster-metrics-exporter
 
 [Install]
 WantedBy=multi-user.target

src/config.cr

@@ -10,7 +10,7 @@ module GlusterMetricsExporter
       gluster_executable_path = "/usr/sbin/gluster",
       disable_all_metrics = false,
       log_level = "info",
-      log_dir = "/var/log/glusterfs-metrics",
+      log_dir = "/var/log/gluster-metrics-exporter",
       log_file = "exporter.log",
       gluster_log_dir = "/var/log/glusterfs",
       glusterd_dir = "/var/lib/glusterd",

src/gluster-metrics-exporter.cr

@@ -1,3 +1,5 @@
+require "log"
+
 require "crometheus"
 require "kemal"
 
@@ -6,6 +8,29 @@ require "./args"
 require "./metrics/*"
 
 module GlusterMetricsExporter
+  class ExporterAPILogHandler < Kemal::BaseLogHandler
+    def initialize
+    end
+
+    def call(context : HTTP::Server::Context)
+      elapsed_time = Time.measure { call_next(context) }
+      elapsed_text = elapsed_text(elapsed_time)
+      Log.info &.emit("#{context.request.method} #{context.request.resource}", status_code: "#{context.response.status_code}", duration: "#{elapsed_text}")
+      context
+    end
+
+    def write(message : String)
+      Log.info { message.strip }
+    end
+
+    private def elapsed_text(elapsed)
+      millis = elapsed.total_milliseconds
+      return "#{millis.round(2)}ms" if millis >= 1
+
+      "#{(millis * 1000).round(2)}µs"
+    end
+  end
+
   # A handler to be called before calling Crometheus handler
   class MetricsRunHandler < Kemal::Handler
     def call(env)
@@ -42,8 +67,14 @@ module GlusterMetricsExporter
   def self.run
     parse_args
 
+    Dir.mkdir_p @@config.log_dir
+    logfile = Path[@@config.log_dir].join(@@config.log_file)
+    # TODO: Handle Log level from CLI arg
+    Log.setup(:info, Log::IOBackend.new(File.new(logfile, "a+")))
+
     Crometheus.default_registry.path = @@config.metrics_path
     Kemal.config.port = @@config.port
+    Kemal.config.logger = ExporterAPILogHandler.new
     Kemal.run
   end
 end

tests/Dockerfile

@@ -0,0 +1,28 @@
+FROM ubuntu:20.04
+
+ENV DEBIAN_FRONTEND=noninteractive
+
+RUN apt-get update -yq &&                                                 \
+    apt-get install -y --no-install-recommends python3 libtirpc3 init     \
+    vim python3-pip ssh rsync lvm2 less emacs software-properties-common  \
+    sudo curl wget git build-essential automake autoconf automake libtool \
+    flex bison libssl-dev pkg-config uuid-dev acl-dev zlib1g-dev          \
+    libxml2-dev libxml2-utils liburcu-dev xfsprogs gdb attr               \
+    libgoogle-perftools-dev zfsutils-linux screen libsqlite3-dev sqlite3  \
+    glusterfs-server
+
+RUN echo "root:kadalu" | chpasswd
+
+RUN sed -i 's/#PermitRootLogin yes/PermitRootLogin yes/' /etc/ssh/sshd_config \
+  && sed -i 's/#PermitRootLogin prohibit-password/PermitRootLogin yes/'       \
+    /etc/ssh/sshd_config &&                                                   \
+  sed -i.save -e "s#udev_sync = 1#udev_sync = 0#"                             \
+    -e "s#udev_rules = 1#udev_rules = 0#"                                     \
+    -e "s#use_lvmetad = 1#use_lvmetad = 0#"                                   \
+    -e "s#obtain_device_list_from_udev = 1#obtain_device_list_from_udev = 0#" \
+    /etc/lvm/lvm.conf &&                                                      \
+  systemctl mask getty.target
+
+RUN curl -fsSL https://crystal-lang.org/install.sh | sudo bash
+
+cmd ["/usr/sbin/init"]

tests/all/volume_metrics.t

@@ -0,0 +1,30 @@
+# -*- mode: ruby -*-
+load "#{File.dirname(__FILE__)}/../reset.t"
+
+USE_REMOTE_PLUGIN "docker"
+
+nodes = ["gserver1", "gserver2", "gserver3"]
+
+nodes.each do |node|
+  USE_NODE node
+
+  TEST "systemctl start glusterd"
+  puts TEST "systemctl status glusterd"
+  TEST "systemctl start gluster-metrics-exporter"
+  puts TEST "systemctl status gluster-metrics-exporter"
+
+  TEST "mkdir -p /exports/vol1"
+end
+
+USE_NODE nodes[0]
+TEST "gluster peer probe #{nodes[1]}"
+TEST "gluster peer probe #{nodes[2]}"
+TEST "gluster volume create vol1 replica 3 #{nodes[0]}:/exports/vol1/s1 #{nodes[1]}:/exports/vol1/s2 #{nodes[2]}:/exports/vol1/s3 force"
+TEST "gluster volume start vol1"
+puts TEST "curl http://localhost:9713/metrics"
+puts TEST "curl http://localhost:9713/metrics.json"
+
+nodes.each do |node|
+  USE_NODE node
+  puts TEST "cat /var/log/gluster-metrics-exporter/exporter.log"
+end

tests/build-container.sh

@@ -0,0 +1,2 @@
+#!/bin/sh
+docker build . --tag kadalu/gluster-node -f Dockerfile

tests/install.t

@@ -0,0 +1,18 @@
+# -*- mode: ruby -*-
+
+USE_REMOTE_PLUGIN "docker"
+nodes = ["gserver1", "gserver2", "gserver3"]
+
+# Static build Kadalu Storage Manager
+TEST "docker run --rm -i -v $PWD:/workspace -w /workspace crystallang/crystal:1.2.0-alpine /bin/sh -c \"apk add --update --no-cache --force-overwrite sqlite-dev sqlite-static && shards install && shards build --static\""
+
+# Install the Static binary to all containers/nodes
+# and copy the service files
+nodes.each do |node|
+  TEST "docker cp ./bin/gluster-metrics-exporter #{node}:/usr/sbin/gluster-metrics-exporter"
+  TEST "docker cp extra/gluster-metrics-exporter.service #{node}:/lib/systemd/system/"
+end
+
+# Sanity test
+USE_NODE nodes[0]
+puts TEST "gluster-metrics-exporter --version"

tests/reset.t

@@ -0,0 +1,15 @@
+# -*- mode: ruby -*-
+
+USE_REMOTE_PLUGIN "docker"
+nodes = ["gserver1", "gserver2", "gserver3"]
+
+nodes.each do |node|
+  USE_NODE node
+  RUN "systemctl stop glusterd"
+  RUN "systemctl disable glusterd"
+  RUN "systemctl stop gluster-metrics-exporter"
+  RUN "systemctl disable gluster-metrics-exporter"
+  RUN "rm -rf /var/lib/glusterd"
+  RUN "rm -rf /var/log/gluster"
+  RUN "rm -rf /var/log/gluster-metrics-exporter"
+end

tests/setup.t

@@ -0,0 +1,19 @@
+# -*- mode: ruby -*-
+USE_REMOTE_PLUGIN "docker"
+nodes = ["gserver1", "gserver2", "gserver3"]
+
+# Start three or N storage nodes(Containers)
+USE_NODE "local"
+nodes.each do |node|
+  USE_NODE "local"
+  RUN "docker stop #{node}"
+  RUN "docker rm #{node}"
+end
+
+RUN "docker network rm g1"
+TEST "docker network create g1"
+
+nodes.each do |node|
+  USE_NODE "local"
+  TEST "docker run -d -v /sys/fs/cgroup/:/sys/fs/cgroup:ro --privileged --name #{node} --hostname #{node} --network g1 kadalu/gluster-node"
+end