Add ClickHouse Plugin

CHANGELOG.md

@@ -17,6 +17,7 @@ NOTE: As semantic versioning states all 0.y.z releases can contain breaking chan
 - [#113](https://github.com/kobsio/kobs/pull/113): Allow and improve customization of axis scaling.
 - [#116](https://github.com/kobsio/kobs/pull/116): Open details when clicking on Jaeger chart nodes.
 - [#119](https://github.com/kobsio/kobs/pull/119): Add Flux plugin to view and reconcile [Flux](https://fluxcd.io) resources.
+- [#122](https://github.com/kobsio/kobs/pull/122): Add ClickHouse plugin, to query show logs ingested by the [kobsio/fluent-bit-clickhouse](https://github.com/kobsio/fluent-bit-clickhouse) Fluent Bit plugin.
 
 ### Fixed
 

app/package.json

@@ -5,6 +5,7 @@
   "proxy": "http://localhost:15220",
   "dependencies": {
     "@kobsio/plugin-applications": "*",
+    "@kobsio/plugin-clickhouse": "*",
     "@kobsio/plugin-core": "*",
     "@kobsio/plugin-dashboards": "*",
     "@kobsio/plugin-elasticsearch": "*",

app/src/index.tsx

@@ -18,6 +18,7 @@ import fluxPlugin from '@kobsio/plugin-flux';
 import opsgeniePlugin from '@kobsio/plugin-opsgenie';
 import markdownPlugin from '@kobsio/plugin-markdown';
 import rssPlugin from '@kobsio/plugin-rss';
+import clickhousePlugin from '@kobsio/plugin-clickhouse';
 
 ReactDOM.render(
   <React.StrictMode>
@@ -34,6 +35,7 @@ ReactDOM.render(
       ...opsgeniePlugin,
       ...markdownPlugin,
       ...rssPlugin,
+      ...clickhousePlugin,
     }} />
   </React.StrictMode>,
   document.getElementById('root')

cmd/kobs/plugins/plugins.go

@@ -12,6 +12,7 @@ import (
 	// Import all plugins, which should be used with the kobs instance. By default this are all first party plugins from
 	// the plugins folder.
 	"github.com/kobsio/kobs/plugins/applications"
+	"github.com/kobsio/kobs/plugins/clickhouse"
 	"github.com/kobsio/kobs/plugins/dashboards"
 	"github.com/kobsio/kobs/plugins/elasticsearch"
 	"github.com/kobsio/kobs/plugins/flux"
@@ -39,6 +40,7 @@ type Config struct {
 	Opsgenie      opsgenie.Config      `json:"opsgenie"`
 	Markdown      markdown.Config      `json:"markdown"`
 	RSS           rss.Config           `json:"rss"`
+	Clickhouse    clickhouse.Config    `json:"clickhouse"`
 }
 
 // Router implements the router for the plugins package. This only registeres one route which is used to return all the
@@ -63,8 +65,8 @@ func Register(clusters *clusters.Clusters, config Config) chi.Router {
 	router.Get("/", router.getPlugins)
 
 	// Register all plugins
-	router.Mount(applications.Route, applications.Register(clusters, router.plugins, config.Applications))
 	router.Mount(resources.Route, resources.Register(clusters, router.plugins, config.Resources))
+	router.Mount(applications.Route, applications.Register(clusters, router.plugins, config.Applications))
 	router.Mount(teams.Route, teams.Register(clusters, router.plugins, config.Teams))
 	router.Mount(dashboards.Route, dashboards.Register(clusters, router.plugins, config.Dashboards))
 	router.Mount(prometheus.Route, prometheus.Register(clusters, router.plugins, config.Prometheus))
@@ -73,6 +75,7 @@ func Register(clusters *clusters.Clusters, config Config) chi.Router {
 	router.Mount(kiali.Route, kiali.Register(clusters, router.plugins, config.Kiali))
 	router.Mount(flux.Route, flux.Register(clusters, router.plugins, config.Flux))
 	router.Mount(opsgenie.Route, opsgenie.Register(clusters, router.plugins, config.Opsgenie))
+	router.Mount(clickhouse.Route, clickhouse.Register(clusters, router.plugins, config.Clickhouse))
 	router.Mount(markdown.Route, markdown.Register(clusters, router.plugins, config.Markdown))
 	router.Mount(rss.Route, rss.Register(clusters, router.plugins, config.RSS))
 

docs/configuration/plugins.md

@@ -5,6 +5,7 @@ Plugins can be used to extend the functions of kobs. They can be configured usin
 | Field | Type | Description | Required |
 | ----- | ---- | ----------- | -------- |
 | applications | [Applications](#applications) | Configure the caching behaviour for the applications plugin. | No |
+| clickhouse | [[]ClickHouse](#clickhouse) | Configure multiple ClickHouse instances, which can be used within kobs. | No |
 | elasticsearch | [[]Elasticsearch](#elasticsearch) | Configure multiple Elasticsearch instances, which can be used within kobs. | No |
 | jaeger | [[]Jaeger](#jaeger) | Configure multiple Jaeger instances, which can be used within kobs. | No |
 | kiali | [[]Kiali](#kiali) | Configure multiple Kiali instances, which can be used within kobs. | No |
@@ -28,6 +29,31 @@ plugins:
 | topologyCacheDuration | [duration](https://pkg.go.dev/time#ParseDuration) | The duration for how long the topology graph should be cached. The default value is `1h`. | No |
 | teamsCacheDuration | [duration](https://pkg.go.dev/time#ParseDuration) | The duration for how long the teams for an application should be cached. The default value is `1h`. | No |
 
+## ClickHouse
+
+The following config can be used to grant kobs access to a ClickHouse instance running at `clickhouse-clickhouse.logging.svc.cluster.local:9000`, where the logs are save in a database named `logs`. To access ClickHouse the user `admin` with the password `admin` is used.
+
+```yaml
+plugins:
+  clickhouse:
+    - name: ClickHouse
+      description: ClickHouse is a fast open-source OLAP database management system.
+      address: clickhouse-clickhouse.logging.svc.cluster.local:9000
+      database: logs
+      username: admin
+      password: admin
+      type: logs
+```
+
+| Field | Type | Description | Required |
+| ----- | ---- | ----------- | -------- |
+| name | string | Name of the ClickHouse instance. | Yes |
+| displayName | string | Name of the ClickHouse as it is shown in the UI. | Yes |
+| descriptions | string | Description of the ClickHouse instance. | No |
+| address | string | Address of the ClickHouse instance. | Yes |
+| username | string | Username to access a ClickHouse instance. | No |
+| password | string | Password to access a ClickHouse instance. | No |
+| type | string | The type which should be used for the ClickHouse instance. Currently the only supported value is `logs`. The `logs` mode should be used together with the [kobsio/fluent-bit-clickhouse](https://github.com/kobsio/fluent-bit-clickhouse) plugin to collect logs via Fluent Bit and save them in ClickHouse. |
 
 ## Elasticsearch
 

docs/plugins/clickhouse.md

@@ -0,0 +1,97 @@
+# ClickHouse
+
+!!! warning
+    The ClickHouse plugin is in a very early stage and might be changed heavily in the future.
+
+The [ClickHouse](https://clickhouse.tech) plugin can be used to get the data from a configured ClickHouse instance.
+
+The ClickHouse plugin can be used together with the [kobsio/fluent-bit-clickhouse](https://github.com/kobsio/fluent-bit-clickhouse) output plugin for [Fluent Bit](https://fluentbit.io). For this the `type` in the plugin options must be set to `logs`. You can then use the specified [Query Syntax](#query-syntax) to get the logs from ClickHouse.
+
+![Logs](assets/clickhouse-logs.png)
+
+## Options
+
+The following options can be used for a panel with the ClickHouse plugin:
+
+| Field | Type | Description | Required |
+| ----- | ---- | ----------- | -------- |
+| type | string | Set the type for which you want to use the ClickHouse instance. Currently the only supported value is `logs`. | Yes |
+| queries | [[]Query](#query) | A list of queries, which can be selected by the user. | Yes |
+
+### Query
+
+| Field | Type | Description | Required |
+| ----- | ---- | ----------- | -------- |
+| name | string | A name for the ClickHouse query, which is displayed in the select box. | Yes |
+| query | string | The query which should be run against ClickHouse. See [Query Syntax](#query-syntax) for more information on the syntax, when ClickHouse is used in the `logs` mode. | Yes |
+| fields | []string | A list of fields to display in the results table. If this field is omitted, the whole document is displayed in the results table. | No |
+
+```yaml
+---
+apiVersion: kobs.io/v1beta1
+kind: Dashboard
+spec:
+  placeholders:
+    - name: namespace
+      description: The workload namespace
+    - name: app
+      description: The workloads app label
+  rows:
+    - size: -1
+      panels:
+        - title: Istio Logs
+          colSpan: 12
+          plugin:
+            name: clickhouse
+            options:
+              showChart: true
+              queries:
+                - name: Istio Logs
+                  query: "namespace='bookinfo' _and_ app='bookinfo' _and_ container_name='istio-proxy' _and_ content.upstream_cluster~'inbound.*'"
+                  fields:
+                    - "pod_name"
+                    - "content.authority"
+                    - "content.route_name"
+                    - "content.protocol"
+                    - "content.method"
+                    - "content.path"
+                    - "content.response_code"
+                    - "content.upstream_service_time"
+                    - "content.bytes_received"
+                    - "content.bytes_sent"
+```
+
+## Query Syntax
+
+### Operators
+
+kobs supports multiple operators which can be used in a query to retrieve logs from ClickHouse:
+
+| Operator | Description | Example |
+| -------- | ----------- | ------- |
+| `(` and `)` | Multiple terms or clauses can be grouped together with parentheses, to form sub-queries. | `cluster='kobs-demo' _and_ (namespace='bookinfo' _or_ namespace='istio-system')` |
+| `_not_` | Exclude the term from the query. | `cluster='kobs-demo' _and_ _not_ namespace='bookinfo'` |
+| `_and_` | Both terms must be included in the results. | `namespace='bookinfo' _and_ app='bookinfo'` |
+| `_or_` | The result can contain one of the given terms. | `namespace='bookinfo' _or_ namespace='istio-system'` |
+| `=` | The field must have this value. | `namespace='bookinfo'` |
+| `!=` | The field should not have this value. | `namespace!='bookinfo'` |
+| `>` | The value of the field must be greater than the specified value. | `content.response_code>499` |
+| `>=` | The value of the field must be greater than or equal to the specified value. | `content.response_code>=500` |
+| `<` | The value of the field must be lower than the specified value. | `content.response_code<500` |
+| `<=` | The value of the field must be lower than or equal to the specified value. | `content.response_code<=499` |
+| `~` | The value of the field must match the regular expression. | `content.upstream_cluster~'inbound.*'` |
+
+### Standard Fields
+
+- `timestamp`: The timestamp for when the log line was written.
+- `cluster`: The name of the cluster as it is set by Fluent Bit.
+- `namespace`: The namespace of the Pod.
+- `app`: The value of the `app` or `k8s-app` label of the Pod.
+- `pod_name`: The name of the Pod.
+- `container_name`: The name of the container from the Pod.
+- `host`: The name of the host where the Pod is running on.
+- `log`: The complete log line as it was written by the container.
+
+### Examples
+
+- `namespace='bookinfo' _and_ app='bookinfo' _and_ container_name='istio-proxy' _and_ content.upstream_cluster~'inbound.*'`: Select all inbound Istio logs from the bookinfo app in the bookinfo namespace.

go.mod

@@ -3,6 +3,7 @@ module github.com/kobsio/kobs
 go 1.16
 
 require (
+	github.com/ClickHouse/clickhouse-go v1.4.5
 	github.com/fluxcd/helm-controller/api v0.11.2
 	github.com/fluxcd/kustomize-controller/api v0.13.3
 	github.com/fluxcd/pkg/apis/meta v0.10.1
@@ -17,6 +18,7 @@ require (
 	github.com/prometheus/common v0.30.0
 	github.com/sirupsen/logrus v1.8.1
 	github.com/spf13/pflag v1.0.5
+	github.com/stretchr/testify v1.7.0
 	k8s.io/api v0.21.3
 	k8s.io/apiextensions-apiserver v0.21.3
 	k8s.io/apimachinery v0.21.3

go.sum

@@ -45,6 +45,8 @@ github.com/Azure/go-autorest/logger v0.2.0/go.mod h1:T9E3cAhj2VqvPOtCYAvby9aBXkZ
 github.com/Azure/go-autorest/tracing v0.6.0/go.mod h1:+vhtPC754Xsa23ID7GlGsrdKBpUA79WCAKPPZVC2DeU=
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
 github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
+github.com/ClickHouse/clickhouse-go v1.4.5 h1:FfhyEnv6/BaWldyjgT2k4gDDmeNwJ9C4NbY/MXxJlXk=
+github.com/ClickHouse/clickhouse-go v1.4.5/go.mod h1:EaI/sW7Azgz9UATzd5ZdZHRUhHgv5+JMS9NSr2smCJI=
 github.com/Knetic/govaluate v3.0.1-0.20171022003610-9aa49832a739+incompatible/go.mod h1:r7JcOSlj0wfOMncg0iLm8Leh48TZaKVeNIfJntJ2wa0=
 github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46/go.mod h1:3wb06e3pkSAbeQ52E9H9iFoQsEEwGN64994WTCIhntQ=
 github.com/NYTimes/gziphandler v1.1.1/go.mod h1:n/CVRwUEOgIxrgPvAQhUUr9oeUtvrhMomdKFjzJNB0c=
@@ -80,6 +82,8 @@ github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+Ce
 github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
 github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
 github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs=
+github.com/bkaradzic/go-lz4 v1.0.0 h1:RXc4wYsyz985CkXXeX04y4VnZFGG8Rd43pRaHsOXAKk=
+github.com/bkaradzic/go-lz4 v1.0.0/go.mod h1:0YdlkowM3VswSROI7qDxhRvJ3sLhlFrRRwjwegp5jy4=
 github.com/bketelsen/crypt v0.0.3-0.20200106085610-5cbc8cc4026c/go.mod h1:MKsuJmJgSg28kpZDP6UIiPt0e0Oz0kqKNGyRaWEPv84=
 github.com/blang/semver v3.5.1+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk=
 github.com/casbin/casbin/v2 v2.1.2/go.mod h1:YcPU1XXisHhLzuxH9coDNf2FbKpjGlbCg3n9yuLkIJQ=
@@ -94,6 +98,8 @@ github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5P
 github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU=
 github.com/clbanning/x2j v0.0.0-20191024224557-825249438eec/go.mod h1:jMjuTZXRI4dUb/I5gc9Hdhagfvm9+RyrPryS/auMzxE=
 github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
+github.com/cloudflare/golz4 v0.0.0-20150217214814-ef862a3cdc58 h1:F1EaeKL/ta07PY/k9Os/UFtwERei2/XzGemhpGnBKNg=
+github.com/cloudflare/golz4 v0.0.0-20150217214814-ef862a3cdc58/go.mod h1:EOBUe0h4xcZ5GoxqC5SDxFQ8gwyZPKQoEzownBlhI80=
 github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc=
 github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa/go.mod h1:zn76sxSg3SzpJ0PPJaLDCu+Bu0Lg3sKTORVIj19EIF8=
 github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd/go.mod h1:sE/e/2PUdi/liOCUjSTXgM1o87ZssimdTWN964YiIeI=
@@ -330,6 +336,7 @@ github.com/jaegertracing/jaeger v1.15.1 h1:7QzNAXq+4ko9GtCjozDNAp2uonoABu+B2Rk94
 github.com/jaegertracing/jaeger v1.15.1/go.mod h1:LUWPSnzNPGRubM8pk0inANGitpiMOOxihXx0+53llXI=
 github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI=
 github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k=
+github.com/jmoiron/sqlx v1.2.0/go.mod h1:1FEQNm3xlJgrMD+FBdI9+xvCksHtbpVBBw5dYhBSsks=
 github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo=
 github.com/jpillora/backoff v1.0.0 h1:uvFg412JmmHBHw7iwprIxkPMI+sGQ4kzOWsMeHnm2EA=
 github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4=
@@ -361,6 +368,7 @@ github.com/kr/pty v1.1.5/go.mod h1:9r2w37qlBe7rQ6e1fg1S/9xpWHSnaqNdHD3WcMdbPDA=
 github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
 github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
 github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
+github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
 github.com/lightstep/lightstep-tracer-common/golang/gogo v0.0.0-20190605223551-bc2310a04743/go.mod h1:qklhhLq1aX+mtWk9cPHPzaBjWImj5ULL6C7HFJtXQMM=
 github.com/lightstep/lightstep-tracer-go v0.18.1/go.mod h1:jlF1pusYV4pidLvZ+XD0UBX0ZE6WURAspgAczcDHrL4=
 github.com/lyft/protoc-gen-validate v0.0.13/go.mod h1:XbGvPuh87YZc5TdIa2/I4pLk0QoUACkjt2znoq26NVQ=
@@ -372,6 +380,7 @@ github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaO
 github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4=
 github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4=
 github.com/mattn/go-runewidth v0.0.2/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU=
+github.com/mattn/go-sqlite3 v1.9.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc=
 github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
 github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 h1:I0XW9+e1XWDxdcEniV4rQAIOPUGDq67JSCiRCgGCZLI=
 github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4=
@@ -461,6 +470,7 @@ github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/9
 github.com/performancecopilot/speed v3.0.0+incompatible/go.mod h1:/CLtqpZ5gBg1M9iaPbIdPPGyKcA8hKdoy6hAWba7Yac=
 github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU=
 github.com/pierrec/lz4 v1.0.2-0.20190131084431-473cd7ce01a1/go.mod h1:3/3N9NVKO0jef7pBehbT1qWhCMrIgbYNnFAZCqQ5LRc=
+github.com/pierrec/lz4 v2.0.5+incompatible h1:2xWsjqPFWcplujydGg4WmhC/6fZqK42wMM8aXeqhl0I=
 github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY=
 github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=

mkdocs.yml

@@ -35,6 +35,7 @@ nav:
   - Plugins:
       - Getting Started: plugins/getting-started.md
       - Applications: plugins/applications.md
+      - ClickHouse: plugins/clickhouse.md
       - Dashboards: plugins/dashboards.md
       - Elasticsearch: plugins/elasticsearch.md
       - Flux: plugins/flux.md