kobsio · ricoberger · Nov 19, 2021 · Nov 19, 2021
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -42,6 +42,7 @@ NOTE: As semantic versioning states all 0.y.z releases can contain breaking chan
 - [#194](https://github.com/kobsio/kobs/pull/194): [elasticsearch] Use pagination instead of infinite scrolling to display logs.
 - [#195](https://github.com/kobsio/kobs/pull/195): [istio] Display upstream cluster instead of authority in the React UI and ignore query string in path.
 - [#197](https://github.com/kobsio/kobs/pull/197): [clickhouse] Change break down filter for aggregation, to allow more complex filters.
+- [#210](https://github.com/kobsio/kobs/pull/210): [klogs] :warning: _Breaking change:_ :warning: Rename `clickhouse` plugin to `klogs`.
 
 ## [v0.6.0](https://github.com/kobsio/kobs/releases/tag/v0.6.0) (2021-10-11)
 

diff --git a/app/package.json b/app/package.json
@@ -5,20 +5,23 @@
   "proxy": "http://localhost:15220",
   "dependencies": {
     "@kobsio/plugin-applications": "*",
-    "@kobsio/plugin-clickhouse": "*",
     "@kobsio/plugin-core": "*",
     "@kobsio/plugin-dashboards": "*",
     "@kobsio/plugin-elasticsearch": "*",
     "@kobsio/plugin-flux": "*",
+    "@kobsio/plugin-grafana": "*",
+    "@kobsio/plugin-harbor": "*",
     "@kobsio/plugin-istio": "*",
     "@kobsio/plugin-jaeger": "*",
     "@kobsio/plugin-kiali": "*",
+    "@kobsio/plugin-klogs": "*",
     "@kobsio/plugin-markdown": "*",
     "@kobsio/plugin-opsgenie": "*",
     "@kobsio/plugin-prometheus": "*",
     "@kobsio/plugin-resources": "*",
     "@kobsio/plugin-rss": "*",
     "@kobsio/plugin-sonarqube": "*",
+    "@kobsio/plugin-sql": "*",
     "@kobsio/plugin-teams": "*",
     "@kobsio/plugin-users": "*",
     "@types/react": "^17.0.0",

diff --git a/app/src/index.tsx b/app/src/index.tsx
@@ -6,7 +6,6 @@ import './index.css';
 // Import plugins. Here we import all plugins, which we want to add to the current kobs app. By default this are all
 // first party plugins from the /plugins folder.
 import { App } from '@kobsio/plugin-core';
-import clickhousePlugin from '@kobsio/plugin-clickhouse';
 import applicationsPlugin from '@kobsio/plugin-applications';
 import dashboardsPlugin from '@kobsio/plugin-dashboards';
 import elasticsearchPlugin from '@kobsio/plugin-elasticsearch';
@@ -16,6 +15,7 @@ import harborPlugin from '@kobsio/plugin-harbor';
 import istioPlugin from '@kobsio/plugin-istio';
 import jaegerPlugin from '@kobsio/plugin-jaeger';
 import kialiPlugin from '@kobsio/plugin-kiali';
+import klogsPlugin from '@kobsio/plugin-klogs';
 import markdownPlugin from '@kobsio/plugin-markdown';
 import opsgeniePlugin from '@kobsio/plugin-opsgenie';
 import prometheusPlugin from '@kobsio/plugin-prometheus';
@@ -30,7 +30,6 @@ ReactDOM.render(
   <React.StrictMode>
     <App plugins={{
       ...applicationsPlugin,
-      ...clickhousePlugin,
       ...dashboardsPlugin,
       ...elasticsearchPlugin,
       ...fluxPlugin,
@@ -39,6 +38,7 @@ ReactDOM.render(
       ...istioPlugin,
       ...jaegerPlugin,
       ...kialiPlugin,
+      ...klogsPlugin,
       ...markdownPlugin,
       ...opsgeniePlugin,
       ...prometheusPlugin,

diff --git a/cmd/kobs/plugins/plugins.go b/cmd/kobs/plugins/plugins.go
@@ -12,7 +12,6 @@ import (
 	// Import all plugins, which should be used with the kobs instance. By default this are all first party plugins from
 	// the plugins folder.
 	"github.com/kobsio/kobs/plugins/applications"
-	"github.com/kobsio/kobs/plugins/clickhouse"
 	"github.com/kobsio/kobs/plugins/dashboards"
 	"github.com/kobsio/kobs/plugins/elasticsearch"
 	"github.com/kobsio/kobs/plugins/flux"
@@ -21,6 +20,7 @@ import (
 	"github.com/kobsio/kobs/plugins/istio"
 	"github.com/kobsio/kobs/plugins/jaeger"
 	"github.com/kobsio/kobs/plugins/kiali"
+	"github.com/kobsio/kobs/plugins/klogs"
 	"github.com/kobsio/kobs/plugins/markdown"
 	"github.com/kobsio/kobs/plugins/opsgenie"
 	"github.com/kobsio/kobs/plugins/prometheus"
@@ -35,7 +35,6 @@ import (
 // Config holds the configuration for all plugins. We have to add the configuration for all the imported plugins.
 type Config struct {
 	Applications  applications.Config  `json:"applications"`
-	Clickhouse    clickhouse.Config    `json:"clickhouse"`
 	Dashboards    dashboards.Config    `json:"dashboards"`
 	Elasticsearch elasticsearch.Config `json:"elasticsearch"`
 	Flux          flux.Config          `json:"flux"`
@@ -44,6 +43,7 @@ type Config struct {
 	Istio         istio.Config         `json:"istio"`
 	Jaeger        jaeger.Config        `json:"jaeger"`
 	Kiali         kiali.Config         `json:"kiali"`
+	Klogs         klogs.Config         `json:"klogs"`
 	Opsgenie      opsgenie.Config      `json:"opsgenie"`
 	Prometheus    prometheus.Config    `json:"prometheus"`
 	Markdown      markdown.Config      `json:"markdown"`
@@ -84,10 +84,10 @@ func Register(clusters *clusters.Clusters, config Config) chi.Router {
 	dashboardsRouter := dashboards.Register(clusters, router.plugins, config.Dashboards)
 	prometheusRouter, prometheusInstances := prometheus.Register(clusters, router.plugins, config.Prometheus)
 	elasticsearchRouter := elasticsearch.Register(clusters, router.plugins, config.Elasticsearch)
-	clickhouseRouter, clickhouseInstances := clickhouse.Register(clusters, router.plugins, config.Clickhouse)
+	klogsRouter, klogsInstances := klogs.Register(clusters, router.plugins, config.Klogs)
 	jaegerRouter := jaeger.Register(clusters, router.plugins, config.Jaeger)
 	kialiRouter := kiali.Register(clusters, router.plugins, config.Kiali)
-	istioRouter := istio.Register(clusters, router.plugins, config.Istio, prometheusInstances, clickhouseInstances)
+	istioRouter := istio.Register(clusters, router.plugins, config.Istio, prometheusInstances, klogsInstances)
 	grafanaRouter := grafana.Register(clusters, router.plugins, config.Grafana)
 	harborRouter := harbor.Register(clusters, router.plugins, config.Harbor)
 	fluxRouter := flux.Register(clusters, router.plugins, config.Flux)
@@ -105,7 +105,7 @@ func Register(clusters *clusters.Clusters, config Config) chi.Router {
 	router.Mount(dashboards.Route, dashboardsRouter)
 	router.Mount(prometheus.Route, prometheusRouter)
 	router.Mount(elasticsearch.Route, elasticsearchRouter)
-	router.Mount(clickhouse.Route, clickhouseRouter)
+	router.Mount(klogs.Route, klogsRouter)
 	router.Mount(jaeger.Route, jaegerRouter)
 	router.Mount(kiali.Route, kialiRouter)
 	router.Mount(istio.Route, istioRouter)

diff --git a/deploy/docker/kobs/config.yaml b/deploy/docker/kobs/config.yaml
@@ -60,10 +60,10 @@ plugins:
   #     apiKey: ${OPSGENIE_API_KEY}
   #     url: https://<mycompany>.app.eu.opsgenie.com
 
-  clickhouse:
-    - name: clickhouse-logging
-      displayName: ClickHouse (Logging)
-      description: ClickHouse is a fast open-source OLAP database management system.
+  klogs:
+    - name: klogs
+      displayName: klogs
+      description: Fast, scalable and reliable logging using Fluent Bit, Kafka and ClickHouse.
       address: localhost:9000
       database: logs
       username: ${CLICKHOUSE_USERNAME}

diff --git a/docs/configuration/plugins.md b/docs/configuration/plugins.md
@@ -5,13 +5,13 @@ Plugins can be used to extend the functions of kobs. They can be configured usin
 | Field | Type | Description | Required |
 | ----- | ---- | ----------- | -------- |
 | applications | [Applications](#applications) | Configure the caching behaviour for the applications plugin. | No |
-| clickhouse | [[]ClickHouse](#clickhouse) | Configure multiple ClickHouse instances, which can be used within kobs. | No |
 | elasticsearch | [[]Elasticsearch](#elasticsearch) | Configure multiple Elasticsearch instances, which can be used within kobs. | No |
 | grafana | [[]Grafana](#grafana) | Configure multiple Grafana instances, which can be used within kobs. | No |
 | harbor | [[]Harbor](#harbor) | Configure multiple Harbor instances, which can be used within kobs. | No |
 | istio | [[]Istio](#istio) | Configure multiple Istio instances, which can be used within kobs. | No |
 | jaeger | [[]Jaeger](#jaeger) | Configure multiple Jaeger instances, which can be used within kobs. | No |
 | kiali | [[]Kiali](#kiali) | Configure multiple Kiali instances, which can be used within kobs. | No |
+| klogs | [[]klogs](#klogs) | Configure multiple klogs instances, which can be used within kobs. | No |
 | opsgenie | [[]Opsgenie](#opsgenie) | Configure the Opsgenie API, which can be used within kobs. | No |
 | prometheus | [[]Prometheus](#prometheus) | Configure multiple Prometheus instances, which can be used within kobs. | No |
 | resources | [Resources](#resources) | Configuration for the resources plugin. | No |
@@ -34,33 +34,6 @@ plugins:
 | topologyCacheDuration | [duration](https://pkg.go.dev/time#ParseDuration) | The duration for how long the topology graph should be cached. The default value is `1h`. | No |
 | teamsCacheDuration | [duration](https://pkg.go.dev/time#ParseDuration) | The duration for how long the teams for an application should be cached. The default value is `1h`. | No |
 
-## ClickHouse
-
-The ClickHouse plugin provides a user interface for the [kobsio/fluent-bit-clickhouse](https://github.com/kobsio/fluent-bit-clickhouse) Fluent Bit plugin.
-
-The following config can be used to grant kobs access to a ClickHouse instance running at `clickhouse-clickhouse.logging.svc.cluster.local:9000`, where the logs are save in a database named `logs`. To access ClickHouse the user `admin` with the password `admin` is used.
-
-```yaml
-plugins:
-  clickhouse:
-    - name: ClickHouse
-      description: ClickHouse is a fast open-source OLAP database management system.
-      address: clickhouse-clickhouse.logging.svc.cluster.local:9000
-      database: logs
-      username: admin
-      password: admin
-```
-
-| Field | Type | Description | Required |
-| ----- | ---- | ----------- | -------- |
-| name | string | Name of the ClickHouse instance. | Yes |
-| displayName | string | Name of the ClickHouse as it is shown in the UI. | Yes |
-| descriptions | string | Description of the ClickHouse instance. | No |
-| address | string | Address of the ClickHouse instance. | Yes |
-| username | string | Username to access a ClickHouse instance. | No |
-| password | string | Password to access a ClickHouse instance. | No |
-| materializedColumns | []string | A list of materialized columns. See [kobsio/fluent-bit-clickhouse](https://github.com/kobsio/fluent-bit-clickhouse#configuration) for more information. | No |
-
 ## Elasticsearch
 
 The following config can be used to grant kobs access to a Elasticsearch instance running on `elasticsearch.kobs.io` and is protected with basic authentication. The credentials will be provided by the environment variables `ES_USERANME` and `ES_PASSWORD`.
@@ -135,7 +108,7 @@ plugins:
 
 ## Istio
 
-The following configuration can be used to access a Istio instances using a Prometheus plugin named `prometheus` and an Clickhouse plugin named `clickhouse`.
+The following configuration can be used to access a Istio instances using a Prometheus plugin named `prometheus` and an klogs plugin named `klogs`.
 
 ```yaml
 plugins:
@@ -146,9 +119,9 @@ plugins:
       prometheus:
         enabled: true
         name: prometheus
-      clickhouse:
+      klogs:
         enabled: true
-        name: clickhouse
+        name: klogs
 ```
 
 | Field | Type | Description | Required |
@@ -158,8 +131,8 @@ plugins:
 | descriptions | string | Description of the Istio instance. | No |
 | prometheus.enabled | boolean | Enabled the Prometheus integration for Istio. | No |
 | prometheus.name | string | The name of the Prometheus instance which should be used for the Istio instance. | No |
-| clickhouse.enabled | boolean | Enabled the Clickhouse integration for Istio. | No |
-| clickhouse.name | string | The name of the Clickhouse instance which should be used for the Istio instance. | No |
+| klogs.enabled | boolean | Enabled the klogs integration for Istio. | No |
+| klogs.name | string | The name of the klogs instance which should be used for the Istio instance. | No |
 
 ## Jaeger
 
@@ -211,6 +184,33 @@ plugins:
 | traffic.failure | number | Threshold to mark edges with failures. This must be a number between `0` and `100`. The default value is `5`. | No |
 | traffic.degraded | number | Threshold to mark edges with degraded performance. This must be a number between `0` and `100`. The default value is `1`. | No |
 
+## klogs
+
+The klogs plugin provides a user interface for the [kobsio/klogs](https://github.com/kobsio/klogs) Fluent Bit plugin, which allows you to collect logs via Fluent Bit and save them in ClickHouse.
+
+The following config can be used to grant kobs access to a ClickHouse instance running at `clickhouse-clickhouse.logging.svc.cluster.local:9000`, where the logs are save in a database named `logs`. To access ClickHouse the user `admin` with the password `admin` is used.
+
+```yaml
+plugins:
+  klogs:
+    - name: klogs
+      description: Fast, scalable and reliable logging using Fluent Bit, Kafka and ClickHouse.
+      address: clickhouse-clickhouse.logging.svc.cluster.local:9000
+      database: logs
+      username: admin
+      password: admin
+```
+
+| Field | Type | Description | Required |
+| ----- | ---- | ----------- | -------- |
+| name | string | Name of the ClickHouse instance. | Yes |
+| displayName | string | Name of the ClickHouse as it is shown in the UI. | Yes |
+| descriptions | string | Description of the ClickHouse instance. | No |
+| address | string | Address of the ClickHouse instance. | Yes |
+| username | string | Username to access a ClickHouse instance. | No |
+| password | string | Password to access a ClickHouse instance. | No |
+| materializedColumns | []string | A list of materialized columns. See [kobsio/klogs](https://github.com/kobsio/klogs#configuration) for more information. | No |
+
 ## Opsgenie
 
 The following configuration can be used to access the Opsgenie API.

diff --git a/...plugins/assets/clickhouse-aggregation.png → docs/plugins/assets/klogs-aggregation.png b/...plugins/assets/clickhouse-aggregation.png → docs/plugins/assets/klogs-aggregation.png
diff --git a/docs/plugins/assets/clickhouse-logs.png → docs/plugins/assets/klogs-logs.png b/docs/plugins/assets/clickhouse-logs.png → docs/plugins/assets/klogs-logs.png
diff --git a/docs/plugins/istio.md b/docs/plugins/istio.md
@@ -1,6 +1,6 @@
 # Istio
 
-The Istio plugin can be used to observe your service mesh within kobs. For that kobs uses an existing Prometheus and Clickhouse instance to get the metrics for your applications which are part of the service mesh and to build a simple topology graph for them.
+The Istio plugin can be used to observe your service mesh within kobs. For that kobs uses an existing Prometheus and klogs instance to get the metrics for your applications which are part of the service mesh and to build a simple topology graph for them.
 
 ![Overview](assets/istio-page-overview.png)
 
@@ -10,7 +10,7 @@ For each application you have a metrics, top and tap view. The metrics view show
 
 ![Metrics Details](assets/istio-page-metrics.png)
 
-The top view can be used to get the aggregated metrics, based on the request direction, service, method and path. These metrics are generated based on the access logs via the Clickhouse plugin. The top view shows the number of requests, the best, worst, average, and last request duration and the success rate. You can also go to the tap view, for a specific request type.
+The top view can be used to get the aggregated metrics, based on the request direction, service, method and path. These metrics are generated based on the access logs via the klogs plugin. The top view shows the number of requests, the best, worst, average, and last request duration and the success rate. You can also go to the tap view, for a specific request type.
 
 ![Top](assets/istio-page-top.png)
 

diff --git a/docs/plugins/clickhouse.md → docs/plugins/klogs.md b/docs/plugins/clickhouse.md → docs/plugins/klogs.md
@@ -1,15 +1,12 @@
-# ClickHouse
+# klogs
 
-!!! note
-    The ClickHouse plugin can only be used with the [kobsio/fluent-bit-clickhouse](https://github.com/kobsio/fluent-bit-clickhouse) Fluent Bit plugin. If you want to use kobs to run raw SQL commands against a ClickHouse instance you can use the [SQL plugin](sql.md).
+The klogs plugin can be used together with the [kobsio/klogs](https://github.com/kobsio/klogs) output plugin for [Fluent Bit](https://fluentbit.io). You can then use the specified [Query Syntax](#query-syntax) to get the logs from ClickHouse.
 
-The ClickHouse plugin can be used together with the [kobsio/fluent-bit-clickhouse](https://github.com/kobsio/fluent-bit-clickhouse) output plugin for [Fluent Bit](https://fluentbit.io). You can then use the specified [Query Syntax](#query-syntax) to get the logs from ClickHouse.
-
-![Logs](assets/clickhouse-logs.png)
+![Logs](assets/klogs-logs.png)
 
 ## Options
 
-The following options can be used for a panel with the ClickHouse plugin:
+The following options can be used for a panel with the klogs plugin:
 
 | Field | Type | Description | Required |
 | ----- | ---- | ----------- | -------- |
@@ -21,8 +18,8 @@ The following options can be used for a panel with the ClickHouse plugin:
 
 | Field | Type | Description | Required |
 | ----- | ---- | ----------- | -------- |
-| name | string | A name for the ClickHouse query, which is displayed in the select box. | Yes |
-| query | string | The query which should be run against ClickHouse. See [Query Syntax](#query-syntax) for more information on the syntax, when ClickHouse is used in the `logs` mode. | Yes |
+| name | string | A name for the klogs query, which is displayed in the select box. | Yes |
+| query | string | The query which should be run against ClickHouse. See [Query Syntax](#query-syntax) for more information on the syntax. | Yes |
 | fields | []string | A list of fields to display in the results table. If this field is omitted, the whole document is displayed in the results table. This field is only available for the `logs`. | No |
 | order | string | Order for the returned logs. Must be `ascending` or `descending`. The default value for this field is `descending`. | No |
 | orderBy | string | The name of the field, by which the results should be orderd. The default value for this field is `timestamp`. | No |
@@ -38,7 +35,7 @@ spec:
         - title: Istio Logs
           colSpan: 12
           plugin:
-            name: clickhouse
+            name: klogs
             options:
               type: logs
               queries:
@@ -83,7 +80,7 @@ spec:
         - title: Number of Logs per App
           colSpan: 6
           plugin:
-            name: clickhouse-logging
+            name: klogs
             options:
               type: aggregation
               aggregation:
@@ -98,7 +95,7 @@ spec:
         - title: Log Levels for MyApplication
           colSpan: 6
           plugin:
-            name: clickhouse-logging
+            name: klogs
             options:
               type: aggregation
               aggregation:
@@ -112,7 +109,7 @@ spec:
         - title: Request Duration for MyApplication by Response Code
           colSpan: 12
           plugin:
-            name: clickhouse-logging
+            name: klogs
             options:
               type: aggregation
               aggregation:
@@ -126,7 +123,7 @@ spec:
                     - content.response_code
 ```
 
-![Aggregation Example](assets/clickhouse-aggregation.png)
+![Aggregation Example](assets/klogs-aggregation.png)
 
 ### Aggregation Options
 

diff --git a/mkdocs.yml b/mkdocs.yml
@@ -47,7 +47,6 @@ nav:
   - Plugins:
       - Getting Started: plugins/getting-started.md
       - Applications: plugins/applications.md
-      - ClickHouse: plugins/clickhouse.md
       - Dashboards: plugins/dashboards.md
       - Elasticsearch: plugins/elasticsearch.md
       - Flux: plugins/flux.md
@@ -56,6 +55,7 @@ nav:
       - Istio: plugins/istio.md
       - Jaeger: plugins/jaeger.md
       - Kiali: plugins/kiali.md
+      - klogs: plugins/klogs.md
       - Markdown: plugins/markdown.md
       - Opsgenie: plugins/opsgenie.md
       - Prometheus: plugins/prometheus.md