Use ClickHouse plugin only for logs

CHANGELOG.md

@@ -62,6 +62,7 @@ NOTE: As semantic versioning states all 0.y.z releases can contain breaking chan
 - [#144](https://github.com/kobsio/kobs/pull/144): Avoid timeouts for long running requests in the ClickHouse plugin.
 - [#147](https://github.com/kobsio/kobs/pull/147): Improve query performance for ClickHouse plugin and allow custom values for the maximum amount of documents, which should be returned (see [#133](https://github.com/kobsio/kobs/pull/133)).
 - [#148](https://github.com/kobsio/kobs/pull/148): Improve reliability of kobs, by do not checking the database connection for a configured ClickHouse instance.
+- [#150](https://github.com/kobsio/kobs/pull/150): :warning: *Breaking change:* :warning: The ClickHouse plugin can now only be used together with the [kobsio/fluent-bit-clickhouse](https://github.com/kobsio/fluent-bit-clickhouse) output plugin for [Fluent Bit](https://fluentbit.io). For raw SQL queries against a ClickHouse instance the SQL plugin added in [#149](https://github.com/kobsio/kobs/pull/149) can be used.
 
 ## [v0.5.0](https://github.com/kobsio/kobs/releases/tag/v0.5.0) (2021-08-03)
 

docs/configuration/plugins.md

@@ -32,6 +32,8 @@ plugins:
 
 ## ClickHouse
 
+The ClickHouse plugin provides a user interface for the [kobsio/fluent-bit-clickhouse](https://github.com/kobsio/fluent-bit-clickhouse) Fluent Bit plugin.
+
 The following config can be used to grant kobs access to a ClickHouse instance running at `clickhouse-clickhouse.logging.svc.cluster.local:9000`, where the logs are save in a database named `logs`. To access ClickHouse the user `admin` with the password `admin` is used.
 
 ```yaml
@@ -43,7 +45,6 @@ plugins:
       database: logs
       username: admin
       password: admin
-      type: logs
 ```
 
 | Field | Type | Description | Required |
@@ -54,7 +55,6 @@ plugins:
 | address | string | Address of the ClickHouse instance. | Yes |
 | username | string | Username to access a ClickHouse instance. | No |
 | password | string | Password to access a ClickHouse instance. | No |
-| type | string | The type which should be used for the ClickHouse instance. This must be `sql` or `logs`. While the `sql` mode allows you to use raw SQL queries, the `logs` mode should be used together with the [kobsio/fluent-bit-clickhouse](https://github.com/kobsio/fluent-bit-clickhouse) plugin to collect logs via Fluent Bit and save them in ClickHouse. |
 
 ## Elasticsearch
 

docs/plugins/clickhouse.md

@@ -1,11 +1,9 @@
 # ClickHouse
 
-!!! warning
-    The ClickHouse plugin is in a very early stage and might be changed heavily in the future.
+!!! note
+    The ClickHouse plugin can only be used with the [kobsio/fluent-bit-clickhouse](https://github.com/kobsio/fluent-bit-clickhouse) Fluent Bit plugin. If you want to use kobs to run raw SQL commands against a ClickHouse instance you can use the [SQL plugin](sql.md).
 
-The [ClickHouse](https://clickhouse.tech) plugin can be used to get the data from a configured ClickHouse instance.
-
-The ClickHouse plugin can be used together with the [kobsio/fluent-bit-clickhouse](https://github.com/kobsio/fluent-bit-clickhouse) output plugin for [Fluent Bit](https://fluentbit.io). For this the `type` in the plugin options must be set to `logs`. You can then use the specified [Query Syntax](#query-syntax) to get the logs from ClickHouse.
+The ClickHouse plugin can be used together with the [kobsio/fluent-bit-clickhouse](https://github.com/kobsio/fluent-bit-clickhouse) output plugin for [Fluent Bit](https://fluentbit.io). You can then use the specified [Query Syntax](#query-syntax) to get the logs from ClickHouse.
 
 ![Logs](assets/clickhouse-logs.png)
 
@@ -15,8 +13,7 @@ The following options can be used for a panel with the ClickHouse plugin:
 
 | Field | Type | Description | Required |
 | ----- | ---- | ----------- | -------- |
-| type | string | Set the type for which you want to use the ClickHouse instance. This must be `sql` or `logs` | Yes |
-| showChart | boolean | If this is `true` the chart with the distribution of the Documents over the selected time range will be shown. This option is only available when type is `logs`. | No |
+| type | string | Set the type which should be used to visualize your logs. Currently this must be `logs`. | Yes |
 | queries | [[]Query](#query) | A list of queries, which can be selected by the user. | Yes |
 
 ### Query
@@ -48,7 +45,6 @@ spec:
           plugin:
             name: clickhouse
             options:
-              showChart: true
               queries:
                 - name: Istio Logs
                   query: "namespace='bookinfo' _and_ app='bookinfo' _and_ container_name='istio-proxy' _and_ content.upstream_cluster~'inbound.*'"

plugins/clickhouse/clickhouse.go

@@ -25,12 +25,6 @@ var (
 // Config is the structure of the configuration for the clickhouse plugin.
 type Config []instance.Config
 
-type logsResponse struct {
-	Documents []map[string]interface{} `json:"documents"`
-	Fields    []string                 `json:"fields"`
-	Offset    int64                    `json:"offset"`
-}
-
 // Router implements the router for the resources plugin, which can be registered in the router for our rest api.
 type Router struct {
 	*chi.Mux
@@ -48,35 +42,6 @@ func (router *Router) getInstance(name string) *instance.Instance {
 	return nil
 }
 
-func (router *Router) getSQL(w http.ResponseWriter, r *http.Request) {
-	name := chi.URLParam(r, "name")
-	query := r.URL.Query().Get("query")
-
-	log.WithFields(logrus.Fields{"name": name, "query": query}).Tracef("getSQL")
-
-	i := router.getInstance(name)
-	if i == nil {
-		errresponse.Render(w, r, nil, http.StatusBadRequest, "Could not find instance name")
-		return
-	}
-
-	rows, columns, err := i.GetSQL(r.Context(), query)
-	if err != nil {
-		errresponse.Render(w, r, err, http.StatusBadRequest, "Could not get result for SQL query")
-		return
-	}
-
-	data := struct {
-		Rows    [][]interface{} `json:"rows"`
-		Columns []string        `json:"columns"`
-	}{
-		rows,
-		columns,
-	}
-
-	render.JSON(w, r, data)
-}
-
 // getLogs implements the special handling when the user selected the "logs" options for the "view" configuration. This
 // options is intended to use together with the kobsio/fluent-bit-clickhouse Fluent Bit plugin and provides a custom
 // query language to get the logs from ClickHouse.
@@ -204,16 +169,11 @@ func Register(clusters *clusters.Clusters, plugins *plugin.Plugins, config Confi
 
 		instances = append(instances, instance)
 
-		var options map[string]interface{}
-		options = make(map[string]interface{})
-		options["type"] = cfg.Type
-
 		plugins.Append(plugin.Plugin{
 			Name:        cfg.Name,
 			DisplayName: cfg.DisplayName,
 			Description: cfg.Description,
 			Type:        "clickhouse",
-			Options:     options,
 		})
 	}
 
@@ -223,7 +183,6 @@ func Register(clusters *clusters.Clusters, plugins *plugin.Plugins, config Confi
 		instances,
 	}
 
-	router.Get("/sql/{name}", router.getSQL)
 	router.Get("/logs/{name}", router.getLogs)
 
 	return router

plugins/clickhouse/pkg/instance/instance.go

@@ -36,41 +36,6 @@ type Instance struct {
 	client   *sql.DB
 }
 
-// GetSQL returns all rows for the user provided SQL query.
-func (i *Instance) GetSQL(ctx context.Context, query string) ([][]interface{}, []string, error) {
-	rows, err := i.client.QueryContext(ctx, query)
-	if err != nil {
-		return nil, nil, err
-	}
-	defer rows.Close()
-
-	var columns []string
-	columns, err = rows.Columns()
-	if err != nil {
-		return nil, nil, err
-	}
-	columnsLen := len(columns)
-
-	var result [][]interface{}
-
-	for rows.Next() {
-		var r []interface{}
-		r = make([]interface{}, columnsLen)
-
-		for i := 0; i < columnsLen; i++ {
-			r[i] = new(interface{})
-		}
-
-		if err := rows.Scan(r...); err != nil {
-			return nil, nil, err
-		}
-
-		result = append(result, r)
-	}
-
-	return result, columns, nil
-}
-
 // GetLogs parses the given query into the sql syntax, which is then run against the ClickHouse instance. The returned
 // rows are converted into a document schema which can be used by our UI.
 func (i *Instance) GetLogs(ctx context.Context, query, order, orderBy string, maxDocuments, limit, offset, timeStart, timeEnd int64) ([]map[string]interface{}, []string, int64, int64, []Bucket, int64, int64, error) {

plugins/clickhouse/src/components/page/Page.tsx

@@ -1,22 +1,95 @@
-import React from 'react';
+import { PageSection, PageSectionVariants, Title } from '@patternfly/react-core';
+import React, { useEffect, useState } from 'react';
+import { useHistory, useLocation } from 'react-router-dom';
 
+import { IOptions } from '../../utils/interfaces';
 import { IPluginPageProps } from '@kobsio/plugin-core';
-import LogsPage from './LogsPage';
-import SQLPage from './SQLPage';
-
-const Page: React.FunctionComponent<IPluginPageProps> = ({
-  name,
-  displayName,
-  description,
-  options,
-}: IPluginPageProps) => {
-  if (options && options.type && options.type === 'logs') {
-    return <LogsPage name={name} displayName={displayName} description={description} />;
-  } else if (options && options.type && options.type === 'sql') {
-    return <SQLPage name={name} displayName={displayName} description={description} />;
-  }
-
-  return null;
+import Logs from './Logs';
+import LogsToolbar from './LogsToolbar';
+import { getOptionsFromSearch } from '../../utils/helpers';
+
+const Page: React.FunctionComponent<IPluginPageProps> = ({ name, displayName, description }: IPluginPageProps) => {
+  const location = useLocation();
+  const history = useHistory();
+  const [options, setOptions] = useState<IOptions>(getOptionsFromSearch(location.search));
+
+  // changeOptions is used to change the options for an ClickHouse query. Instead of directly modifying the options
+  // state we change the URL parameters.
+  const changeOptions = (opts: IOptions): void => {
+    const fields = opts.fields ? opts.fields.map((field) => `&field=${field}`) : [];
+
+    history.push({
+      pathname: location.pathname,
+      search: `?query=${opts.query}&order=${opts.order}&orderBy=${opts.orderBy}&maxDocuments=${
+        opts.maxDocuments
+      }&time=${opts.times.time}&timeEnd=${opts.times.timeEnd}&timeStart=${opts.times.timeStart}${
+        fields.length > 0 ? fields.join('') : ''
+      }`,
+    });
+  };
+
+  // selectField is used to add a field as parameter, when it isn't present and to remove a fields from as parameter,
+  // when it is already present via the changeOptions function.
+  const selectField = (field: string): void => {
+    let tmpFields: string[] = [];
+    if (options.fields) {
+      tmpFields = [...options.fields];
+    }
+
+    if (tmpFields.includes(field)) {
+      tmpFields = tmpFields.filter((f) => f !== field);
+    } else {
+      tmpFields.push(field);
+    }
+
+    changeOptions({ ...options, fields: tmpFields });
+  };
+
+  // useEffect is used to set the options every time the search location for the current URL changes. The URL is changed
+  // via the changeOptions function. When the search location is changed we modify the options state.
+  useEffect(() => {
+    setOptions(getOptionsFromSearch(location.search));
+  }, [location.search]);
+
+  return (
+    <React.Fragment>
+      <PageSection variant={PageSectionVariants.light}>
+        <Title headingLevel="h6" size="xl">
+          {displayName}
+          <span className="pf-u-font-size-md pf-u-font-weight-normal" style={{ float: 'right' }}>
+            <a href="https://kobs.io/plugins/clickhouse/" target="_blank" rel="noreferrer">
+              Documentation
+            </a>
+          </span>
+        </Title>
+        <p>{description}</p>
+        <LogsToolbar
+          query={options.query}
+          order={options.order}
+          orderBy={options.orderBy}
+          maxDocuments={options.maxDocuments}
+          fields={options.fields}
+          times={options.times}
+          setOptions={changeOptions}
+        />
+      </PageSection>
+
+      <PageSection style={{ minHeight: '100%' }} variant={PageSectionVariants.default}>
+        {options.query.length > 0 ? (
+          <Logs
+            name={name}
+            fields={options.fields}
+            query={options.query}
+            order={options.order}
+            orderBy={options.orderBy}
+            maxDocuments={options.maxDocuments}
+            selectField={selectField}
+            times={options.times}
+          />
+        ) : null}
+      </PageSection>
+    </React.Fragment>
+  );
 };
 
 export default Page;