diff --git a/CHANGELOG.md b/CHANGELOG.md index 284865334..a26d8eb96 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -62,6 +62,7 @@ NOTE: As semantic versioning states all 0.y.z releases can contain breaking chan - [#144](https://github.com/kobsio/kobs/pull/144): Avoid timeouts for long running requests in the ClickHouse plugin. - [#147](https://github.com/kobsio/kobs/pull/147): Improve query performance for ClickHouse plugin and allow custom values for the maximum amount of documents, which should be returned (see [#133](https://github.com/kobsio/kobs/pull/133)). - [#148](https://github.com/kobsio/kobs/pull/148): Improve reliability of kobs, by do not checking the database connection for a configured ClickHouse instance. +- [#150](https://github.com/kobsio/kobs/pull/150): :warning: *Breaking change:* :warning: The ClickHouse plugin can now only be used together with the [kobsio/fluent-bit-clickhouse](https://github.com/kobsio/fluent-bit-clickhouse) output plugin for [Fluent Bit](https://fluentbit.io). For raw SQL queries against a ClickHouse instance the SQL plugin added in [#149](https://github.com/kobsio/kobs/pull/149) can be used. ## [v0.5.0](https://github.com/kobsio/kobs/releases/tag/v0.5.0) (2021-08-03) diff --git a/docs/configuration/plugins.md b/docs/configuration/plugins.md index 5f4a8c433..5b0d1fcb6 100644 --- a/docs/configuration/plugins.md +++ b/docs/configuration/plugins.md @@ -32,6 +32,8 @@ plugins: ## ClickHouse +The ClickHouse plugin provides a user interface for the [kobsio/fluent-bit-clickhouse](https://github.com/kobsio/fluent-bit-clickhouse) Fluent Bit plugin. + The following config can be used to grant kobs access to a ClickHouse instance running at `clickhouse-clickhouse.logging.svc.cluster.local:9000`, where the logs are save in a database named `logs`. To access ClickHouse the user `admin` with the password `admin` is used. ```yaml @@ -43,7 +45,6 @@ plugins: database: logs username: admin password: admin - type: logs ``` | Field | Type | Description | Required | @@ -54,7 +55,6 @@ plugins: | address | string | Address of the ClickHouse instance. | Yes | | username | string | Username to access a ClickHouse instance. | No | | password | string | Password to access a ClickHouse instance. | No | -| type | string | The type which should be used for the ClickHouse instance. This must be `sql` or `logs`. While the `sql` mode allows you to use raw SQL queries, the `logs` mode should be used together with the [kobsio/fluent-bit-clickhouse](https://github.com/kobsio/fluent-bit-clickhouse) plugin to collect logs via Fluent Bit and save them in ClickHouse. | ## Elasticsearch diff --git a/docs/plugins/clickhouse.md b/docs/plugins/clickhouse.md index daa3fd92f..ec5557a3d 100644 --- a/docs/plugins/clickhouse.md +++ b/docs/plugins/clickhouse.md @@ -1,11 +1,9 @@ # ClickHouse -!!! warning - The ClickHouse plugin is in a very early stage and might be changed heavily in the future. +!!! note + The ClickHouse plugin can only be used with the [kobsio/fluent-bit-clickhouse](https://github.com/kobsio/fluent-bit-clickhouse) Fluent Bit plugin. If you want to use kobs to run raw SQL commands against a ClickHouse instance you can use the [SQL plugin](sql.md). -The [ClickHouse](https://clickhouse.tech) plugin can be used to get the data from a configured ClickHouse instance. - -The ClickHouse plugin can be used together with the [kobsio/fluent-bit-clickhouse](https://github.com/kobsio/fluent-bit-clickhouse) output plugin for [Fluent Bit](https://fluentbit.io). For this the `type` in the plugin options must be set to `logs`. You can then use the specified [Query Syntax](#query-syntax) to get the logs from ClickHouse. +The ClickHouse plugin can be used together with the [kobsio/fluent-bit-clickhouse](https://github.com/kobsio/fluent-bit-clickhouse) output plugin for [Fluent Bit](https://fluentbit.io). You can then use the specified [Query Syntax](#query-syntax) to get the logs from ClickHouse. ![Logs](assets/clickhouse-logs.png) @@ -15,8 +13,7 @@ The following options can be used for a panel with the ClickHouse plugin: | Field | Type | Description | Required | | ----- | ---- | ----------- | -------- | -| type | string | Set the type for which you want to use the ClickHouse instance. This must be `sql` or `logs` | Yes | -| showChart | boolean | If this is `true` the chart with the distribution of the Documents over the selected time range will be shown. This option is only available when type is `logs`. | No | +| type | string | Set the type which should be used to visualize your logs. Currently this must be `logs`. | Yes | | queries | [[]Query](#query) | A list of queries, which can be selected by the user. | Yes | ### Query @@ -48,7 +45,6 @@ spec: plugin: name: clickhouse options: - showChart: true queries: - name: Istio Logs query: "namespace='bookinfo' _and_ app='bookinfo' _and_ container_name='istio-proxy' _and_ content.upstream_cluster~'inbound.*'" diff --git a/plugins/clickhouse/clickhouse.go b/plugins/clickhouse/clickhouse.go index 4414b7a53..ac8557c7a 100644 --- a/plugins/clickhouse/clickhouse.go +++ b/plugins/clickhouse/clickhouse.go @@ -25,12 +25,6 @@ var ( // Config is the structure of the configuration for the clickhouse plugin. type Config []instance.Config -type logsResponse struct { - Documents []map[string]interface{} `json:"documents"` - Fields []string `json:"fields"` - Offset int64 `json:"offset"` -} - // Router implements the router for the resources plugin, which can be registered in the router for our rest api. type Router struct { *chi.Mux @@ -48,35 +42,6 @@ func (router *Router) getInstance(name string) *instance.Instance { return nil } -func (router *Router) getSQL(w http.ResponseWriter, r *http.Request) { - name := chi.URLParam(r, "name") - query := r.URL.Query().Get("query") - - log.WithFields(logrus.Fields{"name": name, "query": query}).Tracef("getSQL") - - i := router.getInstance(name) - if i == nil { - errresponse.Render(w, r, nil, http.StatusBadRequest, "Could not find instance name") - return - } - - rows, columns, err := i.GetSQL(r.Context(), query) - if err != nil { - errresponse.Render(w, r, err, http.StatusBadRequest, "Could not get result for SQL query") - return - } - - data := struct { - Rows [][]interface{} `json:"rows"` - Columns []string `json:"columns"` - }{ - rows, - columns, - } - - render.JSON(w, r, data) -} - // getLogs implements the special handling when the user selected the "logs" options for the "view" configuration. This // options is intended to use together with the kobsio/fluent-bit-clickhouse Fluent Bit plugin and provides a custom // query language to get the logs from ClickHouse. @@ -204,16 +169,11 @@ func Register(clusters *clusters.Clusters, plugins *plugin.Plugins, config Confi instances = append(instances, instance) - var options map[string]interface{} - options = make(map[string]interface{}) - options["type"] = cfg.Type - plugins.Append(plugin.Plugin{ Name: cfg.Name, DisplayName: cfg.DisplayName, Description: cfg.Description, Type: "clickhouse", - Options: options, }) } @@ -223,7 +183,6 @@ func Register(clusters *clusters.Clusters, plugins *plugin.Plugins, config Confi instances, } - router.Get("/sql/{name}", router.getSQL) router.Get("/logs/{name}", router.getLogs) return router diff --git a/plugins/clickhouse/pkg/instance/instance.go b/plugins/clickhouse/pkg/instance/instance.go index 72ffc8ca8..8296bf312 100644 --- a/plugins/clickhouse/pkg/instance/instance.go +++ b/plugins/clickhouse/pkg/instance/instance.go @@ -36,41 +36,6 @@ type Instance struct { client *sql.DB } -// GetSQL returns all rows for the user provided SQL query. -func (i *Instance) GetSQL(ctx context.Context, query string) ([][]interface{}, []string, error) { - rows, err := i.client.QueryContext(ctx, query) - if err != nil { - return nil, nil, err - } - defer rows.Close() - - var columns []string - columns, err = rows.Columns() - if err != nil { - return nil, nil, err - } - columnsLen := len(columns) - - var result [][]interface{} - - for rows.Next() { - var r []interface{} - r = make([]interface{}, columnsLen) - - for i := 0; i < columnsLen; i++ { - r[i] = new(interface{}) - } - - if err := rows.Scan(r...); err != nil { - return nil, nil, err - } - - result = append(result, r) - } - - return result, columns, nil -} - // GetLogs parses the given query into the sql syntax, which is then run against the ClickHouse instance. The returned // rows are converted into a document schema which can be used by our UI. func (i *Instance) GetLogs(ctx context.Context, query, order, orderBy string, maxDocuments, limit, offset, timeStart, timeEnd int64) ([]map[string]interface{}, []string, int64, int64, []Bucket, int64, int64, error) { diff --git a/plugins/clickhouse/src/components/page/LogsPage.tsx b/plugins/clickhouse/src/components/page/LogsPage.tsx deleted file mode 100644 index 1f02839f0..000000000 --- a/plugins/clickhouse/src/components/page/LogsPage.tsx +++ /dev/null @@ -1,95 +0,0 @@ -import { PageSection, PageSectionVariants, Title } from '@patternfly/react-core'; -import React, { useEffect, useState } from 'react'; -import { useHistory, useLocation } from 'react-router-dom'; - -import { IOptions } from '../../utils/interfaces'; -import { IPluginPageProps } from '@kobsio/plugin-core'; -import Logs from './Logs'; -import LogsToolbar from './LogsToolbar'; -import { getOptionsFromSearch } from '../../utils/helpers'; - -const LogsPage: React.FunctionComponent = ({ name, displayName, description }: IPluginPageProps) => { - const location = useLocation(); - const history = useHistory(); - const [options, setOptions] = useState(getOptionsFromSearch(location.search)); - - // changeOptions is used to change the options for an ClickHouse query. Instead of directly modifying the options - // state we change the URL parameters. - const changeOptions = (opts: IOptions): void => { - const fields = opts.fields ? opts.fields.map((field) => `&field=${field}`) : []; - - history.push({ - pathname: location.pathname, - search: `?query=${opts.query}&order=${opts.order}&orderBy=${opts.orderBy}&maxDocuments=${ - opts.maxDocuments - }&time=${opts.times.time}&timeEnd=${opts.times.timeEnd}&timeStart=${opts.times.timeStart}${ - fields.length > 0 ? fields.join('') : '' - }`, - }); - }; - - // selectField is used to add a field as parameter, when it isn't present and to remove a fields from as parameter, - // when it is already present via the changeOptions function. - const selectField = (field: string): void => { - let tmpFields: string[] = []; - if (options.fields) { - tmpFields = [...options.fields]; - } - - if (tmpFields.includes(field)) { - tmpFields = tmpFields.filter((f) => f !== field); - } else { - tmpFields.push(field); - } - - changeOptions({ ...options, fields: tmpFields }); - }; - - // useEffect is used to set the options every time the search location for the current URL changes. The URL is changed - // via the changeOptions function. When the search location is changed we modify the options state. - useEffect(() => { - setOptions(getOptionsFromSearch(location.search)); - }, [location.search]); - - return ( - - - - {displayName} - <span className="pf-u-font-size-md pf-u-font-weight-normal" style={{ float: 'right' }}> - <a href="https://kobs.io/plugins/clickhouse/" target="_blank" rel="noreferrer"> - Documentation - </a> - </span> - -

{description}

- - - - - {options.query.length > 0 ? ( - - ) : null} - - - ); -}; - -export default LogsPage; diff --git a/plugins/clickhouse/src/components/page/Page.tsx b/plugins/clickhouse/src/components/page/Page.tsx index 6aa0f4799..4903e2272 100644 --- a/plugins/clickhouse/src/components/page/Page.tsx +++ b/plugins/clickhouse/src/components/page/Page.tsx @@ -1,22 +1,95 @@ -import React from 'react'; +import { PageSection, PageSectionVariants, Title } from '@patternfly/react-core'; +import React, { useEffect, useState } from 'react'; +import { useHistory, useLocation } from 'react-router-dom'; +import { IOptions } from '../../utils/interfaces'; import { IPluginPageProps } from '@kobsio/plugin-core'; -import LogsPage from './LogsPage'; -import SQLPage from './SQLPage'; - -const Page: React.FunctionComponent = ({ - name, - displayName, - description, - options, -}: IPluginPageProps) => { - if (options && options.type && options.type === 'logs') { - return ; - } else if (options && options.type && options.type === 'sql') { - return ; - } - - return null; +import Logs from './Logs'; +import LogsToolbar from './LogsToolbar'; +import { getOptionsFromSearch } from '../../utils/helpers'; + +const Page: React.FunctionComponent = ({ name, displayName, description }: IPluginPageProps) => { + const location = useLocation(); + const history = useHistory(); + const [options, setOptions] = useState(getOptionsFromSearch(location.search)); + + // changeOptions is used to change the options for an ClickHouse query. Instead of directly modifying the options + // state we change the URL parameters. + const changeOptions = (opts: IOptions): void => { + const fields = opts.fields ? opts.fields.map((field) => `&field=${field}`) : []; + + history.push({ + pathname: location.pathname, + search: `?query=${opts.query}&order=${opts.order}&orderBy=${opts.orderBy}&maxDocuments=${ + opts.maxDocuments + }&time=${opts.times.time}&timeEnd=${opts.times.timeEnd}&timeStart=${opts.times.timeStart}${ + fields.length > 0 ? fields.join('') : '' + }`, + }); + }; + + // selectField is used to add a field as parameter, when it isn't present and to remove a fields from as parameter, + // when it is already present via the changeOptions function. + const selectField = (field: string): void => { + let tmpFields: string[] = []; + if (options.fields) { + tmpFields = [...options.fields]; + } + + if (tmpFields.includes(field)) { + tmpFields = tmpFields.filter((f) => f !== field); + } else { + tmpFields.push(field); + } + + changeOptions({ ...options, fields: tmpFields }); + }; + + // useEffect is used to set the options every time the search location for the current URL changes. The URL is changed + // via the changeOptions function. When the search location is changed we modify the options state. + useEffect(() => { + setOptions(getOptionsFromSearch(location.search)); + }, [location.search]); + + return ( + + + + {displayName} + <span className="pf-u-font-size-md pf-u-font-weight-normal" style={{ float: 'right' }}> + <a href="https://kobs.io/plugins/clickhouse/" target="_blank" rel="noreferrer"> + Documentation + </a> + </span> + +

{description}

+ + + + + {options.query.length > 0 ? ( + + ) : null} + + + ); }; export default Page; diff --git a/plugins/clickhouse/src/components/page/SQL.tsx b/plugins/clickhouse/src/components/page/SQL.tsx deleted file mode 100644 index 4e7311e5b..000000000 --- a/plugins/clickhouse/src/components/page/SQL.tsx +++ /dev/null @@ -1,79 +0,0 @@ -import { Alert, AlertActionLink, AlertVariant, Card, Spinner } from '@patternfly/react-core'; -import { QueryObserverResult, useQuery } from 'react-query'; -import React from 'react'; -import { useHistory } from 'react-router-dom'; - -import { ISQLData } from '../../utils/interfaces'; -import SQLTable from '../panel/SQLTable'; - -interface ISQLProps { - name: string; - query: string; -} - -const SQL: React.FunctionComponent = ({ name, query }: ISQLProps) => { - const history = useHistory(); - - const { isError, isFetching, error, data, refetch } = useQuery( - ['clickhouse/sql', query], - async () => { - try { - const response = await fetch(`/api/plugins/clickhouse/sql/${name}?query=${encodeURIComponent(query)}`, { - method: 'get', - }); - const json = await response.json(); - - if (response.status >= 200 && response.status < 300) { - return json; - } else { - if (json.error) { - throw new Error(json.error); - } else { - throw new Error('An unknown error occured'); - } - } - } catch (err) { - throw err; - } - }, - ); - - if (isFetching) { - return ( -
- -
- ); - } - - if (isError) { - return ( - - history.push('/')}>Home - > => refetch()}> - Retry - - - } - > -

{error?.message}

- - ); - } - - if (!data) { - return null; - } - - return ( - - - - ); -}; - -export default SQL; diff --git a/plugins/clickhouse/src/components/page/SQLPage.tsx b/plugins/clickhouse/src/components/page/SQLPage.tsx deleted file mode 100644 index 84136e8aa..000000000 --- a/plugins/clickhouse/src/components/page/SQLPage.tsx +++ /dev/null @@ -1,47 +0,0 @@ -import { PageSection, PageSectionVariants, Title } from '@patternfly/react-core'; -import React, { useEffect, useState } from 'react'; -import { useHistory, useLocation } from 'react-router-dom'; - -import { IPluginPageProps } from '@kobsio/plugin-core'; -import SQL from './SQL'; -import SQLToolbar from './SQLToolbar'; -import { getQueryFromSearch } from '../../utils/helpers'; - -const SQLPage: React.FunctionComponent = ({ name, displayName, description }: IPluginPageProps) => { - const location = useLocation(); - const history = useHistory(); - const [query, setQuery] = useState(getQueryFromSearch(location.search)); - - // changeOptions is used to change the options for an ClickHouse query. Instead of directly modifying the options - // state we change the URL parameters. - const changeOptions = (q: string): void => { - history.push({ - pathname: location.pathname, - search: `?query=${q}`, - }); - }; - - // useEffect is used to set the options every time the search location for the current URL changes. The URL is changed - // via the changeOptions function. When the search location is changed we modify the options state. - useEffect(() => { - setQuery(getQueryFromSearch(location.search)); - }, [location.search]); - - return ( - - - - {displayName} - -

{description}

- - - - - {query.length > 0 && } - - - ); -}; - -export default SQLPage; diff --git a/plugins/clickhouse/src/components/page/SQLToolbar.tsx b/plugins/clickhouse/src/components/page/SQLToolbar.tsx deleted file mode 100644 index 8506a6522..000000000 --- a/plugins/clickhouse/src/components/page/SQLToolbar.tsx +++ /dev/null @@ -1,65 +0,0 @@ -import { - Button, - ButtonVariant, - TextArea, - Toolbar, - ToolbarContent, - ToolbarGroup, - ToolbarItem, - ToolbarToggleGroup, -} from '@patternfly/react-core'; -import { FilterIcon, SearchIcon } from '@patternfly/react-icons'; -import React, { useState } from 'react'; - -interface ISQLToolbarProps { - query: string; - setQuery: (data: string) => void; -} - -const SQLToolbar: React.FunctionComponent = ({ query, setQuery }: ISQLToolbarProps) => { - const [data, setData] = useState(query); - - // changeQuery changes the value of a query. - const changeQuery = (value: string): void => { - setData(value); - }; - - // onEnter is used to detect if the user pressed the "ENTER" key. If this is the case we are calling the setOptions - // function to trigger the search. - // use "SHIFT" + "ENTER". - const onEnter = (e: React.KeyboardEvent | undefined): void => { - if (e?.key === 'Enter' && !e.shiftKey) { - e.preventDefault(); - setQuery(data); - } - }; - - return ( - - - } breakpoint="lg"> - - -