From a73652c2bbc0ab388894a843c2553fb38bfe4f39 Mon Sep 17 00:00:00 2001 From: ricoberger Date: Tue, 5 Oct 2021 20:47:34 +0200 Subject: [PATCH] Add support to visualize logs in ClickHouse It is now possible to visualize logs from ClickHouse via bar or pie charts. For that a new link to the visualization page was added to the logs page. To generate a pie or bar chart a user must provide a query and a time range. Besides these two option a user can also select the operation which should be used for the visualization, the field on which this operation should be run, a field to group the results, a sorting order and a limit. NOTE: Currently the user must now if he uses a string or number field. Maybe we can get a list of fields from ClickHouse in the future and the let the user select the field via a select box, where we also now if the field is a number or a string. --- CHANGELOG.md | 1 + plugins/clickhouse/clickhouse.go | 78 +++++++++++ plugins/clickhouse/package.json | 3 + plugins/clickhouse/pkg/instance/instance.go | 47 +++++++ plugins/clickhouse/pkg/instance/structs.go | 6 + .../clickhouse/pkg/instance/visualization.go | 37 ++++++ .../pkg/instance/visualization_test.go | 40 ++++++ .../clickhouse/src/components/page/Logs.tsx | 10 +- .../src/components/page/LogsPage.tsx | 120 +++++++++++++++++ .../clickhouse/src/components/page/Page.tsx | 111 ++-------------- .../src/components/page/Visualization.tsx | 92 +++++++++++++ .../components/page/VisualizationOptions.tsx | 125 ++++++++++++++++++ .../src/components/page/VisualizationPage.tsx | 93 +++++++++++++ .../components/page/VisualizationToolbar.tsx | 64 +++++++++ .../components/panel/VisualizationChart.tsx | 27 ++++ .../panel/VisualizationChartBar.tsx | 99 ++++++++++++++ .../panel/VisualizationChartPie.tsx | 74 +++++++++++ plugins/clickhouse/src/utils/colors.ts | 61 +++++++++ plugins/clickhouse/src/utils/helpers.ts | 37 +++++- plugins/clickhouse/src/utils/interfaces.ts | 18 +++ yarn.lock | 20 +++ 21 files changed, 1059 insertions(+), 104 deletions(-) create mode 100644 plugins/clickhouse/pkg/instance/visualization.go create mode 100644 plugins/clickhouse/pkg/instance/visualization_test.go create mode 100644 plugins/clickhouse/src/components/page/LogsPage.tsx create mode 100644 plugins/clickhouse/src/components/page/Visualization.tsx create mode 100644 plugins/clickhouse/src/components/page/VisualizationOptions.tsx create mode 100644 plugins/clickhouse/src/components/page/VisualizationPage.tsx create mode 100644 plugins/clickhouse/src/components/page/VisualizationToolbar.tsx create mode 100644 plugins/clickhouse/src/components/panel/VisualizationChart.tsx create mode 100644 plugins/clickhouse/src/components/panel/VisualizationChartBar.tsx create mode 100644 plugins/clickhouse/src/components/panel/VisualizationChartPie.tsx create mode 100644 plugins/clickhouse/src/utils/colors.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index 838ff42f6..a944b16b0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -33,6 +33,7 @@ NOTE: As semantic versioning states all 0.y.z releases can contain breaking chan - [#159](https://github.com/kobsio/kobs/pull/159): Allow users to select a time range within the logs chart in the ClickHouse plugin. - [#160](https://github.com/kobsio/kobs/pull/160): Allow users to sort the returned logs within the documents table in the ClickHouse plugin. - [#161](https://github.com/kobsio/kobs/pull/161): Add support for materialized columns, to improve query performance for most frequently queried field. +- [#162](https://github.com/kobsio/kobs/pull/162): Add support to visualize logs in the ClickHouse plugin. ### Fixed diff --git a/plugins/clickhouse/clickhouse.go b/plugins/clickhouse/clickhouse.go index b1d71b2fd..0fde20378 100644 --- a/plugins/clickhouse/clickhouse.go +++ b/plugins/clickhouse/clickhouse.go @@ -129,6 +129,83 @@ func (router *Router) getLogs(w http.ResponseWriter, r *http.Request) { render.JSON(w, r, data) } +// getVisualization runs an aggregation for the given values and returns an array of data point which can be used in a +// chart in the React UI. All fields except the limit, start and end time must be strings. The mentioned fields must be +// numbers so we can parse them. If the groupBy field isn't present we use the operationField as groupBy field. +func (router *Router) getVisualization(w http.ResponseWriter, r *http.Request) { + name := chi.URLParam(r, "name") + groupBy := r.URL.Query().Get("groupBy") + limit := r.URL.Query().Get("limit") + order := r.URL.Query().Get("order") + operation := r.URL.Query().Get("operation") + operationField := r.URL.Query().Get("operationField") + query := r.URL.Query().Get("query") + timeStart := r.URL.Query().Get("timeStart") + timeEnd := r.URL.Query().Get("timeEnd") + + log.WithFields(logrus.Fields{"name": name, "query": query, "groupBy": groupBy, "limit": limit, "operation": operation, "operationField": operationField, "order": order, "timeStart": timeStart, "timeEnd": timeEnd}).Tracef("getLogs") + + i := router.getInstance(name) + if i == nil { + errresponse.Render(w, r, nil, http.StatusBadRequest, "Could not find instance name") + return + } + + parsedTimeStart, err := strconv.ParseInt(timeStart, 10, 64) + if err != nil { + errresponse.Render(w, r, err, http.StatusBadRequest, "Could not parse start time") + return + } + + parsedTimeEnd, err := strconv.ParseInt(timeEnd, 10, 64) + if err != nil { + errresponse.Render(w, r, err, http.StatusBadRequest, "Could not parse end time") + return + } + + parsedLimit, err := strconv.ParseInt(limit, 10, 64) + if err != nil { + errresponse.Render(w, r, err, http.StatusBadRequest, "Could not parse limit") + return + } + + if groupBy == "" { + groupBy = operationField + } + + done := make(chan bool) + + go func() { + ticker := time.NewTicker(10 * time.Second) + defer ticker.Stop() + + for { + select { + case <-done: + return + case <-ticker.C: + if f, ok := w.(http.Flusher); ok { + // w.WriteHeader(http.StatusProcessing) + w.Write([]byte("\n")) + f.Flush() + } + } + } + }() + + defer func() { + done <- true + }() + + data, err := i.GetVisualization(r.Context(), parsedLimit, groupBy, operation, operationField, order, query, parsedTimeStart, parsedTimeEnd) + if err != nil { + errresponse.Render(w, r, err, http.StatusBadRequest, "Could not get logs") + return + } + + render.JSON(w, r, data) +} + // Register returns a new router which can be used in the router for the kobs rest api. func Register(clusters *clusters.Clusters, plugins *plugin.Plugins, config Config) chi.Router { var instances []*instance.Instance @@ -156,6 +233,7 @@ func Register(clusters *clusters.Clusters, plugins *plugin.Plugins, config Confi } router.Get("/logs/{name}", router.getLogs) + router.Get("/visualization/{name}", router.getVisualization) return router } diff --git a/plugins/clickhouse/package.json b/plugins/clickhouse/package.json index a3138ffa4..ffcb86d42 100644 --- a/plugins/clickhouse/package.json +++ b/plugins/clickhouse/package.json @@ -11,6 +11,9 @@ }, "dependencies": { "@kobsio/plugin-core": "*", + "@nivo/bar": "^0.73.1", + "@nivo/pie": "^0.73.0", + "@nivo/tooltip": "^0.73.0", "@patternfly/react-charts": "^6.15.23", "@patternfly/react-core": "^4.128.2", "@patternfly/react-icons": "^4.10.11", diff --git a/plugins/clickhouse/pkg/instance/instance.go b/plugins/clickhouse/pkg/instance/instance.go index 199a3ab0a..b82fc1c7b 100644 --- a/plugins/clickhouse/pkg/instance/instance.go +++ b/plugins/clickhouse/pkg/instance/instance.go @@ -198,6 +198,53 @@ func (i *Instance) GetLogs(ctx context.Context, query, order, orderBy string, ti return documents, fields, count, time.Now().Sub(queryStartTime).Milliseconds(), buckets, nil } +// GetVisualization build an aggregation query for the given parameters and returns the result as slice of label, value +// pairs. +func (i *Instance) GetVisualization(ctx context.Context, limit int64, groupBy, operation, operationField, order, query string, timeStart, timeEnd int64) ([]VisualizationRow, error) { + var data []VisualizationRow + + // As we also do it for the logs query we have to build our where condition for the SQL query first. + whereConditions := "" + if query != "" { + parsedQuery, err := parseLogsQuery(query, i.materializedColumns) + if err != nil { + return nil, err + } + + whereConditions = fmt.Sprintf("timestamp >= FROM_UNIXTIME(%d) AND timestamp <= FROM_UNIXTIME(%d) AND %s", timeStart, timeEnd, parsedQuery) + } + + // Now we have to transform all the given fields / values into a format, which we can use in our SQL query. This + // query is built in the following and then run against ClickHouse. All the returned rows are added to our data + // slice and returned, so that we can used it later in the React UI. + groupBy = formatField(groupBy, i.materializedColumns) + operationField = formatField(operationField, i.materializedColumns) + order = formatOrder(order) + + sql := fmt.Sprintf("SELECT %s as label, %s(%s) as value FROM %s.logs WHERE %s GROUP BY %s ORDER BY value %s LIMIT %d SETTINGS skip_unavailable_shards = 1", groupBy, operation, operationField, i.database, whereConditions, groupBy, order, limit) + log.WithFields(logrus.Fields{"query": sql}).Tracef("sql query visualization") + rows, err := i.client.QueryContext(ctx, sql) + if err != nil { + return nil, err + } + defer rows.Close() + + for rows.Next() { + var r VisualizationRow + if err := rows.Scan(&r.Label, &r.Value); err != nil { + return nil, err + } + + data = append(data, r) + } + + if err := rows.Err(); err != nil { + return nil, err + } + + return data, nil +} + // New returns a new ClickHouse instance for the given configuration. func New(config Config) (*Instance, error) { if config.WriteTimeout == "" { diff --git a/plugins/clickhouse/pkg/instance/structs.go b/plugins/clickhouse/pkg/instance/structs.go index f244202bc..1901fc4ba 100644 --- a/plugins/clickhouse/pkg/instance/structs.go +++ b/plugins/clickhouse/pkg/instance/structs.go @@ -36,3 +36,9 @@ type Bucket struct { Interval int64 `json:"interval"` Count int64 `json:"count"` } + +// VisualizationRow is the structure of a single row for a visualization. +type VisualizationRow struct { + Label string `json:"label"` + Value float64 `json:"value"` +} diff --git a/plugins/clickhouse/pkg/instance/visualization.go b/plugins/clickhouse/pkg/instance/visualization.go new file mode 100644 index 000000000..8740b61a8 --- /dev/null +++ b/plugins/clickhouse/pkg/instance/visualization.go @@ -0,0 +1,37 @@ +package instance + +import ( + "fmt" + "strings" +) + +// formatField returns the SQL syntax for the given field. If the field is of type string and not in the default fields +// or materialized columns list it must be wrapped in single quotes. +func formatField(field string, materializedColumns []string) string { + field = strings.TrimSpace(field) + + if contains(defaultFields, field) || contains(materializedColumns, field) { + return field + } + + if string(field[0]) == "'" && string(field[len(field)-1]) == "'" { + field = field[1 : len(field)-1] + + if contains(defaultFields, field) || contains(materializedColumns, field) { + return field + } + + return fmt.Sprintf("fields_string.value[indexOf(fields_string.key, '%s')]", field) + } + + return fmt.Sprintf("fields_number.value[indexOf(fields_number.key, '%s')]", field) +} + +// formatOrder returns the order key word which can be used in the SQL query for the given input. +func formatOrder(order string) string { + if order == "descending" { + return "DESC" + } + + return "ASC" +} diff --git a/plugins/clickhouse/pkg/instance/visualization_test.go b/plugins/clickhouse/pkg/instance/visualization_test.go new file mode 100644 index 000000000..1d6d2ee41 --- /dev/null +++ b/plugins/clickhouse/pkg/instance/visualization_test.go @@ -0,0 +1,40 @@ +package instance + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +func TestFormatField(t *testing.T) { + for _, tc := range []struct { + field string + expect string + }{ + {field: "namespace", expect: "namespace"}, + {field: "'namespace'", expect: "namespace"}, + {field: "'content.method'", expect: "fields_string.value[indexOf(fields_string.key, 'content.method')]"}, + {field: "content.duration", expect: "fields_number.value[indexOf(fields_number.key, 'content.duration')]"}, + } { + t.Run(tc.field, func(t *testing.T) { + actual := formatField(tc.field, nil) + require.Equal(t, tc.expect, actual) + }) + } +} + +func TestFormatOrder(t *testing.T) { + for _, tc := range []struct { + order string + expect string + }{ + {order: "descending", expect: "DESC"}, + {order: "ascending", expect: "ASC"}, + {order: "foo bar", expect: "ASC"}, + } { + t.Run(tc.order, func(t *testing.T) { + actual := formatOrder(tc.order) + require.Equal(t, tc.expect, actual) + }) + } +} diff --git a/plugins/clickhouse/src/components/page/Logs.tsx b/plugins/clickhouse/src/components/page/Logs.tsx index c8f090932..5fdd12587 100644 --- a/plugins/clickhouse/src/components/page/Logs.tsx +++ b/plugins/clickhouse/src/components/page/Logs.tsx @@ -22,7 +22,7 @@ import LogsChart from '../panel/LogsChart'; import LogsDocuments from '../panel/LogsDocuments'; import LogsFields from './LogsFields'; -interface IPageLogsProps { +interface ILogsProps { name: string; fields?: string[]; order: string; @@ -35,7 +35,7 @@ interface IPageLogsProps { times: IPluginTimes; } -const PageLogs: React.FunctionComponent = ({ +const Logs: React.FunctionComponent = ({ name, fields, order, @@ -46,11 +46,11 @@ const PageLogs: React.FunctionComponent = ({ changeOrder, selectField, times, -}: IPageLogsProps) => { +}: ILogsProps) => { const history = useHistory(); const { isError, isFetching, isLoading, data, error, refetch } = useQuery( - ['clickhouse/logs', query, order, orderBy, times], + ['clickhouse/logs', name, query, order, orderBy, times], async () => { try { const response = await fetch( @@ -165,4 +165,4 @@ const PageLogs: React.FunctionComponent = ({ ); }; -export default PageLogs; +export default Logs; diff --git a/plugins/clickhouse/src/components/page/LogsPage.tsx b/plugins/clickhouse/src/components/page/LogsPage.tsx new file mode 100644 index 000000000..07b3bfed3 --- /dev/null +++ b/plugins/clickhouse/src/components/page/LogsPage.tsx @@ -0,0 +1,120 @@ +import { Divider, Flex, FlexItem, PageSection, PageSectionVariants, Title } from '@patternfly/react-core'; +import { Link, useHistory, useLocation } from 'react-router-dom'; +import React, { useEffect, useState } from 'react'; + +import { IOptions } from '../../utils/interfaces'; +import { IPluginTimes } from '@kobsio/plugin-core'; +import Logs from './Logs'; +import LogsToolbar from './LogsToolbar'; +import { getOptionsFromSearch } from '../../utils/helpers'; + +interface ILogsPageProps { + name: string; + displayName: string; + description: string; +} + +const LogsPage: React.FunctionComponent = ({ name, displayName, description }: ILogsPageProps) => { + const location = useLocation(); + const history = useHistory(); + const [options, setOptions] = useState(getOptionsFromSearch(location.search)); + + // changeOptions is used to change the options for an ClickHouse query. Instead of directly modifying the options + // state we change the URL parameters. + const changeOptions = (opts: IOptions): void => { + const fields = opts.fields ? opts.fields.map((field) => `&field=${field}`) : []; + + history.push({ + pathname: location.pathname, + search: `?query=${opts.query}&order=${opts.order}&orderBy=${opts.orderBy}&time=${opts.times.time}&timeEnd=${ + opts.times.timeEnd + }&timeStart=${opts.times.timeStart}${fields.length > 0 ? fields.join('') : ''}`, + }); + }; + + // selectField is used to add a field as parameter, when it isn't present and to remove a fields from as parameter, + // when it is already present via the changeOptions function. + const selectField = (field: string): void => { + let tmpFields: string[] = []; + if (options.fields) { + tmpFields = [...options.fields]; + } + + if (tmpFields.includes(field)) { + tmpFields = tmpFields.filter((f) => f !== field); + } else { + tmpFields.push(field); + } + + changeOptions({ ...options, fields: tmpFields }); + }; + + const addFilter = (filter: string): void => { + changeOptions({ ...options, query: `${options.query} ${filter}` }); + }; + + const changeTime = (times: IPluginTimes): void => { + changeOptions({ ...options, times: times }); + }; + + const changeOrder = (order: string, orderBy: string): void => { + changeOptions({ ...options, order: order, orderBy: orderBy }); + }; + + // useEffect is used to set the options every time the search location for the current URL changes. The URL is changed + // via the changeOptions function. When the search location is changed we modify the options state. + useEffect(() => { + setOptions(getOptionsFromSearch(location.search)); + }, [location.search]); + + return ( + + + + {displayName} + <span className="pf-u-font-size-md pf-u-font-weight-normal" style={{ float: 'right' }}> + <Flex> + <FlexItem> + <Link to={`/${name}/visualization`}>Visualization</Link> + </FlexItem> + <Divider isVertical={true} /> + <FlexItem> + <a href="https://kobs.io/plugins/clickhouse/" target="_blank" rel="noreferrer"> + Documentation + </a> + </FlexItem> + </Flex> + </span> + +

{description}

+ + + + + {options.query.length > 0 ? ( + + ) : null} + + + ); +}; + +export default LogsPage; diff --git a/plugins/clickhouse/src/components/page/Page.tsx b/plugins/clickhouse/src/components/page/Page.tsx index 9087a9d66..b41a917a9 100644 --- a/plugins/clickhouse/src/components/page/Page.tsx +++ b/plugins/clickhouse/src/components/page/Page.tsx @@ -1,105 +1,20 @@ -import { PageSection, PageSectionVariants, Title } from '@patternfly/react-core'; -import React, { useEffect, useState } from 'react'; -import { useHistory, useLocation } from 'react-router-dom'; +import { Route, Switch } from 'react-router-dom'; +import React from 'react'; -import { IPluginPageProps, IPluginTimes } from '@kobsio/plugin-core'; -import { IOptions } from '../../utils/interfaces'; -import Logs from './Logs'; -import LogsToolbar from './LogsToolbar'; -import { getOptionsFromSearch } from '../../utils/helpers'; +import { IPluginPageProps } from '@kobsio/plugin-core'; +import LogsPage from './LogsPage'; +import VisualizationPage from './VisualizationPage'; const Page: React.FunctionComponent = ({ name, displayName, description }: IPluginPageProps) => { - const location = useLocation(); - const history = useHistory(); - const [options, setOptions] = useState(getOptionsFromSearch(location.search)); - - // changeOptions is used to change the options for an ClickHouse query. Instead of directly modifying the options - // state we change the URL parameters. - const changeOptions = (opts: IOptions): void => { - const fields = opts.fields ? opts.fields.map((field) => `&field=${field}`) : []; - - history.push({ - pathname: location.pathname, - search: `?query=${opts.query}&order=${opts.order}&orderBy=${opts.orderBy}&time=${opts.times.time}&timeEnd=${ - opts.times.timeEnd - }&timeStart=${opts.times.timeStart}${fields.length > 0 ? fields.join('') : ''}`, - }); - }; - - // selectField is used to add a field as parameter, when it isn't present and to remove a fields from as parameter, - // when it is already present via the changeOptions function. - const selectField = (field: string): void => { - let tmpFields: string[] = []; - if (options.fields) { - tmpFields = [...options.fields]; - } - - if (tmpFields.includes(field)) { - tmpFields = tmpFields.filter((f) => f !== field); - } else { - tmpFields.push(field); - } - - changeOptions({ ...options, fields: tmpFields }); - }; - - const addFilter = (filter: string): void => { - changeOptions({ ...options, query: `${options.query} ${filter}` }); - }; - - const changeTime = (times: IPluginTimes): void => { - changeOptions({ ...options, times: times }); - }; - - const changeOrder = (order: string, orderBy: string): void => { - changeOptions({ ...options, order: order, orderBy: orderBy }); - }; - - // useEffect is used to set the options every time the search location for the current URL changes. The URL is changed - // via the changeOptions function. When the search location is changed we modify the options state. - useEffect(() => { - setOptions(getOptionsFromSearch(location.search)); - }, [location.search]); - return ( - - - - {displayName} - <span className="pf-u-font-size-md pf-u-font-weight-normal" style={{ float: 'right' }}> - <a href="https://kobs.io/plugins/clickhouse/" target="_blank" rel="noreferrer"> - Documentation - </a> - </span> - -

{description}

- - - - - {options.query.length > 0 ? ( - - ) : null} - - + + + + + + + + ); }; diff --git a/plugins/clickhouse/src/components/page/Visualization.tsx b/plugins/clickhouse/src/components/page/Visualization.tsx new file mode 100644 index 000000000..51a47a728 --- /dev/null +++ b/plugins/clickhouse/src/components/page/Visualization.tsx @@ -0,0 +1,92 @@ +import { Alert, AlertActionLink, AlertVariant, Card, CardBody, Spinner } from '@patternfly/react-core'; +import { QueryObserverResult, useQuery } from 'react-query'; +import React from 'react'; +import { useHistory } from 'react-router-dom'; + +import { IVisualizationData, IVisualizationOptions } from '../../utils/interfaces'; +import VisualizationChart from '../panel/VisualizationChart'; + +interface IVisualizationProps { + name: string; + options: IVisualizationOptions; +} + +const Visualization: React.FunctionComponent = ({ name, options }: IVisualizationProps) => { + const history = useHistory(); + + const { isError, isLoading, data, error, refetch } = useQuery( + ['clickhouse/visualization', name, options], + async () => { + try { + const response = await fetch( + `/api/plugins/clickhouse/visualization/${name}?query=${encodeURIComponent(options.query)}&timeStart=${ + options.times.timeStart + }&timeEnd=${options.times.timeEnd}&limit=${options.limit}&groupBy=${options.groupBy}&operation=${ + options.operation + }&operationField=${options.operationField}&order=${options.order}`, + { + method: 'get', + }, + ); + const json = await response.json(); + + if (response.status >= 200 && response.status < 300) { + if (json.error) { + throw new Error(json.error); + } + + return json; + } else { + if (json.error) { + throw new Error(json.error); + } else { + throw new Error('An unknown error occured'); + } + } + } catch (err) { + throw err; + } + }, + ); + + if (isLoading) { + return ( +
+ +
+ ); + } + + if (isError) { + return ( + + history.push('/')}>Home + > => refetch()}> + Retry + + + } + > +

{error?.message}

+ + ); + } + + if (!data) { + return null; + } + + return ( + + + + + + ); +}; + +export default Visualization; diff --git a/plugins/clickhouse/src/components/page/VisualizationOptions.tsx b/plugins/clickhouse/src/components/page/VisualizationOptions.tsx new file mode 100644 index 000000000..86c8e3fab --- /dev/null +++ b/plugins/clickhouse/src/components/page/VisualizationOptions.tsx @@ -0,0 +1,125 @@ +import { + Button, + Card, + CardBody, + Form, + FormGroup, + FormSelect, + FormSelectOption, + TextInput, +} from '@patternfly/react-core'; +import React from 'react'; + +import { IVisualizationOptions } from '../../utils/interfaces'; + +interface IVisualizationOptionsProps { + options: IVisualizationOptions; + setOptions: (data: IVisualizationOptions) => void; + changeOptions: () => void; +} + +const VisualizationOptions: React.FunctionComponent = ({ + options, + setOptions, + changeOptions, +}: IVisualizationOptionsProps) => { + return ( + + +
+ + setOptions({ ...options, operationField: value })} + /> + + + + setOptions({ ...options, operation: value })} + id="vis-options-operation" + name="vis-options-operation" + aria-label="Operation" + > + + + + + + + + + + setOptions({ ...options, groupBy: value })} + /> + + + + setOptions({ ...options, order: value })} + id="vis-options-order" + name="vis-options-order" + aria-label="Order" + > + + + + + + + setOptions({ ...options, limit: value })} + /> + + + + setOptions({ ...options, chart: value })} + id="vis-options-chart" + name="vis-options-chart" + aria-label="Chart" + > + + + + + + +
+ + + ); +}; + +export default VisualizationOptions; diff --git a/plugins/clickhouse/src/components/page/VisualizationPage.tsx b/plugins/clickhouse/src/components/page/VisualizationPage.tsx new file mode 100644 index 000000000..d366f7125 --- /dev/null +++ b/plugins/clickhouse/src/components/page/VisualizationPage.tsx @@ -0,0 +1,93 @@ +import { + Divider, + Flex, + FlexItem, + Grid, + GridItem, + PageSection, + PageSectionVariants, + Title, +} from '@patternfly/react-core'; +import { Link, useHistory, useLocation } from 'react-router-dom'; +import React, { useEffect, useState } from 'react'; + +import { IVisualizationOptions } from '../../utils/interfaces'; +import Visualization from './Visualization'; +import VisualizationOptions from './VisualizationOptions'; +import VisualizationToolbar from './VisualizationToolbar'; +import { getVisualizationOptionsFromSearch } from '../../utils/helpers'; + +interface IVisualizationPageProps { + name: string; + displayName: string; + description: string; +} + +const VisualizationPage: React.FunctionComponent = ({ + name, + displayName, + description, +}: IVisualizationPageProps) => { + const location = useLocation(); + const history = useHistory(); + const [tmpOptions, setTmpOptions] = useState( + getVisualizationOptionsFromSearch(location.search), + ); + const [options, setOptions] = useState(getVisualizationOptionsFromSearch(location.search)); + + // changeOptions is used to change the options for an ClickHouse query. Instead of directly modifying the options + // state we change the URL parameters. + const changeOptions = (): void => { + history.push({ + pathname: location.pathname, + search: `?query=${tmpOptions.query}&time=${tmpOptions.times.time}&timeEnd=${tmpOptions.times.timeEnd}&timeStart=${tmpOptions.times.timeStart}&chart=${tmpOptions.chart}&limit=${tmpOptions.limit}&groupBy=${tmpOptions.groupBy}&operation=${tmpOptions.operation}&operationField=${tmpOptions.operationField}&order=${tmpOptions.order}`, + }); + }; + + // useEffect is used to set the options every time the search location for the current URL changes. The URL is changed + // via the changeOptions function. When the search location is changed we modify the options state. + useEffect(() => { + setOptions(getVisualizationOptionsFromSearch(location.search)); + }, [location.search]); + + return ( + + + + {displayName} + <span className="pf-u-font-size-md pf-u-font-weight-normal" style={{ float: 'right' }}> + <Flex> + <FlexItem> + <Link to={`/${name}`}>Logs</Link> + </FlexItem> + <Divider isVertical={true} /> + <FlexItem> + <a href="https://kobs.io/plugins/clickhouse/" target="_blank" rel="noreferrer"> + Documentation + </a> + </FlexItem> + </Flex> + </span> + +

{description}

+ + + + + + + + + + {options.query !== '' && options.operationField !== '' ? ( + + ) : null} + +

 

+ + + + ); +}; + +export default VisualizationPage; diff --git a/plugins/clickhouse/src/components/page/VisualizationToolbar.tsx b/plugins/clickhouse/src/components/page/VisualizationToolbar.tsx new file mode 100644 index 000000000..ce5b6c06a --- /dev/null +++ b/plugins/clickhouse/src/components/page/VisualizationToolbar.tsx @@ -0,0 +1,64 @@ +import { + TextInput, + Toolbar, + ToolbarContent, + ToolbarGroup, + ToolbarItem, + ToolbarToggleGroup, +} from '@patternfly/react-core'; +import { FilterIcon } from '@patternfly/react-icons'; +import React from 'react'; + +import { IOptionsAdditionalFields, Options, TTime } from '@kobsio/plugin-core'; +import { IVisualizationOptions } from '../../utils/interfaces'; + +interface IVisualizationToolbarProps { + options: IVisualizationOptions; + setOptions: (data: IVisualizationOptions) => void; +} + +const VisualizationToolbar: React.FunctionComponent = ({ + options, + setOptions, +}: IVisualizationToolbarProps) => { + const changeQuery = (value: string): void => { + setOptions({ ...options, query: value }); + }; + + // changeOptions changes the ClickHouse option. If the options are changed via the refresh button of the Options + // component we directly modify the options of the parent component, if not we only change the data of the toolbar + // component and the user can trigger an action via the search button. + const changeOptions = ( + refresh: boolean, + additionalFields: IOptionsAdditionalFields[] | undefined, + time: TTime, + timeEnd: number, + timeStart: number, + ): void => { + setOptions({ ...options, times: { time: time, timeEnd: timeEnd, timeStart: timeStart } }); + }; + + return ( + + + } breakpoint="lg"> + + + + + + + + + + + + ); +}; + +export default VisualizationToolbar; diff --git a/plugins/clickhouse/src/components/panel/VisualizationChart.tsx b/plugins/clickhouse/src/components/panel/VisualizationChart.tsx new file mode 100644 index 000000000..7cf236a79 --- /dev/null +++ b/plugins/clickhouse/src/components/panel/VisualizationChart.tsx @@ -0,0 +1,27 @@ +import React from 'react'; + +import { IVisualizationData } from '../../utils/interfaces'; +import VisualizationChartBar from './VisualizationChartBar'; +import VisualizationChartPie from './VisualizationChartPie'; + +interface IVisualizationChartProps { + chart: string; + operation: string; + data: IVisualizationData[]; +} + +const VisualizationChart: React.FunctionComponent = ({ + chart, + operation, + data, +}: IVisualizationChartProps) => { + if (chart === 'bar') { + return ; + } else if (chart === 'pie') { + return ; + } else { + return null; + } +}; + +export default VisualizationChart; diff --git a/plugins/clickhouse/src/components/panel/VisualizationChartBar.tsx b/plugins/clickhouse/src/components/panel/VisualizationChartBar.tsx new file mode 100644 index 000000000..30abb2997 --- /dev/null +++ b/plugins/clickhouse/src/components/panel/VisualizationChartBar.tsx @@ -0,0 +1,99 @@ +import { BarDatum, ResponsiveBarCanvas } from '@nivo/bar'; +import React, { useRef } from 'react'; +import { SquareIcon } from '@patternfly/react-icons'; +import { TooltipWrapper } from '@nivo/tooltip'; + +import { COLOR_SCALE } from '../../utils/colors'; +import { IVisualizationData } from '../../utils/interfaces'; +import { useDimensions } from '@kobsio/plugin-core'; + +interface IVisualizationChartBarProps { + operation: string; + data: IVisualizationData[]; +} + +const VisualizationChartBar: React.FunctionComponent = ({ + operation, + data, +}: IVisualizationChartBarProps) => { + const refChartContainer = useRef(null); + const chartContainerSize = useDimensions(refChartContainer); + + const barData: BarDatum[] = data.map((datum) => { + return { + label: datum.label, + value: datum.value, + }; + }); + + return ( +
+
+ -.0s', + legend: '', + legendOffset: -40, + legendPosition: 'middle', + }} + axisBottom={{ + legend: '', + tickRotation: 45, + }} + borderColor={{ from: 'color', modifiers: [['darker', 1.6]] }} + borderRadius={0} + borderWidth={0} + colorBy="indexValue" + colors={COLOR_SCALE} + data={barData} + enableLabel={false} + enableGridX={false} + enableGridY={true} + groupMode="stacked" + indexBy="label" + indexScale={{ round: true, type: 'band' }} + isInteractive={true} + keys={['value']} + layout="vertical" + margin={{ bottom: 100, left: 50, right: 0, top: 0 }} + maxValue="auto" + minValue="auto" + reverse={false} + theme={{ + background: '#ffffff', + fontFamily: 'RedHatDisplay, Overpass, overpass, helvetica, arial, sans-serif', + fontSize: 10, + textColor: '#000000', + }} + // eslint-disable-next-line @typescript-eslint/explicit-function-return-type + tooltip={(tooltip) => { + const isFirstHalf = tooltip.index < barData.length / 2; + + return ( + +
+
+ {tooltip.data.label}: {tooltip.data.value} +
+
+ + ); + }} + valueFormat="" + valueScale={{ type: 'linear' }} + /> +
+
+ ); +}; + +export default VisualizationChartBar; diff --git a/plugins/clickhouse/src/components/panel/VisualizationChartPie.tsx b/plugins/clickhouse/src/components/panel/VisualizationChartPie.tsx new file mode 100644 index 000000000..3c06db046 --- /dev/null +++ b/plugins/clickhouse/src/components/panel/VisualizationChartPie.tsx @@ -0,0 +1,74 @@ +import React, { useRef } from 'react'; +import { ResponsivePieCanvas } from '@nivo/pie'; +import { SquareIcon } from '@patternfly/react-icons'; +import { TooltipWrapper } from '@nivo/tooltip'; + +import { COLOR_SCALE } from '../../utils/colors'; +import { IVisualizationData } from '../../utils/interfaces'; +import { useDimensions } from '@kobsio/plugin-core'; + +interface IVisualizationChartPieProps { + operation: string; + data: IVisualizationData[]; +} + +const VisualizationChartPie: React.FunctionComponent = ({ + operation, + data, +}: IVisualizationChartPieProps) => { + const refChartContainer = useRef(null); + const chartContainerSize = useDimensions(refChartContainer); + + return ( +
+
+ { + return ( + +
+
+ {tooltip.datum.label}: {tooltip.datum.value} +
+
+ + ); + }} + value="value" + /> +
+
+ ); +}; + +export default VisualizationChartPie; diff --git a/plugins/clickhouse/src/utils/colors.ts b/plugins/clickhouse/src/utils/colors.ts new file mode 100644 index 000000000..52beceacf --- /dev/null +++ b/plugins/clickhouse/src/utils/colors.ts @@ -0,0 +1,61 @@ +import chart_color_blue_100 from '@patternfly/react-tokens/dist/js/chart_color_blue_100'; +import chart_color_blue_200 from '@patternfly/react-tokens/dist/js/chart_color_blue_200'; +import chart_color_blue_300 from '@patternfly/react-tokens/dist/js/chart_color_blue_300'; +import chart_color_blue_400 from '@patternfly/react-tokens/dist/js/chart_color_blue_400'; +import chart_color_blue_500 from '@patternfly/react-tokens/dist/js/chart_color_blue_500'; +import chart_color_cyan_100 from '@patternfly/react-tokens/dist/js/chart_color_cyan_100'; +import chart_color_cyan_200 from '@patternfly/react-tokens/dist/js/chart_color_cyan_200'; +import chart_color_cyan_300 from '@patternfly/react-tokens/dist/js/chart_color_cyan_300'; +import chart_color_cyan_400 from '@patternfly/react-tokens/dist/js/chart_color_cyan_400'; +import chart_color_cyan_500 from '@patternfly/react-tokens/dist/js/chart_color_cyan_500'; +import chart_color_gold_100 from '@patternfly/react-tokens/dist/js/chart_color_gold_100'; +import chart_color_gold_200 from '@patternfly/react-tokens/dist/js/chart_color_gold_200'; +import chart_color_gold_300 from '@patternfly/react-tokens/dist/js/chart_color_gold_300'; +import chart_color_gold_400 from '@patternfly/react-tokens/dist/js/chart_color_gold_400'; +import chart_color_gold_500 from '@patternfly/react-tokens/dist/js/chart_color_gold_500'; +import chart_color_green_100 from '@patternfly/react-tokens/dist/js/chart_color_green_100'; +import chart_color_green_200 from '@patternfly/react-tokens/dist/js/chart_color_green_200'; +import chart_color_green_300 from '@patternfly/react-tokens/dist/js/chart_color_green_300'; +import chart_color_green_400 from '@patternfly/react-tokens/dist/js/chart_color_green_400'; +import chart_color_green_500 from '@patternfly/react-tokens/dist/js/chart_color_green_500'; +import chart_color_orange_100 from '@patternfly/react-tokens/dist/js/chart_color_orange_100'; +import chart_color_orange_200 from '@patternfly/react-tokens/dist/js/chart_color_orange_200'; +import chart_color_orange_300 from '@patternfly/react-tokens/dist/js/chart_color_orange_300'; +import chart_color_orange_400 from '@patternfly/react-tokens/dist/js/chart_color_orange_400'; +import chart_color_orange_500 from '@patternfly/react-tokens/dist/js/chart_color_orange_500'; + +// We are using the multi color ordered theme from Patternfly for the charts. +// See: https://github.com/patternfly/patternfly-react/blob/main/packages/react-charts/src/components/ChartTheme/themes/light/multi-color-ordered-theme.ts +export const COLOR_SCALE = [ + chart_color_blue_300.value, + chart_color_green_300.value, + chart_color_cyan_300.value, + chart_color_gold_300.value, + chart_color_orange_300.value, + chart_color_blue_100.value, + chart_color_green_500.value, + chart_color_cyan_100.value, + chart_color_gold_100.value, + chart_color_orange_500.value, + chart_color_blue_500.value, + chart_color_green_100.value, + chart_color_cyan_500.value, + chart_color_gold_500.value, + chart_color_orange_100.value, + chart_color_blue_200.value, + chart_color_green_400.value, + chart_color_cyan_200.value, + chart_color_gold_200.value, + chart_color_orange_400.value, + chart_color_blue_400.value, + chart_color_green_200.value, + chart_color_cyan_400.value, + chart_color_gold_400.value, + chart_color_orange_200.value, +]; + +// getColor returns the correct color for a given index. The function is mainly used by the legend for an chart, so that +// we can split the legend and chart into separate components. +export const getColor = (index: number): string => { + return COLOR_SCALE[index % COLOR_SCALE.length]; +}; diff --git a/plugins/clickhouse/src/utils/helpers.ts b/plugins/clickhouse/src/utils/helpers.ts index a9c8d4665..9bc0a00e8 100644 --- a/plugins/clickhouse/src/utils/helpers.ts +++ b/plugins/clickhouse/src/utils/helpers.ts @@ -1,5 +1,5 @@ +import { IOptions, IVisualizationOptions } from './interfaces'; import { TTime, TTimeOptions, formatTime } from '@kobsio/plugin-core'; -import { IOptions } from './interfaces'; // getOptionsFromSearch is used to get the ClickHouse options from a given search location. export const getOptionsFromSearch = (search: string): IOptions => { @@ -29,6 +29,41 @@ export const getOptionsFromSearch = (search: string): IOptions => { }; }; +// getOptionsFromSearch is used to get the ClickHouse options for a visualization from a given search location. +export const getVisualizationOptionsFromSearch = (search: string): IVisualizationOptions => { + const params = new URLSearchParams(search); + + const chart = params.get('chart'); + const limit = params.get('limit'); + const groupBy = params.get('groupBy'); + const operation = params.get('operation'); + const operationField = params.get('operationField'); + const order = params.get('order'); + const query = params.get('query'); + const time = params.get('time'); + const timeEnd = params.get('timeEnd'); + const timeStart = params.get('timeStart'); + + return { + chart: chart ? chart : 'bar', + groupBy: groupBy ? groupBy : '', + limit: limit ? limit : '10', + operation: operation ? operation : 'count', + operationField: operationField ? operationField : '', + order: order ? order : 'ascending', + query: query ? query : '', + times: { + time: time && TTimeOptions.includes(time) ? (time as TTime) : 'last15Minutes', + timeEnd: + time && TTimeOptions.includes(time) && timeEnd ? parseInt(timeEnd as string) : Math.floor(Date.now() / 1000), + timeStart: + time && TTimeOptions.includes(time) && timeStart + ? parseInt(timeStart as string) + : Math.floor(Date.now() / 1000) - 900, + }, + }; +}; + // formatTimeWrapper is a wrapper for our shared formatTime function. It is needed to convert a given time string to the // corresponding timestamp representation, which we need for the formatTime function. export const formatTimeWrapper = (time: string): string => { diff --git a/plugins/clickhouse/src/utils/interfaces.ts b/plugins/clickhouse/src/utils/interfaces.ts index 16abe8bbc..22524eef0 100644 --- a/plugins/clickhouse/src/utils/interfaces.ts +++ b/plugins/clickhouse/src/utils/interfaces.ts @@ -9,6 +9,18 @@ export interface IOptions { times: IPluginTimes; } +// IVisualizationOptions is the interface for all options, which can be set to visualize an aggregation query. +export interface IVisualizationOptions { + query: string; + times: IPluginTimes; + operationField: string; + operation: string; + groupBy: string; + order: string; + limit: string; + chart: string; +} + // IPanelOptions are the options for the panel component of the ClickHouse plugin. export interface IPanelOptions { type: string; @@ -60,3 +72,9 @@ export interface IDomain { x: Date[]; y: number[]; } + +// IVisualizationData is the data returned by ClickHouse to render a chart. +export interface IVisualizationData { + label: string; + value: number; +} diff --git a/yarn.lock b/yarn.lock index f0764d5d4..a603461b5 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2185,6 +2185,15 @@ "@react-spring/web" "9.2.4" lodash "^4.17.21" +"@nivo/arcs@0.73.0": + version "0.73.0" + resolved "https://registry.yarnpkg.com/@nivo/arcs/-/arcs-0.73.0.tgz#2211a0c41e8f6ed67374aeebdad607fbb3a1db2f" + integrity sha512-jIjqr3McQUrDWoP6X4CZh8Tg0HphLZdU6K1IDfna2nkG8Dkr2LcB7Ejt5SFbhff+0SC/hjvjNC//h8vcynl7yA== + dependencies: + "@nivo/colors" "0.73.0" + "@react-spring/web" "9.2.4" + d3-shape "^1.3.5" + "@nivo/axes@0.73.0": version "0.73.0" resolved "https://registry.yarnpkg.com/@nivo/axes/-/axes-0.73.0.tgz#d4982bda3c21d318507e4c61b9cce31549f8c894" @@ -2262,6 +2271,17 @@ "@react-spring/web" "9.2.4" d3-shape "^1.3.5" +"@nivo/pie@^0.73.0": + version "0.73.0" + resolved "https://registry.yarnpkg.com/@nivo/pie/-/pie-0.73.0.tgz#4370bfdaaded5b0ba159cc544548b02373baf55a" + integrity sha512-CWwpK9NSs1hfqvhcVvYJMTk+qD/tuxKDloURjfgJglUOs5m2NT1osDJN64jr02aZ6wZyplkP6WvJuc4K6mej6Q== + dependencies: + "@nivo/arcs" "0.73.0" + "@nivo/colors" "0.73.0" + "@nivo/legends" "0.73.0" + "@nivo/tooltip" "0.73.0" + d3-shape "^1.3.5" + "@nivo/recompose@0.73.0": version "0.73.0" resolved "https://registry.yarnpkg.com/@nivo/recompose/-/recompose-0.73.0.tgz#f228fdc633df5453c67640f59b74368071559e73"