[clickhouse] Change filters in aggregations

CHANGELOG.md

@@ -35,6 +35,7 @@ NOTE: As semantic versioning states all 0.y.z releases can contain breaking chan
 - [#190](https://github.com/kobsio/kobs/pull/190): [core] Unify list layout across plugin.
 - [#194](https://github.com/kobsio/kobs/pull/194): [elasticsearch] Use pagination instead of infinite scrolling to display logs.
 - [#195](https://github.com/kobsio/kobs/pull/195): [istio] Display upstream cluster instead of authority in the React UI and ignore query string in path.
+- [#197](https://github.com/kobsio/kobs/pull/197): [clickhouse] Change break down filter for aggregation, to allow more complex filters.
 
 ## [v0.6.0](https://github.com/kobsio/kobs/releases/tag/v0.6.0) (2021-10-11)
 

docs/plugins/clickhouse.md

@@ -142,15 +142,7 @@ spec:
 | verticalAxisOperation | string | The operation for the vertical axis. This can be `count`, `min`, `max`, `sum` or `avg`. | No |
 | verticalAxisField | string | When the verticalAxisOperation is `min`, `max`, `sum` or `avg`, this must be the name of a field for the vertical axis. | No |
 | breakDownByFields | []string | A list of field names, which should be used to break down the data. | No |
-| breakDownByFilters | [[]Aggregation Filters](#aggregation-filters) | A list of filters, which should be used to break down the data. | No |
-
-### Aggregation Filters
-
-| Field | Type | Description | Required |
-| ----- | ---- | ----------- | -------- |
-| field | string | The name of the field, which should be used for the filter. | Yes |
-| operator | string | The operator, which should be used for comparing the field value. This can be `=`, `<`, `>`, `<=` or `>=`. | Yes |
-| value | string | The value against which the field should be compared for the filter. | Yes |
+| breakDownByFilters | []string | A list of filters, which should be used to break down the data. | No |
 
 ## Query Syntax
 

plugins/clickhouse/pkg/instance/aggregation.go

@@ -32,9 +32,9 @@ type AggregationOptions struct {
 	VerticalAxisOperation string `json:"verticalAxisOperation"`
 	VerticalAxisField     string `json:"verticalAxisField"`
 
-	BreakDownBy        string                          `json:"breakDownBy"`
-	BreakDownByFields  []string                        `json:"breakDownByFields"`
-	BreakDownByFilters []AggregationBreakDownByFilters `json:"breakDownByFilters"`
+	BreakDownBy        string   `json:"breakDownBy"`
+	BreakDownByFields  []string `json:"breakDownByFields"`
+	BreakDownByFilters []string `json:"breakDownByFilters"`
 }
 
 // AggregationTimes is the structure, which defines the time interval for the aggregation.
@@ -43,13 +43,6 @@ type AggregationTimes struct {
 	TimeStart int64 `json:"timeStart"`
 }
 
-// AggregationBreakDownByFilters is the structure of a single filter, which should be applied to an aggregation.
-type AggregationBreakDownByFilters struct {
-	Field    string `json:"field"`
-	Operator string `json:"operator"`
-	Value    string `json:"value"`
-}
-
 // generateFieldName generates the field name for an aggregation. For that we are using the user defined field and we
 // are checking if this field is a default field or a materialized column. If this is the case we can directly use the
 // field name. If it is a custom field, we check against the array of the loaded fields to check if it is a string or
@@ -86,7 +79,7 @@ func getOrderBy(order string) string {
 // buildAggregationQuery is our helper function to build the different parts of the SQL statement for the user defined
 // chart and aggregation. The function returns the SELECT, GROUP BY, ORDER BY and LIMIT statement for the SQL query, to
 // get the results of the aggregation.
-func buildAggregationQuery(chart string, options AggregationOptions, materializedColumns []string, customFields Fields) (string, string, string, string, error) {
+func buildAggregationQuery(chart string, options AggregationOptions, materializedColumns []string, customFields Fields, timeStart, timeEnd int64) (string, string, string, string, error) {
 	var selectStatement, groupByStatement, orderByStatement, limitByStatement string
 
 	if chart != "pie" && chart != "bar" && chart != "line" && chart != "area" {
@@ -164,7 +157,12 @@ func buildAggregationQuery(chart string, options AggregationOptions, materialize
 
 			var breakDownByFilters []string
 			for _, breakDownByFilter := range options.BreakDownByFilters {
-				breakDownByFilters = append(breakDownByFilters, fmt.Sprintf("%s %s %s", generateFieldName(breakDownByFilter.Field, materializedColumns, customFields, false), breakDownByFilter.Operator, breakDownByFilter.Value))
+				f, err := parseLogsQuery(breakDownByFilter, materializedColumns)
+				if err != nil {
+					return "", "", "", "", fmt.Errorf("invalid break down filter")
+				}
+
+				breakDownByFilters = append(breakDownByFilters, f)
 			}
 
 			horizontalAxisField := generateFieldName(options.HorizontalAxisField, materializedColumns, customFields, false)
@@ -208,12 +206,35 @@ func buildAggregationQuery(chart string, options AggregationOptions, materialize
 
 		var breakDownByFilters []string
 		for _, breakDownByFilter := range options.BreakDownByFilters {
-			breakDownByFilters = append(breakDownByFilters, fmt.Sprintf("%s %s %s", generateFieldName(breakDownByFilter.Field, materializedColumns, customFields, false), breakDownByFilter.Operator, breakDownByFilter.Value))
+			f, err := parseLogsQuery(breakDownByFilter, materializedColumns)
+			if err != nil {
+				return "", "", "", "", fmt.Errorf("invalid break down filter")
+			}
+
+			breakDownByFilters = append(breakDownByFilters, f)
 		}
 
 		verticalAxisField := generateFieldName(options.VerticalAxisField, materializedColumns, customFields, true)
 
-		selectStatement = "toStartOfInterval(timestamp, INTERVAL 30 second) AS time"
+		// Create an interval for the selected start and end time, so that we always return the same amount of data
+		// points, so that our charts are rendered in the same ways for each selected time range.
+		var interval int64
+		switch seconds := timeEnd - timeStart; {
+		case seconds <= 2:
+			interval = (timeEnd - timeStart) / 1
+		case seconds <= 10:
+			interval = (timeEnd - timeStart) / 5
+		case seconds <= 30:
+			interval = (timeEnd - timeStart) / 15
+		case seconds <= 60:
+			interval = (timeEnd - timeStart) / 30
+		case seconds <= 120:
+			interval = (timeEnd - timeStart) / 60
+		default:
+			interval = (timeEnd - timeStart) / 100
+		}
+
+		selectStatement = fmt.Sprintf("toStartOfInterval(timestamp, INTERVAL %d second) AS time", interval)
 		if len(breakDownByFields) > 0 {
 			selectStatement = fmt.Sprintf("%s, %s", selectStatement, strings.Join(breakDownByFields, ", "))
 		}
@@ -256,7 +277,7 @@ func (i *Instance) GetAggregation(ctx context.Context, aggregation Aggregation)
 	// Build the SELECT, GROUP BY, ORDER BY and LIMIT statement for the SQL query. When the function returns an error
 	// the user provided an invalid aggregation. If the function doesn't return a ORDER BY or LIMIT statement we can
 	// also omit it in the SQL query.
-	selectStatement, groupByStatement, orderByStatement, limitByStatement, err := buildAggregationQuery(aggregation.Chart, aggregation.Options, i.materializedColumns, i.cachedFields)
+	selectStatement, groupByStatement, orderByStatement, limitByStatement, err := buildAggregationQuery(aggregation.Chart, aggregation.Options, i.materializedColumns, i.cachedFields, aggregation.Times.TimeStart, aggregation.Times.TimeEnd)
 	if err != nil {
 		return nil, nil, err
 	}

plugins/clickhouse/src/components/page/AggregationOptionsFilters.tsx

@@ -11,7 +11,7 @@ import {
 } from '@patternfly/react-core';
 import React, { useState } from 'react';
 
-import { IAggregationOptions, IAggregationOptionsAggregationFilter } from '../../utils/interfaces';
+import { IAggregationOptions } from '../../utils/interfaces';
 
 interface IAggregationOptionsFiltersProps {
   options: IAggregationOptions;
@@ -23,19 +23,19 @@ const AggregationOptionsFilters: React.FunctionComponent<IAggregationOptionsFilt
   setOptions,
 }: IAggregationOptionsFiltersProps) => {
   const [show, setShow] = useState<boolean>(false);
-  const [state, setState] = useState<IAggregationOptionsAggregationFilter>({ field: '', operator: '', value: '' });
+  const [filter, setFilter] = useState<string>('');
 
   const addFilter = (): void => {
     setOptions({
       ...options,
       options: {
         ...options.options,
         breakDownByFilters: options.options?.breakDownByFilters
-          ? [...options.options?.breakDownByFilters, state]
-          : [state],
+          ? [...options.options?.breakDownByFilters, filter]
+          : [filter],
       },
     });
-    setState({ field: '', operator: '', value: '' });
+    setFilter('');
     setShow(false);
   };
 
@@ -58,9 +58,7 @@ const AggregationOptionsFilters: React.FunctionComponent<IAggregationOptionsFilt
           isFlat={true}
           onClick={(): void => removeFilter(index)}
         >
-          <CardBody>
-            {filter.field} {filter.operator} {filter.value}
-          </CardBody>
+          <CardBody>{filter}</CardBody>
         </Card>
       ))}
       <Card style={{ cursor: 'pointer' }} isCompact={true} isFlat={true} onClick={(): void => setShow(true)}>
@@ -82,39 +80,15 @@ const AggregationOptionsFilters: React.FunctionComponent<IAggregationOptionsFilt
         ]}
       >
         <Form isHorizontal={true}>
-          <FormGroup label="Field" fieldId="form-field">
+          <FormGroup label="Filter" fieldId="form-filter">
             <TextInput
-              value={state.field}
+              value={filter}
               isRequired
               type="text"
-              id="form-field"
-              aria-describedby="form-field"
-              name="form-field"
-              onChange={(value): void => setState({ ...state, field: value })}
-            />
-          </FormGroup>
-
-          <FormGroup label="Operator" fieldId="form-operator">
-            <TextInput
-              value={state.operator}
-              isRequired
-              type="text"
-              id="form-operator"
-              aria-describedby="form-operator"
-              name="form-operator"
-              onChange={(value): void => setState({ ...state, operator: value })}
-            />
-          </FormGroup>
-
-          <FormGroup label="Value" fieldId="form-value">
-            <TextInput
-              value={state.value}
-              isRequired
-              type="text"
-              id="form-value"
-              aria-describedby="form-value"
-              name="form-value"
-              onChange={(value): void => setState({ ...state, value: value })}
+              id="form-filter"
+              aria-describedby="form-filter"
+              name="form-filter"
+              onChange={(value): void => setFilter(value)}
             />
           </FormGroup>
         </Form>

plugins/clickhouse/src/components/panel/AggregationChartBarTime.tsx

@@ -2,11 +2,11 @@ import React from 'react';
 import { ResponsiveBarCanvas } from '@nivo/bar';
 
 import { CHART_THEME, COLOR_SCALE, ChartTooltip } from '@kobsio/plugin-core';
-import { IAggregationData, IAggregationOptionsAggregationFilter } from '../../utils/interfaces';
 import { convertToBarChartTimeData, formatFilter } from '../../utils/aggregation';
+import { IAggregationData } from '../../utils/interfaces';
 
 interface IAggregationChartBarTimeProps {
-  filters: IAggregationOptionsAggregationFilter[];
+  filters: string[];
   data: IAggregationData;
 }
 
@@ -66,7 +66,9 @@ const AggregationChartBarTime: React.FunctionComponent<IAggregationChartBarTimeP
           <ChartTooltip
             anchor={isFirstHalf ? 'right' : 'left'}
             color={tooltip.color}
-            label={`${label ? `${label} - ${formatFilter(filter, filters)}` : filter}: ${tooltip.value}`}
+            label={`${label ? `${label} - ${formatFilter(filter, filters)}` : formatFilter(filter, filters)}: ${
+              tooltip.value
+            }`}
             position={[0, 20]}
             title={tooltip.data.time as string}
           />

plugins/clickhouse/src/components/panel/AggregationChartBarTop.tsx

@@ -2,11 +2,11 @@ import React from 'react';
 import { ResponsiveBarCanvas } from '@nivo/bar';
 
 import { CHART_THEME, COLOR_SCALE, ChartTooltip } from '@kobsio/plugin-core';
-import { IAggregationData, IAggregationOptionsAggregationFilter } from '../../utils/interfaces';
 import { convertToBarChartTopData, formatFilter } from '../../utils/aggregation';
+import { IAggregationData } from '../../utils/interfaces';
 
 interface IAggregationChartBarTopProps {
-  filters: IAggregationOptionsAggregationFilter[];
+  filters: string[];
   data: IAggregationData;
 }
 

plugins/clickhouse/src/components/panel/AggregationChartLine.tsx

@@ -2,14 +2,14 @@ import React from 'react';
 import { ResponsiveLineCanvas } from '@nivo/line';
 
 import { CHART_THEME, COLOR_SCALE, ChartTooltip } from '@kobsio/plugin-core';
-import { IAggregationData, IAggregationOptionsAggregationFilter } from '../../utils/interfaces';
 import { convertToLineChartData, formatAxisBottom, formatFilter } from '../../utils/aggregation';
+import { IAggregationData } from '../../utils/interfaces';
 
 interface IAggregationChartLineProps {
   isArea: boolean;
   startTime: number;
   endTime: number;
-  filters: IAggregationOptionsAggregationFilter[];
+  filters: string[];
   data: IAggregationData;
 }
 

plugins/clickhouse/src/utils/aggregation.ts

@@ -1,7 +1,7 @@
 import { Datum, Serie } from '@nivo/line';
 import { BarDatum } from '@nivo/bar';
 
-import { IAggregationData, IAggregationDataRow, IAggregationOptionsAggregationFilter } from './interfaces';
+import { IAggregationData, IAggregationDataRow } from './interfaces';
 import { formatTime } from '@kobsio/plugin-core';
 
 // formatAxisBottom calculates the format for the bottom axis based on the specified start and end time.
@@ -27,14 +27,14 @@ export const convertToBarChartTimeData = (data: IAggregationData): BarDatum[] =>
   // times is an array of all times returned from the API. This array is then used to create the bar data. We also have
   // to parse the time to convert the UTC formatted time to the local time.
   const times = data.rows
-    .map((row) => formatTime(Math.floor(new Date(row.time).getTime() / 1000)))
+    .map((row) => formatTime(Math.floor(new Date(row.time as string).getTime() / 1000)))
     .filter((value, index, self) => self.indexOf(value) === index);
 
   return times.map((time) => {
     return {
       time: time,
       ...generateKeys(
-        data.rows.filter((row) => formatTime(Math.floor(new Date(row.time).getTime() / 1000)) === time),
+        data.rows.filter((row) => formatTime(Math.floor(new Date(row.time as string).getTime() / 1000)) === time),
         labelColumns,
         dataColumns,
       ),
@@ -78,8 +78,8 @@ export const convertToLineChartData = (data: IAggregationData): Serie[] => {
         seriesData[i].push({
           filter: dataColumns[i],
           label: label,
-          x: new Date(row.time),
-          y: row[dataColumns[i]],
+          x: new Date(row.time as string),
+          y: row.hasOwnProperty(dataColumns[i]) ? row[dataColumns[i]] : null,
         });
       }
     }
@@ -96,7 +96,7 @@ export const convertToLineChartData = (data: IAggregationData): Serie[] => {
 };
 
 // formatFilter returns a readable string for the applied filters.
-export const formatFilter = (filter: string, filters: IAggregationOptionsAggregationFilter[]): string => {
+export const formatFilter = (filter: string, filters: string[]): string => {
   const filterParts = filter.split('_data');
   let formattedFilter = '';
 
@@ -113,10 +113,9 @@ export const formatFilter = (filter: string, filters: IAggregationOptionsAggrega
   return formattedFilter;
 };
 
-const formatFilterValue = (filter: string, filters: IAggregationOptionsAggregationFilter[]): string => {
+const formatFilterValue = (filter: string, filters: string[]): string => {
   if (filter.startsWith('_filter')) {
-    const breakDownByFilter = filters[parseInt(filter.slice(-1))];
-    return `${breakDownByFilter.field} ${breakDownByFilter.operator} ${breakDownByFilter.value}`;
+    return filters[parseInt(filter.slice(-1))];
   }
 
   return filter;
@@ -151,7 +150,7 @@ const generateKeys = (
     const label = getLabel(row, labelColumns);
 
     for (const dataColumn of dataColumns) {
-      keys[`${label} - ${dataColumn}`] = row[dataColumn];
+      keys[`${label} - ${dataColumn}`] = row.hasOwnProperty(dataColumn) ? row[dataColumn] : null;
     }
   }
 

plugins/clickhouse/src/utils/interfaces.ts

@@ -94,16 +94,7 @@ export interface IAggregationOptionsAggregation {
   verticalAxisField: string;
 
   breakDownByFields: string[];
-  breakDownByFilters: IAggregationOptionsAggregationFilter[];
-}
-
-// IAggregationOptionsAggregationFilter are the filters which can be set for an aggregation. Each filter required the
-// name of a field, an operator like ">", "<", ">=", "<=" or "=" and a value against which the field value should be
-// compared.
-export interface IAggregationOptionsAggregationFilter {
-  field: string;
-  operator: string;
-  value: string;
+  breakDownByFilters: string[];
 }
 
 // IAggregationData is the data returned by the aggregation API call. It contains a list of columns and a list of rows.
@@ -116,5 +107,5 @@ export interface IAggregationData {
 // key and the cell value for the row/column as value. The value could be a string, for the selected fields in the
 // aggregation or a number for the value of the fields combination.
 export interface IAggregationDataRow {
-  [key: string]: string | number;
+  [key: string]: string | number | null;
 }