Skip to content

Commit

Permalink
Elasticsearch: Visualize logs in Explore (#17605)
Browse files Browse the repository at this point in the history
* explore: try to use existing mode when switching datasource

* elasticsearch: initial explore logs support

* Elasticsearch: Adds ElasticsearchOptions type
Updates tests accordingly

* Elasticsearch: Adds typing to query method

* Elasticsearch: Makes maxConcurrentShardRequests optional

* Explore: Allows empty query for elasticsearch datasource

* Elasticsearch: Unifies ElasticsearchQuery interface definition
Removes check for context === 'explore'

* Elasticsearch: Removes context property from ElasticsearchQuery interface
Adds field property
Removes metricAggs property
Adds typing to metrics property

* Elasticsearch: Runs default 'empty' query when 'clear all' button is pressed

* Elasticsearch: Removes index property from ElasticsearchOptions interface

* Elasticsearch: Removes commented code from ElasticsearchQueryField.tsx

* Elasticsearch: Adds comment warning usage of for...in to elastic_response.ts

* Elasticsearch: adds tests related to log queries
  • Loading branch information
marefr authored and kaydelaney committed Jun 24, 2019
1 parent 2fb45ee commit eecd8d1
Show file tree
Hide file tree
Showing 17 changed files with 617 additions and 64 deletions.
3 changes: 3 additions & 0 deletions devenv/datasources.yaml
Expand Up @@ -153,6 +153,9 @@ datasources:
interval: Daily
timeField: "@timestamp"
esVersion: 70
timeInterval: "10s"
logMessageField: message
logLevelField: fields.level

- name: gdev-elasticsearch-v7-metricbeat
type: elasticsearch
Expand Down
6 changes: 1 addition & 5 deletions pkg/api/frontendsettings.go
Expand Up @@ -115,11 +115,7 @@ func (hs *HTTPServer) getFrontendSettingsMap(c *m.ReqContext) (map[string]interf
}
}

if ds.Type == m.DS_ES {
dsMap["index"] = ds.Database
}

if ds.Type == m.DS_INFLUXDB {
if (ds.Type == m.DS_INFLUXDB) || (ds.Type == m.DS_ES) {
dsMap["database"] = ds.Database
}

Expand Down
2 changes: 1 addition & 1 deletion public/app/features/explore/state/reducers.ts
Expand Up @@ -240,7 +240,7 @@ export const itemReducer = reducerFactory<ExploreItemState>({} as ExploreItemSta
const supportsGraph = datasourceInstance.meta.metrics;
const supportsLogs = datasourceInstance.meta.logs;

let mode = ExploreMode.Metrics;
let mode = state.mode || ExploreMode.Metrics;
const supportedModes: ExploreMode[] = [];

if (supportsGraph) {
Expand Down
@@ -0,0 +1,91 @@
import _ from 'lodash';
import React from 'react';
// @ts-ignore
import PluginPrism from 'slate-prism';
// @ts-ignore
import Prism from 'prismjs';

// dom also includes Element polyfills
import QueryField from 'app/features/explore/QueryField';
import { ExploreQueryFieldProps } from '@grafana/ui';
import { ElasticDatasource } from '../datasource';
import { ElasticsearchOptions, ElasticsearchQuery } from '../types';

interface Props extends ExploreQueryFieldProps<ElasticDatasource, ElasticsearchQuery, ElasticsearchOptions> {}

interface State {
syntaxLoaded: boolean;
}

class ElasticsearchQueryField extends React.PureComponent<Props, State> {
plugins: any[];

constructor(props: Props, context: React.Context<any>) {
super(props, context);

this.plugins = [
PluginPrism({
onlyIn: (node: any) => node.type === 'code_block',
getSyntax: (node: any) => 'lucene',
}),
];

this.state = {
syntaxLoaded: false,
};
}

componentDidMount() {
this.onChangeQuery('', true);
}

componentWillUnmount() {}

componentDidUpdate(prevProps: Props) {
// if query changed from the outside (i.e. cleared via explore toolbar)
if (!this.props.query.isLogsQuery) {
this.onChangeQuery('', true);
}
}

onChangeQuery = (value: string, override?: boolean) => {
// Send text change to parent
const { query, onChange, onRunQuery } = this.props;
if (onChange) {
const nextQuery: ElasticsearchQuery = { ...query, query: value, isLogsQuery: true };
onChange(nextQuery);

if (override && onRunQuery) {
onRunQuery();
}
}
};

render() {
const { queryResponse, query } = this.props;
const { syntaxLoaded } = this.state;

return (
<>
<div className="gf-form-inline gf-form-inline--nowrap">
<div className="gf-form gf-form--grow flex-shrink-1">
<QueryField
additionalPlugins={this.plugins}
initialQuery={query.query}
onChange={this.onChangeQuery}
onRunQuery={this.props.onRunQuery}
placeholder="Enter a Lucene query"
portalOrigin="elasticsearch"
syntaxLoaded={syntaxLoaded}
/>
</div>
</div>
{queryResponse && queryResponse.error ? (
<div className="prom-query-field-info text-error">{queryResponse.error.message}</div>
) : null}
</>
);
}
}

export default ElasticsearchQueryField;
2 changes: 2 additions & 0 deletions public/app/plugins/datasource/elasticsearch/config_ctrl.ts
Expand Up @@ -11,6 +11,8 @@ export class ElasticConfigCtrl {
const defaultMaxConcurrentShardRequests = this.current.jsonData.esVersion >= 70 ? 5 : 256;
this.current.jsonData.maxConcurrentShardRequests =
this.current.jsonData.maxConcurrentShardRequests || defaultMaxConcurrentShardRequests;
this.current.jsonData.logMessageField = this.current.jsonData.logMessageField || '';
this.current.jsonData.logLevelField = this.current.jsonData.logLevelField || '';
}

indexPatternTypes = [
Expand Down
87 changes: 64 additions & 23 deletions public/app/plugins/datasource/elasticsearch/datasource.ts
@@ -1,11 +1,17 @@
import angular from 'angular';
import angular, { IQService } from 'angular';
import _ from 'lodash';
import { DataSourceApi, DataSourceInstanceSettings, DataQueryRequest, DataQueryResponse } from '@grafana/ui';
import { ElasticResponse } from './elastic_response';
import { IndexPattern } from './index_pattern';
import { ElasticQueryBuilder } from './query_builder';
import { toUtc } from '@grafana/ui/src/utils/moment_wrapper';
import * as queryDef from './query_def';
import { BackendSrv } from 'app/core/services/backend_srv';
import { TemplateSrv } from 'app/features/templating/template_srv';
import { TimeSrv } from 'app/features/dashboard/services/TimeSrv';
import { ElasticsearchOptions, ElasticsearchQuery } from './types';

export class ElasticDatasource {
export class ElasticDatasource extends DataSourceApi<ElasticsearchQuery, ElasticsearchOptions> {
basicAuth: string;
withCredentials: boolean;
url: string;
Expand All @@ -17,23 +23,44 @@ export class ElasticDatasource {
maxConcurrentShardRequests: number;
queryBuilder: ElasticQueryBuilder;
indexPattern: IndexPattern;
logMessageField?: string;
logLevelField?: string;

/** @ngInject */
constructor(instanceSettings, private $q, private backendSrv, private templateSrv, private timeSrv) {
constructor(
instanceSettings: DataSourceInstanceSettings<ElasticsearchOptions>,
private $q: IQService,
private backendSrv: BackendSrv,
private templateSrv: TemplateSrv,
private timeSrv: TimeSrv
) {
super(instanceSettings);
this.basicAuth = instanceSettings.basicAuth;
this.withCredentials = instanceSettings.withCredentials;
this.url = instanceSettings.url;
this.name = instanceSettings.name;
this.index = instanceSettings.index;
this.timeField = instanceSettings.jsonData.timeField;
this.esVersion = instanceSettings.jsonData.esVersion;
this.indexPattern = new IndexPattern(instanceSettings.index, instanceSettings.jsonData.interval);
this.interval = instanceSettings.jsonData.timeInterval;
this.maxConcurrentShardRequests = instanceSettings.jsonData.maxConcurrentShardRequests;
this.index = instanceSettings.database;
const settingsData = instanceSettings.jsonData || ({} as ElasticsearchOptions);

this.timeField = settingsData.timeField;
this.esVersion = settingsData.esVersion;
this.indexPattern = new IndexPattern(this.index, settingsData.interval);
this.interval = settingsData.timeInterval;
this.maxConcurrentShardRequests = settingsData.maxConcurrentShardRequests;
this.queryBuilder = new ElasticQueryBuilder({
timeField: this.timeField,
esVersion: this.esVersion,
});
this.logMessageField = settingsData.logMessageField || '';
this.logLevelField = settingsData.logLevelField || '';

if (this.logMessageField === '') {
this.logMessageField = null;
}

if (this.logLevelField === '') {
this.logLevelField = null;
}
}

private request(method, url, data?) {
Expand Down Expand Up @@ -200,7 +227,6 @@ export class ElasticDatasource {
}

testDatasource() {
this.timeSrv.setTime({ from: 'now-1m', to: 'now' }, true);
// validate that the index exist and has date field
return this.getFields({ type: 'date' }).then(
dateFields => {
Expand Down Expand Up @@ -240,10 +266,10 @@ export class ElasticDatasource {
return angular.toJson(queryHeader);
}

query(options) {
query(options: DataQueryRequest<ElasticsearchQuery>): Promise<DataQueryResponse> {
let payload = '';
const targets = _.cloneDeep(options.targets);
const sentTargets = [];
const sentTargets: ElasticsearchQuery[] = [];

// add global adhoc filters to timeFilter
const adhocFilters = this.templateSrv.getAdhocFilters(this.name);
Expand All @@ -253,38 +279,53 @@ export class ElasticDatasource {
continue;
}

if (target.alias) {
target.alias = this.templateSrv.replace(target.alias, options.scopedVars, 'lucene');
}

let queryString = this.templateSrv.replace(target.query, options.scopedVars, 'lucene');
// Elasticsearch queryString should always be '*' if empty string
if (!queryString || queryString === '') {
queryString = '*';
}
const queryObj = this.queryBuilder.build(target, adhocFilters, queryString);

let queryObj;
if (target.isLogsQuery) {
target.bucketAggs = [queryDef.defaultBucketAgg()];
target.metrics = [queryDef.defaultMetricAgg()];
queryObj = this.queryBuilder.getLogsQuery(target, queryString);
} else {
if (target.alias) {
target.alias = this.templateSrv.replace(target.alias, options.scopedVars, 'lucene');
}

queryObj = this.queryBuilder.build(target, adhocFilters, queryString);
}

const esQuery = angular.toJson(queryObj);

const searchType = queryObj.size === 0 && this.esVersion < 5 ? 'count' : 'query_then_fetch';
const header = this.getQueryHeader(searchType, options.range.from, options.range.to);
payload += header + '\n';

payload += esQuery + '\n';

sentTargets.push(target);
}

if (sentTargets.length === 0) {
return this.$q.when([]);
return Promise.resolve({ data: [] });
}

payload = payload.replace(/\$timeFrom/g, options.range.from.valueOf());
payload = payload.replace(/\$timeTo/g, options.range.to.valueOf());
payload = payload.replace(/\$timeFrom/g, options.range.from.valueOf().toString());
payload = payload.replace(/\$timeTo/g, options.range.to.valueOf().toString());
payload = this.templateSrv.replace(payload, options.scopedVars);

const url = this.getMultiSearchUrl();

return this.post(url, payload).then(res => {
return new ElasticResponse(sentTargets, res).getTimeSeries();
const er = new ElasticResponse(sentTargets, res);
if (sentTargets.some(target => target.isLogsQuery)) {
return er.getLogs(this.logMessageField, this.logLevelField);
}

return er.getTimeSeries();
});
}

Expand Down Expand Up @@ -380,8 +421,8 @@ export class ElasticDatasource {
const header = this.getQueryHeader(searchType, range.from, range.to);
let esQuery = angular.toJson(this.queryBuilder.getTermsQuery(queryDef));

esQuery = esQuery.replace(/\$timeFrom/g, range.from.valueOf());
esQuery = esQuery.replace(/\$timeTo/g, range.to.valueOf());
esQuery = esQuery.replace(/\$timeFrom/g, range.from.valueOf().toString());
esQuery = esQuery.replace(/\$timeTo/g, range.to.valueOf().toString());
esQuery = header + '\n' + esQuery + '\n';

const url = this.getMultiSearchUrl();
Expand Down

0 comments on commit eecd8d1

Please sign in to comment.